flor 0.9.5 → 0.10.0

data/lib/flor/unit/storage.rb

@@ -1,26 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
 
 require 'zlib'
 
@@ -119,41 +96,13 @@ module Flor
       @db.tables.each { |t| @db[t].delete if t.to_s.match(/^flor_/) }
     end
 
-    def load_exids
-
-      synchronize do
-
-        # only take messages that are 'created' and for whose exid
-        # there are no loaded messages
-
-        # TODO update status to 'created' for messages that have been
-        #      'loaded' for too long
-
-        @db[:flor_messages]
-          .select(:exid)
-          .where(exid:
-            @db[:flor_messages].select(:exid).where(status: 'created'))
-          .exclude(exid:
-            @db[:flor_messages].select(:exid).where(status: 'loaded'))
-          .order(:mtime)
-          .collect { |r| r[:exid] }
-      end
-
-    rescue => err
-
-      @unit.logger.warn("#{self.class}#load_exids", err, '(returning [])')
-
-      []
-    end
-
     def load_execution(exid)
 
       synchronize do
 
         e = @db[:flor_executions]
           .select(:id, :content)
-          .where(exid: exid) # status active or terminated doesn't matter
-          .first
+          .first(exid: exid) # status active or terminated doesn't matter
 
         return {
           'exid' => exid, 'nodes' => {}, 'errors' => [], 'tasks' => {},
@@ -163,9 +112,9 @@ module Flor
 
         ex = from_blob(e[:content])
 
-        fail("couldn't parse execution (db id #{e[:id]})") unless ex
+        fail("couldn't parse execution (db id #{e[:id].to_i})") unless ex
 
-        ex['id'] = e[:id]
+        ex['id'] = e[:id].to_i
         ex['size'] = e[:content].length
 
         ex
@@ -193,6 +142,7 @@ module Flor
 
       data = to_blob(ex)
       ex['size'] = data.length
+      u = @unit.identifier
 
       transync do
 
@@ -201,11 +151,12 @@ module Flor
         if id
 
           @db[:flor_executions]
-            .where(id: id)
+            .where(id: id.to_i)
             .update(
               content: data,
               status: status,
-              mtime: now)
+              mtime: now,
+              munit: u)
 
         else
 
@@ -217,7 +168,10 @@ module Flor
                 content: data,
                 status: 'active',
                 ctime: now,
-                mtime: now)
+                mtime: now,
+                cunit: u,
+                munit: u)
+              .to_i
         end
 
         remove_nodes(ex, status, now)
@@ -232,37 +186,83 @@ module Flor
       raise err
     end
 
-    def fetch_messages(exid)
+    def load_messages(exe_count)
+
+      exe_count += 2
+        # load two more, could prove useful if they vanish like "petits pains"
+
+      synchronize do
+
+        _exids_being_processed =
+          @db[:flor_messages]
+            .select(:exid)
+            .exclude(status: %w[ created consumed ])
+        _exids =
+          @db[:flor_messages]
+            .select(:exid)
+            .exclude(exid: _exids_being_processed)
+            .limit(exe_count)
+        @db[:flor_messages]
+          .where(exid: _exids, status: 'created')
+          .inject({}) { |h, m| (h[m[:exid]] ||= []) << m; h }
+      end
+
+    rescue => err
+
+      @unit.logger.warn(
+        "#{self.class}#load_messages()", err, '(returning {})')
+
+      {}
+    end
+
+    def reserve_all_messages(messages)
+
+      now = Flor.tstamp
+      count = 0
 
       transync do
 
-        # TODO weave in [some] optimistic locking here
+        messages.each do |m|
 
-        mids = []
+          c = @db[:flor_messages]
+            .where(
+              id: m[:id].to_i, status: 'created',
+              mtime: m[:mtime], munit: m[:munit])
+            .update(
+              status: 'reserved', mtime: now, munit: @unit.identifier)
 
-        ms = @db[:flor_messages]
-          .select(:id, :content)
-          .where(status: 'created', exid: exid)
-          .order(:id)
-          .collect { |m|
-            r = from_blob(m[:content]) || {}
-            mid = m[:id]; r['mid'] = mid; mids << mid;
-            r }
+          raise Sequel::Rollback if c != 1
 
-        @db[:flor_messages]
-          .where(id: mids)
-          .update(status: 'loaded', mtime: Flor.tstamp)
-             #
-             # flag them as "loaded" so that other scheduler don't pick them
+          count += 1
+        end
+      end
 
-        ms
+      count == messages.size
+        # true means success: all the messages could be reserved,
+        # executor is clear to work on the execution
+
+    rescue => err
+
+      @unit.logger.warn(
+        "#{self.class}#reserve_all_messages()", err, '(returning false)')
+
+      false
+        # failure
+    end
+
+    def any_message?
+
+      synchronize do
+
+        @db[:flor_messages].count(status: 'created') > 0
       end
 
     rescue => err
 
-      @unit.logger.warn("#{self.class}#fetch_messages()", err, '(returning [])')
+      @unit.logger.warn(
+        "#{self.class}#any_message?()", err, '(returning false)')
 
-      []
+      false
     end
 
     def fetch_traps(exid)
@@ -277,7 +277,8 @@ module Flor
 
     rescue => err
 
-      @unit.logger.warn("#{self.class}#fetch_traps()", err, '(returning [])')
+      @unit.logger.warn(
+        "#{self.class}#fetch_traps()", err, '(returning [])')
 
       []
     end
@@ -288,11 +289,14 @@ module Flor
 
         if @archive
           @db[:flor_messages]
-            .where(id: messages.collect { |m| m['mid'] }.compact)
-            .update(status: 'consumed', mtime: Flor.tstamp)
+            .where(
+              id: messages.collect { |m| m['mid'] }.compact)
+            .update(
+              status: 'consumed', mtime: Flor.tstamp, munit: @unit.identifier)
         else
           @db[:flor_messages]
-            .where(id: messages.collect { |m| m['mid'] }.compact)
+            .where(
+              id: messages.collect { |m| m['mid'] }.compact)
             .delete
         end
       end
@@ -303,45 +307,29 @@ module Flor
       raise err
     end
 
-    def load_timers
-
-      synchronize do
-
-        timers
-          .select(:id, :content)
-          .where(status: 'active')
-          .order(:id)
-          .all
-      end
-
-    rescue => err
-
-      @unit.logger.warn("#{self.class}#load_timers()", err, '(returning [])')
-
-      []
-    end
-
     def put_messages(ms, syn=true)
 
       return if ms.empty?
 
       n = Flor.tstamp
+      u = @unit.identifier
 
       synchronize(syn) do
 
         @db[:flor_messages]
           .import(
             [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime ],
+              :status, :ctime, :mtime, :cunit, :munit ],
             ms.map { |m|
               [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
-                'created', n, n ]
+                'created', n, n, u, u ]
             })
       end
 
-      @unit.wake_up_executions(ms.collect { |m| m['exid'] }.uniq)
+      @unit.wake_up
 
     rescue => err
+
       Thread.current[:sto_errored_items] = ms
       raise err
     end
@@ -351,6 +339,20 @@ module Flor
       put_messages([ m ])
     end
 
+    def unreserve_messages(max_sec)
+
+      tstamp = Flor.tstamp(Time.now - max_sec)
+      tstamp = tstamp[0..tstamp.rindex('.')]
+
+      synchronize do
+
+        @db[:flor_messages]
+          .where(status: 'reserved')
+          .where { mtime < tstamp }
+          .update(status: 'created')
+      end
+    end
+
     def put_timer(message)
 
       type, string = determine_type_and_schedule(message)
@@ -358,24 +360,27 @@ module Flor
       next_time = compute_next_time(type, string)
 
       now = Flor.tstamp
+      u = @unit.identifier
 
-      id =
-        synchronize do
-          @db[:flor_timers].insert(
-            domain: Flor.domain(message['exid']),
-            exid: message['exid'],
-            nid: message['nid'],
-            type: type,
-            schedule: string,
-            ntime: next_time,
-            content: to_blob(message),
-            count: 0,
-            status: 'active',
-            ctime: now,
-            mtime: now)
-        end
+      synchronize do
+
+        @db[:flor_timers].insert(
+          domain: Flor.domain(message['exid']),
+          exid: message['exid'],
+          nid: message['nid'],
+          type: type,
+          schedule: string,
+          ntime: next_time,
+          content: to_blob(message),
+          count: 0,
+          status: 'active',
+          ctime: now,
+          mtime: now,
+          cunit: u,
+          munit: u)
+      end
 
-      @unit.timers[id]
+      @unit.wake_up
 
     rescue => err
 
@@ -383,51 +388,20 @@ module Flor
       raise err
     end
 
-    # Returns the timer if it is rescheduling
-    #
-    def trigger_timer(timer)
+    def trigger_timers
 
-      r = nil
-
-      transync do
+      synchronize do
 
-# TODO: cron/every stop conditions maybe?
+        load_timers.each do |t|
 
-        if timer.type != 'at' && timer.type != 'in'
+          @db.transaction do
 
-          @db[:flor_timers]
-            .where(id: timer.id)
-            .update(
-              count: timer.count + 1,
-              ntime: compute_next_time(timer.type, timer.schedule),
-              mtime: Flor.tstamp)
-          r = timers[timer.id]
-
-        elsif @archive
-
-          @db[:flor_timers]
-            .where(id: timer.id)
-            .update(
-              count: timer.count + 1,
-              status: 'triggered',
-              mtime: Flor.tstamp)
+            next unless reschedule_timer(t) == 1
 
-        else
-
-          @db[:flor_timers]
-            .where(id: timer.id)
-            .delete
+            trigger_timer(t)
+          end
         end
-
-        put_messages([ timer.to_trigger_message ], false)
       end
-
-      r
-
-    rescue => err
-
-      Thread.current[:sto_errored_items] = [ timer ]
-      raise err
     end
 
     def put_trap(node, tra)
@@ -435,9 +409,10 @@ module Flor
       exid = node['exid']
       dom = Flor.domain(exid)
       now = Flor.tstamp
+      u = @unit.identifier
 
       id =
-        transync do
+        synchronize do
 
           @db[:flor_traps].insert(
             domain: dom,
@@ -452,7 +427,9 @@ module Flor
             content: to_blob(tra),
             status: 'active',
             ctime: now,
-            mtime: now)
+            mtime: now,
+            cunit: u,
+            munit: u)
         end
 
       traps[id]
@@ -475,7 +452,8 @@ module Flor
           nid: nid,
           tracer: tracer,
           text: text,
-          ctime: Flor.tstamp)
+          ctime: Flor.tstamp,
+          cunit: @unit.identifier)
       end
     end
 
@@ -493,12 +471,89 @@ module Flor
             nid: msg['nid'],
             type: 'tasker',
             name: tname,
-            ctime: Flor.tstamp)
+            ctime: Flor.tstamp,
+            cunit: @unit.identifier)
       end
     end
 
+    def fetch_next_time
+
+      t =
+        synchronize do
+          @db[:flor_timers]
+            .select(:ntime)
+            .order(:ntime)
+            .first(status: 'active')
+        end
+
+      t ? t[:ntime].split('.').first : nil
+
+    rescue => err
+
+      @unit.logger.warn(
+        "#{self.class}#fetch_next_time()", err, '(returning nil)')
+
+      nil
+    end
+
     protected
 
+    def load_timers
+
+      now = Flor.tstamp
+      no = now[0, now.rindex('.')]
+
+      timers
+        .where(status: 'active')
+        .where { ntime <= no }
+        .order(:ntime)
+        .all
+
+    rescue => err
+
+      @unit.logger.warn("#{self.class}#load_timers()", err, '(returning [])')
+
+      []
+    end
+
+    def trigger_timer(t)
+
+      put_messages([ t.to_trigger_message ], false)
+    end
+
+    def reschedule_timer(t)
+
+      w = { id: t.id.to_i, status: 'active', mtime: t.mtime, munit: t.munit }
+
+      if t.type != 'at' && t.type != 'in'
+
+        @db[:flor_timers]
+          .where(w)
+          .update(
+            count: t.count.to_i + 1,
+            status: 'active',
+            ntime: compute_next_time(t.type, t.schedule, t.ntime_t),
+            mtime: Flor.tstamp,
+            munit: @unit.identifier)
+
+      elsif @archive
+
+        @db[:flor_timers]
+          .where(w)
+          .update(
+            count: t.count.to_i + 1,
+            status: 'triggered',
+            mtime: Flor.tstamp,
+            munit: @unit.identifier)
+
+      else
+
+        @db[:flor_timers]
+          .where(w)
+          .delete
+      end
+    end
+
     def remove_nodes(exe, status, now)
 
       exid = exe['exid']
@@ -520,6 +575,7 @@ module Flor
 
     def update_pointers(exe, status, now)
 
+# TODO archive old pointers???
       exid = exe['exid']
 
       if status == 'terminated'
@@ -537,20 +593,21 @@ module Flor
           # Delete pointers to gone nodes.
 
       dom = Flor.domain(exid)
+      u = @unit.identifier
 
       pointers =
         exe['nodes'].inject([]) { |a, (nid, node)|
           ts = node['tags']
-          ts.each { |t| a << [ dom, exid, nid, 'tag', t, nil, now ] } if ts
+          ts.each { |t| a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts
           a
         }
 
       pointers +=
         (exe['nodes']['0'] || { 'vars' => {} })['vars'].collect { |k, v|
           case v; when Integer, String, TrueClass, FalseClass
-            [ dom, exid, '0', 'var', k, v.to_s, now ]
+            [ dom, exid, '0', 'var', k, v.to_s, now, u ]
           when NilClass
-            [ dom, exid, '0', 'var', k, nil, now ]
+            [ dom, exid, '0', 'var', k, nil, now, u ]
           else
             nil
           end
@@ -558,21 +615,21 @@ module Flor
 
       pointers +=
         exe['tasks'].collect { |nid, v|
-          [ dom, exid, nid, 'tasker', v['tasker'], v['name'], now ]
+          [ dom, exid, nid, 'tasker', v['tasker'], v['name'], now, u ]
         }
 
       cps = @db[:flor_pointers] # current pointers
         .where(exid: exid)
         .select(:nid, :type, :name)
         .all
-      pointers.reject! { |_, _, ni, ty, na, _, _|
+      pointers.reject! { |_, _, ni, ty, na, _, _, _|
         cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
           #
           # don't insert when already inserted
 
       @db[:flor_pointers]
         .import(
-          [ :domain, :exid, :nid, :type, :name, :value, :ctime ],
+          [ :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit ],
           pointers)
     end
 
@@ -590,7 +647,7 @@ module Flor
       nil
     end
 
-    def compute_next_time(type, string)
+    def compute_next_time(type, string, from=nil)
 
       f =
         case type
@@ -601,7 +658,7 @@ module Flor
           else Fugit.parse(string)
         end
 
-      nt = f.is_a?(Time) ? f : f.next_time(Time.now) # local...
+      nt = f.is_a?(Time) ? f : f.next_time(from || Time.now) # local...
 
       Flor.tstamp(nt.utc)
     end