flor 0.17.0 → 1.1.1

data/lib/flor/unit/spooler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 

data/lib/flor/unit/storage.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 require 'zlib'
 
@@ -6,6 +7,14 @@ module Flor
 
   class Storage
 
+    MESSAGE_COLUMNS = [
+      :domain, :exid, :point, :content,
+      :status, :ctime, :mtime, :cunit, :munit
+        ].freeze
+    POINTER_COLUMNS = [
+      :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit
+        ].freeze
+
     attr_reader :unit, :db, :models
 
     attr_reader :mutex
@@ -30,9 +39,11 @@ module Flor
 #p [ :disconnected, @db.object_id ]
     end
 
-    def db_version
+    def db_version(opts={})
+
+      table, column = migration_table_and_column(opts)
 
-      (@db[:schema_info].first rescue {})[:version]
+      (@db[table].first rescue {})[column]
     end
 
     def migration_version
@@ -78,26 +89,28 @@ module Flor
       opts = [ to, from, opts ].find { |e| e.is_a?(Hash) } || {}
       opts[:target] ||= to if to.is_a?(Integer)
       opts[:current] ||= from if from.is_a?(Integer)
+
+      opts[:table], opts[:column] = migration_table_and_column(opts)
         #
         # defaults for the migration version table:
         # { table: :schema_info,
         #   column: :version }
 
       skip =
+        opts[:sparse_migrations] ||
         @unit.conf['db_sparse_migrations'] ||
-        @unit.conf['sto_sparse_migrations'] ||
-        opts[:sparse_migrations]
+        @unit.conf['sto_sparse_migrations']
       if skip && ! opts.has_key?(:allow_missing_migration_files)
         opts[:allow_missing_migration_files] = true
       end
 
       dir =
+        opts[:migrations] ||
+        opts[:migration_dir] ||
         @unit.conf['db_migrations'] ||
         @unit.conf['db_migration_dir'] ||
         @unit.conf['sto_migrations'] ||
         @unit.conf['sto_migration_dir'] ||
-        opts[:migrations] ||
-        opts[:migration_dir] ||
         Flor.migration_dir
 
       synchronize do
@@ -106,9 +119,15 @@ module Flor
       end
     end
 
+    # Delete tables in the storage database that begin with "flor_"
+    # and have more than 2 columns (the Sequel schema_info table has 1 column
+    # as of this writing)
+    #
     def delete_tables
 
-      @db.tables.each { |t| @db[t].delete if t.to_s.match(/^flor_/) }
+      @db.tables.each { |t|
+        @db[t].delete \
+          if t.to_s.match(/^flor_/) && @db[t].columns.size > 2 }
     end
 
     def load_execution(exid)
@@ -321,35 +340,39 @@ module Flor
       n = Flor.tstamp
       u = @unit.identifier
 
-      synchronize(syn) do
+      id =
+        synchronize(syn) do
 
-        stored, unstored = ms.partition { |m| m['mid'] }
+          stored, unstored = ms.partition { |m| m['mid'] }
 
-        #
-        # de-reserve any previously stored message, might happen
-        # for "terminated" messages that got queued back to let
-        # other messages get processed
+          #
+          # de-reserve any previously stored message, might happen
+          # for "terminated" messages that got queued back to let
+          # other messages get processed
 
-        @db[:flor_messages]
-          .where(id: stored.collect { |m| m['mid'] })
-          .update(status: 'created', mtime: n, munit: u) \
-            if stored.any?
+          @db[:flor_messages]
+            .where(id: stored.collect { |m| m['mid'] })
+            .update(status: 'created', mtime: n, munit: u) \
+              if stored.any?
 
-        #
-        # store new messages
+          #
+          # store new messages
 
-        @db[:flor_messages]
-          .import(
-            [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime, :cunit, :munit ],
-            unstored.map { |m|
-              [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
-                'created', n, n, u, u ] }) \
-                  if unstored.any?
-      end
+          @db[:flor_messages]
+            .import(
+              MESSAGE_COLUMNS,
+              unstored.map { |m|
+                [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
+                  'created', n, n, u, u ] }) \
+                    if unstored.any?
+
+          @db[:flor_messages].max(:id)
+        end
 
       @unit.wake_up
 
+      id
+
     rescue => err
 
       Thread.current[:sto_errored_items] = ms
@@ -501,25 +524,6 @@ module Flor
       end
     end
 
-    def put_task_pointer(msg, tname, tconf)
-
-      exid = msg['exid']
-      dom = Flor.domain(exid)
-
-      synchronize do
-
-        @db[:flor_pointers]
-          .insert(
-            domain: dom,
-            exid: exid,
-            nid: msg['nid'],
-            type: 'tasker',
-            name: tname,
-            ctime: Flor.tstamp,
-            cunit: @unit.identifier)
-      end
-    end
-
     def fetch_next_time
 
       t =
@@ -542,6 +546,18 @@ module Flor
 
     protected
 
+    def migration_table_and_column(opts={})
+
+      [ (opts[:migration_table] ||
+         @unit.conf['db_migration_table'] ||
+         @unit.conf['sto_migration_table'] ||
+         :schema_info).to_sym,
+        (opts[:migration_column] ||
+          @unit.conf['db_migration_column'] ||
+          @unit.conf['sto_migration_column'] ||
+          :version).to_sym ]
+    end
+
     def _commaify(o)
 
       if Flor.is_regex_tree?(o)
@@ -570,14 +586,13 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .update(
             status: 'consumed', mtime: n, munit: u)
 
         @db[:flor_messages]
           .import(
-            [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime, :cunit, :munit ],
+            MESSAGE_COLUMNS,
             messages
               .select { |m|
                 ! m['mid'] && POINTS_TO_ARCHIVE.include?(m['point']) }
@@ -593,19 +608,16 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .delete
       end
     end
 
     def load_timers
 
-      now = Flor.tstamp
-      no = now[0, now.rindex('.')]
-
       timers
         .where(status: 'active')
-        .where { ntime <= no }
+        .where { ntime <= Flor.tstam }
         .order(:ntime)
         .all
 
@@ -675,9 +687,9 @@ module Flor
 
     def update_pointers(exe, status, now)
 
-# Q Should we archive old pointers?
-#   Well, it might be better to only archive the execution and leave
-#   in there enough information...
+# Q  Should we archive old pointers?
+# A  Well, it might be better to only archive the execution and leave
+#    in there enough information...
 
       exid = exe['exid']
 
@@ -703,23 +715,25 @@ module Flor
 
           ts = node['tags']
           ts.each { |t|
-            a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts
+            a << [ dom, exid, nid, 'tag', t, nil, now, u, nil ] } if ts
 
           vs = nid == '0' ? node['vars'] : nil
           vs.each { |k, v|
             case v; when Numeric, String, TrueClass, FalseClass, NilClass
-              a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
+              a << [ dom, exid, '0', 'var', k, v.to_s, now, u, nil ]
             when Array, Hash
               s = '(array)'; s = '(object)' if v.is_a?(Hash)
-              a << [ dom, exid, '0', 'var', k, s, now, u ]
+              a << [ dom, exid, '0', 'var', k, s, now, u, nil ]
             else
-              a << [ dom, exid, '0', 'var', k, nil, now, u ]
+              a << [ dom, exid, '0', 'var', k, nil, now, u, nil ]
             end } if vs
 
-          #ta = node['heap'] == 'task' ? node['task'] : nil
-          ta = node['task']
-          a << [ dom, exid, nid, 'tasker', ta['tasker'], ta['name'], now, u ] \
-            if ta
+          if ta = node['task']
+            tasker = ta['tasker']
+            name = ta['name']
+            content = { message: node['message'], atts: node['atts'] }
+            a << [ dom, exid, nid, 'tasker', tasker, name, now, u, content ]
+          end
 
           a }
 
@@ -727,17 +741,35 @@ module Flor
         .where(exid: exid)
         .select(:nid, :type, :name)
         .all
-      pointers.reject! { |_, _, ni, ty, na, _, _, _|
+      pointers.reject! { |_, _, ni, ty, na, _, _, _, _|
         cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
           #
           # don't insert when already inserted
 
+      if pointer_columns.include?(:content)
+        pointers.each { |ptr|
+          c = ptr[8]; ptr[8] = to_blob(c) if c }
+      else
+        pointers.each { |ptr|
+          ptr.pop }
+      end
+
       @db[:flor_pointers]
         .import(
-          [ :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit ],
+          pointer_columns,
           pointers)
     end
 
+    def pointer_columns
+
+      @pointer_columns ||=
+        if @db[:flor_pointers].columns.include?(:content)
+          POINTER_COLUMNS + [ :content ]
+        else
+          POINTER_COLUMNS
+        end
+    end
+
     def determine_type_and_schedule(message)
 
       t, s = message['type'], message['string']
@@ -783,15 +815,27 @@ module Flor
       def error(msg); @unit.logger.db_log(:error, msg); end
     end
 
-    def connect
+    def derive_db
+
+      db = @unit.conf['sto_db']
+
+      return db if db
 
       uri = @unit.conf['sto_uri']
 
-      #uri = DB.uri if uri == 'DB' && defined?(DB)
-      uri = (Kernel.const_get(uri).uri rescue uri) if uri.match(/\A[A-Z]+\z/)
-        # for cases where `sto_uri: "DB"`
+      fail ArgumentError.new("no 'sto_uri' conf, cannot connect to db") \
+        unless uri
+
+      return Kernel.const_get(uri) \
+        if uri.is_a?(String) && uri.match(/\A[A-Z]+\z/)
+          # for cases where uri == 'DB'
+
+      Sequel.connect(uri)
+    end
+
+    def connect
 
-      @db = Sequel.connect(uri)
+      @db = derive_db
 
       class << @db; attr_accessor :flor_unit; end
       @db.flor_unit = @unit

data/lib/flor/unit/taskers.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -28,6 +29,29 @@ module Flor
     alias task_name taskname
 
     def vars; @message['vars']; end
+    alias variables vars
+
+    def set_payload(h)
+      fail TypeError.new("not a hash but a #{fs.class}") unless h.is_a?(Hash)
+      @message['payload'] = h
+    end
+    alias set_fields set_payload
+
+    def set_vars(h)
+      fail TypeError.new("not a hash but a #{fs.class}") unless h.is_a?(Hash)
+      @message['vars'] = h
+    end
+    alias set_variables set_vars
+
+    #def merge_into_payload(h)
+    #  @message['payload'].merge(h)
+    #end
+    #alias merge_into_fields merge_into_payload
+    #def merge_into_vars(h)
+    #  @message['vars'].merge(h)
+    #end
+      #
+      # no for now, payload.merge(h) and vars.merge(h) do suffice
 
     def execution
 
@@ -55,7 +79,11 @@ module Flor
 
     def reply(message=@message, force=false)
 
-      @ganger.return(message) if force || @ganger
+      fail ArgumentError.new(
+        "argument to reply must be a Hash but is #{message.class}"
+      ) unless message.is_a?(Hash)
+
+      @ganger.return(derive_message(message)) if force || @ganger
 
       [] # very important, return no further messages
     end
@@ -65,6 +93,92 @@ module Flor
       reply(
         Flor.to_error_message(@message, error))
     end
+
+    # So that #reply may be called with
+    # ```
+    # reply
+    # reply(@message)
+    # reply(payload: {})
+    # reply(ret: 123)
+    # reply(ret: 123, set: { a: 1 }, unset: [ :b ])
+    # ```
+    #
+    def derive_message(m)
+
+      exid = m['exid']
+      nid = m['nid']
+      pl = m['payload']
+
+      return m if Flor.is_exid?(exid) && Flor.is_nid?(nid) && pl.is_a?(Hash)
+
+      m = Flor.to_string_keyed_hash(m)
+      h = Flor.dup_message(@message)
+      ks = m.keys
+
+      if ks == [ 'payload' ]
+
+        h['payload'] = m['payload']
+
+      elsif (ks & %w[ ret set unset ]).size > 0
+
+        pl = (h['payload'] ||= {})
+
+        pl['ret'] = m['ret'] if m.has_key?('ret')
+        (m['set'] || {}).each { |k, v| pl[k] = v }
+        (m['unset'] || []).each { |k| pl.delete(k.to_s) }
+
+      else
+
+        h['payload'] = m
+      end
+
+      h
+    end
+  end
+
+  # A BasicTasker with stages (pre / on / post)
+  #
+  class StagedBasicTasker < BasicTasker
+
+    def call_task
+
+      call_one_of(:pre_task)
+      call_one_of(:on_task, :task)
+    end
+
+    def call_detask
+
+      call_one_of(:pre_detask, :pre_cancel)
+      call_one_of(:on_detask, :on_cancel, :detask, :cancel)
+    end
+
+    protected
+
+    def call_one_of(*ms)
+
+      m = ms.flatten.find { |mm| respond_to?(mm) }
+
+      send(m) if m
+    end
+
+    def reply(message=@message, force=false)
+
+      fail ArgumentError.new(
+        "argument to reply must be a Hash but is #{message.class}"
+      ) unless message.is_a?(Hash)
+
+      pt = @message['point']
+
+      ms = [ "post_#{pt}" ]; ms << :post_cancel if pt == 'detask'
+        #
+      call_one_of(ms)
+
+      msg = derive_message(message)
+
+      @ganger.return(msg) if force || @ganger
+
+      [] # very important, return no further messages
+    end
   end
 end