flor 1.0.0 → 1.2.1

data/lib/flor/unit/executor.rb

@@ -109,15 +109,13 @@ module Flor
     rescue Exception => exc
 
 # TODO eventually, have a dump dir
+
       fn =
-        [
-          'flor',
-          @unit.conf['env'], @unit.identifier, @exid,
-          'r' + counter('runs').to_s
-        ].collect(&:to_s).join('_') + '.dump'
+        [ 'flor', @unit.conf['env'], @unit.identifier, @exid,
+          'r' + counter('runs').to_s ].collect(&:to_s).join('_') + '.dump'
 
       @unit.logger.error(
-        "#{self.class}#do_run()", exc, "(dumping to #{fn})")
+        "#{self.class}#do_run()", exc, "(dumping to #{fn} ...)")
 
       File.open(fn, 'wb') do |f|
         f.puts(Flor.to_pretty_s({
@@ -134,19 +132,26 @@ module Flor
         f.puts(on_do_run_exc(exc))
       end
 
+      @unit.logger.error(
+        "#{self.class}#do_run()", exc, "(dumped to #{fn})")
+
       #puts on_do_run_exc(exc)
         # dump notification above
     end
 
     def task(message)
 
-      return error_reply(
-        node(message['nid']),
-        message,
-        "don't know how to apply #{message['tasker'].inspect}"
-      ) if message['routed'] == false
-        #
-        # use an error message similar to what the core executor would emit
+      if message['routed'] == false
+
+        t = message['tasker']
+        n = node(message['nid'])
+
+        msg = n['heat0'] != t ?
+          "tasker #{t.inspect} not found" :
+          "don't know how to apply #{t.inspect}"
+
+        return error_reply(n, message, msg)
+      end
 
       @unit.ganger.task(self, message)
     end

data/lib/flor/unit/ganger.rb

@@ -54,6 +54,9 @@ module Flor
          (@unit.loader.tasker(domain, 'ganger', message) ||
           @unit.loader.tasker(domain, 'tasker', message))) ||
         @unit.loader.tasker(domain, tname, message)
+#puts "=" * 80
+#pp tconf
+#puts "=" * 80
 
       fail ArgumentError.new(
         "tasker #{tname.inspect} not found"

data/lib/flor/unit/gangers.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Flor
+
+  # A ModuleGanger accepts a `module:` conf entry that points to a Ruby
+  # module. The tasker implementations are searched for among the classes
+  # in the given module.
+  #
+  # Among the tasker classes (classes that respond to on_task, on_detask, ...)
+  # it selects the first tasker that matches the tasker name.
+  #
+  class ModuleGanger
+
+    def initialize(service, conf, message)
+
+      @service = service
+      @conf = conf
+      @message = message
+    end
+
+    def task
+
+      tas = @message['tasker']
+      clas = list_tasker_classes
+      cla = clas.find { |c| tasker_name(c) == tas }
+
+      return [ Flor.dup_and_merge(@message, 'routed' => false) ] \
+        unless cla
+
+      call_tasker(cla)
+    end
+
+    alias detask task
+
+    protected
+
+    def list_tasker_classes
+
+      mod_name = @conf['module']
+
+      fail ArgumentError.new('ganger module: configuration entry missing') \
+        unless mod_name
+
+      mod = Flor.const_lookup(mod_name) rescue nil
+
+      fail ArgumentError.new("ganger cannot find module #{mod_name.inspect}") \
+        unless mod
+
+      list_classes(mod, [])
+    end
+
+    def list_classes(start, r)
+
+      # place leave classes on top if possible
+      # within a level, sort alphabetically
+
+      clas = start.constants.collect { |co| start.const_get(co) }
+      clas, mods = clas.partition { |c| c.is_a?(Class) }
+
+      mods.each { |m| list_classes(m, r) }
+      r.concat(clas.select { |c| tasker?(c) }.sort_by { |c| c.name })
+
+      r
+    end
+
+    TASKER_METHODS = [
+      :on, :on_message,
+      :task, :on_task,
+      :detask, :on_detask, :cancel, :on_cancel
+        ].freeze
+
+    def tasker?(cla)
+
+      return true if (TASKER_METHODS & cla.public_instance_methods).any?
+      return true if (TASKER_METHODS & cla.public_methods).any?
+      false
+    end
+
+    def tasker_name(cla)
+
+      if cla.public_instance_methods.include?(:tasker_name)
+
+        unless cla.respond_to?(:_ganged)
+          class << cla
+            attr_accessor :_ganged
+          end
+          cla._ganged = cla.allocate
+        end
+
+        call_tasker_name(cla._ganged)
+
+      elsif cla.public_methods.include?(:tasker_name)
+
+        call_tasker_name(cla)
+
+      else
+
+        cla.name.split('::').last.gsub(/Tasker\z/, '')
+          .gsub(/([a-z])([A-Z])/) { |_| $1 + '_' + $2.downcase }
+          .gsub(/([A-Z])/) { |c| c.downcase }
+      end
+    end
+
+    def call_tasker_name(o)
+
+      case i = o.method(:tasker_name).arity
+      when 1 then o.tasker_name(@message)
+      when 2 then o.tasker_name(@conf, @message)
+      when 3 then o.tasker_name(@service, @conf, @message)
+      when -1 then o.tasker_name(
+        service: @service, conf: @conf, message: @message)
+      else o.tasker_name
+      end
+    end
+
+    def call_tasker(c)
+
+      cnf = @conf.merge('class' => c)
+
+      @service.unit.caller
+        .call(@service, cnf, @message)
+    end
+  end
+end
+

data/lib/flor/unit/loader.rb

@@ -46,7 +46,7 @@ module Flor
         .collect { |pa| [ pa, expose_d(pa, {}) ] }
         .select { |pa, d| Flor.sub_domain?(d, domain) }
         .sort_by { |pa, d| d.count('.') }
-        .inject({}) { |vars, (pa, _)| vars.merge!(eval(pa, {})) }
+        .inject({}) { |vars, (pa, _)| vars.merge!(eval_variables(pa, {})) }
     end
 
     #def procedures(path)
@@ -87,25 +87,28 @@ module Flor
         .select { |pa| pa.index('/lib/taskers/') }
         .collect { |pa| [ pa, *expose_dn(pa, {}) ] }
         .select { |pa, d, n|
-          Flor.sub_domain?([ d, n ].join('.'), domain) ||
+          Flor.sub_domain?([ d, n ], domain) ||
           (n == name && Flor.sub_domain?(d, domain)) }
         .sort_by { |pa, d, n| d.count('.') }
         .last
 
       return nil unless pat
 
-      conf = eval(pat, message)
+      conf = eval_tasker_conf(pat, message)
 
       return conf if nam == name
 
-      conf = conf[name]
+      cnf = conf[name]
 
-      return nil unless conf
+      return nil unless cnf
 
-      (conf.is_a?(Array) ? conf : [ conf ])
-        .each { |h| h['_path'] = pat }
+      extras = conf.select { |_, v| ! v.is_a?(Hash) }
+      extras['_path'] = pat
 
-      conf
+      (cnf.is_a?(Array) ? cnf : [ cnf ])
+        .each { |h| h.merge!(extras) }
+
+      cnf
     end
 
     def hooks(domain)
@@ -119,8 +122,8 @@ module Flor
         .select { |pa, d| Flor.sub_domain?(d, domain) }
         .sort_by { |pa, d| d.count('.') }
         .collect { |pa, d|
-          eval(pa, {})
-            .each_with_index { |h, i| h['_path'] = pa + ":#{i}" } }
+          eval_hook_conf(pa, {})
+            .each_with_index { |h, i| h['_path'] = "#{pa}:#{i}" } }
         .flatten(1)
     end
 
@@ -247,6 +250,35 @@ module Flor
       end
     end
 
+    def eval_variables(path, context)
+      eval(path, context)
+    end
+    def eval_tasker_conf(path, context)
+      eval(path, context)
+    end
+# TODO like in eval_hook_conf, reject fautly tasker confs...
+# TODO like in eval_hook_conf, reject fautly variables...
+
+    def eval_hook_conf(path, context)
+
+      a = eval(path, context)
+
+      fail ArgumentError.new(
+        "hook conf at #{path} must be an array of hashes"
+      ) unless a.is_a?(Array)
+
+      a.each do |e|
+        fail ArgumentError.new(
+          "hook conf at #{path} has non-hash entry #{e.inspect}"
+        ) unless e.is_a?(Hash)
+        fail ArgumentError.new(
+          "hook conf at #{path} has incorrect point #{e['point'].inspect}"
+        ) unless e['point'].is_a?(String)
+      end
+
+      a
+    end
+
     def eval(path, context)
 
       ext =

data/lib/flor/unit/logger.rb

@@ -71,7 +71,8 @@ module Flor
       dbi = ' ' + dbi if dbi.length > 0
 
       txt = elts.collect(&:to_s).join(' ')
-      err = elts.find { |e| e.is_a?(Exception) }
+
+      err = find_err(elts)
 
       head = "#{stp} #{@uni}#{dbi} #{lvl} "
 
@@ -198,7 +199,8 @@ module Flor
 
       return unless @unit.conf['log_err']
 
-      @out.puts(Flor.msg_to_detail_s(executor, message, opts.merge(flag: true)))
+      s = Flor.msg_to_detail_s(executor, message, opts.merge(flag: true))
+      @out.puts(s) if s
     end
 
     def log_src(source, opts, log_opts={})
@@ -245,6 +247,13 @@ module Flor
       message[0..k + 2 + 4] + "(...len#{i - (k + 2 + 1)})" + message[i..-1]
     end
 
+    def find_err(elts)
+
+      elts.find { |e| e.is_a?(Exception) } ||
+      (defined?(Java) &&
+       elts.find { |e| e.class.ancestors.include?(Java::JavaLang::Error) })
+    end
+
     class Out
 
       attr_reader :unit

data/lib/flor/unit/models/message.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+
 module Flor
 
   class Message < FlorModel
@@ -19,6 +20,10 @@ module Flor
     #
     #  index :exid
     #end
+
+    def nid; data['nid']; end
+    def tasker; data['tasker']; end
+    alias payload data
   end
 end
 

data/lib/flor/unit/models/pointer.rb

@@ -30,7 +30,7 @@ module Flor
     #end
 
     # If the pointer is a "var" pointer, returns the full value
-    # for the variable, as fund in the execution's node "0".
+    # for the variable, as found in the execution's node "0".
     #
     def full_value
 
@@ -38,6 +38,25 @@ module Flor
 
       node['vars'][name]
     end
+
+    def attd
+
+      data['atts'].inject({}) { |h, (k, v)| h[k] = v if k; h }
+
+    rescue; []
+    end
+
+    def attl
+
+      data['atts'].inject([]) { |a, (k, v)| a << v if k == nil; a }
+
+    rescue; []
+    end
+
+    def att_texts
+
+      attl.select { |e| e.is_a?(String) }
+    end
   end
 end
 

data/lib/flor/unit/scheduler.rb

@@ -138,11 +138,7 @@ module Flor
 
       # TODO heartbeat, every x minutes, when idle, log something
 
-      fail(
-        "database not ready, " +
-        "db ver: #{@storage.db_version.inspect}, " +
-        "mig ver: #{@storage.migration_version}"
-      ) if !! @conf['sto_migration_check'] && @storage.ready?
+      check_migration_version
 
       @thread_status = :running
 
@@ -162,6 +158,15 @@ module Flor
       self
     end
 
+    def check_migration_version
+
+      fail(
+        "database not ready, " +
+        "db ver: #{@storage.db_version.inspect}, " +
+        "mig ver: #{@storage.migration_version}"
+      ) if !! @conf['sto_migration_check'] && @storage.ready?
+    end
+
     def stop
 
       @thread_status = :stop
@@ -227,6 +232,9 @@ module Flor
       @last_queued_message_id =
         @storage.put_message(message)
 
+      # Nota bene:
+      # the #wait method is added to the Scheduler by Flor::WaitList
+
       if opts[:wait]
         wait(message['exid'], opts)
       else
@@ -587,7 +595,7 @@ module Flor
 
     rescue Exception => ex
 
-      puts on_start_exc(ex)
+      puts(on_start_exc(ex))
     end
 
     def prepare_message(point, args)
@@ -675,9 +683,9 @@ module Flor
     def should_wake_up?
 
       return true if @wake_up
-      return true if Time.now - @reloaded_at >= reload_after
+      return true if (Time.now - @reloaded_at) >= reload_after
 
-      @next_time && @next_time <= Flor.tstamp.split('.').first
+      @next_time && (@next_time <= Flor.tstam)
     end
 
     def unreserve_messages

data/lib/flor/unit/storage.rb

@@ -7,6 +7,14 @@ module Flor
 
   class Storage
 
+    MESSAGE_COLUMNS = [
+      :domain, :exid, :point, :content,
+      :status, :ctime, :mtime, :cunit, :munit
+        ].freeze
+    POINTER_COLUMNS = [
+      :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit
+        ].freeze
+
     attr_reader :unit, :db, :models
 
     attr_reader :mutex
@@ -352,8 +360,7 @@ module Flor
 
           @db[:flor_messages]
             .import(
-              [ :domain, :exid, :point, :content,
-                :status, :ctime, :mtime, :cunit, :munit ],
+              MESSAGE_COLUMNS,
               unstored.map { |m|
                 [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
                   'created', n, n, u, u ] }) \
@@ -517,25 +524,6 @@ module Flor
       end
     end
 
-    def put_task_pointer(msg, tname, tconf)
-
-      exid = msg['exid']
-      dom = Flor.domain(exid)
-
-      synchronize do
-
-        @db[:flor_pointers]
-          .insert(
-            domain: dom,
-            exid: exid,
-            nid: msg['nid'],
-            type: 'tasker',
-            name: tname,
-            ctime: Flor.tstamp,
-            cunit: @unit.identifier)
-      end
-    end
-
     def fetch_next_time
 
       t =
@@ -598,14 +586,13 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .update(
             status: 'consumed', mtime: n, munit: u)
 
         @db[:flor_messages]
           .import(
-            [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime, :cunit, :munit ],
+            MESSAGE_COLUMNS,
             messages
               .select { |m|
                 ! m['mid'] && POINTS_TO_ARCHIVE.include?(m['point']) }
@@ -621,19 +608,16 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .delete
       end
     end
 
     def load_timers
 
-      now = Flor.tstamp
-      no = now[0, now.rindex('.')]
-
       timers
         .where(status: 'active')
-        .where { ntime <= no }
+        .where { ntime <= Flor.tstam }
         .order(:ntime)
         .all
 
@@ -703,9 +687,9 @@ module Flor
 
     def update_pointers(exe, status, now)
 
-# Q Should we archive old pointers?
-#   Well, it might be better to only archive the execution and leave
-#   in there enough information...
+# Q  Should we archive old pointers?
+# A  Well, it might be better to only archive the execution and leave
+#    in there enough information...
 
       exid = exe['exid']
 
@@ -731,23 +715,25 @@ module Flor
 
           ts = node['tags']
           ts.each { |t|
-            a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts
+            a << [ dom, exid, nid, 'tag', t, nil, now, u, nil ] } if ts
 
           vs = nid == '0' ? node['vars'] : nil
           vs.each { |k, v|
             case v; when Numeric, String, TrueClass, FalseClass, NilClass
-              a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
+              a << [ dom, exid, '0', 'var', k, v.to_s, now, u, nil ]
             when Array, Hash
               s = '(array)'; s = '(object)' if v.is_a?(Hash)
-              a << [ dom, exid, '0', 'var', k, s, now, u ]
+              a << [ dom, exid, '0', 'var', k, s, now, u, nil ]
             else
-              a << [ dom, exid, '0', 'var', k, nil, now, u ]
+              a << [ dom, exid, '0', 'var', k, nil, now, u, nil ]
             end } if vs
 
-          #ta = node['heap'] == 'task' ? node['task'] : nil
-          ta = node['task']
-          a << [ dom, exid, nid, 'tasker', ta['tasker'], ta['name'], now, u ] \
-            if ta
+          if ta = node['task']
+            tasker = ta['tasker']
+            name = ta['name']
+            content = { message: node['message'], atts: node['atts'] }
+            a << [ dom, exid, nid, 'tasker', tasker, name, now, u, content ]
+          end
 
           a }
 
@@ -755,17 +741,35 @@ module Flor
         .where(exid: exid)
         .select(:nid, :type, :name)
         .all
-      pointers.reject! { |_, _, ni, ty, na, _, _, _|
+      pointers.reject! { |_, _, ni, ty, na, _, _, _, _|
         cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
           #
           # don't insert when already inserted
 
+      if pointer_columns.include?(:content)
+        pointers.each { |ptr|
+          c = ptr[8]; ptr[8] = to_blob(c) if c }
+      else
+        pointers.each { |ptr|
+          ptr.pop }
+      end
+
       @db[:flor_pointers]
         .import(
-          [ :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit ],
+          pointer_columns,
           pointers)
     end
 
+    def pointer_columns
+
+      @pointer_columns ||=
+        if @db[:flor_pointers].columns.include?(:content)
+          POINTER_COLUMNS + [ :content ]
+        else
+          POINTER_COLUMNS
+        end
+    end
+
     def determine_type_and_schedule(message)
 
       t, s = message['type'], message['string']
@@ -822,8 +826,9 @@ module Flor
       fail ArgumentError.new("no 'sto_uri' conf, cannot connect to db") \
         unless uri
 
-      #uri = uri.to_s
-      #return Kernel.const_get(uri) if uri.match(/\A[A-Z]+\z/)
+      return Kernel.const_get(uri) \
+        if uri.is_a?(String) && uri.match(/\A[A-Z]+\z/)
+          # for cases where uri == 'DB'
 
       Sequel.connect(uri)
     end
@@ -844,8 +849,11 @@ module Flor
           # NB: -1 means "check at every use"
       end
 
-      @db_logger = DbLogger.new(@unit)
-      @db.loggers << @db_logger
+      if @unit.conf['sto_db_logger'] != false
+
+        @db_logger = DbLogger.new(@unit)
+        @db.loggers << @db_logger
+      end
     end
 
     class << self