flor 0.15.0 → 0.16.0

data/lib/flor/unit/executor.rb

@@ -58,23 +58,21 @@ module Flor
 
       (@unit.conf['exe_max_messages'] || 77).times do |i|
 
+        break if @shutdown
+
         m = @messages.shift
         break unless m
-        break if @shutdown
 
-        if m['point'] == 'terminated' && @messages.any?
-          #
-          # try to handle 'terminated' last
-          #
-          @messages << m
-          m = @messages.shift
-        end
+        m = (@messages << m).shift \
+          if m['point'] == 'terminated' && @messages.any?
+            #
+            # handle 'terminated' messages last
 
         ms = process(m)
 
         @consumed << m
 
-        ims, oms = ms.partition { |m| m['exid'] == @exid }
+        ims, oms = ms.partition { |mm| mm['exid'] == @exid }
           # qui est "in", qui est "out"?
 
         counter_add('omsgs', oms.size)
@@ -95,7 +93,7 @@ module Flor
       t0 = Flor.tstamp(t0)
 
       @unit.logger.log_run_end(self, t0, du)
-      @unit.hooker.notify(self, make_end_message(t0, du))
+      @unit.hooker.notify(self, make_end_message(t0, du, @execution['size']))
 
       @consumed.clear
 
@@ -131,6 +129,29 @@ module Flor
         # dump notification above
     end
 
+    def task(message)
+
+      return error_reply(
+        node(message['nid']),
+        message,
+        "don't know how to apply #{message['tasker'].inspect}"
+      ) if message['routed'] == false
+        #
+        # use an error message similar to what the core executor would emit
+
+      @unit.ganger.task(self, message)
+    end
+    alias detask task
+
+    def return(message)
+
+      [ { 'point' => 'receive',
+          'exid' => message['exid'],
+          'nid' => message['nid'],
+          'payload' => message['payload'],
+          'tasker' => message['tasker'] } ]
+    end
+
     def schedule(message)
 
       @unit.schedule(message)
@@ -186,7 +207,7 @@ module Flor
       io.string
     end
 
-    def make_end_message(start, duration)
+    def make_end_message(start, duration, execution_size)
 
       m = {}
       m['point'] = 'end'
@@ -196,7 +217,7 @@ module Flor
       m['consumed'] = @consumed.size
       m['counters'] = Flor.dup(@execution['counters'])
       m['nodes'] = @execution['nodes'].size
-      m['size'] = @execution['size']
+      m['execution_size'] = execution_size
       m['archive_size'] = @unit.archive[@exid].size if @unit.archive
       m['er'] = @execution['counters']['runs'] # "emitting run"
       m['pr'] = m['er'] # "processing run"

data/lib/flor/unit/ganger.rb

@@ -62,7 +62,16 @@ module Flor
 
       message['vars'] = gather_vars(executor, tconf, message)
 
-      @unit.caller.call(self, tconf, message)
+      m = Flor.dup(message)
+        #
+        # the tasker gets a copy of the message (and it can play with it
+        # to its heart content), meanwhile the message is handed to the
+        # "post" notifiers.
+
+      @unit.caller.call(self, tconf, m)
+        #
+        # might return a re-routing message,
+        # especially if it's a domain tasker
     end
 
     def return(message)

data/lib/flor/unit/hook.rb

@@ -1,6 +1,8 @@
 
 module Flor
 
+  # Used by Flor::Loader to prepare hooks from hooks.json files.
+  #
   class Hook
 
     def initialize(unit, exid, h)
@@ -29,7 +31,6 @@ module Flor
       [ "hook#{object_id}", opts, self, nil ]
     end
 
-
     def notify(executor, message)
 
       @unit.caller.call(executor, @h, message)

data/lib/flor/unit/hooker.rb

@@ -22,7 +22,7 @@ module Flor
 
     def [](name)
 
-      h = @hooks.find { |n, o, h, b| n == name }
+      h = @hooks.find { |n, _, _, _| n == name }
 
       h ? h[2] || h[3] : nil
     end
@@ -46,7 +46,7 @@ module Flor
         end
       end
 
-      hook = hook.new(@unit) if hook.is_a?(Class)
+      hook = instantiate_hook(hook) if hook.is_a?(Class)
 
       @hooks << [ name, opts, hook, block ]
     end
@@ -73,6 +73,17 @@ module Flor
 
     protected
 
+    def instantiate_hook(hook_class)
+
+      a =
+        case i = hook_class.instance_method(:initialize).arity
+        when 0, 1 then [ @unit ][0, i]
+        else []
+        end
+
+      hook_class.new(*a)
+    end
+
     def o(opts, *keys)
 
       array = false

data/lib/flor/unit/loader.rb

@@ -25,7 +25,7 @@ module Flor
         .collect { |pa| [ pa, expose_d(pa, {}) ] }
         .select { |pa, d| is_subdomain?(domain, d) }
         .sort_by { |pa, d| d.count('.') }
-        .inject({}) { |vars, (pa, d)| vars.merge!(eval(pa, {})) }
+        .inject({}) { |vars, (pa, _)| vars.merge!(eval(pa, {})) }
     end
 
     #def procedures(path)
@@ -45,7 +45,7 @@ module Flor
         name = name[0..m[1].length - 1]
       end
 
-      path, d, n = (Dir[File.join(@root, '**/*.{flo,flor}')])
+      path, _, _ = (Dir[File.join(@root, '**/*.{flo,flor}')])
         .select { |f| f.index('/lib/') }
         .collect { |pa| [ pa, *expose_dn(pa, opts) ] }
         .select { |pa, d, n| n == name && is_subdomain?(domain, d) }
@@ -62,7 +62,7 @@ module Flor
 
       domain, name = split_dn(domain, name)
 
-      path, d, n = Dir[File.join(@root, '**/*.json')]
+      pat, _, nam = Dir[File.join(@root, '**/*.json')]
         .select { |pa| pa.index('/lib/taskers/') }
         .collect { |pa| [ pa, *expose_dn(pa, {}) ] }
         .select { |pa, d, n|
@@ -71,18 +71,18 @@ module Flor
         .sort_by { |pa, d, n| d.count('.') }
         .last
 
-      return nil unless path
+      return nil unless pat
 
-      conf = eval(path, message)
+      conf = eval(pat, message)
 
-      return conf if n == name
+      return conf if nam == name
 
       conf = conf[name]
 
       return nil unless conf
 
       (conf.is_a?(Array) ? conf : [ conf ])
-        .each { |h| h['_path'] = path }
+        .each { |h| h['_path'] = pat }
 
       conf
     end

data/lib/flor/unit/logger.rb

@@ -68,7 +68,6 @@ module Flor
       line = "#{stp} #{@uni} #{lvl} #{txt}"
 
       if err
-        sts = ' ' * stp.length
         dig = lvl[0, 1] + Digest::MD5.hexdigest(line)[0, 4]
         @out.puts("#{stp} #{@uni} #{lvl} #{dig} #{txt}")
         err.backtrace.each { |lin| @out.puts("  #{dig} #{@uni} #{lin}") }
@@ -117,6 +116,11 @@ module Flor
       @out.puts "#{_c.blg}sto#{_c.rs} t#{Thread.current.object_id} #{level.upcase} #{msg}"
     end
 
+    def size_to_s(s)
+
+      "%.2fk" % (s.to_f / 1024)
+    end
+
     def log_run_start(executor)
 
       return unless @unit.conf['log_run']
@@ -128,14 +132,11 @@ module Flor
       s << _c.dg
       s << "    /--- #{_c.lg}run starts#{_c.dg} "
       s << "#{executor.class} #{executor.object_id} #{execution['exid']}"
-      s << "\n    |   "
-      s << { thread: Thread.current.object_id }.inspect
-      s << "\n    |   "
-      s << {
+      s << "\n    |   "; s << Flor.to_dnc(thread: Thread.current.object_id)
+      s << "\n    |   "; s << Flor.to_dnc(
         counters: execution['counters'],
         nodes: execution['nodes'].size,
-        size: execution['size']
-      }.inspect
+        execution_size: size_to_s(execution['size']))
       s << _c.rs
 
       @out.puts(s.string)
@@ -153,23 +154,20 @@ module Flor
 
       s << _c.dg
       s << "    |   run ends #{self.class} #{self.object_id} #{exid}"
-      s << "\n    |   "; s << { started: tstamp, took: duration }.inspect
-      s << "\n    |   "; s << {
+      #s << "\n    |   "; s << Flor.to_dnc(started: tstamp, took: duration)
+      s << "\n    |   "; s << Flor.to_dnc(started: tstamp, took: duration)
+      s << "\n    |   "; s << Flor.to_dnc(
         thread: Thread.current.object_id,
         consumed: executor.consumed.count,
-        traps: executor.traps.count,
-      }.inspect
-      s << "\n    |   "; s << {
+        traps: executor.traps.count)
+      s << "\n    |   "; s << Flor.to_dnc(
         #own_traps: @traps.reject { |t| t.texid == nil }.size, # FIXME
         counters: execution['counters'],
         nodes: execution['nodes'].size,
-        size: execution['size']
-      }.inspect
+        execution_size: size_to_s(execution['size']))
       if @unit.archive
         s << "\n    |   "
-        s << {
-          archive_size: (@unit.archive[exid].size rescue '???')
-        }.inspect
+        s << Flor.to_dnc(archive_size: (@unit.archive[exid].size rescue '???'))
       end
       s << "\n    \\--- ."
       s << _c.rs

data/lib/flor/unit/models.rb

@@ -15,8 +15,8 @@ module Flor
     end
 
     def fetch_rows(sql); yield([]); end
-    #DJV 
-    # add mising methods from dummy adaptor
+    #DJV
+    # add missing methods from dummy adaptor
     def typecast_value_boolean(opts={});true;end
     def test_connection();true;end
     #DJV
@@ -26,6 +26,8 @@ module Flor
 
   class FlorModel < Sequel::Model(DummySequelAdapter::DB)
 
+    self.require_valid_table = false
+
     def _data
 
       d = Flor::Storage.from_blob(content)

data/lib/flor/unit/models/execution.rb

@@ -3,6 +3,21 @@ module Flor
 
   class Execution < FlorModel
 
+    #create_table :flor_executions do
+    #
+    #  primary_key :id, type: :Integer
+    #  String :domain, null: false
+    #  String :exid, null: false
+    #  File :content # JSON
+    #  String :status, null: false # 'active' or something else like 'archived'
+    #  String :ctime, null: false
+    #  String :mtime, null: false
+    #  String :cunit
+    #  String :munit
+    #
+    #  index :exid
+    #end
+
     def nodes; data['nodes']; end
     def zero_node; nodes['0']; end
 
@@ -43,63 +58,93 @@ module Flor
       zero_node['vars']
     end
 
-    # class methods
+    def to_h
 
-    def self.by_status(s)
+      h = super
 
-      self.where(status: s)
-    end
+      h[:size] = self[:content].size
 
-    def self.terminated
+      m = h[:meta] = {}
+      cs = m[:counts] = {}
+      is = m[:nids] = { tasks: [], failures: [] }
 
-      by_status('terminated')
+      fs = 0
+      ts = 0
+      nodes.each do |k, v|
+        if v['task']
+          ts += 1
+          is[:tasks] << k
+        end
+        if v['failure']
+          fs += 1
+          is[:failures] << k
+        end
+      end
+      cs[:nodes] = nodes.count
+      cs[:failures] = fs
+      cs[:tasks] = ts
+
+      h
     end
 
-    def self.by_tag(name)
+    class << self
 
-      _exids = self.db[:flor_pointers]
-        .where(type: 'tag', name: name, value: nil)
-        .select(:exid)
-        .distinct
+      def by_status(s)
 
-      self.where(status: 'active', exid: _exids)
-    end
+        where(status: s)
+      end
+
+      def terminated
+
+        by_status('terminated')
+      end
 
-    def self.by_var(name, value=:no)
+      def by_tag(name)
 
-      w = { type: 'var', name: name }
+        _exids = db[:flor_pointers]
+          .where(type: 'tag', name: name, value: nil)
+          .select(:exid)
+          .distinct
 
-      case value; when nil
-        w[:value] = nil
-      when :no
-        # no w[:value] "constraining"
-      else
-        w[:value] = value.to_s
+        where(status: 'active', exid: _exids)
       end
 
-      _exids = self.db[:flor_pointers]
-        .where(w)
-        .select(:exid)
-        .distinct
+      def by_var(name, value=:no)
 
-      self.where(status: 'active', exid: _exids)
-    end
+        w = { type: 'var', name: name }
+
+        case value; when nil
+          w[:value] = nil
+        when :no
+          # no w[:value] "constraining"
+        else
+          w[:value] = value.to_s
+        end
 
-    def self.by_tasker(name, taskname=:no)
+        _exids = db[:flor_pointers]
+          .where(w)
+          .select(:exid)
+          .distinct
 
-      w = { type: 'tasker', name: name }
-      w[:value] = taskname if taskname != :no
+        where(status: 'active', exid: _exids)
+      end
 
-      _exids = self.db[:flor_pointers]
-        .where(w)
-        .select(:exid)
-        .distinct
+      def by_tasker(name, taskname=:no)
 
-      self.where(status: 'active', exid: _exids)
-    end
+        w = { type: 'tasker', name: name }
+        w[:value] = taskname if taskname != :no
 
-#    def self.by_task(name)
-#    end
+        _exids = db[:flor_pointers]
+          .where(w)
+          .select(:exid)
+          .distinct
+
+        where(status: 'active', exid: _exids)
+      end
+
+#      def by_task(name)
+#      end
+    end
   end
 end
 

data/lib/flor/unit/models/message.rb

@@ -2,6 +2,22 @@
 module Flor
 
   class Message < FlorModel
+
+    #create_table :flor_messages do
+    #
+    #  primary_key :id, type: :Integer
+    #  String :domain, null: false
+    #  String :exid, null: false
+    #  String :point, null: false # 'execute', 'task', 'receive', ...
+    #  File :content # JSON
+    #  String :status, null: false
+    #  String :ctime, null: false
+    #  String :mtime, null: false
+    #  String :cunit
+    #  String :munit
+    #
+    #  index :exid
+    #end
   end
 end