flor 0.17.0 → 0.18.0

data/lib/flor/punit/task.rb

@@ -68,6 +68,9 @@ class Flor::Pro::Task < Flor::Procedure
     # clean_up assign: 'alan'
     # "clean up" assign: 'alan'
 
+    @node['message'] = message
+      # keep copy for Executor#return integrity enforcement
+
     nis = atts(nil)
     ta = att('by', 'for', 'assign')
     tn = att('with', 'task')
@@ -79,8 +82,7 @@ class Flor::Pro::Task < Flor::Procedure
 
     attl, attd = determine_atts
 
-    @node['task'] =
-      { 'tasker' => tasker, 'name' => taskname }
+    @node['task'] = { 'tasker' => tasker, 'name' => taskname }
 
     wrap(
       'point' => 'task',
@@ -96,6 +98,9 @@ class Flor::Pro::Task < Flor::Procedure
 
     close_node
 
+    @node['message'] = message
+      # keep copy for Executor#return integrity enforcement
+
     attl, attd = determine_atts
 
     wrap(
@@ -103,6 +108,7 @@ class Flor::Pro::Task < Flor::Procedure
       'exid' => exid, 'nid' => nid,
       'tags' => list_tags,
       'tasker' => att(nil),
+      'taskname' => @node['task']['name'],
       'attl' => attl, 'attd' => attd,
       'payload' => determine_payload)
   end

data/lib/flor/punit/trap.rb

@@ -178,6 +178,37 @@ class Flor::Pro::Trap < Flor::Procedure
   # twice.
   #
   #
+  # ## the payload: directive
+  #
+  # The `payload:` attribute tells the trap what payload to hand to the
+  # trap handler.
+  #
+  # When `payload: 'event'`, the handler is given a copy of the fields as
+  # seen in the event that triggered the trap.
+  #
+  # When `payload: 'trap'`, the handler is given a copy of the fields as
+  # they were when and where the trap was set.
+  #
+  # When `payload: { a: 'A' }`, the given payload is used (here
+  # `{ 'a' => 'A' }`).
+  #
+  #
+  # ## blocking trap
+  #
+  # One can use a trap without a function. It will block the execution branch
+  # until the expect event, signal, etc occurs
+  # ```
+  # sequence
+  #   technical_writer 'prepare documentation'
+  #   team_manager 'submit documentation to review team'
+  #   trap signal: 'green' # go on after the 'green' signal...
+  #   team_manager 'deploy documentation'
+  # ```
+  # Here the execution will block after the team manager submits the doc
+  # for review. When the 'green' signal comes, the flow resumes to 'deploy
+  # documentation'.
+  #
+  #
   # ## see also
   #
   # On and signal.

data/lib/flor/unit/executor.rb

@@ -153,11 +153,15 @@ module Flor
 
     def return(message)
 
-      [ { 'point' => 'receive',
-          'exid' => message['exid'],
-          'nid' => message['nid'],
-          'payload' => message['payload'],
-          'tasker' => message['tasker'] } ]
+      n = @execution['nodes'][message['nid']] || {}
+      m = n['message'] || {}
+      c = m['cause']
+
+      rm = message.dup
+      rm['point'] = 'receive'
+      rm['cause'] = c if c # preserve 'cause' for routing
+
+      [ rm ]
     end
 
     def schedule(message)

data/lib/flor/unit/ganger.rb

@@ -22,6 +22,12 @@ module Flor
     def shutdown
     end
 
+    # Used by flor when it looks up for a variable and finds nothing.
+    # The last step is to ask the ganger if it knows about a tasker under
+    # the given (domain and) name.
+    #
+    # If it returns true, flor knows there is a tasker under that name.
+    #
     def has_tasker?(exid, name)
 
       #return false if RESERVED_NAMES.include?(name)
@@ -34,6 +40,9 @@ module Flor
         @unit.loader.tasker(d, name))
     end
 
+    # Called by Flor::Scheduler. The ganger then has to hand the task
+    # (the message) to the proper tasker.
+    #
     def task(executor, message)
 
       domain = message['exid'].split('-', 2).first
@@ -50,12 +59,11 @@ module Flor
       ) unless tconf
 
       if tconf.is_a?(Array)
-        tconf =
-          tconf.find { |h| h['point'] == message['point'] } ||
-          tconf.find { |h| h['point'] == nil }
-        tconf ||=
-          tconf.find { |h| h['point'] == 'cancel' } \
-            if message['point'] == 'detask'
+
+        points = [ nil, message['point'] ]
+        points << 'detask' if points.include?('cancel')
+
+        tconf = tconf.find { |h| points.include?(h['point']) }
       end
 
       message['tconf'] = tconf unless tconf['include_tconf'] == false
@@ -74,6 +82,9 @@ module Flor
         # especially if it's a domain tasker
     end
 
+    # Called by the tasker implementations when they're done with a task
+    # and want to hand it back to flor. It might be a failure message.
+    #
     def return(message)
 
       @unit.return(message)
@@ -120,6 +131,10 @@ module Flor
       }.compact
     end
 
+    # By default, taskers don't see the flor variables in the execution.
+    # If 'include_vars' or 'exclude_vars' is present in the configuration
+    # of the tasker, some or all of the variables are passed.
+    #
     def gather_vars(executor, tconf, message)
 
       # try to return before a potentially costly call to executor.vars(nid)

data/lib/flor/unit/models.rb

@@ -42,9 +42,9 @@ module Flor
 
       exid = @values[:exid]; return nil unless exid
 
-      @execution = nil if reload
+      @flor_model_cache_execution = nil if reload
 
-      @execution ||= unit.executions[exid: exid]
+      @flor_model_cache_execution ||= unit.executions[exid: exid]
     end
 
     # Returns the node hash linked to this model
@@ -64,17 +64,20 @@ module Flor
       nod ? nod['payload'] : nil
     end
 
-    def _data
-
-      d = Flor::Storage.from_blob(content)
-      d['id'] = id if d.is_a?(Hash)
+    def data(cache=true)
 
-      d
+      cache ? (@flor_model_cache_data = _data) : _data
     end
 
-    def data(cache=true)
+    def refresh
 
-      cache ? (@data = _data) : _data
+      instance_variables
+        .each do |k|
+          instance_variable_set(k, nil) \
+            if k.to_s.start_with?('@flor_model_cache_')
+        end
+
+      super
     end
 
     def to_h
@@ -88,6 +91,34 @@ module Flor
         h
       end
     end
+
+    class << self
+
+      def from_h(h)
+
+        cols = columns
+
+        h
+          .inject({}) { |r, (k, v)|
+            k = k.to_sym
+            if k == :data
+              r[:content] = Flor.to_blob(v)
+            elsif cols.include?(k)
+              r[k] = v
+            end
+            r }
+      end
+    end
+
+    protected
+
+    def _data
+
+      d = Flor::Storage.from_blob(content)
+      d['id'] = id if d.is_a?(Hash)
+
+      d
+    end
   end
 
   MODELS = [ :executions, :timers, :traces, :traps, :pointers, :messages ]

data/lib/flor/unit/models/trap.rb

@@ -88,9 +88,11 @@ module Flor
 
       c = c - 1
       data['count'] = c
-      self[:status] = c > 0 ? 'active' : 'consumed'
+      self[:status] = s = (c > 0) ? 'active' : 'consumed'
 
-      self.update(content: Flor::Storage.to_blob(@data), status: self[:status])
+      self.update(
+        content: Flor::Storage.to_blob(@flor_model_cache_data),
+        status: s)
 
       c < 1
     end
@@ -107,9 +109,15 @@ module Flor
       if sig = (message['point'] == 'signal' && message['name'])
         args << [ 'sig', sig ]
       end
-      if dat['pl'] == 'event'
+
+      case pl = dat['pl']
+      when 'event'
         args << [ 'payload', msg['payload'] ]
-        msg['payload'] = Flor.dup(message['payload']) # FIXME try without this line...
+        msg['payload'] = Flor.dup(message['payload'])
+      #when 'trap'
+      when Hash
+        msg['payload'] = Flor.dup(pl)
+      #else
       end
 
       { 'point' => 'trigger',

data/lib/flor/unit/scheduler.rb

@@ -238,7 +238,7 @@ module Flor
           message
         else
           message
-            .select { |k, _| %w[ exid nid payload tasker ].include?(k) }
+            .select { |k, _| %w[ exid nid payload tasker cause ].include?(k) }
             .merge!('point' => 'return')
         end
 
@@ -249,12 +249,21 @@ module Flor
 
     def cancel(exid, *as)
 
-      queue(*prepare_message('cancel', [ exid, *as ]))
+      msg, opts = prepare_message('cancel', [ exid, *as ])
+      msg['nid'] ||= '0'
+
+      queue(msg, opts)
     end
 
     def kill(exid, *as)
 
-      queue(*prepare_message('kill', [ exid, *as ]))
+      msg, opts = prepare_message('kill', [ exid, *as ])
+
+      msg['point'] = 'cancel'
+      msg['flavour'] = 'kill'
+      msg['nid'] ||= '0'
+
+      queue(msg, opts)
     end
 
     def signal(name, h={})
@@ -262,21 +271,38 @@ module Flor
       h[:payload] ||= {}
       h[:name] ||= name
 
-      queue(*prepare_message('signal', [ h ]))
+      msg, opts = prepare_message('signal', [ h ])
+
+      fail ArgumentError.new('missing :name string key') \
+        unless msg['name'].is_a?(String)
+
+      queue(msg, opts)
     end
 
     def re_apply(exid, *as)
 
-      queue(*prepare_message('cancel', [ exid, *as, { re_apply: true } ]))
+      msg, opts = prepare_message('cancel', [ exid, *as ])
+
+      msg['on_receive_last'] = prepare_re_apply_messages(msg, opts)
+
+      queue(msg, opts)
     end
     alias reapply re_apply
 
     def add_branches(exid, *as)
 
-      m = prepare_message('add-branches', [ exid, *as ]).first
+      msg, opts = prepare_message('add-branches', [ exid, *as ])
+
+      msg['point'] = 'add'
+      msg['trees'] = prepare_trees(opts)
 
-      exe = @storage.executions[exid: m['exid']]
-      pnid = m['nid']
+      msg['tnid'] = tnid =
+        opts.delete(:tnid) || msg.delete('nid')
+      msg['nid'] =
+        msg.delete('nid') || opts.delete(:pnid) || Flor.parent_nid(tnid)
+
+      exe = @storage.executions[exid: msg['exid']]
+      pnid = msg['nid']
       ptree = exe.lookup_tree(pnid)
 
       fail ArgumentError.new(
@@ -286,7 +312,7 @@ module Flor
         # not likely to happen, since leaves reply immediately
 
       size = ptree[1].size
-      tnid = (m['tnid'] ||= Flor.make_child_nid(pnid, size))
+      tnid = (msg['tnid'] ||= Flor.make_child_nid(pnid, size))
 
       cid = Flor.child_id(tnid)
 
@@ -306,19 +332,23 @@ module Flor
         "node #{pnid} has #{size} branch#{size == 1 ? '' : 'es'}"
       ) if cid > size
 
-      queue(m)
+      queue(msg, opts)
     end
     alias add_branch add_branches
 
     def add_iterations(exid, *as)
 
-      m = prepare_message('add-iterations', [ exid, *as ]).first
+      msg, opts = prepare_message('add-iterations', [ exid, *as ])
+
+      msg['point'] = 'add'
+      msg['elements'] = prepare_elements(opts)
+      msg['nid'] = msg.delete('nid') || opts.delete(:pnid)
 
-      exe = @storage.executions[exid: m['exid']]
-      nid = m['nid']
+      exe = @storage.executions[exid: msg['exid']]
+      nid = msg['nid']
 
       fail ArgumentError.new(
-        "cannot add iteration to missing execution #{m['exid'].inspect}"
+        "cannot add iteration to missing execution #{msg['exid'].inspect}"
       ) unless exe
 
       fail ArgumentError.new(
@@ -329,7 +359,7 @@ module Flor
         "cannot add iteration to missing node #{nid.inspect}"
       ) unless exe.lookup_tree(nid)
 
-      queue(m)
+      queue(msg, opts)
     end
     alias add_iteration add_iterations
 
@@ -387,6 +417,136 @@ module Flor
       ex ? ex.execution : nil
     end
 
+    DUMP_KEYS = %w[ timestamp executions timers traps pointers ]
+
+    # Dumps all or some of the executions to a JSON string.
+    # See Scheduler#load for importing.
+    #
+    # unit.dump -> string  # returns a JSON string of all executions
+    # unit.dump(io) -> io  # dumps the JSON to the given IO instance
+    #
+    # unit.dump(exid: i)  # dumps only the given execution
+    # unit.dump(exids: [ i0, i1 ])  # dumps only the givens executions
+    # unit.dump(domain: d)            # dumps exes from domains,
+    # unit.dump(domains: [ d0, d1 ])  # and their subdomains
+    # unit.dump(sdomain: d)            # dumps strictly from given domains,
+    # unit.dump(sdomains: [ d0, d1 ])  # doesn't look at subdomains
+    #
+    # unit.dump() { |h| ... }  # modify the has right before it's turned to JSON
+    #
+    def dump(io=nil, opts=nil, &block)
+
+      io, opts = nil, io if io.is_a?(Hash)
+      opts ||= {}
+
+      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
+        #
+      exis = o[:exid]
+      doms = o[:domain]
+      sdms = o[:strict_domain] || o[:sdomain]
+        #
+      filter = lambda { |q|
+        q = q.where(
+          exid: exis) if exis
+        q = q.where {
+          Sequel.|(*doms
+            .inject([]) { |a, d|
+              a.concat([
+                { domain: d },
+                Sequel.like(:domain, d + '.%') ]) }) } if doms
+        q = q.where(
+          domain: sdms) if sdms
+        q }
+
+      exs, tms, tps, pts =
+        storage.db.transaction {
+          [ filter[executions].collect(&:to_h),
+            filter[timers].collect(&:to_h),
+            filter[traps].collect(&:to_h),
+            filter[pointers].collect(&:to_h) ] }
+
+      o = io ? io : StringIO.new
+
+      h = {
+        timestamp: Flor.tstamp,
+        executions: exs,
+        timers: tms,
+        traps: tps,
+        pointers: pts }
+
+      block.call(h) if block
+
+      JSON.dump(h, o)
+
+      io ? io : o.string
+    end
+
+    # Read a previous JSON dump and loads it into the storage.
+    # Can be useful when testing, dumping once and reloading multiple times
+    # to test variants.
+    #
+    # load(string) -> h  # load all executions from given JSON string
+    #                    # returns object inserted stat hash
+    # load(io)  # load all executions from the given IO
+    # load(io, close: true)  # load from the given IO and close it after read
+    #
+    # load(x, exid: i)            # load only given executions,
+    # load(x, exids: [ i0, i1 ])  # ignore the rest of the data in the source
+    # load(x, domain: d)            # load only exes from given domains,
+    # load(x, domains: [ d0, d1 ])  # and their subdomains
+    # load(x, sdomain: d)            # load only exes from strict domains,
+    # load(x, sdomains: [ d0, d1 ])  # ignores exes in their subdomains
+    #
+    def load(string_or_io, opts={}, &block)
+
+      s = string_or_io
+      s = s.read if s.respond_to?(:read)
+      string_or_io.close if string_or_io.respond_to?(:close) && opts[:close]
+      h = JSON.load(s)
+
+      mks = DUMP_KEYS - h.keys
+      fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?
+
+      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
+        #
+      exis = o[:exid]
+      doms = o[:domain]
+      sdms = o[:strict_domain] || o[:sdomain]
+        #
+      doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms
+
+      counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }
+
+      storage.db.transaction do
+
+        (DUMP_KEYS - %w[ timestamp ]).each do |k|
+
+          y = k.to_sym
+          cla = storage.send(k)
+          cols = cla.columns
+
+          rows = h[k]
+            .inject([]) { |a, hh|
+
+              next a if exis && ! exis.include?(hh['exid'])
+              next a if doms && ! doms.find { |d| d.match(hh['domain']) }
+              next a if sdms && ! sdms.include?(hh['domain'])
+
+              counts[y] += 1
+              counts[:total] += 1
+
+              vals = cla.from_h(hh)
+              a << cols.collect { |c| vals[c] } }
+
+          cla.import(cols, rows) if rows.any?
+        end
+
+        block.call(h) if block
+      end
+
+      counts
+    end
+
     protected
 
     def tick
@@ -412,8 +572,11 @@ module Flor
         notify(nil, make_idle_message)
       end
 
-      sleep [ @heart_rate - (Time.now - t0), 0 ].max #\
-        #unless should_wake_up?
+      if @idle_count < 1
+        sleep 0.001
+      else
+        sleep([ @heart_rate - (Time.now - t0), 0.001 ].max)
+      end
 
     rescue Exception => ex
 
@@ -422,8 +585,8 @@ module Flor
 
     def prepare_message(point, args)
 
-      h =
-        args.inject({}) { |hh, a|
+      h = args
+        .inject({}) { |hh, a|
           if a.is_a?(Hash) then a.each { |k, v| hh[k.to_s] = v }
           elsif ! hh.has_key?('exid') then hh['exid'] = a
           elsif ! hh.has_key?('nid') then hh['nid'] = a
@@ -441,37 +604,8 @@ module Flor
         end
       end
 
-      if point == 'kill'
-
-        msg['point'] = 'cancel'
-        msg['flavour'] = 'kill'
-
-      elsif point == 'add-branches'
-
-        msg['point'] = 'add'
-        msg['trees'] = prepare_trees(opts)
-
-        msg['tnid'] = tnid =
-          opts.delete(:tnid) || msg.delete('nid')
-        msg['nid'] =
-          msg.delete('nid') || opts.delete(:pnid) || Flor.parent_nid(tnid)
-
-      elsif point == 'add-iterations'
-
-        msg['point'] = 'add'
-        msg['elements'] = prepare_elements(opts)
-        msg['nid'] = msg.delete('nid') || opts.delete(:pnid)
-      end
-
-      if opts[:re_apply]
-
-        msg['on_receive_last'] = prepare_re_apply_messages(msg, opts)
-      end
-
       fail ArgumentError.new('missing :exid key') \
         unless msg['exid'].is_a?(String)
-      fail ArgumentError.new('missing :name string key') \
-        if point == 'signal' && ! msg['name'].is_a?(String)
 
       [ msg, opts ]
     end
@@ -503,8 +637,9 @@ module Flor
 
     def prepare_re_apply_messages(msg, opts)
 
-      fail ArgumentError.new("missing 'payload' to re_apply") \
-        unless msg['payload']
+      pl = msg['payload']
+
+      fail ArgumentError.new("missing 'payload' to re_apply") unless pl
 
       t = Flor.parse(opts[:tree], Flor.caller_fname, {})
 
@@ -512,7 +647,7 @@ module Flor
           'exid' => msg['exid'], 'nid' => msg['nid'],
           'from' => 'parent',
           'tree' => t,
-          'payload' => msg['payload'] } ]
+          'payload' => pl } ]
     end
 
     def make_idle_message