flor 0.10.0 → 0.11.0

data/CHANGELOG.md

@@ -2,6 +2,18 @@
 # flor CHANGELOG.md
 
 
+## flor 0.11.0  released 2017-03-17
+
+- Simplification of the tasker configuration files
+  (alignment on hooks configuration files)
+- Introduction of lib/hooks/
+- go for ; and | (same level) and \ (child level)
+- Implement "twig"
+- Implement basic spooler (var/spool/)
+- Introduce runner service
+- Fix Storage#any_message (misuse of Sequel Dataset#count)
+
+
 ## flor 0.10.0  released 2017-03-03
 
 - Enhance shell, bring in bin/flosh (though not in gem)

data/README.md

@@ -21,9 +21,13 @@ Flor is a "Ruby workflow engine", if that makes any sense.
 * All in all should be easy to maintain (engine itself and executions running
   on top of it)
 
+## Quickstart
+
+See [quickstart/](quickstart/).
+
 ## Documentation
 
-see [doc/](doc/).
+See [doc/](doc/).
 
 
 ## LICENSE

data/fail.txt

@@ -0,0 +1,3 @@
+rspec ./spec/unit/push_and_runs_spec.rb:30 # Flor unit push (run spanning) works
+rspec ./spec/unit/scheduler_spec.rb:371 # Flor unit Flor::Scheduler sch_msg_max_res_time flags as "active" messages that have been reserved for too long
+rspec ./spec/unit/waiter_spec.rb:127 # Flor::Waiter as a launch option lets wait until the scheduler gets idle

data/lib/flor.rb

@@ -12,7 +12,7 @@ require 'raabro'
 
 module Flor
 
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'
 end
 
 require 'flor/colours'

data/lib/flor/conf.rb

@@ -88,7 +88,7 @@ module Flor
     def self.get_class(conf, key)
 
       if v = conf[key]
-        Kernel.const_get(v)
+        Flor.const_lookup(v)
       else
         nil
       end

data/lib/flor/core/executor.rb

@@ -5,13 +5,16 @@ module Flor
 
     attr_reader :unit
     attr_reader :execution
+    attr_reader :hooks
     attr_reader :traps
 
-    def initialize(unit, traps, execution)
+    def initialize(unit, hooks, traps, execution)
 
       @unit = unit
       @execution = execution
-      @traps = traps
+
+      @hooks = hooks # raw hooks if any, fresh from the loader
+      @traps = traps # array of Trap instances
     end
 
     def conf; @unit.conf; end
@@ -54,17 +57,32 @@ module Flor
       counter_add(key, 1)
     end
 
+    def trigger_trap(trap, message)
+
+      del, msgs = trap.trigger(self, message)
+      @traps.delete(trap) if del
+
+      msgs
+    end
+
     def trigger_hook(hook, message)
 
       hook.notify(self, message)
     end
 
-    def trigger_trap(trap, message)
+    def trigger_block(block, opts, message)
 
-      del, msgs = trap.trigger(self, message)
-      @traps.delete(trap) if del
+      r =
+        if block.arity == 1
+          block.call(message)
+        elsif block.arity == 2
+          block.call(message, opts)
+        else
+          block.call(self, message, opts)
+        end
 
-      msgs
+      r.is_a?(Array) && r.all? { |e| e.is_a?(Hash) } ? r : []
+        # be lenient with block hooks, help them return an array
     end
 
     # Given a nid, returns a copy of all the var the node sees
@@ -88,6 +106,16 @@ module Flor
       vs
     end
 
+    def traps_and_hooks
+
+      @htraps = nil if @htraps && @htraps.size != @traps.size
+
+      @htraps ||= @traps.collect(&:to_hook)
+      @hhooks ||= @hooks.collect(&:to_hook)
+
+      @htraps + @hhooks
+    end
+
     protected
 
     def make_node(message)

data/lib/flor/core/node.rb

@@ -152,6 +152,8 @@ class Flor::Node
         @node
       elsif cat == 'v'
         lookup_var(@node, mod, key)
+      elsif cat == 't'
+        lookup_tag(mod, key)
       else
         lookup_field(mod, key)
       end
@@ -371,6 +373,17 @@ class Flor::Node
     lookup_var_name(parent_node(node), val)
   end
 
+  def lookup_tag(mod, key)
+
+    nids =
+      @execution['nodes'].inject([]) do |a, (nid, n)|
+        a << nid if n['tags'] && n['tags'].include?(key)
+        a
+      end
+
+    nids.empty? ? [ '_nul', nil, -1 ] : nids
+  end
+
   def lookup_field(mod, key)
 
     Flor.deep_get(payload.current, key)[1]
@@ -378,7 +391,8 @@ class Flor::Node
 
   def key_split(key) # => category, mode, key
 
-    m = key.match(/\A(?:([lgd]?)((?:v|var|variable)|w|f|fld|field)\.)?(.+)\z/)
+    m = key.match(
+      /\A(?:([lgd]?)((?:v|var|variable)|w|f|fld|field|t|tag)\.)?(.+)\z/)
 
     #fail ArgumentError.new("couldn't split key #{key.inspect}") unless m
       # spare that

data/lib/flor/core/texecutor.rb

@@ -64,6 +64,7 @@ module Flor
 
       super(
         TransientUnit.new(conf),
+        [], # no hooks
         [], # no traps
         {
           'exid' => Flor.generate_exid('eval', 'u0'),
@@ -166,8 +167,17 @@ module Flor
           ls.reject! { |l| l.strip[0, 1] == '#' }
           s = ls.join("\n").strip
         end
+
+      a, b = s[0, 1], s[-1, 1]
+
       s =
-        "{\n#{s}\n}" unless s[0, 1] == '{' && s[-1, 1] == '}'
+        if (a == '{' && b == '}') || (a == '[' && b == ']')
+          s
+        elsif s.match(/[^\r\n{]+:/) || s == ''
+          "{\n#{s}\n}"
+        else
+          "[\n#{s}\n]"
+        end
 
       vs = Hash.new { |h, k| k }
       class << vs
@@ -188,11 +198,16 @@ module Flor
         fail ae
       end
 
-      h = Flor.dup(r['payload']['ret'])
+      o = Flor.dup(r['payload']['ret'])
 
-      h.merge!('_path' => path) unless path.match(/[\r\n]/)
+      if o.is_a?(Hash)
+        o['_path'] = path unless path.match(/[\r\n]/)
+        o['root'] ||= Flor.relativize_path(vs['root'])
+      elsif o.is_a?(Array)
+        o.each { |e| e['_path'] = path } unless path.match(/[\r\n]/)
+      end
 
-      h
+      o
     end
 
     def self.determine_root(path)

data/lib/flor/djan.rb

@@ -156,7 +156,7 @@ module Flor
       indent_space(opts)
 
       if (
-        x.match(/\A[^: \b\f\n\r\t"',()\[\]{}#\\]+\z/) == nil ||
+        x.match(/\A[^: \b\f\n\r\t"',()\[\]{}#\\+%\/><^!=-]+\z/) == nil ||
         x.to_i.to_s == x ||
         x.to_f.to_s == x
       )

data/lib/flor/flor.rb

@@ -11,6 +11,11 @@ module Flor
   #
   # miscellaneous functions
 
+  def self.env_i(k)
+
+    v = ENV[k]; (v && v.match(/\A\d+\z/)) ? v.to_i : nil
+  end
+
   def self.dup(o)
 
     Marshal.load(Marshal.dump(o))
@@ -49,7 +54,7 @@ module Flor
     h['kla'] = o.class.to_s
     t = nil
 
-    if o.is_a?(Exception)
+    if o.is_a?(::Exception)
 
       h['msg'] = o.message
 
@@ -68,6 +73,8 @@ module Flor
     end
 
     h['trc'] = t[0..(t.rindex { |l| l.match(/\/lib\/flor\//) }) + 1] if t
+    h['cwd'] = Dir.pwd
+    h['rlp'] = $: if o.is_a?(::LoadError)
 
     h
   end
@@ -89,7 +96,7 @@ module Flor
 
     path = path[from.length + 1..-1] if path[0, from.length] == from
 
-    path
+    path || '.'
   end
 
   def self.is_array_of_messages?(o)
@@ -101,6 +108,25 @@ module Flor
       e.keys.all? { |k| k.is_a?(String) } }
   end
 
+  def self.h_fetch_a(h, *keys)
+
+    default = keys.last.is_a?(String) ? [] : keys.pop
+
+    k = keys.find { |k| h.has_key?(k) }
+    v = k ? h[k] : nil
+
+    v_to_a(v) || default
+  end
+
+  def self.v_to_a(o)
+
+    return o if o.is_a?(Array)
+    return o.split(',') if o.is_a?(String)
+    return nil if o.nil?
+
+    fail ArgumentError.new("cannot turn instance of #{o.class} into an array")
+  end
+
   #
   # functions about time
 
@@ -132,6 +158,15 @@ module Flor
     isostamp(true, false, false, t)
   end
 
+  def self.tamp(t=Time.now)
+
+    t = t.utc
+    s = StringIO.new
+    s << t.strftime('%Y%m%dT%H%M%S') << sprintf('.%06dZ', t.usec)
+
+    s.string
+  end
+
   # hour stamp
   #
   def self.hstamp(t=Time.now)

data/lib/flor/parser.rb

@@ -10,7 +10,6 @@ module Flor
     def dot(i); str(nil, i, '.'); end
     def colon(i); str(nil, i, ':'); end
     def comma(i); str(nil, i, ','); end
-    def bslash(i); str(nil, i, '\\'); end
 
     def pstart(i); str(nil, i, '('); end
     def pend(i); str(nil, i, ')'); end
@@ -72,11 +71,10 @@ module Flor
     def postval(i); rep(nil, i, :eol, 0); end
 
     def comma_eol(i); seq(nil, i, :comma, :eol, :ws_star); end
-    def bslash_eol(i); seq(nil, i, :bslash, :eol, :ws_star); end
-    def sep(i); alt(nil, i, :comma_eol, :bslash_eol, :ws_star); end
+    def sep(i); alt(nil, i, :comma_eol, :ws_star); end
 
     def comma_qmark_eol(i); seq(nil, i, :comma, '?', :eol); end
-    def coll_sep(i); alt(nil, i, :bslash_eol, :comma_qmark_eol, :ws_star); end
+    def coll_sep(i); alt(nil, i, :comma_qmark_eol, :ws_star); end
 
     def ent(i); seq(:ent, i, :key, :postval, :colon, :postval, :exp, :postval); end
     def ent_qmark(i); rep(nil, i, :ent, 0, 1); end
@@ -125,15 +123,20 @@ module Flor
     alias exp eor
 
     def key(i); seq(:key, i, :exp); end
-    def keycol(i); seq(nil, i, :key, :ws_star, :colon, :eol); end
+    def keycol(i); seq(nil, i, :key, :ws_star, :colon, :eol, :ws_star); end
 
     def att(i); seq(:att, i, :sep, :keycol, '?', :exp); end
     def head(i); seq(:head, i, :exp); end
-    def indent(i); rex(:indent, i, /[|; \t]*/); end
+    def indent(i); rex(:indent, i, /[ \t]*/); end
     def node(i); seq(:node, i, :indent, :head, :att, '*'); end
 
+    def linjoin(i); rex(nil, i, /[ \t]*[\\|;][ \t]*/); end
+    def outjnl(i); seq(nil, i, :linjoin, :comment, '?', :retnew); end
+    def outnlj(i); seq(nil, i, :ws_star, :comment, '?', :retnew, :linjoin); end
+    def outdent(i); alt(:outdent, i, :outjnl, :outnlj, :eol); end
+
     def line(i)
-      seq(:line, i, :node, '?', :eol)
+      seq(:line, i, :node, '?', :outdent)
     end
     def panode(i)
       seq(:panode, i, :pstart, :eol, :ws_star, :line, '*', :eol, :pend)
@@ -151,7 +154,7 @@ module Flor
 
     def rewrite_par(t)
 
-      Nod.new(t.lookup(:node)).to_a
+      Nod.new(t.lookup(:node), nil).to_a
     end
 
     def rewrite_ref(t); [ t.string, [], ln(t) ]; end
@@ -237,7 +240,7 @@ module Flor
       attr_accessor :parent, :indent
       attr_reader :children
 
-      def initialize(tree)
+      def initialize(tree, outdent)
 
         @parent = nil
         @indent = -1
@@ -245,15 +248,21 @@ module Flor
         @children = []
         @line = 0
 
+        @outdent = outdent ? outdent.strip : nil
+        @outdent = nil if @outdent && @outdent.size < 1
+
         read(tree) if tree
       end
 
       def append(node)
 
-        if node.indent == :east
-          node.indent = self.indent + 2
-        elsif node.indent == :south
-          node.indent = self.indent
+        if @outdent
+          if @outdent.index('\\')
+            node.indent = self.indent + 2
+          elsif @outdent.index('|') || @outdent.index(';')
+            node.indent = self.indent
+          end
+          @outdent = nil
         end
 
         if node.indent > self.indent
@@ -270,6 +279,21 @@ module Flor
 
         cn = @children.collect(&:to_a)
 
+        # make sure `[ 1 2 3 ] timeout: '2h'`
+        # turns into `_arr timeout: '2h'; 1 | 2 | 3`
+
+        if (
+          @head.is_a?(Array) &&
+          (@head[0] == '_arr' || @head[0] == '_obj') &&
+          #Flor.is_array_of_trees?(@head[1]) &&
+          cn.all? { |c| c[0] == '_att' && c[1].size > 1 }
+        )
+          cn = @head[1] + cn
+          @head = @head[0]
+          att, non_att = cn.partition { |c| c[0] == '_att' }
+          cn = att + non_att
+        end
+
         # detect if/unless suffix
 
         atts =
@@ -279,7 +303,6 @@ module Flor
             c.size == 1 && %w[ if unless ].include?(c[0][0]) && c[0][1] == []
           }
 
-        #return cn.first if @head == 'sequence' && @line == 0 && cn.size == 1
         return [ @head, cn, @line ] unless i
 
         # rewrite if/unless suffix
@@ -295,20 +318,20 @@ module Flor
 
       def read(tree)
 
-        if it = tree.lookup(:indent)
-
-          s = it.string
-          semicount = s.count(';')
-          pipe = s.index('|')
-
-          @indent =
-            if semicount == 1 then :east
-            elsif semicount > 1 || pipe then :south
-            else s.length; end
-        end
+        #if it = tree.lookup(:indent)
+        #  #s = it.string
+        #  #semicount = s.count(';')
+        #  #pipe = s.index('|')
+        #  #
+        #  #@indent =
+        #  #  if semicount == 1 then :east
+        #  #  elsif semicount > 1 || pipe then :south
+        #  #  else s.length; end
+        #end
+        @indent = tree.lookup(:indent).string.length
 
         ht = tree.lookup(:head)
-        @line = Lang.line_number(ht)
+        @line = Flor::Lang.line_number(ht)
 
         @head = Flor::Lang.rewrite(ht.c0)
         @head = @head[0] if @head[0].is_a?(String) && @head[1] == []
@@ -341,10 +364,13 @@ module Flor
 
     def rewrite_flor(t)
 
-      prev = root = Nod.new(nil)
+      prev = root = Nod.new(nil, nil)
 
-      t.gather(:node).each do |nt|
-        n = Nod.new(nt)
+      #t.gather(:node).each do |nt|
+      t.gather(:line).each do |lt|
+        nt = lt.lookup(:node); next unless nt
+        ot = lt.lookup(:outdent).string
+        n = Nod.new(nt, ot)
         prev.append(n)
         prev = n
       end