flor 0.18.0 → 1.2.0

data/lib/flor/unit/caller_jruby.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Flor
+
+  class Caller
+
+    def split_cmd(cmd)
+
+      #cmd.split(/ +/)
+        # too naive
+
+#Raabro.pp(Flor::Caller::CmdParser.parse(cmd, debug: 3), colours: true)
+      Flor::Caller::CmdParser.parse(cmd)
+    end
+
+    class ProcessStatus
+
+      attr_reader :process, :exitstatus
+
+      def initialize(process, exitstatus)
+
+        @process = process
+        @exitstatus = exitstatus
+      end
+
+      def pid; @process.pid; rescue; nil; end
+    end
+
+    def spawn(conf, data)
+
+      t0 = Time.now
+
+      cmd = conf['cmd']
+      henv, *acmd = split_cmd(cmd)
+
+      to = Fugit.parse(conf['timeout'] || '14s')
+      to = to.is_a?(Fugit::Duration) ? to.to_sec : 14
+      to = 0 if to < 0 # no timeout
+
+      builder = java.lang.ProcessBuilder.new(*acmd)
+      henv.each { |k, v| builder.environment.put(k, v) }
+
+      process = builder.start
+      pid = process.respond_to?(:pid) ?  process.pid : nil
+
+      o = process.outputStream.to_io
+      i = process.inputStream.to_io
+      f = process.errorStream.to_io
+
+      o.write(data)
+      o.close
+
+      ex = timeout(to) { process.waitFor }
+
+      status = ProcessStatus.new(process, ex)
+
+      fail SpawnNonZeroExitError.new(conf, { to: to, t0: t0 }, status, i, f) \
+        if status.exitstatus != 0
+
+      [ i.read, status ]
+
+    rescue => err
+
+      if pid
+        Process.detach(pid)
+        (Process.kill(9, pid) rescue nil) unless Flor.no?(conf['on_error_kill'])
+      else
+        process.destroy rescue nil
+      end
+
+      raise err if err.is_a?(SpawnError)
+      raise WrappedSpawnError.new(conf, { to: to, t0: t0, pid: pid }, err)
+
+    ensure
+
+      [ i, o, f ].each { |x| x.close rescue nil }
+    end
+
+    module CmdParser include Raabro
+
+      # parsing
+
+      def separator(i); rex(nil, i, /[ 	]+/); end
+
+      def dqstring(i); rex(:string, i, /"(\\"|[^"])*"/); end
+      def sqstring(i); rex(:string, i, /'(\\'|[^'])*'/); end
+
+      def word(i); rex(:word, i, /[^ 	"']+/); end
+      def item(i); alt(nil, i, :word, :sqstring, :dqstring); end
+      def cmd(i); jseq(:cmd, i, :item, :separator); end
+
+      def equal(i); rex(nil, i, /[ 	]*=[ 	]*/); end
+      def vval(i); alt(:vval, i, :word, :sqstring, :dqstring); end
+      def vkey(i); rex(:vkey, i, /[a-zA-Z_][a-zA-Z_0-9]*/); end
+      def var(i); seq(:var, i, :vkey, :equal, :vval, :separator); end
+
+      def cmdline(i); seq(:cmdline, i, :var, '*', :cmd) end
+
+      # rewriting
+
+      def rewrite_word(t); t.string; end
+
+      def rewrite_string(t)
+        s = t.string[1..-2]
+        s == 'sleep' ? t.string : s # ah, the awful thing :-(
+      end
+        #
+        # because of `ruby -e "sleep"` :-( is there another way?
+
+      def rewrite_cmd(t)
+
+        t.subgather(nil).collect { |tt| rewrite(tt) }
+      end
+
+      def rewrite_vars(ts)
+
+        ts.inject({}) { |h, t|
+          k = t.lookup(:vkey).string
+          v = t.lookup(:vval).string; v = v[1..-2] unless t.lookup(:word)
+          h[k] = v
+          h }
+      end
+
+      def rewrite_cmdline(t)
+
+#Raabro.pp(t, colours: true)
+        [ rewrite_vars(t.subgather(:var)),
+          *t.lookup(:cmd).subgather(nil).collect { |tt| rewrite(tt) } ]
+      end
+    end
+  end
+end
+

data/lib/flor/unit/dump.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Flor
+
+  class << self
+
+    # See Scheduler#dump for the details
+    #
+    def dump(db_or_unit_or_uri, io=nil, opts=nil, &block)
+
+      derive_unit(db_or_unit_or_uri)
+        .dump(io, opts, &block)
+    end
+
+    # See Scheduler#load for the details
+    #
+    def load(db_or_unit_or_uri, string_or_io, opts={}, &block)
+
+      derive_unit(db_or_unit_or_uri)
+        .load(string_or_io, opts, &block)
+    end
+
+    protected
+
+    def derive_unit(db_or_unit_or_uri)
+
+      case o = db_or_unit_or_uri
+      when Flor::Unit then o
+      when Sequel::Database then Flor::Unit.new(sto_db: o)
+      when String then Flor::Unit.new(sto_uri: o)
+      else fail ArgumentError.new("cannot derive flor Unit out of #{o.inspect}")
+      end
+    end
+  end
+end
+

data/lib/flor/unit/executor.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -108,15 +109,13 @@ module Flor
     rescue Exception => exc
 
 # TODO eventually, have a dump dir
+
       fn =
-        [
-          'flor',
-          @unit.conf['env'], @unit.identifier, @exid,
-          'r' + counter('runs').to_s
-        ].collect(&:to_s).join('_') + '.dump'
+        [ 'flor', @unit.conf['env'], @unit.identifier, @exid,
+          'r' + counter('runs').to_s ].collect(&:to_s).join('_') + '.dump'
 
       @unit.logger.error(
-        "#{self.class}#do_run()", exc, "(dumping to #{fn})")
+        "#{self.class}#do_run()", exc, "(dumping to #{fn} ...)")
 
       File.open(fn, 'wb') do |f|
         f.puts(Flor.to_pretty_s({
@@ -133,19 +132,26 @@ module Flor
         f.puts(on_do_run_exc(exc))
       end
 
+      @unit.logger.error(
+        "#{self.class}#do_run()", exc, "(dumped to #{fn})")
+
       #puts on_do_run_exc(exc)
         # dump notification above
     end
 
     def task(message)
 
-      return error_reply(
-        node(message['nid']),
-        message,
-        "don't know how to apply #{message['tasker'].inspect}"
-      ) if message['routed'] == false
-        #
-        # use an error message similar to what the core executor would emit
+      if message['routed'] == false
+
+        t = message['tasker']
+        n = node(message['nid'])
+
+        msg = n['heat0'] != t ?
+          "tasker #{t.inspect} not found" :
+          "don't know how to apply #{t.inspect}"
+
+        return error_reply(n, message, msg)
+      end
 
       @unit.ganger.task(self, message)
     end

data/lib/flor/unit/ganger.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -53,6 +54,9 @@ module Flor
          (@unit.loader.tasker(domain, 'ganger', message) ||
           @unit.loader.tasker(domain, 'tasker', message))) ||
         @unit.loader.tasker(domain, tname, message)
+#puts "=" * 80
+#pp tconf
+#puts "=" * 80
 
       fail ArgumentError.new(
         "tasker #{tname.inspect} not found"
@@ -66,11 +70,15 @@ module Flor
         tconf = tconf.find { |h| points.include?(h['point']) }
       end
 
+      fail ArgumentError.new(
+        "tconf #{tconf.inspect} not a hash"
+      ) unless tconf.is_a?(Hash)
+
       message['tconf'] = tconf unless tconf['include_tconf'] == false
 
       message['vars'] = gather_vars(executor, tconf, message)
 
-      m = dup_message(message)
+      m = Flor.dup_message(message)
         #
         # the tasker gets a copy of the message (and it can play with it
         # to its heart content), meanwhile the message is handed to the
@@ -92,17 +100,6 @@ module Flor
 
     protected
 
-    def dup_message(m)
-
-      tc = m.delete('tconf')
-      m1 = Flor.dup(m)
-      m1['tconf'] = tc.inject({}) { |h, (k, v)|
-        h[k] = k == 'class' ? v : Flor.dup(v); h } \
-          if tc
-
-      m1
-    end
-
     def var_match(k, filter)
 
       filter.each { |f| return true if (f.is_a?(String) ? k == f : f.match(k)) }

data/lib/flor/unit/gangers.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Flor
+
+  # A ModuleGanger accepts a `module:` conf entry that points to a Ruby
+  # module. The tasker implementations are searched for among the classes
+  # in the given module.
+  #
+  # Among the tasker classes (classes that respond to on_task, on_detask, ...)
+  # it selects the first tasker that matches the tasker name.
+  #
+  class ModuleGanger
+
+    def initialize(service, conf, message)
+
+      @service = service
+      @conf = conf
+      @message = message
+    end
+
+    def task
+
+      tas = @message['tasker']
+      clas = list_tasker_classes
+      cla = clas.find { |c| tasker_name(c) == tas }
+
+      return [ Flor.dup_and_merge(@message, 'routed' => false) ] \
+        unless cla
+
+      call_tasker(cla)
+    end
+
+    alias detask task
+
+    protected
+
+    def list_tasker_classes
+
+      mod_name = @conf['module']
+
+      fail ArgumentError.new('ganger module: configuration entry missing') \
+        unless mod_name
+
+      mod = Flor.const_lookup(mod_name) rescue nil
+
+      fail ArgumentError.new("ganger cannot find module #{mod_name.inspect}") \
+        unless mod
+
+      list_classes(mod, [])
+    end
+
+    def list_classes(start, r)
+
+      # place leave classes on top if possible
+      # within a level, sort alphabetically
+
+      clas = start.constants.collect { |co| start.const_get(co) }
+      clas, mods = clas.partition { |c| c.is_a?(Class) }
+
+      mods.each { |m| list_classes(m, r) }
+      r.concat(clas.select { |c| tasker?(c) }.sort_by { |c| c.name })
+
+      r
+    end
+
+    TASKER_METHODS = [
+      :on, :on_message,
+      :task, :on_task,
+      :detask, :on_detask, :cancel, :on_cancel
+        ].freeze
+
+    def tasker?(cla)
+
+      return true if (TASKER_METHODS & cla.public_instance_methods).any?
+      return true if (TASKER_METHODS & cla.public_methods).any?
+      false
+    end
+
+    def tasker_name(cla)
+
+      if cla.public_instance_methods.include?(:tasker_name)
+
+        unless cla.respond_to?(:_ganged)
+          class << cla
+            attr_accessor :_ganged
+          end
+          cla._ganged = cla.allocate
+        end
+
+        call_tasker_name(cla._ganged)
+
+      elsif cla.public_methods.include?(:tasker_name)
+
+        call_tasker_name(cla)
+
+      else
+
+        cla.name.split('::').last.gsub(/Tasker\z/, '')
+          .gsub(/([a-z])([A-Z])/) { |_| $1 + '_' + $2.downcase }
+          .gsub(/([A-Z])/) { |c| c.downcase }
+      end
+    end
+
+    def call_tasker_name(o)
+
+      case i = o.method(:tasker_name).arity
+      when 1 then o.tasker_name(@message)
+      when 2 then o.tasker_name(@conf, @message)
+      when 3 then o.tasker_name(@service, @conf, @message)
+      when -1 then o.tasker_name(
+        service: @service, conf: @conf, message: @message)
+      else o.tasker_name
+      end
+    end
+
+    def call_tasker(c)
+
+      cnf = @conf.merge('class' => c)
+
+      @service.unit.caller
+        .call(@service, cnf, @message)
+    end
+  end
+end
+

data/lib/flor/unit/hloader.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
 
 module Flor
 
   # A loader which keeps everything in a Hash, while the traditional/default
   # Flor::Loader reads from a file tree.
   #
-  class HashLoader
+  class HashLoader < CoreLoader
 
     # NB: tasker configuration entries start with "loa_", like for Flor::Loader
 
@@ -27,9 +28,6 @@ module Flor
       self.environment = (@unit.conf['lod_environment'] || {})
     end
 
-    def shutdown
-    end
-
     HCATS = {
       'v' => 'variables',
       'var' => 'variables',
@@ -53,9 +51,14 @@ module Flor
       path = path.to_s
       path = path + '.' if c == 'hooks' && path.length > 0 && path[-1, 1] != '.'
 
-      value = block ?
-        block_to_class(c, block) :
-        Flor.to_string_keyed_hash(value)
+      value =
+        if block
+          block_to_class(c, block)
+        elsif value.is_a?(Proc)
+          block_to_class(c, value)
+        else
+          Flor.to_string_keyed_hash(value)
+        end
 
       e = (@environment[c] ||= [])
       e << [ *split(path), value ]
@@ -132,6 +135,30 @@ module Flor
         .collect { |h| Flor::Hook.new(@unit, exid, h) }
     end
 
+    def domains(start=nil)
+
+      start ||= ''
+
+      @environment.values
+        .flatten(1)
+        .collect { |x| x.first }
+        .select { |dm| dm && dm.length > 0 }
+        .select { |dm| Flor.sub_domain?(start, dm) }
+        .uniq
+        .sort
+    end
+
+    def definitions(start=nil)
+
+      start ||= ''
+
+      @environment.values
+        .flatten(1)
+        .collect { |x| x[0, 2].join('.') }
+        .select { |fl| Flor.sub_domain?(start, fl) }
+        .sort
+    end
+
     protected
 
     def recompose(h)