flor 0.16.1 → 0.16.2

data/lib/flor/unit/spooler.rb

@@ -43,9 +43,11 @@ module Flor
 
     def determine_spool_dir
 
-      d = @unit.conf['spo_dir'] || 'var/spool'
-      d = File.join(@unit.conf['root'], d) if d
-      d = nil unless File.directory?(d)
+      r = @unit.conf['root']
+      return nil unless r
+
+      d = File.join(r, @unit.conf['spo_dir'] || 'var/spool')
+      return nil unless File.directory?(d)
 
       d
     end

data/lib/flor/unit/storage.rb

@@ -80,14 +80,25 @@ module Flor
       opts[:current] ||= from if from.is_a?(Integer)
         #
         # defaults for the migration version table:
-        #:table => :schema_info
-        #:column => :version
+        # { table: :schema_info,
+        #   column: :version }
+
+      skip =
+        @unit.conf['db_sparse_migrations'] ||
+        @unit.conf['sto_sparse_migrations'] ||
+        opts[:sparse_migrations]
+      if skip && ! opts.has_key?(:allow_missing_migration_files)
+        opts[:allow_missing_migration_files] = true
+      end
 
       dir =
         @unit.conf['db_migrations'] ||
-        File.absolute_path(
-          File.join(
-            File.dirname(__FILE__), '..', 'migrations'))
+        @unit.conf['db_migration_dir'] ||
+        @unit.conf['sto_migrations'] ||
+        @unit.conf['sto_migration_dir'] ||
+        opts[:migrations] ||
+        opts[:migration_dir] ||
+        Flor.migration_dir
 
       synchronize do
 
@@ -664,7 +675,9 @@ module Flor
 
     def update_pointers(exe, status, now)
 
-# TODO should we archive old pointers?
+# Q Should we archive old pointers?
+#   Well, it might be better to only archive the execution and leave
+#   in there enough information...
 
       exid = exe['exid']
 
@@ -694,9 +707,12 @@ module Flor
 
           vs = nid == '0' ? node['vars'] : nil
           vs.each { |k, v|
-            case v; when Integer, String, TrueClass, FalseClass
+            case v; when Numeric, String, TrueClass, FalseClass, NilClass
               a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
-            when NilClass
+            when Array, Hash
+              s = '(array)'; s = '(object)' if v.is_a?(Hash)
+              a << [ dom, exid, '0', 'var', k, s, now, u ]
+            else
               a << [ dom, exid, '0', 'var', k, nil, now, u ]
             end } if vs
 
@@ -793,19 +809,30 @@ module Flor
       @db.loggers << @db_logger
     end
 
-    def self.to_blob(h)
+    class << self
+
+      def to_blob(h)
 
-      Sequel.blob(Zlib::Deflate.deflate(JSON.dump(h)))
+        h ? Sequel.blob(Zlib::Deflate.deflate(JSON.dump(h))) : nil
 #rescue => e; pp h; raise e
-    end
+      end
 
-    def self.from_blob(content)
+      def from_blob(content)
 
-      JSON.parse(Zlib::Inflate.inflate(content))
+        content ? JSON.parse(Zlib::Inflate.inflate(content)) : nil
+      end
     end
 
     def to_blob(h); self.class.to_blob(h); end
     def from_blob(content); self.class.from_blob(content); end
   end
+
+  # module Flor
+
+  class << self
+
+    def to_blob(h); ::Flor::Storage.to_blob(h); end
+    def from_blob(content); ::Flor::Storage.from_blob(content); end
+  end
 end
 

data/lib/flor/unit/waiter.rb

@@ -5,11 +5,12 @@ module Flor
 
     def initialize(exid, opts)
 
-      serie, timeout, on_timeout, repeat =
+      serie, timeout, on_timeout =
         expand_args(opts)
 
+      # TODO fail if the serie mixes msg_waiting with row_waiting...
+
       @exid = exid
-      @original_serie = repeat ? Flor.dup(serie) : nil
       @serie = serie
       @timeout = timeout
       @on_timeout = on_timeout
@@ -17,39 +18,81 @@ module Flor
       @queue = []
       @mutex = Mutex.new
       @var = ConditionVariable.new
+
+      @executor = nil
+    end
+
+    ROW_PSEUDO_POINTS = %w[ status tag tasker var variable ]
+      # "tasker", not "task", since "task" is already a message point
+
+    def row_waiter?
+
+      @serie.find { |_, points|
+        points.find { |po|
+          pos = po.split(':')
+          pos.length > 1 && ROW_PSEUDO_POINTS.include?(pos[0]) } }
+    end
+
+    def msg_waiter?
+
+      @serie.find { |_, points|
+        points.find { |po|
+          ! ROW_PSEUDO_POINTS.include?(po.split(':').first) } }
     end
 
     def to_s
 
-      "#{super[0..-2]}#{
-        { exid: @exid,
-          original_serie: @original_serie,
-          timeout: @timeout }.inspect
-      }>"
+      "#{super[0..-2]}#{{ exid: @exid, timeout: @timeout }.inspect}>"
     end
 
     def notify(executor, message)
 
+      @executor = executor
+        # could be handy
+
       @mutex.synchronize do
 
         return false unless match?(message)
 
         @serie.shift
-        return false unless @serie.empty?
+        return false if @serie.any?
 
         @queue << [ executor, message ]
         @var.signal
       end
 
-      # then...
-      # returning false: do not remove me, I want to listen/wait further
-      # returning true: remove me
+      true # serie over, remove me
+    end
+
+    def check(unit, rs)
+
+      @mutex.synchronize do
 
-      return true unless @original_serie
+        row = nil
 
-      @serie = Flor.dup(@original_serie) # reset serie
+        loop do
 
-      false # do not remove me
+          break if @serie.empty?
+
+          row = row_match?(unit, rs)
+          return false unless row
+
+          @serie.shift
+        end
+
+        @queue << [ unit, row ]
+        @var.signal
+      end
+
+      true # serie over, remove me
+
+    rescue => err
+
+#puts "!" * 80; p err
+      unit.logger.warn(
+        "#{self.class}#check()", err, '(returning true, aka remove me)')
+
+      true # remove me
     end
 
     def wait
@@ -63,7 +106,8 @@ module Flor
 
           if @queue.empty?
             fail RuntimeError.new(
-              "timeout for #{self.to_s}"
+              "timeout for #{self.to_s}, " +
+              "msg_waiter? #{ !! msg_waiter?}, row_waiter? #{ !! row_waiter?}"
             ) if @on_timeout == 'fail'
             return { 'exid' => @exid, 'timed_out' => @on_timeout }
           end
@@ -73,6 +117,37 @@ module Flor
       end
     end
 
+    def to_query_hashes
+
+      @serie
+        .inject([ [], [] ]) { |a, (nid, points)|
+
+          points.each do |point|
+
+            ss = point.split(':')
+
+            h = {}
+            h[:exid] = @exid if @exid
+            h[:nid] = nid if nid
+
+            case ss[0]
+            when 'status'
+              h[:status] = ss[1]
+              a[0] << h
+            when 'tag', 'tasker', 'var', 'variable'
+              t = ss[0]; t = 'var' if t == 'variable'
+              h[:type] = t
+              h[:name] = ss[1]
+              h[:value] = ss[2] if ss[2]
+              a[1] << h
+            else
+              fail ArgumentError, "cannot turn to query_hash, #{self.inspect}"
+            end
+          end
+
+          a }
+    end
+
     protected
 
     def match?(message)
@@ -85,15 +160,70 @@ module Flor
       mnid = message['nid']
 
       return false if nid && mnid && nid != mnid
-      return false if ! points.include?(mpoint)
+
+      return false unless points.find { |point|
+        ps = point.split(':')
+        next false if ps[0] != mpoint
+        next false if ps[1] && ! message['tags'].include?(ps[1])
+        true }
 
       true
     end
 
+    def row_match?(unit, rs)
+
+      nid, points = @serie.first
+
+      row = nil
+
+      points.find { |point|
+        ps = point.split(':')
+        row = send("row_match_#{ps[0]}?", unit, rs, nid, ps[1..-1]) }
+
+      row
+    end
+
+    def row_match_status?(unit, rs, _, cdr)
+
+      rs[0].find { |exe|
+        (@exid == nil || exe.exid == @exid) &&
+        exe.status == cdr.first }
+    end
+
+    def row_match_tag?(unit, rs, nid, (name, value))
+
+      rs[1].find { |ptr|
+        ptr.type == 'tag' &&
+        (@exid == nil || ptr.exid == @exid) &&
+        (nid == nil || ptr.nid == nid) &&
+        (name == nil || ptr.name == name) &&
+        (value == nil || ptr.value == value) }
+    end
+
+    def row_match_var?(unit, rs, nid, (name, value))
+
+      rs[1].find { |ptr|
+        ptr.type == 'var' &&
+        (@exid == nil || ptr.exid == @exid) &&
+        (nid == nil || ptr.nid == nid) &&
+        (name == nil || ptr.name == name) &&
+        (value == nil || ptr.value == value.to_s) }
+    end
+    alias row_match_variable?  row_match_var?
+
+    def row_match_tasker?(unit, rs, nid, (name, value))
+
+      rs[1].find { |ptr|
+        ptr.type == 'tasker' &&
+        (@exid == nil || ptr.exid == @exid) &&
+        (nid == nil || ptr.nid == nid) &&
+        (name == nil || ptr.name == name) &&
+        (value == nil || ptr.value == value) }
+    end
+
     def expand_args(opts)
 
       owait = opts[:wait]
-      orepeat = opts[:repeat] || false
       otimeout = opts[:timeout]
       oontimeout = opts[:on_timeout] || opts[:ontimeout] || 'fail'
 
@@ -101,33 +231,42 @@ module Flor
       when nil, true
         [ [ [ nil, %w[ failed terminated ] ] ], # serie
           otimeout,
-          oontimeout,
-          orepeat ]
+          oontimeout ]
       when Numeric
         [ [ [ nil, %w[ failed terminated ] ] ], # serie
           owait, # timeout
-          oontimeout,
-          orepeat ]
+          oontimeout ]
       when String, Array
         [ parse_serie(owait), # serie
           otimeout,
-          oontimeout,
-          orepeat ]
+          oontimeout ]
       else
         fail ArgumentError.new(
           "don't know how to deal with #{owait.inspect} (#{owait.class})")
       end
     end
 
+    WAIT_REX =
+      %r{
+        \A
+        ([0-9_\-]+)?[ ]*
+        (
+          [a-z]+(?::[^:|;,\s]+){0,2}
+          (?:[|, ][a-z]+(:[^:|;,\s]+){0,2})*
+        )\z
+      }x
+
     def parse_serie(s)
 
       return s if s.is_a?(Array) && s.collect(&:class).uniq == [ Array ]
 
       (s.is_a?(String) ? s.split(';') : s)
         .collect { |ss|
-          ni, pt = ss.strip.match(/\A([0-9_\-]+)? *([a-z|, ]+)\z/)[1, 2]
-          [ ni, pt.split(/[|,]/).collect(&:strip) ]
-        }
+          m = ss.strip.match(WAIT_REX)
+          fail ArgumentError.new(
+            "cannot parse #{ss.strip.inspect} wait directive") unless m
+          ni, pt = m[1, 2]
+          [ ni, pt.split(/[|,]/).collect(&:strip) ] }
     end
   end
 end

data/lib/flor/unit/wlist.rb

@@ -7,6 +7,9 @@ module Flor
     #
     # `wtl_default_timeout`:
     #   when #launch ing or #wait ing, set the default timeout, in seconds
+    #
+    # `wtl_row_frequency`:
+    #   sleep time between row waiter checks, defaults to 1
 
     DEFAULT_TIMEOUT = Flor.env_i('FLOR_DEFAULT_TIMEOUT')
 
@@ -16,16 +19,25 @@ module Flor
       @unit.hooker.add('wlist', self)
 
       @mutex = Mutex.new
-      @waiters = []
+      @msg_waiters = []
+      @row_waiters = []
 
       @unit.instance_eval do
         def wait(exid, opts=true, more=nil)
           @hooker['wlist'].wait(exid, opts, more)
         end
       end
+
+      @row_thread = nil
+      @row_thread_status = nil
+      @row_frequency = @unit.conf['wtl_row_frequency'] || 1
     end
 
     def shutdown
+
+      @row_thread_status = :shutdown
+
+      nil
     end
 
     def notify(executor, message)
@@ -34,12 +46,12 @@ module Flor
 
         to_remove = []
 
-        @waiters.each do |w|
+        @msg_waiters.each do |w|
           remove = w.notify(executor, message)
           to_remove << w if remove
         end
 
-        @waiters -= to_remove
+        @msg_waiters -= to_remove
 
       end if message['consumed']
 
@@ -49,7 +61,7 @@ module Flor
     def wait(exid, opts, more)
 
       exid, opts =
-        if opts == true && exid == 'idle'
+        if opts == true && ! Flor.is_exid?(exid)
           [ nil, { wait: exid } ]
         elsif opts == true || opts.is_a?(String)
           [ exid, { wait: opts } ]
@@ -65,11 +77,92 @@ module Flor
 
       @mutex.synchronize do
 
-        (@waiters << Waiter.new(exid, opts)).last
+        waiter = Waiter.new(exid, opts)
+
+        fail ArgumentError.new(
+          "unit is stopped, it cannot wait for #{[ exid, opts ].inspect}"
+        ) if waiter.msg_waiter? && @unit.stopped?
+
+        waiters = [ @msg_waiters, @row_waiters ]
+        ts = [ 'msg', 'row' ]
+        if waiter.row_waiter?; waiters.reverse!; ts.reverse!; end
+
+        fail ArgumentError.new(
+          "cannot add a #{ts[0]} waiter, since there are already #{ts[1]} ones"
+        ) if waiters[1].any?
+
+        waiters.first << waiter
+
+        start_row_thread if @row_waiters.any?
+
+        waiter
 
       end.wait
         # returns the response message
     end
+
+    protected
+
+    def start_row_thread
+
+      return if @row_thread_status == :shutdown
+
+      @row_thread = nil if @row_thread && ! @row_thread.alive?
+      @row_thread_status = :running
+
+      @row_thread ||=
+        Thread.new do
+          loop do
+            sleep(@row_frequency)
+            break if [ :stop, :shutdown ].include?(@row_thread_status)
+            break if @row_waiters.empty?
+            check
+          end
+        end
+    end
+
+    def check
+
+      @mutex.synchronize do
+
+        to_remove = []
+
+        rs = row_query_all
+
+        @row_waiters.each do |w|
+          remove = w.check(@unit, rs)
+          to_remove << w if remove
+        end
+
+        @row_waiters -= to_remove
+      end
+
+    rescue => err
+
+      @unit.logger.warn("#{self.class}#check()", err)
+    end
+
+    def row_query_all
+
+      exes, ptrs =
+        @row_waiters.inject([ [], [] ]) { |a, w|
+          es, ps = w.to_query_hashes
+          a[0].concat(es)
+          a[1].concat(ps)
+          a }
+
+      [ row_query(:executions, exes), row_query(:pointers, ptrs) ]
+    end
+
+    def row_query(klass, hs)
+
+      return [] if hs.empty?
+
+      q = @unit.send(klass).where(hs.shift)
+      hs.each { |h| q = q.or(h) }
+
+      q.all
+    end
   end
 end