flor 0.18.0 → 1.2.0

data/lib/flor/unit/models.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -92,6 +93,12 @@ module Flor
       end
     end
 
+    alias to_dump_h to_h
+      #
+      # Downsteam, #to_h answers are more complete, more standalone
+      # whereas, #to_dump_h answers are more compact (see florist).
+      # For now, they're just the same, it specializes downstream.
+
     class << self
 
       def from_h(h)
@@ -135,7 +142,7 @@ module Flor
 
       @models[key] ||=
         parent_module.const_set(
-          "#{c}#{@db.object_id.to_s.gsub('-', 'M')}",
+          "#{c}#{self.object_id.to_s.gsub('-', 'M')}",
           Class.new(parent_module.const_get(c)) do
             self.dataset = s.db["#{table_prefix}#{key}".to_sym]
             self.unit = s.unit

data/lib/flor/unit/models/execution.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -98,6 +99,56 @@ module Flor
       h
     end
 
+    def lookup_nodes(query, opts={})
+
+      @node_index ||= nodes
+        .sort_by { |k, _| k }
+        .inject({}) { |h, (k, v)|
+          #
+          # nid => [ node ]
+          #
+          h[k] = [ v ]
+          #
+          # code => [ node0, node1, ... ]
+          #
+          t =
+            (lookup_tree(v['parent'])[1][Flor.child_id(v['nid'])] rescue nil) ||
+            lookup_tree(v['nid'])
+          s = Flor.tree_to_flor(t, chop: true)
+          (h[s] ||= []) << v
+          #
+          # tag => [ node0, node1, ... ]
+          #
+          ts = v['tags']
+          ts.each { |t| (h[t] ||= []) << v } if ts
+          #
+          h }
+
+      @node_index
+        .select { |k, v|
+          case query
+          when Regexp then k.match(query)
+          else k == query
+          end }
+        .values
+        .flatten(1)
+    end
+
+    def lookup_node(query, opts={})
+
+      lookup_nodes(query, opts).first
+    end
+
+    def lookup_nids(query, opts={})
+
+      lookup_nodes(query, opts).collect { |n| n['nid'] }
+    end
+
+    def lookup_nid(query, opts={})
+
+      lookup_node(query, opts)['nid']
+    end
+
     class << self
 
       def by_status(s)

data/lib/flor/unit/models/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 
 module Flor
 
@@ -18,6 +20,10 @@ module Flor
     #
     #  index :exid
     #end
+
+    def nid; data['nid']; end
+    def tasker; data['tasker']; end
+    alias payload data
   end
 end
 

data/lib/flor/unit/models/pointer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -29,7 +30,7 @@ module Flor
     #end
 
     # If the pointer is a "var" pointer, returns the full value
-    # for the variable, as fund in the execution's node "0".
+    # for the variable, as found in the execution's node "0".
     #
     def full_value
 
@@ -37,6 +38,25 @@ module Flor
 
       node['vars'][name]
     end
+
+    def attd
+
+      data['atts'].inject({}) { |h, (k, v)| h[k] = v if k; h }
+
+    rescue; []
+    end
+
+    def attl
+
+      data['atts'].inject([]) { |a, (k, v)| a << v if k == nil; a }
+
+    rescue; []
+    end
+
+    def att_texts
+
+      attl.select { |e| e.is_a?(String) }
+    end
   end
 end
 

data/lib/flor/unit/models/timer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 

data/lib/flor/unit/models/trace.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 

data/lib/flor/unit/models/trap.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -124,7 +125,7 @@ module Flor
         'exid' => self.exid,
         'nid' => self.nid,
         'type' => 'trap',
-        'trap' => to_hash,
+        'trap' => to_trap_h,
         'trap_id' => self.id,
         'message' => msg,
         'sm' => message['m'] }
@@ -133,7 +134,7 @@ module Flor
             #'dbg' => xx }
     end
 
-    def to_hash
+    def to_trap_h
 
       values
         .inject({}) { |h, (k, v)| h[k.to_s] = v if k != :content; h }

data/lib/flor/unit/scheduler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 
@@ -9,6 +10,7 @@ module Flor
     attr_reader :logger
 
     attr_reader :thread_status
+    attr_reader :last_queued_message_id
 
     attr_reader :archive
 
@@ -136,11 +138,7 @@ module Flor
 
       # TODO heartbeat, every x minutes, when idle, log something
 
-      fail(
-        "database not ready, " +
-        "db ver: #{@storage.db_version.inspect}, " +
-        "mig ver: #{@storage.migration_version}"
-      ) if !! @conf['sto_migration_check'] && @storage.ready?
+      check_migration_version
 
       @thread_status = :running
 
@@ -160,6 +158,15 @@ module Flor
       self
     end
 
+    def check_migration_version
+
+      fail(
+        "database not ready, " +
+        "db ver: #{@storage.db_version.inspect}, " +
+        "mig ver: #{@storage.migration_version}"
+      ) if !! @conf['sto_migration_check'] && @storage.ready?
+    end
+
     def stop
 
       @thread_status = :stop
@@ -222,7 +229,11 @@ module Flor
 
     def queue(message, opts={})
 
-      @storage.put_message(message)
+      @last_queued_message_id =
+        @storage.put_message(message)
+
+      # Nota bene:
+      # the #wait method is added to the Scheduler by Flor::WaitList
 
       if opts[:wait]
         wait(message['exid'], opts)
@@ -356,9 +367,13 @@ module Flor
       ) unless nid
 
       fail ArgumentError.new(
-        "cannot add iteration to missing node #{nid.inspect}"
+        "cannot add iteration to node #{nid.inspect} not present in tree"
       ) unless exe.lookup_tree(nid)
 
+      fail ArgumentError.new(
+        "cannot add iteration to node #{nid.inspect} not present in execution"
+      ) unless exe.nodes.has_key?(nid)
+
       queue(msg, opts)
     end
     alias add_iteration add_iterations
@@ -434,16 +449,14 @@ module Flor
     #
     # unit.dump() { |h| ... }  # modify the has right before it's turned to JSON
     #
+    # unit.dump(hash: true)  # returns the hash (instead of JSONing it)
+    #
     def dump(io=nil, opts=nil, &block)
 
       io, opts = nil, io if io.is_a?(Hash)
       opts ||= {}
 
-      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
-        #
-      exis = o[:exid]
-      doms = o[:domain]
-      sdms = o[:strict_domain] || o[:sdomain]
+      exis, doms, sdms = extract_dump_and_load_filters(opts)
         #
       filter = lambda { |q|
         q = q.where(
@@ -458,25 +471,25 @@ module Flor
           domain: sdms) if sdms
         q }
 
-      exs, tms, tps, pts =
+      hash =
         storage.db.transaction {
-          [ filter[executions].collect(&:to_h),
-            filter[timers].collect(&:to_h),
-            filter[traps].collect(&:to_h),
-            filter[pointers].collect(&:to_h) ] }
 
-      o = io ? io : StringIO.new
+          h = {
+            timestamp: Flor.tstamp,
+            executions: filter[executions].collect(&:to_h),
+            timers: filter[timers].collect(&:to_h),
+            traps: filter[traps].collect(&:to_h),
+            pointers: filter[pointers].collect(&:to_h) }
 
-      h = {
-        timestamp: Flor.tstamp,
-        executions: exs,
-        timers: tms,
-        traps: tps,
-        pointers: pts }
+          block.call(h) if block
 
-      block.call(h) if block
+          h }
 
-      JSON.dump(h, o)
+      return hash if opts[:hash] || opts[:h]
+
+      o = io ? io : StringIO.new
+
+      JSON.dump(hash, o)
 
       io ? io : o.string
     end
@@ -507,12 +520,7 @@ module Flor
       mks = DUMP_KEYS - h.keys
       fail Flor::FlorError.new("missing keys #{mks.inspect}") if mks.any?
 
-      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
-        #
-      exis = o[:exid]
-      doms = o[:domain]
-      sdms = o[:strict_domain] || o[:sdomain]
-        #
+      exis, doms, sdms = extract_dump_and_load_filters(opts)
       doms = doms.collect { |d| /\A#{d}(\.#{Flor::NAME_REX})*\z/ } if doms
 
       counts = { executions: 0, timers: 0, traps: 0, pointers: 0, total: 0 }
@@ -541,7 +549,7 @@ module Flor
           cla.import(cols, rows) if rows.any?
         end
 
-        block.call(h) if block
+        block.call(h, counts) if block
       end
 
       counts
@@ -549,6 +557,13 @@ module Flor
 
     protected
 
+    def extract_dump_and_load_filters(opts)
+
+      o = lambda { |k| v = opts[k] || opts["#{k}s".to_sym]; v ? Array(v) : nil }
+
+      [ o[:exid], o[:domain], o[:strict_domain] || o[:sdomain] ]
+    end
+
     def tick
 
       t0 = Time.now
@@ -580,7 +595,7 @@ module Flor
 
     rescue Exception => ex
 
-      puts on_start_exc(ex)
+      puts(on_start_exc(ex))
     end
 
     def prepare_message(point, args)
@@ -668,9 +683,9 @@ module Flor
     def should_wake_up?
 
       return true if @wake_up
-      return true if Time.now - @reloaded_at >= reload_after
+      return true if (Time.now - @reloaded_at) >= reload_after
 
-      @next_time && @next_time <= Flor.tstamp.split('.').first
+      @next_time && (@next_time <= Flor.tstam)
     end
 
     def unreserve_messages

data/lib/flor/unit/spooler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module Flor
 

data/lib/flor/unit/storage.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 require 'zlib'
 
@@ -6,6 +7,14 @@ module Flor
 
   class Storage
 
+    MESSAGE_COLUMNS = [
+      :domain, :exid, :point, :content,
+      :status, :ctime, :mtime, :cunit, :munit
+        ].freeze
+    POINTER_COLUMNS = [
+      :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit
+        ].freeze
+
     attr_reader :unit, :db, :models
 
     attr_reader :mutex
@@ -30,9 +39,11 @@ module Flor
 #p [ :disconnected, @db.object_id ]
     end
 
-    def db_version
+    def db_version(opts={})
+
+      table, column = migration_table_and_column(opts)
 
-      (@db[:schema_info].first rescue {})[:version]
+      (@db[table].first rescue {})[column]
     end
 
     def migration_version
@@ -79,34 +90,27 @@ module Flor
       opts[:target] ||= to if to.is_a?(Integer)
       opts[:current] ||= from if from.is_a?(Integer)
 
-      opts[:table] = (
-        @unit.conf['db_migration_table'] ||
-        @unit.conf['sto_migration_table'] ||
-        :schema_info).to_sym
-      opts[:column] = (
-        @unit.conf['db_migration_column'] ||
-        @unit.conf['sto_migration_column'] ||
-        :version).to_sym
-          #
-          # defaults for the migration version table:
-          # { table: :schema_info,
-          #   column: :version }
+      opts[:table], opts[:column] = migration_table_and_column(opts)
+        #
+        # defaults for the migration version table:
+        # { table: :schema_info,
+        #   column: :version }
 
       skip =
+        opts[:sparse_migrations] ||
         @unit.conf['db_sparse_migrations'] ||
-        @unit.conf['sto_sparse_migrations'] ||
-        opts[:sparse_migrations]
+        @unit.conf['sto_sparse_migrations']
       if skip && ! opts.has_key?(:allow_missing_migration_files)
         opts[:allow_missing_migration_files] = true
       end
 
       dir =
+        opts[:migrations] ||
+        opts[:migration_dir] ||
         @unit.conf['db_migrations'] ||
         @unit.conf['db_migration_dir'] ||
         @unit.conf['sto_migrations'] ||
         @unit.conf['sto_migration_dir'] ||
-        opts[:migrations] ||
-        opts[:migration_dir] ||
         Flor.migration_dir
 
       synchronize do
@@ -336,35 +340,39 @@ module Flor
       n = Flor.tstamp
       u = @unit.identifier
 
-      synchronize(syn) do
+      id =
+        synchronize(syn) do
 
-        stored, unstored = ms.partition { |m| m['mid'] }
+          stored, unstored = ms.partition { |m| m['mid'] }
 
-        #
-        # de-reserve any previously stored message, might happen
-        # for "terminated" messages that got queued back to let
-        # other messages get processed
+          #
+          # de-reserve any previously stored message, might happen
+          # for "terminated" messages that got queued back to let
+          # other messages get processed
 
-        @db[:flor_messages]
-          .where(id: stored.collect { |m| m['mid'] })
-          .update(status: 'created', mtime: n, munit: u) \
-            if stored.any?
+          @db[:flor_messages]
+            .where(id: stored.collect { |m| m['mid'] })
+            .update(status: 'created', mtime: n, munit: u) \
+              if stored.any?
 
-        #
-        # store new messages
+          #
+          # store new messages
 
-        @db[:flor_messages]
-          .import(
-            [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime, :cunit, :munit ],
-            unstored.map { |m|
-              [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
-                'created', n, n, u, u ] }) \
-                  if unstored.any?
-      end
+          @db[:flor_messages]
+            .import(
+              MESSAGE_COLUMNS,
+              unstored.map { |m|
+                [ Flor.domain(m['exid']), m['exid'], m['point'], to_blob(m),
+                  'created', n, n, u, u ] }) \
+                    if unstored.any?
+
+          @db[:flor_messages].max(:id)
+        end
 
       @unit.wake_up
 
+      id
+
     rescue => err
 
       Thread.current[:sto_errored_items] = ms
@@ -516,25 +524,6 @@ module Flor
       end
     end
 
-    def put_task_pointer(msg, tname, tconf)
-
-      exid = msg['exid']
-      dom = Flor.domain(exid)
-
-      synchronize do
-
-        @db[:flor_pointers]
-          .insert(
-            domain: dom,
-            exid: exid,
-            nid: msg['nid'],
-            type: 'tasker',
-            name: tname,
-            ctime: Flor.tstamp,
-            cunit: @unit.identifier)
-      end
-    end
-
     def fetch_next_time
 
       t =
@@ -557,6 +546,18 @@ module Flor
 
     protected
 
+    def migration_table_and_column(opts={})
+
+      [ (opts[:migration_table] ||
+         @unit.conf['db_migration_table'] ||
+         @unit.conf['sto_migration_table'] ||
+         :schema_info).to_sym,
+        (opts[:migration_column] ||
+          @unit.conf['db_migration_column'] ||
+          @unit.conf['sto_migration_column'] ||
+          :version).to_sym ]
+    end
+
     def _commaify(o)
 
       if Flor.is_regex_tree?(o)
@@ -585,14 +586,13 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .update(
             status: 'consumed', mtime: n, munit: u)
 
         @db[:flor_messages]
           .import(
-            [ :domain, :exid, :point, :content,
-              :status, :ctime, :mtime, :cunit, :munit ],
+            MESSAGE_COLUMNS,
             messages
               .select { |m|
                 ! m['mid'] && POINTS_TO_ARCHIVE.include?(m['point']) }
@@ -608,19 +608,16 @@ module Flor
 
         @db[:flor_messages]
           .where(
-            id: messages.collect { |m| m['mid'] }.compact)
+            id: messages.collect { |m| m['mid'] }.uniq.compact)
           .delete
       end
     end
 
     def load_timers
 
-      now = Flor.tstamp
-      no = now[0, now.rindex('.')]
-
       timers
         .where(status: 'active')
-        .where { ntime <= no }
+        .where { ntime <= Flor.tstam }
         .order(:ntime)
         .all
 
@@ -690,9 +687,9 @@ module Flor
 
     def update_pointers(exe, status, now)
 
-# Q Should we archive old pointers?
-#   Well, it might be better to only archive the execution and leave
-#   in there enough information...
+# Q  Should we archive old pointers?
+# A  Well, it might be better to only archive the execution and leave
+#    in there enough information...
 
       exid = exe['exid']
 
@@ -718,23 +715,25 @@ module Flor
 
           ts = node['tags']
           ts.each { |t|
-            a << [ dom, exid, nid, 'tag', t, nil, now, u ] } if ts
+            a << [ dom, exid, nid, 'tag', t, nil, now, u, nil ] } if ts
 
           vs = nid == '0' ? node['vars'] : nil
           vs.each { |k, v|
             case v; when Numeric, String, TrueClass, FalseClass, NilClass
-              a << [ dom, exid, '0', 'var', k, v.to_s, now, u ]
+              a << [ dom, exid, '0', 'var', k, v.to_s, now, u, nil ]
             when Array, Hash
               s = '(array)'; s = '(object)' if v.is_a?(Hash)
-              a << [ dom, exid, '0', 'var', k, s, now, u ]
+              a << [ dom, exid, '0', 'var', k, s, now, u, nil ]
             else
-              a << [ dom, exid, '0', 'var', k, nil, now, u ]
+              a << [ dom, exid, '0', 'var', k, nil, now, u, nil ]
             end } if vs
 
-          #ta = node['heap'] == 'task' ? node['task'] : nil
-          ta = node['task']
-          a << [ dom, exid, nid, 'tasker', ta['tasker'], ta['name'], now, u ] \
-            if ta
+          if ta = node['task']
+            tasker = ta['tasker']
+            name = ta['name']
+            content = { message: node['message'], atts: node['atts'] }
+            a << [ dom, exid, nid, 'tasker', tasker, name, now, u, content ]
+          end
 
           a }
 
@@ -742,17 +741,35 @@ module Flor
         .where(exid: exid)
         .select(:nid, :type, :name)
         .all
-      pointers.reject! { |_, _, ni, ty, na, _, _, _|
+      pointers.reject! { |_, _, ni, ty, na, _, _, _, _|
         cps.find { |cp| cp[:nid] == ni && cp[:type] == ty && cp[:name] == na } }
           #
           # don't insert when already inserted
 
+      if pointer_columns.include?(:content)
+        pointers.each { |ptr|
+          c = ptr[8]; ptr[8] = to_blob(c) if c }
+      else
+        pointers.each { |ptr|
+          ptr.pop }
+      end
+
       @db[:flor_pointers]
         .import(
-          [ :domain, :exid, :nid, :type, :name, :value, :ctime, :cunit ],
+          pointer_columns,
           pointers)
     end
 
+    def pointer_columns
+
+      @pointer_columns ||=
+        if @db[:flor_pointers].columns.include?(:content)
+          POINTER_COLUMNS + [ :content ]
+        else
+          POINTER_COLUMNS
+        end
+    end
+
     def determine_type_and_schedule(message)
 
       t, s = message['type'], message['string']
@@ -798,15 +815,27 @@ module Flor
       def error(msg); @unit.logger.db_log(:error, msg); end
     end
 
-    def connect
+    def derive_db
+
+      db = @unit.conf['sto_db']
+
+      return db if db
 
       uri = @unit.conf['sto_uri']
 
-      #uri = DB.uri if uri == 'DB' && defined?(DB)
-      uri = (Kernel.const_get(uri).uri rescue uri) if uri.match(/\A[A-Z]+\z/)
-        # for cases where `sto_uri: "DB"`
+      fail ArgumentError.new("no 'sto_uri' conf, cannot connect to db") \
+        unless uri
+
+      return Kernel.const_get(uri) \
+        if uri.is_a?(String) && uri.match(/\A[A-Z]+\z/)
+          # for cases where uri == 'DB'
+
+      Sequel.connect(uri)
+    end
+
+    def connect
 
-      @db = Sequel.connect(uri)
+      @db = derive_db
 
       class << @db; attr_accessor :flor_unit; end
       @db.flor_unit = @unit