flor 0.9.5 → 0.10.0

data/lib/flor/unit/logger.rb

@@ -1,26 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
 
 module Flor
 
@@ -103,6 +80,8 @@ module Flor
     def db_log(level, msg)
 
       return unless @unit.conf['log_sto']
+#return unless msg.match(/flor_timers/)
+#p Time.now
 
       _c = Flor.colours(out: @out)
 
@@ -140,17 +119,19 @@ module Flor
       @out.puts(s.string)
     end
 
-    def log_run_end(executor, t0)
+    def log_run_end(executor, tstamp, duration)
 
       return unless @unit.conf['log_run']
 
       execution = executor.execution
+      exid = executor.exid
+
       _c = Flor.colours(out: @out)
       s = StringIO.new
 
       s << _c.dg
-      s << "    |   run ends #{self.class} #{self.object_id} #{@exid}"
-      s << "\n    |   "; s << { took: Time.now - t0 }.inspect
+      s << "    |   run ends #{self.class} #{self.object_id} #{exid}"
+      s << "\n    |   "; s << { started: tstamp, took: duration }.inspect
       s << "\n    |   "; s << {
         thread: Thread.current.object_id,
         consumed: executor.consumed.count,
@@ -165,7 +146,7 @@ module Flor
       if @unit.archive
         s << "\n    |   "
         s << {
-          archive_size: @unit.archive[@exid].size
+          archive_size: (@unit.archive[exid].size rescue '???')
         }.inspect
       end
       s << "\n    \\--- ."

data/lib/flor/unit/models.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 

data/lib/flor/unit/models/execution.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 
@@ -43,6 +19,19 @@ module Flor
         .find { |n| n['failure'] && n['status'] != 'triggered-on-error' }
     end
 
+    def full_tree
+
+      tree = nodes['0']['tree']
+
+      nodes.each do |nid, n|
+        next if nid == '0'
+        t = n['tree']; next unless t
+# TODO
+      end
+
+      tree
+    end
+
     # class methods
 
     def self.by_status(s)

data/lib/flor/unit/models/pointer.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 

data/lib/flor/unit/models/timer.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 

data/lib/flor/unit/models/trace.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 

data/lib/flor/unit/models/trap.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 

data/lib/flor/unit/scheduler.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 
@@ -67,21 +43,30 @@ module Flor
       @hooker.add('wlist', Flor::WaitList)
 
       @heart_rate = @conf[:sch_heart_rate] || 0.3
-      @reload_frequency = @conf[:sch_reload_frequency] || 60
-      @max_executors = @conf[:sch_max_executors] || 1
+      @reload_after = @conf[:sch_reload_after] || 60
+        #
+      @wake_up = true
+      @next_time = nil
+      @reloaded_at = Time.now
 
-      @mutex = Mutex.new
+      @msg_max_res_time = @conf[:sch_msg_max_res_time] || 10 * 60
 
-      @reloaded_at = nil
-      @timers = []
-      @exids = []
+      @idle_count = 0
 
+      @max_executors = @conf[:sch_max_executors] || 1
+        #
       @executors = []
 
-      @archive = nil # used, so far, only for testing
-
       c = @conf['constant']
+        #
       Kernel.const_set(c, self) if c
+
+      @archive = nil # used, so far, only for testing
+    end
+
+    def name
+
+      @conf['unit'] || @conf['uni_name'] || 'u0'
     end
 
     def storage_mutex
@@ -91,7 +76,20 @@ module Flor
 
     def identifier
 
-      @identifier ||= 's' + Digest::MD5.hexdigest(self.object_id.to_s)[0, 5]
+      @identifier ||=
+        begin
+          ai =
+            Socket.ip_address_list.find { |a| a.ipv4_private? } ||
+            Socket.ip_address_list.find { |a| a.ip_address != '::1' }
+          ip =
+            ai ? ai.ip_address : '::1'
+          [
+            'sch', self.name,
+            'i' + ip,
+            'p' + Process.pid.to_s,
+            'o' + (self.object_id % 100_000).to_s(32)
+          ].join('-')
+        end
     end
 
     def has_tasker?(exid, tname)
@@ -157,23 +155,36 @@ module Flor
         else
 
           Thread.new do
-#p [ :unit_scheduler, :thread, Thread.current.object_id ]
+
             loop do
 
               begin
 
-                t0 = Time.now
-
                 Thread.stop if @thread_status == :stop
                 break if @thread_status == :shutdown
 
-                reload
-                trigger_timers
-                trigger_executions
+                t0 = Time.now
+
+                if should_wake_up?
+
+                  unreserve_messages
+
+                  trigger_timers
+                  trigger_executions
 
-                sleep [ @heart_rate - (Time.now - t0), 0 ].max
+                  reload_next_time
+                  reload_wake_up
+                  @reloaded_at = Time.now
+
+                elsif @executors.empty?
+
+                  @idle_count += 1
+                  notify(nil, make_idle_message)
+                end
+
+                sleep [ @heart_rate - (Time.now - t0), 0 ].max #\
+                  #unless should_wake_up?
 
-              #rescue => er
               rescue Exception => ex
 
                 puts on_start_exc(ex)
@@ -228,16 +239,16 @@ module Flor
       end
 
       fail ArgumentError.new(
-        "flow not found in #{Flor.truncate(source_or_path, 35).inspect}"
+        "flow not found in #{Flor.truncate_string(source_or_path, 35).inspect}"
       ) unless source # will anyway fail badly if src is a tree (array of ...)
 
       @archive ||= {} if opts[:archive]
         # all subsequent launches will be `archive: true` ...
 
-      unit = opts[:unit] || @conf['unit'] || 'u0'
-
       @logger.log_src(source, opts)
 
+      unit = opts[:unit] || self.name
+
       exid = Flor.generate_exid(domain, unit)
       msg = Flor.make_launch_msg(exid, source, opts)
 
@@ -260,17 +271,16 @@ module Flor
       end
     end
 
-    def prepare_message(point, h)
-
-      msg = { 'point' => point }
-      [ :exid, :name, :nid, :payload ].each { |k| msg[k.to_s] = h[k] }
+    def return(message)
 
-      fail ArgumentError.new('missing :exid key') \
-        unless msg['exid'].is_a?(String)
-      fail ArgumentError.new('missing :name string key') \
-        if point == 'signal' && ! msg['name'].is_a?(String)
+      queue({
+        'point' => 'return',
+        'exid' => message['exid'],
+        'nid' => message['nid'],
+        'payload' => message['payload'],
+        'tasker' => message['tasker'] })
 
-      msg
+      nil
     end
 
     def cancel(h)
@@ -285,21 +295,30 @@ module Flor
       queue(prepare_message('signal', h), h)
     end
 
-    def put_timer(message)
+    def re_apply(h)
 
-      timer = @storage.put_timer(message)
+      queue(prepare_message('cancel', h.merge(re_apply: true)), h)
+    end
+
+    def put_timer(message)
 
-      @mutex.synchronize { @timers.push(timer).sort_by!(&:ntime) }
+      #timer = @storage.put_timer(message)
+      #@mutex.synchronize { @timers.push(timer).sort_by!(&:ntime) }
+      @storage.put_timer(message)
     end
 
-    def wake_up_executions(exids)
+    def wake_up
 
-      @mutex.synchronize { @exids.concat(exids).uniq! } if exids.any?
+      @wake_up = true
     end
 
     def notify(executor, o)
 
-      @hooker.notify(executor, o)
+      if executor
+        @hooker.notify(executor, o)
+      else
+        @hooker.wlist.notify(nil, o)
+      end
 
     rescue => err
       puts '-sch' * 19
@@ -319,9 +338,9 @@ module Flor
       #@storage.remove_node(exid, n)
         # done in Storage#put_execution
 
-      @mutex.synchronize do
-        @timers.reject! { |t| t.exid == exid && t.nid == n['nid'] }
-      end
+      #@mutex.synchronize do
+      #  @timers.reject! { |t| t.exid == exid && t.nid == n['nid'] }
+      #end
 
       (@archive[exid] ||= {})[n['nid']] = Flor.dup(n) if @archive
     end
@@ -341,98 +360,108 @@ module Flor
 
     protected
 
-#    # return [ domain, tree ]
-#    #
-#    def extract_domain_and_tree(s, opts)
-#
-#      if Flor.potential_domain_name?(s)
-#
-#        path = [ opts[:domain], s ].compact.join('.')
-#        elts = path.split('.')
-#        flow = @loader.library(path)
-#
-#        fail ArgumentError.new(
-#          "flow not found at #{path.inspect}"
-#        ) unless flow
-#
-#        [ elts[0..-2].join('.'), flow ]
-#
-#      else
-#
-#        [ opts[:domain], s ]
-#      end
-#    end
-
-    def reload
-
-      now = Time.now
-
-      return if @reloaded_at && (now - @reloaded_at < @reload_frequency)
-
-      @mutex.synchronize do
-
-        @reloaded_at = now
-        @timers = load_timers
-        @exids = load_exids
-      end
+    def prepare_on_receive_last(h)
+
+      ei = h[:exid] || h['exid']
+      ni = h[:nid] || h['nid']
+      t = h[:tree] || h['tree']
+      pl = h[:payload] || h['payload']
+
+      fail ArgumentError.new('missing :payload to re_apply') unless pl
+
+      t = Flor::Lang.parse(t, 're_apply', {})
+
+      [
+        { 'point' => 'execute',
+          'exid' => ei, 'nid' => ni,
+          'from' => 'parent',
+          'tree' => t,
+          'payload' => pl }
+      ]
     end
 
-    def load_timers
+    def prepare_message(point, h)
+
+      msg = { 'point' => point }
+
+      [ :exid, :name, :nid, :payload, :on_receive_last ]
+        .each { |k| v = h[k] || h[k.to_s]; msg[k.to_s] = v if v }
+
+      if h[:re_apply]
+        msg['on_receive_last'] = prepare_on_receive_last(h)
+      end
+
+      fail ArgumentError.new('missing :exid key') \
+        unless msg['exid'].is_a?(String)
+      fail ArgumentError.new('missing :name string key') \
+        if point == 'signal' && ! msg['name'].is_a?(String)
 
-      return [] if @thread_status != :running
-      @storage.load_timers.sort_by(&:ntime)
+      msg
     end
 
-    def load_exids
+    def make_idle_message
+
+      m = {}
+      m['point'] = 'idle'
+      m['idle_count'] = @idle_count
+      m['consumed'] = true
 
-      return [] if @thread_status != :running
-      @storage.load_exids
+      m
     end
 
-    def trigger_timers
+    def should_wake_up?
 
-      now = Time.now.utc
-      to_re_add = []
+      return true if Time.now - @reloaded_at >= @reload_after
 
-      loop do
+      return true if @wake_up
+      return false unless @next_time
 
-        timer = @timers.first
-        break if timer == nil || timer.ntime_t > now
+      @next_time <= Flor.tstamp.split('.').first
+    end
 
-        t = @mutex.synchronize { @timers.shift }
-        r = @storage.trigger_timer(t)
-        to_re_add << r if r
-      end
+    def unreserve_messages
 
-      if to_re_add.any?
+      c = @storage.unreserve_messages(@msg_max_res_time)
 
-        @mutex.synchronize do
-          @timers.concat(to_re_add)
-          @timers.sort_by! { |t| t.ntime }
-        end
-      end
+      @logger.info(
+        "#{self.class}#unreserve_messages", "#{c} message#{c > 1 ? 's' : ''}"
+      ) if c > 0
+    end
+
+    def trigger_timers
+
+      @storage.trigger_timers
     end
 
     def trigger_executions
 
-      return if @exids.empty?
+      @executors.select! { |e| e.alive? }
+        # drop done executors
 
-      exid = nil
+      free_executor_count = @max_executors - @executors.size
 
-      while exid = @mutex.synchronize { @exids.shift }
+      return if free_executor_count < 1
 
-        @executors = @executors.select { |e| e.alive? }
-          # drop done executors
+      messages = @storage.load_messages(free_executor_count)
 
-        break if @executors.size > @max_executors
-        next if @executors.find { |e| e.exid == exid }
+      messages.each do |exid, ms|
 
-        @executors << UnitExecutor.new(self, exid).run
+        next unless @storage.reserve_all_messages(ms)
 
-        exid = nil
+        @idle_count = 0
+
+        @executors << UnitExecutor.new(self, ms).run
       end
+    end
+
+    def reload_next_time
+
+      @next_time = @storage.fetch_next_time
+    end
+
+    def reload_wake_up
 
-      @mutex.synchronize { @exids.unshift(exid) } if exid
+      @wake_up = @storage.any_message?
     end
   end
 end