flor 0.9.5 → 0.10.0

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 # flor CHANGELOG.md
 
 
+## flor 0.10.0  released 2017-03-03
+
+- Enhance shell, bring in bin/flosh (though not in gem)
+- Rework "deep" tools (accept square bracket indexes)
+- Implement vanilla "case"
+- Implement "graft"
+- Link unit name and unit identifier
+- Scheduler rework, emphasis on optimistic locking for messages and timers
+
+
 ## flor 0.9.5  released 2017-02-10
 
 - Don't load exids for which there are "loaded" messages

data/Makefile

@@ -34,16 +34,20 @@ push: build
 ## flor tasks ##
 
 RUBY=bundle exec ruby
+#RUBY=bundle exec ruby --disable-did_you_mean
+  # gem uninstall did_you_mean
+
 FLOR_ENV?=dev
 TO?=nil
 FROM?=nil
 
+
 migrate:
 	$(RUBY) -Ilib -e "require 'flor/unit'; Flor::Unit.new('envs/$(FLOR_ENV)/etc/conf.json').storage.migrate($(TO), $(FROM))"
 
 start:
 	$(RUBY) -Ilib -e "require 'flor/unit'; Flor::Unit.new('envs/$(FLOR_ENV)/etc/conf.json').start.join"
-s: start
+#s: start
 
 
 ## misc tasks ##
@@ -62,9 +66,13 @@ mk:
 doc:
 	$(RUBY) -Imak -r 'doc' -e "make_procedures_doc()"
 
-repl:
-	$(RUBY) -Ilib -r 'flor/tools/repl' -e 'Flor::Tools::Repl.new("$(FLOR_ENV)")'
-r: repl
+shell:
+	$(RUBY) -Ilib -r 'flor/tools/shell' -e 'Flor::Tools::Shell.new'
+sh: shell
+
+cleanshell:
+	rm -f envs/shell/var/flor.db
+	rm -fR envs/shell/var/tasks/*
 
-.PHONY: doc repl
+.PHONY: doc shell cleanshell
 

data/README.md

@@ -25,41 +25,6 @@ Flor is a "Ruby workflow engine", if that makes any sense.
 
 see [doc/](doc/).
 
-## Running the specs
-
-(Most of the time, as developer of flor, I'm writing specs, running them with `FLOR_DEBUG=dbg` and hammering the code until the specs are green. The following lines are about setting `FLOR_DEBUG` for flor development).
-
-##### setting FLOR_DEBUG
-
-This is a targetted run of a spec file:
-```
-FLOR_DEBUG=msg,err bundle exec rspec spec/punit/cancel_spec.rb
-```
-
-* `msg` displays the flor messages in a summary, colored format
-* `err` displays errors with details, when and if they happen
-* `src` displays the source before it gets parsed and launched
-* `tree` displays the syntax tree as parsed from the source, right before launch
-* `run` shows info about each [run](doc/glossary.md#run) that just ended
-* `sto` displays debug information about the [storage](doc/glossary.md#storage), it's mostly SQL statements
-
-#### `FLOR_DEBUG=dbg` and `FLOR_DEBUG=all`
-
-There are two shortcuts for the flags above:
-```
-FLOR_DEBUG=dbg bundle exec rspec spec/punit/cancel_spec.rb
-  # is equivalent to
-FLOR_DEBUG=msg,err,src,tree,run bundle exec rspec spec/punit/cancel_spec.rb
-```
-and
-```
-FLOR_DEBUG=all bundle exec rspec spec/punit/cancel_spec.rb
-  # is equivalent to
-FLOR_DEBUG=msg,err,src,tree,run,log,sto bundle exec rspec spec/punit/cancel_spec.rb
-```
-
-I tend to use `FLOR_DEBUG=dbg` when developping flor.
-
 
 ## LICENSE
 

data/flor.gemspec

@@ -37,6 +37,7 @@ A Ruby workflow engine (ruote next generation)
     # so Sequel is not a runtime dependency
 
   s.add_development_dependency 'rspec', '3.4.0'
+  s.add_development_dependency 'terminal-table'
 
   s.require_path = 'lib'
 end

data/lib/flor.rb

@@ -1,32 +1,10 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
 
 require 'pp'
 require 'json'
 require 'logger'
 require 'thread'
 require 'digest'
+require 'socket'
 
 require 'munemo'
 require 'raabro'
@@ -34,17 +12,21 @@ require 'raabro'
 
 module Flor
 
-  VERSION = '0.9.5'
+  VERSION = '0.10.0'
 end
 
 require 'flor/colours'
+require 'flor/djan'
 
+require 'flor/id'
 require 'flor/log'
 require 'flor/flor'
+require 'flor/deep'
 require 'flor/dollar'
 require 'flor/errors'
 require 'flor/parser'
 require 'flor/conf'
+require 'flor/changes'
 require 'flor/to_string'
 
 require 'flor/core'

data/lib/flor/changes.rb

@@ -0,0 +1,26 @@
+
+module Flor::Changes
+  extend self
+
+  def apply(h, cs)
+
+    #return cs if cs.is_a?(Hash)
+
+    h = Flor.dup(h)
+    cs.each { |c| do_apply(h, c) }
+
+    h
+  end
+
+  protected
+
+  def do_apply(h, c)
+
+    case c['op']
+    when 'add' then Flor.deep_insert(h, c['path'], c['value'])
+    when 'replace' then Flor.deep_set(h, c['path'], c['value'])
+    when 'remove' then Flor.deep_unset(h, c['path'])
+    end
+  end
+end
+

data/lib/flor/colours.rb

@@ -1,27 +1,3 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 
@@ -41,8 +17,8 @@ module Flor
     brown yellow purple magenta dark_grey dark_gray light_grey light_gray
 
     rd red bl blue bu blue ba black bk black gn green gr green dg dark_gray
-    gy light_gray lg light_gray yl yellow ma magenta rs reset br bright
-    un underlined rv reverse bn blink blg bg_light_gray bri bright
+    gy light_gray lg light_gray yl yellow y yellow ma magenta rs reset
+    br bright bri bright un underlined rv reverse bn blink blg bg_light_gray
     und underlined rev reverse
   ]]
 
@@ -50,9 +26,13 @@ module Flor
 
     Flor::COLS.each do |k, v|
       if v.match(/\A\d/)
-        class_eval("def #{k}; \"[#{v}m\"; end")
+        class_eval(%{
+          def #{k}(s=nil)
+           s ? "[#{v}m" + s + "": "[#{v}m"
+           end })
       else
-        class_eval("alias #{k} #{v}")
+        class_eval(
+          "alias #{k} #{v}")
       end
     end
   end
@@ -61,7 +41,7 @@ module Flor
 
     Flor::COLS.each do |k, v|
       if v.match(/\A\d/)
-        class_eval("def #{k}; ''; end")
+        class_eval("def #{k}(s=''); s; end")
       else
         class_eval("alias #{k} #{v}")
       end
@@ -78,8 +58,14 @@ module Flor
 
   def self.colours(opts={})
 
-    return @colours if opts[:color] == true || opts[:colour] == true
-    return @no_colours if opts[:color] == false || opts[:colour] == false
+    c = nil
+      #
+    [ :color, :colour, :colors, :colours ].each do |k|
+      if opts.has_key?(k); c = opts[k]; break; end
+    end
+
+    return @colours if c == true
+    return @no_colours if c == false
 
     o = opts[:out] || $stdout
 
@@ -87,5 +73,53 @@ module Flor
       @colours :
       @no_colours
   end
+
+  def self.decolour(s)
+    s.gsub(/\x1b\[\d+(;\d+)?m/, '')
+  end
+
+  def self.no_colour_length(s)
+
+    decolour(s).length
+  end
+
+  def self.truncate_string(s, maxlen, post='...')
+
+    ncl = no_colour_length(s)
+    r = StringIO.new
+    l = 0
+
+    s.scan(/(\x1b\[\d+(?:;\d+)?m|[^\x1b]+)/) do |ss, _|
+      if ss[0, 1] == ""
+        r << ss
+      else
+#p({ r: r.string, l: l, ssl: ss.length, maxlen: maxlen, reml: maxlen - l })
+        ss = ss[0, maxlen - l]
+        r << ss
+        l += ss.length
+        break if l >= maxlen
+      end
+    end
+
+    return r.string if l < maxlen
+
+    if post.is_a?(String)
+      r << post
+    elsif post.is_a?(Proc)
+      r << post.call(ncl, maxlen, s)
+    end
+
+    r.string
+  end
+
+  class << self
+
+    alias decolor decolour
+
+    alias bw_length no_colour_length
+    alias nocolor_length no_colour_length
+    alias no_color_length no_colour_length
+    alias nocolour_length no_colour_length
+  end
 end
 

data/lib/flor/conf.rb

@@ -1,55 +1,100 @@
-#--
-# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-
 
 module Flor
 
   module Conf
 
-    #
     # * uni_ unit prefix (or no prefix like for :env)
     # * sch_ scheduler prefix
     # * sto_ storage prefix
     # * exe_ executor prefix
     # * log_ logger prefix
+    # * loa_ loader prefix
+    # * gan_ ganger prefix
+    # * (tas_ tasker prefix)
+    #
     #
+    # * :uni_name
+    #   The name for the unit (Scheduler+Storage) pair.
+    #   The name must match the following regex: `/\A[a-zA-Z0-9_]+\z/`
+    #   Can also be set via the FLOR_UNIT environment variable.
     #
     # * :sch_heart_rate
-    #   defaults to 0.3s, checks for exids to run and timers to triggers
-    #   at least every 0.3s
+    #   Defaults to 0.3s, every 0.3s the scheduler checks its @wake_up and
+    #   @next_time fields to determine if it has to run (no db query)
+    #   @wake_up is set to true when there are incoming messages give to
+    #   this unit, @next_time simply holds the timestamp for the next timer
+    #   that has to trigger
     #
-    # * :sch_reload_frequency
-    #   resync (reload) with db after how much time? (defaults to 60 (seconds))
+    # * :sch_reload_after
+    #   reload/resync with db after how much time? (defaults to 60 (seconds))
+    #   minimizes communication with db in idle periods
     #
     # * :sch_max_executors
-    #   how many executor thread at most? (defaults to 7, 1 is OK in certain
+    #   How many executor thread at most? (defaults to 7, 1 is OK in certain
     #   environments)
     #
     # * :exe_max_messages
-    #   how many messages will an executor run in a single session
-    #   (before quitting and passing the hand)
+    #   How many messages will an executor run in a single session
+    #   (before quitting and passing the hand) Defaults to 77
+    #   An executor will not run indefinitely as long as they are messages.
+    #   The goal is to prevent an execution from monopolizing an executor.
+    #
+    # * :sch_msg_max_res_time
+    #   Should an executor fail, its reserved messages may stay put.
+    #   Each time the scheduler goes to the database to trigger messages,
+    #   it first updates messages that have been "reserved" for too long
+    #   and sets them back as "created" so that they may be picked again.
+    #   This values defaults to 10 * 60 seconds. After 10 minutes in the
+    #   "reserved" state, messages are put back in the "created" state
+    #   (by a running unit (scheduler) if any).
+    #
+    # And finally:
+    #
+    # * :flor_debug or :debug
     #
+    #   Usually set via the FLOR_DEBUG environment variable.
+    #   * `msg` displays the flor messages in a summary, colored format
+    #   * `err` displays errors with details, when and if they happen
+    #   * `src` displays the source before it gets parsed and launched
+    #   * `tree` displays the syntax tree as parsed from the source, right
+    #     before launch
+    #   * `run` shows info about each [run](doc/glossary.md#run) that just ended
+    #   * `sto` displays debug information about the
+    #           [storage](doc/glossary.md#storage), it's mostly SQL statements
+    #
+    #   * `stdout` states that the debug messages must go to STDOUT
+    #   * `stderr` states that the debug messages must go to STDERR
+    #
+    #   For example `debug: 'msg,stdout'`
+
+    def self.read(s)
+
+      h = {}
+      h.merge!(Flor::ConfExecutor.interpret(s))
+      h.merge!(interpret_flor_debug(h['flor_debug'] || h['debug']))
+
+      h
+    end
+
+    def self.read_env
+
+      h = {}
+      h.merge!(interpret_env)
+      h.merge!(interpret_flor_debug(ENV['FLOR_DEBUG']))
+
+      h
+    end
+
+    def self.get_class(conf, key)
+
+      if v = conf[key]
+        Kernel.const_get(v)
+      else
+        nil
+      end
+    end
+
+    protected # somehow
 
     LOG_DBG_KEYS = %w[ dbg msg err src tree tree_rw run ]
     LOG_ALL_KEYS = %w[ all log sto ] + LOG_DBG_KEYS
@@ -75,34 +120,19 @@ module Flor
         if a.include?('colours') || a.include?('colors')
           # LOG_DEBUG=colours forces colors
 
-      if a.include?('stdout')
-        h['log_out'] = 'stdout'
-      elsif a.include?('stderr')
-        h['log_out'] = 'stderr'
-      end
+      h['log_out'] = 'stdout' if h.delete('stdout')
+      h['log_out'] = 'stderr' if h.delete('stderr')
 
       h
     end
 
-    def self.read(s)
-
-      h = Flor::ConfExecutor.interpret(s)
+    def self.interpret_env
 
-      h.merge!(interpret_flor_debug(h['flor_debug']))
-    end
+      h = {}
+      u = ENV['FLOR_UNIT']
+      h['unit'] = u if u
 
-    def self.read_env
-
-      interpret_flor_debug(ENV['FLOR_DEBUG'])
-    end
-
-    def self.get_class(conf, key)
-
-      if v = conf[key]
-        Kernel.const_get(v)
-      else
-        nil
-      end
+      h
     end
   end
 end