flor 0.0.1 → 0.9.0

data/lib/flor/to_string.rb

@@ -0,0 +1,81 @@
+#--
+# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
+
+module Flor
+
+  def self.to_s(o=nil, k=nil)
+
+    return 'FlorModule' if o == nil && k == nil
+      # should it emerge somewhere...
+
+    return o.collect { |e| Flor.to_s(e, k) }.join("\n") if o.is_a?(Array)
+
+    if o.is_a?(Hash)
+
+      return send("message_#{k}_to_s", o) if k && o['point'].is_a?(String)
+      return message_to_s(o) if o['point'].is_a?(String)
+
+      return send("node_#{k}_to_s", o) if k && o.has_key?('parent')
+      return node_to_s(o) if o['parent'].is_a?(String)
+    end
+
+    return [ o, k ].inspect if k
+    o.inspect
+  end
+
+  def self.message_to_s(m)
+
+    s = StringIO.new
+    s << '(msg ' << m['nid'] << ' ' << m['point']
+    %w[ from flavour ].each { |k|
+      s << ' ' << k << ':' << m[k].to_s if m.has_key?(k) }
+    s << ')'
+
+    s.string
+  end
+
+  def self.node_status_to_s(n)
+
+    s = StringIO.new
+    stas = n['status'].reverse
+    while sta = stas.shift
+      s << '(status ' << (sta['status'] || 'o') # o for open
+      s << ' pt:' << sta['point']
+      if f = sta['flavour']; s << ' fla:' << f; end
+      if f = sta['from']; s << ' fro:' << f; end
+      if m = sta['m']; s << ' m:' << m; end
+      s << ')'
+      s << "\n" if stas.any?
+    end
+
+    s.string
+  end
+
+  def self.node_to_s(n) # there is already a .node_to_s in log.rb
+
+    n.inspect
+  end
+end
+

data/lib/flor/tools/env.rb

@@ -0,0 +1,103 @@
+#--
+# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
+require 'fileutils'
+
+
+module Flor; end
+module Flor::Tools; end
+
+module Flor::Tools::Env
+
+  def self.make(path, envname=nil, opts={})
+
+    if envname.is_a?(Hash)
+      opts = envname
+      envname = nil
+    end
+
+    opts[:env] = envname || 'production'
+    opts[:sto_uri] ||= 'sqlite://tmp/dev.db'
+    opts[:gitkeep] = true unless opts.has_key?(:gitkeep)
+
+    mkdir(path, envname) if envname
+
+    mk_etc(path, envname, opts)
+    mk_lib(path, envname, opts)
+    mk_usr(path, envname, opts)
+    mk_var(path, envname, opts)
+  end
+
+  # protected, somehow
+
+  def self.mk_etc(*ps, opts)
+
+    mkdir(*ps, 'etc')
+    mkdir(*ps, 'etc', 'variables')
+    touch(*ps, 'etc', 'variables', '.gitkeep') if opts[:gitkeep]
+
+    write(*ps, 'etc', 'conf.json') do
+      "env: #{opts[:env]}\n" +
+      "sto_uri: #{opts[:sto_uri].inspect}"
+    end
+  end
+
+  def self.mk_lib(*ps, opts)
+
+    mkdir(*ps, 'lib')
+    mkdir(*ps, 'lib', 'flows')
+    touch(*ps, 'lib', 'flows', '.gitkeep') if opts[:gitkeep]
+    mkdir(*ps, 'lib', 'taskers')
+    touch(*ps, 'lib', 'taskers', '.gitkeep') if opts[:gitkeep]
+  end
+
+  def self.mk_usr(*ps, opts)
+
+    mkdir(*ps, 'usr')
+
+    if opts[:acme] == false
+      touch(*ps, 'usr', '.gitkeep') if opts[:gitkeep]
+    else
+      mkdir(*ps, 'usr', 'com.acme')
+      mk_etc(*ps, 'usr', 'com.acme', opts)
+      mk_lib(*ps, 'usr', 'com.acme', opts)
+    end
+  end
+
+  def self.mk_var(*ps, opts)
+
+    mkdir(*ps, 'var')
+    mkdir(*ps, 'var', 'log')
+    touch(*ps, 'var', 'log', '.gitkeep') if opts[:gitkeep]
+  end
+
+  def self.mkdir(*ps); FileUtils.mkdir(File.join(*ps.compact)); end
+  def self.touch(*ps); FileUtils.touch(File.join(*ps.compact)); end
+
+  def self.write(*ps, &block)
+
+    File.open(File.join(*ps.compact), 'wb') { |f| f.write(block.call) }
+  end
+end
+

data/lib/flor/tools/repl.rb

@@ -0,0 +1,231 @@
+
+require 'flor'
+require 'flor/unit'
+
+
+module Flor::Tools
+
+  class Repl
+
+    def initialize(env)
+
+      @unit = Flor::Unit.new("envs/#{env}/etc/conf.json")
+
+      #pp @unit.conf
+      @unit.conf[:unit] = 'repl'
+
+      #unit.hooker.add('journal', Flor::Journal)
+      if @unit.conf['sto_uri'].match(/memory/)
+        @unit.storage.delete_tables
+        @unit.storage.migrate
+      end
+      @unit.start
+
+      @lines = []
+      @payload = {}
+      @vars = {}
+
+      @outcome = nil
+
+      @unit.hook do |message|
+
+        if ! message['consumed']
+          # do nothing
+        elsif %w[ terminated failed ].include?(message['point'])
+          @outcome = message
+          out = Flor.to_pretty_s(@outcome)
+          col = message['point'] == 'failed' ? _rd : _gr
+          out = out.gsub(/"point"=>"([^"]+)"/, "\"point\"=>\"#{col}\\1#{_yl}\"")
+          out = "\n" + _yl + out + _rs
+          out = out.split("\n").collect { |l| '  ' + l }.join("\n")
+          print(out)
+        end
+      end
+
+      @_c, @_dg, @_yl, @_bl, @_lg, @_gr, @_lr, @_rd = Flor.colours({})
+
+      do_loop
+    end
+
+    # reset dark_grey light_yellow blue light_grey light_green light_red red
+    attr_reader :_rs, :_dg, :_yl, :_bl, :_lg, :_gr, :_lr, :_rd
+
+    protected
+
+    def prompt
+
+      "flor l#{@lines.size} > "
+    end
+
+    def do_loop
+
+      loop do
+
+        line = prompt_and_read
+
+        break unless line
+        next if line.strip == ''
+
+        cmd = "cmd_#{line.split(/\s/).first}".to_sym
+
+        if cmd.size > 4 && methods.include?(cmd)
+          begin
+            send(cmd, line)
+          rescue StandardError, NotImplementedError => err
+            p err
+            err.backtrace[0, 7].each { |l| puts "  #{l}" }
+          end
+        else
+          @lines << line
+        end
+      end
+
+      $stdout.puts
+    end
+
+    def hlp_launch
+      %{ launches the current execution code }
+    end
+    def cmd_launch(line)
+
+      exid = @unit.launch(@lines.join("\n"), vars: @vars, payload: @payload)
+      puts "  launched #{_yl}#{exid}#{_rs}"
+    end
+
+    alias hlp_run hlp_launch
+    alias cmd_run cmd_launch
+
+    def hlp_help
+      %{ displays this help }
+    end
+    def cmd_help(line)
+
+      puts
+      puts "## available commands:"
+      puts
+      COMMANDS.each do |cmd|
+        print "* #{cmd}"
+        if hlp = (send("hlp_#{cmd}") rescue nil); print " - #{hlp.strip}"; end
+        puts
+      end
+      puts
+    end
+
+    def hlp_exit
+      %{ exits this repl, with the given int exit code or 0 }
+    end
+    def cmd_exit(line)
+
+      exit(line.split(/\s+/)[1].to_i)
+    end
+
+    def hlp_list
+      %{ lists the lines of the current execution code }
+    end
+    def do_list(lines)
+
+      lw = [ 2, lines.size.to_s.length ].max
+      sw = 5 - lw
+
+      lines.each_with_index do |l, i|
+        puts "#{_dg}% #{sw}s%0#{lw}i #{_yl}%s#{_rs}" % [ '', i + 1, l ]
+      end
+    end
+    def cmd_list(line)
+
+      do_list(@lines)
+    end
+
+    def hlp_parse
+      %{ parses the current execution code and displays its tree }
+    end
+    def cmd_parse(line)
+
+      Flor.print_tree(
+        Flor::Lang.parse(@lines.join("\n"), nil, {}),
+        '0',
+        headers: false)
+    end
+
+    def hlp_new
+      %w{ erases current execution code, vars and payload }
+    end
+    def cmd_new(line)
+
+      @lines = []
+      @vars = {}
+      @payload = {}
+    end
+
+    def fname(line)
+
+      line.split(/\s+/)[1]
+    end
+
+    def hlp_save
+      %{ saves the current execution code to the given file }
+    end
+    def cmd_save(line)
+
+      File.open(fname(line), 'wb') { |f| f.puts @lines }
+    end
+
+    def hlp_cat
+      %{ outputs the content of the give file }
+    end
+    def cmd_cat(line)
+
+      do_list(File.readlines(fname(line)).collect(&:chomp))
+    end
+
+    def hlp_load
+      %{ loads a file as execution code }
+    end
+    def cmd_load(line)
+
+      @lines = File.readlines(fname(line)).collect(&:chomp)
+    end
+
+    def cmd_cont(line)
+
+      fail NotImplementedError
+    end
+
+    def hlp_edit
+      %{ saves the current execution code to .tmp.flo and opens it for edition }
+    end
+    def cmd_edit(line)
+
+      cmd_save('save .tmp.flo')
+      system('$EDITOR .tmp.flo')
+      cmd_load('load .tmp.flo')
+      FileUtils.rm('.tmp.flo')
+    end
+
+    #
+    # use Readline if possible
+
+    COMMANDS = self.allocate.methods \
+      .select { |m| m.to_s.match(/^cmd_/) }.collect { |m| m[4..-1] }.sort
+
+    begin
+      require 'readline'
+      def prompt_and_read
+        Readline.readline(prompt, true)
+      end
+      Readline.completion_proc =
+        proc { |s|
+          r = /^#{Regexp.escape(s)}/
+          COMMANDS.grep(r) + Dir["#{s}*"].grep(r)
+        }
+      #Readline.completion_append_character =
+      #  " "
+    rescue LoadError => le
+      def prompt_and_read
+        print(prompt)
+        ($stdin.readline rescue false)
+      end
+    end
+  end
+end
+

data/lib/flor/unit/executor.rb

@@ -0,0 +1,260 @@
+#--
+# Copyright (c) 2015-2017, John Mettraux, jmettraux+flor@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
+
+module Flor
+
+  class UnitExecutor < Flor::Executor
+
+    attr_reader :exid
+
+    def initialize(unit, exid)
+
+      super(
+        unit,
+        unit.storage.fetch_traps(exid),
+        unit.storage.load_execution(exid))
+
+      @exid = exid
+      @messages = unit.storage.fetch_messages(exid)
+      @consumed = []
+
+      @alive = true
+      @shutdown = false
+
+      @thread = nil
+    end
+
+    def alive?; @alive; end
+
+    def run
+
+      @thread = Thread.new { do_run }
+#p [ :unit_executor, :thread, @thread.object_id ]
+
+      self
+    end
+
+    def shutdown
+
+      @shutdown = true
+      @thread.join
+    end
+
+    protected
+
+    def do_run
+
+      log_run_start if @unit.conf['log_run']
+
+      counter_next('runs')
+
+      t0 = Time.now
+
+      (@unit.conf['exe_max_messages'] || 77).times do |i|
+
+        m = @messages.shift
+        break unless m
+        break if @shutdown
+
+        if m['point'] == 'terminated' && @messages.any?
+          #
+          # try to handle 'terminated' last
+          #
+          @messages << m
+          m = @messages.shift
+        end
+
+        point = m['point']
+
+        ms = process(m)
+
+        @consumed << m
+          #
+        #@consumed << m unless ms.include?(m)
+          # TODO what if msg is held / pushed back?
+
+        ims, oms = ms.partition { |m| m['exid'] == @exid }
+          # qui est "in", qui est "out"?
+
+        counter_add('omsgs', oms.size)
+          # keep track of "out" messages, messages to other executions
+
+        @messages.concat(ims)
+        @unit.storage.put_messages(oms)
+      end
+
+      @unit.storage.consume(@consumed)
+
+      @alive = false
+
+      @unit.storage.put_execution(@execution)
+      @unit.storage.put_messages(@messages)
+
+      log_run_end(t0) if @unit.conf['log_run']
+
+      @consumed.clear
+
+    rescue Exception => exc
+
+# TODO eventually, have a dump dir
+      fn =
+        [
+          'flor',
+          @unit.conf['env'], @unit.identifier, @exid,
+          'r' + counter('runs').to_s
+        ].collect(&:to_s).join('_') + '.dump'
+
+      @unit.logger.error(
+        "#{self.class}#do_run()", exc, "(dumping to #{fn})")
+
+      File.open(fn, 'wb') do |f|
+        #f.puts(JSON.pretty_generate({
+        f.puts(Flor.to_pretty_s({
+          execution: @execution,
+          messages: @messages,
+          consumed: @consumed,
+          traps: @traps.collect(&:to_h),
+          exid: @exid,
+          alive: @alive,
+          shutdown: @shutdown,
+          thread: [ @thread.object_id, @thread.to_s ]
+        }))
+        f.puts('-' * 80)
+        f.puts(on_do_run_exc(exc))
+      end
+
+      puts on_do_run_exc(exc)
+    end
+
+    def schedule(message)
+
+      @unit.put_timer(message)
+
+      []
+    end
+
+    def trigger(message)
+
+      m = message['message']
+
+      m['nid'] = Flor.sub_nid(m['nid'], counter_next('subs')) \
+        if m['point'] == 'execute'
+
+      [ m ]
+    end
+
+    def log_run_start
+
+      _c = Flor.colours
+
+      s = StringIO.new
+
+      s << _c.dg
+      s << "    /--- #{_c.lg}run starts#{_c.dg} "
+      s << "#{self.class} #{self.object_id} #{@exid}"
+      s << "\n    |   "
+      s << { thread: Thread.current.object_id }.inspect
+      s << "\n    |   "
+      s << {
+        counters: @execution['counters'],
+        nodes: @execution['nodes'].size,
+        size: @execution['size']
+      }.inspect
+      s << _c.rs
+
+      puts s.string
+    end
+
+    def log_run_end(t0)
+
+      _c = Flor.colours
+
+      s = StringIO.new
+
+      s << _c.dg
+      s << "    |   run ends #{self.class} #{self.object_id} #{@exid}"
+      s << "\n    |   "; s << { took: Time.now - t0 }.inspect
+      s << "\n    |   "; s << {
+        thread: Thread.current.object_id,
+        consumed: @consumed.size,
+        traps: @traps.size,
+      }.inspect
+      s << "\n    |   "; s << {
+        #own_traps: @traps.reject { |t| t.texid == nil }.size, # FIXME
+        counters: @execution['counters'],
+        nodes: @execution['nodes'].size,
+        size: @execution['size']
+      }.inspect
+      if @unit.archive
+        s << "\n    |   "
+        s << {
+          archive_size: @unit.archive[@exid].size
+        }.inspect
+      end
+      s << "\n    \\--- ."
+      s << _c.rs
+
+      puts s.string
+    end
+
+    def on_do_run_exc(e)
+
+      io = StringIO.new
+      thr = Thread.current
+
+      head, kind =
+        e.is_a?(StandardError) ? [ '=exe', 'error' ] : [ '!exe', 'exception' ]
+
+      t = head[0, 2] + Time.now.to_f.to_s.split('.').last
+      io.puts ' /' + t + ' ' + head * 17
+      io.puts " |#{t} + in #{self.class}#do_run"
+      io.puts " |#{t} #{kind}: #{e.inspect}"
+      io.puts " |#{t} db:  #{@unit.storage.db.class} #{@unit.storage.db.hash}"
+      io.puts " |#{t} thread: t#{thr.object_id} #{thr.inspect}"
+      if @execution
+        io.puts " |#{t} exe:"
+        io.puts " |#{t}   exid: #{@execution['exid'].inspect}"
+        io.puts " |#{t}   counters: #{@execution['counters'].inspect}"
+      end
+      if @messages
+        io.puts " |#{t} messages:"
+        io.puts " |#{t}   #{@messages.map { |m| [ m['mid'], m['point'] ] }.inspect}"
+      end
+      if is = Thread.current[:sto_errored_items]
+        io.puts " |#{t} storage errored items/models:"
+        is.each do |i|
+          io.puts " |#{t}   * #{i.inspect}"
+        end
+      end
+      io.puts " |#{t} #{kind}: #{e.inspect}"
+      io.puts " |#{t} backtrace:"
+      e.backtrace.each { |l| io.puts "|#{t} #{l}" }
+      io.puts ' \\' + t + ' ' + (head * 17) + ' .'
+
+      io.string
+    end
+  end
+end
+