oflow 0.8.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41a62977484dfde369899d13d67f34f41e88d643
-  data.tar.gz: 80f9dc4d7fb42859bfd4b292393b53857ac418b8
+  metadata.gz: 2a972ca61c61251a79b699cce7cab0bd3a3810e9
+  data.tar.gz: 8a995d9f63fcd512f802520cf5097491d6edfbff
 SHA512:
-  metadata.gz: a32921aa1cfae8f6053f571d5b9fad0bf473f2365f46b36f12ff5866ea913e8836d6f5463b46f7fe05cdc839dd93455f985aabaccc2ae1ecf85455f84353beb0
-  data.tar.gz: 04ad4e9db6a86bb6ea02ab1f31f405f8abcb34c7af516e3598c0549414a5fbb704224eb91935a635884daad0837e1e3a8eb99805c0f074a54946fc31b818052e
+  metadata.gz: 5a31cb845ae3322e83fdcb51e8dead9f12991d6f4b4fdf3daa6f9c60ae04618fde0e220339c4a047370c482a2026d23c4901986ce5c0b4fffc3908c019655245
+  data.tar.gz: e8bca27d81e86ed6c390146fcc80a2071451f7fb2b0a73269d3a9443267910b382050aad46595560c7b1092684e769a5355d72ccef6b64f8334d559dfdd62cc3

data/README.md

@@ -25,13 +25,21 @@ Follow [@peterohler on Twitter](http://twitter.com/#!/peterohler) for announceme
 
 ## Release Notes
 
-### Next Release 0.8
+### Current Release 1.0
+
+ - Add an Actors/Task that starts a application and exchanges data using stdout
+   and stdin.
+
+ - Add an Actors/Task that callout to a remote application and exchange data
+   using JSON over stdin and stdout.
+
+### Release 0.8
 
  - A somewhat non trivial example. GemChart collects statistics on my gems and
    stores those statistics. It also provides a web interface to view the
    statistics as a graph.
 
-### Current Release 0.7
+### Release 0.7
 
  - Simplified the APIs and structure.
 
@@ -139,22 +147,16 @@ Hello World!
 
 ## Future Features
 
- - .svg file input for configuration.
-
- - Visio file input for configuration.
-
- - CallOut Actor that uses pipes and fork to use a non-Ruby actor.
-
- - Cross linking Tasks and Flows.
-
- - Dynamic links to Tasks and Flows.
-
  - High performance C version. Proof of concept puts the performance range at
    around 10M operations per second where an operation is one task execution per
    thread.
 
  - HTTP/Websockets based inpector.
 
+ - .svg file input for configuration.
+
+ - Visio file input for configuration.
+
 # Links
 
 ## Links of Interest

data/lib/oflow/actor.rb

@@ -51,6 +51,10 @@ module OFlow
       {}
     end
 
+    def busy?()
+      false
+    end
+
     class Spec
       attr_reader :op
       attr_reader :type

data/lib/oflow/actors.rb

@@ -15,3 +15,5 @@ require 'oflow/actors/merger'
 require 'oflow/actors/persister'
 require 'oflow/actors/timer'
 require 'oflow/actors/httpserver'
+require 'oflow/actors/shellone'
+require 'oflow/actors/shellrepeat'

data/lib/oflow/actors/shellone.rb

@@ -0,0 +1,79 @@
+
+require 'open3'
+
+module OFlow
+  module Actors
+    
+    # 
+    class ShellOne < Actor
+
+      attr_reader :dir
+      attr_reader :cmd
+      attr_reader :timeout
+
+      def initialize(task, options)
+        super
+        @dir = options[:dir]
+        @dir = '.' if @dir.nil?
+        @dir = File.expand_path(@dir.strip)
+        
+        @cmd = options[:cmd]
+        @timeout = options.fetch(:timeout, 1.0).to_f
+        @timeout = 0.001 if 0.001 > @timeout
+      end
+
+      def perform(op, box)
+        input = Oj.dump(box.contents, mode: :compat, indent: 0)
+        i, o, e, _ = Open3.popen3(@cmd, chdir: @dir)
+        i.write(input)
+        i.close
+        giveup = Time.now + @timeout
+        ra = [e, o]
+
+        out = ''
+        err = ''
+        ec = false # stderr closed flag
+        oc = false # stdout closed flag
+        while true
+          rem = giveup - Time.now
+          raise Exception.new("Timed out waiting for output.") if 0.0 > rem
+          rs, _, es = select(ra, nil, ra, rem)
+          unless es.nil?
+            es.each do |io|
+              ec |= io == e
+              oc |= io == o
+            end
+          end
+          break if ec && oc
+          unless rs.nil?
+            rs.each do |io|
+              if io == e && !ec
+                if io.closed? || io.eof?
+                  ec = true
+                  next
+                end
+                err += io.read_nonblock(1000)
+              elsif io == o && !oc
+                if io.closed? || io.eof?
+                  oc = true
+                  next
+                end
+                out += io.read_nonblock(1000)
+              end
+            end
+          end
+          break if ec && oc
+        end
+        if 0 < err.length
+          raise Exception.new(err)
+        end
+        output = Oj.load(out, mode: :compat)
+        o.close
+        e.close
+
+        task.ship(nil, Box.new(output, box.tracker))
+      end
+
+    end # ShellOne
+  end # Actors
+end # OFlow

data/lib/oflow/actors/shellrepeat.rb

@@ -0,0 +1,110 @@
+
+require 'open3'
+require 'thread'
+
+module OFlow
+  module Actors
+    
+    # 
+    class ShellRepeat < Actor
+
+      attr_reader :dir
+      attr_reader :cmd
+      attr_reader :timeout
+      attr_reader :out
+
+      def initialize(task, options)
+        super
+        @dir = options[:dir]
+        @dir = '.' if @dir.nil?
+        @dir = File.expand_path(@dir.strip)
+        
+        @cmd = options[:cmd]
+        @timeout = options.fetch(:timeout, 1.0).to_f
+        @timeout = 0.001 if 0.001 > @timeout
+        @in = nil
+        @out = nil
+        @err = nil
+        @pid = nil
+        @outThread = nil
+        @ctxs = {}
+        @ctxCnt = 0
+        @killLock = Mutex.new
+      end
+
+      def perform(op, box)
+        if :kill == op
+          status = kill()
+          task.ship(:killed, Box.new(status, box.tracker))
+          return
+        end
+        if @pid.nil?
+          @in, @out, @err, wt = Open3.popen3(@cmd, chdir: @dir)
+          @pid = wt[:pid]
+          @outThread = Thread.start(self) do |me|
+            Thread.current[:name] = me.task.full_name() + "-out"
+            Oj.load(me.out, mode: :compat) do |o|
+              begin
+                k = o["ctx"]
+                raise Exception.new("missing context in #{cmd} reply") if k.nil?
+                raise Exception.new("context not found in #{cmd} reply for #{k}") unless me.hasCtx?(k)
+                ctx = me.clearCtx(k)
+                me.task.ship(nil, Box.new(o["out"], ctx))
+              rescue Exception => e
+                me.task.handle_error(e)
+              end
+            end
+            @outThread = nil
+            kill()
+          end
+        end
+        if @in.closed?
+          kill()
+          return
+        end
+        @ctxCnt += 1
+        @ctxs[@ctxCnt] = box.tracker
+        wrap = { "ctx" => @ctxCnt, "in" => box.contents }
+        input = Oj.dump(wrap, mode: :compat, indent: 0)
+        @in.write(input + "\n")
+        @in.flush
+      end
+
+      def busy?()
+        !@ctxs.empty?
+      end
+
+      def getCtx(ctx)
+        @ctxs[ctx]
+      end
+
+      def hasCtx?(ctx)
+        @ctxs.has_key?(ctx)
+      end
+
+      def clearCtx(ctx)
+        @ctxs.delete(ctx)
+      end
+
+      def kill()
+        status = nil
+        @killLock.synchronize do
+          # kill but don't wait for an exit. Leave it orphaned so a new app can be
+          # started.
+          status = Process.kill("HUP", @pid) unless @pid.nil?
+          @in.close() unless @in.nil?
+          @out.close() unless @out.nil?
+          @err.close() unless @err.nil?
+          Thread.kill(@outThread) unless @outThread.nil?
+          @in = nil
+          @out = nil
+          @err = nil
+          @pid = nil
+          @outThread = nil
+        end
+        status
+      end
+
+    end # ShellRepeat
+  end # Actors
+end # OFlow

data/lib/oflow/task.rb

@@ -241,7 +241,7 @@ module OFlow
     # Returns the true if any requests are queued or a request is being processed.
     # @return [true|false] true if busy, false otherwise
     def busy?()
-      !@current_req.nil? || !@queue.empty?
+      !@current_req.nil? || !@queue.empty? || @actor.busy?
     end
 
     # Returns the default timeout for the time to wait for the Task to be

data/lib/oflow/test/actorwrap.rb

@@ -52,12 +52,25 @@ module OFlow
           begin
             @actor.perform(op, box)
           rescue Exception => e
-            ship(:error, Box.new([e, full_name()]))
+            handle_error(e)
           end
         end
         nil
       end
 
+      def handle_error(e)
+        #puts "*** #{e.class}: #{e.message}"
+        @history << Action.new(:error, Box.new([e, full_name()]))
+      end
+
+      def join(timeout)
+        giveup = Time.now + timeout
+        while @actor.busy?
+          raise Exception.new("Timed out") if giveup < Time.now
+          sleep(0.1)
+        end
+      end
+
       # Task API that adds entry to history.
       def ship(dest, box)
         @history << Action.new(dest, box)

data/lib/oflow/version.rb

@@ -1,5 +1,5 @@
 
 module OFlow
   # Current version of the module. 
-  VERSION = '0.8.0'
+  VERSION = '1.0.0'
 end

data/test/actors/bad.rb

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# broken code intentionally
+
+x = y

data/test/actors/doubler.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(File.dirname(__FILE__)) unless $:.include? File.dirname(File.dirname(__FILE__))
+
+[ File.dirname(__FILE__),
+  File.join(File.dirname(__FILE__), "../../lib"),
+  File.join(File.dirname(__FILE__), "../../../oj/ext"),
+  File.join(File.dirname(__FILE__), "../../../oj/lib"),
+].each { |path| $: << path unless $:.include?(path) }
+
+require 'oj'
+
+input = Oj.load(STDIN, mode: :compat)
+input.map! { |v| v * 2 }
+puts Oj.dump(input, mode: :compat, indent: 0)

data/test/actors/shellone_test.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(File.dirname(__FILE__)) unless $:.include? File.dirname(File.dirname(__FILE__))
+
+require 'helper'
+require 'oflow'
+require 'oflow/test'
+
+class ShellOneTest < ::MiniTest::Test
+
+  def test_shellone_config
+    root_dir = File.expand_path(File.dirname(File.dirname(__FILE__)))
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::ShellOne, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'somewhere',
+                                     cmd: 'pwd',
+                                     timeout: 0.5)
+    assert_equal(File.join(root_dir, 'somewhere'), t.actor.dir, 'dir set from options')
+    assert_equal('pwd', t.actor.cmd, 'cmd set from options')
+    assert_equal(0.5, t.actor.timeout, 'timeout set from options')
+  end
+
+  def test_shellone_simple
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::ShellOne, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'actors',
+                                     cmd: './doubler.rb',
+                                     timeout: 0.5)
+    t.receive(nil, ::OFlow::Box.new([1,2,3]))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+
+    assert_equal([2,4,6], t.history[0].box.contents, 'should  have correct contents in shipment')
+  end
+
+  def test_shellone_bad
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::ShellOne, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'actors',
+                                     cmd: './bad.rb',
+                                     timeout: 0.5)
+    t.receive(nil, ::OFlow::Box.new([1,2,3]))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+
+    assert_equal("Array", t.history[0].box.contents.class.name, 'should have an Array in shipment')
+    assert_equal("Exception", t.history[0].box.contents[0].class.name, 'should have an error in shipment')
+  end
+
+end
+

data/test/actors/shellrepeat_test.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(File.dirname(__FILE__)) unless $:.include? File.dirname(File.dirname(__FILE__))
+
+require 'helper'
+require 'oflow'
+require 'oflow/test'
+
+class ShellRepeatTest < ::MiniTest::Test
+
+  def test_shellrepeat_config
+    root_dir = File.expand_path(File.dirname(File.dirname(__FILE__)))
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::ShellRepeat, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'somewhere',
+                                     cmd: 'tripler.rb',
+                                     timeout: 0.5)
+    assert_equal(File.join(root_dir, 'somewhere'), t.actor.dir, 'dir set from options')
+    assert_equal('tripler.rb', t.actor.cmd, 'cmd set from options')
+    assert_equal(0.5, t.actor.timeout, 'timeout set from options')
+  end
+
+  def test_shellrepeat_simple
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::ShellRepeat, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'actors',
+                                     cmd: './tripler.rb',
+                                     timeout: 0.5)
+    t.receive(nil, ::OFlow::Box.new([1,2,3]))
+    t.join(1.0)
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+
+    assert_equal([3,6,9], t.history[0].box.contents, 'should  have correct contents in shipment')
+    t.receive(nil, ::OFlow::Box.new([3,2,1]))
+    t.join(1.0)
+    assert_equal(2, t.history.size, 'two entries should be in the history')
+    # optional
+    t.receive(:kill, ::OFlow::Box.new([1,2,3]))
+    t.join(1.0)
+  end
+
+end
+

data/test/actors/timer_test.rb

@@ -66,7 +66,7 @@ class TimerTest < ::MiniTest::Test
     assert_equal(Time, t.actor.start.class, 'is the start time a Time?')
     assert(0.1 > (Time.now() + 2 - t.actor.start), 'is the start time now + 2?')
 
-    assert_raises(::OFlow::ConfigError) do
+    assert_raises(ArgumentError) do
       ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::Timer, start: 'now')
     end
   end
@@ -83,7 +83,7 @@ class TimerTest < ::MiniTest::Test
     assert_equal(Time, t.actor.stop.class, 'is the stop time a Time?')
     assert(0.1 > (Time.now() + 2 - t.actor.stop), 'is the stop time now + 2?')
 
-    assert_raises(::OFlow::ConfigError) do
+    assert_raises(ArgumentError) do
       ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::Timer, stop: 'now')
     end
   end

data/test/actors/tripler.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(File.dirname(__FILE__)) unless $:.include? File.dirname(File.dirname(__FILE__))
+
+[ File.dirname(__FILE__),
+  File.join(File.dirname(__FILE__), "../../lib"),
+  File.join(File.dirname(__FILE__), "../../../oj/ext"),
+  File.join(File.dirname(__FILE__), "../../../oj/lib"),
+].each { |path| $: << path unless $:.include?(path) }
+
+require 'oj'
+
+Oj.load(STDIN, mode: :compat) do |input|
+  ctx = input["ctx"]
+  a = input["in"]
+  a.map! { |v| v * 3 }
+  out = { "ctx" => ctx, "out" => a }
+  puts Oj.dump(out, mode: :compat, indent: 0)
+  STDOUT.flush
+end

data/test/all_tests.rb

@@ -24,3 +24,4 @@ require 'actors/merger_test'
 require 'actors/persister_test'
 require 'actors/timer_test'
 require 'actors/httpserver_test'
+require 'actors/shellone_test'

data/test/d2.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(File.dirname(__FILE__)) unless $:.include? File.dirname(File.dirname(__FILE__))
+
+[ File.dirname(__FILE__),
+  File.join(File.dirname(__FILE__), "../../lib"),
+  File.join(File.dirname(__FILE__), "../../../oj/ext"),
+  File.join(File.dirname(__FILE__), "../../../oj/lib"),
+].each { |path| $: << path unless $:.include?(path) }
+
+#input = gets()
+
+#puts "*** input: #{input}"
+puts "*** #{`pwd`}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oflow
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-22 00:00:00.000000000 Z
+date: 2015-02-08 00:00:00.000000000 Z
 dependencies: []
 description: Operations Workflow in Ruby. This implements a workflow/process flow
   using multiple task nodes that each have their own queues and execution thread.
@@ -31,6 +31,8 @@ files:
 - lib/oflow/actors/merger.rb
 - lib/oflow/actors/persister.rb
 - lib/oflow/actors/relay.rb
+- lib/oflow/actors/shellone.rb
+- lib/oflow/actors/shellrepeat.rb
 - lib/oflow/actors/timer.rb
 - lib/oflow/actors/trigger.rb
 - lib/oflow/box.rb
@@ -50,16 +52,22 @@ files:
 - lib/oflow/test/actorwrap.rb
 - lib/oflow/tracker.rb
 - lib/oflow/version.rb
+- test/actors/bad.rb
 - test/actors/balancer_test.rb
+- test/actors/doubler.rb
 - test/actors/httpserver_test.rb
 - test/actors/log_test.rb
 - test/actors/merger_test.rb
 - test/actors/persister_test.rb
+- test/actors/shellone_test.rb
+- test/actors/shellrepeat_test.rb
 - test/actors/timer_test.rb
+- test/actors/tripler.rb
 - test/actorwrap_test.rb
 - test/all_tests.rb
 - test/box_test.rb
 - test/collector.rb
+- test/d2.rb
 - test/flow_basic_test.rb
 - test/flow_cfg_error_test.rb
 - test/flow_linked_test.rb
@@ -92,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: oflow
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Operations Workflow in Ruby