oflow 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5ebeca0ca2d46e8134511f55eeb680352c49a0a
-  data.tar.gz: 7aacf1ca2bde1fd1413695f607a6d087bfbc8935
+  metadata.gz: c60c057bc56ccdf1457715ab9e7c73af54614bb2
+  data.tar.gz: b20b4b16b01f875c7848bb9f442ca3e67213cda7
 SHA512:
-  metadata.gz: 4d8a98f7a6362f81f1fe83d56017b5cf09ced84f59be798fa60edbd647031a83772e0045ac2559d720d79b2367559e18471e24aadc80a789305932cb529c7a7b
-  data.tar.gz: ab9864fcea8cb033cbe1c7926ef8edee98934a741d2eae89a7db1f8a7b9e3ac85de436a50931e7110652c38ec0158530f047b0c6e0591ffe696e76465a026e29
+  metadata.gz: a22ef32835f66cb4cccc81330a76d82c25a03ce3c646a878ea9ebe4b89fc4b2011747e540a53fe1e87c7e530e34407af8309139e5a87bbec1cd124edc3157930
+  data.tar.gz: d9e903de2ecafa7b7cb386182bcc194905f7d571801dccde61ebbab82e4236f4b28626854d61d8f2556164d8e710536d120e5a1ec103a0edf10d9752af2d52bb

data/README.md

@@ -25,6 +25,10 @@ Follow [@peterohler on Twitter](http://twitter.com/#!/peterohler) for announceme
 
 ## Release Notes
 
+### Release 0.5
+
+ - Added Persister Actor that acts as a simple local store database.
+
 ### Release 0.4
 
  - Added support for dynamic Timer options updates.

data/lib/oflow/actors.rb

@@ -11,4 +11,5 @@ require 'oflow/actors/errorhandler'
 
 require 'oflow/actors/balancer'
 require 'oflow/actors/merger'
+require 'oflow/actors/persister'
 require 'oflow/actors/timer'

data/lib/oflow/actors/persister.rb

@@ -0,0 +1,267 @@
+require 'oj'
+
+module OFlow
+  module Actors
+
+    # Actor that persists records to the local file system as JSON
+    # representations of the records. Records can be the whole contents of the
+    # box received or a sub element of the contents. The key to the records are
+    # keys provided either in the record data or outside the data but somewhere
+    # else in the box received. Options for maintaining historic records and
+    # sequence number locking are included. If no sequence number is provide the
+    # Persister will assume there is no checking required and write anyway.
+    #
+    # Records are stored as JSON with the filename as the key and sequence
+    # number. The format of the file name is <key>~<seq>.json. As an example, a
+    # record stored with a key of 'first' and a sequence number of 3 (third time
+    # saved) would be 'first~3.json.
+    class Persister < Actor
+
+      attr_reader :dir
+      attr_reader :key_path
+      attr_reader :seq_path
+      attr_reader :data_path
+      attr_reader :historic
+
+      # Initializes the persister with options of:
+      # @param [Hash] options with keys of
+      #  - :dir [String] directory to store the persisted records
+      #  - :key_data [String] path to record data (default: nil (all))
+      #  - :key_path [String] path to key for the record (default: 'key')
+      #  - :seq_path [String] path to sequence for the record (default: 'seq')
+      #  - :cache [Boolean] if true, cache records in memory
+      #  - :historic [Boolean] if true, do not delete previous versions
+      def initialize(task, options)
+        super
+        @dir = options[:dir]
+        if @dir.nil?
+          @dir = File.join('db', task.full_name.gsub(':', '/'))
+        end
+        @key_path = options.fetch(:key_path, 'key')
+        @seq_path = options.fetch(:seq_path, 'seq')
+        @data_path = options.fetch(:data_path, nil) # nil means all contents
+        if options.fetch(:cache, true)
+          # key is record key, value is [seq, rec]
+          @cache = {}
+        else
+          @cache = nil
+        end
+        @historic = options.fetch(:historic, false)
+
+        if Dir.exist?(@dir)
+          unless @cache.nil?
+            Dir.glob(File.join('**', '*.json')).each do |path|
+              path = File.join(@dir, path)
+              if File.symlink?(path)
+                rec = load(path)
+                unless @cache.nil?
+                  key, seq = key_seq_from_path(path)
+                  @cache[key] = [seq, rec]
+                end
+              end
+            end
+          end
+        else
+          `mkdir -p #{@dir}`
+        end
+      end
+
+      def perform(op, box)
+        dest = box.contents[:dest]
+        result = nil
+        case op
+        when :insert, :create
+          result = insert(box)
+        when :get, :read
+          result = read(box)
+        when :update
+          result = update(box)
+        when :delete, :remove
+          result = delete(box)
+        when :query
+          result = query(box)
+        when :clear
+          result = clear(box)
+        else
+          raise OpError.new(task.full_name, op)
+        end
+        task.ship(dest, Box.new(result, box.tracker))
+      end
+
+      def insert(box)
+        key = box.get(@key_path)
+        raise KeyError.new(:insert) if key.nil?
+        box = box.set(@seq_path, 1)
+        rec = box.get(@data_path)
+        @cache[key] = [1, rec] unless @cache.nil?
+        save(rec, key, 1)
+      end
+
+      # Returns true if the actor is caching records.
+      def caching?()
+        !@cache.nil?
+      end
+
+      def read(box)
+        # Should be a Hash.
+        key = box.contents[:key]
+        raise KeyError(:read) if key.nil?
+        if @cache.nil?
+          linkpath = File.join(@dir, "#{key}.json")
+          rec = load(linkpath)
+        else
+          unless (seq_rec = @cache[key]).nil?
+            rec = seq_rec[1]
+          end
+        end
+        # If not found rec will be nil, that is okay.
+        rec
+      end
+
+      def update(box)
+        key = box.get(@key_path)
+        raise KeyError.new(:update) if key.nil?
+        seq = box.get(@seq_path)
+        if @cache.nil?
+          if (seq_rec = @cache[key]).nil?
+            raise NotFoundError.new(key)
+          end
+          seq = seq_rec[0] if seq.nil?
+        else
+          seq = 0
+          has_rec = false
+          Dir.glob(File.join(@dir, '**', "#{key}*.json")).each do |path|
+            if File.symlink?(path)
+              has_rec = true
+              next
+            end
+            _, s = key_seq_from_path(path)
+            seq = s if seq < s
+          end
+        end
+        raise NotFoundError.new(key) unless has_rec
+        raise SeqError.new(:update, key) if seq.nil? || 0 == seq
+
+        seq += 1
+        box = box.set(@seq_path, seq)
+        rec = box.get(@data_path)
+        @cache[key] = [seq, rec] unless @cache.nil?
+        rec = save(rec, key, seq)
+        delete_historic(key, seq) unless @historic
+        rec
+      end
+
+      def delete(box)
+        key = box.get(@key_path)
+        @cache.delete(key) unless @cache.nil?
+        linkpath = File.join(@dir, "#{key}.json")
+        File.delete(linkpath)
+        delete_historic(key, nil) unless @historic
+        nil
+      end
+
+      def query(box)
+        recs = {}
+        expr = box.get('expr')
+        if expr.nil?
+          if @cache.nil?
+            Dir.glob(File.join(@dir, '**/*.json')).each do |path|
+              recs[File.basename(path)[0..-6]] = load(path) if File.symlink?(path)
+            end
+          else
+            @cache.each do |key,seq_rec|
+              recs[key] = seq_rec[1]
+            end
+          end
+        elsif expr.is_a?(Proc)
+          if @cache.nil?
+            Dir.glob(File.join(@dir, '**/*.json')).each do |path|
+              next unless File.symlink?(path)
+              rec = load(path)
+              key, seq = key_seq_from_path(path)
+              recs[key] = rec if expr.call(rec, key, seq)
+            end
+          else
+            @cache.each do |key,seq_rec|
+              rec = seq_rec[1]
+              recs[key] = rec if expr.call(rec, key, seq_rec[0])
+            end
+          end
+        else
+          # TBD add support for string safe expressions in the future
+          raise Exception.new("expr can only be a Proc, not a #{expr.class}")
+        end
+        recs
+      end
+
+      def clear(box)
+        @cache = {} unless @cache.nil?
+        `rm -rf #{@dir}`
+        # remake the dir in preparation for future inserts
+        `mkdir -p #{@dir}`
+        nil
+      end
+
+      # internal use only
+      def save(rec, key, seq)
+        filename = "#{key}~#{seq}.json"
+        path = File.join(@dir, filename)
+        linkpath = File.join(@dir, "#{key}.json")
+        raise ExistsError.new(key, seq) if File.exist?(path)
+        Oj.to_file(path, rec, :mode => :object)
+        begin
+          File.delete(linkpath)
+        rescue Exception
+          # ignore
+        end
+        File.symlink(filename, linkpath)
+        rec
+      end
+
+      def load(path)
+        return nil unless File.exist?(path)
+        Oj.load_file(path, :mode => :object)
+      end
+
+      def delete_historic(key, seq)
+        Dir.glob(File.join(@dir, '**', "#{key}~*.json")).each do |path|
+          _, s = key_seq_from_path(path)
+          next if s == seq
+          File.delete(path)
+        end
+      end
+
+      def key_seq_from_path(path)
+        path = File.readlink(path) if File.symlink?(path)
+        base = File.basename(path)[0..-6] # strip off '.json'
+        a = base.split('~')
+        [a[0..-2].join('~'), a[-1].to_i]
+      end
+
+      class KeyError < Exception
+        def initialize(op)
+          super("No key found for #{op}")
+        end
+      end # KeyError
+
+      class SeqError < Exception
+        def initialize(op, key)
+          super("No sequence number found for #{op} of #{key}")
+        end
+      end # SeqError
+
+      class ExistsError < Exception
+        def initialize(key, seq)
+          super("#{key}:#{seq} already exists")
+        end
+      end # ExistsError
+
+      class NotFoundError < Exception
+        def initialize(key)
+          super("#{key} not found")
+        end
+      end # NotFoundError
+
+    end # Persister
+  end # Actors
+end # OFlow

data/lib/oflow/errors.rb

@@ -29,6 +29,14 @@ module OFlow
     end
   end # ConfigError
 
+  # An Exception indicating an invalid operation used in a call to receive() or
+  # perform().
+  class OpError < Exception
+    def initialize(name, op)
+      super("'#{op}' is not a valid operation for #{name}.")
+    end
+  end # OpError
+
   # An Exception raised when no destination is found.
   class LinkError < Exception
     def initialize(dest)

data/lib/oflow/hastasks.rb

@@ -21,7 +21,10 @@ module OFlow
       yield(f) if block_given?
       f.resolve_all_links()
       # Wait to validate until at the top so up-links don't fail validation.
-      f.validate() if Env == self
+      if Env == self
+        f.validate() 
+        f.start()
+      end
       f
     end
 
@@ -33,6 +36,8 @@ module OFlow
     # @param block [Proc] block to yield to with the new Task instance
     # @return [Task] new Task
     def task(name, actor_class, options={}, &block)
+      has_state = options.has_key?(:state)
+      options[:state] = Task::STOPPED unless has_state
       t = Task.new(self, name, actor_class, options)
       @tasks[t.name] = t
       yield(t) if block_given?

data/lib/oflow/task.rb

@@ -55,7 +55,7 @@ module OFlow
       @actor = actor_class.new(self, options)
       raise Exception.new("#{actor} does not respond to the perform() method.") unless @actor.respond_to?(:perform)
 
-      @state = RUNNING
+      @state = options.fetch(:state, RUNNING)
       return unless @actor.with_own_thread()
 
       @loop = Thread.start(self) do |me|

data/lib/oflow/test/actorwrap.rb

@@ -49,8 +49,13 @@ module OFlow
         if @starting
           @before << [op, box]
         else
-          @actor.perform(op, box)
+          begin
+            @actor.perform(op, box)
+          rescue Exception => e
+            ship(:error, Box.new([e, full_name()]))
+          end
         end
+        nil
       end
 
       # Task API that adds entry to history.

data/lib/oflow/version.rb

@@ -1,5 +1,5 @@
 
 module OFlow
   # Current version of the module. 
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/test/actors/persister_test.rb

@@ -0,0 +1,410 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+[ File.dirname(__FILE__),
+  File.join(File.dirname(__FILE__), "../../lib"),
+  File.join(File.dirname(__FILE__), "..")
+].each { |path| $: << path unless $:.include?(path) }
+
+require 'test/unit'
+require 'oflow'
+require 'oflow/test'
+
+class PersisterTest < ::Test::Unit::TestCase
+
+  def test_persister_config
+    t = ::OFlow::Test::ActorWrap.new('test', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     dir: 'db/something',
+                                     key_path: 'key',
+                                     cache: false,
+                                     data_path: 'data',
+                                     with_tracker: true,
+                                     with_seq_num: true,
+                                     historic: true,
+                                     seq_path: 'seq')
+    assert_equal('db/something', t.actor.dir, 'dir set from options')
+    assert_equal('key', t.actor.key_path, 'key_path set from options')
+    assert_equal('seq', t.actor.seq_path, 'seq_path set from options')
+    assert_equal('data', t.actor.data_path, 'data_path set from options')
+    assert_equal(false, t.actor.caching?, 'cache set from options')
+    assert_equal(true, t.actor.historic, 'historic set from options')
+    assert(Dir.exist?(t.actor.dir), 'dir exists')
+    `rm -r #{t.actor.dir}`
+
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED)
+    assert_equal('db/test/persist', t.actor.dir, 'dir set from options')
+    assert_equal('key', t.actor.key_path, 'key_path set from options')
+    assert_equal('seq', t.actor.seq_path, 'seq_path set from options')
+    assert_equal(true, t.actor.caching?, 'cache set from options')
+    assert_equal(nil, t.actor.data_path, 'data_path set from options')
+    assert_equal(false, t.actor.historic, 'historic set from options')
+    assert(Dir.exist?(t.actor.dir), 'dir exists')
+    `rm -r #{t.actor.dir}`
+  end
+
+  def test_persister_historic
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     historic: true)
+    # insert
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, key: 'one', data: 0}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:here, t.history[0].dest, 'should have shipped to :here')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # read
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:read, t.history[0].dest, 'should have shipped to :read')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # update
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 1, data: 1}))
+    # no seq so try to find max
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', data: 2}))
+    assert_equal(2, t.history.size, 'one entry for each update expected')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(3, files.size, 'should be 3 history historic files')
+    # make sure current object is last one saved
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal({:dest=>:here, :key=>'one', :data=>2, :seq=>3},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # delete
+    t.reset()
+    t.receive(:delete, ::OFlow::Box.new({dest: :deleted, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:deleted, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(3, files.size, 'should be 3 history historic files')
+    # make sure object was deleted
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # query
+    10.times do |i|
+      t.receive(:insert, ::OFlow::Box.new({dest: :here, key: "rec-#{i}", data: i}))
+    end
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: nil}))
+    assert_equal(10, t.history[0].box.contents.size, 'query with nil returns all')
+
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: Proc.new{|rec,key,seq| 4 < rec[:data] }}))
+    assert_equal(5, t.history[0].box.contents.size, 'query return check')
+
+    # clear
+    t.reset()
+    t.receive(:clear, ::OFlow::Box.new({dest: :cleared}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    files = Dir.glob(File.join(t.actor.dir, '**/*.json'))
+    assert_equal(0, files.size, 'should be no files')
+  end
+
+  def test_persister_historic_cached
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     historic: true,
+                                     cache: true)
+    assert(t.actor.caching?, 'verify caching is on')
+
+    # insert
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, key: 'one', data: 0}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:here, t.history[0].dest, 'should have shipped to :here')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'insert record contents')
+
+    # read
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:read, t.history[0].dest, 'should have shipped to :read')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # update
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 1, data: 1}))
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 2, data: 2}))
+    assert_equal(2, t.history.size, 'one entry for each update expected')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(3, files.size, 'should be 3 history historic files')
+    # make sure current object is last one saved
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal({:dest=>:here, :key=>'one', :data=>2, :seq=>3},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # delete
+    t.reset()
+    t.receive(:delete, ::OFlow::Box.new({dest: :deleted, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:deleted, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(3, files.size, 'should be 3 history historic files')
+    # make sure object was deleted
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # query
+    10.times do |i|
+      t.receive(:insert, ::OFlow::Box.new({dest: :here, key: "rec-#{i}", data: i}))
+    end
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: nil}))
+    assert_equal(10, t.history[0].box.contents.size, 'query with nil returns all')
+
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: Proc.new{|rec,key,seq| 4 < rec[:data] }}))
+    assert_equal(5, t.history[0].box.contents.size, 'query return check')
+
+    # clear
+    t.reset()
+    t.receive(:clear, ::OFlow::Box.new({dest: :cleared}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    files = Dir.glob(File.join(t.actor.dir, '**/*.json'))
+    assert_equal(0, files.size, 'should be no files')
+  end
+
+  def test_persister_not_historic
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     historic: false)
+    # insert
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, key: 'one', data: 0}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:here, t.history[0].dest, 'should have shipped to :here')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # read
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:read, t.history[0].dest, 'should have shipped to :read')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # update
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 1, data: 1}))
+    # no seq so try to find max
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', data: 2}))
+    assert_equal(2, t.history.size, 'one entry for each update expected')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(1, files.size, 'should be just one file')
+    # make sure current object is last one saved
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal({:dest=>:here, :key=>'one', :data=>2, :seq=>3},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # delete
+    t.reset()
+    t.receive(:delete, ::OFlow::Box.new({dest: :deleted, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:deleted, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(0, files.size, 'should no historic files')
+    # make sure object was deleted
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # query
+    10.times do |i|
+      t.receive(:insert, ::OFlow::Box.new({dest: :here, key: "rec-#{i}", data: i}))
+    end
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: nil}))
+    assert_equal(10, t.history[0].box.contents.size, 'query with nil returns all')
+
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: Proc.new{|rec,key,seq| 4 < rec[:data] }}))
+    assert_equal(5, t.history[0].box.contents.size, 'query return check')
+
+    # clear
+    t.reset()
+    t.receive(:clear, ::OFlow::Box.new({dest: :cleared}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    files = Dir.glob(File.join(t.actor.dir, '**/*.json'))
+    assert_equal(0, files.size, 'should be no files')
+  end
+
+  def test_persister_not_historic_cached
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     historic: false,
+                                     cache: true)
+    # insert
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, key: 'one', data: 0}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:here, t.history[0].dest, 'should have shipped to :here')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # read
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:read, t.history[0].dest, 'should have shipped to :read')
+    assert_equal({:dest=>:here, :key=>'one', :data=>0, :seq=>1},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # update
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 1, data: 1}))
+    # no seq so try to find max
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', data: 2}))
+    assert_equal(2, t.history.size, 'one entry for each update expected')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(1, files.size, 'should be just one file')
+    # make sure current object is last one saved
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal({:dest=>:here, :key=>'one', :data=>2, :seq=>3},
+                 t.history[0].box.contents, 'should correct contents in shipment')
+
+    # delete
+    t.reset()
+    t.receive(:delete, ::OFlow::Box.new({dest: :deleted, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:deleted, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(0, files.size, 'should no historic files')
+    # make sure object was deleted
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # query
+    10.times do |i|
+      t.receive(:insert, ::OFlow::Box.new({dest: :here, key: "rec-#{i}", data: i}))
+    end
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: nil}))
+    assert_equal(10, t.history[0].box.contents.size, 'query with nil returns all')
+
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: Proc.new{|rec,key,seq| 4 < rec[:data] }}))
+    assert_equal(5, t.history[0].box.contents.size, 'query return check')
+
+    # clear
+    t.reset()
+    t.receive(:clear, ::OFlow::Box.new({dest: :cleared}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    files = Dir.glob(File.join(t.actor.dir, '**/*.json'))
+    assert_equal(0, files.size, 'should be no files')
+  end
+
+  def test_persister_data_path
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED,
+                                     historic: false,
+                                     cache: true,
+                                     data_path: 'data')
+    # insert
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, key: 'one', data: 0}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:here, t.history[0].dest, 'should have shipped to :here')
+    assert_equal(0, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # read
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:read, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(0, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # update
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', seq: 1, data: 1}))
+    # no seq so try to find max
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'one', data: 2}))
+    assert_equal(2, t.history.size, 'one entry for each update expected')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(1, files.size, 'should be just one file')
+    # make sure current object is last one saved
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(2, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # delete
+    t.reset()
+    t.receive(:delete, ::OFlow::Box.new({dest: :deleted, key: 'one'}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(:deleted, t.history[0].dest, 'should have shipped to :read')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    # check for 3 files in the db
+    files = Dir.glob(File.join(t.actor.dir, '**/*~[0123456789].json'))
+    assert_equal(0, files.size, 'should no historic files')
+    # make sure object was deleted
+    t.reset()
+    t.receive(:read, ::OFlow::Box.new({dest: :read, key: 'one'}))
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+
+    # query
+    10.times do |i|
+      t.receive(:insert, ::OFlow::Box.new({dest: :here, key: "rec-#{i}", data: i}))
+    end
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: nil}))
+    assert_equal(10, t.history[0].box.contents.size, 'query with nil returns all')
+
+    t.reset()
+    t.receive(:query, ::OFlow::Box.new({dest: :query, expr: Proc.new{|rec,key,seq| 4 < rec }}))
+    assert_equal(5, t.history[0].box.contents.size, 'query return check')
+
+    # clear
+    t.reset()
+    t.receive(:clear, ::OFlow::Box.new({dest: :cleared}))
+    assert_equal(1, t.history.size, 'one entry should be in the history')
+    assert_equal(nil, t.history[0].box.contents, 'should correct contents in shipment')
+    files = Dir.glob(File.join(t.actor.dir, '**/*.json'))
+    assert_equal(0, files.size, 'should be no files')
+  end
+
+  def test_persister_errors
+    `rm -rf db/test/persist`
+    t = ::OFlow::Test::ActorWrap.new('persist', ::OFlow::Actors::Persister, state: ::OFlow::Task::BLOCKED)
+
+    # insert with no key
+    t.receive(:insert, ::OFlow::Box.new({dest: :here, nokey: 'one', data: 0}))
+    action = t.history[0]
+    assert_equal(:error, action.dest, 'insert with no key destination')
+    assert_equal(::OFlow::Actors::Persister::KeyError, action.box.contents[0].class, 'insert with key error')
+
+    # update non-existant record
+    t.reset()
+    t.receive(:update, ::OFlow::Box.new({dest: :here, key: 'not-me', data: 0}))
+    action = t.history[0]
+    assert_equal(:error, action.dest, 'error destination')
+    assert_equal(::OFlow::Actors::Persister::NotFoundError, action.box.contents[0].class, 'insert with not found error')
+  end
+
+end

data/test/all_tests.rb

@@ -22,8 +22,8 @@ require 'flow_nest_test'
 require 'flow_tracker_test'
 
 # Actor tests
-require 'actors/log_test'
-require 'actors/timer_test'
 require 'actors/balancer_test'
+require 'actors/log_test'
 require 'actors/merger_test'
-
+require 'actors/persister_test'
+require 'actors/timer_test'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oflow
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-08 00:00:00.000000000 Z
+date: 2014-02-16 00:00:00.000000000 Z
 dependencies: []
 description: Operations Workflow in Ruby. This implements a workflow/process flow
   using multiple task nodes that each have their own queues and execution thread.
@@ -28,6 +28,7 @@ files:
 - lib/oflow/actors/ignore.rb
 - lib/oflow/actors/log.rb
 - lib/oflow/actors/merger.rb
+- lib/oflow/actors/persister.rb
 - lib/oflow/actors/relay.rb
 - lib/oflow/actors/timer.rb
 - lib/oflow/box.rb
@@ -52,6 +53,7 @@ files:
 - test/actors/balancer_test.rb
 - test/actors/log_test.rb
 - test/actors/merger_test.rb
+- test/actors/persister_test.rb
 - test/actors/timer_test.rb
 - test/actorwrap_test.rb
 - test/all_tests.rb