rflow 1.0.1 → 1.1.0

data/spec/rflow/forward_to_output_port_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 class RFlow
   describe ForwardToOutputPort do
     before(:each) do
-      ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+      ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
       Configuration.migrate_database
     end
 
@@ -12,18 +12,19 @@ class RFlow
     let(:generator) do
       RFlow::Components::GenerateIntegerSequence.new.tap do |c|
         c.configure!({})
-        c.out.direct_connect ruby_proc_filter.filtered
       end
     end
 
-    let(:ruby_proc_filter) do
+    let(:accept_evens) do
       RFlow::Components::RubyProcFilter.new.tap do |c|
         c.configure! 'filter_proc_string' => 'message % 2 == 0'
       end
     end
 
     it 'should place the messages on the output port, regardless of the filter' do
-      ruby_proc_filter.filtered.collect_messages(nil, messages) do
+      generator.out.direct_connect accept_evens.filtered
+
+      accept_evens.filtered.collect_messages(nil, messages) do
         5.times { generator.generate }
       end
       expect(messages.map(&:data).map(&:data_object)).to eq([0, 1, 2, 3, 4])

data/spec/rflow/logger_spec.rb

@@ -17,16 +17,16 @@ class RFlow
 
     before(:each) { initialize_logger }
 
-    it "should initialize correctly" do
+    it 'should initialize correctly' do
       expect(File.exist?(log_file_path)).to be true
 
-      logger.error "TESTTESTTEST"
+      logger.error 'TESTTESTTEST'
       expect(File.read(log_file_path)).to match(/TESTTESTTEST/)
 
       logger.close
     end
 
-    it "should reopen correctly" do
+    it 'should reopen correctly' do
       moved_path = log_file_path + '.old'
 
       expect(File.exist?(log_file_path)).to be true
@@ -36,13 +36,13 @@ class RFlow
 
       logger.reopen
 
-      logger.error "TESTTESTTEST"
+      logger.error 'TESTTESTTEST'
       expect(File.read(log_file_path)).to match(/TESTTESTTEST/)
       expect(File.read(moved_path)).not_to match(/TESTTESTTEST/)
 
       logger.close
     end
 
-    it "should toggle log level"
+    it 'should toggle log level'
   end
 end

data/spec/rflow/message/data/raw_spec.rb

@@ -7,11 +7,11 @@ class RFlow
       describe 'Raw Avro Schema' do
         let(:schema) { Configuration.available_data_types['RFlow::Message::Data::Raw']['avro'] }
 
-        it "should load the schema" do
+        it 'should load the schema' do
           expect(schema).not_to be_nil
         end
 
-        it "should encode and decode an object" do
+        it 'should encode and decode an object' do
           raw = {'raw' => Array.new(256) { rand(256) }.pack('c*')}
 
           expect { encode_avro(schema, raw) }.to_not raise_error

data/spec/rflow/message/data_spec.rb

@@ -9,13 +9,13 @@ class RFlow
       let(:valid_schema) { '{"type": "string"}' }
       let(:serialized_string) { encode_avro(valid_schema, string) }
 
-      context "if created without a schema" do
-        it "should throw an exception" do
+      context 'if created without a schema' do
+        it 'should throw an exception' do
           expect { Data.new(nil) }.to raise_error(ArgumentError, /^Invalid schema/)
         end
       end
 
-      context "if created with an invalid schema for the serialization" do
+      context 'if created with an invalid schema for the serialization' do
         ['avro', :avro].each do |it|
           it "should throw an exception for serialization type #{it.inspect}" do
             expect { Data.new(invalid_schema, it) }.to raise_error(ArgumentError, /^Invalid schema/)
@@ -23,14 +23,14 @@ class RFlow
         end
       end
 
-      context "if created with a valid avro schema" do
+      context 'if created with a valid avro schema' do
         ['avro', :avro].each do |it|
           it "should instantiate correctly for serialization type #{it.inspect}" do
             expect { Data.new(valid_schema, it) }.to_not raise_error
           end
         end
 
-        context "if created with a non-avro data serialization" do
+        context 'if created with a non-avro data serialization' do
           ['unknown', :unknown, 'xml', :xml].each do |it|
             it "should throw an exception for serialization type #{it.inspect}" do
               expect { Data.new(valid_schema, it) }.to raise_error(
@@ -39,15 +39,15 @@ class RFlow
           end
         end
 
-        context "if created with an avro serialization" do
+        context 'if created with an avro serialization' do
           ['avro', :avro].each do |it|
             it "should instantiate correctly for serialization type #{it.inspect}" do
               expect { Data.new(valid_schema, it) }.to_not raise_error
             end
           end
 
-          context "if created with a serialized data object" do
-            it "should instantiate correctly" do
+          context 'if created with a serialized data object' do
+            it 'should instantiate correctly' do
               expect { Data.new(valid_schema, 'avro', serialized_string )}.to_not raise_error
             end
           end

data/spec/rflow/message_spec.rb

@@ -4,70 +4,70 @@ require 'rflow/message'
 
 class RFlow
   describe Message do
-    context "if created with an unknown data type" do
-      it "should throw an exception" do
+    context 'if created with an unknown data type' do
+      it 'should throw an exception' do
         expect { Message.new('non_existent_data_type') }.to raise_error(
           ArgumentError, "Data type 'non_existent_data_type' with serialization_type 'avro' not found")
       end
     end
 
-    context "if created with a known data type" do
+    context 'if created with a known data type' do
       before(:all) do
         @schema = '{"type": "string"}'
         Configuration.add_available_data_type(:string_type, 'avro', @schema)
       end
 
-      it "should instantiate correctly" do
+      it 'should instantiate correctly' do
         expect { Message.new('string_type') }.to_not raise_error
       end
 
-      context "if created with empty provenance" do
-        context "if created with an unknown data serialization" do
+      context 'if created with empty provenance' do
+        context 'if created with an unknown data serialization' do
           ['unknown', :unknown].each do |it|
             it "should throw an exception for #{it.inspect}" do
-              expect { Message.new('string_type', [], it) }.to raise_error(
+              expect { Message.new('string_type', [], {}, it) }.to raise_error(
                 ArgumentError, "Data type 'string_type' with serialization_type 'unknown' not found")
             end
           end
         end
 
-        context "if created with a known data serialization" do
+        context 'if created with a known data serialization' do
           ['avro', :avro].each do |it|
             it "should instantiate correctly for #{it.inspect}" do
-              expect { Message.new('string_type', [], it) }.to_not raise_error
+              expect { Message.new('string_type', [], {}, it) }.to_not raise_error
             end
           end
 
-          context "if created with a mismatched schema" do
+          context 'if created with a mismatched schema' do
             it
           end
-          context "if created with a matched schema" do
+          context 'if created with a matched schema' do
             it
           end
 
-          context "if created with a nil schema" do
-            context "if created with a serialized data object" do
+          context 'if created with a nil schema' do
+            context 'if created with a serialized data object' do
               let(:serialized_string) { encode_avro(@schema, 'this is a string to be serialized') }
 
-              it "should instantiate correctly" do
-                expect { Message.new('string_type', [], 'avro', nil, serialized_string) }.to_not raise_error
+              it 'should instantiate correctly' do
+                expect { Message.new('string_type', [], {}, 'avro', nil, serialized_string) }.to_not raise_error
               end
             end
           end
         end
       end
 
-      context "if created with invalid provenance" do
+      context 'if created with invalid provenance' do
         let(:invalid_processing_event_hash) { {'started_at' => 'bad time string'} }
         let(:invalid_provenance) { [invalid_processing_event_hash] }
 
-        it "should throw an exception" do
+        it 'should throw an exception' do
           expect { Message.new('string_type', invalid_provenance) }.to raise_error(
             ArgumentError, 'invalid date: "bad time string"')
         end
       end
 
-      context "if created with valid provenance" do
+      context 'if created with valid provenance' do
         let(:valid_xmlschema_time) { '2001-01-01T01:01:01.000001Z' }
         let(:valid_processing_event_hash) { {'component_instance_uuid' => 'uuid', 'started_at' => valid_xmlschema_time } }
         let(:valid_processing_event) { Message::ProcessingEvent.new('uuid', valid_xmlschema_time, valid_xmlschema_time, 'context') }
@@ -77,11 +77,11 @@ class RFlow
            valid_processing_event]
         end
 
-        it "should instantiate correctly" do
+        it 'should instantiate correctly' do
           expect { Message.new('string_type', valid_provenance) }.to_not raise_error
         end
 
-        it "should correctly set the provenance processing events" do
+        it 'should correctly set the provenance processing events' do
           Message.new('string_type', valid_provenance).provenance[1].tap do |p|
             expect(p.component_instance_uuid).to eq('uuid')
             expect(p.started_at).to eq(Time.xmlschema(valid_xmlschema_time))
@@ -90,16 +90,50 @@ class RFlow
           end
         end
 
-        it "should to_hash its provenance correctly" do
+        it 'should to_hash its provenance correctly' do
           expect(Message.new('string_type', valid_provenance).provenance.map(&:to_hash)).to eq([
-            {"component_instance_uuid" => "uuid", "started_at" => nil, "completed_at" => nil, "context" => nil},
-            {"component_instance_uuid" => "uuid", "started_at" => valid_xmlschema_time, "completed_at" => nil, "context" => nil},
-            {"component_instance_uuid" => "uuid", "started_at" => valid_xmlschema_time, "completed_at" => valid_xmlschema_time, "context" => "context"}])
+            {'component_instance_uuid' => 'uuid', 'started_at' => nil, 'completed_at' => nil, 'context' => nil},
+            {'component_instance_uuid' => 'uuid', 'started_at' => valid_xmlschema_time, 'completed_at' => nil, 'context' => nil},
+            {'component_instance_uuid' => 'uuid', 'started_at' => valid_xmlschema_time, 'completed_at' => valid_xmlschema_time, 'context' => 'context'}])
         end
+
+        it "should not serialize data that doesn't match the schema" do
+          message = Message.new('string_type')
+          expect { message.to_avro }.to raise_error(::Avro::IO::AvroTypeError)
+          message.data.data_object = 'data'
+          expect { message.to_avro }.to_not raise_error
+        end
+      end
+
+      context 'if given properties' do
+        it 'should initialize and serialize with valid properties' do
+          properties = { 'boom' => 'boom', 'down' => 'town' }
+          message = Message.new('string_type', nil, properties)
+          expect(message.properties).to eq(properties)
+
+          message.data.data_object = 'data'
+
+          Message.from_avro(message.to_avro).tap do |processed|
+            expect(processed.properties).to eq(properties)
+            expect(processed.properties['boom']).to eq(message.properties['boom'])
+          end
+        end
+
+        it 'should stringify non-string properties' do
+          properties = { :boom => :town }
+          message = Message.new('string_type', nil, properties)
+
+          message.data.data_object = 'data'
+
+          Message.from_avro(message.to_avro).tap do |processed|
+            expect(processed.properties['boom']).to eq(message.properties[:boom].to_s)
+          end
+        end
+
       end
 
-      context "if correctly created" do
-        it "should serialize and deserialize correctly to/from avro" do
+      context 'if correctly created' do
+        it 'should serialize and deserialize correctly to/from avro' do
           message = Message.new('string_type').tap do |m|
             m.provenance << Message::ProcessingEvent.new('UUID')
             m.data.data_object = 'teh awesome'
@@ -112,8 +146,8 @@ class RFlow
         end
       end
 
-      context "if data extensions exist" do
-        it "should extend the data element with the extension" do
+      context 'if data extensions exist' do
+        it 'should extend the data element with the extension' do
           module ExtensionModule; def ext_method; end; end
 
           message = Message.new('string_type')
@@ -126,7 +160,7 @@ class RFlow
       end
     end
 
-    it "should correctly handle large raw types" do
+    it 'should correctly handle large raw types' do
       message = Message.new('RFlow::Message::Data::Raw').tap do |m|
         m.data.raw = Array.new(101) { rand(256) }.pack('c*')
       end

data/spec/rflow_spec.rb

@@ -37,7 +37,7 @@ describe RFlow do
 
     # Grab all the pids from the log, which seems to be the only
     # reliable way to get them
-    log_lines.map {|line| /\((\d+)\)/.match(line)[1].to_i }.uniq
+    log_lines.map {|line| /\(\s*(\d+)\s*\)/.match(line)[1].to_i }.uniq
   end
 
   def run_and_shutdown(app_name, expected_worker_count)
@@ -64,7 +64,7 @@ describe RFlow do
     yield # verify output
 
     # Terminate the master
-    expect(Process.kill("TERM", master_pid)).to eq(1)
+    expect(Process.kill('TERM', master_pid)).to eq(1)
 
     # Make sure everything is dead after a second
     sleep 2
@@ -72,7 +72,7 @@ describe RFlow do
       expect { Process.kill(0, pid) }.to raise_error(Errno::ESRCH)
     end
   rescue Exception => e
-    Process.kill("TERM", master_pid) if master_pid
+    Process.kill('TERM', master_pid) if master_pid
     raise
   end
 
@@ -94,13 +94,13 @@ describe RFlow do
 
   after(:each) { Dir.chdir @original_directory_path }
 
-  context "when executing from the test script" do
+  context 'when executing from the test script' do
     before(:all) { load @extensions_file_name }
 
     describe '.run!' do
       def run_rflow_with_dsl(&block)
         rflow_thread = Thread.new do
-          ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+          ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
           RFlow::Configuration.migrate_database
           RFlow::Configuration::RubyDSL.configure {|c| block.call(c) }
           RFlow::Configuration.merge_defaults!
@@ -117,7 +117,7 @@ describe RFlow do
         rflow_thread.join
       end
 
-      it "should run a non-sharded workflow" do
+      it 'should run a non-sharded workflow' do
         run_rflow_with_dsl do |c|
           c.setting 'rflow.log_level', 'ERROR'
           c.setting 'rflow.application_directory_path', @temp_directory_path
@@ -158,7 +158,7 @@ describe RFlow do
         end
       end
 
-      it "should run a sharded workflow" do
+      it 'should run a sharded workflow' do
         run_rflow_with_dsl do |c|
           c.setting 'rflow.log_level', 'ERROR'
           c.setting 'rflow.application_directory_path', @temp_directory_path
@@ -207,7 +207,7 @@ describe RFlow do
         end
       end
 
-      it "should deliver broadcast messages to every copy of a shard" do
+      it 'should deliver broadcast messages to every copy of a shard' do
         run_rflow_with_dsl do |c|
           c.setting 'rflow.log_level', 'FATAL'
           c.setting 'rflow.application_directory_path', @temp_directory_path
@@ -249,8 +249,8 @@ describe RFlow do
     end
   end
 
-  context "when executing via the rflow binary" do
-    context "with a simple ruby DSL config file" do
+  context 'when executing via the rflow binary' do
+    context 'with a simple ruby DSL config file' do
       before(:each) do
         write_config_file <<-EOF
           RFlow::Configuration::RubyDSL.configure do |c|
@@ -259,14 +259,14 @@ describe RFlow do
         EOF
       end
 
-      it "should load a ruby dsl file into a sqlite DB" do
+      it 'should load a ruby dsl file into a sqlite DB' do
         load_database
 
-        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: db_file_name
+        ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: db_file_name
         expect(RFlow::Configuration::Setting.where(:name => 'mysetting').first.value).to eq('myvalue')
       end
 
-      it "should not load a database if the database file already exists" do
+      it 'should not load a database if the database file already exists' do
         File.open(db_file_name, 'w') {|file| file.write 'boom' }
 
         r = execute_rflow("load -d #{db_file_name} -c #{config_file_name}")
@@ -278,7 +278,7 @@ describe RFlow do
       end
     end
 
-    context "with a component that runs subshells" do
+    context 'with a component that runs subshells' do
       let(:app_name) { 'sharded_subshell_test' }
 
       before(:each) do
@@ -300,15 +300,15 @@ describe RFlow do
         load_database
       end
 
-      it "should run successfully daemonize and run in the background" do
+      it 'should run successfully daemonize and run in the background' do
         run_and_shutdown app_name, 1 do # 1 default worker
           expect(File.exist?(File.join(@temp_directory_path, 'out1'))).to be true
-          File.readlines('out1').each {|line| expect(line).to match /\w+ \w+ \d+ \d+:\d+:\d+ \w+ \d+/ }
+          File.readlines('out1').each {|line| expect(line).to match /\w+ \w+\s+\d+ \d+:\d+:\d+ \w+ \d+/ }
         end
       end
     end
 
-    context "with a complex, sharded ruby DSL config file" do
+    context 'with a complex, sharded ruby DSL config file' do
       let(:app_name) { 'sharded_bin_test' }
 
       before(:each) do
@@ -352,7 +352,7 @@ describe RFlow do
         expect(r[:stdout]).to match /error/i
       end
 
-      it "should daemonize and run in the background" do
+      it 'should daemonize and run in the background' do
         output_files = {
           'out1'    => [0, 3, 6, 9] * 3,
           'out2'    => (20..30).to_a * 2,

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
+- John Stoneham
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2016-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools
@@ -44,14 +45,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -86,26 +87,26 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.5
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.5
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: em-zeromq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.0
 - !ruby/object:Gem::Dependency
@@ -114,14 +115,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -181,7 +182,8 @@ dependencies:
 description: A Ruby flow-based programming framework that utilizes ZeroMQ for component
   connections and Avro for serialization
 email:
-- michael.artz@redjack.com
+- john.stoneham@redjack.com
+- mlartz@gmail.com
 executables:
 - rflow
 extensions: []
@@ -196,7 +198,6 @@ files:
 - Gemfile
 - Guardfile
 - LICENSE
-- NOTES
 - README.VAGRANT
 - README.md
 - Rakefile
@@ -277,7 +278,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.3.0
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A Ruby flow-based programming framework