rflow 1.0.1 → 1.1.0

data/schema/message.avsc

@@ -22,6 +22,12 @@
              }
          }
         },
+        {"name": "properties",
+         "type": {
+             "type": "map",
+             "values": "string"
+         }
+        },
         {"name": "data_serialization_type",
          "type": {
              "type": "enum",
@@ -33,4 +39,3 @@
         {"name": "data", "type": "bytes"}
     ]
 }
-

data/spec/rflow/component/port_spec.rb

@@ -2,39 +2,39 @@ require 'spec_helper'
 
 class RFlow
   class Component
-    context "Input and output ports" do
+    context 'Input and output ports' do
       let(:connection) { RFlow::Connection.new(RFlow::Configuration::NullConnectionConfiguration.new) }
       let(:message) { RFlow::Message.new('RFlow::Message::Data::Raw') }
 
       describe Port do
-        it "should not be connected" do
+        it 'should not be connected' do
           expect(described_class.new(nil)).not_to be_connected
         end
       end
 
       describe HashPort do
-        it "should not be connected" do
+        it 'should not be connected' do
           expect(described_class.new(nil)).not_to be_connected
         end
       end
 
       [InputPort, OutputPort].each do |c|
         describe c do
-          context "#add_connection" do
-            it "should add the connection" do
+          context '#add_connection' do
+            it 'should add the connection' do
               described_class.new(nil).tap do |port|
                 port.add_connection(nil, connection)
-                expect(port[nil]).to include connection
+                expect(port[nil].connections).to include connection
               end
             end
           end
 
-          context "#remove_connection" do
-            it "should remove the connection" do
+          context '#remove_connection' do
+            it 'should remove the connection' do
               described_class.new(nil).tap do |port|
                 port.add_connection(nil, connection)
                 port.remove_connection(nil, connection)
-                expect(port[nil]).not_to include connection
+                expect(port[nil].connections).not_to include connection
               end
             end
           end
@@ -42,8 +42,8 @@ class RFlow
       end
 
       describe InputPort do
-        context "#connect!" do
-          it "should be connected" do
+        context '#connect!' do
+          it 'should be connected' do
             expect(connection).to receive(:connect_input!)
 
             described_class.new(nil).tap do |port|
@@ -57,8 +57,8 @@ class RFlow
       end
 
       describe OutputPort do
-        context "#connect!" do
-          it "should be connected" do
+        context '#connect!' do
+          it 'should be connected' do
             expect(connection).to receive(:connect_output!)
 
             described_class.new(nil).tap do |port|
@@ -70,8 +70,8 @@ class RFlow
           end
         end
 
-        context "#add_connection" do
-          it "should deliver messages to the new connection" do
+        context '#add_connection' do
+          it 'should deliver messages to the new connection' do
             described_class.new(nil).tap do |port|
               port.connect!
               port.all_connections # trigger caching
@@ -85,8 +85,8 @@ class RFlow
           end
         end
 
-        context "#remove_connection" do
-          it "should not deliver messages to the old connection" do
+        context '#remove_connection' do
+          it 'should not deliver messages to the old connection' do
             described_class.new(nil).tap do |port|
               allow(connection).to receive(:connect_output!)
               port.add_connection(nil, connection)

data/spec/rflow/components/clock_spec.rb

@@ -4,7 +4,7 @@ class RFlow
   module Components
     describe Clock do
       before(:each) do
-        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+        ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
         Configuration.migrate_database
       end
 

data/spec/rflow/configuration/ruby_dsl_spec.rb

@@ -5,11 +5,11 @@ class RFlow
   class Configuration
     describe RubyDSL do
       before(:each) do
-        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+        ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
         Configuration.migrate_database
       end
 
-      it "should correctly process an empty DSL" do
+      it 'should correctly process an empty DSL' do
         described_class.configure {}
 
         expect(Shard).to have(0).shards
@@ -18,7 +18,7 @@ class RFlow
         expect(Connection).to have(0).connections
       end
 
-      it "should correctly process a component declaration" do
+      it 'should correctly process a component declaration' do
         described_class.configure do |c|
           c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
         end
@@ -35,7 +35,7 @@ class RFlow
         end
       end
 
-      it "should correctly process a connect declaration" do
+      it 'should correctly process a connect declaration' do
         described_class.configure do |c|
           c.component 'first', 'First'
           c.component 'second', 'Second'
@@ -109,32 +109,32 @@ class RFlow
         end
       end
 
-      it "should correctly process shard declarations" do
+      it 'should correctly process shard declarations' do
         described_class.configure do |c|
           c.component 'first', 'First', :opt1 => 'opt1'
 
-          c.shard "s1", :process => 2 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's1', :process => 2 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
-          c.shard "s2", :type => :process, :count => 10 do |s|
+          c.shard 's2', :type => :process, :count => 10 do |s|
             s.component 'third', 'Third'
             s.component 'fourth', 'Fourth'
           end
 
-          c.process "s3", :count => 10 do |s|
+          c.process 's3', :count => 10 do |s|
             s.component 'fifth', 'Fifth'
           end
 
-          c.shard "s-ignored", :type => :process, :count => 10 do
+          c.shard 's-ignored', :type => :process, :count => 10 do
             # ignored because there are no components
           end
 
-          c.thread "s4", :count => 10 do |s|
+          c.thread 's4', :count => 10 do |s|
             s.component 'sixth', 'Sixth'
           end
 
-          c.shard "s5", :type => :thread, :count => 10 do |s|
+          c.shard 's5', :type => :thread, :count => 10 do |s|
             s.component 'seventh', 'Seventh'
           end
 
@@ -173,12 +173,12 @@ class RFlow
            'third#out=>sixth#in'])
       end
 
-      it "should generate PUSH-PULL inproc ZeroMQ connections for in-shard connections" do
+      it 'should generate PUSH-PULL inproc ZeroMQ connections for in-shard connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 2 do |s|
+          c.shard 's1', :process => 2 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in'
@@ -205,15 +205,15 @@ class RFlow
         end
       end
 
-      it "should generate PUSH-PULL ipc ZeroMQ connections for one-to-one inter-shard connections" do
+      it 'should generate PUSH-PULL ipc ZeroMQ connections for one-to-one inter-shard connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 1 do |s|
+          c.shard 's1', :process => 1 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 1 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 1 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in'
@@ -240,15 +240,15 @@ class RFlow
         end
       end
 
-      it "should generate PUSH-PULL ipc ZeroMQ connections for one-to-many inter-shard connections" do
+      it 'should generate PUSH-PULL ipc ZeroMQ connections for one-to-many inter-shard connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 1 do |s|
+          c.shard 's1', :process => 1 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 3 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in'
@@ -275,15 +275,15 @@ class RFlow
         end
       end
 
-      it "should generate PUSH-PULL ipc ZeroMQ connections for many-to-one inter-shard connections" do
+      it 'should generate PUSH-PULL ipc ZeroMQ connections for many-to-one inter-shard connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 3 do |s|
+          c.shard 's1', :process => 3 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 1 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 1 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in'
@@ -310,15 +310,15 @@ class RFlow
         end
       end
 
-      it "should generate PUSH-PULL brokered ZeroMQ connections for many-to-many inter-shard connections" do
+      it 'should generate PUSH-PULL brokered ZeroMQ connections for many-to-many inter-shard connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 3 do |s|
+          c.shard 's1', :process => 3 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 3 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in'
@@ -345,15 +345,15 @@ class RFlow
         end
       end
 
-      it "should generate PUB-SUB ipc ZeroMQ connections for one-to-many broadcast connections" do
+      it 'should generate PUB-SUB ipc ZeroMQ connections for one-to-many broadcast connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 1 do |s|
+          c.shard 's1', :process => 1 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 3 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in', :delivery => 'broadcast'
@@ -380,15 +380,15 @@ class RFlow
         end
       end
 
-      it "should generate PUB-SUB brokered ZeroMQ connections for many-to-many broadcast connections" do
+      it 'should generate PUB-SUB brokered ZeroMQ connections for many-to-many broadcast connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 3 do |s|
+          c.shard 's1', :process => 3 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
           end
 
-          c.shard "s2", :process => 3 do |s|
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          c.shard 's2', :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in', :delivery => 'broadcast'
@@ -415,12 +415,12 @@ class RFlow
         end
       end
 
-      it "should generate PUB-SUB brokered ZeroMQ connections for many-to-many in-shard broadcast connections" do
+      it 'should generate PUB-SUB brokered ZeroMQ connections for many-to-many in-shard broadcast connections' do
         described_class.configure do |c|
 
-          c.shard "s1", :process => 3 do |s|
+          c.shard 's1', :process => 3 do |s|
             s.component 'first', 'First', :opt1 => 'opt1'
-            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+            s.component 'second', 'Second', :opt1 => 'opt1', 'opt2' => 'opt2'
           end
 
           c.connect 'first#out' => 'second#in', :delivery => 'broadcast'
@@ -447,7 +447,7 @@ class RFlow
         end
       end
 
-      it "should not allow two components with the same name" do
+      it 'should not allow two components with the same name' do
         expect {
           described_class.configure do |c|
             c.component 'first', 'First'
@@ -456,13 +456,13 @@ class RFlow
         }.to raise_error(ActiveRecord::RecordInvalid)
       end
 
-      it "should not allow two shards with the same name" do
+      it 'should not allow two shards with the same name' do
         expect {
           described_class.configure do |c|
-            c.shard("s1", :process => 2) {|c| c.component 'x', 'y' }
-            c.shard("s1", :process => 2) {|c| c.component 'z', 'q' }
+            c.shard('s1', :process => 2) {|c| c.component 'x', 'y' }
+            c.shard('s1', :process => 2) {|c| c.component 'z', 'q' }
           end
-        }.to raise_error
+        }.to raise_error(ActiveRecord::RecordInvalid)
       end
     end
   end

data/spec/rflow/configuration_spec.rb

@@ -5,12 +5,12 @@ class RFlow
   describe Configuration do
     describe '.add_available_data_type' do
       context 'if passed a data_serialization that is not avro' do
-        it "should throw an exception" do
+        it 'should throw an exception' do
           expect { Configuration.add_available_data_type('A', 'boom', 'schema') }.to raise_error(
             ArgumentError, "Data serialization_type must be 'avro' for 'A'")
         end
 
-        it "should not update the available_data_types" do
+        it 'should not update the available_data_types' do
           expect {
             Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
           }.not_to change { Configuration.available_data_types.size }
@@ -18,18 +18,18 @@ class RFlow
       end
     end
 
-    describe "Data Extensions" do
-      describe ".add_available_data_extension" do
+    describe 'Data Extensions' do
+      describe '.add_available_data_extension' do
         context 'if passed a non-module data extension' do
-          it "should throw an exception" do
+          it 'should throw an exception' do
             expect {
               Configuration.add_available_data_extension('data_type', 'NOTAMODULE')
-            }.to raise_error(ArgumentError, "Invalid data extension NOTAMODULE for data_type.  Only Ruby Modules allowed")
+            }.to raise_error(ArgumentError, 'Invalid data extension NOTAMODULE for data_type.  Only Ruby Modules allowed')
           end
         end
 
-        context "if passed a valid Module as a data extension" do
-          it "should update the available_data_extensions" do
+        context 'if passed a valid Module as a data extension' do
+          it 'should update the available_data_extensions' do
             expect {
               Configuration.add_available_data_extension('data_type', Module.new)
             }.to change { Configuration.available_data_extensions['data_type'].size }.by(1)

data/spec/rflow/forward_to_input_port_spec.rb

@@ -3,38 +3,67 @@ require 'spec_helper'
 class RFlow
   describe ForwardToInputPort do
     before(:each) do
-      ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+      ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ':memory:'
       Configuration.migrate_database
     end
 
-    let(:filtered_messages) { [] }
+    let(:accepted_messages) { [] }
     let(:dropped_messages) { [] }
 
     let(:generator) do
       RFlow::Components::GenerateIntegerSequence.new.tap do |c|
         c.configure!({})
-        c.out.direct_connect ruby_proc_filter.in
       end
     end
 
-    let(:ruby_proc_filter) do
+    let(:accept_evens) do
       RFlow::Components::RubyProcFilter.new.tap do |c|
         c.configure! 'filter_proc_string' => 'message.data.data_object % 2 == 0'
       end
     end
 
     it 'should forward generated integers to be filtered by the proc filter' do
-      ruby_proc_filter.filtered.collect_messages(nil, filtered_messages) do
-        ruby_proc_filter.dropped.collect_messages(nil, dropped_messages) do
+      generator.out.direct_connect accept_evens.in
+
+      accept_evens.filtered.collect_messages(nil, accepted_messages) do
+        accept_evens.dropped.collect_messages(nil, dropped_messages) do
+          5.times { generator.generate }
+        end
+      end
+
+      expect(accepted_messages).to have(3).messages
+      expect(accepted_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
+      expect(dropped_messages).to have(2).messages
+      expect(dropped_messages.map(&:data).map(&:data_object)).to eq([1, 3])
+    end
+
+    it 'should forward integers from the union of subports' do
+      generator.even_odd_out.direct_connect accept_evens.in
+
+      accept_evens.filtered.collect_messages(nil, accepted_messages) do
+        accept_evens.dropped.collect_messages(nil, dropped_messages) do
           5.times { generator.generate }
         end
       end
 
-      expect(filtered_messages).to have(3).messages
-      expect(filtered_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
+      expect(accepted_messages).to have(3).messages
+      expect(accepted_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
       expect(dropped_messages).to have(2).messages
       expect(dropped_messages.map(&:data).map(&:data_object)).to eq([1, 3])
     end
+
+    it 'should forward integers from a subport' do
+      generator.even_odd_out['even'].direct_connect accept_evens.in
+
+      accept_evens.filtered.collect_messages(nil, accepted_messages) do
+        accept_evens.dropped.collect_messages(nil, dropped_messages) do
+          5.times { generator.generate }
+        end
+      end
+
+      expect(accepted_messages).to have(3).messages
+      expect(accepted_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
+      expect(dropped_messages).to have(0).messages
+    end
   end
 end
-