rflow 1.0.0a4 → 1.0.0a5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 91749755100dd4bfa1eb64f8b455e42cbb9e6b1a
-  data.tar.gz: 537aa8a39d393a99e8b4f8bf674eb8d38eb87169
+  metadata.gz: ef1b7b3ae0fb683425fec5c44f0c1d39f7f29935
+  data.tar.gz: acc3b05b5b26d9286fe51c4c0d3e6a028d9c0f46
 SHA512:
-  metadata.gz: f107c920c6255585fe9508c50cdea49dc0c3f39f8e8cbd321cb3ec04f3997c1e733923d0fcc153dc632a134af650ec37fe55402a266f87a093ae19b431d5dcde
-  data.tar.gz: 537459d7fed23e2f025b8b06f1fa22738b39fc1235bd54376edef8d880683915769e47d5c8b94f90979bd0add24726c1749cd010f02241f3385a17d2d4ca14aa
+  metadata.gz: d040a8d35aff247bd4200c2b5ac8f4ba4c9b8b9dc48a31292e68db1db07cc3e568a2a33067c600f54d99e3346bc60640f7338f395d9c9ddbc5caf7c7f725145d
+  data.tar.gz: 3f8b6b770e61ad2eb2ad17aa647d516d761d042c38491d958bbcb7b85f1b0d91fc80fa0cdd9d5a6e27af1308d05c2c3324c2e9a595bb1b076eedbb781c1c66a8

data/README.md

@@ -64,10 +64,11 @@ work. (You will probably get errors saying arcane things like
 
 * __Connection__ - a directed link between an output port and an input
   port.  RFlow supports generalized connection types; however, only
-  ZeroMQ links are currently used.
+  ZeroMQ links are currently used.  Round-robin and broadcast message
+  delivery are supported on a per-link basis.
 
 * __Message__ - a bit of serialized data that is sent out an output
-  port and recieved on an input port. Due to the serialization,
+  port and received on an input port. Due to the serialization,
   message types and schemas are explicitly defined. In a departure
   from "pure" FBP, RFlow supports sending multiple message types via a
   single connection.
@@ -228,7 +229,7 @@ The `provenance` is a way for a component to annotate a message with a
 bit of data that should (by convention) be carried through the
 workflow with the message, as well as being copied to derived
 messages. For example, a TCP server component would spin up a TCP
-server and, upon recieving a connection and packets on a session, it
+server and, upon receiving a connection and packets on a session, it
 would marshal the packets into `RFlow::Messsage`s and send them out
 its output ports. Messages received on its input port, however, need
 to have a way to be matched to the corresponding underlying TCP
@@ -237,7 +238,6 @@ component to add a bit of metadata (namely an identifier for the TCP
 connection) such that later messages that contain the same provenance
 can be matched to the correct underlying TCP connection.
 
-
 The other parts of the message envelope are related to the embedded
 data object. In addition to the data object itself (which is encoded
 with a specific Avro schema), there are a few fields that describe the
@@ -319,7 +319,6 @@ message.data.int = 1024
 messaga.data.default?   # => false
 ```
 
-
 ## RFlow Workflow Configuration
 
 RFlow currently stores its configuration in a SQLite database which
@@ -351,12 +350,13 @@ identify specific components.
   either a `RFlow::Configuration::InputPort` or
   `RFlow::Configuration::OutputPort`.
 
-* connections - a connection between two ports via foriegn keys
+* connections - a connection between two ports via foreign keys
   `input_port_uuid` and `output_port_uuid`. Like ports, connections
   are typed via AR STI (`RFlow::Configuration::ZMQConnection` and
-  'RFlow::Configuration::BrokeredZMGConnection` are the only
+  `RFlow::Configuration::BrokeredZMQConnection` are the only
   supported values for now) and have a YAML serialized `options`
-  hash. A connection also (potentially) defines the port keys.
+  hash and a `delivery` type (`round-robin` or `broadcast`).
+  A connection also (potentially) defines the port keys.
 
 RFlow also provides a RubyDSL for configuration-like file to be used
 to load the database:
@@ -411,9 +411,10 @@ ZeroMQ communication between components in the same shard uses ZeroMQ's
 components in different shards is accomplished with a ZeroMQ `ipc` socket.
 In the case of a many-to-many connection (many workers in a producing
 shard and many workers in a consuming shard), a ZeroMQ message broker
-process is created to route the messages appropriately. Senders round-robin
-to receivers and receivers fair-queue the messages from the senders.
-Load balancing based on receiver responsiveness is not currently implemented.
+process is created to route the messages appropriately. By default,
+senders round-robin to receivers, though broadcast delivery can be chosen
+instead. Receivers fair-queue the messages from senders.  Load balancing
+based on receiver responsiveness is not currently implemented.
 
 To define a custom shard in the Ruby DSL, use the `shard` method. For
 example:
@@ -462,7 +463,9 @@ RFlow::Configuration::RubyDSL.configure do |config|
   config.connect 'generate_ints1#out' => 'filter#in'
   config.connect 'generate_ints2#out' => 'filter#in'
   config.connect 'filter#filtered' => 'replicate#in'
-  config.connect 'filter#out' => 'output1#in'
+  # choosing broadcast delivery delivers a copy to each worker for
+  # the shard
+  config.connect 'filter#out' => 'output1#in', :delivery => 'broadcast'
   config.connect 'filter#filtered' => 'output2#in'
 end
 ```

data/lib/rflow/component/port.rb

@@ -83,6 +83,26 @@ class RFlow
         connections_for[key] << connection
       end
 
+      # Removes a connection from a given key
+      def remove_connection(key, connection)
+        RFlow.logger.debug "Removing #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) from port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
+        connections_for[key].delete(connection)
+      end
+
+      def collect_messages(key, receiver)
+        begin
+          connection = RFlow::MessageCollectingConnection.new.tap do |c|
+            c.messages = receiver
+          end
+          add_connection key, connection
+
+          yield if block_given?
+          connection
+        ensure
+          remove_connection key, connection if connection && block_given?
+        end
+      end
+
       def direct_connect(other_port)
         case other_port
         when InputPort; add_connection nil, ForwardToInputPort.new(other_port)

data/lib/rflow/shard.rb

@@ -31,6 +31,7 @@ class RFlow
             run_components!
           rescue Exception => e
             RFlow.logger.error "Error in worker, shutting down: #{e.class.name}: #{e.message}, because: #{e.backtrace.inspect}"
+            exit! 1
           end
         end
 
@@ -68,6 +69,11 @@ class RFlow
       end
 
       def shutdown!(signal)
+        RFlow.logger.debug "Shutting down components"
+        @components.each do |component|
+          RFlow.logger.debug "Shutting down component '#{component.name}' (#{component.uuid})"
+          component.shutdown!
+        end
         EM.stop_event_loop
         super
       end

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "1.0.0a4"
+  VERSION = "1.0.0a5"
 end

data/spec/rflow/component/port_spec.rb

@@ -31,6 +31,22 @@ class RFlow
           end
         end
       end
+
+      context "#(add|remove)_connection" do
+        it "should remove the connection" do
+          connection = double('connection')
+          allow(connection).to receive(:name)
+          allow(connection).to receive(:uuid)
+          allow(connection).to receive(:input_port_key)
+
+          described_class.new(nil).tap do |port|
+            port.add_connection(nil, connection)
+            expect(port[nil]).to include connection
+            port.remove_connection(nil, connection)
+            expect(port[nil]).not_to include connection
+          end
+        end
+      end
     end
 
     describe OutputPort do

data/spec/rflow/components/clock_spec.rb

@@ -7,18 +7,15 @@ class RFlow
         ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
         Configuration.migrate_database
       end
-      let(:message_connection) { RFlow::MessageCollectingConnection.new }
+
+      let(:messages) { [] }
 
       def clock(args = {})
         Clock.new.tap do |c|
           c.configure! args
-          c.tick_port.connect!
-          c.tick_port.add_connection nil, message_connection
         end
       end
 
-      def messages; message_connection.messages; end
-
       it 'defaults configuration nicely' do
         clock.tap do |c|
           expect(c.clock_name).to eq('Clock')
@@ -51,14 +48,16 @@ class RFlow
 
       it 'should generate a tick message when asked' do
         clock.tap do |c|
-          now = Integer(Time.now.to_f * 1000)
-          expect(messages).to be_empty
-          c.tick
-          expect(messages).to have(1).message
-          messages.first.tap do |m|
-            expect(m.data_type_name).to eq('RFlow::Message::Clock::Tick')
-            expect(m.data.name).to eq('Clock')
-            expect(m.data.timestamp).to be >= now
+          c.tick_port.collect_messages(nil, messages) do
+            now = Integer(Time.now.to_f * 1000)
+            expect(messages).to be_empty
+            c.tick
+            expect(messages).to have(1).message
+            messages.first.tap do |m|
+              expect(m.data_type_name).to eq('RFlow::Message::Clock::Tick')
+              expect(m.data.name).to eq('Clock')
+              expect(m.data.timestamp).to be >= now
+            end
           end
         end
       end

data/spec/rflow/forward_to_input_port_spec.rb

@@ -7,8 +7,8 @@ class RFlow
       Configuration.migrate_database
     end
 
-    let(:filtered_message_connection) { RFlow::MessageCollectingConnection.new }
-    let(:dropped_message_connection) { RFlow::MessageCollectingConnection.new }
+    let(:filtered_messages) { [] }
+    let(:dropped_messages) { [] }
 
     let(:generator) do
       RFlow::Components::GenerateIntegerSequence.new.tap do |c|
@@ -20,16 +20,16 @@ class RFlow
     let(:ruby_proc_filter) do
       RFlow::Components::RubyProcFilter.new.tap do |c|
         c.configure! 'filter_proc_string' => 'message.data.data_object % 2 == 0'
-        c.filtered.add_connection nil, filtered_message_connection
-        c.dropped.add_connection nil, dropped_message_connection
       end
     end
 
-    def filtered_messages; filtered_message_connection.messages; end
-    def dropped_messages; dropped_message_connection.messages; end
-
     it 'should forward generated integers to be filtered by the proc filter' do
-      5.times { generator.generate }
+      ruby_proc_filter.filtered.collect_messages(nil, filtered_messages) do
+        ruby_proc_filter.dropped.collect_messages(nil, dropped_messages) do
+          5.times { generator.generate }
+        end
+      end
+
       expect(filtered_messages).to have(3).messages
       expect(filtered_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
       expect(dropped_messages).to have(2).messages

data/spec/rflow/forward_to_output_port_spec.rb

@@ -7,7 +7,7 @@ class RFlow
       Configuration.migrate_database
     end
 
-    let(:message_connection) { RFlow::MessageCollectingConnection.new }
+    let(:messages) { [] }
 
     let(:generator) do
       RFlow::Components::GenerateIntegerSequence.new.tap do |c|
@@ -19,14 +19,13 @@ class RFlow
     let(:ruby_proc_filter) do
       RFlow::Components::RubyProcFilter.new.tap do |c|
         c.configure! 'filter_proc_string' => 'message % 2 == 0'
-        c.filtered.add_connection nil, message_connection
       end
     end
 
-    def messages; message_connection.messages; end
-
     it 'should place the messages on the output port, regardless of the filter' do
-      5.times { generator.generate }
+      ruby_proc_filter.filtered.collect_messages(nil, messages) do
+        5.times { generator.generate }
+      end
       expect(messages.map(&:data).map(&:data_object)).to eq([0, 1, 2, 3, 4])
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.0.0a4
+  version: 1.0.0a5
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-20 00:00:00.000000000 Z
+date: 2014-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools