rflow 1.0.0a2 → 1.0.0a3

data/lib/rflow/broker.rb

@@ -0,0 +1,18 @@
+require 'rflow/child_process'
+
+class RFlow
+  # A message broker to mediate messages along a connection.
+  # The broker runs in a child process and will not return from spawn!.
+  class Broker < ChildProcess
+    class << self
+      def build(config)
+        case config.class.name
+        when 'RFlow::Configuration::ZMQStreamer'
+          RFlow::Connections::ZMQStreamer.new(config)
+        else
+          raise ArgumentError, 'Only ZMQ brokers currently supported'
+        end
+      end
+    end
+  end
+end

data/lib/rflow/child_process.rb

@@ -35,8 +35,10 @@ class RFlow
       unhandle_signals
     end
 
+    def run_process; end
+
     def shutdown!(signal)
-      RFlow.logger.info "Shutting down #{@name} due to #{signal}"
+      RFlow.logger.info "Shutting down due to #{signal}"
       unhandle_signals
     end
 

data/lib/rflow/component.rb

@@ -24,16 +24,7 @@ class RFlow
 
         # Create the port accessor method based on the port name
         define_method name.to_s.to_sym do
-          port = ports.by_name[name.to_s]
-          return port if port
-
-          # If the port was not connected, return a port-like object
-          # that can respond/log but doesn't send any data.  Note,
-          # it won't be available in the 'by_uuid' collection, as it
-          # doesn't have a configured uuid
-          RFlow.logger.debug "'#{self.name}##{name}' not connected, creating a disconnected port"
-
-          DisconnectedPort.new(OpenStruct.new(:name => name, :uuid => 0)).tap {|d| ports << d }
+          ports.by_name[name.to_s]
         end
       end
 
@@ -50,33 +41,49 @@ class RFlow
 
         RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
         begin
-          component = RFlow.configuration.available_components[config.specification]
+          component_class = RFlow.configuration.available_components[config.specification]
 
-          if component
+          if component_class
             RFlow.logger.debug "Component found in configuration.available_components['#{config.specification}']"
-            component.new(config)
           else
             RFlow.logger.debug "Component not found in configuration.available_components, constantizing component '#{config.specification}'"
-            config.specification.constantize.new(config)
+            component_class = config.specification.constantize
+          end
+
+          component_class.new(uuid: config.uuid, name: config.name).tap do |component|
+            config.input_ports.each {|p| component.configure_input_port! p.name, uuid: p.uuid }
+            config.output_ports.each {|p| component.configure_output_port! p.name, uuid: p.uuid }
+
+            config.input_ports.each do |p|
+              p.input_connections.each do |c|
+                component.send(p.name.to_sym).add_connection c.input_port_key, Connection.build(c)
+              end
+            end
+
+            config.output_ports.each do |p|
+              p.output_connections.each do |c|
+                component.send(p.name.to_sym).add_connection c.output_port_key, Connection.build(c)
+              end
+            end
           end
         rescue NameError => e
-          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' was not found"
+          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' could not be loaded (#{e.message})"
         rescue Exception => e
-          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}"
+          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}, because: #{e.backtrace.inspect}"
         end
       end
     end
 
-    attr_reader :uuid, :name, :ports
+    attr_accessor :uuid, :name
+    attr_reader :ports
 
-    def initialize(config)
-      @config = config
-      @uuid = config.uuid
-      @name = config.name
+    def initialize(args = {})
+      @name = args[:name]
+      @uuid = args[:uuid]
       @ports = PortCollection.new
 
-      configure_ports!
-      configure_connections!
+      self.class.defined_input_ports.each {|name, _| ports << InputPort.new(self, name: name) }
+      self.class.defined_output_ports.each {|name, _| ports << OutputPort.new(self, name: name) }
     end
 
     # Returns a list of connected input ports.  Each port will have
@@ -87,15 +94,33 @@ class RFlow
     # one or more keys associated with the particular connection.
     def output_ports; ports.by_type["RFlow::Component::OutputPort"]; end
 
-    # Returns a list of disconnected output ports.
-    def disconnected_ports; ports.by_type["RFlow::Component::DisconnectedPort"]; end
+    def configure_input_port!(port_name, options = {})
+      RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) input port '#{port_name}' (#{options[:uuid]})"
+      unless self.class.defined_input_ports.include? port_name
+        raise ArgumentError, "Input port '#{port_name}' not defined on component '#{self.class}'"
+      end
+      ports.by_name[port_name].uuid = options[:uuid]
+    end
+
+    def configure_output_port!(port_name, options = {})
+      RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) output port '#{port_name}' (#{options[:uuid]})"
+      unless self.class.defined_output_ports.include? port_name
+        raise ArgumentError, "Output port '#{port_name}' not defined on component '#{self.class}'"
+      end
+      ports.by_name[port_name].uuid = options[:uuid]
+    end
 
     # Tell the component to establish its ports' connections, i.e. make
     # the connection.  Uses the underlying connection object.  Also
     # establishes the callbacks for each of the input ports
-    def connect!
+    def connect_inputs!
       input_ports.each {|port| port.recv_callback = method(:process_message) }
       input_ports.each(&:connect!)
+    end
+
+    # Tell the component to establish its ports' connections, i.e. make
+    # the connection. Uses the underlying connection object.
+    def connect_outputs!
       output_ports.each(&:connect!)
     end
 
@@ -137,40 +162,5 @@ class RFlow
     # before the global RFlow exit.  Sublcasses should implement to
     # cleanup any leftover state, e.g. flush file handles, etc
     def cleanup!; end
-
-    private
-    def configure_ports!
-      @config.input_ports.each do |p|
-        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with input port '#{p.name}' (#{p.uuid})"
-        unless self.class.defined_input_ports.include? p.name
-          raise ArgumentError, "Input port '#{p.name}' not defined on component '#{self.class}'"
-        end
-        ports << InputPort.new(p)
-      end
-
-      @config.output_ports.each do |p|
-        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with output port '#{p.name}' (#{p.uuid})"
-        unless self.class.defined_output_ports.include? p.name
-          raise ArgumentError, "Output port '#{p.name}' not defined on component '#{self.class}'"
-        end
-        ports << OutputPort.new(p)
-      end
-    end
-
-    def configure_connections!
-      @config.input_ports.each do |p|
-        p.input_connections.each do |c|
-          RFlow.logger.debug "Configuring input port '#{p.name}' (#{p.uuid}) key '#{c.input_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
-          ports.by_uuid[p.uuid].add_connection c.input_port_key, Connection.build(c)
-        end
-      end
-
-      @config.output_ports.each do |p|
-        p.output_connections.each do |c|
-          RFlow.logger.debug "Configuring output port '#{p.name}' (#{p.uuid}) key '#{c.output_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
-          ports.by_uuid[p.uuid].add_connection c.output_port_key, Connection.build(c)
-        end
-      end
-    end
   end
 end

data/lib/rflow/component/port.rb

@@ -9,28 +9,25 @@ class RFlow
       end
     end
 
-    # Collection class to make it easier to index by both names,
-    # UUIDs, and types.
+    # Collection class to make it easier to index by both names
+    # and types.
     class PortCollection
-      attr_reader :ports, :by_uuid, :by_name, :by_type
+      attr_reader :ports, :by_name, :by_type
 
       def initialize
         @ports = []
-        @by_uuid = {}
         @by_name = {}
         @by_type = Hash.new {|hash, key| hash[key.to_s] = []}
       end
 
       def <<(port)
-        by_uuid[port.uuid.to_s] = port
         by_name[port.name.to_s] = port
         by_type[port.class.to_s] << port
         ports << port
         self
       end
 
-      # Enumerate through each connected (or disconnected but
-      # referenced) port
+      # Enumerate through each port
       # TODO: simplify with enumerators and procs
       def each
         ports.each {|port| yield port }
@@ -38,7 +35,12 @@ class RFlow
     end
 
     class Port
-      attr_reader :connected
+      attr_reader :connected, :component
+
+      def initialize(component)
+        @component = component
+      end
+
       def connected?; connected; end
     end
 
@@ -49,16 +51,16 @@ class RFlow
     # result in the same message being sent to all indexed
     # connections.
     class HashPort < Port
-      attr_reader :config, :name, :uuid
+      attr_accessor :name, :uuid
 
       protected
       attr_reader :connections_for
 
       public
-      def initialize(config)
-        @config = config
-        @name = config.name
-        @uuid = config.uuid
+      def initialize(component, args = {})
+        super(component)
+        self.uuid = args[:uuid]
+        self.name = args[:name]
         @connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
       end
 
@@ -77,9 +79,18 @@ class RFlow
 
       # Adds a connection for a given key
       def add_connection(key, connection)
+        RFlow.logger.debug "Attaching #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) to port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
         connections_for[key] << connection
       end
 
+      def direct_connect(other_port)
+        case other_port
+        when InputPort; add_connection nil, ForwardToInputPort.new(other_port)
+        when OutputPort; add_connection nil, ForwardToOutputPort.new(other_port)
+        else raise ArgumentError, "Unknown port type #{other_port.class.name}"
+        end
+      end
+
       # Return a list of connected keys
       def keys
         connections_for.keys
@@ -142,7 +153,5 @@ class RFlow
         all_connections.send_message(message)
       end
     end
-
-    class DisconnectedPort < HashPort; end
   end
 end

data/lib/rflow/configuration.rb

@@ -211,6 +211,7 @@ class RFlow
     def [](name); Setting.find_by_name(name).value rescue nil; end
     def settings; Setting.all; end
     def shards; Shard.all; end
+    def connections; Connection.all; end
     def shard(uuid); Shard.find_by_uuid uuid; end
     def components; Component.all; end
     def component(uuid); Component.find_by_uuid uuid; end

data/lib/rflow/configuration/connection.rb

@@ -46,6 +46,9 @@ class RFlow
       # allow defaults to use other parameters in the connection to
       # construct the appropriate default value.
       def self.default_options; {}; end
+
+      # By default, no broker processes are required to manage a connection.
+      def brokers; []; end
     end
 
     # STI Subclass for ZMQ connections and their required options
@@ -62,31 +65,46 @@ class RFlow
       end
     end
 
-    # STI Subclass for AMQP connections and their required options
-    class AMQPConnection < Connection
+    # STI Subclass for brokered ZMQ connections and their required options
+    #
+    # We name the IPCs to resemble a quasi-component. Outputting to this
+    # connection goes to the 'in' of the IPC pair. Reading input from this
+    # connection comes from the 'out' of the IPC pair.
+    #
+    # The broker shuttles messages between the two to support the many-to-many
+    # delivery pattern.
+    class BrokeredZMQConnection < Connection
       def self.default_options
         {
-          'host'     => 'localhost',
-          'port'     => 5672,
-          'insist'   => true,
-          'vhost'    => '/',
-          'username' => 'guest',
-          'password' => 'guest',
-
-          # If a queue is created, these are the default parameters
-          # for said queue type
-          'queue_passive'     => false,
-          'queue_durable'     => true,
-          'queue_exclusive'   => false,
-          'queue_auto_delete' => false,
-          'queue_nowait'      => true,
+          'output_socket_type'    => 'PUSH',
+          'output_address'        => lambda{|conn| "ipc://rflow.#{conn.uuid}.in"},
+          'output_responsibility' => 'connect',
+          'input_socket_type'     => 'PULL',
+          'input_address'         => lambda{|conn| "ipc://rflow.#{conn.uuid}.out"},
+          'input_responsibility'  => 'connect',
         }
       end
+
+      # A brokered ZMQ connection requires one broker process.
+      def brokers
+        @brokers ||= [ZMQStreamer.new(self)]
+      end
+    end
+
+    # Represents the broker process configuration. No special parameters
+    # that can't be derived from the connection. Not persisted in the database -
+    # it's encapsulated in the nature of the connection.
+    class ZMQStreamer
+      attr_reader :connection
+
+      def initialize(connection)
+        @connection = connection
+      end
     end
 
     # for testing purposes
     class NullConfiguration
-      attr_accessor :name, :uuid, :options
+      attr_accessor :name, :uuid, :options, :input_port_key, :output_port_key
     end
   end
 end

data/lib/rflow/configuration/ruby_dsl.rb

@@ -42,6 +42,16 @@ class RFlow
         @current_shard = default_shard
       end
 
+      # shortcut
+      def process(name, options = {}, &block)
+        shard(name, options.merge(:type => :process), &block)
+      end
+
+      # shortcut
+      def thread(name, options = {}, &block)
+        shard(name, options.merge(:type => :thread), &block)
+      end
+
       # DSL method to specify a component.  Expects a name,
       # specification, and set of component specific options, that
       # must be marshallable into the database (i.e. should all be strings)
@@ -85,12 +95,12 @@ class RFlow
       def self.configure
         config_file = self.new
         yield config_file
-        config_file.process
+        config_file.process_objects
       end
 
       # Method to process the 'DSL' objects into the config database
       # via ActiveRecord
-      def process
+      def process_objects
         process_setting_specs
         process_shard_specs
         process_connection_specs
@@ -154,8 +164,8 @@ class RFlow
 
       # For each given connection, break up each input/output
       # component/port specification, ensure that the component
-      # already exists in the database (by name).  Also, only supports
-      # ZeroMQ ipc sockets
+      # already exists in the database (by name). Chooses the best
+      # connection type for any pair of components.
       def process_connection_specs
         connection_specs.each do |spec|
           begin
@@ -175,11 +185,39 @@ class RFlow
             input_port = input_component.input_ports.find_or_initialize_by_name :name => spec[:input_port_name]
             input_port.save!
 
-            RFlow::Configuration::ZMQConnection.create!(:name => spec[:name],
-                                                        :output_port_key => spec[:output_port_key],
-                                                        :input_port_key => spec[:input_port_key],
-                                                        :output_port => output_port,
-                                                        :input_port => input_port)
+            output_shards = output_component.shard.count
+            input_shards = input_component.shard.count
+
+            in_shard_connection = output_component.shard == input_component.shard
+            one_to_one = output_shards == 1 && input_shards == 1
+            one_to_many = output_shards == 1 && input_shards > 1
+            many_to_one = output_shards > 1 && input_shards == 1
+            many_to_many = output_shards > 1 && input_shards > 1
+
+            connection_type = many_to_many ? RFlow::Configuration::BrokeredZMQConnection : RFlow::Configuration::ZMQConnection
+
+            conn = connection_type.create!(:name => spec[:name],
+                                           :output_port_key => spec[:output_port_key],
+                                           :input_port_key => spec[:input_port_key],
+                                           :output_port => output_port,
+                                           :input_port => input_port)
+
+            # bind on the cardinality-1 side, connect on the cardinality-n side
+            if in_shard_connection
+              conn.options['output_responsibility'] = 'connect'
+              conn.options['input_responsibility'] = 'bind'
+              conn.options['output_address'] = "inproc://rflow.#{conn.uuid}"
+              conn.options['input_address'] = "inproc://rflow.#{conn.uuid}"
+            elsif many_to_one
+              conn.options['output_responsibility'] = 'connect'
+              conn.options['input_responsibility'] = 'bind'
+            elsif one_to_many
+              conn.options['output_responsibility'] = 'bind'
+              conn.options['input_responsibility'] = 'connect'
+            end
+
+            conn.save!
+            conn
           rescue Exception => e
             # TODO: Figure out why an ArgumentError doesn't put the
             # offending message into e.message, even though it is printed