rflow 0.0.5 → 1.0.0a1

data/lib/rflow/component.rb

@@ -1,3 +1,5 @@
+require 'ostruct'
+
 require 'rflow/message'
 require 'rflow/component/port'
 
@@ -8,7 +10,7 @@ class RFlow
       RFlow::Configuration.add_available_component(subclass)
     end
 
-        
+
     # The Component class methods used in the creation of a component
     class << self
       def defined_input_ports
@@ -23,7 +25,7 @@ class RFlow
       # Port defintions only have names
 
       # TODO: consider class-based UUIDs to identify component types
-      
+
       # Define an input port with a given name
       def input_port(port_name)
         define_port(defined_input_ports, port_name)
@@ -41,89 +43,142 @@ class RFlow
 
         # Create the port accessor method based on the port name
         define_method port_name.to_s.to_sym do
-          port = ports.by_name[port_name.to_s] 
+          port = ports.by_name[port_name.to_s]
           return port if port
-          
+
           # If the port was not connected, return a port-like object
           # that can respond/log but doesn't send any data.  Note,
           # it won't be available in the 'by_uuid' collection, as it
-          # doesn't have a configured instance_uuid
+          # doesn't have a configured uuid
           RFlow.logger.debug "'#{self.name}##{port_name}' not connected, creating a disconnected port"
-          disconnected_port = DisconnectedPort.new(port_name, 0)
+
+          disconnected_port = DisconnectedPort.new(OpenStruct.new(:name => port_name, :uuid => 0))
           ports << disconnected_port
           disconnected_port
         end
       end
+
+
+      # Attempt to instantiate a component described by the config
+      # specification. This assumes that the specification of a
+      # component is a fully qualified Ruby class that has already
+      # been loaded. It will first attempt to find subclasses of
+      # RFlow::Component (in the available_components hash) and then
+      # attempt to constantize the specification into a different
+      # class. Future releases will support external (i.e. non-managed
+      # components), but the current stuff only supports Ruby classes
+      def build(config)
+        if config.managed?
+          RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
+          begin
+            RFlow.logger.debug RFlow.configuration.available_components.inspect
+            instantiated_component = if RFlow.configuration.available_components.include? config.specification
+                                       RFlow.logger.debug "Component found in configuration.available_components['#{config.specification}']"
+                                       RFlow.configuration.available_components[config.specification].new(config)
+                                     else
+                                       RFlow.logger.debug "Component not found in configuration.available_components, constantizing component '#{config.specification}'"
+                                       config.specification.constantize.new(config)
+                                     end
+          rescue NameError => e
+            error_message = "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' was not found"
+            RFlow.logger.error error_message
+            raise RuntimeError, error_message
+          rescue Exception => e
+            error_message = "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}"
+            RFlow.logger.error error_message
+            raise RuntimeError, error_message
+          end
+        else
+          error_message = "Non-managed components not yet implemented for component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
+          RFlow.logger.error error_message
+          raise NotImplementedError, error_message
+        end
+
+        instantiated_component
+      end
     end
 
-    attr_reader :instance_uuid
-    attr_reader :name
-    attr_reader :configuration
-    attr_reader :ports
-    
-    def initialize(uuid, name=nil, configuration=nil)
-      @instance_uuid = uuid
-      @name = name
+    attr_reader :config, :uuid, :name, :ports
+
+    def initialize(config)
+      @config = config
+      @uuid = config.uuid
+      @name = config.name
       @ports = PortCollection.new
-      @configuration = configuration
+
+      configure_ports!
+      configure_connections!
+      configure!(config.options)
     end
 
-    
+
     # Returns a list of connected input ports.  Each port will have
     # one or more keys associated with a particular connection.
     def input_ports
       ports.by_type["RFlow::Component::InputPort"]
     end
 
-    
+
     # Returns a list of connected output ports.  Each port will have
     # one or more keys associated with the particular connection.
     def output_ports
       ports.by_type["RFlow::Component::OutputPort"]
     end
 
-    
+
     # Returns a list of disconnected output ports.
     def disconnected_ports
       ports.by_type["RFlow::Component::DisconnectedPort"]
     end
-    
 
-    # TODO: DRY up the following two methods by factoring out into a meta-method
-    
-    def configure_input_port!(port_name, port_instance_uuid, port_options={})
-      unless self.class.defined_input_ports.include? port_name
-        raise ArgumentError, "Input port '#{port_name}' not defined on component '#{self.class}'"
+
+    def configure_ports!
+      # Send the port configuration to each component
+      config.input_ports.each do |input_port_config|
+        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with input port '#{input_port_config.name}' (#{input_port_config.uuid})"
+        configure_input_port!(input_port_config)
       end
-      ports << InputPort.new(port_name, port_instance_uuid, port_options)
+
+      config.output_ports.each do |output_port_config|
+        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with output port '#{output_port_config.name}' (#{output_port_config.uuid})"
+        configure_output_port!(output_port_config)
+      end
+    end
+
+
+    def configure_input_port!(port_config)
+      unless self.class.defined_input_ports.include? port_config.name
+        raise ArgumentError, "Input port '#{port_config.name}' not defined on component '#{self.class}'"
+      end
+      ports << InputPort.new(port_config)
     end
 
-    
-    def configure_output_port!(port_name, port_instance_uuid, port_options={})
-      unless self.class.defined_output_ports.include? port_name
-        raise ArgumentError, "Output port '#{port_name}' not defined on component '#{self.class}'"
+
+    def configure_output_port!(port_config)
+      unless self.class.defined_output_ports.include? port_config.name
+        raise ArgumentError, "Output port '#{port_config.name}' not defined on component '#{self.class}'"
       end
-      ports << OutputPort.new(port_name, port_instance_uuid, port_options)
+      ports << OutputPort.new(port_config)
     end
 
 
-    # Only supports Ruby types.
-    # TODO: figure out how to dynamically load the built-in
-    # connections, or require them at the top of the file and not rely
-    # on rflow.rb requiring 'rflow/connections'
-    def configure_connection!(port_instance_uuid, port_key, connection_type, connection_uuid, connection_name=nil, connection_options={})
-      case connection_type
-      when 'RFlow::Configuration::ZMQConnection'
-        connection = RFlow::Connections::ZMQConnection.new(connection_uuid, connection_name, connection_options)
-      else
-        raise ArgumentError, "Only ZMQConnections currently supported"
+    def configure_connections!
+      config.input_ports.each do |input_port_config|
+        input_port_config.input_connections.each do |input_connection_config|
+          RFlow.logger.debug "Configuring input port '#{input_port_config.name}' (#{input_port_config.uuid}) key '#{input_connection_config.input_port_key}' with #{input_connection_config.type.to_s} connection '#{input_connection_config.name}' (#{input_connection_config.uuid})"
+          ports.by_uuid[input_port_config.uuid].add_connection(input_connection_config.input_port_key, Connection.build(input_connection_config))
+        end
       end
 
-      ports.by_uuid[port_instance_uuid.to_s].add_connection(port_key, connection)
-      connection
+      config.output_ports.each do |output_port_config|
+        output_port_config.output_connections.each do |output_connection_config|
+          RFlow.logger.debug "Configuring output port '#{output_port_config.name}' (#{output_port_config.uuid}) key '#{output_connection_config.output_port_key}' with #{output_connection_config.type.to_s} connection '#{output_connection_config.name}' (#{output_connection_config.uuid})"
+          ports.by_uuid[output_port_config.uuid].add_connection(output_connection_config.output_port_key, Connection.build(output_connection_config))
+        end
+      end
     end
 
-    
+
     # Tell the component to establish it's ports' connections, i.e. make
     # the connection.  Uses the underlying connection object.  Also
     # establishes the callbacks for each of the input ports
@@ -141,26 +196,26 @@ class RFlow
           end
         end
       end
-        
+
       output_ports.each do |output_port|
         output_port.connect!
       end
     end
 
-    
+
     def to_s
-      string = "Component '#{name}' (#{instance_uuid})\n"
+      string = "Component '#{name}' (#{uuid})\n"
       ports.each do |port|
         port.keys.each do |port_key|
           port[port_key].each do |connection|
-            string << "\t#{port.class.to_s} '#{port.name}' (#{port.instance_uuid}) key '#{port_key}' connection '#{connection.name}' (#{connection.instance_uuid})\n"
+            string << "\t#{port.class.to_s} '#{port.name}' (#{port.uuid}) key '#{port_key}' connection '#{connection.name}' (#{connection.uuid})\n"
           end
         end
       end
       string
     end
 
-    
+
     # Method that should be overridden by a subclass to provide for
     # component-specific configuration.  The subclass should use the
     # self.configuration attribute (@configuration) to store its
@@ -168,7 +223,7 @@ class RFlow
     # parameter is from the RFlow configuration database and is (most
     # likely) a hash.  Don't assume that the keys are symbols
     def configure!(deserialized_configuration); end
-    
+
     # Main component running method.  Subclasses should implement if
     # they want to set up any EventMachine stuffs (servers, clients,
     # etc)
@@ -187,6 +242,6 @@ class RFlow
     # before the global RFlow exit.  Sublcasses should implement to
     # cleanup any leftover state, e.g. flush file handles, etc
     def cleanup!; end
-    
+
   end # class Component
 end # class RFlow

data/lib/rflow/component/port.rb

@@ -10,7 +10,7 @@ class RFlow
           connection.send_message(message)
         end
       end
-    end  
+    end
 
     # Collection class to make it easier to index by both names,
     # UUIDs, and types.
@@ -25,7 +25,7 @@ class RFlow
       end
 
       def <<(port)
-        by_uuid[port.instance_uuid.to_s] = port
+        by_uuid[port.uuid.to_s] = port
         by_name[port.name.to_s] = port
         by_type[port.class.to_s] << port
         ports << port
@@ -42,12 +42,14 @@ class RFlow
         end
       end
     end
-    
 
-    # Bare superclass for (potential) later methods.  Currently empty
-    class Port; end
 
-    
+    class Port
+      attr_reader :connected
+      def connected?; @connected; end
+    end
+
+
     # Allows for a list of connections to be assigned to each port/key
     # combination.  Note that binding an input port to an un-indexed
     # output port will result in messages from all indexed connections
@@ -55,14 +57,16 @@ class RFlow
     # result in the same message being sent to all indexed
     # connections.
     class HashPort < Port
-      attr_reader :name, :instance_uuid, :options, :connections_for
-      
-      def initialize(name, instance_uuid, options={})
-        @name = name
-        @instance_uuid = instance_uuid
+      attr_reader :config, :name, :uuid, :connections_for
+
+      def initialize(config)
+        @config = config
+        @name = config.name
+        @uuid = config.uuid
         @connections_for = Hash.new {|hash, key| hash[key] = Array.new.extend(ConnectionCollection)}
       end
 
+
       # Returns an extended Array of all the connections that should
       # be sent/received on this port.  Merges the nil-keyed port
       # (i.e. any connections for a port without a key) to those
@@ -79,7 +83,7 @@ class RFlow
         connections_for[key] << connection
       end
 
-      
+
       # Return a list of connected keys
       def keys
         connections_for.keys
@@ -94,13 +98,13 @@ class RFlow
         end
       end
 
-      
+
       # Send a message to all connections on all keys for this port,
       # but only once per connection.
       def send_message(message)
         all_connections.send_message(message)
       end
-      
+
 
       # Should be overridden.  Called when it is time to actually
       # establish the connection
@@ -113,38 +117,35 @@ class RFlow
           connections
         end.flatten.uniq.extend(ConnectionCollection)
       end
-      
+
     end
 
-    
+
     class InputPort < HashPort
       def connect!
         connections_for.each do |port_key, connections|
           connections.each do |connection|
             connection.connect_input!
+            @connected = true
           end
         end
       end
     end
 
-    
+
     class OutputPort < HashPort
       def connect!
         connections_for.each do |port_key, keyed_connections|
           keyed_connections.each do |connection|
             connection.connect_output!
+            @connected = true
           end
         end
       end
     end
 
+
     class DisconnectedPort < HashPort; end
-    
+
   end
 end
-
-__END__
-
-out[even] -> a
-out[odd]  -> b
-out[nil]  -> c

data/lib/rflow/components/raw.rb

@@ -3,14 +3,14 @@ require 'rflow/components/raw/extensions'
 class RFlow
   module Components
     module Raw
-      
+
       # Load the schemas
       SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
-      
+
       SCHEMA_FILES = {
         'raw.avsc' => 'RFlow::Message::Data::Raw',
       }
-      
+
       SCHEMA_FILES.each do |file_name, data_type_name|
         schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
         RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
@@ -20,7 +20,7 @@ class RFlow
       RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Raw',
                                                         RFlow::Components::Raw::Extensions::RawExtension)
 
-      
+
     end
   end
 end

data/lib/rflow/components/raw/extensions.rb

@@ -1,9 +1,9 @@
 class RFlow
   module Components
     module Raw
-      
+
       module Extensions
-        
+
         module RawExtension
           def self.extended(base_data)
             base_data.data_object ||= {'raw' => ''}

data/lib/rflow/configuration.rb

@@ -17,13 +17,13 @@ class RFlow
     # An exception class
     class ConfigurationInvalid < StandardError; end
 
-    
+
     # A class to hold DB config and connection information
     class ConfigDB < ActiveRecord::Base
         self.abstract_class = true
     end
-    
-    
+
+
     # A collection class for data extensions that supports a naive
     # prefix-based 'inheritance' on lookup.  When looking up a key
     # with [] all existing keys will be examined to determine if the
@@ -55,10 +55,10 @@ class RFlow
       def clear
         @hash.clear
       end
-      
+
     end
 
-    
+
     class << self
 
       # A collection of data types (schemas) indexed by their name and
@@ -119,7 +119,7 @@ class RFlow
       available_data_extensions.add data_type_name, data_extension
     end
 
-    
+
     # Used when RFlow::Component is subclassed to add another
     # available component to the list.
     def self.add_available_component(component)
@@ -141,8 +141,8 @@ class RFlow
       ConfigDB.establish_connection(:adapter => "sqlite3",
                                     :database  => config_database_path)
     end
-    
-    
+
+
     # Using default ActiveRecord migrations, attempt to migrate the
     # database to the latest version.
     def self.migrate_database
@@ -160,7 +160,7 @@ class RFlow
       load config_file_path
     end
 
-    
+
     # Connect to the configuration database, migrate it to the latest
     # version, and process a config file if provided.
     def self.initialize_database(config_database_path, config_file_path=nil)
@@ -172,7 +172,7 @@ class RFlow
                                               :database  => config_database_path)
 
       migrate_database
-      
+
       expanded_config_file_path = File.expand_path config_file_path if config_file_path
 
       working_dir = Dir.getwd
@@ -183,12 +183,14 @@ class RFlow
       end
 
       RFlow.logger.debug "Defaulting non-existing config values"
-      merge_defaults! 
+      merge_defaults!
 
       Dir.chdir working_dir
+
+      self.new(config_database_path)
     end
 
-    
+
     # Make sure that the configuration has all the necessary values set.
     def self.merge_defaults!
       Setting::DEFAULTS.each do |name, default_value_or_proc|
@@ -202,8 +204,8 @@ class RFlow
         end
       end
     end
-    
-    
+
+
     attr_accessor :config_database_path
     attr_accessor :cached_settings
     attr_accessor :cached_components
@@ -211,19 +213,24 @@ class RFlow
     attr_accessor :cached_connections
 
 
-    def initialize(config_database_path)
+    def initialize(config_database_path=nil)
       @cached_settings = Hash.new
       @cached_components = Hash.new
       @cached_ports = []
       @cached_connections = []
 
-      @config_database_path = config_database_path
-      self.class.establish_config_database_connection(config_database_path)
+      # If there is not a config DB path, assume that an AR
+      # conncection has already been established
+      if config_database_path
+        @config_database_path = config_database_path
+        self.class.establish_config_database_connection(config_database_path)
+      end
 
       # Validate the connected database.  TODO: make this more
       # complete, i.e. validate the various columns
       begin
         Setting.first
+        Shard.first
         Component.first
         Port.first
         Connection.first
@@ -234,44 +241,54 @@ class RFlow
       end
     end
 
-    
+
     def to_s
       string = "Configuration:\n"
+
       settings.each do |setting|
         string << "Setting: '#{setting.name}' = '#{setting.value}'\n"
       end
-      components.each do |component|
-        string << "Component '#{component.name}' as #{component.specification} (#{component.uuid})\n"
-        component.output_ports.each do |output_port|
-          output_port.output_connections.each do |output_connection|
-            input_port = output_connection.input_port
-            string << "\tOutputPort '#{output_port.name}' key '#{output_connection.output_port_key}' (#{output_port.uuid}) =>\n"
-            string << "\t\tConnection '#{output_connection.name}' as #{output_connection.type} (#{output_connection.uuid}) =>\n"
-            string << "\t\tInputPort '#{input_port.name}' key '#{output_connection.input_port_key}' (#{input_port.uuid}) Component '#{input_port.component.name}' (#{input_port.component.uuid})\n"
+
+      shards.each do |shard|
+        string << "Shard #{shard.name} (#{shard.uuid}), type #{shard.class.name}, count #{shard.count}\n"
+        shard.components.each do |component|
+          string << "  Component '#{component.name}' as #{component.specification} (#{component.uuid})\n"
+          component.output_ports.each do |output_port|
+            output_port.output_connections.each do |output_connection|
+              input_port = output_connection.input_port
+              string << "    OutputPort '#{output_port.name}' key '#{output_connection.output_port_key}' (#{output_port.uuid}) =>\n"
+              string << "      Connection '#{output_connection.name}' as #{output_connection.type} (#{output_connection.uuid}) =>\n"
+              string << "      InputPort '#{input_port.name}' key '#{output_connection.input_port_key}' (#{input_port.uuid}) Component '#{input_port.component.name}' (#{input_port.component.uuid})\n"
+            end
           end
         end
       end
       string
     end
-      
+
     # Helper method to access settings with minimal syntax
     def [](setting_name)
       Setting.find_by_name(setting_name).value rescue nil
     end
 
-    
+    def settings
+      Setting.all
+    end
+
+    def shards
+      Shard.all
+    end
+
+    def shard(shard_uuid)
+      Shard.find_by_uuid shard_uuid
+    end
+
     def components
       Component.all
     end
 
-    
-    def component(component_instance_uuid)
-      Component.find_by_uuid component_instance_uuid
-    end
-    
-    
-    def settings
-      Setting.all
+    def component(component_uuid)
+      Component.find_by_uuid component_uuid
     end
 
     def available_components
@@ -282,6 +299,7 @@ end
 
 # Load the models
 require 'rflow/configuration/setting'
+require 'rflow/configuration/shard'
 require 'rflow/configuration/component'
 require 'rflow/configuration/port'
 require 'rflow/configuration/connection'