rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow/configuration/component.rb

@@ -3,13 +3,10 @@ require 'rflow/configuration/uuid_keyed'
 
 class RFlow
   class Configuration
-    class Component < ConfigDB
+    class Component < ConfigurationItem
       include UUIDKeyed
       include ActiveModel::Validations
 
-      class ComponentInvalid < StandardError; end
-      class ComponentNotFound < StandardError; end
-
       serialize :options, Hash
 
       belongs_to :shard, :primary_key => 'uuid', :foreign_key => 'shard_uuid'

data/lib/rflow/configuration/connection.rb

@@ -3,7 +3,7 @@ require 'rflow/configuration/uuid_keyed'
 
 class RFlow
   class Configuration
-    class Connection < ConfigDB
+    class Connection < ConfigurationItem
       class ConnectionInvalid < StandardError; end
 
       include UUIDKeyed
@@ -19,10 +19,9 @@ class RFlow
       validates_uniqueness_of :uuid
       validates_presence_of :output_port_uuid, :input_port_uuid
 
-      validate :all_required_options_present
+      validate :all_required_options_present?
 
-
-      def all_required_options_present
+      def all_required_options_present?
         self.class.required_options.each do |option_name|
           unless self.options.include? option_name.to_s
             errors.add(:options, "must include #{option_name} for #{self.class.to_s}")
@@ -30,49 +29,41 @@ class RFlow
         end
       end
 
-
       def merge_default_options!
         self.options ||= {}
-        self.class.default_options.each do |option_name, default_value_or_proc|
-          self.options[option_name.to_s] ||= default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call(self) : default_value_or_proc
+        self.class.default_options.each do |name, default_value_or_proc|
+          self.options[name.to_s] ||= default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call(self) : default_value_or_proc
         end
       end
 
-
       # Should return a list of require option names which will be
       # used in validations.  To be overridden.
       def self.required_options; []; end
 
-
       # Should return a hash of default options, where the keys are
       # the option names and the values are either default option
       # values or Procs that take a single connection argument.  This
       # allow defaults to use other parameters in the connection to
       # construct the appropriate default value.
       def self.default_options; {}; end
-
     end
 
-
     # STI Subclass for ZMQ connections and their required options
     class ZMQConnection < Connection
-
       def self.default_options
         {
           'output_socket_type'    => 'PUSH',
-          'output_address'       => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
+          'output_address'        => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
           'output_responsibility' => 'connect',
           'input_socket_type'     => 'PULL',
-          'input_address'        => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
+          'input_address'         => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
           'input_responsibility'  => 'bind',
         }
       end
     end
 
-
     # STI Subclass for AMQP connections and their required options
     class AMQPConnection < Connection
-
       def self.default_options
         {
           'host'     => 'localhost',
@@ -91,7 +82,11 @@ class RFlow
           'queue_nowait'      => true,
         }
       end
+    end
 
+    # for testing purposes
+    class NullConfiguration
+      attr_accessor :name, :uuid, :options
     end
   end
 end

data/lib/rflow/configuration/port.rb

@@ -3,12 +3,10 @@ require 'rflow/configuration/uuid_keyed'
 
 class RFlow
   class Configuration
-    class Port < ConfigDB
+    class Port < ConfigurationItem
       include UUIDKeyed
       include ActiveModel::Validations
 
-      class PortInvalid < StandardError; end
-
       belongs_to :component,  :primary_key => 'uuid', :foreign_key => 'component_uuid'
 
       # TODO: Make some sort of component/port validation work here
@@ -16,12 +14,12 @@ class RFlow
     end
 
     # STI-based classes
-    class InputPort < Port;
+    class InputPort < Port
       has_many :input_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
       has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
     end
 
-    class OutputPort < Port;
+    class OutputPort < Port
       has_many :output_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
       has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
     end

data/lib/rflow/configuration/ruby_dsl.rb

@@ -2,57 +2,53 @@ require 'rflow/configuration'
 
 class RFlow
   class Configuration
-
     # Ruby DSL config file controller.
     # TODO: more docs and examples
     class RubyDSL
-      attr_accessor :setting_specs, :shard_specs, :connection_specs, :allocated_system_ports
+      private
+      attr_accessor :setting_specs, :shard_specs, :connection_specs, :default_shard
 
+      public
       def initialize
+        @default_shard = {:name => "DEFAULT", :type => :process, :count => 1, :components => []}
+        @current_shard = default_shard
+
         @setting_specs = []
-        @shard_specs = [{:name => "DEFAULT", :type => :process, :count => 1, :components => []}]
+        @shard_specs = [default_shard]
         @connection_specs = []
-
-        @current_shard = @shard_specs.first
-      end
-
-      # Helper function to extract the line of the config that
-      # specified the operation.  Useful in printing helpful error messages
-      def get_config_line(call_history)
-        call_history.first.split(':in').first
       end
 
       # DSL method to specify a name/value pair.  RFlow core uses the
       # 'rflow.' prefix on all of its settings.  Custom settings
       # should use a custom (unique) prefix
-      def setting(setting_name, setting_value)
-        setting_specs << {:name => setting_name.to_s, :value => setting_value.to_s, :config_line => get_config_line(caller)}
+      def setting(name, value)
+        setting_specs << {:name => name.to_s, :value => value.to_s, :config_line => get_config_line(caller)}
       end
 
       # DSL method to specify a shard block for either a process or thread
-      def shard(shard_name, shard_options={})
-        raise ArgumentError, "Cannot use DEFAULT as a shard name" if shard_name == 'DEFAULT'
-        shard_type = if shard_options[:thread] || shard_options[:type] == :thread
-                       :thread
-                     else
-                       :process
-                     end
-
-        shard_count = shard_options[shard_type] || shard_options[:count] || 1
-
-        @current_shard = {:name => shard_name, :type => shard_type, :count => shard_count, :components => [], :config_line => get_config_line(caller)}
-        @shard_specs << @current_shard
+      def shard(name, options = {})
+        raise ArgumentError, "Cannot use DEFAULT as a shard name" if name == 'DEFAULT'
+        raise ArgumentError, "Cannot nest shards" if @current_shard != default_shard
+
+        type = if options[:thread] || options[:type] == :thread; :thread
+               else :process
+               end
+
+        count = options[type] || options[:count] || 1
+
+        @current_shard = {:name => name, :type => type, :count => count, :components => [], :config_line => get_config_line(caller)}
+        shard_specs << @current_shard
         yield self
-        @current_shard = @shard_specs.first
+        @current_shard = default_shard
       end
 
       # DSL method to specify a component.  Expects a name,
       # specification, and set of component specific options, that
       # must be marshallable into the database (i.e. should all be strings)
-      def component(component_name, component_specification, component_options={})
+      def component(name, specification, options = {})
         @current_shard[:components] << {
-          :name => component_name,
-          :specification => component_specification.to_s, :options => component_options,
+          :name => name,
+          :specification => specification.to_s, :options => options,
           :config_line => get_config_line(caller)
         }
       end
@@ -67,9 +63,8 @@ class RFlow
       # progamming syntax, i.e.
       #  connect 'componentA#arrayport[2]' => 'componentB#in[1]'
       # Uses the model to assign random UUIDs
-      def connect(connection_hash)
-        config_file_line = get_config_line(caller)
-        connection_hash.each do |output_string, input_string|
+      def connect(hash)
+        hash.each do |output_string, input_string|
           output_component_name, output_port_name, output_port_key = parse_connection_string(output_string)
           input_component_name, input_port_name, input_port_key = parse_connection_string(input_string)
 
@@ -81,21 +76,18 @@ class RFlow
             :input_component_name => input_component_name,
             :input_port_name => input_port_name, :input_port_key => input_port_key,
             :input_string => input_string,
-            :config_line => config_file_line,
+            :config_line => get_config_line(caller)
           }
         end
       end
 
-      # Splits the connection string into component/port parts
-      COMPONENT_PORT_STRING_REGEX = /^(\w+)#(\w+)(?:\[([^\]]+)\])?$/
-      def parse_connection_string(connection_string)
-        matched = COMPONENT_PORT_STRING_REGEX.match(connection_string)
-        raise ArgumentError, "Invalid component/port string specification: #{connection_string}" unless matched
-        # component_name, port_name, port_key
-        [matched[1], matched[2], (matched[3] || nil)]
+      # Method called within the config file itself
+      def self.configure
+        config_file = self.new
+        yield config_file
+        config_file.process
       end
 
-
       # Method to process the 'DSL' objects into the config database
       # via ActiveRecord
       def process
@@ -104,37 +96,54 @@ class RFlow
         process_connection_specs
       end
 
+      private
+      # Helper function to extract the line of the config that
+      # specified the operation.  Useful in printing helpful error messages
+      def get_config_line(call_history)
+        call_history.first.split(':in').first
+      end
+
+      # Splits the connection string into component/port parts
+      COMPONENT_PORT_STRING_REGEX = /^(\w+)#(\w+)(?:\[([^\]]+)\])?$/
+
+      def parse_connection_string(string)
+        matched = COMPONENT_PORT_STRING_REGEX.match(string)
+        raise ArgumentError, "Invalid component/port string specification: #{string}" unless matched
+        component_name, port_name, port_key = matched.captures
+        [component_name, port_name, port_key]
+      end
 
       # Iterates through each setting specified in the DSL and
       # creates rows in the database corresponding to the setting
       def process_setting_specs
-        setting_specs.each do |setting_spec|
-          RFlow.logger.debug "Found config file setting '#{setting_spec[:name]}' = (#{Dir.getwd}) '#{setting_spec[:value]}'"
-          RFlow::Configuration::Setting.create! :name => setting_spec[:name], :value => setting_spec[:value]
+        setting_specs.each do |spec|
+          RFlow.logger.debug "Found config file setting '#{spec[:name]}' = (#{Dir.getwd}) '#{spec[:value]}'"
+          RFlow::Configuration::Setting.create! :name => spec[:name], :value => spec[:value]
         end
       end
 
-
       # Iterates through each shard specified in the DSL and creates
       # rows in the database corresponding to the shard and included
       # components
       def process_shard_specs
-        @shard_specs.each do |shard_spec|
-          RFlow.logger.debug "Found #{shard_spec[:type]} shard '#{shard_spec[:name]}', creating"
-
-          shard_class = case shard_spec[:type]
-                        when :process
-                          RFlow::Configuration::ProcessShard
-                        when :thread
-                          RFlow::Configuration::ThreadShard
-                        else
-                          raise RFlow::Configuration::Shard::ShardInvalid, "Invalid shard: #{shard_spec.inspect}"
-                        end
-
-          shard = shard_class.create! :name => shard_spec[:name], :count => shard_spec[:count]
-
-          shard_spec[:components].each do |component_spec|
-            RFlow.logger.debug "Shard '#{shard_spec[:name]}' found component '#{component_spec[:name]}', creating"
+        shard_specs.each do |spec|
+          RFlow.logger.debug "Found #{spec[:type]} shard '#{spec[:name]}', creating"
+
+          if spec[:components].empty?
+            RFlow.logger.warn "Skipping shard '#{spec[:name]}' because it has no components"
+            next
+          end
+
+          clazz = case spec[:type]
+                  when :process; RFlow::Configuration::ProcessShard
+                  when :thread; RFlow::Configuration::ThreadShard
+                  else raise RFlow::Configuration::Shard::ShardInvalid, "Invalid shard: #{spec.inspect}"
+                  end
+
+          shard = clazz.create! :name => spec[:name], :count => spec[:count]
+
+          spec[:components].each do |component_spec|
+            RFlow.logger.debug "Shard '#{spec[:name]}' found component '#{component_spec[:name]}', creating"
             RFlow::Configuration::Component.create!(:shard => shard,
                                                     :name => component_spec[:name],
                                                     :specification => component_spec[:specification],
@@ -143,65 +152,42 @@ class RFlow
         end
       end
 
-
-      # Iterates through each component specified in the DSL and uses
-      # 'process_connection' to insert all the parts of the connection
-      # into the database
-      def process_connection_specs
-        connection_specs.each do |connection_spec|
-          process_connection_spec(connection_spec)
-        end
-      end
-
-      # For the given connection, break up each input/output
+      # For each given connection, break up each input/output
       # component/port specification, ensure that the component
       # already exists in the database (by name).  Also, only supports
       # ZeroMQ ipc sockets
-      def process_connection_spec(connection_spec)
-        RFlow.logger.debug "Found connection from '#{connection_spec[:output_string]}' to '#{connection_spec[:input_string]}', creating"
-
-        # an input port can be associated with multiple outputs, but
-        # an output port can only be associated with one input
-        output_component = RFlow::Configuration::Component.find_by_name connection_spec[:output_component_name]
-        raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:output_component_name]}" unless output_component
-        output_port = output_component.output_ports.find_or_initialize_by_name :name => connection_spec[:output_port_name]
-        output_port.save!
-
-        input_component = RFlow::Configuration::Component.find_by_name connection_spec[:input_component_name]
-        raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:input_component_name]}" unless input_component
-        input_port = input_component.input_ports.find_or_initialize_by_name :name => connection_spec[:input_port_name]
-        input_port.save!
-
-        connection = RFlow::Configuration::ZMQConnection.new(:name => connection_spec[:name],
-                                                             :output_port_key => connection_spec[:output_port_key],
-                                                             :input_port_key => connection_spec[:input_port_key])
-
-        connection.output_port = output_port
-        connection.input_port = input_port
-        connection.save!
-
-      rescue RFlow::Configuration::Component::ComponentNotFound => e
-        error_message = "Component '#{e.message}' not found at #{connection_spec[:config_line]}"
-        RFlow.logger.error error_message
-        raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
-      rescue Exception => e
-        # TODO: Figure out why an ArgumentError doesn't put the
-        # offending message into e.message, even though it is printed
-        # out if not caught
-        error_message = "#{e.class}: #{e.message} at config '#{connection_spec[:config_line]}'"
-        RFlow.logger.debug "Exception #{e.class} - " + error_message
-        RFlow.logger.error error_message
-        raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
-      end
-
-
-      # Method called within the config file itself
-      def self.configure
-        config_file = self.new
-        yield config_file
-        config_file.process
+      def process_connection_specs
+        connection_specs.each do |spec|
+          begin
+            RFlow.logger.debug "Found connection from '#{spec[:output_string]}' to '#{spec[:input_string]}', creating"
+
+            # an input port can be associated with multiple outputs, but
+            # an output port can only be associated with one input
+            output_component = RFlow::Configuration::Component.find_by_name spec[:output_component_name]
+            raise RFlow::Configuration::Connection::ConnectionInvalid,
+              "Component '#{spec[:output_component_name]}' not found at #{spec[:config_line]}" unless output_component
+            output_port = output_component.output_ports.find_or_initialize_by_name :name => spec[:output_port_name]
+            output_port.save!
+
+            input_component = RFlow::Configuration::Component.find_by_name spec[:input_component_name]
+            raise RFlow::Configuration::Connection::ConnectionInvalid,
+              "Component '#{spec[:input_component_name]}' not found at #{spec[:config_line]}" unless input_component
+            input_port = input_component.input_ports.find_or_initialize_by_name :name => spec[:input_port_name]
+            input_port.save!
+
+            RFlow::Configuration::ZMQConnection.create!(:name => spec[:name],
+                                                        :output_port_key => spec[:output_port_key],
+                                                        :input_port_key => spec[:input_port_key],
+                                                        :output_port => output_port,
+                                                        :input_port => input_port)
+          rescue Exception => e
+            # TODO: Figure out why an ArgumentError doesn't put the
+            # offending message into e.message, even though it is printed
+            # out if not caught
+            raise RFlow::Configuration::Connection::ConnectionInvalid, "#{e.class}: #{e.message} at config '#{spec[:config_line]}'"
+          end
+        end
       end
-
-    end # class RubyDSL
-  end # class Configuration
-end # class RFlow
+    end
+  end
+end

data/lib/rflow/configuration/setting.rb

@@ -3,9 +3,7 @@ require 'rflow/configuration/uuid_keyed'
 
 class RFlow
   class Configuration
-    class Setting < ConfigDB
-      class SettingInvalid < StandardError; end
-
+    class Setting < ConfigurationItem
       include ActiveModel::Validations
 
       self.primary_key = 'name'
@@ -13,33 +11,30 @@ class RFlow
       attr_accessible :name, :value
 
       DEFAULTS = {
-        'rflow.application_name' => 'rflow',
-
+        'rflow.application_name'           => 'rflow',
         'rflow.application_directory_path' => '.',
-        'rflow.pid_directory_path'         => 'run', #lambda {File.join(Setting['rflow.application_directory_path'], 'run')},
-        'rflow.log_directory_path'         => 'log', #lambda {File.join(Setting['rflow.application_directory_path'], 'log')},
-
+        'rflow.pid_directory_path'         => 'run', # relative to rflow.application_directory_path
+        'rflow.log_directory_path'         => 'log', # relative to rflow.application_directory_path
         'rflow.log_file_path' => lambda {File.join(Setting['rflow.log_directory_path'], Setting['rflow.application_name'] + '.log')},
         'rflow.pid_file_path' => lambda {File.join(Setting['rflow.pid_directory_path'], Setting['rflow.application_name'] + '.pid')},
-
         'rflow.log_level' => 'INFO',
       }
 
       DIRECTORY_PATHS = [
-                         'rflow.application_directory_path',
-                         'rflow.pid_directory_path',
-                         'rflow.log_directory_path',
-                        ]
-      FILE_PATHS = [
-                    'rflow.log_file_path',
-                    'rflow.pid_file_path',
-                   ]
+        'rflow.application_directory_path',
+        'rflow.pid_directory_path',
+        'rflow.log_directory_path',
+      ]
 
+      FILE_PATHS = [
+        'rflow.log_file_path',
+        'rflow.pid_file_path',
+      ]
 
       # TODO: fix these validations, as they run without the
-      #application directory path context for subdirectories
-      #validate :valid_directory_path, :if => :directory_path?
-      #validate :valid_writable_path, :if => :directory_path?
+      # application directory path context for subdirectories
+      #validate :valid_directory_path?, :if => :directory_path?
+      #validate :valid_writable_path?, :if => :directory_path?
 
       # TODO: Think about making this a regex check to pull in other,
       # externally-defined settings
@@ -47,22 +42,21 @@ class RFlow
         DIRECTORY_PATHS.include? self.name
       end
 
-      def valid_directory_path
+      def valid_directory_path?
         unless File.directory? self.value
           errors.add :value, "setting '#{self.name}' is not a directory ('#{File.expand_path self.value}')"
         end
       end
 
-      def valid_writable_path
+      def valid_writable_path?
         unless File.writable? self.value
           errors.add :value, "setting '#{self.name}' is not writable ('#{File.expand_path self.value}')"
         end
       end
 
-      def self.[](setting_name)
-        Setting.find(setting_name).value rescue nil
+      def self.[](name)
+        Setting.find(name).value rescue nil
       end
-
     end
   end
 end