rflow 0.0.5 → 1.0.0a1

data/lib/rflow/configuration/component.rb

@@ -6,12 +6,13 @@ class RFlow
     class Component < ConfigDB
       include UUIDKeyed
       include ActiveModel::Validations
-      
+
       class ComponentInvalid < StandardError; end
       class ComponentNotFound < StandardError; end
 
       serialize :options, Hash
 
+      belongs_to :shard, :primary_key => 'uuid', :foreign_key => 'shard_uuid'
       has_many :input_ports,  :primary_key => 'uuid', :foreign_key => 'component_uuid'
       has_many :output_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
 
@@ -19,9 +20,7 @@ class RFlow
       #has_many :input_connections, :through => :input_ports, :source => :input_connections
       #has_many :output_connections, :through => :output_ports, :source => :output_connection
 
-      
       validates_uniqueness_of :name
-
     end
   end
 end

data/lib/rflow/configuration/connection.rb

@@ -10,18 +10,18 @@ class RFlow
       include ActiveModel::Validations
 
       serialize :options, Hash
-      
+
       belongs_to :input_port, :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
       belongs_to :output_port,:primary_key => 'uuid', :foreign_key => 'output_port_uuid'
 
       before_create :merge_default_options!
-      
+
       validates_uniqueness_of :uuid
       validates_presence_of :output_port_uuid, :input_port_uuid
 
       validate :all_required_options_present
 
-      
+
       def all_required_options_present
         self.class.required_options.each do |option_name|
           unless self.options.include? option_name.to_s
@@ -30,7 +30,7 @@ class RFlow
         end
       end
 
-      
+
       def merge_default_options!
         self.options ||= {}
         self.class.default_options.each do |option_name, default_value_or_proc|
@@ -43,7 +43,7 @@ class RFlow
       # used in validations.  To be overridden.
       def self.required_options; []; end
 
-      
+
       # Should return a hash of default options, where the keys are
       # the option names and the values are either default option
       # values or Procs that take a single connection argument.  This
@@ -53,7 +53,7 @@ class RFlow
 
     end
 
-    
+
     # STI Subclass for ZMQ connections and their required options
     class ZMQConnection < Connection
 
@@ -61,15 +61,15 @@ class RFlow
         {
           'output_socket_type'    => 'PUSH',
           'output_address'       => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
-          'output_responsibility' => 'bind',
+          'output_responsibility' => 'connect',
           'input_socket_type'     => 'PULL',
           'input_address'        => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
-          'input_responsibility'  => 'connect',
+          'input_responsibility'  => 'bind',
         }
       end
     end
 
-    
+
     # STI Subclass for AMQP connections and their required options
     class AMQPConnection < Connection
 
@@ -95,4 +95,3 @@ class RFlow
     end
   end
 end
-

data/lib/rflow/configuration/migrations/{20010101000001_create_settings.rb → 20010101000000_create_settings.rb}

@@ -3,11 +3,11 @@ class CreateSettings < ActiveRecord::Migration
     create_table(:settings, :id => false) do |t|
       t.string :name, :primary => true
       t.text   :value
- 
+
       t.timestamps
     end
   end
- 
+
   def self.down
     drop_table :settings
   end

data/lib/rflow/configuration/migrations/20010101000001_create_shards.rb

@@ -0,0 +1,21 @@
+class CreateShards < ActiveRecord::Migration
+  def self.up
+    create_table(:shards, :id => false) do |t|
+      t.string :uuid, :limit => 36, :primary => true
+      t.string :name
+      t.integer :count
+
+      # STI
+      t.string :type
+
+      t.timestamps
+    end
+
+    add_index :shards, :uuid, :unique => true
+    add_index :shards, :name, :unique => true
+  end
+
+  def self.down
+    drop_table :shards
+  end
+end

data/lib/rflow/configuration/migrations/20010101000002_create_components.rb

@@ -6,13 +6,18 @@ class CreateComponents < ActiveRecord::Migration
       t.boolean :managed, :default => true
       t.text   :specification
       t.text   :options
-      
+
+      # UUID version of belongs_to :shard
+      t.string :shard_uuid
+
       t.timestamps
     end
 
     add_index :components, :uuid, :unique => true
+    add_index :components, :name, :unique => true
+    add_index :components, :shard_uuid
   end
- 
+
   def self.down
     drop_table :components
   end

data/lib/rflow/configuration/migrations/20010101000003_create_ports.rb

@@ -7,9 +7,9 @@ class CreatePorts < ActiveRecord::Migration
       # For STI
       t.text   :type
 
-      # UUID version of belongs_to :component 
+      # UUID version of belongs_to :component
       t.string :component_uuid
-      
+
       t.timestamps
     end
 
@@ -17,7 +17,7 @@ class CreatePorts < ActiveRecord::Migration
     add_index :ports, :component_uuid
     add_index :ports, [:component_uuid, :name], :unique => true
   end
- 
+
   def self.down
     drop_table :ports
   end

data/lib/rflow/configuration/migrations/20010101000004_create_connections.rb

@@ -14,13 +14,13 @@ class CreateConnections < ActiveRecord::Migration
       t.string  :input_port_key, :default => '0'
 
       t.text :options
-      
+
       t.timestamps
     end
 
     add_index :connections, :uuid, :unique => true
   end
- 
+
   def self.down
     drop_table :connections
   end

data/lib/rflow/configuration/port.rb

@@ -6,7 +6,7 @@ class RFlow
     class Port < ConfigDB
       include UUIDKeyed
       include ActiveModel::Validations
-      
+
       class PortInvalid < StandardError; end
 
       belongs_to :component,  :primary_key => 'uuid', :foreign_key => 'component_uuid'
@@ -18,13 +18,12 @@ class RFlow
     # STI-based classes
     class InputPort < Port;
       has_many :input_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
-      has_many :output_ports, :through => :connections
+      has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
     end
 
     class OutputPort < Port;
       has_many :output_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
-      has_many :input_ports, :through => :connections
+      has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
     end
   end
 end
-

data/lib/rflow/configuration/ruby_dsl.rb

@@ -6,12 +6,14 @@ class RFlow
     # Ruby DSL config file controller.
     # TODO: more docs and examples
     class RubyDSL
-      attr_accessor :setting_specs, :component_specs, :connection_specs, :allocated_system_ports
-      
+      attr_accessor :setting_specs, :shard_specs, :connection_specs, :allocated_system_ports
+
       def initialize
         @setting_specs = []
-        @component_specs = []
+        @shard_specs = [{:name => "DEFAULT", :type => :process, :count => 1, :components => []}]
         @connection_specs = []
+
+        @current_shard = @shard_specs.first
       end
 
       # Helper function to extract the line of the config that
@@ -19,7 +21,7 @@ class RFlow
       def get_config_line(call_history)
         call_history.first.split(':in').first
       end
-      
+
       # DSL method to specify a name/value pair.  RFlow core uses the
       # 'rflow.' prefix on all of its settings.  Custom settings
       # should use a custom (unique) prefix
@@ -27,11 +29,28 @@ class RFlow
         setting_specs << {:name => setting_name.to_s, :value => setting_value.to_s, :config_line => get_config_line(caller)}
       end
 
+      # DSL method to specify a shard block for either a process or thread
+      def shard(shard_name, shard_options={})
+        raise ArgumentError, "Cannot use DEFAULT as a shard name" if shard_name == 'DEFAULT'
+        shard_type = if shard_options[:thread] || shard_options[:type] == :thread
+                       :thread
+                     else
+                       :process
+                     end
+
+        shard_count = shard_options[shard_type] || shard_options[:count] || 1
+
+        @current_shard = {:name => shard_name, :type => shard_type, :count => shard_count, :components => [], :config_line => get_config_line(caller)}
+        @shard_specs << @current_shard
+        yield self
+        @current_shard = @shard_specs.first
+      end
+
       # DSL method to specify a component.  Expects a name,
       # specification, and set of component specific options, that
       # must be marshallable into the database (i.e. should all be strings)
       def component(component_name, component_specification, component_options={})
-        component_specs << {
+        @current_shard[:components] << {
           :name => component_name,
           :specification => component_specification.to_s, :options => component_options,
           :config_line => get_config_line(caller)
@@ -55,9 +74,9 @@ class RFlow
           input_component_name, input_port_name, input_port_key = parse_connection_string(input_string)
 
           connection_specs << {
-            :name => output_string + '=>' + input_string, 
+            :name => output_string + '=>' + input_string,
             :output_component_name => output_component_name,
-            :output_port_name => output_port_name, :output_port_key => output_port_key, 
+            :output_port_name => output_port_name, :output_port_key => output_port_key,
             :output_string => output_string,
             :input_component_name => input_component_name,
             :input_port_name => input_port_name, :input_port_key => input_port_key,
@@ -76,16 +95,16 @@ class RFlow
         [matched[1], matched[2], (matched[3] || nil)]
       end
 
-      
+
       # Method to process the 'DSL' objects into the config database
       # via ActiveRecord
       def process
         process_setting_specs
-        process_component_specs
+        process_shard_specs
         process_connection_specs
       end
 
-      
+
       # Iterates through each setting specified in the DSL and
       # creates rows in the database corresponding to the setting
       def process_setting_specs
@@ -95,17 +114,36 @@ class RFlow
         end
       end
 
-      
-      # Iterates through each component specified in the DSL and
-      # creates rows in the database corresponding to the component.
-      def process_component_specs
-        component_specs.each do |component_spec|
-          RFlow.logger.debug "Found component '#{component_spec[:name]}', creating"
-          RFlow::Configuration::Component.create! :name => component_spec[:name], :specification => component_spec[:specification], :options => component_spec[:options]
+
+      # Iterates through each shard specified in the DSL and creates
+      # rows in the database corresponding to the shard and included
+      # components
+      def process_shard_specs
+        @shard_specs.each do |shard_spec|
+          RFlow.logger.debug "Found #{shard_spec[:type]} shard '#{shard_spec[:name]}', creating"
+
+          shard_class = case shard_spec[:type]
+                        when :process
+                          RFlow::Configuration::ProcessShard
+                        when :thread
+                          RFlow::Configuration::ThreadShard
+                        else
+                          raise RFlow::Configuration::Shard::ShardInvalid, "Invalid shard: #{shard_spec.inspect}"
+                        end
+
+          shard = shard_class.create! :name => shard_spec[:name], :count => shard_spec[:count]
+
+          shard_spec[:components].each do |component_spec|
+            RFlow.logger.debug "Shard '#{shard_spec[:name]}' found component '#{component_spec[:name]}', creating"
+            RFlow::Configuration::Component.create!(:shard => shard,
+                                                    :name => component_spec[:name],
+                                                    :specification => component_spec[:specification],
+                                                    :options => component_spec[:options])
+          end
         end
       end
 
-      
+
       # Iterates through each component specified in the DSL and uses
       # 'process_connection' to insert all the parts of the connection
       # into the database
@@ -121,36 +159,22 @@ class RFlow
       # ZeroMQ ipc sockets
       def process_connection_spec(connection_spec)
         RFlow.logger.debug "Found connection from '#{connection_spec[:output_string]}' to '#{connection_spec[:input_string]}', creating"
-        
+
         # an input port can be associated with multiple outputs, but
         # an output port can only be associated with one input
         output_component = RFlow::Configuration::Component.find_by_name connection_spec[:output_component_name]
         raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:output_component_name]}" unless output_component
         output_port = output_component.output_ports.find_or_initialize_by_name :name => connection_spec[:output_port_name]
         output_port.save!
-        
+
         input_component = RFlow::Configuration::Component.find_by_name connection_spec[:input_component_name]
         raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:input_component_name]}" unless input_component
         input_port = input_component.input_ports.find_or_initialize_by_name :name => connection_spec[:input_port_name]
         input_port.save!
 
-        # Create a unique ZMQ address
-#        zmq_address = "ipc://run/rflow.#{output_component.uuid}.#{output_port.uuid}"
-#        if connection_spec[:output_port_key]
-#          zmq_address << ".#{connection_spec[:output_port_key].gsub(/[^\w]/, '').downcase}"
-#        end
-          
         connection = RFlow::Configuration::ZMQConnection.new(:name => connection_spec[:name],
                                                              :output_port_key => connection_spec[:output_port_key],
                                                              :input_port_key => connection_spec[:input_port_key])
-#                                                             :options => {
-#                                                               'output_socket_type' => "PUSH",
-#                                                               'output_address' => zmq_address,
-#                                                               'output_responsibility' => "bind",
-#                                                               'input_socket_type' => "PULL",
-#                                                               'input_address' => zmq_address,
-#                                                               'input_responsibility' => "connect",
-#                                                             })
 
         connection.output_port = output_port
         connection.input_port = input_port
@@ -170,7 +194,7 @@ class RFlow
         raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
       end
 
-      
+
       # Method called within the config file itself
       def self.configure
         config_file = self.new

data/lib/rflow/configuration/setting.rb

@@ -8,19 +8,20 @@ class RFlow
 
       include ActiveModel::Validations
 
-      set_primary_key :name
+      self.primary_key = 'name'
+
       attr_accessible :name, :value
-      
+
       DEFAULTS = {
         'rflow.application_name' => 'rflow',
-        
+
         'rflow.application_directory_path' => '.',
         'rflow.pid_directory_path'         => 'run', #lambda {File.join(Setting['rflow.application_directory_path'], 'run')},
         'rflow.log_directory_path'         => 'log', #lambda {File.join(Setting['rflow.application_directory_path'], 'log')},
 
         'rflow.log_file_path' => lambda {File.join(Setting['rflow.log_directory_path'], Setting['rflow.application_name'] + '.log')},
         'rflow.pid_file_path' => lambda {File.join(Setting['rflow.pid_directory_path'], Setting['rflow.application_name'] + '.pid')},
-        
+
         'rflow.log_level' => 'INFO',
       }
 
@@ -41,7 +42,7 @@ class RFlow
       #validate :valid_writable_path, :if => :directory_path?
 
       # TODO: Think about making this a regex check to pull in other,
-      # externally-defined settings 
+      # externally-defined settings
       def directory_path?
         DIRECTORY_PATHS.include? self.name
       end
@@ -51,7 +52,7 @@ class RFlow
           errors.add :value, "setting '#{self.name}' is not a directory ('#{File.expand_path self.value}')"
         end
       end
-      
+
       def valid_writable_path
         unless File.writable? self.value
           errors.add :value, "setting '#{self.name}' is not writable ('#{File.expand_path self.value}')"
@@ -61,7 +62,7 @@ class RFlow
       def self.[](setting_name)
         Setting.find(setting_name).value rescue nil
       end
-      
+
     end
   end
 end

data/lib/rflow/configuration/shard.rb

@@ -0,0 +1,24 @@
+require 'active_record'
+require 'rflow/configuration/uuid_keyed'
+
+class RFlow
+  class Configuration
+
+    class Shard < ConfigDB
+      include UUIDKeyed
+      include ActiveModel::Validations
+
+      class ShardInvalid < StandardError; end
+
+      has_many :components, :primary_key => 'uuid', :foreign_key => 'shard_uuid'
+
+      validates_presence_of :name
+      validates_uniqueness_of :name
+      validates_numericality_of :count, :only_integer => true, :greater_than => 0
+    end
+
+    # STI-based classes
+    class ProcessShard < Shard; end
+    class ThreadShard < Shard; end
+  end
+end