rflow 0.0.5

data/lib/rflow/configuration/component.rb

@@ -0,0 +1,27 @@
+require 'active_record'
+require 'rflow/configuration/uuid_keyed'
+
+class RFlow
+  class Configuration
+    class Component < ConfigDB
+      include UUIDKeyed
+      include ActiveModel::Validations
+      
+      class ComponentInvalid < StandardError; end
+      class ComponentNotFound < StandardError; end
+
+      serialize :options, Hash
+
+      has_many :input_ports,  :primary_key => 'uuid', :foreign_key => 'component_uuid'
+      has_many :output_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
+
+      #TODO: Get this to work
+      #has_many :input_connections, :through => :input_ports, :source => :input_connections
+      #has_many :output_connections, :through => :output_ports, :source => :output_connection
+
+      
+      validates_uniqueness_of :name
+
+    end
+  end
+end

data/lib/rflow/configuration/connection.rb

@@ -0,0 +1,98 @@
+require 'active_record'
+require 'rflow/configuration/uuid_keyed'
+
+class RFlow
+  class Configuration
+    class Connection < ConfigDB
+      class ConnectionInvalid < StandardError; end
+
+      include UUIDKeyed
+      include ActiveModel::Validations
+
+      serialize :options, Hash
+      
+      belongs_to :input_port, :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
+      belongs_to :output_port,:primary_key => 'uuid', :foreign_key => 'output_port_uuid'
+
+      before_create :merge_default_options!
+      
+      validates_uniqueness_of :uuid
+      validates_presence_of :output_port_uuid, :input_port_uuid
+
+      validate :all_required_options_present
+
+      
+      def all_required_options_present
+        self.class.required_options.each do |option_name|
+          unless self.options.include? option_name.to_s
+            errors.add(:options, "must include #{option_name} for #{self.class.to_s}")
+          end
+        end
+      end
+
+      
+      def merge_default_options!
+        self.options ||= {}
+        self.class.default_options.each do |option_name, default_value_or_proc|
+          self.options[option_name.to_s] ||= default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call(self) : default_value_or_proc
+        end
+      end
+
+
+      # Should return a list of require option names which will be
+      # used in validations.  To be overridden.
+      def self.required_options; []; end
+
+      
+      # Should return a hash of default options, where the keys are
+      # the option names and the values are either default option
+      # values or Procs that take a single connection argument.  This
+      # allow defaults to use other parameters in the connection to
+      # construct the appropriate default value.
+      def self.default_options; {}; end
+
+    end
+
+    
+    # STI Subclass for ZMQ connections and their required options
+    class ZMQConnection < Connection
+
+      def self.default_options
+        {
+          'output_socket_type'    => 'PUSH',
+          'output_address'       => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
+          'output_responsibility' => 'bind',
+          'input_socket_type'     => 'PULL',
+          'input_address'        => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
+          'input_responsibility'  => 'connect',
+        }
+      end
+    end
+
+    
+    # STI Subclass for AMQP connections and their required options
+    class AMQPConnection < Connection
+
+      def self.default_options
+        {
+          'host'     => 'localhost',
+          'port'     => 5672,
+          'insist'   => true,
+          'vhost'    => '/',
+          'username' => 'guest',
+          'password' => 'guest',
+
+          # If a queue is created, these are the default parameters
+          # for said queue type
+          'queue_passive'     => false,
+          'queue_durable'     => true,
+          'queue_exclusive'   => false,
+          'queue_auto_delete' => false,
+          'queue_nowait'      => true,
+        }
+      end
+
+    end
+  end
+end
+

data/lib/rflow/configuration/migrations/20010101000001_create_settings.rb

@@ -0,0 +1,14 @@
+class CreateSettings < ActiveRecord::Migration
+  def self.up
+    create_table(:settings, :id => false) do |t|
+      t.string :name, :primary => true
+      t.text   :value
+ 
+      t.timestamps
+    end
+  end
+ 
+  def self.down
+    drop_table :settings
+  end
+end

data/lib/rflow/configuration/migrations/20010101000002_create_components.rb

@@ -0,0 +1,19 @@
+class CreateComponents < ActiveRecord::Migration
+  def self.up
+    create_table(:components, :id => false) do |t|
+      t.string :uuid, :limit => 36, :primary => true
+      t.string :name
+      t.boolean :managed, :default => true
+      t.text   :specification
+      t.text   :options
+      
+      t.timestamps
+    end
+
+    add_index :components, :uuid, :unique => true
+  end
+ 
+  def self.down
+    drop_table :components
+  end
+end

data/lib/rflow/configuration/migrations/20010101000003_create_ports.rb

@@ -0,0 +1,24 @@
+class CreatePorts < ActiveRecord::Migration
+  def self.up
+    create_table(:ports, :id => false) do |t|
+      t.string :uuid, :limit => 36, :primary => true
+      t.string :name
+
+      # For STI
+      t.text   :type
+
+      # UUID version of belongs_to :component 
+      t.string :component_uuid
+      
+      t.timestamps
+    end
+
+    add_index :ports, :uuid, :unique => true
+    add_index :ports, :component_uuid
+    add_index :ports, [:component_uuid, :name], :unique => true
+  end
+ 
+  def self.down
+    drop_table :ports
+  end
+end

data/lib/rflow/configuration/migrations/20010101000004_create_connections.rb

@@ -0,0 +1,27 @@
+class CreateConnections < ActiveRecord::Migration
+  def self.up
+    create_table(:connections, :id => false) do |t|
+      t.string :uuid, :limit => 36, :primary => false
+      t.string :name
+
+      # To allow for multiple types of connections
+      t.string :type
+
+      # Data flows from an output port to an input port
+      t.string  :output_port_uuid
+      t.string  :output_port_key, :default => '0'
+      t.string  :input_port_uuid
+      t.string  :input_port_key, :default => '0'
+
+      t.text :options
+      
+      t.timestamps
+    end
+
+    add_index :connections, :uuid, :unique => true
+  end
+ 
+  def self.down
+    drop_table :connections
+  end
+end

data/lib/rflow/configuration/port.rb

@@ -0,0 +1,30 @@
+require 'active_record'
+require 'rflow/configuration/uuid_keyed'
+
+class RFlow
+  class Configuration
+    class Port < ConfigDB
+      include UUIDKeyed
+      include ActiveModel::Validations
+      
+      class PortInvalid < StandardError; end
+
+      belongs_to :component,  :primary_key => 'uuid', :foreign_key => 'component_uuid'
+
+      # TODO: Make some sort of component/port validation work here
+      #validate :component_has_defined_port
+    end
+
+    # STI-based classes
+    class InputPort < Port;
+      has_many :input_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
+      has_many :output_ports, :through => :connections
+    end
+
+    class OutputPort < Port;
+      has_many :output_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
+      has_many :input_ports, :through => :connections
+    end
+  end
+end
+

data/lib/rflow/configuration/ruby_dsl.rb

@@ -0,0 +1,183 @@
+require 'rflow/configuration'
+
+class RFlow
+  class Configuration
+
+    # Ruby DSL config file controller.
+    # TODO: more docs and examples
+    class RubyDSL
+      attr_accessor :setting_specs, :component_specs, :connection_specs, :allocated_system_ports
+      
+      def initialize
+        @setting_specs = []
+        @component_specs = []
+        @connection_specs = []
+      end
+
+      # Helper function to extract the line of the config that
+      # specified the operation.  Useful in printing helpful error messages
+      def get_config_line(call_history)
+        call_history.first.split(':in').first
+      end
+      
+      # DSL method to specify a name/value pair.  RFlow core uses the
+      # 'rflow.' prefix on all of its settings.  Custom settings
+      # should use a custom (unique) prefix
+      def setting(setting_name, setting_value)
+        setting_specs << {:name => setting_name.to_s, :value => setting_value.to_s, :config_line => get_config_line(caller)}
+      end
+
+      # DSL method to specify a component.  Expects a name,
+      # specification, and set of component specific options, that
+      # must be marshallable into the database (i.e. should all be strings)
+      def component(component_name, component_specification, component_options={})
+        component_specs << {
+          :name => component_name,
+          :specification => component_specification.to_s, :options => component_options,
+          :config_line => get_config_line(caller)
+        }
+      end
+
+      # DSL method to specify a connection between a
+      # component/output_port and another component/input_port.  The
+      # component/port specification is a string where the names of
+      # the two elements are separated by '#', and the "connection" is
+      # specified by a Ruby Hash, i.e.:
+      #  connect 'componentA#output' => 'componentB#input'
+      # Array ports are specified with an key suffix in standard
+      # progamming syntax, i.e.
+      #  connect 'componentA#arrayport[2]' => 'componentB#in[1]'
+      # Uses the model to assign random UUIDs
+      def connect(connection_hash)
+        config_file_line = get_config_line(caller)
+        connection_hash.each do |output_string, input_string|
+          output_component_name, output_port_name, output_port_key = parse_connection_string(output_string)
+          input_component_name, input_port_name, input_port_key = parse_connection_string(input_string)
+
+          connection_specs << {
+            :name => output_string + '=>' + input_string, 
+            :output_component_name => output_component_name,
+            :output_port_name => output_port_name, :output_port_key => output_port_key, 
+            :output_string => output_string,
+            :input_component_name => input_component_name,
+            :input_port_name => input_port_name, :input_port_key => input_port_key,
+            :input_string => input_string,
+            :config_line => config_file_line,
+          }
+        end
+      end
+
+      # Splits the connection string into component/port parts
+      COMPONENT_PORT_STRING_REGEX = /^(\w+)#(\w+)(?:\[([^\]]+)\])?$/
+      def parse_connection_string(connection_string)
+        matched = COMPONENT_PORT_STRING_REGEX.match(connection_string)
+        raise ArgumentError, "Invalid component/port string specification: #{connection_string}" unless matched
+        # component_name, port_name, port_key
+        [matched[1], matched[2], (matched[3] || nil)]
+      end
+
+      
+      # Method to process the 'DSL' objects into the config database
+      # via ActiveRecord
+      def process
+        process_setting_specs
+        process_component_specs
+        process_connection_specs
+      end
+
+      
+      # Iterates through each setting specified in the DSL and
+      # creates rows in the database corresponding to the setting
+      def process_setting_specs
+        setting_specs.each do |setting_spec|
+          RFlow.logger.debug "Found config file setting '#{setting_spec[:name]}' = (#{Dir.getwd}) '#{setting_spec[:value]}'"
+          RFlow::Configuration::Setting.create! :name => setting_spec[:name], :value => setting_spec[:value]
+        end
+      end
+
+      
+      # Iterates through each component specified in the DSL and
+      # creates rows in the database corresponding to the component.
+      def process_component_specs
+        component_specs.each do |component_spec|
+          RFlow.logger.debug "Found component '#{component_spec[:name]}', creating"
+          RFlow::Configuration::Component.create! :name => component_spec[:name], :specification => component_spec[:specification], :options => component_spec[:options]
+        end
+      end
+
+      
+      # Iterates through each component specified in the DSL and uses
+      # 'process_connection' to insert all the parts of the connection
+      # into the database
+      def process_connection_specs
+        connection_specs.each do |connection_spec|
+          process_connection_spec(connection_spec)
+        end
+      end
+
+      # For the given connection, break up each input/output
+      # component/port specification, ensure that the component
+      # already exists in the database (by name).  Also, only supports
+      # ZeroMQ ipc sockets
+      def process_connection_spec(connection_spec)
+        RFlow.logger.debug "Found connection from '#{connection_spec[:output_string]}' to '#{connection_spec[:input_string]}', creating"
+        
+        # an input port can be associated with multiple outputs, but
+        # an output port can only be associated with one input
+        output_component = RFlow::Configuration::Component.find_by_name connection_spec[:output_component_name]
+        raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:output_component_name]}" unless output_component
+        output_port = output_component.output_ports.find_or_initialize_by_name :name => connection_spec[:output_port_name]
+        output_port.save!
+        
+        input_component = RFlow::Configuration::Component.find_by_name connection_spec[:input_component_name]
+        raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:input_component_name]}" unless input_component
+        input_port = input_component.input_ports.find_or_initialize_by_name :name => connection_spec[:input_port_name]
+        input_port.save!
+
+        # Create a unique ZMQ address
+#        zmq_address = "ipc://run/rflow.#{output_component.uuid}.#{output_port.uuid}"
+#        if connection_spec[:output_port_key]
+#          zmq_address << ".#{connection_spec[:output_port_key].gsub(/[^\w]/, '').downcase}"
+#        end
+          
+        connection = RFlow::Configuration::ZMQConnection.new(:name => connection_spec[:name],
+                                                             :output_port_key => connection_spec[:output_port_key],
+                                                             :input_port_key => connection_spec[:input_port_key])
+#                                                             :options => {
+#                                                               'output_socket_type' => "PUSH",
+#                                                               'output_address' => zmq_address,
+#                                                               'output_responsibility' => "bind",
+#                                                               'input_socket_type' => "PULL",
+#                                                               'input_address' => zmq_address,
+#                                                               'input_responsibility' => "connect",
+#                                                             })
+
+        connection.output_port = output_port
+        connection.input_port = input_port
+        connection.save!
+
+      rescue RFlow::Configuration::Component::ComponentNotFound => e
+        error_message = "Component '#{e.message}' not found at #{connection_spec[:config_line]}"
+        RFlow.logger.error error_message
+        raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
+      rescue Exception => e
+        # TODO: Figure out why an ArgumentError doesn't put the
+        # offending message into e.message, even though it is printed
+        # out if not caught
+        error_message = "#{e.class}: #{e.message} at config '#{connection_spec[:config_line]}'"
+        RFlow.logger.debug "Exception #{e.class} - " + error_message
+        RFlow.logger.error error_message
+        raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
+      end
+
+      
+      # Method called within the config file itself
+      def self.configure
+        config_file = self.new
+        yield config_file
+        config_file.process
+      end
+
+    end # class RubyDSL
+  end # class Configuration
+end # class RFlow

data/lib/rflow/configuration/setting.rb

@@ -0,0 +1,67 @@
+require 'active_record'
+require 'rflow/configuration/uuid_keyed'
+
+class RFlow
+  class Configuration
+    class Setting < ConfigDB
+      class SettingInvalid < StandardError; end
+
+      include ActiveModel::Validations
+
+      set_primary_key :name
+      attr_accessible :name, :value
+      
+      DEFAULTS = {
+        'rflow.application_name' => 'rflow',
+        
+        'rflow.application_directory_path' => '.',
+        'rflow.pid_directory_path'         => 'run', #lambda {File.join(Setting['rflow.application_directory_path'], 'run')},
+        'rflow.log_directory_path'         => 'log', #lambda {File.join(Setting['rflow.application_directory_path'], 'log')},
+
+        'rflow.log_file_path' => lambda {File.join(Setting['rflow.log_directory_path'], Setting['rflow.application_name'] + '.log')},
+        'rflow.pid_file_path' => lambda {File.join(Setting['rflow.pid_directory_path'], Setting['rflow.application_name'] + '.pid')},
+        
+        'rflow.log_level' => 'INFO',
+      }
+
+      DIRECTORY_PATHS = [
+                         'rflow.application_directory_path',
+                         'rflow.pid_directory_path',
+                         'rflow.log_directory_path',
+                        ]
+      FILE_PATHS = [
+                    'rflow.log_file_path',
+                    'rflow.pid_file_path',
+                   ]
+
+
+      # TODO: fix these validations, as they run without the
+      #application directory path context for subdirectories
+      #validate :valid_directory_path, :if => :directory_path?
+      #validate :valid_writable_path, :if => :directory_path?
+
+      # TODO: Think about making this a regex check to pull in other,
+      # externally-defined settings 
+      def directory_path?
+        DIRECTORY_PATHS.include? self.name
+      end
+
+      def valid_directory_path
+        unless File.directory? self.value
+          errors.add :value, "setting '#{self.name}' is not a directory ('#{File.expand_path self.value}')"
+        end
+      end
+      
+      def valid_writable_path
+        unless File.writable? self.value
+          errors.add :value, "setting '#{self.name}' is not writable ('#{File.expand_path self.value}')"
+        end
+      end
+
+      def self.[](setting_name)
+        Setting.find(setting_name).value rescue nil
+      end
+      
+    end
+  end
+end