rflow 0.0.5

data/lib/rflow/component/port.rb

@@ -0,0 +1,150 @@
+class RFlow
+  class Component
+
+    # TODO: make this into a class to limit the amount of extensions
+    # that we have to do when operating on these "Arrays", i.e. when
+    # adding two together
+    module ConnectionCollection
+      def send_message(message)
+        each do |connection|
+          connection.send_message(message)
+        end
+      end
+    end  
+
+    # Collection class to make it easier to index by both names,
+    # UUIDs, and types.
+    class PortCollection
+      attr_reader :ports, :by_uuid, :by_name, :by_type
+
+      def initialize
+        @ports = Array.new
+        @by_uuid = Hash.new
+        @by_name = Hash.new
+        @by_type = Hash.new {|hash, key| hash[key.to_s] = []}
+      end
+
+      def <<(port)
+        by_uuid[port.instance_uuid.to_s] = port
+        by_name[port.name.to_s] = port
+        by_type[port.class.to_s] << port
+        ports << port
+        self
+      end
+
+
+      # Enumerate through each connected (or disconnected but
+      # referenced) port
+      # TODO: simplify with enumerators and procs
+      def each
+        ports.each do |port|
+          yield port
+        end
+      end
+    end
+    
+
+    # Bare superclass for (potential) later methods.  Currently empty
+    class Port; end
+
+    
+    # Allows for a list of connections to be assigned to each port/key
+    # combination.  Note that binding an input port to an un-indexed
+    # output port will result in messages from all indexed connections
+    # being received.  Similarly, sending to an unindexed port will
+    # result in the same message being sent to all indexed
+    # connections.
+    class HashPort < Port
+      attr_reader :name, :instance_uuid, :options, :connections_for
+      
+      def initialize(name, instance_uuid, options={})
+        @name = name
+        @instance_uuid = instance_uuid
+        @connections_for = Hash.new {|hash, key| hash[key] = Array.new.extend(ConnectionCollection)}
+      end
+
+      # Returns an extended Array of all the connections that should
+      # be sent/received on this port.  Merges the nil-keyed port
+      # (i.e. any connections for a port without a key) to those
+      # specific for the key, so should only be used to read a list of
+      # connections, not to add new ones.  Use add_connection to add a
+      # new connection for a given key.
+      def [](key)
+         (connections_for[key] + connections_for[nil]).extend(ConnectionCollection)
+      end
+
+
+      # Adds a connection for a given key
+      def add_connection(key, connection)
+        connections_for[key] << connection
+      end
+
+      
+      # Return a list of connected keys
+      def keys
+        connections_for.keys
+      end
+
+
+      # Enumerate through all the ConnectionCollections
+      # TODO: simplify with enumerators and procs
+      def each
+        connections_for.values.each do |connections|
+          yield connections
+        end
+      end
+
+      
+      # Send a message to all connections on all keys for this port,
+      # but only once per connection.
+      def send_message(message)
+        all_connections.send_message(message)
+      end
+      
+
+      # Should be overridden.  Called when it is time to actually
+      # establish the connection
+      def connect!; raise NotImplementedError, "Raw ports do not know which direction to connect"; end
+
+      private
+
+      def all_connections
+        @all_connections ||= connections_for.map do |port_key, connections|
+          connections
+        end.flatten.uniq.extend(ConnectionCollection)
+      end
+      
+    end
+
+    
+    class InputPort < HashPort
+      def connect!
+        connections_for.each do |port_key, connections|
+          connections.each do |connection|
+            connection.connect_input!
+          end
+        end
+      end
+    end
+
+    
+    class OutputPort < HashPort
+      def connect!
+        connections_for.each do |port_key, keyed_connections|
+          keyed_connections.each do |connection|
+            connection.connect_output!
+          end
+        end
+      end
+    end
+
+    class DisconnectedPort < HashPort; end
+    
+  end
+end
+
+__END__
+
+out[even] -> a
+out[odd]  -> b
+out[nil]  -> c

data/lib/rflow/components.rb

@@ -0,0 +1,10 @@
+require 'rflow/component'
+
+class RFlow
+  module Components
+  end
+end
+
+require 'rflow/components/raw'
+
+

data/lib/rflow/components/raw.rb

@@ -0,0 +1,26 @@
+require 'rflow/components/raw/extensions'
+
+class RFlow
+  module Components
+    module Raw
+      
+      # Load the schemas
+      SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
+      
+      SCHEMA_FILES = {
+        'raw.avsc' => 'RFlow::Message::Data::Raw',
+      }
+      
+      SCHEMA_FILES.each do |file_name, data_type_name|
+        schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
+        RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
+      end
+
+      # Load the data extensions
+      RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Raw',
+                                                        RFlow::Components::Raw::Extensions::RawExtension)
+
+      
+    end
+  end
+end

data/lib/rflow/components/raw/extensions.rb

@@ -0,0 +1,18 @@
+class RFlow
+  module Components
+    module Raw
+      
+      module Extensions
+        
+        module RawExtension
+          def self.extended(base_data)
+            base_data.data_object ||= {'raw' => ''}
+          end
+
+          def raw; data_object['raw']; end
+          def raw=(new_raw); data_object['raw'] = new_raw; end
+        end
+      end
+    end
+  end
+end

data/lib/rflow/configuration.rb

@@ -0,0 +1,290 @@
+require 'rflow/util'
+
+class RFlow
+
+  # Contains all the configuration data and methods for RFlow.
+  # Interacts directly with underlying sqlite database, and keeps a
+  # registry of available data types, extensions, and components.
+  # Also includes an external DSL, RubyDSL, that can be used in
+  # crafting config-like files that load the database.
+  #
+  # Configuration provides a MVC-like framework for config files,
+  # where the models are the Setting, Component, Port, and Connection
+  # subclasses, the controllers are things like RubyDSL, and the views
+  # are defined relative to the controllers
+  class Configuration
+
+    # An exception class
+    class ConfigurationInvalid < StandardError; end
+
+    
+    # A class to hold DB config and connection information
+    class ConfigDB < ActiveRecord::Base
+        self.abstract_class = true
+    end
+    
+    
+    # A collection class for data extensions that supports a naive
+    # prefix-based 'inheritance' on lookup.  When looking up a key
+    # with [] all existing keys will be examined to determine if the
+    # existing key is a string prefix of the lookup key. All the
+    # results are consolidated into a single, flattened array.
+    class DataExtensionCollection
+
+      def initialize
+        # TODO: choose a different data structure ...
+        @hash = Hash.new {|hash, key| hash[key] = Array.new}
+      end
+
+      # Return an array of all of the values that have keys that are
+      # prefixes of the lookup key.
+      def [](key)
+        key_string = key.to_s
+        @hash.map do |data_type, extensions|
+          key_string.start_with?(data_type) ? extensions : nil
+        end.flatten.compact
+      end
+
+      # Adds a data extension for a given data type to the collection
+      def add(data_type, extension)
+        @hash[data_type.to_s] << extension
+      end
+
+      # Remove all elements from the collection.  Useful for testing,
+      # not much else
+      def clear
+        @hash.clear
+      end
+      
+    end
+
+    
+    class << self
+
+      # A collection of data types (schemas) indexed by their name and
+      # their schema type ('avro').
+      def available_data_types
+        @available_data_types ||= Hash.new {|hash, key| hash[key] = Hash.new}
+      end
+
+      # A DataExtensionCollection to hold available extensions that
+      # will be applied to the de-serialized data types
+      def available_data_extensions
+        @available_data_extensions ||= DataExtensionCollection.new
+      end
+
+      # A Hash of defined components, usually automatically populated
+      # when a component subclasses RFlow::Component
+      def available_components
+        @available_components ||= Hash.new
+      end
+    end
+
+    # TODO: refactor each of these add_available_* into collections to
+    # make DRYer.  Also figure out what to do with all to to_syms
+
+    # Add a schema to the available_data_types class attribute.
+    # Schema is indexed by data_type_name and schema/serialization
+    # type.  'avro' is currently the only supported
+    # data_serialization_type.
+    def self.add_available_data_type(data_type_name, data_serialization_type, data_schema)
+      unless data_serialization_type == 'avro'
+        error_message = "Data serialization_type must be 'avro' for '#{data_type_name}'"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+
+      if available_data_types[data_type_name.to_s].include? data_serialization_type.to_s
+        error_message = "Data type '#{data_type_name}' already defined for serialization_type '#{data_serialization_type}'"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+
+      available_data_types[data_type_name.to_s][data_serialization_type.to_s] = data_schema
+    end
+
+    # Add a data extension to the available_data_extensions class
+    # attributes.  The data_extension parameter should be the name of
+    # a ruby module that will extend RFlow::Message::Data object to
+    # provide additional methods/capability.  Naive, prefix-based
+    # inheritance is possible, see available_data_extensions or the
+    # DataExtensionCollection class
+    def self.add_available_data_extension(data_type_name, data_extension)
+      unless data_extension.is_a? Module
+        error_message = "Invalid data extension #{data_extension} for #{data_type_name}.  Only Ruby Modules allowed"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+
+      available_data_extensions.add data_type_name, data_extension
+    end
+
+    
+    # Used when RFlow::Component is subclassed to add another
+    # available component to the list.
+    def self.add_available_component(component)
+      if available_components.include?(component.name)
+        error_message = "Component already '#{component.name}' already defined"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+      available_components[component.name] = component
+    end
+
+
+    # Connect to the configuration sqlite database, but use the
+    # ConfigDB subclass to protect the connection information from
+    # other ActiveRecord apps (i.e. Rails)
+    def self.establish_config_database_connection(config_database_path)
+      RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
+      ActiveRecord::Base.logger = RFlow.logger
+      ConfigDB.establish_connection(:adapter => "sqlite3",
+                                    :database  => config_database_path)
+    end
+    
+    
+    # Using default ActiveRecord migrations, attempt to migrate the
+    # database to the latest version.
+    def self.migrate_database
+      RFlow.logger.debug "Applying default migrations to config database"
+      migrations_directory_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
+#      ActiveRecord::Migration.verbose = RFlow.logger
+      ActiveRecord::Migrator.migrate migrations_directory_path
+    end
+
+
+    # Load the config file, which should load/process/store all the
+    # elements.  Only run this after the database has been setup
+    def self.process_config_file(config_file_path)
+      RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{config_file_path}'"
+      load config_file_path
+    end
+
+    
+    # Connect to the configuration database, migrate it to the latest
+    # version, and process a config file if provided.
+    def self.initialize_database(config_database_path, config_file_path=nil)
+      RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{config_database_path}'"
+
+      RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
+      ActiveRecord::Base.logger = RFlow.logger
+      ActiveRecord::Base.establish_connection(:adapter => "sqlite3",
+                                              :database  => config_database_path)
+
+      migrate_database
+      
+      expanded_config_file_path = File.expand_path config_file_path if config_file_path
+
+      working_dir = Dir.getwd
+      Dir.chdir File.dirname(config_database_path)
+
+      if config_file_path
+        process_config_file(expanded_config_file_path)
+      end
+
+      RFlow.logger.debug "Defaulting non-existing config values"
+      merge_defaults! 
+
+      Dir.chdir working_dir
+    end
+
+    
+    # Make sure that the configuration has all the necessary values set.
+    def self.merge_defaults!
+      Setting::DEFAULTS.each do |name, default_value_or_proc|
+        setting = Setting.find_or_create_by_name(:name => name,
+                                                 :value => default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc)
+        unless setting.valid?
+          error_message = setting.errors.map do |attribute, error_string|
+            error_string
+          end.join ', '
+          raise RuntimeError, error_message
+        end
+      end
+    end
+    
+    
+    attr_accessor :config_database_path
+    attr_accessor :cached_settings
+    attr_accessor :cached_components
+    attr_accessor :cached_ports
+    attr_accessor :cached_connections
+
+
+    def initialize(config_database_path)
+      @cached_settings = Hash.new
+      @cached_components = Hash.new
+      @cached_ports = []
+      @cached_connections = []
+
+      @config_database_path = config_database_path
+      self.class.establish_config_database_connection(config_database_path)
+
+      # Validate the connected database.  TODO: make this more
+      # complete, i.e. validate the various columns
+      begin
+        Setting.first
+        Component.first
+        Port.first
+        Connection.first
+      rescue ActiveRecord::StatementInvalid => e
+        error_message = "Invalid schema in configuration database: #{e.message}"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+    end
+
+    
+    def to_s
+      string = "Configuration:\n"
+      settings.each do |setting|
+        string << "Setting: '#{setting.name}' = '#{setting.value}'\n"
+      end
+      components.each do |component|
+        string << "Component '#{component.name}' as #{component.specification} (#{component.uuid})\n"
+        component.output_ports.each do |output_port|
+          output_port.output_connections.each do |output_connection|
+            input_port = output_connection.input_port
+            string << "\tOutputPort '#{output_port.name}' key '#{output_connection.output_port_key}' (#{output_port.uuid}) =>\n"
+            string << "\t\tConnection '#{output_connection.name}' as #{output_connection.type} (#{output_connection.uuid}) =>\n"
+            string << "\t\tInputPort '#{input_port.name}' key '#{output_connection.input_port_key}' (#{input_port.uuid}) Component '#{input_port.component.name}' (#{input_port.component.uuid})\n"
+          end
+        end
+      end
+      string
+    end
+      
+    # Helper method to access settings with minimal syntax
+    def [](setting_name)
+      Setting.find_by_name(setting_name).value rescue nil
+    end
+
+    
+    def components
+      Component.all
+    end
+
+    
+    def component(component_instance_uuid)
+      Component.find_by_uuid component_instance_uuid
+    end
+    
+    
+    def settings
+      Setting.all
+    end
+
+    def available_components
+      self.class.available_components
+    end
+  end
+end
+
+# Load the models
+require 'rflow/configuration/setting'
+require 'rflow/configuration/component'
+require 'rflow/configuration/port'
+require 'rflow/configuration/connection'
+
+# Incorporate various config file processors
+require 'rflow/configuration/ruby_dsl'