rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow/component/port.rb

@@ -1,14 +1,11 @@
 class RFlow
   class Component
-
     # TODO: make this into a class to limit the amount of extensions
     # that we have to do when operating on these "Arrays", i.e. when
     # adding two together
     module ConnectionCollection
       def send_message(message)
-        each do |connection|
-          connection.send_message(message)
-        end
+        each {|connection| connection.send_message(message) }
       end
     end
 
@@ -18,9 +15,9 @@ class RFlow
       attr_reader :ports, :by_uuid, :by_name, :by_type
 
       def initialize
-        @ports = Array.new
-        @by_uuid = Hash.new
-        @by_name = Hash.new
+        @ports = []
+        @by_uuid = {}
+        @by_name = {}
         @by_type = Hash.new {|hash, key| hash[key.to_s] = []}
       end
 
@@ -32,24 +29,19 @@ class RFlow
         self
       end
 
-
       # Enumerate through each connected (or disconnected but
       # referenced) port
       # TODO: simplify with enumerators and procs
       def each
-        ports.each do |port|
-          yield port
-        end
+        ports.each {|port| yield port }
       end
     end
 
-
     class Port
       attr_reader :connected
-      def connected?; @connected; end
+      def connected?; connected; end
     end
 
-
     # Allows for a list of connections to be assigned to each port/key
     # combination.  Note that binding an input port to an un-indexed
     # output port will result in messages from all indexed connections
@@ -57,16 +49,19 @@ class RFlow
     # result in the same message being sent to all indexed
     # connections.
     class HashPort < Port
-      attr_reader :config, :name, :uuid, :connections_for
+      attr_reader :config, :name, :uuid
+
+      protected
+      attr_reader :connections_for
 
+      public
       def initialize(config)
         @config = config
         @name = config.name
         @uuid = config.uuid
-        @connections_for = Hash.new {|hash, key| hash[key] = Array.new.extend(ConnectionCollection)}
+        @connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
       end
 
-
       # Returns an extended Array of all the connections that should
       # be sent/received on this port.  Merges the nil-keyed port
       # (i.e. any connections for a port without a key) to those
@@ -74,78 +69,80 @@ class RFlow
       # connections, not to add new ones.  Use add_connection to add a
       # new connection for a given key.
       def [](key)
-         (connections_for[key] + connections_for[nil]).extend(ConnectionCollection)
+        case key
+        when nil; connections_for[nil]
+        else connections_for[key] + connections_for[nil]
+        end.extend(ConnectionCollection)
       end
 
-
       # Adds a connection for a given key
       def add_connection(key, connection)
         connections_for[key] << connection
       end
 
-
       # Return a list of connected keys
       def keys
         connections_for.keys
       end
 
-
       # Enumerate through all the ConnectionCollections
       # TODO: simplify with enumerators and procs
       def each
-        connections_for.values.each do |connections|
-          yield connections
-        end
+        connections_for.values.each {|connections| yield connections }
       end
 
-
-      # Send a message to all connections on all keys for this port,
-      # but only once per connection.
       def send_message(message)
-        all_connections.send_message(message)
+        def connect!; raise NotImplementedError, "Raw ports do not know how to send messages"; end
       end
 
-
       # Should be overridden.  Called when it is time to actually
       # establish the connection
       def connect!; raise NotImplementedError, "Raw ports do not know which direction to connect"; end
 
       private
-
       def all_connections
-        @all_connections ||= connections_for.map do |port_key, connections|
-          connections
-        end.flatten.uniq.extend(ConnectionCollection)
+        @all_connections ||= connections_for.values.flatten.uniq.extend(ConnectionCollection)
       end
-
     end
 
-
     class InputPort < HashPort
       def connect!
-        connections_for.each do |port_key, connections|
+        connections_for.each do |key, connections|
           connections.each do |connection|
             connection.connect_input!
             @connected = true
           end
         end
       end
-    end
 
+      def recv_callback=(callback)
+        connections_for.each do |key, connections|
+          connections.each do |connection|
+            connection.recv_callback = Proc.new do |message|
+              callback.call self, key, connection, message
+            end
+          end
+        end
+      end
+    end
 
     class OutputPort < HashPort
       def connect!
-        connections_for.each do |port_key, keyed_connections|
-          keyed_connections.each do |connection|
+        connections_for.each do |key, connections|
+          connections.each do |connection|
             connection.connect_output!
             @connected = true
           end
         end
       end
-    end
 
+      # Send a message to all connections on all keys for this port,
+      # but only once per connection.
+      def send_message(message)
+        all_connections.send_message(message)
+      end
+    end
 
     class DisconnectedPort < HashPort; end
-
   end
 end

data/lib/rflow/components.rb

@@ -1,10 +1,6 @@
 require 'rflow/component'
-
-class RFlow
-  module Components
-  end
-end
-
+require 'rflow/components/clock'
+require 'rflow/components/integer'
 require 'rflow/components/raw'
-
-
+require 'rflow/components/replicate'
+require 'rflow/components/ruby_proc_filter'

data/lib/rflow/components/clock.rb

@@ -0,0 +1,49 @@
+class RFlow
+  module Components
+    class Clock < Component
+      module Tick
+        SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
+        SCHEMA_FILES = {'tick.avsc' => 'RFlow::Message::Clock::Tick'}
+        SCHEMA_FILES.each do |file_name, data_type_name|
+          schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
+          RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
+        end
+        module Extension
+          def self.extended(base_data); base_data.data_object ||= {}; end
+          def name; data_object['name']; end
+          def name=(new_name); data_object['name'] = new_name; end
+          def timestamp; data_object['timestamp']; end
+          def timestamp=(new_ts); data_object['timestamp'] = new_ts; end
+        end
+        RFlow::Configuration.add_available_data_extension('RFlow::Message::Clock::Tick', Extension)
+      end
+
+      output_port :tick_port
+
+      DEFAULT_CONFIG = {
+        'name' => 'Clock',
+        'tick_interval' => 1
+      }
+
+      attr_reader :config, :tick_interval
+
+      def configure!(config)
+        @config = DEFAULT_CONFIG.merge config
+        @tick_interval = Float(@config['tick_interval'])
+      end
+
+      def clock_name; config['name']; end
+
+      def run!
+        @timer = EventMachine::PeriodicTimer.new(tick_interval) { tick }
+      end
+
+      def tick
+        tick_port.send_message(RFlow::Message.new('RFlow::Message::Clock::Tick').tap do |m|
+          m.data.name = clock_name
+          m.data.timestamp = Integer(Time.now.to_f * 1000) # ms since epoch
+        end)
+      end
+    end
+  end
+end

data/lib/rflow/components/integer.rb

@@ -0,0 +1,39 @@
+class RFlow
+  module Components
+    Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', '{"type": "long"}')
+
+    class GenerateIntegerSequence < Component
+      output_port :out
+      output_port :even_odd_out
+
+      def configure!(config)
+        @start = config['start'].to_i
+        @finish = config['finish'].to_i
+        @step = config['step'] ? config['step'].to_i : 1
+        # If interval seconds is not given, it will default to 0
+        @interval_seconds = config['interval_seconds'].to_i
+      end
+
+      # Note that this uses the timer (sometimes with 0 interval) so as
+      # not to block the reactor
+      def run!
+        @timer = EM::PeriodicTimer.new(@interval_seconds) { generate }
+      end
+
+      def generate
+        Message.new('RFlow::Message::Data::Integer').tap do |m|
+          m.data.data_object = @start
+          out.send_message m
+          if @start % 2 == 0
+            even_odd_out['even'].send_message m
+          else
+            even_odd_out['odd'].send_message m
+          end
+        end
+
+        @start += @step
+        @timer.cancel if @start > @finish && @timer
+      end
+    end
+  end
+end

data/lib/rflow/components/raw.rb

@@ -1,10 +1,17 @@
-require 'rflow/components/raw/extensions'
-
 class RFlow
   module Components
     module Raw
+      module Extensions
+        module RawExtension
+          def self.extended(base_data)
+            base_data.data_object ||= {'raw' => ''}
+          end
+
+          def raw; data_object['raw']; end
+          def raw=(new_raw); data_object['raw'] = new_raw; end
+        end
+      end
 
-      # Load the schemas
       SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
 
       SCHEMA_FILES = {
@@ -16,11 +23,8 @@ class RFlow
         RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
       end
 
-      # Load the data extensions
       RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Raw',
                                                         RFlow::Components::Raw::Extensions::RawExtension)
-
-
     end
   end
 end

data/lib/rflow/components/replicate.rb

@@ -0,0 +1,20 @@
+class RFlow
+  module Components
+    class Replicate < Component
+      input_port :in
+      output_port :out
+      output_port :errored
+
+      def process_message(input_port, input_port_key, connection, message)
+        out.each do |connections|
+          begin
+            connections.send_message message
+          rescue Exception => e
+            RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
+            errored.send_message message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rflow/components/ruby_proc_filter.rb

@@ -0,0 +1,27 @@
+class RFlow
+  module Components
+    class RubyProcFilter < Component
+      input_port :in
+      output_port :filtered
+      output_port :dropped
+      output_port :errored
+
+      def configure!(config)
+        @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
+      end
+
+      def process_message(input_port, input_port_key, connection, message)
+        begin
+          if @filter_proc.call(message)
+            filtered.send_message message
+          else
+            dropped.send_message message
+          end
+        rescue Exception => e
+          RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
+          errored.send_message message
+        end
+      end
+    end
+  end
+end

data/lib/rflow/configuration.rb

@@ -1,7 +1,4 @@
-require 'rflow/util'
-
 class RFlow
-
   # Contains all the configuration data and methods for RFlow.
   # Interacts directly with underlying sqlite database, and keeps a
   # registry of available data types, extensions, and components.
@@ -13,58 +10,47 @@ class RFlow
   # subclasses, the controllers are things like RubyDSL, and the views
   # are defined relative to the controllers
   class Configuration
-
-    # An exception class
-    class ConfigurationInvalid < StandardError; end
-
-
-    # A class to hold DB config and connection information
-    class ConfigDB < ActiveRecord::Base
-        self.abstract_class = true
-    end
-
-
     # A collection class for data extensions that supports a naive
     # prefix-based 'inheritance' on lookup.  When looking up a key
     # with [] all existing keys will be examined to determine if the
     # existing key is a string prefix of the lookup key. All the
     # results are consolidated into a single, flattened array.
     class DataExtensionCollection
-
       def initialize
         # TODO: choose a different data structure ...
-        @hash = Hash.new {|hash, key| hash[key] = Array.new}
+        @extensions_for = Hash.new {|hash, key| hash[key] = []}
       end
 
       # Return an array of all of the values that have keys that are
       # prefixes of the lookup key.
       def [](key)
-        key_string = key.to_s
-        @hash.map do |data_type, extensions|
-          key_string.start_with?(data_type) ? extensions : nil
-        end.flatten.compact
+        @extensions_for.
+          find_all {|data_type, _| key.to_s.start_with?(data_type) }.
+          flat_map {|_, extensions| extensions }
       end
 
       # Adds a data extension for a given data type to the collection
       def add(data_type, extension)
-        @hash[data_type.to_s] << extension
+        @extensions_for[data_type.to_s] << extension
       end
 
       # Remove all elements from the collection.  Useful for testing,
       # not much else
       def clear
-        @hash.clear
+        @extensions_for.clear
       end
-
     end
 
+    # Base class for persisted RFlow configuration items.
+    class ConfigurationItem < ActiveRecord::Base
+      self.abstract_class = true
+    end
 
     class << self
-
       # A collection of data types (schemas) indexed by their name and
       # their schema type ('avro').
       def available_data_types
-        @available_data_types ||= Hash.new {|hash, key| hash[key] = Hash.new}
+        @available_data_types ||= Hash.new {|hash, key| hash[key] = {}}
       end
 
       # A DataExtensionCollection to hold available extensions that
@@ -76,172 +62,128 @@ class RFlow
       # A Hash of defined components, usually automatically populated
       # when a component subclasses RFlow::Component
       def available_components
-        @available_components ||= Hash.new
+        @available_components ||= {}
       end
-    end
 
-    # TODO: refactor each of these add_available_* into collections to
-    # make DRYer.  Also figure out what to do with all to to_syms
-
-    # Add a schema to the available_data_types class attribute.
-    # Schema is indexed by data_type_name and schema/serialization
-    # type.  'avro' is currently the only supported
-    # data_serialization_type.
-    def self.add_available_data_type(data_type_name, data_serialization_type, data_schema)
-      unless data_serialization_type == 'avro'
-        error_message = "Data serialization_type must be 'avro' for '#{data_type_name}'"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
-      end
+      # TODO: refactor each of these add_available_* into collections to
+      # make DRYer.  Also figure out what to do with all to to_syms
 
-      if available_data_types[data_type_name.to_s].include? data_serialization_type.to_s
-        error_message = "Data type '#{data_type_name}' already defined for serialization_type '#{data_serialization_type}'"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
-      end
+      # Add a schema to the available_data_types class attribute.
+      # Schema is indexed by data_type_name and schema/serialization
+      # type.  'avro' is currently the only supported serialization_type.
+      def add_available_data_type(name, serialization_type, schema)
+        raise ArgumentError, "Data serialization_type must be 'avro' for '#{name}'" unless serialization_type == 'avro'
 
-      available_data_types[data_type_name.to_s][data_serialization_type.to_s] = data_schema
-    end
+        if available_data_types[name.to_s].include? serialization_type.to_s
+          raise ArgumentError, "Data type '#{name}' already defined for serialization_type '#{serialization_type}'"
+        end
 
-    # Add a data extension to the available_data_extensions class
-    # attributes.  The data_extension parameter should be the name of
-    # a ruby module that will extend RFlow::Message::Data object to
-    # provide additional methods/capability.  Naive, prefix-based
-    # inheritance is possible, see available_data_extensions or the
-    # DataExtensionCollection class
-    def self.add_available_data_extension(data_type_name, data_extension)
-      unless data_extension.is_a? Module
-        error_message = "Invalid data extension #{data_extension} for #{data_type_name}.  Only Ruby Modules allowed"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
+        available_data_types[name.to_s][serialization_type.to_s] = schema
       end
 
-      available_data_extensions.add data_type_name, data_extension
-    end
-
+      # Add a data extension to the available_data_extensions class
+      # attributes.  The data_extension parameter should be the name of
+      # a ruby module that will extend RFlow::Message::Data object to
+      # provide additional methods/capability.  Naive, prefix-based
+      # inheritance is possible, see available_data_extensions or the
+      # DataExtensionCollection class
+      def add_available_data_extension(data_type_name, extension)
+        unless extension.is_a? Module
+          raise ArgumentError, "Invalid data extension #{extension} for #{data_type_name}.  Only Ruby Modules allowed"
+        end
 
-    # Used when RFlow::Component is subclassed to add another
-    # available component to the list.
-    def self.add_available_component(component)
-      if available_components.include?(component.name)
-        error_message = "Component already '#{component.name}' already defined"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
+        available_data_extensions.add data_type_name, extension
       end
-      available_components[component.name] = component
-    end
 
+      # Used when RFlow::Component is subclassed to add another
+      # available component to the list.
+      def add_available_component(component)
+        if available_components.include?(component.name)
+          raise ArgumentError, "Component already '#{component.name}' already defined"
+        end
+        available_components[component.name] = component
+      end
 
-    # Connect to the configuration sqlite database, but use the
-    # ConfigDB subclass to protect the connection information from
-    # other ActiveRecord apps (i.e. Rails)
-    def self.establish_config_database_connection(config_database_path)
-      RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
-      ActiveRecord::Base.logger = RFlow.logger
-      ConfigDB.establish_connection(:adapter => "sqlite3",
-                                    :database  => config_database_path)
-    end
-
+      # Connect to the configuration sqlite database, but use the
+      # ConfigurationItem class to protect the connection information from
+      # other ActiveRecord apps (i.e. Rails)
+      def establish_config_database_connection(database_path)
+        RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{database_path}'"
+        ActiveRecord::Base.logger = RFlow.logger
+        ConfigurationItem.establish_connection(:adapter => "sqlite3", :database => database_path)
+      end
 
-    # Using default ActiveRecord migrations, attempt to migrate the
-    # database to the latest version.
-    def self.migrate_database
-      RFlow.logger.debug "Applying default migrations to config database"
-      migrations_directory_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
-#      ActiveRecord::Migration.verbose = RFlow.logger
-      ActiveRecord::Migrator.migrate migrations_directory_path
-    end
+      # Using default ActiveRecord migrations, attempt to migrate the
+      # database to the latest version.
+      def migrate_database
+        RFlow.logger.debug "Applying default migrations to config database"
+        migrations_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
+        ActiveRecord::Migration.verbose = false
+        ActiveRecord::Migrator.migrate migrations_path
+      end
 
+      # Load the config file, which should load/process/store all the
+      # elements.  Only run this after the database has been setup
+      def process_config_file(path)
+        RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{path}'"
+        load path
+      end
 
-    # Load the config file, which should load/process/store all the
-    # elements.  Only run this after the database has been setup
-    def self.process_config_file(config_file_path)
-      RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{config_file_path}'"
-      load config_file_path
-    end
+      # Connect to the configuration database, migrate it to the latest
+      # version, and process a config file if provided.
+      def initialize_database(database_path, config_file_path = nil)
+        RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{database_path}'"
 
+        # TODO should not need this line
+        ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => database_path)
 
-    # Connect to the configuration database, migrate it to the latest
-    # version, and process a config file if provided.
-    def self.initialize_database(config_database_path, config_file_path=nil)
-      RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{config_database_path}'"
+        establish_config_database_connection database_path
+        migrate_database
 
-      RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
-      ActiveRecord::Base.logger = RFlow.logger
-      ActiveRecord::Base.establish_connection(:adapter => "sqlite3",
-                                              :database  => config_database_path)
+        working_dir = Dir.getwd
+        Dir.chdir File.dirname(database_path)
 
-      migrate_database
+        if config_file_path
+          process_config_file File.expand_path(config_file_path)
+        end
 
-      expanded_config_file_path = File.expand_path config_file_path if config_file_path
+        RFlow.logger.debug "Defaulting non-existing config values"
+        merge_defaults!
 
-      working_dir = Dir.getwd
-      Dir.chdir File.dirname(config_database_path)
+        Dir.chdir working_dir
 
-      if config_file_path
-        process_config_file(expanded_config_file_path)
+        self.new(database_path)
       end
 
-      RFlow.logger.debug "Defaulting non-existing config values"
-      merge_defaults!
-
-      Dir.chdir working_dir
-
-      self.new(config_database_path)
-    end
-
-
-    # Make sure that the configuration has all the necessary values set.
-    def self.merge_defaults!
-      Setting::DEFAULTS.each do |name, default_value_or_proc|
-        setting = Setting.find_or_create_by_name(:name => name,
-                                                 :value => default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc)
-        unless setting.valid?
-          error_message = setting.errors.map do |attribute, error_string|
-            error_string
-          end.join ', '
-          raise RuntimeError, error_message
+      # Make sure that the configuration has all the necessary values set.
+      def merge_defaults!
+        Setting::DEFAULTS.each do |name, default_value_or_proc|
+          value = default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc
+          setting = Setting.find_or_create_by_name(:name => name, :value => value)
+          unless setting.valid?
+            raise RuntimeError, setting.errors.map {|_, msg| msg }.join(', ')
+          end
         end
       end
     end
 
-
-    attr_accessor :config_database_path
-    attr_accessor :cached_settings
-    attr_accessor :cached_components
-    attr_accessor :cached_ports
-    attr_accessor :cached_connections
-
-
-    def initialize(config_database_path=nil)
-      @cached_settings = Hash.new
-      @cached_components = Hash.new
-      @cached_ports = []
-      @cached_connections = []
-
+    def initialize(database_path = nil)
       # If there is not a config DB path, assume that an AR
-      # conncection has already been established
-      if config_database_path
-        @config_database_path = config_database_path
-        self.class.establish_config_database_connection(config_database_path)
+      # connection has already been established
+      if database_path
+        @database_path = database_path
+        Configuration.establish_config_database_connection(database_path)
       end
 
-      # Validate the connected database.  TODO: make this more
-      # complete, i.e. validate the various columns
+      # Validate the connected database.
+      # TODO: make this more complete, i.e. validate the various columns
       begin
-        Setting.first
-        Shard.first
-        Component.first
-        Port.first
-        Connection.first
+        [Setting, Shard, Component, Port, Connection].each(&:first)
       rescue ActiveRecord::StatementInvalid => e
-        error_message = "Invalid schema in configuration database: #{e.message}"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
+        raise ArgumentError, "Invalid schema in configuration database: #{e.message}"
       end
     end
 
-
     def to_s
       string = "Configuration:\n"
 
@@ -266,34 +208,13 @@ class RFlow
       string
     end
 
-    # Helper method to access settings with minimal syntax
-    def [](setting_name)
-      Setting.find_by_name(setting_name).value rescue nil
-    end
-
-    def settings
-      Setting.all
-    end
-
-    def shards
-      Shard.all
-    end
-
-    def shard(shard_uuid)
-      Shard.find_by_uuid shard_uuid
-    end
-
-    def components
-      Component.all
-    end
-
-    def component(component_uuid)
-      Component.find_by_uuid component_uuid
-    end
-
-    def available_components
-      self.class.available_components
-    end
+    def [](name); Setting.find_by_name(name).value rescue nil; end
+    def settings; Setting.all; end
+    def shards; Shard.all; end
+    def shard(uuid); Shard.find_by_uuid uuid; end
+    def components; Component.all; end
+    def component(uuid); Component.find_by_uuid uuid; end
+    def available_components; Configuration.available_components; end
   end
 end