rflow 1.3.0 → 1.3.1

data/lib/rflow/components/clock.rb

@@ -1,43 +1,94 @@
 class RFlow
+  # @!parse
+  #   class Message
+  #     # Clock messages.
+  #     module Clock
+  #       # Message emitted by {RFlow::Components::Clock}. Of course the real class is {RFlow::Message}
+  #       # with type +RFlow::Message::Clock::Tick+.
+  #       class Tick
+  #         # @!attribute name
+  #         #   The name of the clock.
+  #         #   @return [String]
+  #
+  #         # @!attribute timestamp
+  #         #   The timestamp of the tick, in milliseconds from epoch.
+  #         #   @return [Integer]
+  #
+  #         # Just here to force Yard to create documentation.
+  #         # @!visibility private
+  #         def initialize; end
+  #       end
+  #     end
+  #   end
+
+  # Components.
   module Components
+    # A clock. It ticks every _n_ seconds. Get it?
+    #
+    # Accepts config parameters:
+    # - +name+ - name of the clock, to disambiguate more than one
+    # - +tick_interval+ - how long to wait between ticks
+    #
+    # Emits {RFlow::Message}s whose internal type is {RFlow::Message::Clock::Tick}.
     class Clock < Component
+      # @!visibility private
       module Tick
+        # @!visibility private
         SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
+        # @!visibility private
         SCHEMA_FILES = {'tick.avsc' => 'RFlow::Message::Clock::Tick'}
         SCHEMA_FILES.each do |file_name, data_type_name|
           schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
           RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
         end
+        # @!visibility private
         module Extension
+          # @!visibility private
           def self.extended(base_data); base_data.data_object ||= {}; end
+          # @!visibility private
           def name; data_object['name']; end
+          # @!visibility private
           def name=(new_name); data_object['name'] = new_name; end
+          # @!visibility private
           def timestamp; data_object['timestamp']; end
+          # @!visibility private
           def timestamp=(new_ts); data_object['timestamp'] = new_ts; end
         end
         RFlow::Configuration.add_available_data_extension('RFlow::Message::Clock::Tick', Extension)
       end
 
+      # @!attribute [r] tick_port
+      #   Outputs {RFlow::Message::Clock::Tick} messages.
+      #   @return [Component::OutputPort]
       output_port :tick_port
 
+      # Default configuration.
       DEFAULT_CONFIG = {
         'name' => 'Clock',
         'tick_interval' => 1
       }
 
+      # @!visibility private
       attr_reader :config, :tick_interval
 
+      # RFlow-called method at startup.
+      # @param config [Hash] configuration from the RFlow config file
+      # @return [void]
       def configure!(config)
         @config = DEFAULT_CONFIG.merge config
         @tick_interval = Float(@config['tick_interval'])
       end
 
+      # @!visibility private
       def clock_name; config['name']; end
 
+      # RFlow-called method at startup.
+      # @return [void]
       def run!
         @timer = EventMachine::PeriodicTimer.new(tick_interval) { tick }
       end
 
+      # @!visibility private
       def tick
         tick_port.send_message(RFlow::Message.new('RFlow::Message::Clock::Tick').tap do |m|
           m.data.name = clock_name

data/lib/rflow/components/integer.rb

@@ -1,11 +1,46 @@
 class RFlow
+  # @!parse
+  #   class Message
+  #     class Data
+  #       # Message emitted by {RFlow::Components::GenerateIntegerSequence}.
+  #       # Of course the real class is {RFlow::Message} with type +RFlow::Message::Data::Integer+.
+  #       #
+  #       # {RFlow::Message::Data#data_object} will return the integer.
+  #       class Integer
+  #         # Just here to force Yard to create documentation.
+  #         # @!visibility private
+  #         def initialize; end
+  #       end
+  #     end
+  #   end
+
+  # Components.
   module Components
     Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', '{"type": "long"}')
 
+    # An integer sequence generator that ticks every _n_ seconds.
+    #
+    # Accepts config parameters:
+    # - +start+ - the number to start at (defaults to +0+)
+    # - +finish+ - the number to finish at (defaults to +0+; no numbers greater than this will be emitted)
+    # - +step+ - the number to step (defaults to +1+)
+    # - +interval_seconds+ - how long to wait, in seconds, between ticks (defaults to +0+)
+    #
+    # Emits {RFlow::Message}s whose internal type is {RFlow::Message::Data::Integer}.
     class GenerateIntegerSequence < Component
+      # @!attribute [r] out
+      #   Outputs {RFlow::Message::Data::Integer} messages.
+      #   @return [Component::OutputPort]
       output_port :out
+      # @!attribute [r] even_odd_out
+      #   Outputs the same messages as {out}. Also addressable with subports +even+ and +odd+
+      #   to select those subsequences.
+      #   @return [Component::OutputPort]
       output_port :even_odd_out
 
+      # RFlow-called method at startup.
+      # @param config [Hash] configuration from the RFlow config file
+      # @return [void]
       def configure!(config)
         @start = config['start'].to_i
         @finish = config['finish'].to_i
@@ -14,12 +49,15 @@ class RFlow
         @interval_seconds = config['interval_seconds'].to_i
       end
 
-      # Note that this uses the timer (sometimes with 0 interval) so as
-      # not to block the reactor
+      # RFlow-called method at startup.
+      # @return [void]
       def run!
+        # Note that this uses the timer (sometimes with 0 interval) so as
+        # not to block the reactor.
         @timer = EM::PeriodicTimer.new(@interval_seconds) { generate }
       end
 
+      # @!visibility private
       def generate
         Message.new('RFlow::Message::Data::Integer').tap do |m|
           m.data.data_object = @start

data/lib/rflow/components/log.rb

@@ -1,23 +1,62 @@
 class RFlow
+  # @!parse
+  #   class Message
+  #     class Data
+  #       # RFlow format defined for log messages which can be emitted by components.
+  #       # Of course the real class is {RFlow::Message}
+  #       # with type +RFlow::Message::Data::Log+.
+  #       class Log
+  #         # @!attribute timestamp
+  #         #   The timestamp of the log, in ms since epoch.
+  #         #   @return [Integer]
+  #
+  #         # @!attribute level
+  #         #   The log level (INFO, WARN, ERROR, etc.).
+  #         #   @return [String]
+  #
+  #         # @!attribute text
+  #         #   The text of the log message.
+  #         #   @return [String]
+  #
+  #         # Just here to force Yard to create documentation.
+  #         # @!visibility private
+  #         def initialize; end
+  #       end
+  #     end
+  #   end
+
+  # Components.
   module Components
+    # @!visibility private
     module Log
+      # @!visibility private
       module Extensions
+        # @!visibility private
         module LogExtension
+          # @!visibility private
           def self.extended(base_data)
             base_data.data_object ||= {'timestamp' => 0, 'level' => 'INFO', 'text' => ''}
           end
 
+          # @!visibility private
           def timestamp; data_object['timestamp']; end
+          # @!visibility private
           def timestamp=(new_timestamp); data_object['timestamp'] = new_timestamp; end
+          # @!visibility private
           def level; data_object['level']; end
+          # @!visibility private
           def level=(new_level); data_object['level'] = new_level; end
+          # @!visibility private
           def text; data_object['text']; end
+          # @!visibility private
           def text=(new_text); data_object['text'] = new_text; end
         end
       end
 
+      # @!visibility private
       SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
 
+      # @!visibility private
       SCHEMA_FILES = {
         'log.avsc' => 'RFlow::Message::Data::Log',
       }

data/lib/rflow/components/raw.rb

@@ -1,19 +1,46 @@
 class RFlow
+  # @!parse
+  #   class Message
+  #     class Data
+  #       # RFlow format defined for raw-data messages which can be emitted by components.
+  #       # Of course the real class is {RFlow::Message}
+  #       # with type +RFlow::Message::Data::Raw+.
+  #       class Raw
+  #         # @!attribute raw
+  #         #   The raw data.
+  #         #   @return [String]
+  #
+  #         # Just here to force Yard to create documentation.
+  #         # @!visibility private
+  #         def initialize; end
+  #       end
+  #     end
+  #   end
+
+  # Components.
   module Components
+    # @!visibility private
     module Raw
+      # @!visibility private
       module Extensions
+        # @!visibility private
         module RawExtension
+          # @!visibility private
           def self.extended(base_data)
             base_data.data_object ||= {'raw' => ''}
           end
 
+          # @!visibility private
           def raw; data_object['raw']; end
+          # @!visibility private
           def raw=(new_raw); data_object['raw'] = new_raw; end
         end
       end
 
+      # @!visibility private
       SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
 
+      # @!visibility private
       SCHEMA_FILES = {
         'raw.avsc' => 'RFlow::Message::Data::Raw',
       }

data/lib/rflow/components/replicate.rb

@@ -1,10 +1,29 @@
 class RFlow
+  # Components.
   module Components
+    # A component that replicates all inbound messages onto a single out port in
+    # order to easily support a many-to-many connection pattern (connect all the
+    # ins to this component and all the outs to this component instead of
+    # all of the ins to all of the outs).
+    #
+    # Emits {RFlow::Message}s of whatever type was sent in. Any messages with
+    # problems being sent to {out} will be sent to {errored} instead.
     class Replicate < Component
+      # @!attribute [r] in
+      #   Receives {RFlow::Message}s.
+      #   @return [Component::InputPort]
       input_port :in
+      # @!attribute [r] out
+      #   Outputs {RFlow::Message}s.
+      #   @return [Component::OutputPort]
       output_port :out
+      # @!attribute [r] errored
+      #   Outputs {RFlow::Messages}s that could not be sent to {errored}.
+      #   @return [Component::OutputPort]
       output_port :errored
 
+      # RFlow-called method on message arrival.
+      # @return [void]
       def process_message(input_port, input_port_key, connection, message)
         out.each do |connections|
           begin

data/lib/rflow/components/ruby_proc_filter.rb

@@ -1,15 +1,39 @@
 class RFlow
+  # Components.
   module Components
+    # Component that filters messages based on Ruby defined in the RFlow config file.
+    # Inbound messages will be sent out {filtered} if the predicate returns truthy,
+    # {dropped} if it returns falsey, or {errored} if it raises an exception.
+    #
+    # Accept config parameter +filter_proc_string+ which is the text of a +lambda+
+    # receiving a message +message+. For example, +message.data.data_object['foo'] > 2+.
     class RubyProcFilter < Component
+      # @!attribute [r] in
+      #   Receives {RFlow::Message}s.
+      #   @return [Component::InputPort]
       input_port :in
+      # @!attribute [r] filtered
+      #   Outputs {RFlow::Message}s that pass the filter predicate.
+      #   @return [Component::OutputPort]
       output_port :filtered
+      # @!attribute [r] dropped
+      #   Outputs {RFlow::Message}s that do not pass the filter predicate.
+      #   @return [Component::OutputPort]
       output_port :dropped
+      # @!attribute [r] errored
+      #   Outputs {RFlow::Message}s that raise from the filter predicate.
+      #   @return [Component::OutputPort]
       output_port :errored
 
+      # RFlow-called method at startup.
+      # @param config [Hash] configuration from the RFlow config file
+      # @return [void]
       def configure!(config)
         @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
       end
 
+      # RFlow-called method on message arrival.
+      # @return [void]
       def process_message(input_port, input_port_key, connection, message)
         begin
           if @filter_proc.call(message)

data/lib/rflow/configuration.rb

@@ -1,18 +1,18 @@
 class RFlow
   # Contains all the configuration data and methods for RFlow.
-  # Interacts directly with underlying sqlite database, and keeps a
+  # Interacts directly with underlying SQLite database, and keeps a
   # registry of available data types, extensions, and components.
   # Also includes an external DSL, RubyDSL, that can be used in
   # crafting config-like files that load the database.
   #
-  # Configuration provides a MVC-like framework for config files,
-  # where the models are the Setting, Component, Port, and Connection
+  # {Configuration} provides a MVC-like framework for config files,
+  # where the models are the {Setting}, {Component}, {Port}, and {Connection}
   # subclasses, the controllers are things like RubyDSL, and the views
-  # are defined relative to the controllers
+  # are defined relative to the controllers.
   class Configuration
     # A collection class for data extensions that supports a naive
     # prefix-based 'inheritance' on lookup.  When looking up a key
-    # with [] all existing keys will be examined to determine if the
+    # with {[]} all existing keys will be examined to determine if the
     # existing key is a string prefix of the lookup key. All the
     # results are consolidated into a single, flattened array.
     class DataExtensionCollection
@@ -23,6 +23,7 @@ class RFlow
 
       # Return an array of all of the values that have keys that are
       # prefixes of the lookup key.
+      # @return [Array]
       def [](key)
         @extensions_for.
           find_all {|data_type, _| key.to_s.start_with?(data_type) }.
@@ -30,12 +31,14 @@ class RFlow
       end
 
       # Adds a data extension for a given data type to the collection
+      # @return [void]
       def add(data_type, extension)
         @extensions_for[data_type.to_s] << extension
       end
 
       # Remove all elements from the collection.  Useful for testing,
       # not much else
+      # @return [void]
       def clear
         @extensions_for.clear
       end
@@ -49,29 +52,32 @@ class RFlow
     class << self
       # A collection of data types (schemas) indexed by their name and
       # their schema type ('avro').
+      # @return [Hash]
       def available_data_types
         @available_data_types ||= Hash.new {|hash, key| hash[key] = {}}
       end
 
-      # A DataExtensionCollection to hold available extensions that
-      # will be applied to the de-serialized data types
+      # A {DataExtensionCollection} to hold available extensions that
+      # will be applied to the de-serialized data types.
+      # @return [DataExtensionCollection]
       def available_data_extensions
         @available_data_extensions ||= DataExtensionCollection.new
       end
 
       # A Hash of defined components, usually automatically populated
-      # when a component subclasses RFlow::Component
+      # when a component subclasses {RFlow::Component}.
+      # @return [Hash]
       def available_components
         @available_components ||= {}
       end
 
-      # TODO: refactor each of these add_available_* into collections to
-      # make DRYer.  Also figure out what to do with all to to_syms
-
-      # Add a schema to the available_data_types class attribute.
-      # Schema is indexed by data_type_name and schema/serialization
-      # type.  'avro' is currently the only supported serialization_type.
+      # Add a schema to the {available_data_types} class attribute.
+      # Schema is indexed by +name+ and +serialization_type+.
+      # +avro+ is currently the only supported +serialization_type+.
+      # @return [void]
       def add_available_data_type(name, serialization_type, schema)
+        # TODO: refactor each of these add_available_* into collections to
+        # make DRYer.  Also figure out what to do with all to to_syms
         raise ArgumentError, "Data serialization_type must be 'avro' for '#{name}'" unless serialization_type == 'avro'
 
         if available_data_types[name.to_s].include? serialization_type.to_s
@@ -81,12 +87,13 @@ class RFlow
         available_data_types[name.to_s][serialization_type.to_s] = schema
       end
 
-      # Add a data extension to the available_data_extensions class
-      # attributes.  The data_extension parameter should be the name of
-      # a ruby module that will extend RFlow::Message::Data object to
-      # provide additional methods/capability.  Naive, prefix-based
-      # inheritance is possible, see available_data_extensions or the
-      # DataExtensionCollection class
+      # Add a data extension to the {available_data_extensions} class
+      # attribute. The +extension+ parameter should be the name of
+      # a ruby module that will extend {RFlow::Message::Data} to
+      # provide additional methods/capability. Naive, prefix-based
+      # inheritance is possible, see {available_data_extensions} or
+      # {DataExtensionCollection}.
+      # @return [void]
       def add_available_data_extension(data_type_name, extension)
         unless extension.is_a? Module
           raise ArgumentError, "Invalid data extension #{extension} for #{data_type_name}.  Only Ruby Modules allowed"
@@ -95,8 +102,9 @@ class RFlow
         available_data_extensions.add data_type_name, extension
       end
 
-      # Used when RFlow::Component is subclassed to add another
+      # Used when {RFlow::Component} is subclassed to add another
       # available component to the list.
+      # @return [void]
       def add_available_component(component)
         if available_components.include?(component.name)
           raise ArgumentError, "Component already '#{component.name}' already defined"
@@ -104,9 +112,10 @@ class RFlow
         available_components[component.name] = component
       end
 
-      # Connect to the configuration sqlite database, but use the
-      # ConfigurationItem class to protect the connection information from
-      # other ActiveRecord apps (i.e. Rails)
+      # Connect to the configuration SQLite database, but use
+      # {ConfigurationItem} to protect the connection information from
+      # other ActiveRecord apps (i.e. Rails).
+      # @return [void]
       def establish_config_database_connection(database_path)
         RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{database_path}'"
         ActiveRecord::Base.logger = RFlow.logger
@@ -115,6 +124,7 @@ class RFlow
 
       # Using default ActiveRecord migrations, attempt to migrate the
       # database to the latest version.
+      # @return [void]
       def migrate_database
         RFlow.logger.debug 'Applying default migrations to config database'
         migrations_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
@@ -123,7 +133,8 @@ class RFlow
       end
 
       # Load the config file, which should load/process/store all the
-      # elements.  Only run this after the database has been setup
+      # elements. Only run this after the database has been setup
+      # @return [void]
       def process_config_file(path)
         RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{path}'"
         load path
@@ -131,6 +142,7 @@ class RFlow
 
       # Connect to the configuration database, migrate it to the latest
       # version, and process a config file if provided.
+      # @return [void]
       def initialize_database(database_path, config_file_path = nil)
         RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{database_path}'"
 
@@ -156,6 +168,7 @@ class RFlow
       end
 
       # Make sure that the configuration has all the necessary values set.
+      # @return [void]
       def merge_defaults!
         Setting::DEFAULTS.each do |name, default_value_or_proc|
           value = default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc
@@ -184,6 +197,8 @@ class RFlow
       end
     end
 
+    # Output the RFlow configuration to a pretty-printed String.
+    # @return [String]
     def to_s
       string = "Configuration:\n"
 
@@ -208,13 +223,36 @@ class RFlow
       string
     end
 
+    # Retrieve a setting value by name from the SQLite database.
+    # @return [Object]
     def [](name); Setting.find_by_name(name).value rescue nil; end
+
+    # Retrieve all the {Setting}s from the SQLite database.
+    # @return [Array<Setting>]
     def settings; Setting.all; end
+
+    # Retrieve all the {Shard}s from the SQLite database.
+    # @return [Array<Shard>]
     def shards; Shard.all; end
+
+    # Retrieve all the {Connection}s from the SQLite database.
+    # @return [Array<Connection>]
     def connections; Connection.all; end
+
+    # Retrieve a single {Shard} by UUID from the SQLite database.
+    # @return [Shard]
     def shard(uuid); Shard.find_by_uuid uuid; end
+
+    # Retrieve all the {Component}s from the SQLite database.
+    # @return [Array<Component>]
     def components; Component.all; end
+
+    # Retrieve a single {Component} by UUID from the SQLite database.
+    # @return [Shard]
     def component(uuid); Component.find_by_uuid uuid; end
+
+    # Retrieve the mapping from component name to {Component}.
+    # @return [Hash]
     def available_components; Configuration.available_components; end
   end
 end