rflow 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f99f280cff598c00599545d4f57e27ee6589b62
-  data.tar.gz: 2b5933fc4baef7b4ae520f820bb0fa48e5e79b1d
+  metadata.gz: a854e3c0cf56feae660eb061c29d9cbaad028a34
+  data.tar.gz: ac21f337ed623cb7a0a96c8612dc17f5e1e63bd2
 SHA512:
-  metadata.gz: 5c436a2293827ff13ffc5f9b5b7732549e9e3948c261d0f9d19d83c565eb694d93129ab8e0133e6afe99deace85c4f7053ca2de8bc3d8e120139e0adca1960ba
-  data.tar.gz: 3ab7e7aa970cf600adbdef69bbab0934cf402741c2baf610dcb5b039edddf32b9576e58b49c421d1a8b67cf6529dc0434fcfb1014769f97fe103a921905dcb84
+  metadata.gz: d580e43a06b900a9cfcb8cd7372d933b698f5dc28c40909a1a2ed4211c342af497829f44e3699b520499dfd7007e70fc6925c88b8354a63278d6402cd504f5bc
+  data.tar.gz: b5d50e712d85913f0052e3a3c3cd5d2c73a4e69280af1cc0d3f92aa1fcc8917181db8a7d5288b1cab17787161543945023f6b170a47cc9d883428197d35173a0

data/.gitignore

@@ -7,3 +7,5 @@ pkg/*
 spec/tmp/
 *.swp
 /.vagrant
+.yardoc
+doc/

data/.yardopts

@@ -1 +1 @@
---output ./doc --main README.md --files schema/*.avsc lib/**/*.rb bin/*.rb - README.md LICENSE
+--output ./doc --main README.md --files schema/*.avsc lib/**/*.rb bin/*.rb --exclude lib/rflow/configuration/migrations --exclude lib/rflow/configuration/ruby_dsl.rb - README.md LICENSE

data/Vagrantfile

@@ -3,14 +3,17 @@
 VAGRANTFILE_API_VERSION = '2'
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.define 'centos62' do |c|
-    c.vm.box = 'jstoneham/rflow-centos62'
-  end
-  config.vm.define 'centos64' do |c|
-    c.vm.box = 'box-cutter/centos64'
-  end
-  config.vm.define 'centos65' do |c|
-    c.vm.box = 'chef/centos-6.5'
+#  config.vm.define 'centos62' do |c|
+#    c.vm.box = 'jstoneham/rflow-centos62'
+#  end
+#  config.vm.define 'centos64' do |c|
+#    c.vm.box = 'box-cutter/centos64'
+#  end
+#  config.vm.define 'centos65' do |c|
+#    c.vm.box = 'chef/centos-6.5'
+#  end
+  config.vm.define 'centos67' do |c|
+    c.vm.box = 'boxcutter/centos67'
   end
 
   config.vm.synced_folder '.', '/rflow'
@@ -25,14 +28,14 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # install RPM dependencies for rflow and zeromq
   config.vm.provision 'shell', privileged: true, inline: <<-EOS
-    curl -O https://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
+    curl -OL https://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
     rpm -ivh epel-release-6-8.noarch.rpm
     yum -y install libyaml-devel patch libffi-devel glibc-headers autoconf gcc-c++ glibc-devel readline-devel zlib-devel openssl-devel automake libtool bison git sqlite-devel rpm-build libuuid-devel vim
   EOS
 
   # build zeromq as vagrant user
   config.vm.provision 'shell', privileged: false, inline: <<-EOS
-    curl -O http://download.zeromq.org/zeromq-3.2.4.tar.gz
+    curl -OL https://archive.org/download/zeromq_3.2.4/zeromq-3.2.4.tar.gz
     rpmbuild -tb zeromq-3.2.4.tar.gz
   EOS
 
@@ -44,6 +47,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # set up RVM and bundler
   config.vm.provision 'shell', privileged: false, inline: <<-EOS
     rm -f .profile
+    gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
     curl -sSL https://get.rvm.io | bash -s stable
     source .rvm/scripts/rvm
     rvm install `cat /rflow/.ruby-version`

data/lib/rflow.rb

@@ -11,16 +11,31 @@ require 'rflow/components'
 require 'rflow/connections'
 require 'rflow/logger'
 
+# The RFlow application.
 class RFlow
   include Log4r
 
   class << self
+    # RFlow's logger, whose message will be routed to logs as specified in the configuration.
+    # @return [RFlow::Logger]
     attr_accessor :logger
-    attr_reader :configuration, :master
+
+    # RFlow's configuration.
+    # @return [RFlow::Configuration]
+    attr_reader :configuration
+
+    # RFlow's master node which oversees all the others.
+    # @return [RFlow::Master]
+    attr_reader :master
 
     RFlow.logger = RFlow::Logger.new({})
   end
 
+  # Start RFlow running. This is the main programmatic entry point to the application.
+  # Pulls in the configuration, sets up logging, and starts the master note.
+  #
+  # @param config_database_path [String] the path to the SQLite configuration database
+  # @param daemonize [boolean] true to fork and daemonize; false to run in the foreground
   def self.run!(config_database_path = nil, daemonize = false)
     @config_database_path = config_database_path
     @daemonize = daemonize

data/lib/rflow/broker.rb

@@ -1,10 +1,14 @@
 require 'rflow/child_process'
 
 class RFlow
-  # A message broker to mediate messages along a connection.
-  # The broker runs in a child process and will not return from spawn!.
+  # A message broker process to mediate messages along a connection.
+  # The broker runs in a child process and will not return from {spawn!}.
   class Broker < ChildProcess
     class << self
+      # Build the broker from the connection configuration.
+      # Only supports {RFlow::Configuration::ZMQStreamer} configurations.
+      # @param config [RFlow::Configuration::ZMQStreamer] the connection configuration
+      # @return [RFlow::Connections::ZMQStreamer]
       def build(config)
         case config.class.name
         when 'RFlow::Configuration::ZMQStreamer'

data/lib/rflow/child_process.rb

@@ -1,11 +1,20 @@
 require 'fcntl'
 
 class RFlow
+  # Encapsulates a child process being managed by RFlow.
   class ChildProcess
-    attr_reader :pid, :name
-
+    # The PID of the child process.
+    # @return [Fixnum]
+    attr_reader :pid
+    # The name of the child process.
+    # @return [String]
+    attr_reader :name
+
+    # Symbolic constant for SIGINFO as this is only defined on BSD and not in Ruby.
     SIGINFO = 29
 
+    # @param name [String] process name
+    # @param role [String] role to be played by the process, for logging (Master, Broker, etc.)
     def initialize(name, role = name)
       @name = name
       @role = role
@@ -13,7 +22,12 @@ class RFlow
 
     # Launch another process to execute the child. The parent
     # process retains the original worker object (with pid and IPC
-    # pipe) to allow for process management
+    # pipe) to allow for process management. Parent will
+    # return once the child starts; child will update its process
+    # name, detach from the process group, set up signal handlers, and
+    # execute {run_child_process}; when that returns, it will
+    # exit with return code 0.
+    # @return [void]
     def spawn!
       establish_child_pipe
       drop_database_connections
@@ -26,6 +40,7 @@ class RFlow
       end
     end
 
+    protected
     def run_child_process
       @child_pipe_w.close
       register_logging_context
@@ -42,6 +57,11 @@ class RFlow
 
     def run_process; end
 
+    public
+    # Called when the child process needs to be shut down, before it dies.
+    # Clears signal handlers.
+    # @param signal [String] SIG*, whichever signal caused the shutdown
+    # @return [void]
     def shutdown!(signal)
       RFlow.logger.info "Shutting down due to #{signal}"
       unhandle_signals

data/lib/rflow/component.rb

@@ -3,22 +3,37 @@ require 'rflow/message'
 require 'rflow/component/port'
 
 class RFlow
+  # Parent class for all RFlow components.
   class Component
     class << self
-      # Keep track of available component subclasses
+      # Keep track of available component subclasses.
+      # @!visibility private
       def inherited(subclass)
         RFlow::Configuration.add_available_component(subclass)
       end
 
-      # Define an input port with a given name
+      # When declaring your component class, defines an input port with a given
+      # name. Will also define a port accessor method named after the port for
+      # retrieving it.
+      #
+      # @param name [String]
+      # @return [void]
       def input_port(name); define_port(defined_input_ports, name); end
 
-      # Define an output port with a given name
+      # When declaring your component class, defines an output port with a
+      # given name. Will also define a port accessor method named after the
+      # port for retrieving it.
+      #
+      # @param name [String]
+      # @return [void]
       def output_port(name); define_port(defined_output_ports, name); end
 
+      # @!visibility private
       def defined_input_ports; @defined_input_ports ||= {}; end
+      # @!visibility private
       def defined_output_ports; @defined_output_ports ||= {}; end
 
+      # @!visibility private
       def define_port(collection, name)
         collection[name.to_s] = true
 
@@ -32,10 +47,14 @@ class RFlow
       # specification. This assumes that the specification of a
       # component is a fully qualified Ruby class that has already
       # been loaded. It will first attempt to find subclasses of
-      # RFlow::Component (in the available_components hash) and then
+      # {RFlow::Component} (in {Configuration#available_components}) and then
       # attempt to constantize the specification into a different
       # class. Future releases will support external (i.e. non-managed
-      # components), but the current stuff only supports Ruby classes
+      # components), but the current stuff only supports Ruby classes.
+      #
+      # @param worker [Shard::Worker] the worker process for the component to run in
+      # @param config [Configuration::Component] the component configuration
+      # @return [RFlow::Component] an instance of the component class
       def build(worker, config)
         raise NotImplementedError, "Non-managed components not yet implemented for component '#{config.name}' as '#{config.specification}' (#{config.uuid})" unless config.managed?
 
@@ -74,9 +93,20 @@ class RFlow
       end
     end
 
-    attr_accessor :uuid, :name
-    attr_reader :ports, :worker
-
+    # The UUID of the component.
+    # @return [String]
+    attr_accessor :uuid
+    # The name of the component.
+    # @return [String]
+    attr_accessor :name
+    # Collection of the component's input and output ports.
+    # @return [PortCollection]
+    attr_reader :ports
+    # Reference to the worker process in which this instance of the component is running.
+    # @return [Shard::Worker]
+    attr_reader :worker
+
+    # @param args [Hash] supported args are +:name+, +:uuid+, +:worker+
     def initialize(args = {})
       @name = args[:name]
       @uuid = args[:uuid]
@@ -87,16 +117,22 @@ class RFlow
       self.class.defined_output_ports.each {|name, _| ports << OutputPort.new(self, name: name) }
     end
 
+    # @!attribute shard [r]
+    # Reference to the component's worker process's {Shard}.
+    # @return [Shard]
     def shard; worker.shard if worker; end
 
     # Returns a list of connected input ports.  Each port will have
     # one or more keys associated with a particular connection.
+    # @return [Array<InputPort>]
     def input_ports; ports.by_type['RFlow::Component::InputPort']; end
 
     # Returns a list of connected output ports.  Each port will have
     # one or more keys associated with the particular connection.
+    # @return [Array<OutputPort>]
     def output_ports; ports.by_type['RFlow::Component::OutputPort']; end
 
+    # @!visibility private
     def configure_input_port!(port_name, options = {})
       RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) input port '#{port_name}' (#{options[:uuid]})"
       unless self.class.defined_input_ports.include? port_name
@@ -105,6 +141,7 @@ class RFlow
       ports.by_name[port_name].uuid = options[:uuid]
     end
 
+    # @!visibility private
     def configure_output_port!(port_name, options = {})
       RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) output port '#{port_name}' (#{options[:uuid]})"
       unless self.class.defined_output_ports.include? port_name
@@ -113,6 +150,7 @@ class RFlow
       ports.by_name[port_name].uuid = options[:uuid]
     end
 
+    # @!visibility private
     # Tell the component to establish its ports' connections, i.e. make
     # the connection.  Uses the underlying connection object.  Also
     # establishes the callbacks for each of the input ports
@@ -121,12 +159,16 @@ class RFlow
       input_ports.each(&:connect!)
     end
 
+    # @!visibility private
     # Tell the component to establish its ports' connections, i.e. make
     # the connection. Uses the underlying connection object.
+    # @!visibility private
     def connect_outputs!
       output_ports.each(&:connect!)
     end
 
+    # Pretty-printed version of the component, its ports, their keys, and their connections.
+    # @return [String]
     def to_s
       string = "Component '#{name}' (#{uuid})\n"
       ports.each do |port|
@@ -141,29 +183,37 @@ class RFlow
 
     # Method that should be overridden by a subclass to provide for
     # component-specific configuration.  The subclass should use the
-    # self.configuration attribute (@configuration) to store its
-    # particular configuration.  The incoming deserialized_configuration
-    # parameter is from the RFlow configuration database and is (most
-    # likely) a hash.  Don't assume that the keys are symbols
+    # {configuration} attribute (+@configuration+) to store its
+    # particular configuration.
+    # @param deserialized_configuration [Hash] from the RFlow configuration database; most likely a Hash. Don't assume that the keys are symbols!
+    # @return [void]
     def configure!(deserialized_configuration); end
 
     # Main component running method.  Subclasses should implement if
     # they want to set up any EventMachine stuffs (servers, clients,
-    # etc)
+    # etc.).
+    # @return [void]
     def run!; end
 
     # Method called when a message is received on an input port.
-    # Subclasses should implement if they want to receive messages
+    # Subclasses should implement if they want to receive messages.
+    # @param input_port [RFlow::Component::InputPort] the input port the message was received on
+    # @param input_port_key [String] if the message was received on a keyed subport, this is the key
+    # @param connection [RFlow::Connection] the connection the message was received on
+    # @param message [RFlow::Message] the message itself
+    # @return [void]
     def process_message(input_port, input_port_key, connection, message); end
 
     # Method called when RFlow is shutting down.  Subclasses should
-    # implment to terminate any servers/clients (or let them finish)
-    # and stop sending new data through the flow
+    # implement to terminate any servers/clients (or let them finish)
+    # and stop sending new data through the flow.
+    # @return [void]
     def shutdown!; end
 
     # Method called after all components have been shutdown! and just
-    # before the global RFlow exit.  Sublcasses should implement to
-    # cleanup any leftover state, e.g. flush file handles, etc
+    # before the global RFlow exit. Sublcasses should implement to
+    # cleanup any leftover state, e.g. flush file handles, etc.
+    # @return [void]
     def cleanup!; end
   end
 end

data/lib/rflow/component/port.rb

@@ -3,7 +3,9 @@ class RFlow
     # TODO: make this into a class to limit the amount of extensions
     # that we have to do when operating on these 'Arrays', i.e. when
     # adding two together
+    # @!visibility private
     module ConnectionCollection
+      # @!visibility private
       def send_message(message)
         each {|connection| connection.send_message(message) }
       end
@@ -12,7 +14,15 @@ class RFlow
     # Collection class to make it easier to index by both names
     # and types.
     class PortCollection
-      attr_reader :ports, :by_name, :by_type
+      # All the ports in the collection.
+      # @return [Array<Port>]
+      attr_reader :ports
+      # All the ports in the collection, indexed by name.
+      # @return [Hash<String, Port>]
+      attr_reader :by_name
+      # All the ports in the collection, indexed by type ({InputPort}, {OutputPort}).
+      # @return [Hash<String, Array<Port>>]
+      attr_reader :by_type
 
       def initialize
         @ports = []
@@ -20,6 +30,9 @@ class RFlow
         @by_type = Hash.new {|hash, key| hash[key.to_s] = []}
       end
 
+      # Add a port to the collection.
+      # @param port [Port] port to add
+      # @return [PortCollection] self
       def <<(port)
         by_name[port.name.to_s] = port
         by_type[port.class.to_s] << port
@@ -27,42 +40,65 @@ class RFlow
         self
       end
 
-      # Enumerate through each port
+      # Enumerate through each port, +yield+ing each.
       # TODO: simplify with enumerators and procs
+      # @return [Array<Port>]
       def each
         ports.each {|port| yield port }
       end
     end
 
+    # An input or output port on a {Component}.
     class Port
-      attr_reader :connected, :component
+      # True if there are connections to the port.
+      # @return [boolean]
+      attr_reader :connected
+      # The {Component} this port belongs to.
+      # @return [Component]
+      attr_reader :component
 
       def initialize(component)
         @component = component
       end
 
+      # Synonym for {connected}.
+      # @return [boolean]
       def connected?; connected; end
     end
 
-    # Stateless class to help with a nicer API
+    # Represents a keyed subport on a {Component} - that is, an input or output port
+    # that has been subscripted with a port name for subdividing the messages being
+    # received or output.
     class HashSubPort
+      # @param hash_port [HashPort] the port to which this subport belongs
+      # @param key [String] the key subscript
       def initialize(hash_port, key)
         @hash_port = hash_port
         @key = key
       end
 
+      # Send a {Message} down all the connections to this subport.
+      # @param message [Message]
+      # @return [void]
       def send_message(message)
         connections.each {|connection| connection.send_message(message) }
       end
 
+      # Retrieve all the connections for this subport.
+      # @return [Array<Connection>]
       def connections
         @hash_port.connections_for(@key)
       end
 
+      # Directly connect this subport to another port.
+      # @param other_port [Port] the other port to connect to
+      # @return [void]
       def direct_connect(other_port)
         @hash_port.direct_connect(@key, other_port)
       end
 
+      # Enumerate the connections to this subport, +yield+ing each.
+      # @return [Array<Connection>]
       def each
         connections.each {|connection| yield connection }
       end
@@ -75,9 +111,16 @@ class RFlow
     # result in the same message being sent to all indexed
     # connections.
     class HashPort < Port
-      attr_accessor :name, :uuid
+      # The name of the port.
+      # @return [String]
+      attr_accessor :name
+      # The UUID of the port.
+      # @return [String]
+      attr_accessor :uuid
 
       public
+      # @param component [Component] the component the port belongs to
+      # @param args [Hash] supported args are +:uuid+ and +:name+
       def initialize(component, args = {})
         super(component)
         self.uuid = args[:uuid]
@@ -86,17 +129,21 @@ class RFlow
       end
 
       # Get the subport for a given key, which can be used to send messages
-      # or direct connection
+      # or direct connection.
+      # @param key [String] the key to subscript with
+      # @return [HashSubPort]
       def [](key)
         HashSubPort.new(self, key)
       end
 
-      # Returns an Array of all the connections that should
-      # be sent/received on this subport.  Merges the nil-keyed port
+      # Returns all the connections that should
+      # be sent/received on this subport.  Merges the +nil+-keyed port
       # (i.e. any connections for a port without a key) to those
       # specific for the key, so should only be used to read a list of
-      # connections, not to add new ones.  Use add_connection to add a
+      # connections, not to add new ones.  Use {add_connection} to add a
       # new connection for a given key.
+      # @param key [String] the key to subscript with
+      # @return [Array<Connection>]
       def connections_for(key)
         case key
         when nil; @connections_for[nil]
@@ -104,20 +151,32 @@ class RFlow
         end
       end
 
-      # Adds a connection for a given key
+      # Adds a connection for a given key.
+      # @param key [String] the key to subscript with
+      # @param connection [Connection] the connection to add
+      # @return [void]
       def add_connection(key, connection)
         RFlow.logger.debug "Attaching #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) to port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
         @connections_for[key] << connection
         @all_connections = nil
       end
 
-      # Removes a connection from a given key
+      # Removes a connection from a given key.
+      # @param key [String] the key to subscript with
+      # @param connection [Connection] the connection to remove
+      # @return [void]
       def remove_connection(key, connection)
         RFlow.logger.debug "Removing #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) from port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
         @connections_for[key].delete(connection)
         @all_connections = nil
       end
 
+      # Collect messages being sent to this port in a {MessageCollectingConnection}
+      # for retrieval later, usually for unit testing purposes. +yield+s after
+      # establishing the new connection.
+      # @param key [String] the key to subscript with
+      # @param receiver [Array] array in which to place arriving messages
+      # @return [MessageCollectingConnection]
       def collect_messages(key, receiver)
         begin
           connection = RFlow::MessageCollectingConnection.new.tap do |c|
@@ -132,6 +191,12 @@ class RFlow
         end
       end
 
+      # Directly connect this port to another port. If it's an input port,
+      # forward messages to that input port; if it's an output port,
+      # forward messages so they appear to come from that output port.
+      # @param key [String] the key to subscript with
+      # @param other_port [Port] the port to forward to
+      # @return [void]
       def direct_connect(key = nil, other_port)
         case other_port
         when InputPort; add_connection key, ForwardToInputPort.new(other_port)
@@ -140,39 +205,62 @@ class RFlow
         end
       end
 
-      # Return a list of connected keys
+      # A list of connected keys.
+      # @return [Array<String>]
       def keys
         @connections_for.keys
       end
 
+      # Enumerate all connections, +yield+ing each.
+      # @return [Array<Connection>]
       def each
         @connections_for.values.each {|connections| yield connections }
       end
 
+      # Override in subclasses to actually send messages places.
+      # @param message [Message] the message to send
+      # @return [void]
       def send_message(message)
         raise NotImplementedError, 'Raw ports do not know how to send messages'
       end
 
-      # Should be overridden.  Called when it is time to actually
-      # establish the connection
+      # Override in subclasses to handle establishing the connection.
+      # @return [void]
       def connect!; raise NotImplementedError, 'Raw ports do not know which direction to connect'; end
 
+      # Retrieve all connections to the port, regardless of key. The resulting +Array+
+      # also supports +send_message(message)+ which will deliver the message on all
+      # connections.
+      # @return [Array<Connection>]
       def all_connections
         @all_connections ||= @connections_for.values.flatten.uniq.extend(ConnectionCollection)
       end
     end
 
+    # An actual {Component} input port.
     class InputPort < HashPort
+      # Connect all the input connections, once everything's been set up.
+      # @return [void]
       def connect!
         @connections_for.each {|key, conns| conns.each {|c| c.connect_input! } }
         @connected = true
       end
 
+      # Add and start up a new {Connection}.
+      # @param key [String] the key to subscript with
+      # @param connection [Connection] the connection to add
+      # @return [void]
       def add_connection(key, connection)
         super
         connection.connect_input! if connected?
       end
 
+      # Once things have been set up, registering the receive callback
+      # will set it on all connections, so that when messages are received,
+      # they are delivered on all connections with appropriate key and connection
+      # information from the context of the connection.
+      # @param callback [Proc] the receive callback
+      # @return [void]
       def recv_callback=(callback)
         @connections_for.each do |key, connections|
           connections.each do |connection|
@@ -184,12 +272,19 @@ class RFlow
       end
     end
 
+    # An actual {Component} output port.
     class OutputPort < HashPort
+      # Connect all the output connections, once everything's been set up.
+      # @return [void]
       def connect!
         @connections_for.each {|key, conns| conns.each {|c| c.connect_output! } }
         @connected = true
       end
 
+      # Add and start up a new {Connection}.
+      # @param key [String] the key to subscript with
+      # @param connection [Connection] the connection to add
+      # @return [void]
       def add_connection(key, connection)
         super
         connection.connect_output! if connected?
@@ -197,6 +292,8 @@ class RFlow
 
       # Send a message to all connections on all keys for this port,
       # but only once per connection.
+      # @param message [RFlow::Message] the message to send
+      # @return [void]
       def send_message(message)
         all_connections.send_message(message)
       end