rflow 1.3.0 → 1.3.1

data/lib/rflow/daemon_process.rb

@@ -1,7 +1,10 @@
 require 'rflow/pid_file'
 
 class RFlow
+  # Encapsulates a master process being managed by RFlow that can run in the foreground
+  # or daemonize.
   class DaemonProcess
+    # Symbolic constant for SIGINFO as this is only defined on BSD and not in Ruby.
     SIGINFO = 29
 
     def initialize(name, role = name, options = {})
@@ -10,6 +13,9 @@ class RFlow
       @pid_file = PIDFile.new(options[:pid_file_path]) if options[:pid_file_path]
     end
 
+    # Daemonize by forking and exiting the parent after handling
+    # IO streams and checking successful start of the new copy.
+    # @return [void]
     def daemonize!
       RFlow.logger.info "#{@name} daemonizing"
       establish_daemon_pipe
@@ -23,6 +29,11 @@ class RFlow
       end
     end
 
+    # Execute the master process. Writes out a pidfile and updates the process
+    # name, installs signal handlers, and spawns all the defined subprocesses.
+    # Finally executes {run_process}; when that returns, it will
+    # exit with the resulting return code.
+    # @return [void]
     def run!
       write_pid_file
       register_logging_context
@@ -38,9 +49,23 @@ class RFlow
       remove_pid_file
     end
 
+    # Default implementation. Subclasses should override to provide logic
+    # for actually spawning subprocesses.
+    # @return [void]
     def spawn_subprocesses; end
+
+    # Default implementation. Subclasses should override to provide logic
+    # for actually doing something useful.
+    # @return [void]
+    def run_process; end
+
+    # A list of {ChildProcess}es to start and signal.
+    # @return [Array<ChildProcess>]
     def subprocesses; []; end
 
+    # Shut down the application. Cleans up the pid file, removes
+    # signal handlers, and signals all child processes with +SIGQUIT+.
+    # @return [void]
     def shutdown!(reason)
       RFlow.logger.info "#{@name} shutting down due to #{reason}"
       remove_pid_file

data/lib/rflow/logger.rb

@@ -1,12 +1,16 @@
 require 'log4r'
 
 class RFlow
+  # The customized logger for RFlow applications that flows to the configured log file.
   class Logger
     extend Forwardable
     include Log4r
 
+    # @!visibility private
     LOG_PATTERN_FORMAT = '%-5l [%d] %x (%-5p) - %M'
+    # @!visibility private
     DATE_METHOD = 'xmlschema(6)'
+    # @!visibility private
     LOG_PATTERN_FORMATTER = PatternFormatter.new :pattern => LOG_PATTERN_FORMAT, :date_method => DATE_METHOD
 
     private
@@ -14,6 +18,8 @@ class RFlow
     attr_accessor :log_file_path, :log_level, :log_name
 
     public
+    # For the current logging context, how wide the field is where we're going to write the context/process name.
+    # @return [Integer]
     attr_accessor :context_width
 
     # make sure Log4r is initialized; ignored if custom levels are already set
@@ -28,6 +34,8 @@ class RFlow
       reconfigure(config, include_stdout)
     end
 
+    # Reconfigure the log file.
+    # @return [void]
     def reconfigure(config, include_stdout = false)
       @log_file_path = config['rflow.log_file_path']
       @log_level = config['rflow.log_level'] || 'WARN'
@@ -43,6 +51,9 @@ class RFlow
       internal_logger
     end
 
+    # Reopen the logs at their configured filesystem locations. Presumably the previous
+    # log files have been renamed by now.
+    # @return [void]
     def reopen
       # TODO: Make this less of a hack, although Log4r doesn't support
       # it, so it might be permanent
@@ -50,14 +61,21 @@ class RFlow
       File.open(log_file.path, 'a') { |tmp_log_file| log_file.reopen(tmp_log_file) }
     end
 
+    # Close the logger.
+    # @return [void]
     def close
       Outputter['rflow.log_file'].close
     end
 
+    # Update the log level.
+    # @return [void]
     def level=(level)
       internal_logger.level = LNAMES.index(level.to_s) || level
     end
 
+    # Toggle the log level between +DEBUG+ and whatever the default is. The previous
+    # level is saved to be toggled back the next time this method is called.
+    # @return [void]
     def toggle_log_level
       original_log_level = LNAMES[internal_logger.level]
       new_log_level = (original_log_level == 'DEBUG' ? log_level : 'DEBUG')
@@ -66,6 +84,8 @@ class RFlow
       internal_logger.level = LNAMES.index new_log_level
     end
 
+    # Send a complete thread dump of the current process out to the logger.
+    # @return [void]
     def dump_threads
       Thread.list.each do |t|
         info "Thread #{t.inspect}:"
@@ -75,18 +95,27 @@ class RFlow
       info 'Thread dump complete.'
     end
 
+    # Clone the logging context so changes to it will not affect the
+    # exiting logging context.
+    # @return [void]
     def clone_logging_context
       Log4r::NDC.clone_stack
     end
 
+    # Replace the current logging context.
+    # @return [void]
     def apply_logging_context(context)
       Log4r::NDC.inherit(context)
     end
 
+    # Clear the current logging context.
+    # @return [void]
     def clear_logging_context
       Log4r::NDC.clear
     end
 
+    # Add more logging context to the stack.
+    # @return [void]
     def add_logging_context(context)
       Log4r::NDC.push context
     end

data/lib/rflow/master.rb

@@ -3,8 +3,15 @@ require 'rflow/shard'
 require 'rflow/broker'
 
 class RFlow
+  # The master/watchdog process for RFlow. Mostly exists to receive +SIGCHLD+ from subprocesses
+  # so it can kill them all with +SIGQUIT+ and get restarted.
   class Master < DaemonProcess
+    # The {Shard}s being managed by the {Master}.
+    # @return [Array<Shard>]
     attr_reader :shards
+
+    # The {Broker}s being managed by the {Master}.
+    # @return [Array<Broker>]
     attr_reader :brokers
 
     def initialize(config)
@@ -14,6 +21,9 @@ class RFlow
       @brokers = config.connections.flat_map(&:brokers).map {|config| Broker.build(config) }
     end
 
+    # Override of {spawn_subprocesses} that actually spawns them,
+    # then calls {Shard#run!} on each.
+    # @return [void]
     def spawn_subprocesses
       RFlow.logger.debug "Running #{brokers.count} brokers" if brokers.count > 0
       brokers.each(&:spawn!)
@@ -22,10 +32,15 @@ class RFlow
       shards.each(&:run!)
     end
 
+    # Override of {subprocesses} that includes the {Broker}s and
+    # every {Shard::Worker} of every {Shard}.
+    # @return [Array<ChildProcess>]
     def subprocesses
       brokers + shards.flat_map(&:workers)
     end
 
+    # Override that starts EventMachine and waits until it gets stopped.
+    # @return [void]
     def run_process
       EM.run do
         # TODO: Monitor the workers

data/lib/rflow/message.rb

@@ -4,11 +4,20 @@ require 'avro'
 require 'rflow/configuration'
 
 class RFlow
+  # Utility methods for doing Avro encoding/decoding.
   class Avro
+    # Decode serialized Avro data.
+    # @param reader [::Avro::IO::DatumReader] reader preconfigured with schema
+    # @param bytes [String] byte string to decode
+    # @return decoded object
     def self.decode(reader, bytes)
       reader.read ::Avro::IO::BinaryDecoder.new(StringIO.new(bytes.force_encoding('BINARY')))
     end
 
+    # Encode data to serialized Avro.
+    # @param writer [::Avro::IO::DatumWriter] writer preconfigured with schema
+    # @param message [String]
+    # @return [String]
     def self.encode(writer, message)
       String.new.force_encoding('BINARY').tap do |result|
         writer.write message, ::Avro::IO::BinaryEncoder.new(StringIO.new(result, 'w'))
@@ -16,15 +25,21 @@ class RFlow
     end
   end
 
+  # A message to be sent around in the RFlow framework.
   class Message
     class << self
+      # @!visibility private
       def schema; @schema ||= ::Avro::Schema.parse(File.read(File.join(File.dirname(__FILE__), '..', '..', 'schema', 'message.avsc'))); end
+      # @!visibility private
       def message_reader; @message_reader ||= ::Avro::IO::DatumReader.new(schema, schema); end
+      # @!visibility private
       def message_writer; @message_writer ||= ::Avro::IO::DatumWriter.new(schema); end
+      # @!visibility private
       def encode(message); RFlow::Avro.encode(message_writer, message); end
 
       # Take in an Avro serialization of a message and return a new
       # Message object.  Assumes the org.rflow.Message Avro schema.
+      # @!visibility private
       def from_avro(bytes)
         message = RFlow::Avro.decode(message_reader, bytes)
         Message.new(message['data_type_name'], message['provenance'], message['properties'],
@@ -33,13 +48,26 @@ class RFlow
       end
     end
 
-    attr_accessor :provenance, :properties
-    attr_reader :data_type_name, :data
+    # The message's provenance information.
+    # @return [Array<ProcessingEvent>]
+    attr_accessor :provenance
+
+    # The message's properties information.
+    # @return [Hash]
+    attr_accessor :properties
+
+    # The data type name of the message.
+    # @return [String]
+    attr_reader :data_type_name
+
+    # The actual data string in the message.
+    # @return [String]
+    attr_reader :data
 
     # When creating a new message as a transformation of an existing
-    # message, its encouraged to copy the provenance and properties of
+    # message, it's encouraged to copy the provenance and properties of
     # the original message into the new message. This allows
-    # downstream components to potentially use these fields
+    # downstream components to potentially use these fields.
     def initialize(data_type_name, provenance = [], properties = {}, serialization_type = 'avro', schema = nil, serialized_data = nil)
       @data_type_name = data_type_name.to_s
 
@@ -86,7 +114,8 @@ class RFlow
     # org.rflow.Message Avro schema.  Note that we have to manually
     # set the encoding for Ruby 1.9, otherwise the stringio would use
     # UTF-8 by default, which would not work correctly, as a serialize
-    # avro string is BINARY, not UTF-8
+    # avro string is BINARY, not UTF-8.
+    # @return [String]
     def to_avro
       # stringify all the properties
       string_properties = Hash[properties.map { |k,v| [k.to_s, v.to_s] }]
@@ -99,9 +128,20 @@ class RFlow
                      'data' => data.to_avro)
     end
 
+    # One processing event in the message's provenance.
     class ProcessingEvent
-      attr_reader :component_instance_uuid, :started_at
-      attr_accessor :completed_at, :context
+      # The UUID of the component doing the processing.
+      # @return [String]
+      attr_reader :component_instance_uuid
+      # The time processing started, in XML schema format.
+      # @return [String]
+      attr_reader :started_at
+      # The time processing ended, in XML schema format.
+      # @return [String]
+      attr_accessor :completed_at
+      # Arbitrary context bytes.
+      # @return [String]
+      attr_accessor :context
 
       def initialize(component_instance_uuid, started_at = nil, completed_at = nil, context = nil)
         @component_instance_uuid = component_instance_uuid
@@ -116,6 +156,8 @@ class RFlow
         @context = context
       end
 
+      # Represent the processing event as a hash.
+      # @return [Hash]
       def to_hash
         {
           'component_instance_uuid' => component_instance_uuid.to_s,
@@ -126,11 +168,21 @@ class RFlow
       end
     end
 
-    # Should proxy most methods to data_object that we can serialize
-    # to avro using the schema.  Extensions should use 'extended' hook
+    # Should proxy most methods to {data_object} that we can serialize
+    # to Avro using the schema. Extensions should use +extended+ hook
     # to apply immediate changes.
     class Data
-      attr_reader :schema_string, :schema, :serialization_type
+      # The string form of the schema the data follows.
+      # @return [String]
+      attr_reader :schema_string
+      # Avro parsed version of the schema the data follows
+      # @return [::Avro::Schema]
+      attr_reader :schema
+      # Serialization type. Currently, always +avro+.
+      # @return [String]
+      attr_reader :serialization_type
+      # The data object for the message.
+      # @return [Object]
       attr_accessor :data_object
 
       def initialize(schema_string, serialization_type = 'avro', serialized_data = nil)
@@ -152,17 +204,22 @@ class RFlow
         end
       end
 
+      # Is the message valid per the Avro schema?
+      # @return [boolean]
       def valid?
         ::Avro::Schema.validate @schema, @data_object
       end
 
+      # Encode the message out to real Avro.
+      # @return [String]
       def to_avro
         RFlow::Avro.encode @writer, @data_object
       end
 
-      # Proxy methods down to the underlying data_object, probably a
+      # Proxy methods down to the underlying {data_object}, probably a
       # Hash.  Hopefully an extension will provide any additional
-      # functionality so this won't be called unless needed
+      # functionality so this won't be called unless needed.
+      # @return [void]
       def method_missing(method_sym, *args, &block)
         @data_object.send(method_sym, *args, &block)
       end

data/lib/rflow/pid_file.rb

@@ -1,4 +1,5 @@
 class RFlow
+  # Represents a file on disk that contains RFlow's PID, for process management.
   class PIDFile
     private
     attr_reader :path
@@ -8,6 +9,8 @@ class RFlow
       @path = path
     end
 
+    # Read the pid file and get the PID from it.
+    # @return [Integer]
     def read
       return nil unless File.exist? path
       contents = File.read(path)
@@ -19,6 +22,8 @@ class RFlow
       end
     end
 
+    # Write a new PID out to the pid file.
+    # @return [Integer] the pid
     def write(pid = $$)
       return unless validate?
 
@@ -42,6 +47,8 @@ class RFlow
       pid
     end
 
+    # Determine if the application is running by checking the running PID and the pidfile.
+    # @return [boolean]
     def running?
       return false unless exist?
       pid = read
@@ -52,18 +59,22 @@ class RFlow
       nil
     end
 
-    # unlinks a PID file at given if it contains the current PID still
+    # Unlinks a PID file if it contains the current PID. Still
     # potentially racy without locking the directory (which is
     # non-portable and may interact badly with other programs), but the
-    # window for hitting the race condition is small
+    # window for hitting the race condition is small.
+    # @return [void]
     def safe_unlink
       (current_process? and unlink) rescue nil
     end
 
+    # Signal the process with the matching PID with a given signal.
+    # @return [void]
     def signal(sig)
       Process.kill(sig, read)
     end
 
+    # @!visibility private
     def to_s
       File.expand_path(path)
     end

data/lib/rflow/shard.rb

@@ -2,14 +2,23 @@ require 'rflow/child_process'
 
 class RFlow
   # An object implementation shared between two processes. The parent
-  # process will instantiate, configure, and run! a shard, at which
-  # point the parent will have access to the shard object and be able
-  # to monitor the underlying processes. The child implementation,
-  # running in a separate process, will not return from spawn!, but
+  # process will instantiate, configure, and run! a {Shard}, at which
+  # point the parent will have access to the {Shard} object and be able
+  # to monitor the underlying {Shard::Worker} processes. The child implementation,
+  # running in a separate process, will not return from +spawn!+, but
   # start an EventMachine reactor.
   class Shard
+    # An actual child process under the {Shard}, which coordinates a set of
+    # identical {Worker}s.
     class Worker < ChildProcess
-      attr_reader :shard, :index
+      # A reference to the {Shard} governing this {Worker}.
+      # @return [Shard]
+      attr_reader :shard
+
+      # Which worker index this is (for example, in a set of 3 {Worker}s,
+      # one would have index 0, one would have index 1, one would have index 2).
+      # @return [Integer]
+      attr_reader :index
 
       def initialize(shard, index = 1)
         super("#{shard.name}-#{index}", 'Worker')
@@ -20,6 +29,8 @@ class RFlow
         @components = shard.config.components.map {|config| Component.build(self, config) }
       end
 
+      # Configure, connect, and actually start running RFlow components.
+      # @return [void]
       def run_process
         EM.run do
           begin
@@ -38,6 +49,7 @@ class RFlow
         RFlow.logger.info 'Shutting down worker after EM stopped'
       end
 
+      protected
       def configure_components!
         RFlow.logger.debug 'Configuring components'
         @components.zip(shard.config.components.map(&:options)).each do |(component, config)|
@@ -68,6 +80,9 @@ class RFlow
         end
       end
 
+      public
+      # Shut down the {Worker}. Shuts down each component and kills EventMachine.
+      # @return [void]
       def shutdown!(signal)
         RFlow.logger.debug 'Shutting down components'
         @components.each do |component|
@@ -79,7 +94,21 @@ class RFlow
       end
     end
 
-    attr_reader :config, :name, :count, :workers
+    # Reference to the {Shard}'s configuration.
+    # @return [Configuration::Shard]
+    attr_reader :config
+
+    # The {Shard}'s name.
+    # @return [String]
+    attr_reader :name
+
+    # The count of workers that should be started.
+    # @return [Integer]
+    attr_reader :count
+
+    # Reference to the actual {Worker}s.
+    # @return [Array<Worker>]
+    attr_reader :workers
 
     def initialize(config)
       @config = config
@@ -89,6 +118,8 @@ class RFlow
       @workers = count.times.map {|i| Worker.new(self, i+1) }
     end
 
+    # Start the shard by spawning and starting all the workers.
+    # @return [void]
     def run!
       RFlow.logger.debug "Running shard #{name} with #{count} workers"
       workers.each(&:spawn!)