rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow/configuration/shard.rb

@@ -3,8 +3,7 @@ require 'rflow/configuration/uuid_keyed'
 
 class RFlow
   class Configuration
-
-    class Shard < ConfigDB
+    class Shard < ConfigurationItem
       include UUIDKeyed
       include ActiveModel::Validations
 
@@ -12,7 +11,6 @@ class RFlow
 
       has_many :components, :primary_key => 'uuid', :foreign_key => 'shard_uuid'
 
-      validates_presence_of :name
       validates_uniqueness_of :name
       validates_numericality_of :count, :only_integer => true, :greater_than => 0
     end

data/lib/rflow/connection.rb

@@ -2,7 +2,6 @@ require 'rflow/message'
 
 class RFlow
   class Connection
-
     class << self
       def build(config)
         case config.type
@@ -15,7 +14,9 @@ class RFlow
     end
 
     attr_accessor :config, :uuid, :name, :options
-    attr_accessor :recv_callback
+    attr_writer :recv_callback
+    protected
+    attr_reader :recv_callback
 
     def initialize(config)
       @config = config
@@ -24,15 +25,13 @@ class RFlow
       @options = config.options
     end
 
-
     # Subclass and implement to be able to handle future 'recv'
     # methods.  Will only be called in the context of a running
     # EventMachine reactor
     def connect_input!
-      raise NotImplementedError, "Raw connections do not support connect_input.  Please subclass and define a connect_output method."
+      raise NotImplementedError, "Raw connections do not support connect_input.  Please subclass and define a connect_input method."
     end
 
-
     # Subclass and implement to be able to handle future 'send'
     # methods.  Will only be called in the context of a running
     # EventMachine reactor
@@ -40,7 +39,6 @@ class RFlow
       raise NotImplementedError, "Raw connections do not support connect_output.  Please subclass and define a connect_output method."
     end
 
-
     # Subclass and implement to handle outgoing messages.  The message
     # will be a RFlow::Message object and the subclasses are expected
     # to marshal it up into something that will be unmarshalled on the
@@ -58,13 +56,52 @@ class RFlow
     def recv_callback
       @recv_callback ||= Proc.new {|message|}
     end
+  end
+
+  # Primarily for testing purposes. Captures whatever messages are sent on it.
+  class MessageCollectingConnection < Connection
+    attr_accessor :messages
+
+    def initialize
+      super(RFlow::Configuration::NullConfiguration.new)
+      @messages = []
+    end
+
+    def send_message(message)
+      @messages << message
+    end
+  end
 
-  end # class Connection
+  # Manually shuffle messages in-process from one output port to another output
+  # port. Can be used to get a Facade pattern effect - to have one component
+  # contain other components within it, shuttling messages in and out without
+  # making the internal component visible to the larger RFlow network.
+  class ForwardToOutputPort < Connection
+    def initialize(receiver, port_name)
+      super(RFlow::Configuration::NullConfiguration.new)
+      @receiver = receiver
+      @port_name = port_name.to_sym
+    end
 
-  class Disconnection < Connection
     def send_message(message)
-      RFlow.logger.debug "Attempting to send without a connection, doing nothing"
+      @receiver.send(@port_name).send_message(message)
     end
   end
 
-end # class RFlow
+  # Manually shuffle messages in-process from one output port to another output
+  # port. Can be used to get a Facade pattern effect - to have one component
+  # contain other components within it, shuttling messages in and out without
+  # making the internal component visible to the larger RFlow network.
+  class ForwardToInputPort < Connection
+    def initialize(receiver, port_name, port_key)
+      super(RFlow::Configuration::NullConfiguration.new)
+      @receiver = receiver
+      @port_name = port_name.to_sym
+      @port_key = port_key
+    end
+
+    def send_message(message)
+      @receiver.process_message(@receiver.send(@port_name), @port_key, self, message)
+    end
+  end
+end

data/lib/rflow/connections.rb

@@ -1,2 +1 @@
-# Load the core connection types
 require 'rflow/connections/zmq_connection'

data/lib/rflow/connections/zmq_connection.rb

@@ -1,12 +1,10 @@
 require 'em-zeromq'
-
 require 'rflow/connection'
 require 'rflow/message'
 
 class RFlow
   module Connections
     class ZMQConnection < RFlow::Connection
-
       class << self
         attr_accessor :zmq_context
 
@@ -18,8 +16,7 @@ class RFlow
           EM::ZeroMQ::Context.new(1)
         end
 
-        # Returns the current ZeroMQ context object or creates it if
-        # it does not exist.
+        # Returns the current ZeroMQ context object or creates it if it does not exist.
         def zmq_context
           @zmq_context ||= create_zmq_context
         end
@@ -27,63 +24,42 @@ class RFlow
 
       def zmq_context; self.class.zmq_context; end
 
+      private
       attr_accessor :input_socket, :output_socket
 
+      public
       def initialize(config)
         super
         validate_options!
-        # Cause the ZMQ context to be created before the reactor is running
-        zmq_context
-      end
-
-
-      def validate_options!
-        # TODO: Normalize/validate configuration
-        missing_options = []
-
-        ['input', 'output'].each do |direction_prefix|
-          ['_socket_type', '_address', '_responsibility'].each do |option_suffix|
-            option_name = "#{direction_prefix}#{option_suffix}"
-            unless options.include? option_name
-              missing_options << option_name
-            end
-          end
-        end
-
-        unless missing_options.empty?
-          raise ArgumentError, "#{self.class.to_s}: configuration missing options: #{missing_options.join ', '}"
-        end
-
-        true
+        zmq_context # cause the ZMQ context to be created before the reactor is running
       end
 
-
       def connect_input!
         RFlow.logger.debug "Connecting input #{uuid} with #{options.find_all {|k, v| k.to_s =~ /input/}}"
         self.input_socket = zmq_context.socket(ZMQ.const_get(options['input_socket_type'].to_sym))
-        input_socket.send(options['input_responsibility'].to_sym,
-                          options['input_address'])
+        input_socket.send(options['input_responsibility'].to_sym, options['input_address'])
 
         input_socket.on(:message) do |*message_parts|
-          message = RFlow::Message.from_avro(message_parts.last.copy_out_string)
-          RFlow.logger.debug "#{name}: Received message of type '#{message_parts.first.copy_out_string}'"
-          message_parts.each { |part| part.close } # avoid memory leaks
-          recv_callback.call(message)
+          begin
+            message = RFlow::Message.from_avro(message_parts.last.copy_out_string)
+            RFlow.logger.debug "#{name}: Received message of type '#{message_parts.first.copy_out_string}'"
+            message_parts.each(&:close) # avoid memory leaks
+            recv_callback.call(message)
+          rescue Exception => e
+            RFlow.logger.error "#{name}: Exception processing message of type '#{message.data_type_name}': #{e.message}, because: #{e.backtrace}"
+          end
         end
 
         input_socket
       end
 
-
       def connect_output!
         RFlow.logger.debug "Connecting output #{uuid} with #{options.find_all {|k, v| k.to_s =~ /output/}}"
         self.output_socket = zmq_context.socket(ZMQ.const_get(options['output_socket_type'].to_sym))
-        output_socket.send(options['output_responsibility'].to_sym,
-                           options['output_address'].to_s)
+        output_socket.send(options['output_responsibility'].to_sym, options['output_address'].to_s)
         output_socket
       end
 
-
       # TODO: fix this tight loop of retries
       def send_message(message)
         RFlow.logger.debug "#{name}: Sending message of type '#{message.data_type_name.to_s}'"
@@ -97,6 +73,26 @@ class RFlow
         end
       end
 
+      private
+      def validate_options!
+        # TODO: Normalize/validate configuration
+        missing_options = []
+
+        ['input', 'output'].each do |direction_prefix|
+          ['_socket_type', '_address', '_responsibility'].each do |option_suffix|
+            option_name = "#{direction_prefix}#{option_suffix}"
+            unless options.include? option_name
+              missing_options << option_name
+            end
+          end
+        end
+
+        unless missing_options.empty?
+          raise ArgumentError, "#{self.class.to_s}: configuration missing options: #{missing_options.join ', '}"
+        end
+
+        true
+      end
     end
   end
 end

data/lib/rflow/daemon_process.rb

@@ -0,0 +1,155 @@
+class RFlow
+  class DaemonProcess
+    def initialize(name, role = name)
+      @name = name
+      @role = role
+    end
+
+    def daemonize!
+      RFlow.logger.info "#{@name} daemonizing"
+      establish_daemon_pipe
+      drop_database_connections
+
+      parent = fork
+      if parent
+        exit_after_daemon_starts
+      else
+        daemonize_process
+      end
+    end
+
+    def run!
+      register_logging_context
+      update_process_name
+      handle_signals
+      spawn_subprocesses
+      signal_successful_start
+
+      RFlow.logger.info "#{@role} started"
+      run_process
+    ensure
+      unhandle_signals
+    end
+
+    def spawn_subprocesses; end
+    def subprocesses; []; end
+
+    def shutdown!(reason)
+      RFlow.logger.info "#{@name} shutting down due to #{reason}"
+      unhandle_signals
+      signal_subprocesses('QUIT')
+      RFlow.logger.info "#{@name} exiting"
+    end
+
+    private
+    def establish_daemon_pipe
+      @daemon_pipe_r, @daemon_pipe_w = IO.pipe
+      [@daemon_pipe_r, @daemon_pipe_w].each {|io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+    end
+
+    # Holding database connections over the fork causes problems. Instead,
+    # let them be automatically restored after the fork.
+    def drop_database_connections
+      ::ActiveRecord::Base.clear_all_connections!
+    end
+
+    def exit_after_daemon_starts
+      @daemon_pipe_w.close
+
+      # Parent waits for a PID on the pipe indicating that the
+      # child successfully started.
+      child_pid = (@daemon_pipe_r.readpartial(16) rescue nil).to_i
+      @daemon_pipe_r.close
+      if child_pid > 1
+        RFlow.logger.info "#{@role} indicated successful daemonization"
+        exit 0
+      else
+        RFlow.logger.error "#{@role} failed to start"
+        exit! 1
+      end
+    end
+
+    def daemonize_process
+      @daemon_pipe_r.close
+      Process.daemon(true, true)
+      close_stdio_streams
+    end
+
+    def close_stdio_streams
+      $stdout.sync = $stderr.sync = true
+      [$stdin, $stdout, $stderr].each do |stream|
+        stream.binmode
+        begin; stream.reopen "/dev/null"; rescue ::Exception; end
+      end
+    end
+
+    def register_logging_context
+      # arrange for process's name to appear in log messages
+      Log4r::NDC.clear
+      Log4r::NDC.push @name
+    end
+
+    def clone_logging_context
+      Log4r::NDC.clone_stack
+    end
+
+    def apply_logging_context(context)
+      Log4r::NDC.inherit(context)
+    end
+
+    def update_process_name
+      # set the visible process name to match the process's name
+      $0 = @name
+    end
+
+    def handle_signals
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD'].each do |signal|
+        trap_signal(signal) do
+          shutdown! signal
+          exit! 0
+        end
+      end
+
+      trap_signal 'SIGUSR1' do
+        RFlow.logger.reopen
+        signal_subprocesses signal
+      end
+
+      trap_signal 'SIGUSR2' do
+        RFlow.logger.toggle_log_level
+        signal_subprocesses signal
+      end
+    end
+
+    def unhandle_signals
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2'].each do |signal|
+        Signal.trap signal, 'DEFAULT'
+      end
+    end
+
+    def trap_signal(signal)
+      # Log4r and traps don't mix, so we need to put it in another thread
+      context = clone_logging_context
+      Signal.trap signal do
+        Thread.new do
+          apply_logging_context context
+          yield
+        end.join
+      end
+    end
+
+    def signal_successful_start
+      if @daemon_pipe_w
+        @daemon_pipe_w.syswrite($$.to_s)
+        @daemon_pipe_w.close rescue nil
+      end
+    end
+
+    def signal_subprocesses(signal)
+      subprocesses.each do |p|
+        RFlow.logger.info "Signaling #{p.name} with #{signal}"
+        Process.kill(signal, p.pid)
+      end
+    end
+  end
+end

data/lib/rflow/logger.rb

@@ -9,37 +9,29 @@ class RFlow
     DATE_METHOD = 'xmlschema(6)'
     LOG_PATTERN_FORMATTER = PatternFormatter.new :pattern => LOG_PATTERN_FORMAT, :date_method => DATE_METHOD
 
-    attr_accessor :config, :internal_logger
-    attr_accessor :log_file_path, :log_level
+    private
+    attr_accessor :internal_logger
+    attr_accessor :log_file_path, :log_level, :log_name
+
+    public
+
+    # make sure Log4r is initialized; ignored if custom levels are already set
+    Log4r.define_levels(*Log4rConfig::LogLevels)
 
     # Delegate log methods to internal logger
-    def_delegators :@internal_logger, :fatal, :error, :warn, :info, :debug
+    def_delegators :@internal_logger,
+      *Log4r::LNAMES.map(&:downcase).map(&:to_sym),
+      *Log4r::LNAMES.map(&:downcase).map {|n| "#{n}?".to_sym }
 
-    def initialize(config, include_stdout=nil)
+    def initialize(config, include_stdout = false)
       @log_file_path = config['rflow.log_file_path']
       @log_level = config['rflow.log_level']
+      @log_name = (config['rflow.application_name'] || File.basename(log_file_path))
 
-      log_name = (config['rflow.application_name'] rescue File.basename(log_file_path))
-      @internal_logger = Log4r::Logger.new(log_name)
-
-      internal_logger.level = LNAMES.index log_level
-      internal_logger.trace = true
-
-      begin
-        internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
-      rescue Exception => e
-        error_message = "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\b#{e.backtrace.join("\n")}"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
-      end
-
-      if include_stdout
-        internal_logger.add StdoutOutputter.new('rflow_stdout', :formatter => LOG_PATTERN_FORMATTER)
-      end
-
-      # RFlow.logger.info "Transitioning to running log file #{log_file_path} at level #{log_level}"
-      Log4r::NDC.clear
-      Log4r::NDC.push(log_name)
+      establish_internal_logger
+      hook_up_logfile
+      hook_up_stdout if include_stdout
+      register_logging_context
 
       internal_logger
     end
@@ -63,5 +55,29 @@ class RFlow
       internal_logger.level = LNAMES.index new_log_level
     end
 
+    private
+    def establish_internal_logger
+      @internal_logger = Log4r::Logger.new(log_name).tap do |logger|
+        logger.level = LNAMES.index log_level
+        logger.trace = true
+      end
+    end
+
+    def hook_up_logfile
+      begin
+        internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
+      rescue Exception => e
+        raise ArgumentError, "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\b#{e.backtrace.join("\n")}"
+      end
+    end
+
+    def hook_up_stdout
+      internal_logger.add StdoutOutputter.new('rflow_stdout', :formatter => LOG_PATTERN_FORMATTER)
+    end
+
+    def register_logging_context
+      Log4r::NDC.clear
+      Log4r::NDC.push(log_name)
+    end
   end
 end