rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow.rb

@@ -1,68 +1,60 @@
 require "rubygems"
 require "bundler/setup"
-
 require 'time'
-
 require 'active_record'
 require 'eventmachine'
 require 'sqlite3'
-
 require 'rflow/configuration'
-
 require 'rflow/master'
 require 'rflow/message'
-
 require 'rflow/components'
 require 'rflow/connections'
-
 require 'rflow/logger'
 
 class RFlow
   include Log4r
 
-  class Error < StandardError; end
-
   class << self
-    attr_accessor :config_database_path
     attr_accessor :logger
-    attr_accessor :configuration
-    attr_accessor :master
+    attr_reader :configuration, :master
   end
 
-  def self.run(config_database_path=nil, daemonize=nil)
-    self.configuration = Configuration.new(config_database_path)
+  def self.run!(config_database_path = nil, daemonize = false)
+    @config_database_path = config_database_path
+    @daemonize = daemonize
 
-    if config_database_path
-      # First change to the config database directory, which might hold
-      # relative paths for the other files/directories, such as the
-      # application_directory_path
-      Dir.chdir File.dirname(config_database_path)
-    end
+    establish_configuration
+    chdir_application_directory
+    setup_logger
+    start_master_node
+  rescue SystemExit => e
+    # Do nothing, just prevent a normal exit from causing an unsightly
+    # error in the logs
+  end
 
-    # Bail unless you have some of the basic information.  TODO:
-    # rethink this when things get more dynamic
+  private
+  def self.establish_configuration
+    @configuration = Configuration.new(@config_database_path)
     unless configuration['rflow.application_directory_path']
-      error_message = "Empty configuration database!  Use a view/controller (such as the RubyDSL) to create a configuration"
-      RFlow.logger.error "Empty configuration database!  Use a view/controller (such as the RubyDSL) to create a configuration"
-      raise ArgumentError, error_message
+      raise ArgumentError, "Empty configuration database!  Use a view/controller (such as the RubyDSL) to create a configuration"
     end
+  end
 
+  def self.chdir_application_directory
+    # First change to the config db directory, which might hold
+    # relative paths for the other files/directories
+    Dir.chdir(File.dirname(@config_database_path)) if @config_database_path
     Dir.chdir configuration['rflow.application_directory_path']
+  end
 
-    self.logger = RFlow::Logger.new(configuration, !daemonize)
-    @master = Master.new(configuration)
-
-    master.daemonize! if daemonize
-    master.run # Runs EM and doesn't return
-
-    # Should never get here
-    logger.warn "going down"
-  rescue SystemExit => e
-    # Do nothing, just prevent a normal exit from causing an unsightly
-    # error in the logs
-  rescue Exception => e
-    logger.fatal "Exception caught: #{e.class} - #{e.message}\n#{e.backtrace.join "\n"}"
-    exit 1
+  def self.setup_logger
+    include_stdout = !@daemonize
+    self.logger = RFlow::Logger.new(configuration, include_stdout)
   end
 
-end # class RFlow
+  def self.start_master_node
+    @master = Master.new(configuration)
+    master.daemonize! if @daemonize
+    master.run! # blocks until EventMachine stops
+  end
+end

data/lib/rflow/child_process.rb

@@ -0,0 +1,112 @@
+class RFlow
+  class ChildProcess
+    attr_reader :pid, :name
+
+    def initialize(name, role = name)
+      @name = name
+      @role = role
+    end
+
+    # Launch another process to execute the child. The parent
+    # process retains the original worker object (with pid and IPC
+    # pipe) to allow for process management
+    def spawn!
+      establish_child_pipe
+      drop_database_connections
+
+      @pid = fork
+      if @pid
+        return_after_child_starts
+      else
+        run_child_process
+      end
+    end
+
+    def run_child_process
+      @child_pipe_w.close
+      register_logging_context
+      update_process_name
+      handle_signals
+
+      RFlow.logger.info "#{@role} started"
+      run_process
+      exit 0
+    ensure
+      unhandle_signals
+    end
+
+    def shutdown!(signal)
+      RFlow.logger.info "Shutting down #{@name} due to #{signal}"
+      unhandle_signals
+    end
+
+    private
+    def establish_child_pipe
+      @child_pipe_r, @child_pipe_w = IO.pipe
+      [@child_pipe_r, @child_pipe_w].each {|io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+    end
+
+    # Holding database connections over the fork causes problems. Instead,
+    # let them be automatically restored after the fork.
+    def drop_database_connections
+      ::ActiveRecord::Base.clear_all_connections!
+    end
+
+    def return_after_child_starts
+      @child_pipe_r.close
+      self
+    end
+
+    def register_logging_context
+      # arrange for child's name to appear in log messages
+      Log4r::NDC.push @name
+    end
+
+    def clone_logging_context
+      Log4r::NDC.clone_stack
+    end
+
+    def apply_logging_context(context)
+      Log4r::NDC.inherit(context)
+    end
+
+    def update_process_name
+      # set the visible process name to match the child's name
+      $0 += " #{@name}"
+    end
+
+    def handle_signals
+      ['SIGTERM', 'SIGINT', 'SIGQUIT'].each do |signal|
+        trap_signal(signal) do
+          shutdown! signal
+          exit! 0
+        end
+      end
+
+      trap_signal 'SIGUSR1' do
+        RFlow.logger.reopen
+      end
+
+      trap_signal 'SIGUSR2' do
+        RFlow.logger.toggle_log_level
+      end
+    end
+
+    def unhandle_signals
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2'].each do |signal|
+        Signal.trap signal, 'DEFAULT'
+      end
+    end
+
+    def trap_signal(signal)
+      # Log4r and traps don't mix, so we need to put it in another thread
+      context = clone_logging_context
+      Signal.trap signal do
+        Thread.new do
+          apply_logging_context context
+          yield
+        end.join
+      end
+    end
+  end
+end

data/lib/rflow/component.rb

@@ -1,64 +1,42 @@
 require 'ostruct'
-
 require 'rflow/message'
 require 'rflow/component/port'
 
 class RFlow
   class Component
-    # Keep track of available component subclasses
-    def self.inherited(subclass)
-      RFlow::Configuration.add_available_component(subclass)
-    end
-
-
-    # The Component class methods used in the creation of a component
     class << self
-      def defined_input_ports
-        @defined_input_ports ||= Hash.new
+      # Keep track of available component subclasses
+      def inherited(subclass)
+        RFlow::Configuration.add_available_component(subclass)
       end
 
-      def defined_output_ports
-        @defined_output_ports ||= Hash.new
-      end
-
-      # TODO: Update the class vs instance stuffs here to be correct
-      # Port defintions only have names
-
-      # TODO: consider class-based UUIDs to identify component types
-
       # Define an input port with a given name
-      def input_port(port_name)
-        define_port(defined_input_ports, port_name)
-      end
+      def input_port(name); define_port(defined_input_ports, name); end
 
       # Define an output port with a given name
-      def output_port(port_name)
-        define_port(defined_output_ports, port_name)
-      end
+      def output_port(name); define_port(defined_output_ports, name); end
+
+      def defined_input_ports; @defined_input_ports ||= {}; end
+      def defined_output_ports; @defined_output_ports ||= {}; end
 
-      # Helper method to keep things DRY for standard component
-      # definition methods input_port and output_port
-      def define_port(collection, port_name)
-        collection[port_name.to_s] = true
+      def define_port(collection, name)
+        collection[name.to_s] = true
 
         # Create the port accessor method based on the port name
-        define_method port_name.to_s.to_sym do
-          port = ports.by_name[port_name.to_s]
+        define_method name.to_s.to_sym do
+          port = ports.by_name[name.to_s]
           return port if port
 
           # If the port was not connected, return a port-like object
           # that can respond/log but doesn't send any data.  Note,
           # it won't be available in the 'by_uuid' collection, as it
           # doesn't have a configured uuid
-          RFlow.logger.debug "'#{self.name}##{port_name}' not connected, creating a disconnected port"
+          RFlow.logger.debug "'#{self.name}##{name}' not connected, creating a disconnected port"
 
-          disconnected_port = DisconnectedPort.new(OpenStruct.new(:name => port_name, :uuid => 0))
-          ports << disconnected_port
-          disconnected_port
+          DisconnectedPort.new(OpenStruct.new(:name => name, :uuid => 0)).tap {|d| ports << d }
         end
       end
 
-
       # Attempt to instantiate a component described by the config
       # specification. This assumes that the specification of a
       # component is a fully qualified Ruby class that has already
@@ -68,37 +46,28 @@ class RFlow
       # class. Future releases will support external (i.e. non-managed
       # components), but the current stuff only supports Ruby classes
       def build(config)
-        if config.managed?
-          RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
-          begin
-            RFlow.logger.debug RFlow.configuration.available_components.inspect
-            instantiated_component = if RFlow.configuration.available_components.include? config.specification
-                                       RFlow.logger.debug "Component found in configuration.available_components['#{config.specification}']"
-                                       RFlow.configuration.available_components[config.specification].new(config)
-                                     else
-                                       RFlow.logger.debug "Component not found in configuration.available_components, constantizing component '#{config.specification}'"
-                                       config.specification.constantize.new(config)
-                                     end
-          rescue NameError => e
-            error_message = "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' was not found"
-            RFlow.logger.error error_message
-            raise RuntimeError, error_message
-          rescue Exception => e
-            error_message = "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}"
-            RFlow.logger.error error_message
-            raise RuntimeError, error_message
+        raise NotImplementedError, "Non-managed components not yet implemented for component '#{config.name}' as '#{config.specification}' (#{config.uuid})" unless config.managed?
+
+        RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
+        begin
+          component = RFlow.configuration.available_components[config.specification]
+
+          if component
+            RFlow.logger.debug "Component found in configuration.available_components['#{config.specification}']"
+            component.new(config)
+          else
+            RFlow.logger.debug "Component not found in configuration.available_components, constantizing component '#{config.specification}'"
+            config.specification.constantize.new(config)
           end
-        else
-          error_message = "Non-managed components not yet implemented for component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
-          RFlow.logger.error error_message
-          raise NotImplementedError, error_message
+        rescue NameError => e
+          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' was not found"
+        rescue Exception => e
+          raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}"
         end
-
-        instantiated_component
       end
     end
 
-    attr_reader :config, :uuid, :name, :ports
+    attr_reader :uuid, :name, :ports
 
     def initialize(config)
       @config = config
@@ -108,114 +77,40 @@ class RFlow
 
       configure_ports!
       configure_connections!
-      configure!(config.options)
     end
 
-
     # Returns a list of connected input ports.  Each port will have
     # one or more keys associated with a particular connection.
-    def input_ports
-      ports.by_type["RFlow::Component::InputPort"]
-    end
-
+    def input_ports; ports.by_type["RFlow::Component::InputPort"]; end
 
     # Returns a list of connected output ports.  Each port will have
     # one or more keys associated with the particular connection.
-    def output_ports
-      ports.by_type["RFlow::Component::OutputPort"]
-    end
-
+    def output_ports; ports.by_type["RFlow::Component::OutputPort"]; end
 
     # Returns a list of disconnected output ports.
-    def disconnected_ports
-      ports.by_type["RFlow::Component::DisconnectedPort"]
-    end
-
-
-    def configure_ports!
-      # Send the port configuration to each component
-      config.input_ports.each do |input_port_config|
-        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with input port '#{input_port_config.name}' (#{input_port_config.uuid})"
-        configure_input_port!(input_port_config)
-      end
-
-      config.output_ports.each do |output_port_config|
-        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with output port '#{output_port_config.name}' (#{output_port_config.uuid})"
-        configure_output_port!(output_port_config)
-      end
-    end
-
-
-    def configure_input_port!(port_config)
-      unless self.class.defined_input_ports.include? port_config.name
-        raise ArgumentError, "Input port '#{port_config.name}' not defined on component '#{self.class}'"
-      end
-      ports << InputPort.new(port_config)
-    end
-
-
-    def configure_output_port!(port_config)
-      unless self.class.defined_output_ports.include? port_config.name
-        raise ArgumentError, "Output port '#{port_config.name}' not defined on component '#{self.class}'"
-      end
-      ports << OutputPort.new(port_config)
-    end
-
+    def disconnected_ports; ports.by_type["RFlow::Component::DisconnectedPort"]; end
 
-    def configure_connections!
-      config.input_ports.each do |input_port_config|
-        input_port_config.input_connections.each do |input_connection_config|
-          RFlow.logger.debug "Configuring input port '#{input_port_config.name}' (#{input_port_config.uuid}) key '#{input_connection_config.input_port_key}' with #{input_connection_config.type.to_s} connection '#{input_connection_config.name}' (#{input_connection_config.uuid})"
-          ports.by_uuid[input_port_config.uuid].add_connection(input_connection_config.input_port_key, Connection.build(input_connection_config))
-        end
-      end
-
-      config.output_ports.each do |output_port_config|
-        output_port_config.output_connections.each do |output_connection_config|
-          RFlow.logger.debug "Configuring output port '#{output_port_config.name}' (#{output_port_config.uuid}) key '#{output_connection_config.output_port_key}' with #{output_connection_config.type.to_s} connection '#{output_connection_config.name}' (#{output_connection_config.uuid})"
-          ports.by_uuid[output_port_config.uuid].add_connection(output_connection_config.output_port_key, Connection.build(output_connection_config))
-        end
-      end
-    end
-
-
-    # Tell the component to establish it's ports' connections, i.e. make
+    # Tell the component to establish its ports' connections, i.e. make
     # the connection.  Uses the underlying connection object.  Also
     # establishes the callbacks for each of the input ports
     def connect!
-      input_ports.each do |input_port|
-        input_port.connect!
-
-        # Create the callbacks for recieving messages as a proc
-        input_port.keys.each do |input_port_key|
-          keyed_connections = input_port[input_port_key]
-          keyed_connections.each do |connection|
-            connection.recv_callback = Proc.new do |message|
-              process_message(input_port, input_port_key, connection, message)
-            end
-          end
-        end
-      end
-
-      output_ports.each do |output_port|
-        output_port.connect!
-      end
+      input_ports.each {|port| port.recv_callback = method(:process_message) }
+      input_ports.each(&:connect!)
+      output_ports.each(&:connect!)
     end
 
-
     def to_s
       string = "Component '#{name}' (#{uuid})\n"
       ports.each do |port|
-        port.keys.each do |port_key|
-          port[port_key].each do |connection|
-            string << "\t#{port.class.to_s} '#{port.name}' (#{port.uuid}) key '#{port_key}' connection '#{connection.name}' (#{connection.uuid})\n"
+        port.keys.each do |key|
+          port[key].each do |connection|
+            string << "\t#{port.class.to_s} '#{port.name}' (#{port.uuid}) key '#{key}' connection '#{connection.name}' (#{connection.uuid})\n"
           end
         end
       end
       string
     end
 
-
     # Method that should be overridden by a subclass to provide for
     # component-specific configuration.  The subclass should use the
     # self.configuration attribute (@configuration) to store its
@@ -243,5 +138,39 @@ class RFlow
     # cleanup any leftover state, e.g. flush file handles, etc
     def cleanup!; end
 
-  end # class Component
-end # class RFlow
+    private
+    def configure_ports!
+      @config.input_ports.each do |p|
+        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with input port '#{p.name}' (#{p.uuid})"
+        unless self.class.defined_input_ports.include? p.name
+          raise ArgumentError, "Input port '#{p.name}' not defined on component '#{self.class}'"
+        end
+        ports << InputPort.new(p)
+      end
+
+      @config.output_ports.each do |p|
+        RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with output port '#{p.name}' (#{p.uuid})"
+        unless self.class.defined_output_ports.include? p.name
+          raise ArgumentError, "Output port '#{p.name}' not defined on component '#{self.class}'"
+        end
+        ports << OutputPort.new(p)
+      end
+    end
+
+    def configure_connections!
+      @config.input_ports.each do |p|
+        p.input_connections.each do |c|
+          RFlow.logger.debug "Configuring input port '#{p.name}' (#{p.uuid}) key '#{c.input_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
+          ports.by_uuid[p.uuid].add_connection c.input_port_key, Connection.build(c)
+        end
+      end
+
+      @config.output_ports.each do |p|
+        p.output_connections.each do |c|
+          RFlow.logger.debug "Configuring output port '#{p.name}' (#{p.uuid}) key '#{c.output_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
+          ports.by_uuid[p.uuid].add_connection c.output_port_key, Connection.build(c)
+        end
+      end
+    end
+  end
+end