rflow 0.0.5 → 1.0.0a1

data/lib/rflow/configuration/uuid_keyed.rb

@@ -5,12 +5,12 @@ class RFlow
     module UUIDKeyed
       def self.included(base)
         base.class_eval do
-          set_primary_key 'uuid'
+          self.primary_key = 'uuid'
           before_create :generate_uuid
-          
+
           def generate_uuid
             self.uuid = UUIDTools::UUID.random_create.to_s
-          end 
+          end
         end
       end
     end

data/lib/rflow/connection.rb

@@ -2,15 +2,26 @@ require 'rflow/message'
 
 class RFlow
   class Connection
-    attr_accessor :instance_uuid, :name, :configuration, :recv_callback
 
-    # Attribute that holds the 
+    class << self
+      def build(config)
+        case config.type
+        when 'RFlow::Configuration::ZMQConnection'
+          RFlow::Connections::ZMQConnection.new(config)
+        else
+          raise ArgumentError, "Only ZMQConnections currently supported"
+        end
+      end
+    end
+
+    attr_accessor :config, :uuid, :name, :options
     attr_accessor :recv_callback
-    
-    def initialize(connection_instance_uuid, connection_name=nil, connection_configuration={})
-      @instance_uuid = connection_instance_uuid
-      @name = connection_name
-      @configuration = connection_configuration
+
+    def initialize(config)
+      @config = config
+      @uuid = config.uuid
+      @name = config.name
+      @options = config.options
     end
 
 
@@ -29,7 +40,7 @@ class RFlow
       raise NotImplementedError, "Raw connections do not support connect_output.  Please subclass and define a connect_output method."
     end
 
-    
+
     # Subclass and implement to handle outgoing messages.  The message
     # will be a RFlow::Message object and the subclasses are expected
     # to marshal it up into something that will be unmarshalled on the
@@ -47,7 +58,7 @@ class RFlow
     def recv_callback
       @recv_callback ||= Proc.new {|message|}
     end
-    
+
   end # class Connection
 
   class Disconnection < Connection
@@ -55,5 +66,5 @@ class RFlow
       RFlow.logger.debug "Attempting to send without a connection, doing nothing"
     end
   end
-  
+
 end # class RFlow

data/lib/rflow/connections/zmq_connection.rb

@@ -1,6 +1,4 @@
-#require 'ffi'
-#require 'ffi-rzmq'
-require 'em-zeromq-mri'
+require 'em-zeromq'
 
 require 'rflow/connection'
 require 'rflow/message'
@@ -14,88 +12,91 @@ class RFlow
 
         def create_zmq_context
           RFlow.logger.debug "Creating a new ZeroMQ context"
-          unless EM.reactor_running?
-            raise RuntimeError, "EventMachine reactor is not running when attempting to create a ZeroMQ context" 
+          if EM.reactor_running?
+            raise RuntimeError, "EventMachine reactor is running when attempting to create a ZeroMQ context"
           end
           EM::ZeroMQ::Context.new(1)
         end
-        
+
         # Returns the current ZeroMQ context object or creates it if
-        # it does not exist.  Assumes that we are within a running
-        # EventMachine reactor
+        # it does not exist.
         def zmq_context
           @zmq_context ||= create_zmq_context
         end
       end
 
-      attr_accessor :socket
+      def zmq_context; self.class.zmq_context; end
+
+      attr_accessor :input_socket, :output_socket
 
-      REQUIRED_OPTION_SUFFIXES = ['_socket_type', '_address', '_responsibility']
+      def initialize(config)
+        super
+        validate_options!
+        # Cause the ZMQ context to be created before the reactor is running
+        zmq_context
+      end
 
-      def self.configuration_errors(configuration)
+
+      def validate_options!
         # TODO: Normalize/validate configuration
-        missing_config_elements = []
+        missing_options = []
 
         ['input', 'output'].each do |direction_prefix|
-          REQUIRED_OPTION_SUFFIXES.each do |option_suffix|
-            config_element = "#{direction_prefix}#{option_suffix}"
-            unless configuration.include? config_element
-              missing_config_elements << config_element
+          ['_socket_type', '_address', '_responsibility'].each do |option_suffix|
+            option_name = "#{direction_prefix}#{option_suffix}"
+            unless options.include? option_name
+              missing_options << option_name
             end
           end
         end
 
-        missing_config_elements
-      end
-
-      
-      def initialize(connection_instance_uuid, connection_name, connection_configuration)
-        configuration_errors = self.class.configuration_errors(connection_configuration)
-        unless configuration_errors.empty?
-          raise ArgumentError, "#{self.class.to_s}: configuration missing elements: #{configuration_errors.join ', '}"
+        unless missing_options.empty?
+          raise ArgumentError, "#{self.class.to_s}: configuration missing options: #{missing_options.join ', '}"
         end
 
-        super
+        true
       end
 
-      
+
       def connect_input!
-        RFlow.logger.debug "Connecting input #{instance_uuid} with #{configuration.find_all {|k, v| k.to_s =~ /input/}}"
-        self.socket = self.class.zmq_context.send(configuration['input_responsibility'],
-                                                  ZMQ.const_get(configuration['input_socket_type'].to_sym),
-                                                  configuration['input_address'],
-                                                  self)
+        RFlow.logger.debug "Connecting input #{uuid} with #{options.find_all {|k, v| k.to_s =~ /input/}}"
+        self.input_socket = zmq_context.socket(ZMQ.const_get(options['input_socket_type'].to_sym))
+        input_socket.send(options['input_responsibility'].to_sym,
+                          options['input_address'])
+
+        input_socket.on(:message) do |*message_parts|
+          message = RFlow::Message.from_avro(message_parts.last.copy_out_string)
+          RFlow.logger.debug "#{name}: Received message of type '#{message_parts.first.copy_out_string}'"
+          message_parts.each { |part| part.close } # avoid memory leaks
+          recv_callback.call(message)
+        end
+
+        input_socket
       end
 
 
       def connect_output!
-        RFlow.logger.debug "Connecting output #{instance_uuid} with #{configuration.find_all {|k, v| k.to_s =~ /output/}}"
-        self.socket = self.class.zmq_context.send(configuration['output_responsibility'].to_s,
-                                                  ZMQ.const_get(configuration['output_socket_type'].to_sym),
-                                                  configuration['output_address'].to_s,
-                                                  self)
+        RFlow.logger.debug "Connecting output #{uuid} with #{options.find_all {|k, v| k.to_s =~ /output/}}"
+        self.output_socket = zmq_context.socket(ZMQ.const_get(options['output_socket_type'].to_sym))
+        output_socket.send(options['output_responsibility'].to_sym,
+                           options['output_address'].to_s)
+        output_socket
       end
 
 
-      def on_readable(socket, message_parts)
-        message = RFlow::Message.from_avro(message_parts.last.copy_out_string)
-        RFlow.logger.debug "#{name}: Received message of type '#{message_parts.first.copy_out_string}'"
-        recv_callback.call(message)
-      end
-
       # TODO: fix this tight loop of retries
       def send_message(message)
         RFlow.logger.debug "#{name}: Sending message of type '#{message.data_type_name.to_s}'"
 
         begin
-          socket.send_msg(message.data_type_name.to_s, message.to_avro)
+          output_socket.send_msg(message.data_type_name.to_s, message.to_avro)
           RFlow.logger.debug "#{name}: Successfully sent message of type '#{message.data_type_name.to_s}'"
         rescue Exception => e
           RFlow.logger.debug "Exception #{e.class}: #{e.message}, retrying send"
           retry
         end
       end
-      
+
     end
   end
 end

data/lib/rflow/logger.rb

@@ -0,0 +1,67 @@
+require 'log4r'
+
+class RFlow
+  class Logger
+    extend Forwardable
+    include Log4r
+
+    LOG_PATTERN_FORMAT = '%l [%d] %x (%p) - %M'
+    DATE_METHOD = 'xmlschema(6)'
+    LOG_PATTERN_FORMATTER = PatternFormatter.new :pattern => LOG_PATTERN_FORMAT, :date_method => DATE_METHOD
+
+    attr_accessor :config, :internal_logger
+    attr_accessor :log_file_path, :log_level
+
+    # Delegate log methods to internal logger
+    def_delegators :@internal_logger, :fatal, :error, :warn, :info, :debug
+
+    def initialize(config, include_stdout=nil)
+      @log_file_path = config['rflow.log_file_path']
+      @log_level = config['rflow.log_level']
+
+      log_name = (config['rflow.application_name'] rescue File.basename(log_file_path))
+      @internal_logger = Log4r::Logger.new(log_name)
+
+      internal_logger.level = LNAMES.index log_level
+      internal_logger.trace = true
+
+      begin
+        internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
+      rescue Exception => e
+        error_message = "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\b#{e.backtrace.join("\n")}"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+
+      if include_stdout
+        internal_logger.add StdoutOutputter.new('rflow_stdout', :formatter => LOG_PATTERN_FORMATTER)
+      end
+
+      # RFlow.logger.info "Transitioning to running log file #{log_file_path} at level #{log_level}"
+      Log4r::NDC.clear
+      Log4r::NDC.push(log_name)
+
+      internal_logger
+    end
+
+    def reopen
+      # TODO: Make this less of a hack, although Log4r doesn't support
+      # it, so it might be permanent
+      log_file = Outputter['rflow.log_file'].instance_variable_get(:@out)
+      File.open(log_file.path, 'a') { |tmp_log_file| log_file.reopen(tmp_log_file) }
+    end
+
+    def close
+      Outputter['rflow.log_file'].close
+    end
+
+    def toggle_log_level
+      original_log_level = LNAMES[logger.level]
+      new_log_level = (original_log_level == 'DEBUG' ? log_level : 'DEBUG')
+
+      internal_logger.warn "Changing log level from #{original_log_level} to #{new_log_level}"
+      internal_logger.level = LNAMES.index new_log_level
+    end
+
+  end
+end

data/lib/rflow/master.rb

@@ -0,0 +1,127 @@
+require 'rflow/pid_file'
+require 'rflow/shard'
+
+class RFlow
+  class Master
+
+    attr_accessor :name, :pid_file, :ready_write
+    attr_accessor :shards
+
+    def initialize(config)
+      @name = config['rflow.application_name']
+      @pid_file = PIDFile.new(config['rflow.pid_file_path'])
+      @shards = config.shards.map do |shard_config|
+        Shard.new(shard_config)
+      end
+    end
+
+    def handle_signals
+      # Gracefully shutdown on termination signals
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD'].each do |signal|
+        Signal.trap signal do
+          # Log4r and traps don't mix, so we need to put it in another thread
+          Thread.new { shutdown(signal) }.join
+        end
+      end
+
+      # Reopen logs on USR1
+      ['SIGUSR1'].each do |signal|
+        Signal.trap signal do
+          Thread.new do
+            RFlow.logger.reopen
+            signal_workers(signal)
+          end.join
+        end
+      end
+
+      # Toggle log level on USR2
+      ['SIGUSR2'].each do |signal|
+        Signal.trap signal do
+          Thread.new do
+            RFlow.logger.toggle_log_level
+            signal_workers(signal)
+          end.join
+        end
+      end
+    end
+
+    def run
+      Log4r::NDC.clear
+      Log4r::NDC.push name
+      $0 = name
+
+      shards.each {|s| s.run!}
+
+      handle_signals
+
+      # Signal the grandparent that we are running
+      if ready_write
+        ready_write.syswrite($$.to_s)
+        ready_write.close rescue nil
+      end
+
+      pid_file.write
+
+      RFlow.logger.info "Master started"
+
+      EM.run do
+        # TODO: Monitor the workers
+      end
+
+      @pid_file.safe_unlink
+    end
+
+    def daemonize!
+      RFlow.logger.info "#{name} daemonizing"
+
+      ready_read, @ready_write = IO.pipe
+      [ready_read, @ready_write].each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+
+      grandparent = $$
+
+      if fork
+        # Grandparent waits for a PID on the pipe indicating that the
+        # master successfully started.
+        @ready_write.close # grandparent does not write
+        master_pid = (ready_read.readpartial(16) rescue nil).to_i
+        unless master_pid > 1
+          RFlow.logger.error "Master failed to start"
+          exit! 1
+        end
+        RFlow.logger.info "Master indicated successful daemonization"
+        exit 0
+      end
+
+      Process.daemon(true, true)
+
+      ready_read.close # master does not read
+
+      # Close standard IO
+      $stdout.sync = $stderr.sync = true
+      $stdin.binmode; $stdout.binmode; $stderr.binmode
+      begin; $stdin.reopen  "/dev/null"; rescue ::Exception; end
+      begin; $stdout.reopen "/dev/null"; rescue ::Exception; end
+      begin; $stderr.reopen "/dev/null"; rescue ::Exception; end
+
+      $$
+    end
+
+    def signal_workers(signal)
+      shards.each do |shard|
+        shard.workers.each do |worker|
+          RFlow.logger.info "Signalling #{worker.name} with #{signal}"
+          Process.kill(signal, worker.pid)
+        end
+      end
+    end
+
+    def shutdown(reason)
+      RFlow.logger.info "#{name} shutting down due to #{reason}"
+      signal_workers('QUIT')
+      pid_file.safe_unlink
+      RFlow.logger.info "#{name} exiting"
+      exit 0
+    end
+
+  end
+end

data/lib/rflow/message.rb

@@ -13,7 +13,7 @@ class RFlow
 
     class << self
       def avro_message_schema; @avro_message_schema ||= Avro::Schema.parse(File.read(File.join(File.dirname(__FILE__), '..', '..', 'schema', 'message.avsc'))); end
-      
+
       def avro_reader;  @avro_reader  ||= Avro::IO::DatumReader.new(avro_message_schema, avro_message_schema); end
       def avro_writer;  @avro_writer  ||= Avro::IO::DatumWriter.new(avro_message_schema); end
       def avro_decoder(io_object); Avro::IO::BinaryDecoder.new(io_object); end
@@ -29,8 +29,8 @@ class RFlow
                     message_hash['data'])
       end
     end
-    
-    
+
+
     # Serialize the current message object to Avro using the
     # org.rflow.Message Avro schema.  Note that we have to manually
     # set the encoding for Ruby 1.9, otherwise the stringio would use
@@ -52,14 +52,14 @@ class RFlow
       self.class.avro_writer.write deserialized_avro_object, self.class.avro_encoder(avro_serialized_message_bytes_stringio)
       avro_serialized_message_bytes
     end
-    
+
 
     attr_reader :data_type_name
     attr_accessor :processing_event
     attr_accessor :provenance
     attr_reader :data, :data_extensions
 
-    
+
     def initialize(data_type_name, provenance=[], data_serialization_type='avro', data_schema_string=nil, serialized_data_object=nil)
       # Default the values, in case someone puts in a nil instead of
       # the default
@@ -79,7 +79,7 @@ class RFlow
                               processing_event_hash_or_object['context'])
         end
       end
-      
+
       # TODO: Make this better.  This check is technically
       # unnecessary, as we are able to completely desrialize the
       # message without needing to resort to the registered schema.
@@ -97,7 +97,7 @@ class RFlow
         RFlow.logger.error error_message
         raise ArgumentError, error_message
       end
-      
+
       @data = Data.new(registered_data_schema_string, data_serialization_type.to_s, serialized_data_object)
 
       # Get the extensions and apply them to the data object to add capability
@@ -115,16 +115,16 @@ class RFlow
       def initialize(component_instance_uuid, started_at=nil, completed_at=nil, context=nil)
         @component_instance_uuid = component_instance_uuid
         @started_at = case started_at
-                      when String then Time.xmlschema(started_at) 
+                      when String then Time.xmlschema(started_at)
                       when Time   then started_at
                       else; nil; end
         @completed_at = case completed_at
-                        when String then Time.xmlschema(completed_at) 
+                        when String then Time.xmlschema(completed_at)
                         when Time   then completed_at
                         else; nil; end
         @context = context
       end
-      
+
       def to_hash
         {
           'component_instance_uuid' => component_instance_uuid.to_s,
@@ -134,7 +134,7 @@ class RFlow
         }
       end
     end
-    
+
     # Should proxy most methods to data_object that we can serialize
     # to avro using the schema.  Extensions should use 'extended' hook
     # to apply immediate changes.
@@ -159,7 +159,7 @@ class RFlow
           RFlow.logger.error error_message
           raise ArgumentError, error_message
         end
-        
+
         if serialized_data_object
           serialized_data_object.force_encoding 'BINARY'
           avro_decoder = Avro::IO::BinaryDecoder.new StringIO.new(serialized_data_object)
@@ -170,7 +170,7 @@ class RFlow
       def valid?
         Avro::Schema.validate @schema, @data_object
       end
-      
+
       def to_avro
         serialized_data_object_bytes = ''
         serialized_data_object_bytes.force_encoding 'BINARY'
@@ -186,6 +186,6 @@ class RFlow
         @data_object.send(method_sym, *args, &block)
       end
     end
-      
+
   end
 end