rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow/master.rb

@@ -1,127 +1,45 @@
+require 'rflow/daemon_process'
 require 'rflow/pid_file'
 require 'rflow/shard'
 
 class RFlow
-  class Master
-
-    attr_accessor :name, :pid_file, :ready_write
-    attr_accessor :shards
+  class Master < DaemonProcess
+    attr_reader :shards
 
     def initialize(config)
-      @name = config['rflow.application_name']
+      super(config['rflow.application_name'], 'Master')
       @pid_file = PIDFile.new(config['rflow.pid_file_path'])
-      @shards = config.shards.map do |shard_config|
-        Shard.new(shard_config)
-      end
+      @shards = config.shards.map {|config| Shard.new(config) }
     end
 
-    def handle_signals
-      # Gracefully shutdown on termination signals
-      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD'].each do |signal|
-        Signal.trap signal do
-          # Log4r and traps don't mix, so we need to put it in another thread
-          Thread.new { shutdown(signal) }.join
-        end
-      end
-
-      # Reopen logs on USR1
-      ['SIGUSR1'].each do |signal|
-        Signal.trap signal do
-          Thread.new do
-            RFlow.logger.reopen
-            signal_workers(signal)
-          end.join
-        end
-      end
-
-      # Toggle log level on USR2
-      ['SIGUSR2'].each do |signal|
-        Signal.trap signal do
-          Thread.new do
-            RFlow.logger.toggle_log_level
-            signal_workers(signal)
-          end.join
-        end
-      end
+    def run!
+      write_pid_file
+      super
+    ensure
+      remove_pid_file
     end
 
-    def run
-      Log4r::NDC.clear
-      Log4r::NDC.push name
-      $0 = name
-
-      shards.each {|s| s.run!}
-
-      handle_signals
-
-      # Signal the grandparent that we are running
-      if ready_write
-        ready_write.syswrite($$.to_s)
-        ready_write.close rescue nil
-      end
-
-      pid_file.write
-
-      RFlow.logger.info "Master started"
-
-      EM.run do
-        # TODO: Monitor the workers
-      end
-
-      @pid_file.safe_unlink
+    def spawn_subprocesses
+      shards.each(&:run!)
     end
 
-    def daemonize!
-      RFlow.logger.info "#{name} daemonizing"
-
-      ready_read, @ready_write = IO.pipe
-      [ready_read, @ready_write].each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
-
-      grandparent = $$
-
-      if fork
-        # Grandparent waits for a PID on the pipe indicating that the
-        # master successfully started.
-        @ready_write.close # grandparent does not write
-        master_pid = (ready_read.readpartial(16) rescue nil).to_i
-        unless master_pid > 1
-          RFlow.logger.error "Master failed to start"
-          exit! 1
-        end
-        RFlow.logger.info "Master indicated successful daemonization"
-        exit 0
-      end
-
-      Process.daemon(true, true)
-
-      ready_read.close # master does not read
-
-      # Close standard IO
-      $stdout.sync = $stderr.sync = true
-      $stdin.binmode; $stdout.binmode; $stderr.binmode
-      begin; $stdin.reopen  "/dev/null"; rescue ::Exception; end
-      begin; $stdout.reopen "/dev/null"; rescue ::Exception; end
-      begin; $stderr.reopen "/dev/null"; rescue ::Exception; end
-
-      $$
+    def subprocesses
+      shards.flat_map(&:workers)
     end
 
-    def signal_workers(signal)
-      shards.each do |shard|
-        shard.workers.each do |worker|
-          RFlow.logger.info "Signalling #{worker.name} with #{signal}"
-          Process.kill(signal, worker.pid)
-        end
+    def run_process
+      EM.run do
+        # TODO: Monitor the workers
       end
     end
 
-    def shutdown(reason)
-      RFlow.logger.info "#{name} shutting down due to #{reason}"
-      signal_workers('QUIT')
-      pid_file.safe_unlink
-      RFlow.logger.info "#{name} exiting"
-      exit 0
+    def shutdown!(reason)
+      remove_pid_file
+      super
     end
 
+    private
+    def write_pid_file; @pid_file.write; end
+    def remove_pid_file; @pid_file.safe_unlink; end
   end
 end

data/lib/rflow/message.rb

@@ -1,127 +1,108 @@
 require 'stringio'
 require 'time'
-
 require 'avro'
-
 require 'rflow/configuration'
 
 class RFlow
+  class Avro
+    def self.decode(reader, bytes)
+      reader.read ::Avro::IO::BinaryDecoder.new(StringIO.new(bytes.force_encoding('BINARY')))
+    end
 
-  # TODO: reduce reliance/expectation on avro serialization in method
-  # names and such.
-  class Message
+    def self.encode(writer, message)
+      String.new.force_encoding('BINARY').tap do |result|
+        writer.write message, ::Avro::IO::BinaryEncoder.new(StringIO.new(result, 'w'))
+      end
+    end
+  end
 
+  class Message
     class << self
-      def avro_message_schema; @avro_message_schema ||= Avro::Schema.parse(File.read(File.join(File.dirname(__FILE__), '..', '..', 'schema', 'message.avsc'))); end
-
-      def avro_reader;  @avro_reader  ||= Avro::IO::DatumReader.new(avro_message_schema, avro_message_schema); end
-      def avro_writer;  @avro_writer  ||= Avro::IO::DatumWriter.new(avro_message_schema); end
-      def avro_decoder(io_object); Avro::IO::BinaryDecoder.new(io_object); end
-      def avro_encoder(io_object); Avro::IO::BinaryEncoder.new(io_object); end
+      def schema; @schema ||= ::Avro::Schema.parse(File.read(File.join(File.dirname(__FILE__), '..', '..', 'schema', 'message.avsc'))); end
+      def message_reader; @message_reader ||= ::Avro::IO::DatumReader.new(schema, schema); end
+      def message_writer; @message_writer ||= ::Avro::IO::DatumWriter.new(schema); end
+      def encode(message); RFlow::Avro.encode(message_writer, message); end
 
       # Take in an Avro serialization of a message and return a new
       # Message object.  Assumes the org.rflow.Message Avro schema.
-      def from_avro(avro_serialized_message_byte_string)
-        avro_serialized_message_byte_stringio = StringIO.new(avro_serialized_message_byte_string)
-        message_hash = avro_reader.read avro_decoder(avro_serialized_message_byte_stringio)
-        Message.new(message_hash['data_type_name'], message_hash['provenance'],
-                    message_hash['data_serialization_type'], message_hash['data_schema'],
-                    message_hash['data'])
+      def from_avro(bytes)
+        message = RFlow::Avro.decode(message_reader, bytes)
+        Message.new(message['data_type_name'], message['provenance'],
+                    message['data_serialization_type'], message['data_schema'],
+                    message['data'])
       end
     end
 
-
-    # Serialize the current message object to Avro using the
-    # org.rflow.Message Avro schema.  Note that we have to manually
-    # set the encoding for Ruby 1.9, otherwise the stringio would use
-    # UTF-8 by default, which would not work correctly, as a serialize
-    # avro string is BINARY, not UTF-8
-    def to_avro
-      avro_serialized_message_bytes = ''
-      avro_serialized_message_bytes.force_encoding 'BINARY'
-      avro_serialized_message_bytes_stringio = StringIO.new(avro_serialized_message_bytes, 'w')
-
-      deserialized_avro_object = {
-        'data_type_name' => self.data_type_name.to_s,
-        'provenance' => self.provenance.map(&:to_hash),
-        'data_serialization_type' => self.data.serialization_type.to_s,
-        'data_schema' => self.data.schema_string,
-        'data' => self.data.to_avro
-      }
-
-      self.class.avro_writer.write deserialized_avro_object, self.class.avro_encoder(avro_serialized_message_bytes_stringio)
-      avro_serialized_message_bytes
-    end
-
-
-    attr_reader :data_type_name
-    attr_accessor :processing_event
+    attr_reader :data_type_name, :data
     attr_accessor :provenance
-    attr_reader :data, :data_extensions
 
-
-    def initialize(data_type_name, provenance=[], data_serialization_type='avro', data_schema_string=nil, serialized_data_object=nil)
-      # Default the values, in case someone puts in a nil instead of
-      # the default
+    def initialize(data_type_name, provenance = [], serialization_type = 'avro', schema = nil, serialized_data = nil)
       @data_type_name = data_type_name.to_s
 
-      # Turn the provenance array of Hashes into an array of
-      # ProcessingEvents for easier access and time validation.  TODO:
-      # do this lazily so as not to create/destroy objects that are
-      # never used
-      @provenance = (provenance || []).map do |processing_event_hash_or_object|
-        if processing_event_hash_or_object.is_a? ProcessingEvent
-          processing_event_hash_or_object
-        else
-          ProcessingEvent.new(processing_event_hash_or_object['component_instance_uuid'],
-                              processing_event_hash_or_object['started_at'],
-                              processing_event_hash_or_object['completed_at'],
-                              processing_event_hash_or_object['context'])
-        end
-      end
-
       # TODO: Make this better.  This check is technically
-      # unnecessary, as we are able to completely desrialize the
+      # unnecessary, as we are able to completely deserialize the
       # message without needing to resort to the registered schema.
-      registered_data_schema_string = RFlow::Configuration.available_data_types[@data_type_name][data_serialization_type.to_s]
-      unless registered_data_schema_string
-        error_message = "Data type '#{@data_type_name}' with serialization_type '#{data_serialization_type}' not found"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
+      registered_schema = RFlow::Configuration.available_data_types[@data_type_name][serialization_type.to_s]
+      unless registered_schema
+        raise ArgumentError, "Data type '#{@data_type_name}' with serialization_type '#{serialization_type}' not found"
       end
 
       # TODO: think about registering the schemas automatically if not
       # found in Configuration
-      if data_schema_string && (registered_data_schema_string != data_schema_string)
-        error_message = "Passed schema ('#{data_schema_string}') does not equal registered schema ('#{registered_data_schema_string}') for data type '#{@data_type_name}' with serialization_type '#{data_serialization_type}'"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
+      if schema && (registered_schema != schema)
+        raise ArgumentError, "Passed schema ('#{schema}') does not equal registered schema ('#{registered_schema}') for data type '#{@data_type_name}' with serialization_type '#{serialization_type}'"
+      end
+
+      # Turn the provenance array of Hashes into an array of
+      # ProcessingEvents for easier access and time validation.
+      # TODO: do this lazily so as not to create/destroy objects that are
+      # never used
+      @provenance = (provenance || []).map do |event|
+        if event.is_a? ProcessingEvent
+          event
+        else
+          ProcessingEvent.new(event['component_instance_uuid'],
+                              event['started_at'], event['completed_at'],
+                              event['context'])
+        end
       end
 
-      @data = Data.new(registered_data_schema_string, data_serialization_type.to_s, serialized_data_object)
+      @data = Data.new(registered_schema, serialization_type.to_s, serialized_data)
 
       # Get the extensions and apply them to the data object to add capability
-      @data_extensions = RFlow::Configuration.available_data_extensions[@data_type_name]
-      @data_extensions.each do |data_extension|
-        RFlow.logger.debug "Extending '#{data_type_name}' with extension '#{data_extension}'"
-        @data.extend data_extension
+      RFlow::Configuration.available_data_extensions[@data_type_name].each do |e|
+        RFlow.logger.debug "Extending '#{data_type_name}' with extension '#{e}'"
+        @data.extend e
       end
     end
 
+    # Serialize the current message object to Avro using the
+    # org.rflow.Message Avro schema.  Note that we have to manually
+    # set the encoding for Ruby 1.9, otherwise the stringio would use
+    # UTF-8 by default, which would not work correctly, as a serialize
+    # avro string is BINARY, not UTF-8
+    def to_avro
+      Message.encode('data_type_name' => data_type_name.to_s,
+                     'provenance' => provenance.map(&:to_hash),
+                     'data_serialization_type' => data.serialization_type.to_s,
+                     'data_schema' => data.schema_string,
+                     'data' => data.to_avro)
+    end
 
     class ProcessingEvent
-      attr_accessor :component_instance_uuid, :started_at, :completed_at, :context
+      attr_reader :component_instance_uuid, :started_at
+      attr_accessor :completed_at, :context
 
-      def initialize(component_instance_uuid, started_at=nil, completed_at=nil, context=nil)
+      def initialize(component_instance_uuid, started_at = nil, completed_at = nil, context = nil)
         @component_instance_uuid = component_instance_uuid
         @started_at = case started_at
-                      when String then Time.xmlschema(started_at)
-                      when Time   then started_at
-                      else; nil; end
+                      when String; Time.xmlschema(started_at)
+                      when Time; started_at
+                      else nil; end
         @completed_at = case completed_at
-                        when String then Time.xmlschema(completed_at)
-                        when Time   then completed_at
-                        else; nil; end
+                        when String; Time.xmlschema(completed_at)
+                        when Time; completed_at
+                        else nil; end
         @context = context
       end
 
@@ -142,41 +123,31 @@ class RFlow
       attr_reader :schema_string, :schema, :serialization_type
       attr_accessor :data_object
 
-      def initialize(schema_string, serialization_type='avro', serialized_data_object=nil)
-        unless serialization_type == 'avro'
-          error_message = "Only Avro serialization_type supported at the moment"
-          RFlow.logger.error error_message
-          raise ArgumentError, error_message
-        end
+      def initialize(schema_string, serialization_type = 'avro', serialized_data = nil)
+        raise ArgumentError, "Only Avro serialization_type supported at the moment" unless serialization_type.to_s == 'avro'
 
         @schema_string = schema_string
-        @serialization_type = serialization_type
+        @serialization_type = serialization_type.to_s
 
         begin
-          @schema = Avro::Schema.parse(schema_string)
+          @schema = ::Avro::Schema.parse(schema_string)
+          @writer = ::Avro::IO::DatumWriter.new(@schema)
         rescue Exception => e
-          error_message = "Invalid schema '#{@schema_string}': #{e}: #{e.message}"
-          RFlow.logger.error error_message
-          raise ArgumentError, error_message
+          raise ArgumentError, "Invalid schema '#{@schema_string}': #{e}: #{e.message}"
         end
 
-        if serialized_data_object
-          serialized_data_object.force_encoding 'BINARY'
-          avro_decoder = Avro::IO::BinaryDecoder.new StringIO.new(serialized_data_object)
-          @data_object = Avro::IO::DatumReader.new(schema, schema).read avro_decoder
+        if serialized_data
+          serialized_data.force_encoding 'BINARY'
+          @data_object = RFlow::Avro.decode(::Avro::IO::DatumReader.new(schema, schema), serialized_data)
         end
       end
 
       def valid?
-        Avro::Schema.validate @schema, @data_object
+        ::Avro::Schema.validate @schema, @data_object
       end
 
       def to_avro
-        serialized_data_object_bytes = ''
-        serialized_data_object_bytes.force_encoding 'BINARY'
-        serialized_data_object_bytes_stringio = StringIO.new(serialized_data_object_bytes)
-        Avro::IO::DatumWriter.new(@schema).write @data_object, Avro::IO::BinaryEncoder.new(serialized_data_object_bytes_stringio)
-        serialized_data_object_bytes
+        RFlow::Avro.encode @writer, @data_object
       end
 
       # Proxy methods down to the underlying data_object, probably a
@@ -186,6 +157,5 @@ class RFlow
         @data_object.send(method_sym, *args, &block)
       end
     end
-
   end
 end

data/lib/rflow/pid_file.rb

@@ -1,52 +1,35 @@
 class RFlow
   class PIDFile
-    attr_reader :pid_file_path
+    private
+    attr_reader :path
 
-    def initialize(pid_file_path)
-      @pid_file_path = pid_file_path
-    end
-
-    def validate?
-      if current_process?
-        RFlow.logger.warn "Already running #{read.to_s}, not writing PID to file '#{to_s}'"
-        return nil
-      elsif running?
-        error_message = "Already running #{read.to_s}, possibly stale PID file '#{to_s}'"
-        RFlow.logger.error error_message
-        raise ArgumentError, error_message
-      elsif exist?
-        RFlow.logger.warn "Found stale PID #{read.to_s} in PID file '#{to_s}', removing"
-        unlink
-      end
-      true
+    public
+    def initialize(path)
+      @path = path
     end
 
     def read
-      return nil unless File.exist? pid_file_path
-      File.read(pid_file_path).to_i
+      return nil unless File.exist? path
+      File.read(path).to_i
     end
 
-    def write(pid=$$)
+    def write(pid = $$)
       return unless validate?
 
       RFlow.logger.debug "Writing PID #{pid} file '#{to_s}'"
       pid_fp = begin
-                 tmp_pid_file_path = File.join(File.dirname(pid_file_path), ".#{File.basename(pid_file_path)}")
-                 File.open(tmp_pid_file_path, File::RDWR|File::CREAT|File::EXCL, 0644)
+                 tmp_path = File.join(File.dirname(path), ".#{File.basename(path)}")
+                 File.open(tmp_path, File::RDWR|File::CREAT|File::EXCL, 0644)
                rescue Errno::EEXIST
                  retry
                end
       pid_fp.syswrite("#{pid}\n")
-      File.rename(pid_fp.path, pid_file_path)
+      File.rename(pid_fp.path, path)
       pid_fp.close
 
       pid
     end
 
-    def exist?
-      File.exist? pid_file_path
-    end
-
     def running?
       return false unless exist?
       pid = read
@@ -57,14 +40,6 @@ class RFlow
       nil
     end
 
-    def current_process?
-      read == $$
-    end
-
-    def unlink
-      File.unlink(pid_file_path)
-    end
-
     # unlinks a PID file at given if it contains the current PID still
     # potentially racy without locking the directory (which is
     # non-portable and may interact badly with other programs), but the
@@ -78,7 +53,32 @@ class RFlow
     end
 
     def to_s
-      File.expand_path(pid_file_path)
+      File.expand_path(path)
+    end
+
+    private
+    def validate?
+      if current_process?
+        return nil
+      elsif running?
+        raise ArgumentError, "Process #{read.to_s} referenced in stale PID file '#{to_s}' still exists; probably attempting to run duplicate RFlow instances"
+      elsif exist?
+        RFlow.logger.warn "Found stale PID #{read.to_s} in PID file '#{to_s}', removing"
+        unlink
+      end
+      true
+    end
+
+    def exist?
+      File.exist? path
+    end
+
+    def current_process?
+      read == $$
+    end
+
+    def unlink
+      File.unlink(path)
     end
   end
 end