rflow 1.0.1 → 1.1.0

data/lib/rflow/configuration.rb

@@ -110,13 +110,13 @@ class RFlow
       def establish_config_database_connection(database_path)
         RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{database_path}'"
         ActiveRecord::Base.logger = RFlow.logger
-        ConfigurationItem.establish_connection(:adapter => "sqlite3", :database => database_path)
+        ConfigurationItem.establish_connection(:adapter => 'sqlite3', :database => database_path)
       end
 
       # Using default ActiveRecord migrations, attempt to migrate the
       # database to the latest version.
       def migrate_database
-        RFlow.logger.debug "Applying default migrations to config database"
+        RFlow.logger.debug 'Applying default migrations to config database'
         migrations_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
         ActiveRecord::Migration.verbose = false
         ActiveRecord::Migrator.migrate migrations_path
@@ -135,7 +135,7 @@ class RFlow
         RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{database_path}'"
 
         # TODO should not need this line
-        ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => database_path)
+        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => database_path)
 
         establish_config_database_connection database_path
         migrate_database
@@ -147,7 +147,7 @@ class RFlow
           process_config_file File.expand_path(config_file_path)
         end
 
-        RFlow.logger.debug "Defaulting non-existing config values"
+        RFlow.logger.debug 'Defaulting non-existing config values'
         merge_defaults!
 
         Dir.chdir working_dir

data/lib/rflow/configuration/ruby_dsl.rb

@@ -10,7 +10,7 @@ class RFlow
 
       public
       def initialize
-        @default_shard = {:name => "DEFAULT", :type => :process, :count => 1, :components => []}
+        @default_shard = {:name => 'DEFAULT', :type => :process, :count => 1, :components => []}
         @current_shard = default_shard
 
         @setting_specs = []
@@ -27,8 +27,8 @@ class RFlow
 
       # DSL method to specify a shard block for either a process or thread
       def shard(name, options = {})
-        raise ArgumentError, "Cannot use DEFAULT as a shard name" if name == 'DEFAULT'
-        raise ArgumentError, "Cannot nest shards" if @current_shard != default_shard
+        raise ArgumentError, 'Cannot use DEFAULT as a shard name' if name == 'DEFAULT'
+        raise ArgumentError, 'Cannot nest shards' if @current_shard != default_shard
 
         type = if options[:thread] || options[:type] == :thread; :thread
                else :process
@@ -66,7 +66,7 @@ class RFlow
       # DSL method to specify a connection between a
       # component/output_port and another component/input_port.  The
       # component/port specification is a string where the names of
-      # the two elements are separated by '#', and the "connection" is
+      # the two elements are separated by '#', and the 'connection' is
       # specified by a Ruby Hash, i.e.:
       #  connect 'componentA#output' => 'componentB#input'
       # Array ports are specified with an key suffix in standard

data/lib/rflow/connection.rb

@@ -10,7 +10,7 @@ class RFlow
         when 'RFlow::Configuration::BrokeredZMQConnection'
           RFlow::Connections::BrokeredZMQConnection.new(config)
         else
-          raise ArgumentError, "Only ZMQConnections currently supported"
+          raise ArgumentError, 'Only ZMQConnections currently supported'
         end
       end
     end
@@ -32,14 +32,14 @@ class RFlow
     # methods.  Will only be called in the context of a running
     # EventMachine reactor
     def connect_input!
-      raise NotImplementedError, "Raw connections do not support connect_input.  Please subclass and define a connect_input method."
+      raise NotImplementedError, 'Raw connections do not support connect_input.  Please subclass and define a connect_input method.'
     end
 
     # Subclass and implement to be able to handle future 'send'
     # methods.  Will only be called in the context of a running
     # EventMachine reactor
     def connect_output!
-      raise NotImplementedError, "Raw connections do not support connect_output.  Please subclass and define a connect_output method."
+      raise NotImplementedError, 'Raw connections do not support connect_output.  Please subclass and define a connect_output method.'
     end
 
     # Subclass and implement to handle outgoing messages.  The message
@@ -47,7 +47,7 @@ class RFlow
     # to marshal it up into something that will be unmarshalled on the
     # other side
     def send_message(message)
-      raise NotImplementedError, "Raw connections do not support send_message.  Please subclass and define a send_message method."
+      raise NotImplementedError, 'Raw connections do not support send_message.  Please subclass and define a send_message method.'
     end
 
     # Parent component will set this attribute if it expects to

data/lib/rflow/connections/zmq_connection.rb

@@ -18,7 +18,7 @@ class RFlow
           version = LibZMQ::version
           RFlow.logger.debug { "Creating a new ZeroMQ context; ZeroMQ version is #{version[:major]}.#{version[:minor]}.#{version[:patch]}" }
           if EM.reactor_running?
-            raise RuntimeError, "EventMachine reactor is running when attempting to create a ZeroMQ context"
+            raise RuntimeError, 'EventMachine reactor is running when attempting to create a ZeroMQ context'
           end
           EM::ZeroMQ::Context.new(1)
         end
@@ -74,15 +74,13 @@ class RFlow
         output_socket
       end
 
-      # TODO: fix this tight loop of retries
       def send_message(message)
         RFlow.logger.debug "#{name}: Sending message of type '#{message.data_type_name.to_s}'"
 
         begin
           output_socket.send_msg(message.data_type_name.to_s, message.to_avro)
         rescue Exception => e
-          RFlow.logger.debug "Exception #{e.class}: #{e.message}, retrying send"
-          retry
+          RFlow.logger.error "Exception #{e.class}: #{e.message}, because: #{e.backtrace}"
         end
       end
 
@@ -159,9 +157,9 @@ class RFlow
                 end
         front.bind(connection.options['output_address'])
         back.bind(connection.options['input_address'])
-        ZMQ::Proxy.new(front, back)
-        back.close
-        front.close
+        while true
+          ZMQ::Proxy.new(front, back)
+        end
       rescue Exception => e
         RFlow.logger.error "Error running message broker: #{e.class}: #{e.message}, because: #{e.backtrace.inspect}"
       ensure

data/lib/rflow/daemon_process.rb

@@ -1,8 +1,11 @@
+require 'rflow/pid_file'
+
 class RFlow
   class DaemonProcess
-    def initialize(name, role = name)
+    def initialize(name, role = name, options = {})
       @name = name
       @role = role
+      @pid_file = PIDFile.new(options[:pid_file_path]) if options[:pid_file_path]
     end
 
     def daemonize!
@@ -19,6 +22,7 @@ class RFlow
     end
 
     def run!
+      write_pid_file
       register_logging_context
       update_process_name
       handle_signals
@@ -29,6 +33,7 @@ class RFlow
       run_process
     ensure
       unhandle_signals
+      remove_pid_file
     end
 
     def spawn_subprocesses; end
@@ -36,6 +41,7 @@ class RFlow
 
     def shutdown!(reason)
       RFlow.logger.info "#{@name} shutting down due to #{reason}"
+      remove_pid_file
       unhandle_signals
       signal_subprocesses('QUIT')
       RFlow.logger.info "#{@name} exiting"
@@ -65,6 +71,7 @@ class RFlow
         exit 0
       else
         RFlow.logger.error "#{@role} failed to start"
+        STDERR.puts "\n\n*** #{@role} failed to start; see log file for details"
         exit! 1
       end
     end
@@ -79,7 +86,7 @@ class RFlow
       $stdout.sync = $stderr.sync = true
       [$stdin, $stdout, $stderr].each do |stream|
         stream.binmode
-        begin; stream.reopen "/dev/null"; rescue ::Exception; end
+        begin; stream.reopen '/dev/null'; rescue ::Exception; end
       end
     end
 
@@ -151,5 +158,8 @@ class RFlow
         Process.kill(signal, p.pid)
       end
     end
+
+    def write_pid_file; @pid_file.write if @pid_file; end
+    def remove_pid_file; @pid_file.safe_unlink if @pid_file; end
   end
 end

data/lib/rflow/logger.rb

@@ -30,7 +30,7 @@ class RFlow
 
     def reconfigure(config, include_stdout = false)
       @log_file_path = config['rflow.log_file_path']
-      @log_level = config['rflow.log_level'] || "WARN"
+      @log_level = config['rflow.log_level'] || 'WARN'
       @log_name = if config['rflow.application_name']; config['rflow.application_name']
                   elsif log_file_path; File.basename(log_file_path)
                   else ''; end

data/lib/rflow/master.rb

@@ -1,5 +1,4 @@
 require 'rflow/daemon_process'
-require 'rflow/pid_file'
 require 'rflow/shard'
 require 'rflow/broker'
 
@@ -9,24 +8,16 @@ class RFlow
     attr_reader :brokers
 
     def initialize(config)
-      super(config['rflow.application_name'], 'Master')
-      @pid_file = PIDFile.new(config['rflow.pid_file_path'])
+      super(config['rflow.application_name'], 'Master', pid_file_path: config['rflow.pid_file_path'])
       @shards = config.shards.map {|config| Shard.new(config) }
       RFlow.logger.context_width = @shards.flat_map(&:workers).map(&:name).map(&:length).max
       @brokers = config.connections.flat_map(&:brokers).map {|config| Broker.build(config) }
     end
 
-    def run!
-      write_pid_file
-      super
-    ensure
-      remove_pid_file
-    end
-
     def spawn_subprocesses
       RFlow.logger.debug "Running #{brokers.count} brokers" if brokers.count > 0
       brokers.each(&:spawn!)
-      RFlow.logger.debug "#{brokers.count} brokers started: #{brokers.map { |w| "#{w.name} (#{w.pid})" }.join(", ")}" if brokers.count > 0
+      RFlow.logger.debug "#{brokers.count} brokers started: #{brokers.map { |w| "#{w.name} (#{w.pid})" }.join(', ')}" if brokers.count > 0
 
       shards.each(&:run!)
     end
@@ -40,14 +31,5 @@ class RFlow
         # TODO: Monitor the workers
       end
     end
-
-    def shutdown!(reason)
-      remove_pid_file
-      super
-    end
-
-    private
-    def write_pid_file; @pid_file.write; end
-    def remove_pid_file; @pid_file.safe_unlink; end
   end
 end

data/lib/rflow/message.rb

@@ -27,32 +27,22 @@ class RFlow
       # Message object.  Assumes the org.rflow.Message Avro schema.
       def from_avro(bytes)
         message = RFlow::Avro.decode(message_reader, bytes)
-        Message.new(message['data_type_name'], message['provenance'],
+        Message.new(message['data_type_name'], message['provenance'], message['properties'],
                     message['data_serialization_type'], message['data_schema'],
                     message['data'])
       end
     end
 
+    attr_accessor :provenance, :properties
     attr_reader :data_type_name, :data
-    attr_accessor :provenance
 
-    def initialize(data_type_name, provenance = [], serialization_type = 'avro', schema = nil, serialized_data = nil)
+    # When creating a new message as a transformation of an existing
+    # message, its encouraged to copy the provenance and properties of
+    # the original message into the new message. This allows
+    # downstream components to potentially use these fields
+    def initialize(data_type_name, provenance = [], properties = {}, serialization_type = 'avro', schema = nil, serialized_data = nil)
       @data_type_name = data_type_name.to_s
 
-      # TODO: Make this better.  This check is technically
-      # unnecessary, as we are able to completely deserialize the
-      # message without needing to resort to the registered schema.
-      registered_schema = RFlow::Configuration.available_data_types[@data_type_name][serialization_type.to_s]
-      unless registered_schema
-        raise ArgumentError, "Data type '#{@data_type_name}' with serialization_type '#{serialization_type}' not found"
-      end
-
-      # TODO: think about registering the schemas automatically if not
-      # found in Configuration
-      if schema && (registered_schema != schema)
-        raise ArgumentError, "Passed schema ('#{schema}') does not equal registered schema ('#{registered_schema}') for data type '#{@data_type_name}' with serialization_type '#{serialization_type}'"
-      end
-
       # Turn the provenance array of Hashes into an array of
       # ProcessingEvents for easier access and time validation.
       # TODO: do this lazily so as not to create/destroy objects that are
@@ -67,6 +57,22 @@ class RFlow
         end
       end
 
+      @properties = properties || {}
+
+      # TODO: Make this better.  This check is technically
+      # unnecessary, as we are able to completely deserialize the
+      # message without needing to resort to the registered schema.
+      registered_schema = RFlow::Configuration.available_data_types[@data_type_name][serialization_type.to_s]
+      unless registered_schema
+        raise ArgumentError, "Data type '#{@data_type_name}' with serialization_type '#{serialization_type}' not found"
+      end
+
+      # TODO: think about registering the schemas automatically if not
+      # found in Configuration
+      if schema && (registered_schema != schema)
+        raise ArgumentError, "Passed schema ('#{schema}') does not equal registered schema ('#{registered_schema}') for data type '#{@data_type_name}' with serialization_type '#{serialization_type}'"
+      end
+
       @data = Data.new(registered_schema, serialization_type.to_s, serialized_data)
 
       # Get the extensions and apply them to the data object to add capability
@@ -82,8 +88,12 @@ class RFlow
     # UTF-8 by default, which would not work correctly, as a serialize
     # avro string is BINARY, not UTF-8
     def to_avro
+      # stringify all the properties
+      string_properties = Hash[properties.map { |k,v| [k.to_s, v.to_s] }]
+
       Message.encode('data_type_name' => data_type_name.to_s,
                      'provenance' => provenance.map(&:to_hash),
+                     'properties' => string_properties.to_hash,
                      'data_serialization_type' => data.serialization_type.to_s,
                      'data_schema' => data.schema_string,
                      'data' => data.to_avro)
@@ -124,7 +134,7 @@ class RFlow
       attr_accessor :data_object
 
       def initialize(schema_string, serialization_type = 'avro', serialized_data = nil)
-        raise ArgumentError, "Only Avro serialization_type supported at the moment" unless serialization_type.to_s == 'avro'
+        raise ArgumentError, 'Only Avro serialization_type supported at the moment' unless serialization_type.to_s == 'avro'
 
         @schema_string = schema_string
         @serialization_type = serialization_type.to_s

data/lib/rflow/pid_file.rb

@@ -10,18 +10,30 @@ class RFlow
 
     def read
       return nil unless File.exist? path
-      File.read(path).to_i
+      contents = File.read(path)
+      if contents.empty?
+        RFlow.logger.warn "Ignoring empty PID file #{path}"
+        nil
+      else
+        contents.to_i
+      end
     end
 
     def write(pid = $$)
       return unless validate?
 
-      RFlow.logger.debug "Writing PID #{pid} file '#{to_s}'"
+      RFlow.logger.debug "Writing PID #{pid} to file '#{to_s}'"
+      tmp_path = File.join(File.dirname(path), ".#{File.basename(path)}")
+      if File.exist? tmp_path
+        RFlow.logger.warn "Deleting stale temp PID file #{tmp_path}"
+        File.delete(tmp_path)
+      end
       pid_fp = begin
-                 tmp_path = File.join(File.dirname(path), ".#{File.basename(path)}")
                  File.open(tmp_path, File::RDWR|File::CREAT|File::EXCL, 0644)
-               rescue Errno::EEXIST
-                 retry
+               rescue Errno::EACCES => e
+                 RFlow.logger.fatal "Access error while writing temp PID file '#{tmp_path}'"
+                 RFlow.logger.fatal "Exception #{e.class}: #{e.message}"
+                 abort
                end
       pid_fp.syswrite("#{pid}\n")
       File.rename(pid_fp.path, path)

data/lib/rflow/shard.rb

@@ -35,11 +35,11 @@ class RFlow
           end
         end
 
-        RFlow.logger.info "Shutting down worker after EM stopped"
+        RFlow.logger.info 'Shutting down worker after EM stopped'
       end
 
       def configure_components!
-        RFlow.logger.debug "Configuring components"
+        RFlow.logger.debug 'Configuring components'
         @components.zip(shard.config.components.map(&:options)).each do |(component, config)|
           RFlow.logger.debug "Configuring component '#{component.name}' (#{component.uuid})"
           component.configure! config
@@ -49,7 +49,7 @@ class RFlow
       # Connect all inputs before all outputs, so connection types that require a 'server'
       # to be established before a 'client' can connect can get themselves ready.
       def connect_components!
-        RFlow.logger.debug "Connecting components"
+        RFlow.logger.debug 'Connecting components'
         @components.each do |component|
           RFlow.logger.debug "Connecting inputs for component '#{component.name}' (#{component.uuid})"
           component.connect_inputs!
@@ -61,7 +61,7 @@ class RFlow
       end
 
       def run_components!
-        RFlow.logger.debug "Running components"
+        RFlow.logger.debug 'Running components'
         @components.each do |component|
           RFlow.logger.debug "Running component '#{component.name}' (#{component.uuid})"
           component.run!
@@ -69,7 +69,7 @@ class RFlow
       end
 
       def shutdown!(signal)
-        RFlow.logger.debug "Shutting down components"
+        RFlow.logger.debug 'Shutting down components'
         @components.each do |component|
           RFlow.logger.debug "Shutting down component '#{component.name}' (#{component.uuid})"
           component.shutdown!
@@ -93,7 +93,7 @@ class RFlow
       RFlow.logger.debug "Running shard #{name} with #{count} workers"
       workers.each(&:spawn!)
 
-      RFlow.logger.debug "#{count} workers started for #{name}: #{workers.map { |w| "#{w.name} (#{w.pid})" }.join(", ")}"
+      RFlow.logger.debug "#{count} workers started for #{name}: #{workers.map { |w| "#{w.name} (#{w.pid})" }.join(', ')}"
       workers
     end
   end

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "1.0.1"
+  VERSION = '1.1.0'
 end

data/rflow.gemspec

@@ -1,37 +1,37 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "rflow/version"
+require 'rflow/version'
 
 Gem::Specification.new do |s|
-  s.name        = "rflow"
+  s.name        = 'rflow'
   s.version     = RFlow::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.required_ruby_version = ">= 1.9"
-  s.authors     = ["Michael L. Artz"]
-  s.email       = ["michael.artz@redjack.com"]
-  s.homepage    = "https://github.com/redjack/rflow"
-  s.license     = "Apache-2.0"
+  s.required_ruby_version = '>= 1.9'
+  s.authors     = ['John Stoneham', 'Michael L. Artz']
+  s.email       = ['john.stoneham@redjack.com', 'mlartz@gmail.com']
+  s.homepage    = 'https://github.com/redjack/rflow'
+  s.license     = 'Apache-2.0'
   s.summary     = %q{A Ruby flow-based programming framework}
   s.description = %q{A Ruby flow-based programming framework that utilizes ZeroMQ for component connections and Avro for serialization}
 
   s.files         = `git ls-files -z`.split("\x0")
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
 
-  s.add_dependency "uuidtools", "~> 2.1"
-  s.add_dependency "log4r", "~> 1.1"
-  s.add_dependency "sys-filesystem", "~> 1.1.2"
+  s.add_dependency 'uuidtools', '~> 2.1'
+  s.add_dependency 'log4r', '~> 1.1'
+  s.add_dependency 'sys-filesystem', '~> 1.1'
 
-  s.add_dependency "sqlite3", "~> 1.3"
-  s.add_dependency "activerecord", "~> 3.2"
+  s.add_dependency 'sqlite3', '~> 1.3'
+  s.add_dependency 'activerecord', '~> 3.2'
 
-  s.add_dependency "avro", "~> 1.7.5"
-  s.add_dependency "em-zeromq", "0.5.0"
+  s.add_dependency 'avro', '~> 1.7'
+  s.add_dependency 'em-zeromq', '~> 0.5.0'
 
-  s.add_development_dependency "bundler", "~> 1.6"
-  s.add_development_dependency "rspec", "~> 3.0"
-  s.add_development_dependency "rspec-collection_matchers", "~> 1.0"
-  s.add_development_dependency "rake", ">= 10.3"
-  s.add_development_dependency "yard", "~> 0.8"
+  s.add_development_dependency 'bundler', '~> 1'
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rspec-collection_matchers', '~> 1.0'
+  s.add_development_dependency 'rake', '>= 10.3'
+  s.add_development_dependency 'yard', '~> 0.8'
 end