rflow 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 039a6516fe34dc10cd018dfb402fca1b0f68d5d8
-  data.tar.gz: 30124661ef4d6d2284c45e80555fe400f5a5db8a
+  metadata.gz: b52acbd7c200a903c0e3c17982a220a2a9ee0169
+  data.tar.gz: 6f16e7be6007f0de84de616bffaa82279bf90a59
 SHA512:
-  metadata.gz: 741a3780c5660ffb2a913a00a14ea8436dcce18d4d6d789aa92cbdc37b0227b68167b3fd35d62f9c788a20bbbfb377e1450e66c27ab14262a95371fe13f49c77
-  data.tar.gz: bca75e4cf91c28e4250a24c5c0fc9381901a9176783e67ec202e90a6ceaa5a890ce1b18850a45cf490acfa146d3a171447058e8226d720d3242aedb7d89bd490
+  metadata.gz: ffcde5466c777d5e36c815142a1b54b8da90fab13f6e7188519148482fdfc961250149a21f59d1998b674ce520f8436dd1a6185750af76063403742767aa4201
+  data.tar.gz: 15ea4a44084a9d8fa11691f45788ec9205b442cbabd4084b18e13d7ad94016e2f9ef37fb6d6e22bd6eeabee392bcf2fbef585cd3cc991c70c47abc87c3bc20e1

data/bin/rflow

@@ -69,82 +69,69 @@ end
 begin
   require 'rflow'
 rescue Exception => e
-  STDERR.puts "Error loading RFlow: #{e.class} - #{e.message}"
+  STDERR.puts "Error loading RFlow: #{e.class} - #{e.message}\n#{e.backtrace.join("\n")}"
   exit 1
 end
 
-# Set up the startup logging, which is distinct from the runtime
-# logging that is defined in the config database.  The startup logging
-# will always go to STDOUT, as well as to the file specified with the
-# '-l' parameter
-startup_logger = Log4r::Logger.new 'startup'
-startup_logger.add Log4r::StdoutOutputter.new('startup_stdout', :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
-startup_logger.level = Log4r::LNAMES.index options[:startup_log_level].to_s
-Log4r::NDC.push('startup')
-
-if options[:startup_log_file_path]
-  begin
-    startup_logger.add Log4r::FileOutputter.new('startup_file', :filename => options[:startup_log_file_path], :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
-  rescue Exception => e
-    startup_logger.fatal "Log file '#{options[:startup_log_file_path]}' problem: #{e.message}"
-    exit 1
-  end
-end
+# Set up the startup logging, which is distinct from the runtime logging that
+# is defined in the config database.  The startup logging will always go to
+# STDOUT, as well as to the file specified with the '-l' parameter
+#
+# This logging setup will be used while we call into RFlow to check on or setup
+# things, like the config database.  We want those log messages to go to the
+# startup log when setting up.  The running log will transition to what is
+# specified in the config database.
+RFlow.logger.reconfigure({'rflow.application_name' => 'startup',
+                          'rflow.log_level' => options[:startup_log_level].to_s,
+                          'rflow.log_file_path' => options[:startup_log_file_path]}, true)
 
 command = ARGV[0]
 unless ['start', 'stop', 'status', 'load'].include? command
-  startup_logger.fatal "Command needs to be one of [start|stop|status|load]\n#{option_parser.help}"
+  RFlow.logger.fatal "Command needs to be one of [start|stop|status|load]\n#{option_parser.help}"
   exit 1
 end
 
 if options[:config_file_path] && command != 'load'
-  startup_logger.fatal "Config file only valid for 'load' command"
+  RFlow.logger.fatal "Config file only valid for 'load' command"
   exit 1
 end
 
 unless options[:config_database_path]
-  startup_logger.warn "Config database not specified, using default 'config.sqlite'"
+  RFlow.logger.warn "Config database not specified, using default 'config.sqlite'"
   options[:config_database_path] = File.expand_path(File.join(Dir.getwd, 'config.sqlite'))
 end
 
-# Set the standard logger to the startup one in the case that we need
-# to call into RFlow to check on or setup things, like the config
-# database.  We want those log messages to go to the startup log when
-# setting up.  The running log will transition to what is specified in
-# the config database
-RFlow.logger = startup_logger
-
 case command
 when 'load'
   # Load the database with the config file, if it exists.  Will
   # otherwise default values (not very useful)
   if options[:config_file_path]
     unless File.exist? options[:config_file_path]
-      startup_logger.fatal "Config file '#{options[:config_file_path]}' not found\n#{option_parser.help}"
+      RFlow.logger.fatal "Config file '#{options[:config_file_path]}' not found\n#{option_parser.help}"
       exit 1
     end
 
     unless File.readable? options[:config_file_path]
-      startup_logger.fatal "Config file '#{options[:config_file_path]}' not readable\n#{option_parser.help}"
+      RFlow.logger.fatal "Config file '#{options[:config_file_path]}' not readable\n#{option_parser.help}"
       exit 1
     end
   end
 
   if File.exist? options[:config_database_path]
-    startup_logger.fatal "Config database '#{options[:config_database_path]}' exists, exiting to prevent accidental overwrite from config file '#{options[:config_file_path]}'"
+    RFlow.logger.fatal "Config database '#{options[:config_database_path]}' exists, exiting to prevent accidental overwrite from config file '#{options[:config_file_path]}'"
     exit 1
   end
 
-  startup_logger.info "Creating config database '#{options[:config_database_path]}'"
+  RFlow.logger.info "Creating config database '#{options[:config_database_path]}'"
   begin
     config = RFlow::Configuration::initialize_database(options[:config_database_path], options[:config_file_path])
   rescue Exception => e
-    startup_logger.fatal "Error initializing configuration database: #{e.message}: #{e.backtrace.join "\n"}"
+    RFlow.logger.fatal "Error initializing configuration database: #{e.message}: #{e.backtrace.join "\n"}"
     exit 1
   end
 
-  startup_logger.warn "Successfully initialized database '#{options[:config_database_path]}' with '#{options[:config_file_path]}'"
-  startup_logger.debug config.to_s
+  RFlow.logger.warn "Successfully initialized database '#{options[:config_database_path]}' with '#{options[:config_file_path]}'"
+  RFlow.logger.debug config.to_s
   exit 0
 end
 
@@ -152,7 +139,7 @@ end
 begin
   config = RFlow::Configuration.new(options[:config_database_path])
 rescue Exception => e
-  startup_logger.fatal "Error loading config database: #{e.class} - #{e.message}"
+  RFlow.logger.fatal "Error loading config database: #{e.class} - #{e.message}"
   exit 1
 end
 
@@ -163,26 +150,26 @@ pid_file = RFlow::PIDFile.new(config['rflow.pid_file_path'])
 case command
 when 'stop'
   if pid_file.running?
-    startup_logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}, terminating"
+    RFlow.logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}, terminating"
     # TODO: check if it actually shut down
     pid_file.signal(:INT)
   else
-    startup_logger.warn "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
+    RFlow.logger.warn "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
     exit 1
   end
   exit 0
 
 when 'status'
   unless pid_file.running?
-    startup_logger.error "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
+    RFlow.logger.error "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
     exit 1
   end
-  startup_logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}"
+  RFlow.logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}"
   exit 0
 
 when 'start'
   if pid_file.running?
-    startup_logger.error "#{config['rflow.application_name']} already running, process #{pid_file.read} found in #{pid_file.to_s}"
+    RFlow.logger.error "#{config['rflow.application_name']} already running, process #{pid_file.read} found in #{pid_file.to_s}"
     exit 1
   end
 end
@@ -191,15 +178,15 @@ end
 
 # require all the gem extensions
 options[:gems].each do |extension_gem|
-  startup_logger.info "Requiring #{extension_gem}"
+  RFlow.logger.info "Requiring #{extension_gem}"
   require extension_gem
 end
 
 # load all the file extensions
 options[:extensions_file_paths].each do |extensions_file_path|
-  startup_logger.info "Loading #{extensions_file_path}"
+  RFlow.logger.info "Loading #{extensions_file_path}"
   unless File.readable? extensions_file_path
-    startup_logger.fatal "Extensions file ('#{Dir.getwd}') '#{extensions_file_path}' not reabable\n#{option_parser.help}"
+    RFlow.logger.fatal "Extensions file ('#{Dir.getwd}') '#{extensions_file_path}' not reabable\n#{option_parser.help}"
     exit 1
   end
   load extensions_file_path
@@ -209,7 +196,7 @@ end
 begin
   RFlow.run! options[:config_database_path], options[:daemonize]
 rescue Exception => e
-  startup_logger.fatal "Error running rflow: #{e.class}: #{e.message}"
+  RFlow.logger.fatal "Error running rflow: #{e.class}: #{e.message}"
   exit(1)
 end
 

data/lib/rflow.rb

@@ -17,6 +17,8 @@ class RFlow
   class << self
     attr_accessor :logger
     attr_reader :configuration, :master
+
+    RFlow.logger = RFlow::Logger.new({})
   end
 
   def self.run!(config_database_path = nil, daemonize = false)
@@ -49,7 +51,7 @@ class RFlow
 
   def self.setup_logger
     include_stdout = !@daemonize
-    self.logger = RFlow::Logger.new(configuration, include_stdout)
+    logger.reconfigure(configuration, include_stdout)
   end
 
   def self.start_master_node

data/lib/rflow/child_process.rb

@@ -26,6 +26,7 @@ class RFlow
       @child_pipe_w.close
       register_logging_context
       update_process_name
+      detach_process_group
       handle_signals
 
       RFlow.logger.info "#{@role} started"
@@ -61,7 +62,7 @@ class RFlow
 
     def register_logging_context
       # arrange for child's name to appear in log messages
-      Log4r::NDC.push @name
+      Log4r::NDC.push sprintf("%-#{RFlow.logger.context_width}s", @name)
     end
 
     def clone_logging_context
@@ -77,6 +78,12 @@ class RFlow
       $0 += " #{@name}"
     end
 
+    # detach from parent process group so shutdown remains orderly (prevent
+    # signals from being delivered to entire group)
+    def detach_process_group
+      Process.setpgid(0, 0)
+    end
+
     def handle_signals
       Signal.trap 'SIGCHLD', 'DEFAULT' # make sure child process can run subshells
 

data/lib/rflow/connections/zmq_connection.rb

@@ -80,7 +80,6 @@ class RFlow
 
         begin
           output_socket.send_msg(message.data_type_name.to_s, message.to_avro)
-          RFlow.logger.debug "#{name}: Successfully sent message of type '#{message.data_type_name.to_s}'"
         rescue Exception => e
           RFlow.logger.debug "Exception #{e.class}: #{e.message}, retrying send"
           retry

data/lib/rflow/daemon_process.rb

@@ -112,12 +112,12 @@ class RFlow
 
       trap_signal 'SIGUSR1' do
         RFlow.logger.reopen
-        signal_subprocesses signal
+        signal_subprocesses 'SIGUSR1'
       end
 
       trap_signal 'SIGUSR2' do
         RFlow.logger.toggle_log_level
-        signal_subprocesses signal
+        signal_subprocesses 'SIGUSR2'
       end
     end
 

data/lib/rflow/logger.rb

@@ -5,7 +5,7 @@ class RFlow
     extend Forwardable
     include Log4r
 
-    LOG_PATTERN_FORMAT = '%l [%d] %x (%p) - %M'
+    LOG_PATTERN_FORMAT = '%-5l [%d] %x (%-5p) - %M'
     DATE_METHOD = 'xmlschema(6)'
     LOG_PATTERN_FORMATTER = PatternFormatter.new :pattern => LOG_PATTERN_FORMAT, :date_method => DATE_METHOD
 
@@ -14,6 +14,7 @@ class RFlow
     attr_accessor :log_file_path, :log_level, :log_name
 
     public
+    attr_accessor :context_width
 
     # make sure Log4r is initialized; ignored if custom levels are already set
     Log4r.define_levels(*Log4rConfig::LogLevels)
@@ -24,9 +25,15 @@ class RFlow
       *Log4r::LNAMES.map(&:downcase).map {|n| "#{n}?".to_sym }
 
     def initialize(config, include_stdout = false)
+      reconfigure(config, include_stdout)
+    end
+
+    def reconfigure(config, include_stdout = false)
       @log_file_path = config['rflow.log_file_path']
-      @log_level = config['rflow.log_level']
-      @log_name = (config['rflow.application_name'] || File.basename(log_file_path))
+      @log_level = config['rflow.log_level'] || "WARN"
+      @log_name = if config['rflow.application_name']; config['rflow.application_name']
+                  elsif log_file_path; File.basename(log_file_path)
+                  else ''; end
 
       establish_internal_logger
       hook_up_logfile
@@ -47,8 +54,12 @@ class RFlow
       Outputter['rflow.log_file'].close
     end
 
+    def level=(level)
+      internal_logger.level = LNAMES.index(level.to_s) || level
+    end
+
     def toggle_log_level
-      original_log_level = LNAMES[logger.level]
+      original_log_level = LNAMES[internal_logger.level]
       new_log_level = (original_log_level == 'DEBUG' ? log_level : 'DEBUG')
 
       internal_logger.warn "Changing log level from #{original_log_level} to #{new_log_level}"
@@ -64,10 +75,12 @@ class RFlow
     end
 
     def hook_up_logfile
-      begin
-        internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
-      rescue Exception => e
-        raise ArgumentError, "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\b#{e.backtrace.join("\n")}"
+      if log_file_path
+        begin
+          internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
+        rescue Exception => e
+          raise ArgumentError, "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\n#{e.backtrace.join("\n")}"
+        end
       end
     end
 

data/lib/rflow/master.rb

@@ -12,6 +12,7 @@ class RFlow
       super(config['rflow.application_name'], 'Master')
       @pid_file = PIDFile.new(config['rflow.pid_file_path'])
       @shards = config.shards.map {|config| Shard.new(config) }
+      RFlow.logger.context_width = @shards.flat_map(&:workers).map(&:name).map(&:length).max
       @brokers = config.connections.flat_map(&:brokers).map {|config| Broker.build(config) }
     end
 

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

data/spec/spec_helper.rb

@@ -8,12 +8,6 @@ require 'tmpdir'
 I18n.enforce_available_locales = true
 
 RSpec.configure do |config|
-  config.before(:all) do
-    RFlow.logger = Log4r::Logger.new 'test'
-    RFlow.logger.add Log4r::StdoutOutputter.new('test_stdout', :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
-    RFlow.logger.level = 5
-  end
-
   config.before(:each) do
     @temp_directory_path = Dir.mktmpdir('rflow')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-30 00:00:00.000000000 Z
+date: 2014-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools