rflow 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/rflow +32 -45
- data/lib/rflow.rb +3 -1
- data/lib/rflow/child_process.rb +8 -1
- data/lib/rflow/connections/zmq_connection.rb +0 -1
- data/lib/rflow/daemon_process.rb +2 -2
- data/lib/rflow/logger.rb +21 -8
- data/lib/rflow/master.rb +1 -0
- data/lib/rflow/version.rb +1 -1
- data/spec/spec_helper.rb +0 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b52acbd7c200a903c0e3c17982a220a2a9ee0169
|
4
|
+
data.tar.gz: 6f16e7be6007f0de84de616bffaa82279bf90a59
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ffcde5466c777d5e36c815142a1b54b8da90fab13f6e7188519148482fdfc961250149a21f59d1998b674ce520f8436dd1a6185750af76063403742767aa4201
|
7
|
+
data.tar.gz: 15ea4a44084a9d8fa11691f45788ec9205b442cbabd4084b18e13d7ad94016e2f9ef37fb6d6e22bd6eeabee392bcf2fbef585cd3cc991c70c47abc87c3bc20e1
|
data/bin/rflow
CHANGED
@@ -69,82 +69,69 @@ end
|
|
69
69
|
begin
|
70
70
|
require 'rflow'
|
71
71
|
rescue Exception => e
|
72
|
-
STDERR.puts "Error loading RFlow: #{e.class} - #{e.message}"
|
72
|
+
STDERR.puts "Error loading RFlow: #{e.class} - #{e.message}\n#{e.backtrace.join("\n")}"
|
73
73
|
exit 1
|
74
74
|
end
|
75
75
|
|
76
|
-
# Set up the startup logging, which is distinct from the runtime
|
77
|
-
#
|
78
|
-
#
|
79
|
-
#
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
startup_logger.add Log4r::FileOutputter.new('startup_file', :filename => options[:startup_log_file_path], :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
|
88
|
-
rescue Exception => e
|
89
|
-
startup_logger.fatal "Log file '#{options[:startup_log_file_path]}' problem: #{e.message}"
|
90
|
-
exit 1
|
91
|
-
end
|
92
|
-
end
|
76
|
+
# Set up the startup logging, which is distinct from the runtime logging that
|
77
|
+
# is defined in the config database. The startup logging will always go to
|
78
|
+
# STDOUT, as well as to the file specified with the '-l' parameter
|
79
|
+
#
|
80
|
+
# This logging setup will be used while we call into RFlow to check on or setup
|
81
|
+
# things, like the config database. We want those log messages to go to the
|
82
|
+
# startup log when setting up. The running log will transition to what is
|
83
|
+
# specified in the config database.
|
84
|
+
RFlow.logger.reconfigure({'rflow.application_name' => 'startup',
|
85
|
+
'rflow.log_level' => options[:startup_log_level].to_s,
|
86
|
+
'rflow.log_file_path' => options[:startup_log_file_path]}, true)
|
93
87
|
|
94
88
|
command = ARGV[0]
|
95
89
|
unless ['start', 'stop', 'status', 'load'].include? command
|
96
|
-
|
90
|
+
RFlow.logger.fatal "Command needs to be one of [start|stop|status|load]\n#{option_parser.help}"
|
97
91
|
exit 1
|
98
92
|
end
|
99
93
|
|
100
94
|
if options[:config_file_path] && command != 'load'
|
101
|
-
|
95
|
+
RFlow.logger.fatal "Config file only valid for 'load' command"
|
102
96
|
exit 1
|
103
97
|
end
|
104
98
|
|
105
99
|
unless options[:config_database_path]
|
106
|
-
|
100
|
+
RFlow.logger.warn "Config database not specified, using default 'config.sqlite'"
|
107
101
|
options[:config_database_path] = File.expand_path(File.join(Dir.getwd, 'config.sqlite'))
|
108
102
|
end
|
109
103
|
|
110
|
-
# Set the standard logger to the startup one in the case that we need
|
111
|
-
# to call into RFlow to check on or setup things, like the config
|
112
|
-
# database. We want those log messages to go to the startup log when
|
113
|
-
# setting up. The running log will transition to what is specified in
|
114
|
-
# the config database
|
115
|
-
RFlow.logger = startup_logger
|
116
|
-
|
117
104
|
case command
|
118
105
|
when 'load'
|
119
106
|
# Load the database with the config file, if it exists. Will
|
120
107
|
# otherwise default values (not very useful)
|
121
108
|
if options[:config_file_path]
|
122
109
|
unless File.exist? options[:config_file_path]
|
123
|
-
|
110
|
+
RFlow.logger.fatal "Config file '#{options[:config_file_path]}' not found\n#{option_parser.help}"
|
124
111
|
exit 1
|
125
112
|
end
|
126
113
|
|
127
114
|
unless File.readable? options[:config_file_path]
|
128
|
-
|
115
|
+
RFlow.logger.fatal "Config file '#{options[:config_file_path]}' not readable\n#{option_parser.help}"
|
129
116
|
exit 1
|
130
117
|
end
|
131
118
|
end
|
132
119
|
|
133
120
|
if File.exist? options[:config_database_path]
|
134
|
-
|
121
|
+
RFlow.logger.fatal "Config database '#{options[:config_database_path]}' exists, exiting to prevent accidental overwrite from config file '#{options[:config_file_path]}'"
|
135
122
|
exit 1
|
136
123
|
end
|
137
124
|
|
138
|
-
|
125
|
+
RFlow.logger.info "Creating config database '#{options[:config_database_path]}'"
|
139
126
|
begin
|
140
127
|
config = RFlow::Configuration::initialize_database(options[:config_database_path], options[:config_file_path])
|
141
128
|
rescue Exception => e
|
142
|
-
|
129
|
+
RFlow.logger.fatal "Error initializing configuration database: #{e.message}: #{e.backtrace.join "\n"}"
|
143
130
|
exit 1
|
144
131
|
end
|
145
132
|
|
146
|
-
|
147
|
-
|
133
|
+
RFlow.logger.warn "Successfully initialized database '#{options[:config_database_path]}' with '#{options[:config_file_path]}'"
|
134
|
+
RFlow.logger.debug config.to_s
|
148
135
|
exit 0
|
149
136
|
end
|
150
137
|
|
@@ -152,7 +139,7 @@ end
|
|
152
139
|
begin
|
153
140
|
config = RFlow::Configuration.new(options[:config_database_path])
|
154
141
|
rescue Exception => e
|
155
|
-
|
142
|
+
RFlow.logger.fatal "Error loading config database: #{e.class} - #{e.message}"
|
156
143
|
exit 1
|
157
144
|
end
|
158
145
|
|
@@ -163,26 +150,26 @@ pid_file = RFlow::PIDFile.new(config['rflow.pid_file_path'])
|
|
163
150
|
case command
|
164
151
|
when 'stop'
|
165
152
|
if pid_file.running?
|
166
|
-
|
153
|
+
RFlow.logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}, terminating"
|
167
154
|
# TODO: check if it actually shut down
|
168
155
|
pid_file.signal(:INT)
|
169
156
|
else
|
170
|
-
|
157
|
+
RFlow.logger.warn "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
|
171
158
|
exit 1
|
172
159
|
end
|
173
160
|
exit 0
|
174
161
|
|
175
162
|
when 'status'
|
176
163
|
unless pid_file.running?
|
177
|
-
|
164
|
+
RFlow.logger.error "#{config['rflow.application_name']} process not found in #{pid_file.to_s}"
|
178
165
|
exit 1
|
179
166
|
end
|
180
|
-
|
167
|
+
RFlow.logger.info "#{config['rflow.application_name']} running, process #{pid_file.read} found in #{pid_file.to_s}"
|
181
168
|
exit 0
|
182
169
|
|
183
170
|
when 'start'
|
184
171
|
if pid_file.running?
|
185
|
-
|
172
|
+
RFlow.logger.error "#{config['rflow.application_name']} already running, process #{pid_file.read} found in #{pid_file.to_s}"
|
186
173
|
exit 1
|
187
174
|
end
|
188
175
|
end
|
@@ -191,15 +178,15 @@ end
|
|
191
178
|
|
192
179
|
# require all the gem extensions
|
193
180
|
options[:gems].each do |extension_gem|
|
194
|
-
|
181
|
+
RFlow.logger.info "Requiring #{extension_gem}"
|
195
182
|
require extension_gem
|
196
183
|
end
|
197
184
|
|
198
185
|
# load all the file extensions
|
199
186
|
options[:extensions_file_paths].each do |extensions_file_path|
|
200
|
-
|
187
|
+
RFlow.logger.info "Loading #{extensions_file_path}"
|
201
188
|
unless File.readable? extensions_file_path
|
202
|
-
|
189
|
+
RFlow.logger.fatal "Extensions file ('#{Dir.getwd}') '#{extensions_file_path}' not reabable\n#{option_parser.help}"
|
203
190
|
exit 1
|
204
191
|
end
|
205
192
|
load extensions_file_path
|
@@ -209,7 +196,7 @@ end
|
|
209
196
|
begin
|
210
197
|
RFlow.run! options[:config_database_path], options[:daemonize]
|
211
198
|
rescue Exception => e
|
212
|
-
|
199
|
+
RFlow.logger.fatal "Error running rflow: #{e.class}: #{e.message}"
|
213
200
|
exit(1)
|
214
201
|
end
|
215
202
|
|
data/lib/rflow.rb
CHANGED
@@ -17,6 +17,8 @@ class RFlow
|
|
17
17
|
class << self
|
18
18
|
attr_accessor :logger
|
19
19
|
attr_reader :configuration, :master
|
20
|
+
|
21
|
+
RFlow.logger = RFlow::Logger.new({})
|
20
22
|
end
|
21
23
|
|
22
24
|
def self.run!(config_database_path = nil, daemonize = false)
|
@@ -49,7 +51,7 @@ class RFlow
|
|
49
51
|
|
50
52
|
def self.setup_logger
|
51
53
|
include_stdout = !@daemonize
|
52
|
-
|
54
|
+
logger.reconfigure(configuration, include_stdout)
|
53
55
|
end
|
54
56
|
|
55
57
|
def self.start_master_node
|
data/lib/rflow/child_process.rb
CHANGED
@@ -26,6 +26,7 @@ class RFlow
|
|
26
26
|
@child_pipe_w.close
|
27
27
|
register_logging_context
|
28
28
|
update_process_name
|
29
|
+
detach_process_group
|
29
30
|
handle_signals
|
30
31
|
|
31
32
|
RFlow.logger.info "#{@role} started"
|
@@ -61,7 +62,7 @@ class RFlow
|
|
61
62
|
|
62
63
|
def register_logging_context
|
63
64
|
# arrange for child's name to appear in log messages
|
64
|
-
Log4r::NDC.push @name
|
65
|
+
Log4r::NDC.push sprintf("%-#{RFlow.logger.context_width}s", @name)
|
65
66
|
end
|
66
67
|
|
67
68
|
def clone_logging_context
|
@@ -77,6 +78,12 @@ class RFlow
|
|
77
78
|
$0 += " #{@name}"
|
78
79
|
end
|
79
80
|
|
81
|
+
# detach from parent process group so shutdown remains orderly (prevent
|
82
|
+
# signals from being delivered to entire group)
|
83
|
+
def detach_process_group
|
84
|
+
Process.setpgid(0, 0)
|
85
|
+
end
|
86
|
+
|
80
87
|
def handle_signals
|
81
88
|
Signal.trap 'SIGCHLD', 'DEFAULT' # make sure child process can run subshells
|
82
89
|
|
@@ -80,7 +80,6 @@ class RFlow
|
|
80
80
|
|
81
81
|
begin
|
82
82
|
output_socket.send_msg(message.data_type_name.to_s, message.to_avro)
|
83
|
-
RFlow.logger.debug "#{name}: Successfully sent message of type '#{message.data_type_name.to_s}'"
|
84
83
|
rescue Exception => e
|
85
84
|
RFlow.logger.debug "Exception #{e.class}: #{e.message}, retrying send"
|
86
85
|
retry
|
data/lib/rflow/daemon_process.rb
CHANGED
@@ -112,12 +112,12 @@ class RFlow
|
|
112
112
|
|
113
113
|
trap_signal 'SIGUSR1' do
|
114
114
|
RFlow.logger.reopen
|
115
|
-
signal_subprocesses
|
115
|
+
signal_subprocesses 'SIGUSR1'
|
116
116
|
end
|
117
117
|
|
118
118
|
trap_signal 'SIGUSR2' do
|
119
119
|
RFlow.logger.toggle_log_level
|
120
|
-
signal_subprocesses
|
120
|
+
signal_subprocesses 'SIGUSR2'
|
121
121
|
end
|
122
122
|
end
|
123
123
|
|
data/lib/rflow/logger.rb
CHANGED
@@ -5,7 +5,7 @@ class RFlow
|
|
5
5
|
extend Forwardable
|
6
6
|
include Log4r
|
7
7
|
|
8
|
-
LOG_PATTERN_FORMAT = '
|
8
|
+
LOG_PATTERN_FORMAT = '%-5l [%d] %x (%-5p) - %M'
|
9
9
|
DATE_METHOD = 'xmlschema(6)'
|
10
10
|
LOG_PATTERN_FORMATTER = PatternFormatter.new :pattern => LOG_PATTERN_FORMAT, :date_method => DATE_METHOD
|
11
11
|
|
@@ -14,6 +14,7 @@ class RFlow
|
|
14
14
|
attr_accessor :log_file_path, :log_level, :log_name
|
15
15
|
|
16
16
|
public
|
17
|
+
attr_accessor :context_width
|
17
18
|
|
18
19
|
# make sure Log4r is initialized; ignored if custom levels are already set
|
19
20
|
Log4r.define_levels(*Log4rConfig::LogLevels)
|
@@ -24,9 +25,15 @@ class RFlow
|
|
24
25
|
*Log4r::LNAMES.map(&:downcase).map {|n| "#{n}?".to_sym }
|
25
26
|
|
26
27
|
def initialize(config, include_stdout = false)
|
28
|
+
reconfigure(config, include_stdout)
|
29
|
+
end
|
30
|
+
|
31
|
+
def reconfigure(config, include_stdout = false)
|
27
32
|
@log_file_path = config['rflow.log_file_path']
|
28
|
-
@log_level = config['rflow.log_level']
|
29
|
-
@log_name =
|
33
|
+
@log_level = config['rflow.log_level'] || "WARN"
|
34
|
+
@log_name = if config['rflow.application_name']; config['rflow.application_name']
|
35
|
+
elsif log_file_path; File.basename(log_file_path)
|
36
|
+
else ''; end
|
30
37
|
|
31
38
|
establish_internal_logger
|
32
39
|
hook_up_logfile
|
@@ -47,8 +54,12 @@ class RFlow
|
|
47
54
|
Outputter['rflow.log_file'].close
|
48
55
|
end
|
49
56
|
|
57
|
+
def level=(level)
|
58
|
+
internal_logger.level = LNAMES.index(level.to_s) || level
|
59
|
+
end
|
60
|
+
|
50
61
|
def toggle_log_level
|
51
|
-
original_log_level = LNAMES[
|
62
|
+
original_log_level = LNAMES[internal_logger.level]
|
52
63
|
new_log_level = (original_log_level == 'DEBUG' ? log_level : 'DEBUG')
|
53
64
|
|
54
65
|
internal_logger.warn "Changing log level from #{original_log_level} to #{new_log_level}"
|
@@ -64,10 +75,12 @@ class RFlow
|
|
64
75
|
end
|
65
76
|
|
66
77
|
def hook_up_logfile
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
78
|
+
if log_file_path
|
79
|
+
begin
|
80
|
+
internal_logger.add FileOutputter.new('rflow.log_file', :filename => log_file_path, :formatter => LOG_PATTERN_FORMATTER)
|
81
|
+
rescue Exception => e
|
82
|
+
raise ArgumentError, "Log file '#{File.expand_path log_file_path}' problem: #{e.message}\n#{e.backtrace.join("\n")}"
|
83
|
+
end
|
71
84
|
end
|
72
85
|
end
|
73
86
|
|
data/lib/rflow/master.rb
CHANGED
@@ -12,6 +12,7 @@ class RFlow
|
|
12
12
|
super(config['rflow.application_name'], 'Master')
|
13
13
|
@pid_file = PIDFile.new(config['rflow.pid_file_path'])
|
14
14
|
@shards = config.shards.map {|config| Shard.new(config) }
|
15
|
+
RFlow.logger.context_width = @shards.flat_map(&:workers).map(&:name).map(&:length).max
|
15
16
|
@brokers = config.connections.flat_map(&:brokers).map {|config| Broker.build(config) }
|
16
17
|
end
|
17
18
|
|
data/lib/rflow/version.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
@@ -8,12 +8,6 @@ require 'tmpdir'
|
|
8
8
|
I18n.enforce_available_locales = true
|
9
9
|
|
10
10
|
RSpec.configure do |config|
|
11
|
-
config.before(:all) do
|
12
|
-
RFlow.logger = Log4r::Logger.new 'test'
|
13
|
-
RFlow.logger.add Log4r::StdoutOutputter.new('test_stdout', :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
|
14
|
-
RFlow.logger.level = 5
|
15
|
-
end
|
16
|
-
|
17
11
|
config.before(:each) do
|
18
12
|
@temp_directory_path = Dir.mktmpdir('rflow')
|
19
13
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael L. Artz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: uuidtools
|