symphony 0.3.0.pre20140327204419 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.gemtest +0 -0
- data/History.rdoc +2 -2
- data/Manifest.txt +2 -1
- data/Rakefile +0 -1
- data/TODO.md +3 -2
- data/USAGE.rdoc +10 -8
- data/bin/symphony +1 -2
- data/lib/symphony.rb +2 -2
- data/lib/symphony/daemon.rb +118 -90
- data/lib/symphony/metrics.rb +2 -1
- data/lib/symphony/queue.rb +17 -1
- data/lib/symphony/signal_handling.rb +1 -0
- data/lib/symphony/task.rb +9 -0
- data/spec/symphony/daemon_spec.rb +13 -0
- data/spec/symphony/queue_spec.rb +5 -5
- data/spec/symphony/task_spec.rb +143 -22
- data/spec/symphony_spec.rb +55 -0
- metadata +5 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2ba69bb4284905cfc71472c6b5629c3c063dd546
|
4
|
+
data.tar.gz: 2bf944cdfce3b98d5eaf91732a9ef62f093eef65
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c656bb9f2ce03e943eda18baec1eac6e7d3c9aad80c771db522cc4c38984e9fdf214fe8ddb0379208775f050877badac843507c0a3cb575ac10c05b8ddbbda6
|
7
|
+
data.tar.gz: 2125f48a7132995aa9a60465e430b656cedf198f48c6e5688e7f317b911f63d0e45c1f2bc5bde4e8b69c7f057397536dcfe39acd60f65c31c2ebb232d16a2e4c
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/.gemtest
ADDED
File without changes
|
data/History.rdoc
CHANGED
data/Manifest.txt
CHANGED
@@ -23,8 +23,9 @@ lib/symphony/tasks/pinger.rb
|
|
23
23
|
lib/symphony/tasks/simulator.rb
|
24
24
|
lib/symphony/tasks/ssh.rb
|
25
25
|
lib/symphony/tasks/sshscript.rb
|
26
|
+
spec/helpers.rb
|
27
|
+
spec/symphony/daemon_spec.rb
|
26
28
|
spec/symphony/mixins_spec.rb
|
27
29
|
spec/symphony/queue_spec.rb
|
28
30
|
spec/symphony/task_spec.rb
|
29
31
|
spec/symphony_spec.rb
|
30
|
-
spec/helpers.rb
|
data/Rakefile
CHANGED
data/TODO.md
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
|
2
2
|
[ ] Convert routing / metrics to 'plugins' syntax (inside/outside for prepend/include)
|
3
|
-
[ ] Process management w/ Daemon
|
4
3
|
[ ] Update pinger/ssh/sshscript
|
5
|
-
|
4
|
+
[ ] Per Task throttling for daemon
|
5
|
+
[ ] Task scaling (min/max settings), check queue backlog during received message
|
6
|
+
[ ] Make the block of routes declared with on() method accept various arities
|
data/USAGE.rdoc
CHANGED
@@ -252,10 +252,7 @@ name of the task. You can override this per-task.
|
|
252
252
|
|
253
253
|
= Plugins
|
254
254
|
|
255
|
-
Plugins change or enhance the behavior of a Symphony::Task.
|
256
|
-
are enabled with the 'plugin' option, with the name of the plugin (or
|
257
|
-
comma separated plugins) as a symbol argument. Symphony currently
|
258
|
-
ships with two.
|
255
|
+
Plugins change or enhance the behavior of a Symphony::Task.
|
259
256
|
|
260
257
|
|
261
258
|
== Metrics
|
@@ -265,8 +262,13 @@ of a task worker to the log. It shows processed message averages and
|
|
265
262
|
resource consumption summaries at the "info" log level, and also changes
|
266
263
|
the process name to display a total count and jobs per second rate.
|
267
264
|
|
268
|
-
|
265
|
+
require 'symphony/metrics'
|
269
266
|
|
267
|
+
class Test < Symphony::Task
|
268
|
+
prepend Symphony::Metrics
|
269
|
+
# ...
|
270
|
+
end
|
271
|
+
|
270
272
|
|
271
273
|
== Routing
|
272
274
|
|
@@ -275,10 +277,10 @@ in the #work method, and instead links individual units of work to
|
|
275
277
|
separate #on declarations. This makes tasks that are designed to
|
276
278
|
receive multiple message topics much easier to maintain and test.
|
277
279
|
|
280
|
+
require 'symphony/routing'
|
281
|
+
|
278
282
|
class Test < Symphony::Task
|
279
|
-
|
280
|
-
# Use the routing plugin
|
281
|
-
plugin :routing
|
283
|
+
include Symphony::Routing
|
282
284
|
|
283
285
|
on 'users.create', 'workstation.create' do |payload, metadata|
|
284
286
|
puts "A user or workstation wants to be created!"
|
data/bin/symphony
CHANGED
data/lib/symphony.rb
CHANGED
@@ -15,11 +15,11 @@ module Symphony
|
|
15
15
|
VERSION = '0.3.0'
|
16
16
|
|
17
17
|
# Version-control revision constant
|
18
|
-
REVISION = %q$Revision$
|
18
|
+
REVISION = %q$Revision: cf61ced7d585 $
|
19
19
|
|
20
20
|
|
21
21
|
# The name of the environment variable to check for config file overrides
|
22
|
-
CONFIG_ENV = '
|
22
|
+
CONFIG_ENV = 'SYMPHONY_CONFIG'
|
23
23
|
|
24
24
|
# The path to the default config file
|
25
25
|
DEFAULT_CONFIG_FILE = 'etc/config.yml'
|
data/lib/symphony/daemon.rb
CHANGED
@@ -3,19 +3,17 @@
|
|
3
3
|
|
4
4
|
require 'configurability'
|
5
5
|
require 'loggability'
|
6
|
-
require 'fcntl'
|
7
|
-
require 'trollop'
|
8
6
|
|
9
7
|
require 'symphony' unless defined?( Symphony )
|
10
|
-
require 'symphony/worker'
|
11
8
|
require 'symphony/task'
|
9
|
+
require 'symphony/signal_handling'
|
12
10
|
|
13
|
-
|
14
|
-
#
|
15
|
-
# contained in the jobs it fetches.
|
11
|
+
# A daemon which manages startup and shutdown of one or more Workers
|
12
|
+
# running Tasks as they are published from a queue.
|
16
13
|
class Symphony::Daemon
|
17
14
|
extend Loggability,
|
18
|
-
Configurability
|
15
|
+
Configurability,
|
16
|
+
Symphony::MethodUtilities
|
19
17
|
|
20
18
|
include Symphony::SignalHandling
|
21
19
|
|
@@ -24,27 +22,42 @@ class Symphony::Daemon
|
|
24
22
|
log_to :symphony
|
25
23
|
|
26
24
|
# Configurability API -- use the 'worker_daemon' section of the config
|
27
|
-
config_key :
|
25
|
+
config_key :symphony
|
26
|
+
|
28
27
|
|
28
|
+
# Default configuration
|
29
|
+
CONFIG_DEFAULTS = {
|
30
|
+
throttle_max: 16,
|
31
|
+
throttle_factor: 1,
|
32
|
+
tasks: []
|
33
|
+
}
|
29
34
|
|
30
35
|
# Signals we understand
|
31
36
|
QUEUE_SIGS = [
|
32
|
-
:QUIT, :INT, :TERM, :HUP,
|
37
|
+
:QUIT, :INT, :TERM, :HUP, :CHLD,
|
33
38
|
# :TODO: :WINCH, :USR1, :USR2, :TTIN, :TTOU
|
34
39
|
]
|
35
40
|
|
36
|
-
# The maximum throttle value caused by failing workers
|
37
|
-
THROTTLE_MAX = 16
|
38
|
-
|
39
|
-
# The factor which controls how much incrementing the throttle factor
|
40
|
-
# affects the pause between workers being started.
|
41
|
-
THROTTLE_FACTOR = 2
|
42
41
|
|
43
42
|
|
44
43
|
#
|
45
44
|
# Class methods
|
46
45
|
#
|
47
46
|
|
47
|
+
##
|
48
|
+
# The maximum throttle factor caused by failing workers
|
49
|
+
singleton_attr_accessor :throttle_max
|
50
|
+
|
51
|
+
##
|
52
|
+
# The factor which controls how much incrementing the throttle factor
|
53
|
+
# affects the pause between workers being started.
|
54
|
+
singleton_attr_accessor :throttle_factor
|
55
|
+
|
56
|
+
##
|
57
|
+
# The Array of Symphony::Task classes that are configured to run
|
58
|
+
singleton_attr_accessor :tasks
|
59
|
+
|
60
|
+
|
48
61
|
### Get the daemon's version as a String.
|
49
62
|
def self::version_string( include_buildnum=false )
|
50
63
|
vstring = "%s %s" % [ self.name, Symphony::VERSION ]
|
@@ -56,37 +69,41 @@ class Symphony::Daemon
|
|
56
69
|
end
|
57
70
|
|
58
71
|
|
59
|
-
###
|
60
|
-
def self::
|
61
|
-
|
72
|
+
### Configurability API -- configure the daemon.
|
73
|
+
def self::configure( config=nil )
|
74
|
+
config = self.defaults.merge( config || {} )
|
62
75
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
76
|
+
self.throttle_max = config[:throttle_max]
|
77
|
+
self.throttle_factor = config[:throttle_factor]
|
78
|
+
|
79
|
+
self.tasks = self.load_configured_tasks( config[:tasks] )
|
80
|
+
end
|
67
81
|
|
68
|
-
opt :config, "The config file to load instead of the default",
|
69
|
-
:type => :string
|
70
|
-
opt :crew_size, "Number of workers to maintain.", :default => DEFAULT_CREW_SIZE
|
71
|
-
opt :queue, "The name of the queue to monitor.", :default => '_default_'
|
72
82
|
|
73
|
-
|
83
|
+
### Load the tasks with the specified +task_names+ and return them
|
84
|
+
### as an Array.
|
85
|
+
def self::load_configured_tasks( task_names )
|
86
|
+
return task_names.map do |task_name|
|
87
|
+
Symphony::Task.get_subclass( task_name )
|
74
88
|
end
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
### Start the daemon.
|
93
|
+
def self::run( args )
|
94
|
+
Loggability.format_with( :color ) if $stdout.tty?
|
75
95
|
|
76
96
|
# Turn on debugging if it's enabled
|
77
|
-
if
|
78
|
-
$DEBUG = true
|
79
|
-
Loggability.level = :debug
|
80
|
-
end
|
97
|
+
Loggability.level = :debug if $DEBUG
|
81
98
|
|
82
99
|
# Now load the config file
|
83
|
-
Symphony.load_config(
|
100
|
+
Symphony.load_config( args.shift )
|
84
101
|
|
85
102
|
# Re-enable debug-level logging if the config reset it
|
86
|
-
Loggability.level = :debug if
|
103
|
+
Loggability.level = :debug if $DEBUG
|
87
104
|
|
88
105
|
# And start the daemon
|
89
|
-
self.new
|
106
|
+
self.new.run
|
90
107
|
end
|
91
108
|
|
92
109
|
|
@@ -95,13 +112,11 @@ class Symphony::Daemon
|
|
95
112
|
#
|
96
113
|
|
97
114
|
### Create a new Daemon instance.
|
98
|
-
def initialize
|
99
|
-
@options = options
|
100
|
-
@queue = Symphony::Queue.new( options.queue )
|
101
|
-
|
115
|
+
def initialize
|
102
116
|
# Process control
|
103
|
-
@
|
104
|
-
|
117
|
+
@tasks = self.class.tasks
|
118
|
+
|
119
|
+
@running_tasks = {}
|
105
120
|
@running = false
|
106
121
|
@shutting_down = false
|
107
122
|
@throttle = 0
|
@@ -115,11 +130,8 @@ class Symphony::Daemon
|
|
115
130
|
public
|
116
131
|
######
|
117
132
|
|
118
|
-
# The
|
119
|
-
attr_reader :
|
120
|
-
|
121
|
-
# The maximum number of children to have running at any given time
|
122
|
-
attr_reader :crew_size
|
133
|
+
# The Hash of PIDs to task class
|
134
|
+
attr_reader :running_tasks
|
123
135
|
|
124
136
|
# A self-pipe for deferred signal-handling
|
125
137
|
attr_reader :selfpipe
|
@@ -145,19 +157,13 @@ class Symphony::Daemon
|
|
145
157
|
|
146
158
|
### Set up the daemon and start running.
|
147
159
|
def run
|
148
|
-
self.log.info "Starting
|
149
|
-
|
150
|
-
# Become session leader if we can
|
151
|
-
if Process.euid.zero?
|
152
|
-
sid = Process.setsid
|
153
|
-
self.log.debug " became session leader of new session: %d" % [ sid ]
|
154
|
-
end
|
160
|
+
self.log.info "Starting task daemon"
|
155
161
|
|
156
162
|
# Set up traps for common signals
|
157
163
|
self.set_signal_traps( *QUEUE_SIGS )
|
158
164
|
|
159
165
|
# Listen for new jobs and handle them as they come in
|
160
|
-
self.
|
166
|
+
self.run_tasks
|
161
167
|
|
162
168
|
# Restore the default signal handlers
|
163
169
|
self.reset_signal_traps( *QUEUE_SIGS )
|
@@ -168,26 +174,22 @@ class Symphony::Daemon
|
|
168
174
|
|
169
175
|
### The main loop of the daemon -- wait for signals, children dying, or jobs, and
|
170
176
|
### take appropriate action.
|
171
|
-
def
|
177
|
+
def run_tasks
|
172
178
|
@running = true
|
173
179
|
|
174
180
|
self.log.debug "Starting supervisor loop..."
|
175
181
|
while self.running?
|
176
182
|
self.start_missing_children unless self.shutting_down?
|
177
|
-
|
178
|
-
timeout = self.throttle_seconds
|
179
|
-
timeout = nil if timeout.zero?
|
180
|
-
|
181
183
|
self.wait_for_signals
|
182
184
|
self.reap_children
|
183
185
|
end
|
184
|
-
self.log.info "Supervisor job loop done."
|
185
186
|
|
186
187
|
rescue => err
|
187
188
|
self.log.fatal "%p in job-handler loop: %s" % [ err.class, err.message ]
|
188
189
|
self.log.debug { ' ' + err.backtrace.join("\n ") }
|
189
190
|
|
190
191
|
ensure
|
192
|
+
self.log.info "Done running tasks."
|
191
193
|
@running = false
|
192
194
|
self.stop
|
193
195
|
end
|
@@ -202,18 +204,18 @@ class Symphony::Daemon
|
|
202
204
|
|
203
205
|
self.log.warn "Stopping children."
|
204
206
|
3.times do |i|
|
205
|
-
self.reap_children
|
207
|
+
self.reap_children
|
206
208
|
sleep( 1 )
|
207
209
|
self.kill_children
|
208
210
|
sleep( 1 )
|
209
|
-
break if self.
|
211
|
+
break if self.running_tasks.empty?
|
210
212
|
sleep( 1 )
|
211
|
-
end unless self.
|
213
|
+
end unless self.running_tasks.empty?
|
212
214
|
|
213
215
|
# Give up on our remaining children.
|
214
216
|
Signal.trap( :CHLD, :IGNORE )
|
215
|
-
if !self.
|
216
|
-
self.log.warn " %d workers remain: sending KILL" % [ self.
|
217
|
+
if !self.running_tasks.empty?
|
218
|
+
self.log.warn " %d workers remain: sending KILL" % [ self.running_tasks.length ]
|
217
219
|
self.kill_children( :KILL )
|
218
220
|
end
|
219
221
|
end
|
@@ -232,11 +234,11 @@ class Symphony::Daemon
|
|
232
234
|
|
233
235
|
### Handle signals.
|
234
236
|
def handle_signal( sig )
|
235
|
-
self.log.debug "Handling signal %s" % [ sig ]
|
237
|
+
self.log.debug "Handling signal %s in PID %d" % [ sig, Process.pid ]
|
236
238
|
case sig
|
237
239
|
when :INT, :TERM
|
238
240
|
if @running
|
239
|
-
self.log.warn "%s signal:
|
241
|
+
self.log.warn "%s signal: graceful shutdown" % [ sig ]
|
240
242
|
@running = false
|
241
243
|
else
|
242
244
|
self.ignore_signals
|
@@ -250,6 +252,7 @@ class Symphony::Daemon
|
|
250
252
|
self.reload_config
|
251
253
|
|
252
254
|
when :CHLD
|
255
|
+
self.log.info "Got SIGCHLD."
|
253
256
|
# Just need to wake up, nothing else necessary
|
254
257
|
|
255
258
|
else
|
@@ -259,10 +262,10 @@ class Symphony::Daemon
|
|
259
262
|
end
|
260
263
|
|
261
264
|
|
262
|
-
###
|
265
|
+
### Start any tasks which aren't already running
|
263
266
|
def start_missing_children
|
264
|
-
|
265
|
-
return
|
267
|
+
missing_tasks = self.find_missing_tasks
|
268
|
+
return if missing_tasks.empty?
|
266
269
|
|
267
270
|
# Return unless the throttle period has lapsed
|
268
271
|
unless self.throttle_seconds < (Time.now - @last_child_started)
|
@@ -271,21 +274,37 @@ class Symphony::Daemon
|
|
271
274
|
return
|
272
275
|
end
|
273
276
|
|
274
|
-
self.log.debug "Starting %d
|
275
|
-
|
276
|
-
pid = self.start_worker
|
277
|
-
self.log.debug " started
|
278
|
-
self.
|
277
|
+
self.log.debug "Starting %d tasks out of %d" % [ missing_tasks.size, self.class.tasks.size ]
|
278
|
+
missing_tasks.each do |task_class|
|
279
|
+
pid = self.start_worker( task_class )
|
280
|
+
self.log.debug " started task %p at pid %d" % [ task_class, pid ]
|
281
|
+
self.running_tasks[ pid ] = task_class
|
279
282
|
end
|
280
283
|
|
281
284
|
@last_child_started = Time.now
|
282
285
|
end
|
283
286
|
|
284
287
|
|
288
|
+
### Examine the running tasks and return any that are missing.
|
289
|
+
def find_missing_tasks
|
290
|
+
missing_tasks = []
|
291
|
+
|
292
|
+
self.class.tasks.uniq.each do |task_type|
|
293
|
+
count = self.class.tasks.count( task_type )
|
294
|
+
missing = count - self.running_tasks.values.count( task_type )
|
295
|
+
missing.times do
|
296
|
+
missing_tasks << task_type
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
return missing_tasks
|
301
|
+
end
|
302
|
+
|
303
|
+
|
285
304
|
### Return the number of seconds between child startup times.
|
286
305
|
def throttle_seconds
|
287
306
|
return 0 unless @throttle.nonzero?
|
288
|
-
return Math.log( @throttle ) *
|
307
|
+
return Math.log( @throttle ) * self.class.throttle_factor
|
289
308
|
end
|
290
309
|
|
291
310
|
|
@@ -295,18 +314,18 @@ class Symphony::Daemon
|
|
295
314
|
self.log.debug "Adjusting worker throttle by %d" % [ adjustment ]
|
296
315
|
@throttle += adjustment
|
297
316
|
@throttle = 0 if @throttle < 0
|
298
|
-
@throttle =
|
317
|
+
@throttle = self.class.throttle_max if @throttle > self.class.throttle_max
|
299
318
|
end
|
300
319
|
|
301
320
|
|
302
321
|
### Kill all current children with the specified +signal+. Returns +true+ if the signal was
|
303
322
|
### sent to one or more children.
|
304
323
|
def kill_children( signal=:TERM )
|
305
|
-
return false if self.
|
324
|
+
return false if self.running_tasks.empty?
|
306
325
|
|
307
|
-
self.log.info "Sending %s signal to %d
|
308
|
-
[ signal, self.
|
309
|
-
Process.kill( signal, *self.
|
326
|
+
self.log.info "Sending %s signal to %d task pids: %p." %
|
327
|
+
[ signal, self.running_tasks.length, self.running_tasks.keys ]
|
328
|
+
Process.kill( signal, *self.running_tasks.keys )
|
310
329
|
|
311
330
|
return true
|
312
331
|
rescue Errno::ESRCH
|
@@ -314,11 +333,18 @@ class Symphony::Daemon
|
|
314
333
|
end
|
315
334
|
|
316
335
|
|
317
|
-
### Start a new Symphony::
|
318
|
-
def start_worker
|
336
|
+
### Start a new Symphony::Task and return its PID.
|
337
|
+
def start_worker( task_class )
|
319
338
|
return if self.shutting_down?
|
320
|
-
|
321
|
-
|
339
|
+
|
340
|
+
self.log.debug "Starting a %p." % [ task_class ]
|
341
|
+
pid = Process.fork do
|
342
|
+
task_class.after_fork
|
343
|
+
task_class.run
|
344
|
+
end
|
345
|
+
Process.setpgid( pid, 0 )
|
346
|
+
|
347
|
+
return pid
|
322
348
|
end
|
323
349
|
|
324
350
|
|
@@ -340,28 +366,30 @@ class Symphony::Daemon
|
|
340
366
|
|
341
367
|
|
342
368
|
### Reap any children that have died within the caller's process group
|
343
|
-
### and remove them from the
|
369
|
+
### and remove them from the Hash of running tasks.
|
344
370
|
def reap_any_child
|
345
371
|
self.log.debug " no pids; waiting on any child in this process group"
|
346
372
|
|
347
|
-
pid, status = Process.waitpid2( -1, Process::WNOHANG )
|
373
|
+
pid, status = Process.waitpid2( -1, Process::WNOHANG|Process::WUNTRACED )
|
374
|
+
self.log.debug " waitpid2 returned: [ %p, %p ]" % [ pid, status ]
|
348
375
|
while pid
|
349
376
|
self.adjust_throttle( status.success? ? -1 : 1 )
|
350
377
|
self.log.debug "Child %d exited: %p." % [ pid, status ]
|
351
|
-
self.
|
378
|
+
self.running_tasks.delete( pid )
|
352
379
|
|
353
|
-
pid, status = Process.waitpid2( -1, Process::WNOHANG )
|
380
|
+
pid, status = Process.waitpid2( -1, Process::WNOHANG|Process::WUNTRACED )
|
381
|
+
self.log.debug " waitpid2 returned: [ %p, %p ]" % [ pid, status ]
|
354
382
|
end
|
355
383
|
end
|
356
384
|
|
357
385
|
|
358
386
|
### Wait on the child associated with the given +pid+, deleting it from the
|
359
|
-
###
|
387
|
+
### running tasks Hash if successful.
|
360
388
|
def reap_specific_child( pid )
|
361
389
|
spid, status = Process.waitpid2( pid )
|
362
390
|
if spid
|
363
391
|
self.log.debug "Child %d exited: %p." % [ spid, status ]
|
364
|
-
self.
|
392
|
+
self.running_tasks.delete( spid )
|
365
393
|
self.adjust_throttle( status.success? ? -1 : 1 )
|
366
394
|
else
|
367
395
|
self.log.debug "Child %d no reapy." % [ pid ]
|
data/lib/symphony/metrics.rb
CHANGED
@@ -28,7 +28,8 @@ module Symphony::Metrics
|
|
28
28
|
|
29
29
|
@log_reporter = Metriks::Reporter::Logger.new(
|
30
30
|
logger: Loggability[ Symphony ],
|
31
|
-
registry: @metriks_registry
|
31
|
+
registry: @metriks_registry,
|
32
|
+
prefix: self.class.name )
|
32
33
|
@proc_reporter = Metriks::Reporter::ProcTitle.new(
|
33
34
|
prefix: self.class.name,
|
34
35
|
registry: @metriks_registry,
|
data/lib/symphony/queue.rb
CHANGED
@@ -209,7 +209,8 @@ class Symphony::Queue
|
|
209
209
|
tag = self.consumer_tag
|
210
210
|
|
211
211
|
# Last argument is *no_ack*, so need to invert the logic
|
212
|
-
self.log.debug "Creating
|
212
|
+
self.log.debug "Creating consumer for the '%s' queue with tag: %s" %
|
213
|
+
[ amqp_queue.name, tag ]
|
213
214
|
cons = Bunny::Consumer.new( amqp_queue.channel, amqp_queue, tag, !ackmode )
|
214
215
|
|
215
216
|
cons.on_delivery do |delivery_info, properties, payload|
|
@@ -309,5 +310,20 @@ class Symphony::Queue
|
|
309
310
|
self.consumer.channel.close
|
310
311
|
end
|
311
312
|
|
313
|
+
|
314
|
+
### Return a human-readable representation of the Queue in a form suitable for debugging.
|
315
|
+
def inspect
|
316
|
+
return "#<%p:%#0x %s (%s) ack: %s, routing: %p, prefetch: %d>" % [
|
317
|
+
self.class,
|
318
|
+
self.object_id * 2,
|
319
|
+
self.name,
|
320
|
+
self.consumer_tag,
|
321
|
+
self.acknowledge ? "yes" : "no",
|
322
|
+
self.routing_keys,
|
323
|
+
self.prefetch,
|
324
|
+
]
|
325
|
+
end
|
326
|
+
|
327
|
+
|
312
328
|
end # class Symphony::Queue
|
313
329
|
|
data/lib/symphony/task.rb
CHANGED
@@ -51,6 +51,14 @@ class Symphony::Task
|
|
51
51
|
end
|
52
52
|
|
53
53
|
|
54
|
+
### Prepare the process after being forked from the Daemon.
|
55
|
+
def self::after_fork
|
56
|
+
self.log.debug "After fork [%d]: Threads: %p" % [ Process.pid, ThreadGroup::Default.list ]
|
57
|
+
Process.setpgrp
|
58
|
+
Symphony.config.install if Symphony.config
|
59
|
+
end
|
60
|
+
|
61
|
+
|
54
62
|
### Inheritance hook -- set some defaults on subclasses.
|
55
63
|
def self::inherited( subclass )
|
56
64
|
super
|
@@ -193,6 +201,7 @@ class Symphony::Task
|
|
193
201
|
end
|
194
202
|
|
195
203
|
|
204
|
+
|
196
205
|
######
|
197
206
|
public
|
198
207
|
######
|
data/spec/symphony/queue_spec.rb
CHANGED
@@ -150,7 +150,7 @@ describe Symphony::Queue do
|
|
150
150
|
|
151
151
|
|
152
152
|
it "subscribes to the message queue with a configured consumer to wait for messages" do
|
153
|
-
amqp_queue = double( "AMQP queue", channel: described_class.amqp_channel )
|
153
|
+
amqp_queue = double( "AMQP queue", name: 'a queue', channel: described_class.amqp_channel )
|
154
154
|
consumer = double( "Bunny consumer", channel: described_class.amqp_channel )
|
155
155
|
|
156
156
|
expect( described_class.amqp_channel ).to receive( :queue ).
|
@@ -202,7 +202,7 @@ describe Symphony::Queue do
|
|
202
202
|
|
203
203
|
|
204
204
|
it "sets up the queue and consumer to only run once if waiting in one-shot mode" do
|
205
|
-
amqp_queue = double( "AMQP queue", channel: described_class.amqp_channel )
|
205
|
+
amqp_queue = double( "AMQP queue", name: 'a queue', channel: described_class.amqp_channel )
|
206
206
|
consumer = double( "Bunny consumer", channel: described_class.amqp_channel )
|
207
207
|
|
208
208
|
expect( described_class.amqp_channel ).to receive( :queue ).
|
@@ -243,7 +243,7 @@ describe Symphony::Queue do
|
|
243
243
|
|
244
244
|
|
245
245
|
it "shuts down the consumer if the queues it's consuming from is deleted on the server" do
|
246
|
-
amqp_queue = double( "AMQP queue", channel: described_class.amqp_channel )
|
246
|
+
amqp_queue = double( "AMQP queue", name: 'a queue', channel: described_class.amqp_channel )
|
247
247
|
consumer = double( "Bunny consumer", channel: described_class.amqp_channel )
|
248
248
|
|
249
249
|
expect( described_class.amqp_channel ).to receive( :queue ).
|
@@ -277,7 +277,7 @@ describe Symphony::Queue do
|
|
277
277
|
|
278
278
|
it "creates a consumer with acknowledgements enabled if it has acknowledgements enabled" do
|
279
279
|
amqp_channel = double( "AMQP channel" )
|
280
|
-
amqp_queue = double( "AMQP queue", channel: amqp_channel )
|
280
|
+
amqp_queue = double( "AMQP queue", name: 'a queue', channel: amqp_channel )
|
281
281
|
consumer = double( "Bunny consumer" )
|
282
282
|
|
283
283
|
# Ackmode argument is actually 'no_ack'
|
@@ -293,7 +293,7 @@ describe Symphony::Queue do
|
|
293
293
|
|
294
294
|
it "creates a consumer with acknowledgements disabled if it has acknowledgements disabled" do
|
295
295
|
amqp_channel = double( "AMQP channel" )
|
296
|
-
amqp_queue = double( "AMQP queue", channel: amqp_channel )
|
296
|
+
amqp_queue = double( "AMQP queue", name: 'a queue', channel: amqp_channel )
|
297
297
|
consumer = double( "Bunny consumer" )
|
298
298
|
|
299
299
|
# Ackmode argument is actually 'no_ack'
|
data/spec/symphony/task_spec.rb
CHANGED
@@ -64,84 +64,205 @@ describe Symphony::Task do
|
|
64
64
|
|
65
65
|
context "a concrete subclass" do
|
66
66
|
|
67
|
-
|
68
|
-
|
67
|
+
let( :task_class ) do
|
68
|
+
Class.new( described_class ) do
|
69
69
|
def self::name; 'ACME::TestingTask'; end
|
70
70
|
end
|
71
71
|
end
|
72
|
+
let( :payload ) {{ "the" => "payload" }}
|
73
|
+
let( :serialized_payload ) { Yajl.dump(payload) }
|
74
|
+
let( :metadata ) {{ :content_type => 'application/json' }}
|
75
|
+
let( :queue ) do
|
76
|
+
obj = Symphony::Queue.for_task( task_class )
|
77
|
+
# Don't really talk to AMQP for messages
|
78
|
+
allow( obj ).to receive( :wait_for_message ) do |oneshot, &callback|
|
79
|
+
callback.call( serialized_payload, metadata )
|
80
|
+
end
|
81
|
+
obj
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
it "puts the process into its own process group after a fork" do
|
86
|
+
expect( Process ).to receive( :setpgrp ).with( no_args )
|
87
|
+
task_class.after_fork
|
88
|
+
end
|
72
89
|
|
73
90
|
|
74
91
|
it "raises an exception if run without specifying any subscriptions" do
|
75
|
-
expect {
|
92
|
+
expect { task_class.run }.to raise_error( ScriptError, /no subscriptions/i )
|
76
93
|
end
|
77
94
|
|
78
95
|
|
79
96
|
it "can set an explicit queue name" do
|
80
|
-
|
81
|
-
expect(
|
97
|
+
task_class.queue_name( 'happy.fun.queue' )
|
98
|
+
expect( task_class.queue_name ).to eq( 'happy.fun.queue' )
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
it "can set the number of messages to prefetch" do
|
103
|
+
task_class.prefetch( 10 )
|
104
|
+
expect( task_class.prefetch ).to eq( 10 )
|
82
105
|
end
|
83
106
|
|
84
107
|
|
85
108
|
it "can retry on timeout instead of rejecting" do
|
86
|
-
|
87
|
-
expect(
|
109
|
+
task_class.timeout_action( :retry )
|
110
|
+
expect( task_class.timeout_action ).to eq( :retry )
|
88
111
|
end
|
89
112
|
|
90
113
|
|
91
114
|
it "provides a default name for its queue based on its name" do
|
92
|
-
expect(
|
115
|
+
expect( task_class.queue_name ).to eq( 'acme.testingtask' )
|
93
116
|
end
|
94
117
|
|
95
118
|
|
96
119
|
it "can declare a pattern to use when subscribing" do
|
97
|
-
|
98
|
-
expect(
|
120
|
+
task_class.subscribe_to( 'foo.test' )
|
121
|
+
expect( task_class.routing_keys ).to include( 'foo.test' )
|
99
122
|
end
|
100
123
|
|
101
124
|
|
102
125
|
it "has acknowledgements enabled by default" do
|
103
|
-
expect(
|
126
|
+
expect( task_class.acknowledge ).to eq( true )
|
104
127
|
end
|
105
128
|
|
106
129
|
|
107
130
|
it "can enable acknowledgements" do
|
108
|
-
|
109
|
-
expect(
|
131
|
+
task_class.acknowledge( true )
|
132
|
+
expect( task_class.acknowledge ).to eq( true )
|
110
133
|
end
|
111
134
|
|
112
135
|
|
113
136
|
it "can disable acknowledgements" do
|
114
|
-
|
115
|
-
expect(
|
137
|
+
task_class.acknowledge( false )
|
138
|
+
expect( task_class.acknowledge ).to eq( false )
|
116
139
|
end
|
117
140
|
|
118
141
|
|
119
142
|
it "can set a timeout" do
|
120
|
-
|
121
|
-
expect(
|
143
|
+
task_class.timeout( 10 )
|
144
|
+
expect( task_class.timeout ).to eq( 10 )
|
122
145
|
end
|
123
146
|
|
124
147
|
|
125
148
|
it "can declare a one-shot work model" do
|
126
|
-
|
127
|
-
expect(
|
149
|
+
task_class.work_model( :oneshot )
|
150
|
+
expect( task_class.work_model ).to eq( :oneshot )
|
128
151
|
end
|
129
152
|
|
130
153
|
|
131
154
|
it "can declare a long-lived work model" do
|
132
|
-
|
133
|
-
expect(
|
155
|
+
task_class.work_model( :longlived )
|
156
|
+
expect( task_class.work_model ).to eq( :longlived )
|
134
157
|
end
|
135
158
|
|
136
159
|
|
137
160
|
it "raises an error if an invalid work model is declared " do
|
138
161
|
expect {
|
139
|
-
|
162
|
+
task_class.work_model( :lazy )
|
140
163
|
}.to raise_error( /unknown work_model/i )
|
141
164
|
end
|
142
165
|
|
143
166
|
|
167
|
+
context "an instance" do
|
168
|
+
|
169
|
+
let( :task_class ) do
|
170
|
+
Class.new( described_class ) do
|
171
|
+
def initialize( * )
|
172
|
+
super
|
173
|
+
@received_messages = []
|
174
|
+
end
|
175
|
+
attr_reader :received_messages
|
176
|
+
|
177
|
+
def work( payload, metadata )
|
178
|
+
self.received_messages << [ payload, metadata ]
|
179
|
+
true
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
let( :task ) { task_class.new(queue) }
|
185
|
+
|
186
|
+
|
187
|
+
it "handles received messages by calling its work method" do
|
188
|
+
expect( queue ).to receive( :wait_for_message ) do |oneshot, &callback|
|
189
|
+
callback.call( serialized_payload, metadata )
|
190
|
+
end
|
191
|
+
|
192
|
+
task.start_handling_messages
|
193
|
+
|
194
|
+
expect( task.received_messages ).to eq([ [payload, metadata] ])
|
195
|
+
end
|
196
|
+
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
|
201
|
+
context "an instance with a timeout" do
|
202
|
+
|
203
|
+
let( :task_class ) do
|
204
|
+
Class.new( described_class ) do
|
205
|
+
timeout 0.2
|
206
|
+
def initialize( * )
|
207
|
+
super
|
208
|
+
@received_messages = []
|
209
|
+
@sleeptime = 0
|
210
|
+
end
|
211
|
+
attr_reader :received_messages
|
212
|
+
attr_accessor :sleeptime
|
213
|
+
|
214
|
+
def work( payload, metadata )
|
215
|
+
self.received_messages << [ payload, metadata ]
|
216
|
+
sleep( self.sleeptime )
|
217
|
+
true
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
let( :task ) { task_class.new(queue) }
|
223
|
+
|
224
|
+
|
225
|
+
it "returns true if the work completes before the timeout" do
|
226
|
+
task.sleeptime = 0
|
227
|
+
expect( task.start_handling_messages ).to be_truthy
|
228
|
+
end
|
229
|
+
|
230
|
+
|
231
|
+
it "raises a Timeout::Error if the work takes longer than the timeout" do
|
232
|
+
task.sleeptime = task_class.timeout + 2
|
233
|
+
expect {
|
234
|
+
task.start_handling_messages
|
235
|
+
}.to raise_error( Timeout::Error, /execution expired/ )
|
236
|
+
end
|
237
|
+
|
238
|
+
|
239
|
+
it "returns false if the work takes longer than the timeout and the timeout_action is set to :retry" do
|
240
|
+
task_class.timeout_action( :retry )
|
241
|
+
task.sleeptime = task_class.timeout + 2
|
242
|
+
expect( task.start_handling_messages ).to be_falsey
|
243
|
+
end
|
244
|
+
|
245
|
+
end
|
246
|
+
|
247
|
+
|
248
|
+
|
249
|
+
context "an instance with no #work method" do
|
250
|
+
|
251
|
+
let( :task ) { task_class.new(queue) }
|
252
|
+
|
253
|
+
it "raises an exception when told to do work" do
|
254
|
+
expect {
|
255
|
+
task.work( 'payload', {} )
|
256
|
+
}.to raise_error( NotImplementedError, /#work/ )
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
260
|
+
|
261
|
+
|
262
|
+
|
144
263
|
end
|
145
264
|
|
265
|
+
|
266
|
+
|
146
267
|
end
|
147
268
|
|
data/spec/symphony_spec.rb
CHANGED
@@ -8,7 +8,62 @@ require 'symphony'
|
|
8
8
|
|
9
9
|
describe Symphony do
|
10
10
|
|
11
|
+
before( :each ) do
|
12
|
+
ENV.delete( 'SYMPHONY_CONFIG' )
|
13
|
+
end
|
11
14
|
|
12
15
|
|
16
|
+
it "will load a default config file if none is specified" do
|
17
|
+
config_object = double( "Configurability::Config object" )
|
18
|
+
expect( Configurability ).to receive( :gather_defaults ).
|
19
|
+
and_return( {} )
|
20
|
+
expect( Configurability::Config ).to receive( :load ).
|
21
|
+
with( described_class::DEFAULT_CONFIG_FILE, {} ).
|
22
|
+
and_return( config_object )
|
23
|
+
expect( config_object ).to receive( :install )
|
24
|
+
|
25
|
+
described_class.load_config
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
it "will load a config file given in an environment variable if none is specified" do
|
30
|
+
ENV['SYMPHONY_CONFIG'] = '/usr/local/etc/config.yml'
|
31
|
+
|
32
|
+
config_object = double( "Configurability::Config object" )
|
33
|
+
expect( Configurability ).to receive( :gather_defaults ).
|
34
|
+
and_return( {} )
|
35
|
+
expect( Configurability::Config ).to receive( :load ).
|
36
|
+
with( '/usr/local/etc/config.yml', {} ).
|
37
|
+
and_return( config_object )
|
38
|
+
expect( config_object ).to receive( :install )
|
39
|
+
|
40
|
+
described_class.load_config
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
it "will load a config file and install it if one is given" do
|
45
|
+
config_object = double( "Configurability::Config object" )
|
46
|
+
expect( Configurability ).to receive( :gather_defaults ).
|
47
|
+
and_return( {} )
|
48
|
+
expect( Configurability::Config ).to receive( :load ).
|
49
|
+
with( 'a/configfile.yml', {} ).
|
50
|
+
and_return( config_object )
|
51
|
+
expect( config_object ).to receive( :install )
|
52
|
+
|
53
|
+
described_class.load_config( 'a/configfile.yml' )
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
it "will override default values when loading the config if they're given" do
|
58
|
+
config_object = double( "Configurability::Config object" )
|
59
|
+
expect( Configurability ).to_not receive( :gather_defaults )
|
60
|
+
expect( Configurability::Config ).to receive( :load ).
|
61
|
+
with( 'a/different/configfile.yml', {database: {dbname: 'test'}} ).
|
62
|
+
and_return( config_object )
|
63
|
+
expect( config_object ).to receive( :install )
|
64
|
+
|
65
|
+
described_class.load_config( 'a/different/configfile.yml', database: {dbname: 'test'} )
|
66
|
+
end
|
67
|
+
|
13
68
|
end
|
14
69
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: symphony
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.0
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Granger
|
@@ -271,6 +271,7 @@ extra_rdoc_files:
|
|
271
271
|
- TODO.md
|
272
272
|
- USAGE.rdoc
|
273
273
|
files:
|
274
|
+
- .gemtest
|
274
275
|
- .simplecov
|
275
276
|
- ChangeLog
|
276
277
|
- History.rdoc
|
@@ -297,6 +298,7 @@ files:
|
|
297
298
|
- lib/symphony/tasks/ssh.rb
|
298
299
|
- lib/symphony/tasks/sshscript.rb
|
299
300
|
- spec/helpers.rb
|
301
|
+
- spec/symphony/daemon_spec.rb
|
300
302
|
- spec/symphony/mixins_spec.rb
|
301
303
|
- spec/symphony/queue_spec.rb
|
302
304
|
- spec/symphony/task_spec.rb
|
@@ -320,9 +322,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
320
322
|
version: 2.0.0
|
321
323
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
322
324
|
requirements:
|
323
|
-
- - '
|
325
|
+
- - '>='
|
324
326
|
- !ruby/object:Gem::Version
|
325
|
-
version:
|
327
|
+
version: 2.0.3
|
326
328
|
requirements: []
|
327
329
|
rubyforge_project: symphony
|
328
330
|
rubygems_version: 2.2.2
|
metadata.gz.sig
CHANGED
Binary file
|