symphony 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,10 @@ require 'symphony' unless defined?( Symphony )
8
8
  require 'symphony/mixins'
9
9
 
10
10
 
11
+ Bunny.extend( Loggability )
12
+ Bunny.log_as( :amqp )
13
+
14
+
11
15
  # An object class that encapsulates queueing logic for Symphony jobs.
12
16
  class Symphony::Queue
13
17
  extend Loggability,
@@ -66,7 +70,7 @@ class Symphony::Queue
66
70
  ### Fetch a Hash of AMQP options.
67
71
  def self::amqp_session_options
68
72
  opts = self.session_opts.merge({
69
- logger: Loggability[ Symphony ],
73
+ logger: Loggability[ Bunny ],
70
74
  })
71
75
  opts[:heartbeat] = opts[:heartbeat].to_sym if opts[:heartbeat].is_a?( String )
72
76
 
@@ -131,6 +135,7 @@ class Symphony::Queue
131
135
  ### Fetch the configured AMQP exchange interface object.
132
136
  def self::amqp_exchange
133
137
  unless self.amqp[:exchange]
138
+ self.log.info "Getting a reference to the %s topic exchange" % [ self.exchange ]
134
139
  self.amqp[:exchange] = self.amqp_channel.topic( self.exchange, passive: true )
135
140
  end
136
141
  return self.amqp[:exchange]
@@ -281,12 +286,14 @@ class Symphony::Queue
281
286
 
282
287
  # Re-raise errors from AMQP
283
288
  rescue Bunny::Exception => err
284
- self.log.error "%p while handling a message: %s" % [ err.class, err.message ]
289
+ self.log.error "%p while handling a message: %s %s" %
290
+ [ err.class, err.message, err.backtrace.first ]
285
291
  self.log.debug " " + err.backtrace.join( "\n " )
286
292
  raise
287
293
 
288
294
  rescue => err
289
- self.log.error "%p while handling a message: %s" % [ err.class, err.message ]
295
+ self.log.error "%p while handling a message: %s %s" %
296
+ [ err.class, err.message, err.backtrace.first ]
290
297
  self.log.debug " " + err.backtrace.join( "\n " )
291
298
  return self.ack_message( delivery_info.delivery_tag, false, false )
292
299
  end
@@ -313,7 +320,7 @@ class Symphony::Queue
313
320
  ### Close the AMQP session associated with this queue.
314
321
  def shutdown
315
322
  self.shutting_down = true
316
- self.consumer.cancel
323
+ self.consumer.cancel if self.consumer
317
324
  end
318
325
 
319
326
 
@@ -70,11 +70,12 @@ module Symphony::Routing
70
70
  ### Return a regular expression that will match messages matching the given
71
71
  ### +routing_key+.
72
72
  def make_routing_pattern( routing_key )
73
- re_string = routing_key.gsub( /\./, '\\.' )
74
- re_string = re_string.gsub( /\*/, '([^\.]*)' )
75
- re_string = re_string.gsub( /#/, '(.*)' )
73
+ re_string = Regexp.escape( routing_key ).
74
+ gsub( /\\\*/, '[^\.]*' ).
75
+ gsub( /\\.\\#/, '(\..*)?' ).
76
+ gsub( /\\#\\./, '(.*\.)?' )
76
77
 
77
- return Regexp.compile( re_string )
78
+ return Regexp.compile( '^' + re_string + '$' )
78
79
  end
79
80
 
80
81
  end # module ClassMethods
@@ -45,13 +45,12 @@ module Symphony::SignalHandling
45
45
  def wait_for_signals( timeout=nil )
46
46
 
47
47
  # Wait on the selfpipe for signals
48
- self.log.debug " waiting for the selfpipe"
48
+ # self.log.debug " waiting for the selfpipe"
49
49
  fds = IO.select( [@selfpipe[:reader]], [], [], timeout )
50
50
  begin
51
51
  rval = @selfpipe[:reader].read_nonblock( 11 )
52
52
  self.log.debug " read from the selfpipe: %p" % [ rval ]
53
53
  rescue Errno::EAGAIN, Errno::EINTR => err
54
- self.log.debug " %p: %s!" % [ err.class, err.message ]
55
54
  # ignore
56
55
  end
57
56
 
@@ -0,0 +1,96 @@
1
+ # -*- ruby -*-
2
+ #encoding: utf-8
3
+
4
+ require 'loggability'
5
+ require 'symphony' unless defined?( Symphony )
6
+
7
+
8
+ # A collection of statistics functions for various Symphony systems. Special
9
+ # thanks to Justin Z. Smith <justin@statisticool.com> for the maths. Good luck
10
+ # with your search for economic collapse in applesauce!
11
+ module Symphony::Statistics
12
+
13
+ # The default number of samples to keep
14
+ DEFAULT_SAMPLE_SIZE = 100
15
+
16
+
17
+ ### Set up some instance variables for tracking statistical info when the object
18
+ ### is created.
19
+ def initialize( * )
20
+ super
21
+ @samples = []
22
+ @sample_size = DEFAULT_SAMPLE_SIZE
23
+ end
24
+
25
+
26
+ # Samples of the number of pending jobs
27
+ attr_reader :samples
28
+
29
+ ##
30
+ # The number of samples to keep for analysis, and required
31
+ # before trending is performed.
32
+ attr_accessor :sample_size
33
+
34
+
35
+ ### Add the specified +value+ as a sample for the current time.
36
+ def add_sample( value )
37
+ @samples << [ Time.now.to_f, value ]
38
+ @samples.pop( @samples.size - self.sample_size ) if @samples.size > self.sample_size
39
+ end
40
+
41
+
42
+ ### Returns +true+ if the samples gathered so far indicate an upwards trend.
43
+ def sample_values_increasing?
44
+ return self.calculate_trend > 3
45
+ end
46
+
47
+
48
+ ### Returns +true+ if the samples gathered so far indicate a downwards trend.
49
+ def sample_values_decreasing?
50
+ return self.calculate_trend < -3
51
+ end
52
+
53
+
54
+ ### Predict the likelihood that an upward trend will continue based on linear regression
55
+ ### analysis of the given samples. If the value returned is >= 3.0, the values are
56
+ ### statistically trending upwards, which in Symphony's case means that the workers are
57
+ ### not handling the incoming work.
58
+ def calculate_trend
59
+ return 0 unless self.samples.size >= self.sample_size
60
+ # Loggability[ Symphony ].debug "%d samples of required %d" % [ self.samples.size, self.sample_size ]
61
+
62
+ x_vec, y_vec = self.samples.transpose
63
+
64
+ y_avg = y_vec.inject( :+ ).to_f / y_vec.size
65
+ x_avg = x_vec.inject( :+ ).to_f / x_vec.size
66
+
67
+ # Find slope and y-intercept.
68
+ #
69
+ n = d = 0
70
+ samples.each do |x, y_val|
71
+ xv = x - x_avg
72
+ n = n + ( xv * ( y_val - y_avg ) )
73
+ d = d + ( xv ** 2 )
74
+ end
75
+
76
+ slope = n/d
77
+ y_intercept = y_avg - ( slope * x_avg )
78
+
79
+ # Find stderr.
80
+ #
81
+ r = s = 0
82
+ samples.each do |x, y_val|
83
+ yv = ( slope * x ) + y_intercept
84
+ r = r + ( (y_val - yv) ** 2 )
85
+ s = s + ( (x - x_avg) ** 2 )
86
+ end
87
+
88
+ stde = Math.sqrt( (r / ( samples.size - 2 )) / s )
89
+
90
+ # Loggability[ Symphony ].debug " job sampling trend is: %f" % [ slope / stde ]
91
+ return slope / stde
92
+ end
93
+
94
+
95
+ end # module Symphony::Statistics
96
+
@@ -30,6 +30,14 @@ class Symphony::Task
30
30
  # Valid work model types
31
31
  WORK_MODELS = %i[ longlived oneshot ]
32
32
 
33
+ # The number of seconds between checks to see if the worker is idle. Note that
34
+ # this is not the same as the idle timeout -- it's how often to check to see if
35
+ # the task has been idle for too long.
36
+ IDLE_CHECK_INTERVAL = 5 # seconds
37
+
38
+ # The default number of seconds to wait for work
39
+ DEFAULT_IDLE_TIMEOUT = IDLE_CHECK_INTERVAL * 2
40
+
33
41
 
34
42
  # Loggability API -- log to symphony's logger
35
43
  log_to :symphony
@@ -41,20 +49,20 @@ class Symphony::Task
41
49
 
42
50
  ### Create a new Task object and listen for work. Exits with the code returned
43
51
  ### by #start when it's done.
44
- def self::run
52
+ def self::run( exit_on_idle=false )
45
53
  if self.subscribe_to.empty?
46
54
  raise ScriptError,
47
55
  "No subscriptions defined. Add one or more patterns using subscribe_to."
48
56
  end
49
57
 
50
- exit self.new( self.queue ).start
58
+ exit self.new( self.queue, exit_on_idle ).start
51
59
  end
52
60
 
53
61
 
54
62
  ### Prepare the process to be forked.
55
63
  def self::before_fork
56
64
  self.log.debug "Before fork [%d]: Threads: %p" % [ Process.pid, ThreadGroup::Default.list ]
57
- # No-op
65
+ Symphony::Queue.reset
58
66
  end
59
67
 
60
68
 
@@ -76,6 +84,7 @@ class Symphony::Task
76
84
  subclass.instance_variable_set( :@prefetch, 10 )
77
85
  subclass.instance_variable_set( :@timeout_action, :reject )
78
86
  subclass.instance_variable_set( :@persistent, false )
87
+ subclass.instance_variable_set( :@idle_timeout, DEFAULT_IDLE_TIMEOUT )
79
88
  end
80
89
 
81
90
 
@@ -209,16 +218,33 @@ class Symphony::Task
209
218
  end
210
219
 
211
220
 
221
+ ### Get/set the maximum number of seconds the worker should wait for events to
222
+ ### arrive before exiting.
223
+ def self::idle_timeout( seconds=nil, options={} )
224
+ unless seconds.nil?
225
+ self.log.info "Setting the idle timeout to %0.2fs." % [ seconds.to_f ]
226
+ @idle_timeout = seconds.to_f
227
+ end
228
+
229
+ return @idle_timeout
230
+ end
231
+
232
+
233
+
234
+
212
235
  #
213
236
  # Instance Methods
214
237
  #
215
238
 
216
239
  ### Create a worker that will listen on the specified +queue+ for a job.
217
- def initialize( queue )
240
+ def initialize( queue, exit_on_idle=false )
218
241
  @queue = queue
219
242
  @signal_handler = nil
220
243
  @shutting_down = false
221
244
  @restarting = false
245
+ @idle_checker = nil
246
+ @last_worked = Time.now
247
+ @exit_on_idle = exit_on_idle
222
248
  end
223
249
 
224
250
 
@@ -239,6 +265,15 @@ class Symphony::Task
239
265
  # Is the task in the process of restarting?
240
266
  attr_predicate_accessor :restarting
241
267
 
268
+ # The Thread that checks for idle timeout
269
+ attr_accessor :idle_checker
270
+
271
+ # The Time the task was last running
272
+ attr_accessor :last_worked
273
+
274
+ # Flag that determines whether this task instance exits when it doesn't have any work.
275
+ attr_predicate :exit_on_idle
276
+
242
277
 
243
278
  ### Set up the task and start handling messages.
244
279
  def start
@@ -254,7 +289,7 @@ class Symphony::Task
254
289
  return rval ? 0 : 1
255
290
 
256
291
  rescue Exception => err
257
- self.log.fatal "%p in %p: %s" % [ err.class, self.class, err.message ]
292
+ self.log.fatal "%p in %p: %s: %s" % [ err.class, self.class, err.message, err.backtrace.first ]
258
293
  self.log.debug { ' ' + err.backtrace.join(" \n") }
259
294
 
260
295
  return :software
@@ -299,27 +334,52 @@ class Symphony::Task
299
334
  def start_handling_messages
300
335
  oneshot = self.class.work_model == :oneshot
301
336
 
302
- return self.queue.wait_for_message( oneshot ) do |payload, metadata|
337
+ rval = nil
338
+ self.queue.wait_for_message( oneshot ) do |payload, metadata|
339
+ self.last_worked = nil
303
340
  work_payload = self.preprocess_payload( payload, metadata )
304
341
 
305
- if self.class.timeout
342
+ rval = if self.class.timeout
306
343
  self.work_with_timeout( work_payload, metadata )
307
344
  else
308
345
  self.work( work_payload, metadata )
309
346
  end
310
- end
347
+
348
+ self.last_worked = Time.now
349
+ end
350
+
351
+ return rval
311
352
  end
312
353
 
313
354
 
314
- ### Start the thread that will deliver signals once they're put on the queue.
355
+ ### Start the thread that will deliver signals once they're put on the queue, and
356
+ ### check for the last time a job was handled by this task process.
315
357
  def start_signal_handler
316
358
  @signal_handler = Thread.new do
317
359
  Thread.current.abort_on_exception = true
318
360
  loop do
319
- self.log.debug "Signal handler: waiting for new signals in the queue."
320
- self.wait_for_signals
361
+ # self.log.debug "Signal handler: waiting for new signals in the queue."
362
+ self.wait_for_signals( IDLE_CHECK_INTERVAL )
363
+ self.check_for_idle_timeout
321
364
  end
322
365
  end
366
+ rescue => err
367
+ self.log.fatal "Signal handler thread crashed: %p: %s" % [ err.class, err.message ]
368
+ self.stop_immediately
369
+ end
370
+
371
+
372
+ ### Check to see if the last run was more than #idle_timeout seconds ago,
373
+ ### and cancelling the task's consumer if so.
374
+ def check_for_idle_timeout
375
+
376
+ # If it's unset, it means it's running now
377
+ return unless self.last_worked && self.exit_on_idle?
378
+
379
+ if (Time.now - self.last_worked) > self.class.idle_timeout
380
+ self.log.debug "Sending stop signal due to idle timeout"
381
+ self.stop_gracefully
382
+ end
323
383
  end
324
384
 
325
385
 
@@ -0,0 +1,165 @@
1
+ # -*- ruby -*-
2
+ #encoding: utf-8
3
+
4
+ require 'set'
5
+ require 'pluggability'
6
+
7
+ require 'symphony' unless defined?( Symphony )
8
+
9
+ # A group for managing groups of tasks.
10
+ class Symphony::TaskGroup
11
+ extend Pluggability,
12
+ Loggability
13
+
14
+
15
+ # Log to the Symphony logger
16
+ log_to :symphony
17
+
18
+ # Pluggability API -- set the directory/directories that will be search when trying to
19
+ # load tasks by name.
20
+ plugin_prefixes 'symphony/task_group'
21
+
22
+
23
+ ### Set up a new task group.
24
+ def initialize( task_class, max_workers )
25
+ @task_class = task_class
26
+ @max_workers = max_workers
27
+ @workers = Set.new
28
+ @last_child_started = Time.now
29
+ @throttle = 0
30
+ @queue = nil
31
+ end
32
+
33
+
34
+ ######
35
+ public
36
+ ######
37
+
38
+ ##
39
+ # The set of worker PIDs
40
+ attr_reader :workers
41
+
42
+ ##
43
+ # The maximum number of workers the group will keep running
44
+ attr_accessor :max_workers
45
+
46
+ ##
47
+ # The Class of the task the worker runs
48
+ attr_reader :task_class
49
+
50
+ ##
51
+ # The Time that the last child started
52
+ attr_reader :last_child_started
53
+
54
+
55
+ ### Start a new Symphony::Task and return its PID.
56
+ def start_worker( exit_on_idle=false )
57
+ self.log.debug "Starting a %p." % [ task_class ]
58
+ task_class.before_fork
59
+
60
+ pid = Process.fork do
61
+ task_class.after_fork
62
+ task_class.run( exit_on_idle )
63
+ end or raise "No PID from forked %p worker?" % [ task_class ]
64
+
65
+ Process.setpgid( pid, 0 )
66
+
67
+ self.log.info "Adding worker %p" % [ pid ]
68
+ self.workers << pid
69
+ @last_child_started = Time.now
70
+
71
+ return pid
72
+ end
73
+
74
+
75
+ ### Stop a worker from the task group.
76
+ def stop_worker
77
+ pid = self.workers.first
78
+ self.signal_processes( :TERM, pid )
79
+ end
80
+
81
+
82
+ ### Stop all of the task group's workers.
83
+ def stop_all_workers
84
+ self.signal_processes( :TERM, *self.workers )
85
+ end
86
+
87
+
88
+ ### Send a SIGHUP to all the group's workers.
89
+ def restart_workers
90
+ self.signal_processes( :HUP, *self.workers )
91
+ end
92
+
93
+
94
+ ### Scale workers up or down based on the task group's work model.
95
+ ### This method needs to return an array of pids that were started, otherwise
96
+ ### nil.
97
+ def adjust_workers
98
+ raise NotImplementedError, "%p needs to provide an implementation of #adjust_workers" % [
99
+ self.class
100
+ ]
101
+ end
102
+
103
+
104
+ ### Handle the exit of the child with the specified +pid+. The +status+ is the
105
+ ### Process::Status returned by waitpid.
106
+ def on_child_exit( pid, status )
107
+ self.workers.delete( pid )
108
+ self.adjust_throttle( status.success? ? -1 : 1 )
109
+ end
110
+
111
+
112
+ ### Returns +true+ if the group of tasks is throttled (i.e., should wait to start any more
113
+ ### children).
114
+ def throttled?
115
+ # Return unless the throttle period has lapsed
116
+ unless self.throttle_seconds < (Time.now - self.last_child_started)
117
+ self.log.warn "Not starting children: throttled for %0.2f seconds" %
118
+ [ self.throttle_seconds ]
119
+ return true
120
+ end
121
+
122
+ return false
123
+ end
124
+
125
+
126
+ ### Return the number of seconds between child startup times.
127
+ def throttle_seconds
128
+ return 0 unless @throttle.nonzero?
129
+ return Math.log( @throttle ) * Symphony.throttle_factor
130
+ end
131
+
132
+
133
+ ### Add +adjustment+ to the throttle value, ensuring that it doesn't go
134
+ ### below zero.
135
+ def adjust_throttle( adjustment=1 )
136
+ self.log.debug "Adjusting worker throttle by %d" % [ adjustment ]
137
+ @throttle += adjustment
138
+ @throttle = 0 if @throttle < 0
139
+ @throttle = Symphony.throttle_max if @throttle > Symphony.throttle_max
140
+ end
141
+
142
+
143
+ #########
144
+ protected
145
+ #########
146
+
147
+ ### Send the specified +signal+ to the process associated with +pid+, handling
148
+ ### harmless errors.
149
+ def signal_processes( signal, *pids )
150
+ self.log.debug "Signalling processes: %p" % [ pids ]
151
+
152
+ # Do them one at a time, as Process.kill will abort on the first error if you
153
+ # pass more than one pid.
154
+ pids.each do |pid|
155
+ begin
156
+ Process.kill( signal, pid )
157
+ rescue Errno::ESRCH => err
158
+ self.log.error "%p when trying to %s worker %d: %s" %
159
+ [ err.class, signal, pid, err.message ]
160
+ end
161
+ end
162
+ end
163
+
164
+ end # class Symphony::TaskGroup
165
+