symphony 0.8.0 → 0.9.0

data/lib/symphony/queue.rb

@@ -8,6 +8,10 @@ require 'symphony' unless defined?( Symphony )
 require 'symphony/mixins'
 
 
+Bunny.extend( Loggability )
+Bunny.log_as( :amqp )
+
+
 # An object class that encapsulates queueing logic for Symphony jobs.
 class Symphony::Queue
 	extend Loggability,
@@ -66,7 +70,7 @@ class Symphony::Queue
 	### Fetch a Hash of AMQP options.
 	def self::amqp_session_options
 		opts = self.session_opts.merge({
-			logger: Loggability[ Symphony ],
+			logger: Loggability[ Bunny ],
 		})
 		opts[:heartbeat] = opts[:heartbeat].to_sym if opts[:heartbeat].is_a?( String )
 
@@ -131,6 +135,7 @@ class Symphony::Queue
 	### Fetch the configured AMQP exchange interface object.
 	def self::amqp_exchange
 		unless self.amqp[:exchange]
+			self.log.info "Getting a reference to the %s topic exchange" % [ self.exchange ]
 			self.amqp[:exchange] = self.amqp_channel.topic( self.exchange, passive: true )
 		end
 		return self.amqp[:exchange]
@@ -281,12 +286,14 @@ class Symphony::Queue
 
 	# Re-raise errors from AMQP
 	rescue Bunny::Exception => err
-		self.log.error "%p while handling a message: %s" % [ err.class, err.message ]
+		self.log.error "%p while handling a message: %s %s" %
+			[ err.class, err.message, err.backtrace.first ]
 		self.log.debug "  " + err.backtrace.join( "\n  " )
 		raise
 
 	rescue => err
-		self.log.error "%p while handling a message: %s" % [ err.class, err.message ]
+		self.log.error "%p while handling a message: %s %s" %
+			[ err.class, err.message, err.backtrace.first ]
 		self.log.debug "  " + err.backtrace.join( "\n  " )
 		return self.ack_message( delivery_info.delivery_tag, false, false )
 	end
@@ -313,7 +320,7 @@ class Symphony::Queue
 	### Close the AMQP session associated with this queue.
 	def shutdown
 		self.shutting_down = true
-		self.consumer.cancel
+		self.consumer.cancel if self.consumer
 	end
 
 

data/lib/symphony/routing.rb

@@ -70,11 +70,12 @@ module Symphony::Routing
 		### Return a regular expression that will match messages matching the given
 		### +routing_key+.
 		def make_routing_pattern( routing_key )
-			re_string = routing_key.gsub( /\./, '\\.' )
-			re_string = re_string.gsub( /\*/, '([^\.]*)' )
-			re_string = re_string.gsub( /#/, '(.*)' )
+			re_string = Regexp.escape( routing_key ).
+				gsub( /\\\*/, '[^\.]*' ).
+				gsub( /\\.\\#/, '(\..*)?' ).
+				gsub( /\\#\\./, '(.*\.)?' )
 
-			return Regexp.compile( re_string )
+			return Regexp.compile( '^' + re_string + '$' )
 		end
 
 	end # module ClassMethods

data/lib/symphony/signal_handling.rb

@@ -45,13 +45,12 @@ module Symphony::SignalHandling
 	def wait_for_signals( timeout=nil )
 
 		# Wait on the selfpipe for signals
-		self.log.debug "  waiting for the selfpipe"
+		# self.log.debug "  waiting for the selfpipe"
 		fds = IO.select( [@selfpipe[:reader]], [], [], timeout )
 		begin
 			rval = @selfpipe[:reader].read_nonblock( 11 )
 			self.log.debug "    read from the selfpipe: %p" % [ rval ]
 		rescue Errno::EAGAIN, Errno::EINTR => err
-			self.log.debug "    %p: %s!" % [ err.class, err.message ]
 			# ignore
 		end
 

data/lib/symphony/statistics.rb

@@ -0,0 +1,96 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'loggability'
+require 'symphony' unless defined?( Symphony )
+
+
+# A collection of statistics functions for various Symphony systems. Special
+# thanks to Justin Z. Smith <justin@statisticool.com> for the maths. Good luck
+# with your search for economic collapse in applesauce!
+module Symphony::Statistics
+
+	# The default number of samples to keep
+	DEFAULT_SAMPLE_SIZE = 100
+
+
+	### Set up some instance variables for tracking statistical info when the object
+	### is created.
+	def initialize( * )
+		super
+		@samples = []
+		@sample_size = DEFAULT_SAMPLE_SIZE
+	end
+
+
+	# Samples of the number of pending jobs
+	attr_reader :samples
+
+	##
+	# The number of samples to keep for analysis, and required
+	# before trending is performed.
+	attr_accessor :sample_size
+
+
+	### Add the specified +value+ as a sample for the current time.
+	def add_sample( value )
+		@samples << [ Time.now.to_f, value ]
+		@samples.pop( @samples.size - self.sample_size ) if @samples.size > self.sample_size
+	end
+
+
+	### Returns +true+ if the samples gathered so far indicate an upwards trend.
+	def sample_values_increasing?
+		return self.calculate_trend > 3
+	end
+
+
+	### Returns +true+ if the samples gathered so far indicate a downwards trend.
+	def sample_values_decreasing?
+		return self.calculate_trend < -3
+	end
+
+
+	### Predict the likelihood that an upward trend will continue based on linear regression
+	### analysis of the given samples. If the value returned is >= 3.0, the values are
+	### statistically trending upwards, which in Symphony's case means that the workers are
+	### not handling the incoming work.
+	def calculate_trend
+		return 0 unless self.samples.size >= self.sample_size
+		# Loggability[ Symphony ].debug "%d samples of required %d" % [ self.samples.size, self.sample_size ]
+
+		x_vec, y_vec = self.samples.transpose
+
+		y_avg = y_vec.inject( :+ ).to_f / y_vec.size
+		x_avg = x_vec.inject( :+ ).to_f / x_vec.size
+
+		# Find slope and y-intercept.
+		#
+		n = d = 0
+		samples.each do |x, y_val|
+			xv = x - x_avg
+			n  = n + ( xv * ( y_val - y_avg ) )
+			d  = d + ( xv ** 2 )
+		end
+
+		slope = n/d
+		y_intercept = y_avg - ( slope * x_avg )
+
+		# Find stderr.
+		#
+		r = s = 0
+		samples.each do |x, y_val|
+			yv = ( slope * x ) + y_intercept
+			r  = r + ( (y_val - yv) ** 2 )
+			s  = s + ( (x - x_avg) ** 2 )
+		end
+
+		stde = Math.sqrt( (r / ( samples.size - 2 )) / s )
+
+		# Loggability[ Symphony ].debug "  job sampling trend is: %f" %  [ slope / stde ]
+		return slope / stde
+	end
+
+
+end # module Symphony::Statistics
+

data/lib/symphony/task.rb

@@ -30,6 +30,14 @@ class Symphony::Task
 	# Valid work model types
 	WORK_MODELS = %i[ longlived oneshot ]
 
+	# The number of seconds between checks to see if the worker is idle. Note that
+	# this is not the same as the idle timeout -- it's how often to check to see if
+	# the task has been idle for too long.
+	IDLE_CHECK_INTERVAL = 5 # seconds
+
+	# The default number of seconds to wait for work
+	DEFAULT_IDLE_TIMEOUT = IDLE_CHECK_INTERVAL * 2
+
 
 	# Loggability API -- log to symphony's logger
 	log_to :symphony
@@ -41,20 +49,20 @@ class Symphony::Task
 
 	### Create a new Task object and listen for work. Exits with the code returned
 	### by #start when it's done.
-	def self::run
+	def self::run( exit_on_idle=false )
 		if self.subscribe_to.empty?
 			raise ScriptError,
 				"No subscriptions defined. Add one or more patterns using subscribe_to."
 		end
 
-		exit self.new( self.queue ).start
+		exit self.new( self.queue, exit_on_idle ).start
 	end
 
 
 	### Prepare the process to be forked.
 	def self::before_fork
 		self.log.debug "Before fork [%d]: Threads: %p" % [ Process.pid, ThreadGroup::Default.list ]
-		# No-op
+		Symphony::Queue.reset
 	end
 
 
@@ -76,6 +84,7 @@ class Symphony::Task
 		subclass.instance_variable_set( :@prefetch, 10 )
 		subclass.instance_variable_set( :@timeout_action, :reject )
 		subclass.instance_variable_set( :@persistent, false )
+		subclass.instance_variable_set( :@idle_timeout, DEFAULT_IDLE_TIMEOUT )
 	end
 
 
@@ -209,16 +218,33 @@ class Symphony::Task
 	end
 
 
+	### Get/set the maximum number of seconds the worker should wait for events to
+	### arrive before exiting.
+	def self::idle_timeout( seconds=nil, options={} )
+		unless seconds.nil?
+			self.log.info "Setting the idle timeout to %0.2fs." % [ seconds.to_f ]
+			@idle_timeout = seconds.to_f
+		end
+
+		return @idle_timeout
+	end
+
+
+
+
 	#
 	# Instance Methods
 	#
 
 	### Create a worker that will listen on the specified +queue+ for a job.
-	def initialize( queue )
+	def initialize( queue, exit_on_idle=false )
 		@queue          = queue
 		@signal_handler = nil
 		@shutting_down  = false
 		@restarting     = false
+		@idle_checker   = nil
+		@last_worked    = Time.now
+		@exit_on_idle   = exit_on_idle
 	end
 
 
@@ -239,6 +265,15 @@ class Symphony::Task
 	# Is the task in the process of restarting?
 	attr_predicate_accessor :restarting
 
+	# The Thread that checks for idle timeout
+	attr_accessor :idle_checker
+
+	# The Time the task was last running
+	attr_accessor :last_worked
+
+	# Flag that determines whether this task instance exits when it doesn't have any work.
+	attr_predicate :exit_on_idle
+
 
 	### Set up the task and start handling messages.
 	def start
@@ -254,7 +289,7 @@ class Symphony::Task
 		return rval ? 0 : 1
 
 	rescue Exception => err
-		self.log.fatal "%p in %p: %s" % [ err.class, self.class, err.message ]
+		self.log.fatal "%p in %p: %s: %s" % [ err.class, self.class, err.message, err.backtrace.first ]
 		self.log.debug { '  ' + err.backtrace.join("  \n") }
 
 		return :software
@@ -299,27 +334,52 @@ class Symphony::Task
 	def start_handling_messages
 		oneshot = self.class.work_model == :oneshot
 
-		return self.queue.wait_for_message( oneshot ) do |payload, metadata|
+		rval = nil
+		self.queue.wait_for_message( oneshot ) do |payload, metadata|
+			self.last_worked = nil
 			work_payload = self.preprocess_payload( payload, metadata )
 
-			if self.class.timeout
+			rval = if self.class.timeout
 				self.work_with_timeout( work_payload, metadata )
 			else
 				self.work( work_payload, metadata )
 			end
-		end
+
+			self.last_worked = Time.now
+	end
+
+		return rval
 	end
 
 
-	### Start the thread that will deliver signals once they're put on the queue.
+	### Start the thread that will deliver signals once they're put on the queue, and
+	### check for the last time a job was handled by this task process.
 	def start_signal_handler
 		@signal_handler = Thread.new do
 			Thread.current.abort_on_exception = true
 			loop do
-				self.log.debug "Signal handler: waiting for new signals in the queue."
-				self.wait_for_signals
+				# self.log.debug "Signal handler: waiting for new signals in the queue."
+				self.wait_for_signals( IDLE_CHECK_INTERVAL )
+				self.check_for_idle_timeout
 			end
 		end
+	rescue => err
+		self.log.fatal "Signal handler thread crashed: %p: %s" % [ err.class, err.message ]
+		self.stop_immediately
+	end
+
+
+	### Check to see if the last run was more than #idle_timeout seconds ago,
+	### and cancelling the task's consumer if so.
+	def check_for_idle_timeout
+
+		# If it's unset, it means it's running now
+		return unless self.last_worked && self.exit_on_idle?
+
+		if (Time.now - self.last_worked) > self.class.idle_timeout
+			self.log.debug "Sending stop signal due to idle timeout"
+			self.stop_gracefully
+		end
 	end
 
 

data/lib/symphony/task_group.rb

@@ -0,0 +1,165 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'set'
+require 'pluggability'
+
+require 'symphony' unless defined?( Symphony )
+
+# A group for managing groups of tasks.
+class Symphony::TaskGroup
+	extend Pluggability,
+	       Loggability
+
+
+	# Log to the Symphony logger
+	log_to :symphony
+
+	# Pluggability API -- set the directory/directories that will be search when trying to
+	# load tasks by name.
+	plugin_prefixes 'symphony/task_group'
+
+
+	### Set up a new task group.
+	def initialize( task_class, max_workers )
+		@task_class         = task_class
+		@max_workers        = max_workers
+		@workers            = Set.new
+		@last_child_started = Time.now
+		@throttle           = 0
+		@queue              = nil
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The set of worker PIDs
+	attr_reader :workers
+
+	##
+	# The maximum number of workers the group will keep running
+	attr_accessor :max_workers
+
+	##
+	# The Class of the task the worker runs
+	attr_reader :task_class
+
+	##
+	# The Time that the last child started
+	attr_reader :last_child_started
+
+
+	### Start a new Symphony::Task and return its PID.
+	def start_worker( exit_on_idle=false )
+		self.log.debug "Starting a %p." % [ task_class ]
+		task_class.before_fork
+
+		pid = Process.fork do
+			task_class.after_fork
+			task_class.run( exit_on_idle )
+		end or raise "No PID from forked %p worker?" % [ task_class ]
+
+		Process.setpgid( pid, 0 )
+
+		self.log.info "Adding worker %p" % [ pid ]
+		self.workers << pid
+		@last_child_started = Time.now
+
+		return pid
+	end
+
+
+	### Stop a worker from the task group.
+	def stop_worker
+		pid = self.workers.first
+		self.signal_processes( :TERM, pid )
+	end
+
+
+	### Stop all of the task group's workers.
+	def stop_all_workers
+		self.signal_processes( :TERM, *self.workers )
+	end
+
+
+	### Send a SIGHUP to all the group's workers.
+	def restart_workers
+		self.signal_processes( :HUP, *self.workers )
+	end
+
+
+	### Scale workers up or down based on the task group's work model.
+	### This method needs to return an array of pids that were started, otherwise
+	### nil.
+	def adjust_workers
+		raise NotImplementedError, "%p needs to provide an implementation of #adjust_workers" % [
+			self.class
+		]
+	end
+
+
+	### Handle the exit of the child with the specified +pid+. The +status+ is the
+	### Process::Status returned by waitpid.
+	def on_child_exit( pid, status )
+		self.workers.delete( pid )
+		self.adjust_throttle( status.success? ? -1 : 1 )
+	end
+
+
+	### Returns +true+ if the group of tasks is throttled (i.e., should wait to start any more
+	### children).
+	def throttled?
+		# Return unless the throttle period has lapsed
+		unless self.throttle_seconds < (Time.now - self.last_child_started)
+			self.log.warn "Not starting children: throttled for %0.2f seconds" %
+				[ self.throttle_seconds ]
+			return true
+		end
+
+		return false
+	end
+
+
+	### Return the number of seconds between child startup times.
+	def throttle_seconds
+		return 0 unless @throttle.nonzero?
+		return Math.log( @throttle ) * Symphony.throttle_factor
+	end
+
+
+	### Add +adjustment+ to the throttle value, ensuring that it doesn't go
+	### below zero.
+	def adjust_throttle( adjustment=1 )
+		self.log.debug "Adjusting worker throttle by %d" % [ adjustment ]
+		@throttle += adjustment
+		@throttle = 0 if @throttle < 0
+		@throttle = Symphony.throttle_max if @throttle > Symphony.throttle_max
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Send the specified +signal+ to the process associated with +pid+, handling
+	### harmless errors.
+	def signal_processes( signal, *pids )
+		self.log.debug "Signalling processes: %p" % [ pids ]
+
+		# Do them one at a time, as Process.kill will abort on the first error if you
+		# pass more than one pid.
+		pids.each do |pid|
+			begin
+				Process.kill( signal, pid )
+			rescue Errno::ESRCH => err
+				self.log.error "%p when trying to %s worker %d: %s" %
+					[ err.class, signal, pid, err.message ]
+			end
+		end
+	end
+
+end # class Symphony::TaskGroup
+