symphony 0.11.1 → 0.12.0

data/History.rdoc

@@ -1,3 +1,17 @@
+== v0.12.0 [2019-06-26] Michael Granger <ged@FaerieMUD.org>
+
+Enhancements:
+
+- Set a proctitle for all tasks, not just those that include the
+  Metrics mixin.
+
+Bugfixes:
+
+- Fix a bunch of bugs with scaling, idle timeout, and message-
+  counting.
+- Clean up unused @pids variable in longlived task group
+
+
 == v0.11.1 [2017-01-31] Mahlon E. Smith <mahlon@martini.nu>
 
 Housekeeping:

data/lib/symphony.rb

@@ -12,10 +12,10 @@ module Symphony
 	       Configurability
 
 	# Library version constant
-	VERSION = '0.11.1'
+	VERSION = '0.12.0'
 
 	# Version-control revision constant
-	REVISION = %q$Revision: 5347e2bfccb1 $
+	REVISION = %q$Revision$
 
 
 	# The name of the environment variable to check for config file overrides

data/lib/symphony/daemon.rb

@@ -69,7 +69,7 @@ class Symphony::Daemon
 	def initialize
 		@task_pids   = {}
 		@task_groups = {}
-		@running             = false
+		@running     = false
 
 		self.set_up_signal_handling
 	end
@@ -233,7 +233,7 @@ class Symphony::Daemon
 			self.log.info "%p no longer configured; stopping its task group." % [ task_class ]
 			self.stop_task_group( group )
 		end
-		end
+	end
 
 
 	### Start a new task group for the given +task_class+ and +max+ number of workers.
@@ -264,7 +264,7 @@ class Symphony::Daemon
 			new_pids = group.adjust_workers or next
 			new_pids.each do |pid|
 				self.task_pids[ pid ] = group
-	end
+			end
 		end
 	end
 
@@ -341,7 +341,9 @@ class Symphony::Daemon
 	### Notify the task group the specified +pid+ belongs to that its child exited
 	### with the specified +status+.
 	def notify_group( pid, status )
+		self.log.debug "Notifying group of reaped child %d: %p" % [ pid, status ]
 		return unless self.running?
+
 		group = self.task_pids[ pid ]
 		group.on_child_exit( pid, status )
 	end

data/lib/symphony/metrics.rb

@@ -31,7 +31,7 @@ module Symphony::Metrics
 			registry: @metriks_registry,
 			prefix: self.class.name )
 		@proc_reporter = Metriks::Reporter::ProcTitle.new(
-			prefix: self.class.name,
+			prefix: self.procname,
 			registry: @metriks_registry,
 			on_error: lambda {|ex| self.log.error(ex) } )
 

data/lib/symphony/queue.rb

@@ -235,7 +235,7 @@ class Symphony::Queue
 		cons = Bunny::Consumer.new( amqp_queue.channel, amqp_queue, tag, !ackmode, false, CONSUMER_ARGS )
 
 		cons.on_delivery do |delivery_info, properties, payload|
-			rval = self.handle_message( delivery_info, properties, payload, &work_callback )
+			self.handle_message( delivery_info, properties, payload, &work_callback )
 			self.log.debug "Done with message %s. Session is %s" %
 					[ delivery_info.delivery_tag, self.class.amqp_session.closed? ? "closed" : "open" ]
 			cons.cancel if self.shutting_down?
@@ -278,8 +278,8 @@ class Symphony::Queue
 			end
 		end
 
-			return queue
-		end
+		return queue
+	end
 
 
 	### Handle each subscribed message.

data/lib/symphony/statistics.rb

@@ -36,7 +36,7 @@ module Symphony::Statistics
 	### Add the specified +value+ as a sample for the current time.
 	def add_sample( value )
 		@samples << [ Time.now.to_f, value ]
-		@samples.pop( @samples.size - self.sample_size ) if @samples.size > self.sample_size
+		@samples.shift( @samples.size - self.sample_size ) if @samples.size > self.sample_size
 		@counter = ( @counter + 1 ) % 3
 	end
 

data/lib/symphony/task.rb

@@ -292,6 +292,8 @@ class Symphony::Task
 	def start
 		rval = nil
 
+		Process.setproctitle( self.procname )
+
 		begin
 			self.restarting = false
 			rval = self.with_signal_handler( *SIGNALS ) do
@@ -349,16 +351,19 @@ class Symphony::Task
 
 		rval = nil
 		self.queue.wait_for_message( oneshot ) do |payload, metadata|
-			self.last_worked = nil
-			work_payload = self.preprocess_payload( payload, metadata )
-
-			rval = if self.class.timeout
-				self.work_with_timeout( work_payload, metadata )
-			else
-				self.work( work_payload, metadata )
+			begin
+				self.last_worked = nil
+				work_payload = self.preprocess_payload( payload, metadata )
+
+				rval = if self.class.timeout
+					self.work_with_timeout( work_payload, metadata )
+				else
+					self.work( work_payload, metadata )
+				end
+			ensure
+				self.last_worked = Time.now
 			end
 
-			self.last_worked = Time.now
 			rval
 		end
 
@@ -390,7 +395,10 @@ class Symphony::Task
 		# If it's unset, it means it's running now
 		return unless self.last_worked && self.exit_on_idle?
 
-		if (Time.now - self.last_worked) > self.class.idle_timeout
+		seconds_idle = Time.now - self.last_worked
+		self.log.debug "%p: idle %0.2fs" % [ self.class, seconds_idle ]
+
+		if seconds_idle > self.class.idle_timeout
 			self.log.debug "Sending stop signal due to idle timeout"
 			self.stop_gracefully
 		end
@@ -470,6 +478,18 @@ class Symphony::Task
 	end
 
 
+	### Return a string for setting the proc title
+	def procname
+		return "%s %s: Symphony: %p (%s) -> %s" % [
+			RUBY_ENGINE,
+			RUBY_VERSION,
+			self.class,
+			self.class.work_model,
+			self.class.queue_name
+		]
+	end
+
+
 	### Handle a hangup signal by re-reading the config and restarting.
 	def on_hangup
 		self.log.info "Hangup signal."

data/lib/symphony/task_group.rb

@@ -64,7 +64,7 @@ class Symphony::TaskGroup
 
 		Process.setpgid( pid, 0 )
 
-		self.log.info "Adding worker %p" % [ pid ]
+		self.log.info "Adding %p worker %p" % [ task_class, pid ]
 		self.workers << pid
 		@last_child_started = Time.now
 

data/lib/symphony/task_group/longlived.rb

@@ -15,10 +15,7 @@ class Symphony::TaskGroup::LongLived < Symphony::TaskGroup
 	### run a maximum of +max_workers+.
 	def initialize( task_class, max_workers )
 		super
-
-		@queue              = nil
-		@pids               = Set.new
-		@started_one_worker = false
+		@queue = nil
 	end
 
 
@@ -26,33 +23,6 @@ class Symphony::TaskGroup::LongLived < Symphony::TaskGroup
 	public
 	######
 
-	# The PIDs of the child this task group manages
-	attr_reader :pids
-
-
-	### Return +true+ if the task group should scale up by one.
-	def needs_a_worker?
-		return true unless self.started_one_worker?
-		return false unless @queue
-		if ( cc = @queue.consumer_count ) >= self.max_workers
-			self.log.debug "Already at max workers (%d)" % [ self.max_workers ]
-			return false
-		else
-			self.log.debug "Not yet at max workers (have %d)" % [ cc ]
-		end
-		self.log.debug "Mean jobcount is %0.2f" % [ self.mean_jobcount ]
-		return self.mean_jobcount > 1 && !self.sample_values_decreasing?
-	end
-
-
-	### Returns +true+ if the group has started at least one worker. Used to avoid
-	### racing to start workers when one worker has started, but we haven't yet connected
-	### to AMQP to get consumer count yet.
-	def started_one_worker?
-		return @started_one_worker
-	end
-
-
 	### If the number of workers is not at the maximum, start some.
 	def adjust_workers
 		self.sample_queue_status
@@ -61,44 +31,68 @@ class Symphony::TaskGroup::LongLived < Symphony::TaskGroup
 
 		if self.needs_a_worker?
 			self.log.info "Too few workers for (%s); spinning one up." % [ self.task_class.name ]
-			pid = self.start_worker( @started_one_worker )
-			self.pids.add( pid )
+			pid = self.start_worker( !self.workers.empty? )
 			return [ pid ]
 		end
 
+		@queue ||= self.get_message_counting_queue
+
 		return nil
 	end
 
 
+	### Return +true+ if the task group should scale up by one.
+	def needs_a_worker?
+		return true if self.workers.empty?
+		return false unless @queue
+
+
+		# Calculate the number of workers across the whole broker
+		if ( cc = @queue.consumer_count ) >= self.max_workers
+			self.log.debug "%p: Already at max workers (%d)" % [ self.task_class, self.max_workers ]
+			return false
+		else
+			self.log.debug "%p: Not yet at max workers (have %d)" % [ self.task_class, cc ]
+		end
+
+		self.log.debug "Mean jobcount is %0.2f" % [ self.mean_jobcount ]
+		return self.mean_jobcount > 1 && !self.sample_values_decreasing?
+	end
+
+
 	### Add the current number of workers to the samples.
 	def sample_queue_status
 		return unless @queue
-		self.add_sample( @queue.message_count )
+
+		count = @queue.message_count
+		self.add_sample( count )
 	end
 
 
 	### Overridden to grab a Bunny::Queue for monitoring when the first
 	### worker starts.
 	def start_worker( exit_on_idle=false )
-		@started_one_worker = true
-
 		pid = super
 		self.log.info "Start a new worker at pid %d" % [ pid ]
 
-		unless @queue
-			begin
-				channel = Symphony::Queue.amqp_channel
-				@queue = channel.queue( self.task_class.queue_name, passive: true, prefetch: 0 )
-				self.log.debug "  got the 0-prefetch queue"
-			rescue Bunny::NotFound => err
-				self.log.info "Child hasn't created the queue yet; deferring"
-				Symphony::Queue.reset
-			end
-		end
-
 		return pid
 	end
 
+
+	### Get a queue for counting the number of messages in the queue for this
+	### worker.
+	def get_message_counting_queue
+		channel = Symphony::Queue.amqp_channel
+		queue = channel.queue( self.task_class.queue_name, passive: true, prefetch: 0 )
+
+		return queue
+	rescue Bunny::NotFound => err
+		self.log.info "Child hasn't created the queue yet; deferring"
+		Symphony::Queue.reset
+
+		return nil
+	end
+
 end # class Symphony::TaskGroup::LongLived
 
 

data/lib/symphony/tasks/auditor.rb

@@ -17,7 +17,7 @@ class Auditor < Symphony::Task
 
 
 	### Create a new Auditor task.
-	def initialize( queue )
+	def initialize( * )
 		super
 		@logdir = Pathname.pwd
 		@logfile = @logdir + 'events.log'

data/lib/symphony/tasks/oneshot_simulator.rb

@@ -40,13 +40,13 @@ class OneshotSimulator < Symphony::Task
 
 		val = Random.rand
 		case
-		when val < 0.33
+		when val < 0.1
 			$stderr.puts "Simulating an error in the task (reject)."
 			raise "OOOOOPS!"
-		when val < 0.66
+		when val < 0.15
 			$stderr.puts "Simulating a soft failure in the task (reject+requeue)."
 			return false
-		when val < 0.88
+		when val < 0.20
 			$stderr.puts "Simulating a timeout case"
 			sleep( self.class.timeout + 1 )
 		else

data/lib/symphony/tasks/simulator.rb

@@ -15,6 +15,9 @@ class Simulator < Symphony::Task
 	# Fetch 100 events at a time
 	prefetch 10
 
+	# Keep the queue around even when the task isn't running
+	persistent true
+
 	# Only allow 2 seconds for work to complete before rejecting or retrying.
 	# timeout 2.0, action: :retry
 
@@ -36,22 +39,22 @@ class Simulator < Symphony::Task
 
 		sleep rand( 0.0 .. 2.0 )
 
-		# val = Random.rand
-		# case
-		# when val < 0.33
-		# 	$stderr.puts "Simulating an error in the task (reject)."
-		# 	raise "OOOOOPS!"
-		# when val < 0.66
-		# 	$stderr.puts "Simulating a soft failure in the task (reject+requeue)."
-		# 	return false
-		# when val < 0.88
-		# 	$stderr.puts "Simulating a timeout case"
-		# 	sleep( self.class.timeout + 1 )
-		# else
-		# 	$stderr.puts "Simulating a successful task run (accept)"
-		# 	puts( payload.inspect )
-		# 	return true
-		# end
+		val = Random.rand
+		case
+		when val < 0.05
+			$stderr.puts "Simulating an error in the task (reject)."
+			raise "OOOOOPS!"
+		when val < 0.10
+			$stderr.puts "Simulating a soft failure in the task (reject+requeue)."
+			return false
+		when val < 0.15
+			$stderr.puts "Simulating a timeout case"
+			sleep( self.class.timeout + 1 ) if self.class.timeout
+		else
+			$stderr.puts "Simulating a successful task run (accept)"
+			puts( payload.inspect )
+			return true
+		end
 
 		true
 	end

data/spec/symphony/daemon_spec.rb

@@ -90,7 +90,7 @@ describe Symphony::Daemon do
 	it "adjusts its tasks when its config is reloaded" do
 		config = Configurability.default_config
 		config.symphony.tasks = [ 'test1', 'test2' ]
-		# config.logging.__default__ = 'debug'
+		config.logging.__default__ = 'fatal'
 		config.install
 
 		allow( Symphony::Task ).to receive( :exit )

data/spec/symphony/task_group/longlived_spec.rb

@@ -6,6 +6,8 @@ require 'symphony/task_group/longlived'
 
 describe Symphony::TaskGroup::LongLived do
 
+	FIRST_PID = 414
+
 	let( :task ) do
 		Class.new( Symphony::Task ) do
 			extend Symphony::MethodUtilities
@@ -30,7 +32,7 @@ describe Symphony::TaskGroup::LongLived do
 
 	let( :pid_generator ) do
 		Enumerator.new do |generator|
-			i = 414
+			i = FIRST_PID
 			loop do
 				generator.yield( i )
 				i += rand( 3 ) + 1
@@ -38,9 +40,6 @@ describe Symphony::TaskGroup::LongLived do
 		end
 	end
 
-	# not enough samples
-	# trending up
-
 
 
 	it "doesn't start anything if it's throttled" do
@@ -61,20 +60,12 @@ describe Symphony::TaskGroup::LongLived do
 
 
 		it "starts an initial worker if it doesn't have any" do
-			allow( Process ).to receive( :setpgid ).with( 414, 0 )
-
-			channel = double( Bunny::Channel )
-			queue = double( Bunny::Queue )
-			expect( Symphony::Queue ).to receive( :amqp_channel ).
-				and_return( channel )
-			expect( channel ).to receive( :queue ).
-				with( task.queue_name, passive: true, prefetch: 0 ).
-				and_return( queue )
+			allow( Process ).to receive( :setpgid ).with( FIRST_PID, 0 )
 
 			task_group.adjust_workers
 
-			expect( task_group.started_one_worker? ).to be_truthy
-			expect( task_group.pids ).to include( 414 )
+			expect( task_group.workers ).to_not be_empty
+			expect( task_group.workers ).to contain_exactly( FIRST_PID )
 		end
 
 
@@ -93,20 +84,19 @@ describe Symphony::TaskGroup::LongLived do
 				and_return( queue )
 
 			expect( queue ).to receive( :consumer_count ) do
-				task_group.pids.size
+				task_group.workers.size
 			end.at_least( :once )
 			expect( queue ).to receive( :message_count ).and_return( *samples )
 
 			start = 1414002605
-			start.upto( start + samples.size ) do |time|
+			start.upto( start + samples.size + 1 ) do |time|
 				Timecop.freeze( time ) do
 					task_group.adjust_workers
 				end
 			end
 
-			expect( task_group.started_one_worker? ).to be_truthy
-			expect( task_group.pids ).to include( 414 )
-			expect( task_group.pids.length ).to eq( 2 )
+			expect( task_group.workers ).to include( FIRST_PID )
+			expect( task_group.workers.length ).to eq( 2 )
 		end
 
 
@@ -125,18 +115,18 @@ describe Symphony::TaskGroup::LongLived do
 				and_return( queue )
 
 			expect( queue ).to receive( :consumer_count ) do
-				task_group.pids.size
+				task_group.workers.size
 			end.at_least( :once )
 			expect( queue ).to receive( :message_count ).and_return( *samples )
 
 			start = 1414002605
-			start.upto( start + samples.size ) do |time|
+			start.upto( start + samples.size + 1 ) do |time|
 				Timecop.freeze( time ) do
 					task_group.adjust_workers
 				end
 			end
 
-			expect( task_group.pids.size ).to eq( 2 )
+			expect( task_group.workers.size ).to eq( 2 )
 		end
 
 
@@ -155,18 +145,18 @@ describe Symphony::TaskGroup::LongLived do
 				and_return( queue )
 
 			expect( queue ).to receive( :consumer_count ) do
-				task_group.pids.size
+				task_group.workers.size
 			end.at_least( :once )
 			expect( queue ).to receive( :message_count ).and_return( *samples )
 
 			start = 1414002605
-			start.upto( start + samples.size ) do |time|
+			start.upto( start + samples.size + 1 ) do |time|
 				Timecop.freeze( time ) do
 					task_group.adjust_workers
 				end
 			end
 
-			expect( task_group.pids.size ).to eq( 2 )
+			expect( task_group.workers.size ).to eq( 2 )
 		end
 
 
@@ -188,13 +178,13 @@ describe Symphony::TaskGroup::LongLived do
 			expect( queue ).to receive( :message_count ).and_return( *samples )
 
 			start = 1414002605
-			start.upto( start + samples.size ) do |time|
+			start.upto( start + samples.size + 1 ) do |time|
 				Timecop.freeze( time ) do
 					task_group.adjust_workers
 				end
 			end
 
-			expect( task_group.pids.size ).to eq( 1 )
+			expect( task_group.workers.size ).to eq( 1 )
 		end
 
 
@@ -213,19 +203,18 @@ describe Symphony::TaskGroup::LongLived do
 				and_return( queue )
 
 			expect( queue ).to receive( :consumer_count ) do
-				task_group.pids.size
+				task_group.workers.size
 			end.at_least( :once )
 			expect( queue ).to receive( :message_count ).and_return( *samples )
 
 			start = 1414002605
-			start.upto( start + samples.size ) do |time|
+			start.upto( start + samples.size + 1 ) do |time|
 				Timecop.freeze( time ) do
 					task_group.adjust_workers
 				end
 			end
 
-			expect( task_group.started_one_worker? ).to be_truthy
-			expect( task_group.pids.size ).to eq( 1 )
+			expect( task_group.workers.size ).to eq( 1 )
 		end
 
 	end