symphony 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f77d5c29817bfb9b9a524045e2a09dce3f07f469
-  data.tar.gz: ff3c97b9eab7dfdc1528cd766f6018b508c36698
+  metadata.gz: 095d96d07d5e07c3b0cecfa55d5705e1c5dddb55
+  data.tar.gz: cfc61ebc105673602505d959956bb2a24a939e3a
 SHA512:
-  metadata.gz: fe1f7745132368e2b7414fa27ac34a8be01a40af388655041dd61718fc6b8dbcf9db44ed9337b60df4ab72150a925dfafbd64b518ecabf9642e89a050d17d6bd
-  data.tar.gz: 119c5a6aebb5d52341f1b901ac29457d6dcb9d810745c59d50b678bd0aec5c0bdafbdaabef506662d78338d7c0a94124c39e3ce4de96c289b4db06349622f74c
+  metadata.gz: 2c9443c031f087f3ae3633be51d70d9dfc7adfd817ac72972cd992a72716525d76c2965303bfbb726d7e06acb5143ec168b3e379b4253cbee1ed1643e25b7397
+  data.tar.gz: 7661d67300b397b7b97e8b24e011c8328bd3737c9f89b44cc668121589e5ec63364c5a957f09c148ea407b872bbda3ec9faf37abe0849d086a7039cf3fb15da9

data/ChangeLog

@@ -1,15 +1,103 @@
+2015-05-29  Michael Granger  <ged@FaerieMUD.org>
+
+	* .gems, .rvm.gems, .rvmrc:
+	Update dev environment
+	[44341600897a] [tip]
+
+2015-02-27  Michael Granger  <ged@FaerieMUD.org>
+
+	* .travis.yml:
+	Don't test under rbx
+	[c6e5888c9d68] [github/master]
+
+	* .travis.yml, Gemfile.ci:
+	Nope, backed out changeset 031ea486d462
+	[3031ee38d8a8]
+
+	* .travis.yml, Gemfile.ci:
+	Try using a specific gemset without metrics to avoid rusage
+	[031ea486d462]
+
+	* .travis.yml:
+	Adding travis-ci config
+	[a4bec430c2ce]
+
+	* .hgignore, .hoerc, .ruby-version, Gemfile, Manifest.txt, Rakefile,
+	UPGRADING.md, USAGE.rdoc, symphony.gemspec:
+	Update project files for prerelease
+	[5f344848f265]
+
+	* lib/symphony/daemon.rb, lib/symphony/task_group.rb,
+	spec/symphony/daemon_spec.rb, spec/symphony/queue_spec.rb:
+	Finish up the daemon task reconfig spec
+	[d3ea6af65086]
+
+2015-02-27  mahlon  <mahlon@martini.nu>
+
+	* Manifest.txt:
+	Update the manifest.
+	[74cfe663a36e]
+
+2015-02-27  Michael Granger  <ged@FaerieMUD.org>
+
+	* lib/symphony/queue.rb, lib/symphony/statistics.rb:
+	Squelch spammy debugging
+	[e098fa10f649]
+
+	* .rvm.gems, .rvmrc, .tm_properties, Manifest.txt, Rakefile, TODO.md,
+	lib/symphony.rb, lib/symphony/daemon.rb, lib/symphony/queue.rb,
+	lib/symphony/signal_handling.rb, lib/symphony/statistics.rb,
+	lib/symphony/task.rb, lib/symphony/task_group.rb,
+	lib/symphony/task_group/longlived.rb,
+	lib/symphony/task_group/oneshot.rb,
+	lib/symphony/tasks/oneshot_simulator.rb,
+	lib/symphony/tasks/simulator.rb, spec/helpers.rb,
+	spec/symphony/daemon_spec.rb, spec/symphony/statistics_spec.rb,
+	spec/symphony/task_group/longlived_spec.rb,
+	spec/symphony/task_group/oneshot_spec.rb,
+	spec/symphony/task_group_spec.rb, spec/symphony_spec.rb:
+	Implement task work models
+	* * * Load components earlier, and move configure() below other method
+	definitions. This fixes a chicken-and-egg with Configurability where
+	loading the config before loading Symphony caused a RuntimeError.
+	[a6b97e9ea2e1]
+
+	* lib/symphony/queue.rb:
+	Include the first backtrace frame when logging job errors
+	[47b5da330502]
+
+2014-10-14  Michael Granger  <ged@FaerieMUD.org>
+
+	* .rvm.gems, Rakefile:
+	Bump minimum version of Bunny to 1.5 for better connection recovery
+	[cd10e63d22d7]
+
+2014-09-03  Michael Granger  <ged@FaerieMUD.org>
+
+	* .hgtags:
+	Added tag v0.8.0 for changeset 8d6669a7e329
+	[af3d65f141e7]
+
+	* .hgsigs:
+	Added signature for changeset 68352ad02888
+	[8d6669a7e329] [v0.8.0]
+
+	* History.rdoc, Manifest.txt, lib/symphony.rb:
+	Bump the minor version, update history.
+	[68352ad02888]
+
 2014-09-01  Michael Granger  <ged@FaerieMUD.org>
 
 	* lib/symphony/routing.rb, spec/symphony/routing_spec.rb:
 	Add route options to routing mixin
-	[328bc1fbb538] [qbase, qtip, route-options.patch, tip]
+	[328bc1fbb538]
 
 2014-05-10  Michael Granger  <ged@FaerieMUD.org>
 
 	* bin/symphony-task:
 	Use the config env stuff already in Symphony.load_config, and keep
 	the config file argument optional for symphony-task.
-	[0d91a148cacd] [qparent]
+	[0d91a148cacd]
 
 2014-05-09  Mahlon E. Smith  <mahlon@laika.com>
 
@@ -75,7 +163,7 @@
 
 	* History.rdoc, lib/symphony.rb:
 	Bump the minor version, update history
-	[fa6a367872f4] [github/master]
+	[fa6a367872f4]
 
 	* TODO.md, lib/symphony/queue.rb, spec/symphony/queue_spec.rb:
 	Update the TO-DO list

data/History.rdoc

@@ -1,3 +1,16 @@
+== v0.9.0 [2015-06-01] Michael Granger <ged@FaerieMUD.org>
+
+Improvements:
+
+- Implement task work models
+- Include the first backtrace frame when logging job errors
+
+Bugfixes:
+
+- Fix the patterns used by Symphony::Routing to match
+  RabbitMQ topic-queue matching.
+
+
 == v0.8.0 [2014-09-01] Michael Granger <ged@FaerieMUD.org>
 
 - Add route options to routing mixin

data/Manifest.txt

@@ -4,7 +4,7 @@ History.rdoc
 Manifest.txt
 README.rdoc
 Rakefile
-TODO.md
+UPGRADING.md
 USAGE.rdoc
 bin/symphony
 bin/symphony-task
@@ -16,14 +16,23 @@ lib/symphony/mixins.rb
 lib/symphony/queue.rb
 lib/symphony/routing.rb
 lib/symphony/signal_handling.rb
+lib/symphony/statistics.rb
 lib/symphony/task.rb
+lib/symphony/task_group.rb
+lib/symphony/task_group/longlived.rb
+lib/symphony/task_group/oneshot.rb
 lib/symphony/tasks/auditor.rb
 lib/symphony/tasks/failure_logger.rb
+lib/symphony/tasks/oneshot_simulator.rb
 lib/symphony/tasks/simulator.rb
 spec/helpers.rb
 spec/symphony/daemon_spec.rb
 spec/symphony/mixins_spec.rb
 spec/symphony/queue_spec.rb
 spec/symphony/routing_spec.rb
+spec/symphony/statistics_spec.rb
+spec/symphony/task_group/longlived_spec.rb
+spec/symphony/task_group/oneshot_spec.rb
+spec/symphony/task_group_spec.rb
 spec/symphony/task_spec.rb
 spec/symphony_spec.rb

data/README.rdoc

@@ -58,7 +58,7 @@ and generate the API documentation.
 
 == License
 
-Copyright (c) 2011-2014, Michael Granger and Mahlon E. Smith
+Copyright (c) 2011-2015, Michael Granger and Mahlon E. Smith
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/Rakefile

@@ -12,7 +12,6 @@ GEMSPEC = 'symphony.gemspec'
 Hoe.plugin :mercurial
 Hoe.plugin :signing
 Hoe.plugin :deveiate
-Hoe.plugin :bundler
 
 Hoe.plugins.delete :rubyforge
 
@@ -27,10 +26,10 @@ hoespec = Hoe.spec 'symphony' do |spec|
 	spec.developer 'Michael Granger', 'ged@FaerieMUD.org'
 	spec.developer 'Mahlon E. Smith', 'mahlon@martini.nu'
 
-	spec.dependency 'configurability', '~> 2.1'
+	spec.dependency 'configurability', '~> 2.2'
 	spec.dependency 'loggability', '~> 0.10'
 	spec.dependency 'pluggability', '~> 0.4'
-	spec.dependency 'bunny', '~> 1.1'
+	spec.dependency 'bunny', '~> 1.5'
 	spec.dependency 'sysexits', '~> 1.1'
 	spec.dependency 'yajl-ruby', '~> 1.2'
 	spec.dependency 'msgpack', '~> 0.5'
@@ -39,6 +38,7 @@ hoespec = Hoe.spec 'symphony' do |spec|
 
 	spec.dependency 'rspec', '~> 3.0', :developer
 	spec.dependency 'simplecov', '~> 0.8', :developer
+	spec.dependency 'timecop', '~> 0.7', :developer
 
 	spec.require_ruby_version( '>=2.0.0' )
 	spec.hg_sign_tags = true if spec.respond_to?( :hg_sign_tags= )
@@ -51,7 +51,7 @@ hoespec.spec.files.delete( '.gemtest' )
 ENV['VERSION'] ||= hoespec.spec.version.to_s
 
 # Run the tests before checking in
-task 'hg:precheckin' => [ :check_history, :check_manifest, :spec ]
+task 'hg:precheckin' => [ :check_history, :check_manifest, :gemspec, :spec ]
 
 # Rebuild the ChangeLog immediately before release
 task :prerelease => 'ChangeLog'
@@ -67,7 +67,7 @@ end
 task :gemspec => GEMSPEC
 file GEMSPEC => hoespec.spec.files do |task|
 	spec = $hoespec.spec
-	spec.version = "#{spec.version}.pre#{Time.now.strftime("%Y%m%d%H%M%S")}"
+	spec.version = "#{spec.version.bump}.0.pre#{Time.now.strftime("%Y%m%d%H%M%S")}"
 	File.open( task.name, 'w' ) do |fh|
 		fh.write( spec.to_ruby )
 	end

data/UPGRADING.md

@@ -0,0 +1,38 @@
+# Upgrading
+
+## From 0.8.x to 0.9.0
+
+This version introduces several new mechanisms for adjusting your workers to handle increased amounts of work.
+
+This is accomplished via a new (backward-compatible) `tasks` config syntax, and a pluggable "work model" that can be set to control how workers running a particular task behave.
+
+Tasks are now run inside of a "task group" which contains the logic of the work model. There are two initial work models that come with Symphony: `longlived` and `oneshot`.
+
+The `longlived` work model is the default, and works similarly to how tasks worked prior to the 0.9 release: it starts up and executes tasks as they arrive, and then shuts down when the Symphony daemon shuts down. If you keep your configuration the same as it was before this release, nothing should change.
+
+However, you can now tell your `longlived` task groups to automatically scale up the number of instances when the amount of work to be done increases. You can do this by converting the `tasks` config section to a Hash, with the task names as the keys and an integer as the value.
+
+The old way:
+
+    symphony:
+      tasks:
+      - audit_logger
+      - failure_logger
+      - payments_processor
+      - user_mailer
+      - thumbnailer
+
+the new way:
+
+    symphony:
+      tasks:
+        audit_logger: 1
+        failure_logger: 1
+        payments_processor: 1
+        user_mailer: 2
+        thumbnailer: 5
+
+The value controls the _maximum_ number of that task class that can be running at one time, and the Symphony daemon will now scale the number of workers up to your specified maximum when the amount of work is trending upwards, and scale it back down to a single worker when work is trending down.
+
+The `oneshot` work model is a new kind of worker that fetches and executes a single task and then exits. It's used for tasks that consume large amounts of memory or other resources that may not be released in between tasks such as 3D rendering or video-processing. 
+

data/USAGE.rdoc

@@ -179,7 +179,7 @@ events.
 	acknowledge false   # (default: true)
 
 
-== Worker Model
+== Work Model
 
 By default, a task signals that is ready for more messages as soon as
 it finishes processing one.  This isn't always the optimal environment

data/lib/symphony.rb

@@ -12,10 +12,10 @@ module Symphony
 	       Configurability
 
 	# Library version constant
-	VERSION = '0.8.0'
+	VERSION = '0.9.0'
 
 	# Version-control revision constant
-	REVISION = %q$Revision: 68352ad02888 $
+	REVISION = %q$Revision: 84c169f1c831 $
 
 
 	# The name of the environment variable to check for config file overrides
@@ -29,6 +29,80 @@ module Symphony
 	log_as :symphony
 
 
+	# Configurability API -- use the 'worker_daemon' section of the config
+	config_key :symphony
+
+
+	# Default configuration
+	CONFIG_DEFAULTS = {
+		throttle_max:     16,
+		throttle_factor:  1,
+		tasks:            [],
+		scaling_interval: 0.1,
+	}
+
+
+	require 'symphony/mixins'
+	require 'symphony/queue'
+	require 'symphony/task'
+	require 'symphony/task_group'
+	extend Symphony::MethodUtilities
+
+	##
+	# The maximum throttle factor caused by failing workers
+	singleton_attr_accessor :throttle_max
+	self.throttle_max = CONFIG_DEFAULTS[:throttle_max]
+
+	##
+	# The factor which controls how much incrementing the throttle factor
+	# affects the pause between workers being started.
+	singleton_attr_accessor :throttle_factor
+	self.throttle_factor = CONFIG_DEFAULTS[:throttle_factor]
+
+	##
+	# The Array of Symphony::Task classes that are configured to run
+	singleton_attr_accessor :tasks
+	self.tasks = CONFIG_DEFAULTS[:tasks]
+
+	##
+	# The maximum amount of time between task group process checks
+	singleton_attr_accessor :scaling_interval
+	self.scaling_interval = CONFIG_DEFAULTS[:scaling_interval]
+
+
+	### Load the tasks with the specified +task_names+ and return them
+	### as an Array.
+	def self::load_configured_tasks( task_config )
+		if task_config.respond_to?( :each_pair )
+			return self.task_config_from_hash( task_config )
+		else
+			return self.task_config_from_array( task_config )
+		end
+	end
+
+
+	### Return the Hash of +tasks+ as a Hash of Classes and the maximum number to
+	### run.
+	def self::task_config_from_hash( task_config )
+		return task_config.each_with_object({}) do |(task_name, max), tasks|
+			task_class = Symphony::Task.get_subclass( task_name )
+			tasks[ task_class ] = max.to_i
+		end
+	end
+
+
+	### Return the Array of +tasks+ as a Hash of Classes and the maximum number to
+	### run.
+	def self::task_config_from_array( task_config )
+		return [] unless task_config
+		return task_config.uniq.each_with_object({}) do |task_name, tasks|
+			max = task_config.count( task_name )
+			task_class = Symphony::Task.get_subclass( task_name )
+			tasks[ task_class ] = max
+		end
+	end
+
+
 	### Get the loaded config (a Configurability::Config object)
 	def self::config
 		Configurability.loaded_config
@@ -48,9 +122,16 @@ module Symphony
 	end
 
 
-	require 'symphony/mixins'
-	require 'symphony/queue'
-	require 'symphony/task'
+	### Configurability API -- configure the daemon.
+	def self::configure( config=nil )
+		config = self.defaults.merge( config || {} )
+
+		self.throttle_max     = config[:throttle_max]
+		self.throttle_factor  = config[:throttle_factor]
+		self.scaling_interval = config[:scaling_interval]
+
+		self.tasks = self.load_configured_tasks( config[:tasks] )
+	end
 
 end # module Symphony
 

data/lib/symphony/daemon.rb

@@ -7,13 +7,12 @@ require 'loggability'
 require 'symphony' unless defined?( Symphony )
 require 'symphony/task'
 require 'symphony/signal_handling'
+require 'symphony/task_group'
 
 # A daemon which manages startup and shutdown of one or more Workers
 # running Tasks as they are published from a queue.
 class Symphony::Daemon
-	extend Loggability,
-	       Configurability,
-	       Symphony::MethodUtilities
+	extend Loggability
 
 	include Symphony::SignalHandling
 
@@ -21,16 +20,6 @@ class Symphony::Daemon
 	# Loggability API -- log to the symphony logger
 	log_to :symphony
 
-	# Configurability API -- use the 'worker_daemon' section of the config
-	config_key :symphony
-
-
-	# Default configuration
-	CONFIG_DEFAULTS = {
-		throttle_max:    16,
-		throttle_factor: 1,
-		tasks:           []
-	}
 
 	# Signals we understand
 	QUEUE_SIGS = [
@@ -39,25 +28,10 @@ class Symphony::Daemon
 	]
 
 
-
 	#
 	# Class methods
 	#
 
-	##
-	# The maximum throttle factor caused by failing workers
-	singleton_attr_accessor :throttle_max
-
-	##
-	# The factor which controls how much incrementing the throttle factor
-	# affects the pause between workers being started.
-	singleton_attr_accessor :throttle_factor
-
-	##
-	# The Array of Symphony::Task classes that are configured to run
-	singleton_attr_accessor :tasks
-
-
 	### Get the daemon's version as a String.
 	def self::version_string( include_buildnum=false )
 		vstring = "%s %s" % [ self.name, Symphony::VERSION ]
@@ -69,26 +43,6 @@ class Symphony::Daemon
 	end
 
 
-	### Configurability API -- configure the daemon.
-	def self::configure( config=nil )
-		config = self.defaults.merge( config || {} )
-
-		self.throttle_max    = config[:throttle_max]
-		self.throttle_factor = config[:throttle_factor]
-
-		self.tasks = self.load_configured_tasks( config[:tasks] )
-	end
-
-
-	### Load the tasks with the specified +task_names+ and return them
-	### as an Array.
-	def self::load_configured_tasks( task_names )
-		return task_names.map do |task_name|
-			Symphony::Task.get_subclass( task_name )
-		end
-	end
-
-
 	### Start the daemon.
 	def self::run( args )
 		Loggability.format_with( :color ) if $stdout.tty?
@@ -113,11 +67,9 @@ class Symphony::Daemon
 
 	### Create a new Daemon instance.
 	def initialize
-		@running_tasks       = {}
+		@task_pids   = {}
+		@task_groups = {}
 		@running             = false
-		@shutting_down       = false
-		@throttle            = 0
-		@last_child_started  = Time.now
 
 		self.set_up_signal_handling
 	end
@@ -127,8 +79,11 @@ class Symphony::Daemon
 	public
 	######
 
-	# The Hash of PIDs to task class
-	attr_reader :running_tasks
+	# The Hash of PID to task group
+	attr_reader :task_pids
+
+	# The Array of running task groups
+	attr_reader :task_groups
 
 	# A self-pipe for deferred signal-handling
 	attr_reader :selfpipe
@@ -136,13 +91,6 @@ class Symphony::Daemon
 	# The Symphony::Queue that jobs will be fetched from
 	attr_reader :queue
 
-	# The Configurability::Config object for the current configuration.
-	attr_reader :config
-
-
-	# Make a delegator for the class's task list
-	define_method( :tasks, &self.method(:tasks) )
-
 
 	### Returns +true+ if the daemon is still running.
 	def running?
@@ -150,12 +98,6 @@ class Symphony::Daemon
 	end
 
 
-	### Returns +true+ if the daemon is shutting down.
-	def shutting_down?
-		return @shutting_down
-	end
-
-
 	### Set up the daemon and start running.
 	def run
 		self.log.info "Starting task daemon"
@@ -168,8 +110,6 @@ class Symphony::Daemon
 
 		# Restore the default signal handlers
 		self.reset_signal_traps( *QUEUE_SIGS )
-
-		exit
 	end
 
 
@@ -177,12 +117,14 @@ class Symphony::Daemon
 	### take appropriate action.
 	def run_tasks
 		@running = true
+		self.create_task_groups
 
 		self.log.debug "Starting supervisor loop..."
 		while self.running?
-			self.start_missing_children unless self.shutting_down?
-			self.wait_for_signals
-			self.reap_children
+			self.tickle_task_groups
+			if self.wait_for_signals( Symphony.scaling_interval )
+				self.reap_children
+			end
 		end
 
 	rescue => err
@@ -191,7 +133,6 @@ class Symphony::Daemon
 
 	ensure
 		self.log.info "Done running tasks."
-		@running = false
 		self.stop
 	end
 
@@ -199,7 +140,7 @@ class Symphony::Daemon
 	### Shut the daemon down gracefully.
 	def stop
 		self.log.warn "Stopping."
-		@shutting_down = true
+		@running = false
 
 		self.ignore_signals( *QUEUE_SIGS )
 
@@ -209,14 +150,14 @@ class Symphony::Daemon
 			sleep( 1 )
 			self.kill_children
 			sleep( 1 )
-			break if self.running_tasks.empty?
+			break if self.task_pids.empty?
 			sleep( 1 )
-		end unless self.running_tasks.empty?
+		end unless self.task_pids.empty?
 
 		# Give up on our remaining children.
 		Signal.trap( :CHLD, :IGNORE )
-		if !self.running_tasks.empty?
-			self.log.warn "  %d workers remain: sending KILL" % [ self.running_tasks.length ]
+		if !self.task_pids.empty?
+			self.log.warn "  %d workers remain: sending KILL" % [ self.task_pids.length ]
 			self.kill_children( :KILL )
 		end
 	end
@@ -224,8 +165,11 @@ class Symphony::Daemon
 
 	### Reload the configuration.
 	def reload_config
-		self.log.warn "Reloading config %p" % [ self.config ]
-		self.config.reload
+		self.log.warn "Reloading config %p" % [ Symphony.config ]
+		Symphony.config.reload
+
+		# And start them up again using the new config.
+		self.create_task_groups
 	end
 
 
@@ -263,70 +207,83 @@ class Symphony::Daemon
 	end
 
 
-	### Start any tasks which aren't already running
-	def start_missing_children
-		missing_tasks = self.find_missing_tasks
-		return if missing_tasks.empty?
+	### Create task groups for each configured task.
+	def create_task_groups
+		old_task_groups = @task_groups || {}
+		@task_groups = {}
+
+		self.log.debug "Managing task groups: %p" % [ old_task_groups ]
 
-		# Return unless the throttle period has lapsed
-		unless self.throttle_seconds < (Time.now - @last_child_started)
-			self.log.info "Not starting children: throttled for %0.2f seconds" %
-				[ self.throttle_seconds ]
-			return
+		Symphony.tasks.each do |task_class, max|
+			# If the task is still configured, restart all of its workers
+			if group = old_task_groups.delete( task_class )
+				self.log.info "%p still configured; restarting its task group." % [ task_class ]
+				self.restart_task_group( group, task_class, max )
+				@task_groups[ task_class ] = group
+
+			# If it's new, just start it up
+			else
+				self.log.info "Starting up new task group for %p" % [ task_class ]
+				@task_groups[ task_class ] = self.start_task_group( task_class, max )
+			end
 		end
 
-		self.log.debug "Starting %d tasks out of %d" % [ missing_tasks.size, self.class.tasks.size ]
-		missing_tasks.each do |task_class|
-			pid = self.start_worker( task_class )
-			self.log.debug "  started task %p at pid %d" % [ task_class, pid ]
-			self.running_tasks[ pid ] = task_class
+		# Any task classes remaining are no longer configured, so stop them.
+		old_task_groups.each do |task_class, group|
+			self.log.info "%p no longer configured; stopping its task group." % [ task_class ]
+			self.stop_task_group( group )
+		end
 		end
 
-		@last_child_started = Time.now
-	end
 
+	### Start a new task group for the given +task_class+ and +max+ number of workers.
+	def start_task_group( task_class, max )
+		self.log.info "Starting a task group for %p" % [ task_class ]
+		Symphony::TaskGroup.create( task_class.work_model, task_class, max )
+	end
 
-	### Examine the running tasks and return any that are missing.
-	def find_missing_tasks
-		missing_tasks = []
 
-		self.class.tasks.uniq.each do |task_type|
-			count = self.class.tasks.count( task_type )
-			missing = count - self.running_tasks.values.count( task_type )
-			missing.times do
-				missing_tasks << task_type
-			end
+	### Tell the specified task +group+ to restart with the specified +max+ number of workers.
+	def restart_task_group( group, task_class, max )
+		self.log.info "Restarting task group for %p" % [ task_class ]
+		group.max_workers = max
+		group.restart_workers
 		end
 
-		return missing_tasks
-	end
-
 
-	### Return the number of seconds between child startup times.
-	def throttle_seconds
-		return 0 unless @throttle.nonzero?
-		return Math.log( @throttle ) * self.class.throttle_factor
+	### Shut down the workers for the specified task group.
+	def stop_task_group( group )
+		self.log.info "Shutting down the task group for %p" % [ group.task_class ]
+		group.stop_all_workers
 	end
 
 
-	### Add +adjustment+ to the throttle value, ensuring that it doesn't go
-	### below zero.
-	def adjust_throttle( adjustment=1 )
-		self.log.debug "Adjusting worker throttle by %d" % [ adjustment ]
-		@throttle += adjustment
-		@throttle = 0 if @throttle < 0
-		@throttle = self.class.throttle_max if @throttle > self.class.throttle_max
+	### Tell the task groups to start or stop children based on their work model.
+	def tickle_task_groups
+		self.task_groups.each do |task_class, group|
+			new_pids = group.adjust_workers or next
+			new_pids.each do |pid|
+				self.task_pids[ pid ] = group
+	end
+		end
 	end
 
 
 	### Kill all current children with the specified +signal+. Returns +true+ if the signal was
 	### sent to one or more children.
 	def kill_children( signal=:TERM )
-		return false if self.running_tasks.empty?
+		return false if self.task_pids.empty?
 
 		self.log.info "Sending %s signal to %d task pids: %p." %
-			 [ signal, self.running_tasks.length, self.running_tasks.keys ]
-		Process.kill( signal, *self.running_tasks.keys )
+			 [ signal, self.task_pids.length, self.task_pids.keys ]
+		self.task_pids.keys.each do |pid|
+			begin
+				Process.kill( signal, pid )
+			rescue Errno::ESRCH => err
+				self.log.error "%p when trying to %s child %d: %s" %
+					[ err.class, signal, pid, err.message ]
+			end
+		end
 
 		return true
 	rescue Errno::ESRCH
@@ -334,22 +291,6 @@ class Symphony::Daemon
 	end
 
 
-	### Start a new Symphony::Task and return its PID.
-	def start_worker( task_class )
-		return if self.shutting_down?
-
-		self.log.debug "Starting a %p." % [ task_class ]
-		task_class.before_fork
-		pid = Process.fork do
-			task_class.after_fork
-			task_class.run
-		end
-		Process.setpgid( pid, 0 )
-
-		return pid
-	end
-
-
 	### Clean up after any children that have died.
 	def reap_children( *pids )
 		self.log.debug "Reaping children."
@@ -362,7 +303,7 @@ class Symphony::Daemon
 				self.reap_specific_child( pid )
 			end
 		end
-	rescue Errno::ECHILD => err
+	rescue Errno::ECHILD
 		self.log.debug "No more children to reap."
 	end
 
@@ -375,9 +316,8 @@ class Symphony::Daemon
 		pid, status = Process.waitpid2( -1, Process::WNOHANG|Process::WUNTRACED )
 		self.log.debug "  waitpid2 returned: [ %p, %p ]" % [ pid, status ]
 		while pid
-			self.adjust_throttle( status.success? ? -1 : 1 )
-			self.log.debug "Child %d exited: %p." % [ pid, status ]
-			self.running_tasks.delete( pid )
+			self.notify_group( pid, status )
+			self.task_pids.delete( pid )
 
 			pid, status = Process.waitpid2( -1, Process::WNOHANG|Process::WUNTRACED )
 			self.log.debug "  waitpid2 returned: [ %p, %p ]" % [ pid, status ]
@@ -388,15 +328,22 @@ class Symphony::Daemon
 	### Wait on the child associated with the given +pid+, deleting it from the
 	### running tasks Hash if successful.
 	def reap_specific_child( pid )
-		spid, status = Process.waitpid2( pid )
-		if spid
-			self.log.debug "Child %d exited: %p." % [ spid, status ]
-			self.running_tasks.delete( spid )
-			self.adjust_throttle( status.success? ? -1 : 1 )
+		pid, status = Process.waitpid2( pid )
+		if pid
+			self.notify_group( pid, status )
+			self.task_pids.delete( pid )
 		else
 			self.log.debug "Child %d no reapy." % [ pid ]
 		end
 	end
 
 
+	### Notify the task group the specified +pid+ belongs to that its child exited
+	### with the specified +status+.
+	def notify_group( pid, status )
+		return unless self.running?
+		group = self.task_pids[ pid ]
+		group.on_child_exit( pid, status )
+	end
+
 end # class Symphony::Daemon