symphony 0.3.0.pre20140327204419

data/lib/symphony/task.rb

@@ -0,0 +1,407 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'set'
+require 'sysexits'
+require 'pluggability'
+require 'loggability'
+
+require 'msgpack'
+require 'yajl'
+require 'yaml'
+
+require 'symphony' unless defined?( Symphony )
+require 'symphony/signal_handling'
+
+
+# A task is the subclassable unit of work that Symphony loads when it starts up.
+class Symphony::Task
+	extend Loggability,
+	       Pluggability,
+	       Sysexits,
+	       Symphony::MethodUtilities
+
+	include Symphony::SignalHandling
+
+
+	# Signal to reset to defaults for the child
+	SIGNALS = %i[ INT TERM HUP CHLD WINCH ]
+
+	# Valid work model types
+	WORK_MODELS = %i[ longlived oneshot ]
+
+
+	# Loggability API -- log to symphony's logger
+	log_to :symphony
+
+	# Pluggability API -- set the directory/directories that will be search when trying to
+	# load tasks by name.
+	plugin_prefixes 'symphony/tasks'
+
+
+	### Create a new Task object and listen for work. Exits with the code returned
+	### by #start when it's done.
+	def self::run
+		if self.subscribe_to.empty?
+			raise ScriptError,
+				"No subscriptions defined. Add one or more patterns using subscribe_to."
+		end
+
+		exit self.new( self.queue ).start
+	end
+
+
+	### Inheritance hook -- set some defaults on subclasses.
+	def self::inherited( subclass )
+		super
+
+		subclass.instance_variable_set( :@routing_keys, Set.new )
+		subclass.instance_variable_set( :@acknowledge, true )
+		subclass.instance_variable_set( :@work_model, :longlived )
+		subclass.instance_variable_set( :@prefetch, 10 )
+		subclass.instance_variable_set( :@timeout_action, :reject )
+	end
+
+
+	### Fetch the Symphony::Queue for this task, creating it if necessary.
+	def self::queue
+		unless @queue
+			@queue = Symphony::Queue.for_task( self )
+		end
+		return @queue
+	end
+
+
+	### Return an queue name derived from the name of the task class.
+	def self::default_queue_name
+		name = self.name || "anonymous task %d" % [ self.object_id ]
+		return name.gsub( /\W+/, '.' ).downcase
+	end
+
+
+	### Return a consumer tag for this task's queue consumer.
+	def self::consumer_tag
+		return "%s.%s.%d" % [
+			self.queue_name,
+			Socket.gethostname.gsub(/\..*$/, ''),
+			Process.pid,
+		]
+	end
+
+
+	#
+	# :section: Declarative Methods
+	# These methods are used to configure how the task interacts with its queue and
+	# how it runs.
+
+
+	### Get/set the name of the queue to consume.
+	def self::queue_name( new_name=nil )
+		if new_name
+			@queue_name = new_name
+		end
+
+		@queue_name ||= self.default_queue_name
+		return @queue_name
+	end
+
+
+	### Set up one or more topic key patterns to use when binding the Task's queue
+	### to the exchange.
+	def self::subscribe_to( *routing_keys )
+		unless routing_keys.empty?
+			self.log.info "Setting task routing keys to: %p." % [ routing_keys ]
+			@routing_keys.replace( routing_keys )
+		end
+
+		return @routing_keys
+	end
+	class << self; alias_method :routing_keys, :subscribe_to ; end
+
+
+	### Enable or disable acknowledgements.
+	def self::acknowledge( new_setting=nil )
+		unless new_setting.nil?
+			self.log.info "Turning task acknowlegement %s." % [ new_setting ? "on" : "off" ]
+			@acknowledge = new_setting
+		end
+
+		return @acknowledge
+	end
+
+
+	### Get/set the maximum number of seconds the job should work on a single
+	### message before giving up.
+	def self::timeout( seconds=nil, options={} )
+		unless seconds.nil?
+			self.log.info "Setting the task timeout to %0.2fs." % [ seconds.to_f ]
+			@timeout = seconds.to_f
+			self.timeout_action( options[:action] )
+		end
+
+		return @timeout
+	end
+
+
+	### Set the action taken when work times out.
+	def self::timeout_action( new_value=nil )
+		if new_value
+			@timeout_action = new_value.to_sym
+		end
+
+		return @timeout_action
+	end
+
+
+	### Alter the work model between oneshot or longlived.
+	def self::work_model( new_setting=nil )
+		if new_setting
+			new_setting = new_setting.to_sym
+			unless WORK_MODELS.include?( new_setting )
+				raise "Unknown work_model %p (must be one of: %s)" %
+					[ new_setting, WORK_MODELS.join(', ') ]
+			end
+
+			self.log.info "Setting task work model to: %p." % [ new_setting ]
+			@work_model = new_setting
+		end
+
+		return @work_model
+	end
+
+
+	### Set the maximum number of messages to prefetch. Ignored if the work_model is
+	### :oneshot.
+	def self::prefetch( count=nil )
+		if count
+			@prefetch = count
+		end
+		return @prefetch
+	end
+
+
+	#
+	# Instance Methods
+	#
+
+	### Create a worker that will listen on the specified +queue+ for a job.
+	def initialize( queue )
+		@queue          = queue
+		@signal_handler = nil
+		@shutting_down  = false
+		@restarting     = false
+	end
+
+
+	######
+	public
+	######
+
+	# The queue that the task consumes messages from
+	attr_reader :queue
+
+	# The signal handler thread
+	attr_reader :signal_handler
+
+	# Is the task in the process of shutting down?
+	attr_predicate_accessor :shutting_down
+
+	# Is the task in the process of restarting?
+	attr_predicate_accessor :restarting
+
+
+	### Set up the task and start handling messages.
+	def start
+		rval = nil
+
+		begin
+			self.restarting = false
+			rval = self.with_signal_handler( *SIGNALS ) do
+				self.start_handling_messages
+			end
+		end while self.restarting?
+
+		return rval ? 0 : 1
+
+	rescue Exception => err
+		self.log.fatal "%p in %p: %s" % [ err.class, self.class, err.message ]
+		self.log.debug { '  ' + err.backtrace.join("  \n") }
+
+		return :software
+	end
+
+
+	### Restart the task after reloading the config.
+	def restart
+		self.restarting = true
+		self.log.warn "Restarting..."
+
+		if Symphony.config.reload
+			self.log.info "  config reloaded"
+		else
+			self.log.info "  no config changes"
+		end
+
+		self.log.info "  resetting queue"
+		Symphony::Queue.reset
+		self.queue.shutdown
+	end
+
+
+	### Stop the task immediately, e.g., when sent a second TERM signal.
+	def stop_immediately
+		self.log.warn "Already in shutdown -- halting immediately."
+		self.shutting_down = true
+		self.ignore_signals( *SIGNALS )
+		self.queue.halt
+	end
+
+
+	### Set the task to stop after what it's doing is completed.
+	def stop_gracefully
+		self.log.warn "Attempting to shut down gracefully."
+		self.shutting_down = true
+		self.queue.shutdown
+	end
+
+
+	### Start consuming messages from the queue, calling #work for each one.
+	def start_handling_messages
+		oneshot = self.class.work_model == :oneshot
+
+		return self.queue.wait_for_message( oneshot ) do |payload, metadata|
+			work_payload = self.preprocess_payload( payload, metadata )
+
+			if self.class.timeout
+				self.work_with_timeout( work_payload, metadata )
+			else
+				self.work( work_payload, metadata )
+			end
+		end
+	end
+
+
+	### Start the thread that will deliver signals once they're put on the queue.
+	def start_signal_handler
+		@signal_handler = Thread.new do
+			Thread.current.abort_on_exception = true
+			loop do
+				self.log.debug "Signal handler: waiting for new signals in the queue."
+				self.wait_for_signals
+			end
+		end
+	end
+
+
+	### Stop the signal handler thread.
+	def stop_signal_handler
+		@signal_handler.exit if @signal_handler
+	end
+
+
+	### Handle signals; called by the signal handler thread with a signal from the
+	### queue.
+	def handle_signal( sig )
+		self.log.debug "Handling signal %s" % [ sig ]
+		case sig
+		when :TERM
+			self.on_terminate
+		when :INT
+			self.on_interrupt
+		when :HUP
+			self.on_hangup
+		when :CHLD
+			self.on_child_exit
+		when :WINCH
+			self.on_window_size_change
+		else
+			self.log.warn "Unhandled signal %s" % [ sig ]
+		end
+	end
+
+
+	### Do any necessary pre-processing on the raw +payload+ according to values in
+	### the given +metadata+.
+	def preprocess_payload( payload, metadata )
+		self.log.debug "Got a %0.2fK %s payload" %
+			[ payload.bytesize / 1024.0, metadata[:content_type] ]
+		work_payload = case metadata[:content_type]
+			when 'application/x-msgpack'
+				MessagePack.unpack( payload )
+			when 'application/json', 'text/javascript'
+				Yajl::Parser.parse( payload )
+			when 'application/x-yaml', 'text/x-yaml'
+				YAML.load( payload )
+			else
+				payload
+			end
+
+		return work_payload
+	end
+
+
+	### Return a consumer tag for this task's queue consumer.
+	def make_consumer_tag
+		return "%s.%s.%d" % [
+			self.queue_name,
+			Socket.gethostname.gsub(/\..*$/, ''),
+			Process.pid,
+		]
+	end
+
+
+	### Do work based on the given message +payload+ and +metadata+.
+	def work( payload, metadata )
+		raise NotImplementedError,
+			"%p doesn't implement required method #work" % [ self.class ]
+	end
+
+
+	### Wrap a timeout around the call to work, and handle timeouts according to
+	### the configured timeout_action.
+	def work_with_timeout( payload, metadata )
+		Timeout.timeout( self.class.timeout ) do
+			return self.work( payload, metadata )
+		end
+	rescue Timeout::Error
+		self.log.error "Timed out while performing work"
+		raise if self.class.timeout_action == :reject
+		return false
+	end
+
+
+	### Handle a child process exiting.
+	def on_child_exit
+		self.log.info "Child exited."
+		Process.waitpid( 0, Process::WNOHANG )
+	end
+
+
+	### Handle a window size change event. No-op by default.
+	def on_window_size_change
+		self.log.info "Window size changed."
+	end
+
+
+	### Handle a hangup signal by re-reading the config and restarting.
+	def on_hangup
+		self.log.info "Hangup signal."
+		self.restart
+	end
+
+
+	### Handle a termination or interrupt signal.
+	def on_terminate
+		self.log.debug "Signalled to shut down."
+
+		if self.shutting_down?
+			self.stop_immediately
+		else
+			self.stop_gracefully
+		end
+	end
+	alias_method :on_interrupt, :on_terminate
+
+
+end # class Symphony::Task
+

data/lib/symphony/tasks/auditor.rb

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'pp'
+require 'pathname'
+require 'tmpdir'
+require 'symphony/task' unless defined?( Symphony::Task )
+require 'symphony/metrics'
+
+
+# A spike to log events
+class Auditor < Symphony::Task
+	prepend Symphony::Metrics
+
+	subscribe_to '#'
+	prefetch 1000
+	queue_name '_audit'
+
+
+	### Create a new Auditor task.
+	def initialize( queue )
+		super
+		@logdir = Pathname.pwd
+		@logfile = @logdir + 'events.log'
+		@log = @logfile.open( File::CREAT|File::APPEND|File::WRONLY, encoding: 'utf-8' )
+		self.log.info "Logfile is: %s" % [ @logfile ]
+	end
+
+
+	######
+	public
+	######
+
+	#
+	# Task API
+	#
+
+	# Log the event.
+	def work( payload, metadata )
+		@log.puts "%d%s [%s]: %p" % [
+			metadata[:delivery_info][:delivery_tag],
+			metadata[:delivery_info][:redelivered] ? '+' : '',
+			metadata[:delivery_info][:routing_key],
+			payload
+		]
+
+		return true
+	end
+
+
+end # class Auditor
+

data/lib/symphony/tasks/failure_logger.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require 'objspace'
+require 'pathname'
+require 'tmpdir'
+require 'symphony/task' unless defined?( Symphony::Task )
+require 'symphony/metrics'
+
+
+# Log events that get published to the dead-letter queue
+class FailureLogger < Symphony::Task
+	prepend Symphony::Metrics
+
+	# Audit all events
+	subscribe_to '#'
+
+	# Connect to a specific queue
+	queue_name '_failures'
+
+
+	### Set up the output device. By default it's STDERR, but it can be anything
+	### that responds to #<<.
+	def initialize( * )
+		super
+		@output = $stderr
+		$stderr.sync = true
+	end
+
+
+	######
+	public
+	######
+
+	#
+	# Task API
+	#
+
+	# Log the failure
+	# :headers=>{
+	#    "x-death"=>[{
+	#       "reason"=>"rejected",
+	#       "queue"=>"auditor",
+	#       "time"=>2014-03-12 18:55:10 -0700,
+	#       "exchange"=>"events",
+	#       "routing-keys"=>["some.stuff"]
+	#    }]
+	# }
+	def work( payload, metadata )
+		self.log_failure( payload, metadata )
+		return true
+	end
+
+
+	### Log one or more +deaths+ of the failed event.
+	def log_failure( payload, metadata )
+		raise "No headers; not a dead-lettered message?" unless
+			metadata[:properties] &&
+			metadata[:properties][:headers]
+		deaths = metadata[:properties][:headers]['x-death'] or
+			raise "No x-death header; not a dead-lettered message?"
+
+		message = self.log_prefix( payload, metadata )
+		message << self.log_deaths( deaths )
+		message << self.log_payload( payload, metadata )
+
+		@output << message << "\n"
+	end
+
+
+	### Return a logging message prefix based on the specified +routing_key+ and
+	### +deaths+.
+	def log_prefix( payload, metadata )
+		return "[%s]: " % [ Time.now.strftime('%Y-%m-%d %H:%M:%S.%4N') ]
+	end
+
+
+	### Return a logging message part based on the specified message +payload+.
+	def log_payload( payload, metadata )
+		return " -- %0.2fKB %s payload: %p" % [
+			ObjectSpace.memsize_of(payload) / 1024.0,
+			metadata[:content_type],
+			payload,
+		]
+	end
+
+
+	### Return a logging message part based on the specified +deaths+.
+	###
+	### deaths - An Array of Hashes derived from the 'x-death' headers of the message
+	###
+	def log_deaths( deaths )
+		message = ''
+		deaths.each do |death|
+			message << " %s-{%s}->%s (%s)" % [
+				death['exchange'],
+				death['routing-keys'].join(','),
+				death['queue'],
+				death['reason'],
+			]
+		end
+
+		return message
+	end
+
+end # class FailureLogger
+

data/lib/symphony/tasks/pinger.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'timeout'
+require 'symphony/task' unless defined?( Symphony::Task )
+
+
+### A proof-of-concept task to determine ssh availability of a host.
+class Symphony::Task::Pinger < Symphony::Task
+
+	# The topic key to subscribe to
+	subscribe_to 'monitor.availability.port',
+	             'host.ping'
+
+	# Send success/failure back to the queue on job completion.  Then true, the
+	# work isn't considered complete until receiving a success ack.  When false,
+	# a worker simply consuming the task is sufficient.
+	acknowledge false # default: true
+
+	# Timeout for performing work.  NOT to be confused with the message TTL
+	# during queue lifetime.
+	timeout 10.minutes # default: no timeout
+
+	# Whether the task should exit after doing its work
+	work_model :oneshot # default: :longlived
+
+
+	# The default port
+	DEFAULT_PORT = 'ssh'
+
+
+	### Create a new Pinger task for the given +job+ and +queue+.
+	def initialize
+		super
+	end
+
+
+	######
+	public
+	######
+
+	# The hostname to ping
+	attr_reader :hostname
+
+	# The (TCP) port to ping
+	attr_reader :port
+
+	# If there is a problem pinging the remote host, this is set to the exception
+	# raised when doing so.
+	attr_reader :problem
+
+
+	#
+	# Task API
+	#
+
+	### Do the ping.
+	def work( payload, metadata )
+		return ping( payload['hostname'], payload['port'] )
+	end
+
+
+end # class Symphony::Task::Pinger
+

data/lib/symphony/tasks/simulator.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+require 'tmpdir'
+require 'symphony/task' unless defined?( Symphony::Task )
+require 'symphony/metrics'
+
+# A spike to test out various task execution outcomes.
+class Simulator < Symphony::Task
+	prepend Symphony::Metrics
+
+	# Simulate processing all events
+	subscribe_to '#'
+
+	# Fetch 100 events at a time
+	prefetch 100
+
+	# Only allow 2 seconds for work to complete before rejecting or retrying.
+	timeout 2.0, action: :retry
+
+
+	######
+	public
+	######
+
+	#
+	# Task API
+	#
+
+	# Do the ping.
+	def work( payload, metadata )
+		if metadata[:properties][:headers] &&
+		   metadata[:properties][:headers]['x-death']
+			puts "Deaths! %p" % [ metadata[:properties][:headers]['x-death'] ]
+		end
+
+		val = Random.rand
+		case
+		when val < 0.33
+			$stderr.puts "Simulating an error in the task (reject)."
+			raise "OOOOOPS!"
+		when val < 0.66
+			$stderr.puts "Simulating a soft failure in the task (reject+requeue)."
+			return false
+		when val < 0.88
+			$stderr.puts "Simulating a timeout case"
+			sleep( self.class.timeout + 1 )
+		else
+			$stderr.puts "Simulating a successful task run (accept)"
+			puts( payload.inspect )
+			return true
+		end
+	end
+
+
+end # class Simulator
+