droid 0.9.5 → 1.0.0

data/lib/droid/heroku/stats.rb

@@ -0,0 +1,30 @@
+module Stats
+	# The MemCache instance used to manipulate stats.
+	def cache
+		MemcacheCluster.cache("heroku:stats")
+	end
+
+	# Increment a stat counter. If the counter does not exist,
+	# yield to the block and use the result as the current counter
+	# value. With no block, the counter will be started at zero.
+	def increment(key, amount=1)
+		if (value = cache.incr(key, amount)).nil?
+			value = yield if block_given?
+			value = (value || 0) + amount
+			cache.add(key, value.to_s, 0, true)
+		end
+	rescue => boom
+		Log.default_error(boom)
+		nil
+	end
+
+	# Set the stat counter to a specific value.
+	def sample(key, value)
+		cache.set(key, value.to_s, 0, true)
+	rescue => boom
+		Log.default_error(boom)
+		nil
+	end
+
+	extend self
+end

data/lib/droid/json_server.rb

@@ -0,0 +1,109 @@
+require 'eventmachine'
+require 'evma_httpserver'
+
+class Droid
+	class JSONServer < ::EM::Connection
+		include ::EM::HttpServer
+
+		def post_init
+			super
+			no_environment_strings
+		end
+
+		def process_http_request
+			return not_found_response if @http_request_method != "GET"
+
+			if @http_request_uri == "/"
+				default_response
+			else
+				method_name = "get_#{@http_request_uri.split("/")[1]}".gsub(/[^\d\w_]/,'').downcase
+				if public_methods.include?(method_name)
+					generate_response(method_name)
+				else
+					not_found_response
+				end
+			end
+		end
+
+		def generate_response(method_name)
+			status, data, content_type = self.send(method_name)
+
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = status
+			response.content_type content_type
+			response.content = data
+			response.send_response
+		end
+
+		def default_response
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = 200
+			response.content_type 'application/json'
+			response.content = {"status" => "OK"}.to_json
+			response.send_response
+		end
+
+		def not_found_response
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = 404
+			response.content_type 'text/plain'
+			response.content = "Not Found"
+			response.send_response
+		end
+
+		def get_droid
+			report_data = Droid::Utilization.report_data
+
+			metrics = {}
+			report_data.each do |topic, data|
+				metrics[topic] = data['msgs']
+			end
+			metrics['latency'] = Droid::Utilization.latency
+
+			summary = Droid::Utilization.report_summary(report_data)
+
+			metrics['total_msgs'] = summary['msgs']
+
+			status = "AMQP: #{summary['msgs']} msgs processed since #{Droid::Utilization.start.utc}"
+
+			# reset metrics data
+			Droid::Utilization.reinit
+
+			data = {
+				'status' => status,
+				'state' => 'ok',
+				'metrics' => hash_to_metrics(metrics)
+			}
+			[200, data.to_json, "application/json"]
+		end
+
+		def hash_to_metrics(hash); self.class.hash_to_metrics(hash); end
+
+		# utility method to convert a ruby hash to a metrics format
+		# that can be consumed by cloudkick
+		def self.hash_to_metrics(hash)
+			hash.collect do |k,v|
+				name = k.to_s
+				value = v
+				type = if v.kind_of?(Integer)
+					'int'
+				elsif v.kind_of?(Float)
+					'float'
+				else
+					'string'
+				end
+
+				# bool -> int conversion
+				if [TrueClass, FalseClass].include?(v.class)
+					value = v ? 1 : 0
+					type = 'int'
+				end
+
+				# if type is really string then it should respond to .to_s
+				value = value.to_s if type == 'string'
+
+				{ 'name' => name, 'type' => type, 'value' => value }
+			end
+		end
+	end
+end

data/lib/droid/monkey.rb

@@ -0,0 +1,8 @@
+module AMQP
+	module Client
+		def reconnect(*args)
+			sleep 10
+			exit 1
+		end
+	end
+end

data/lib/droid/publish.rb

@@ -0,0 +1,24 @@
+require 'droid/utils'
+
+class Droid
+	# publish to queue directly
+	def self.publish_to_q(queue_name, data, opts={}, popts={})
+		q = ::MQ.queue(queue_name)
+		json, popts = Droid::Utils.format_publish(data, opts, popts)
+		q.publish(json, popts)
+		log.info "amqp_publish queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	# publish to exchange directly
+	def self.publish_to_ex(ex_name, data, opts={}, popts={})
+		ex = ::MQ.direct(ex_name)
+		json, popts = Droid::Utils.format_publish(data, opts, popts)
+		ex.publish(json, popts)
+		log.info "amqp_publish exchange=#{ex_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	# default is publish to exchange
+	def self.publish(ex_name, data, opts={}, popts={})
+		publish_to_ex(ex_name, data, opts, popts)
+	end
+end

data/lib/droid/queue.rb

@@ -0,0 +1,196 @@
+class Droid
+	class ExpiredMessage < RuntimeError; end
+
+	class BaseQueue
+		attr_reader :queue_name, :opts
+		attr_reader :q, :ex, :mq
+
+		def initialize(queue_name, opts={})
+			opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
+
+			@queue_name, @opts = queue_name, opts
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			# if we don't specify an exchange name it defaults to the queue_name
+			@ex = @mq.direct(opts[:exchange_name] || queue_name)
+		end
+
+		def temp?
+			false
+		end
+
+		def log
+			Droid.log
+		end
+
+		def tag
+			s = "queue=#{q.name}"
+			s += " exchange=#{ex.name}" if ex
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			setup
+
+			q.bind(ex) if ex
+			q.subscribe(amqp_opts) do |header, message|
+				Droid::Utilization.monitor(q.name, :temp => temp?) do
+					request = Droid::Request.new(self, header, message)
+					log.info "amqp_message #{tag} action=received ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+					begin
+						raise Droid::ExpiredMessage if request.expired?
+						yield request if block_given?
+						finished = Time.now.getgm.to_i
+						log.info "amqp_message action=processed #{tag} elapsed=#{finished-request.start} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+					rescue Droid::ExpiredMessage
+						log.info "amqp_message action=timeout #{tag} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+						request.ack if amqp_opts[:ack]
+					rescue => e
+						request.ack if amqp_opts[:ack]
+						Droid.handle_error(e)
+					end
+				end
+			end
+			log.info "amqp_subscribe #{tag}"
+			self
+		end
+
+		def teardown
+			@q.unsubscribe
+			@mq.close
+			log.info "amqp_unsubscribe #{tag}"
+		end
+
+		def destroy
+			teardown
+		end
+	end
+
+	class WorkerQueue < BaseQueue
+		attr_reader :prefetch
+
+		def initialize(queue_name, opts={})
+			@prefetch = opts.delete(:prefetch) || 1
+			opts[:auto_delete] = false
+
+			super(queue_name, opts)
+		end
+
+		def setup
+			super
+			@mq.prefetch(self.prefetch)
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			amqp_opts[:ack] = true
+			super(amqp_opts, opts) do |request|
+				begin
+					yield request if block_given?
+				ensure
+					request.ack unless amqp_opts[:auto_ack] == false
+				end
+			end
+		end
+	end
+
+	class ListenQueue < BaseQueue
+		def initialize(exchange_name, opts={})
+			opts[:exchange_name] = exchange_name
+			queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
+			super(queue_name, opts)
+		end
+	end
+
+	class ReplyQueue < BaseQueue
+		def initialize(queue_name, opts={})
+			opts[:auto_delete] = true
+			super
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			@ex = nil
+		end
+
+		def temp?
+			true
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			super(amqp_opts, opts) do |request|
+				yield request if block_given?
+				self.destroy
+			end
+		end
+
+		def teardown
+			@q.delete
+			super
+		end
+	end
+
+	module QueueMethods
+		def worker(queue_name, opts={})
+			WorkerQueue.new(queue_name, opts)
+		end
+
+		def listener(exchange_name, opts={})
+			ListenQueue.new(exchange_name, opts)
+		end
+	end
+
+	class BackwardsCompatibleQueue < BaseQueue
+		def initialize(exchange_name, opts={})
+			opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
+			opts[:exchange_name] = exchange_name
+			queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
+			@queue_name, @opts = queue_name, opts
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			@ex = @mq.direct(opts[:exchange_name])
+
+			@mq.prefetch(opts[:prefetch]) if opts[:prefetch]
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			super(amqp_opts, opts) do |request|
+				if block_given?
+					if opts[:detail]
+						yield request, request.header, request.raw_message if block_given?
+					else
+						yield request if block_given?
+					end
+				end
+			end
+
+			self
+		end
+	end
+
+	module BackwardsCompatibleMethods
+		def listen4(key, orig_opts={}, &block)
+			opts = {}
+			amqp_opts = {}
+			subscribe_opts = {}
+
+			if orig_opts[:prefetch] || orig_opts[:ack]
+				opts[:prefetch] = orig_opts[:prefetch] || 1
+				opts[:ack] = true
+			end
+			if orig_opts[:queue]
+				opts[:queue] = orig_opts[:queue]
+			end
+			if orig_opts[:detail]
+				subscribe_opts[:detail] = true
+			end
+
+			BackwardsCompatibleQueue.new(key, opts).subscribe(amqp_opts, subscribe_opts, &block)
+		end
+	end
+end

data/lib/droid/request.rb

@@ -0,0 +1,110 @@
+require 'droid/utils'
+
+class Droid
+	class UnknownReplyTo < RuntimeError; end
+
+	class Request
+		attr_reader :qobj, :header, :raw_message
+		attr_reader :droid_headers, :msg, :start
+
+		def initialize(qobj, header, raw_message)
+			@qobj = qobj
+			@header = header
+			@raw_message = raw_message
+
+			@droid_headers = Droid::Utils.parse_custom_headers(header.headers)
+			@msg = Droid::Utils.parse_message(raw_message)
+
+			@start = Time.now.getgm.to_i
+
+			@acked = false
+		end
+
+		def q
+			qobj.q
+		end
+
+		def ex
+			qobj.ex
+		end
+
+		def mq
+			qobj.mq
+		end
+
+		def [](field)
+			msg[field.to_s]
+		end
+
+		alias :params :msg
+
+		def age
+			return -1 unless droid_headers[:published_on]
+			start - droid_headers[:published_on]
+		end
+
+		def ttl
+			droid_headers[:ttl] || -1
+		end
+
+		def expired?
+			return false if ttl == -1
+			age > ttl
+		end
+
+		def acked?
+			@acked == true
+		end
+
+		def ack
+			return if acked?
+			header.ack
+			@acked = true
+		end
+
+		def reply(data, opts={}, popts={})
+			opts.merge!(default_publish_opts)
+			reply_to = droid_headers[:reply_to] || self.msg['reply_to']
+			raise UnknownReplyTo unless reply_to
+			Droid.publish_to_q(reply_to, data, opts, popts)
+		end
+
+		def publish(name, data, opts={}, popts={}, &block)
+			opts = default_publish_opts.merge(opts)
+			if block
+				opts[:reply_to] ||= Droid::Utils.generate_reply_to(name)
+				Droid::ReplyQueue.new(opts[:reply_to]).subscribe(&block)
+			end
+			Droid.publish(name, data, opts, popts)
+		end
+
+		def requeue(ropts={})
+			h = droid_headers.dup
+			h[:requeued] = true
+			h.delete(:ttl)
+			popts = { :headers => h }
+			ropts[:ttl] ||= 10
+			Droid.publish_to_q(q.name, msg, ropts, popts)
+		end
+
+		def defer(&blk)
+			EM.defer(lambda do
+				begin
+					blk.call
+				rescue => e
+					Droid.handle_error(e)
+				end
+			end)
+		end
+
+		def default_publish_opts
+			{
+				:event_hash => droid_headers[:event_hash]
+			}
+		end
+
+		def data_summary
+			Droid::Utils.format_data_summary(msg, droid_headers)
+		end
+	end
+end