droid19 1.0.2

data/lib/droid/request.rb

@@ -0,0 +1,110 @@
+require 'droid/utils'
+
+class Droid
+	class UnknownReplyTo < RuntimeError; end
+
+	class Request
+		attr_reader :qobj, :header, :raw_message
+		attr_reader :droid_headers, :msg, :start
+
+		def initialize(qobj, header, raw_message)
+			@qobj = qobj
+			@header = header
+			@raw_message = raw_message
+
+			@droid_headers = Droid::Utils.parse_custom_headers(header.headers)
+			@msg = Droid::Utils.parse_message(raw_message)
+
+			@start = Time.now.getgm.to_i
+
+			@acked = false
+		end
+
+		def q
+			qobj.q
+		end
+
+		def ex
+			qobj.ex
+		end
+
+		def mq
+			qobj.mq
+		end
+
+		def [](field)
+			msg[field.to_s]
+		end
+
+		alias :params :msg
+
+		def age
+			return -1 unless droid_headers[:published_on]
+			start - droid_headers[:published_on]
+		end
+
+		def ttl
+			droid_headers[:ttl] || -1
+		end
+
+		def expired?
+			return false if ttl == -1
+			age > ttl
+		end
+
+		def acked?
+			@acked == true
+		end
+
+		def ack
+			return if acked?
+			header.ack
+			@acked = true
+		end
+
+		def reply(data, opts={}, popts={})
+			opts.merge!(default_publish_opts)
+			reply_to = droid_headers[:reply_to] || self.msg['reply_to']
+			raise UnknownReplyTo unless reply_to
+			Droid.reply_to_q(reply_to, data, opts, popts)
+		end
+
+		def publish(name, data, opts={}, popts={}, &block)
+			opts = default_publish_opts.merge(opts)
+			if block
+				opts[:reply_to] ||= Droid::Utils.generate_reply_to(name)
+				Droid::ReplyQueue.new(opts[:reply_to]).subscribe(&block)
+			end
+			Droid.publish(name, data, opts, popts)
+		end
+
+		def requeue(ropts={})
+			h = droid_headers.dup
+			h[:requeued] = true
+			h.delete(:ttl)
+			popts = { :headers => h }
+			ropts[:ttl] ||= 10
+			Droid.publish_to_q(q.name, msg, ropts, popts)
+		end
+
+		def defer(&blk)
+			EM.defer(lambda do
+				begin
+					blk.call
+				rescue => e
+					Droid.handle_error(e)
+				end
+			end)
+		end
+
+		def default_publish_opts
+			{
+				:event_hash => droid_headers[:event_hash]
+			}
+		end
+
+		def data_summary
+			Droid::Utils.format_data_summary(msg, droid_headers)
+		end
+	end
+end

data/lib/droid/sync.rb

@@ -0,0 +1,87 @@
+require 'droid'
+require 'bunny'
+
+class Droid
+	class SyncException < RuntimeError; end
+
+	def self.reset_bunny
+		@@bunny = nil
+	end
+
+	def self.bunny
+		@@bunny ||= begin
+			b = Bunny.new(default_config)
+			b.start
+			b
+		end
+	end
+
+	def self.start(name, opts={})
+		raise SyncException, "start block is not allowed under sync operation"
+	end
+
+	def self.reconnect_on_error
+		SystemTimer::timeout(20) do
+			begin
+				yield if block_given?
+			rescue Bunny::ProtocolError
+				sleep 0.5
+				retry
+			rescue Bunny::ConnectionError
+				sleep 0.5
+				reset_bunny
+				retry
+			rescue Bunny::ServerDownError
+				sleep 0.5
+				reset_bunny
+				retry
+			end
+		end
+	end
+
+	def self.pop(q)
+		begin
+			loop do
+				result = q.pop
+				result = result[:payload] if result.is_a?(Hash)
+				return JSON.parse(result) unless result == :queue_empty
+				sleep 0.1
+			end
+		end
+	end
+
+	def self.call(queue_name, data, opts={}, popts={})
+		opts[:reply_to] ||= Droid::Utils.generate_reply_to(queue_name)
+		q = nil
+		begin
+			reconnect_on_error do
+				q = bunny.queue(opts[:reply_to], :auto_delete => true)
+				publish_to_ex(queue_name, data, opts, popts)
+				pop(q)
+			end
+		ensure
+			if q
+				q.delete rescue nil
+			end
+		end
+	end
+
+	# override publish methods
+	def self.publish_to_q(queue_name, data, opts={}, popts={})
+		reconnect_on_error do
+			q = bunny.queue(queue_name)
+			json, popts = Droid::Utils.format_publish(data, opts, popts)
+			q.publish(json, popts)
+		end
+		log.info "amqp_publish queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	def self.publish_to_ex(ex_name, data, opts={}, popts={})
+		reconnect_on_error do
+			ex = bunny.exchange(ex_name)
+			json, popts = Droid::Utils.format_publish(data, opts, popts)
+			ex.publish(json, popts)
+		end
+		log.info "amqp_publish exchange=#{ex_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+end

data/lib/droid/utilization.rb

@@ -0,0 +1,113 @@
+class Droid
+	module Utilization
+		extend self
+
+		@latency = 0.0
+		def latency=(val); @latency = val; end
+		def latency; @latency; end
+
+		@@start = Time.now.getutc
+		@@data = { }
+
+		def start
+			@@start
+		end
+
+		def reinit
+			@@start = Time.now.getutc
+			@@data = { }
+		end
+
+		def data(topic)
+			@@data[topic] ||= {
+				'msgs' => 0,
+				'time' => 0.0
+			}
+			@@data[topic]
+		end
+
+		def topics
+			@@data.keys
+		end
+
+		def record(topic, secs)
+			d = data(topic)
+			d['msgs'] += 1
+			d['time'] += secs
+		end
+
+		def calc_utilization(topic, t2=nil)
+			d = data(topic)
+			t1 = @@start
+			t2 ||= Time.now.getutc.to_i
+			secs = (t2 - t1)
+			secs = 1 if secs <= 0.0
+			if d['msgs'] == 0
+				avg = 0.0
+			else
+				avg = d['time'] / d['msgs']
+			end
+			utilization = d['time'] / secs
+			{
+				'avg' => avg,
+				'secs' => secs,
+				'utilization' => utilization,
+				'msgs' => d['msgs'],
+				'msgs_per_sec' => d['msgs'] / secs
+			}
+		end
+
+		def monitor(topic, opts={})
+			topic = 'temporary' if opts[:temp]
+
+			t1 = Time.now.getutc
+			begin
+				yield if block_given?
+			ensure
+				t2 = Time.now.getutc
+				record(topic, t2 - t1)
+			end
+		end
+
+		def report_data
+			data = {}
+			t2 = Time.now.getutc
+			topics.each do |topic|
+				data[topic] = calc_utilization(topic, t2)
+			end
+			data
+		end
+
+		def report_summary(data=nil)
+			data ||= report_data
+
+			summary = {
+				'avg' => 0.0,
+				'utilization' => 0.0,
+				'msgs' => 0,
+				'msgs_per_sec' => 0.0,
+				'secs' => 0.0
+			}
+			data.each do |topic, d|
+				summary['utilization'] += d['utilization']
+				summary['msgs'] += d['msgs']
+				summary['msgs_per_sec'] += d['msgs_per_sec']
+				summary['avg'] += d['avg']
+				summary['secs'] += d['secs']
+			end
+			if data.size < 1
+				summary['avg'] = 0.0
+			else
+				summary['avg'] /= data.size
+			end
+
+			summary
+		end
+
+		def report
+			data = report_data
+			summary = report_summary(data)
+			{ 'data' => data, 'summary' => summary }
+		end
+	end
+end

data/lib/droid/utils.rb

@@ -0,0 +1,113 @@
+require 'socket'
+require 'digest/md5'
+
+class Droid
+	DEFAULT_TTL = 300
+
+	class BadPayload < RuntimeError; end
+
+	module Utils
+		def self.parse_message(raw)
+			return { } unless raw
+			JSON.parse(raw)
+		end
+
+		def self.parse_custom_headers(headers)
+			return { } unless headers
+
+			h = headers.dup
+
+			h[:published_on] = h[:published_on].to_i
+
+			h[:ttl] = h[:ttl].to_i rescue -1
+			h[:ttl] = -1 if h[:ttl] == 0
+
+			h
+		end
+
+		def self.create_event_hash
+			s = Time.now.to_s + self.object_id.to_s + rand(100).to_s
+			'd' + Digest::MD5.hexdigest(s)
+		end
+
+		def self.extract_custom_headers(hash, opts={}, popts={})
+			popts[:headers] ||= {}
+			headers = popts[:headers]
+
+			headers[:published_on] ||= hash.delete('published_on') || opts[:published_on] || Time.now.getgm.to_i
+			headers[:ttl] ||= hash.delete('ttl') || (opts[:ttl] || Droid::DEFAULT_TTL).to_i
+			headers[:reply_to] ||= opts[:reply_to] if opts[:reply_to]
+
+			# this is the event hash that gets transferred through various publish/reply actions
+			headers[:event_hash] ||= hash.delete('event_hash') || opts[:event_hash] || create_event_hash
+
+			# this value should be unique for each published/received message pair
+			headers[:message_id] ||= create_event_hash
+
+			# some strange behavior with integers makes it better to
+			# convert all amqp headers to strings to avoid any problems
+			headers.each { |k,v| headers[k] = v.to_s }
+
+			[hash, headers]
+		end
+
+		def self.format_publish(data, opts={}, popts={})
+			raise Droid::BadPayload unless data.is_a?(Hash)
+
+			hash, headers = extract_custom_headers(data, opts, popts)
+
+			popts[:content_type] ||= 'application/json'
+
+			[hash.to_json, popts]
+		end
+
+		def self.generate_queue(exchange_name, second_name=nil)
+			second_name ||= $$
+			"#{generate_name_for_instance(exchange_name)}.#{second_name}"
+		end
+
+		def self.generate_name_for_instance(name)
+			"#{name}.#{Socket.gethostname}"
+		end
+
+		def self.generate_reply_to(name)
+			"temp.reply.#{name}.#{self.generate_sym}"
+		end
+
+		def self.generate_sym
+			values = [
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x1000000),
+				rand(0x1000000),
+			]
+			"%04x%04x%04x%04x%04x%06x%06x" % values
+		end
+
+		def self.data_summary(json)
+			return '-> (empty)' if json.empty?
+			summary = json.map do |k,v|
+				v = v.to_s
+				v = v[0..37] + '...' if v.size > 40
+				"#{k}=#{v}"
+			end.join(', ')
+			"-> #{summary}"
+		end
+
+		def self.format_data_summary(data, headers)
+			"(data) " + Droid::Utils.data_summary(data) + " (headers) " + Droid::Utils.data_summary(headers)
+		end
+	end
+
+	# Need this to be backwards compatible
+	def self.gen_queue(droid, name)
+		dn = droid
+		dn = dn.name if dn.respond_to?(:name)
+		dn ||= "d"
+		dn = dn.gsub(" ", "")
+		Droid::Utils.generate_queue(name, droid)
+	end
+end

data/lib/heroku_droid.rb

@@ -0,0 +1 @@
+require 'droid/heroku'

data/lib/local_stats.rb

@@ -0,0 +1 @@
+require 'droid/heroku/local_stats'

data/lib/memcache_cluster.rb

@@ -0,0 +1 @@
+require 'droid/heroku/memcache_cluster'

data/lib/stats.rb

@@ -0,0 +1 @@
+require 'droid/heroku/stats'

data/spec/publish_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe 'AMQP Publish' do
+	before do
+		@json, @publish_opts = Droid::Utils.format_publish({:x => 1, :y => 2}, {}, {})
+		Droid::Utils.stubs(:format_publish).with({:x => 1, :y => 2}, {}, {}).returns([@json, @publish_opts])
+	end
+
+	it "publishes a message to a queue" do
+		@q = mock('queue')
+		::MQ.expects(:queue).with('topic').returns(@q)
+		@q.expects(:publish).with(@json, @publish_opts)
+		Droid.publish_to_q('topic', :x => 1, :y => 2)
+	end
+
+	it "publishes a message to an exchange" do
+		@ex = mock('exchange')
+		::MQ.expects(:direct).with('topic').returns(@ex)
+		@ex.expects(:publish).with(@json, @publish_opts)
+		Droid.publish_to_ex('topic', :x => 1, :y => 2)
+	end
+
+	it "by default publishes to an exchange" do
+		Droid.expects(:publish_to_ex).with('topic', {:x => 1, :y => 2}, {}, {})
+		Droid.publish('topic', :x => 1, :y => 2)
+	end
+end