droid 0.9.5 → 1.0.0

data/lib/droid/sync.rb

@@ -0,0 +1,88 @@
+require 'droid'
+require 'bunny'
+require 'system_timer'
+
+class Droid
+	class SyncException < RuntimeError; end
+
+	def self.reset_bunny
+		@@bunny = nil
+	end
+
+	def self.bunny
+		@@bunny ||= begin
+			b = Bunny.new(default_config)
+			b.start
+			b
+		end
+	end
+
+	def self.start(name, opts={})
+		raise SyncException, "start block is not allowed under sync operation"
+	end
+
+	def self.reconnect_on_error
+		SystemTimer::timeout(20) do
+			begin
+				yield if block_given?
+			rescue Bunny::ProtocolError
+				sleep 0.5
+				retry
+			rescue Bunny::ConnectionError
+				sleep 0.5
+				reset_bunny
+				retry
+			rescue Bunny::ServerDownError
+				sleep 0.5
+				reset_bunny
+				retry
+			end
+		end
+	end
+
+	def self.pop(q)
+		begin
+			loop do
+				result = q.pop
+				result = result[:payload] if result.is_a?(Hash)
+				return JSON.parse(result) unless result == :queue_empty
+				sleep 0.1
+			end
+		end
+	end
+
+	def self.call(queue_name, data, opts={}, popts={})
+		opts[:reply_to] ||= Droid::Utils.generate_reply_to(queue_name)
+		q = nil
+		begin
+			reconnect_on_error do
+				q = bunny.queue(opts[:reply_to], :auto_delete => true)
+				publish_to_ex(queue_name, data, opts, popts)
+				pop(q)
+			end
+		ensure
+			if q
+				q.delete rescue nil
+			end
+		end
+	end
+
+	# override publish methods
+	def self.publish_to_q(queue_name, data, opts={}, popts={})
+		reconnect_on_error do
+			q = bunny.queue(queue_name)
+			json, popts = Droid::Utils.format_publish(data, opts, popts)
+			q.publish(json, popts)
+		end
+		log.info "amqp_publish queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	def self.publish_to_ex(ex_name, data, opts={}, popts={})
+		reconnect_on_error do
+			ex = bunny.exchange(ex_name)
+			json, popts = Droid::Utils.format_publish(data, opts, popts)
+			ex.publish(json, popts)
+		end
+		log.info "amqp_publish exchange=#{ex_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+end

data/lib/droid/utilization.rb

@@ -0,0 +1,113 @@
+class Droid
+	module Utilization
+		extend self
+
+		@latency = 0.0
+		def latency=(val); @latency = val; end
+		def latency; @latency; end
+
+		@@start = Time.now
+		@@data = { }
+
+		def start
+			@@start
+		end
+
+		def reinit
+			@@start = Time.now
+			@@data = { }
+		end
+
+		def data(topic)
+			@@data[topic] ||= {
+				'msgs' => 0,
+				'time' => 0.0
+			}
+			@@data[topic]
+		end
+
+		def topics
+			@@data.keys
+		end
+
+		def record(topic, secs)
+			d = data(topic)
+			d['msgs'] += 1
+			d['time'] += secs
+		end
+
+		def calc_utilization(topic, t2=nil)
+			d = data(topic)
+			t1 = @@start
+			t2 ||= Time.now
+			secs = (t2 - t1)
+			secs = 1 if secs <= 0.0
+			if d['msgs'] == 0
+				avg = 0.0
+			else
+				avg = d['time'] / d['msgs']
+			end
+			utilization = d['time'] / secs
+			{
+				'avg' => avg,
+				'secs' => secs,
+				'utilization' => utilization,
+				'msgs' => d['msgs'],
+				'msgs_per_sec' => d['msgs'] / secs
+			}
+		end
+
+		def monitor(topic, opts={})
+			topic = 'temporary' if opts[:temp]
+
+			t1 = Time.now
+			begin
+				yield if block_given?
+			ensure
+				t2 = Time.now
+				record(topic, t2 - t1)
+			end
+		end
+
+		def report_data
+			data = {}
+			t2 = Time.now
+			topics.each do |topic|
+				data[topic] = calc_utilization(topic, t2)
+			end
+			data
+		end
+
+		def report_summary(data=nil)
+			data ||= report_data
+
+			summary = {
+				'avg' => 0.0,
+				'utilization' => 0.0,
+				'msgs' => 0,
+				'msgs_per_sec' => 0.0,
+				'secs' => 0.0
+			}
+			data.each do |topic, d|
+				summary['utilization'] += d['utilization']
+				summary['msgs'] += d['msgs']
+				summary['msgs_per_sec'] += d['msgs_per_sec']
+				summary['avg'] += d['avg']
+				summary['secs'] += d['secs']
+			end
+			if data.size < 1
+				summary['avg'] = 0.0
+			else
+				summary['avg'] /= data.size
+			end
+
+			summary
+		end
+
+		def report
+			data = report_data
+			summary = report_summary(data)
+			{ 'data' => data, 'summary' => summary }
+		end
+	end
+end

data/lib/droid/utils.rb

@@ -0,0 +1,113 @@
+require 'socket'
+require 'digest/md5'
+
+class Droid
+	DEFAULT_TTL = 300
+
+	class BadPayload < RuntimeError; end
+
+	module Utils
+		def self.parse_message(raw)
+			return { } unless raw
+			JSON.parse(raw)
+		end
+
+		def self.parse_custom_headers(headers)
+			return { } unless headers
+
+			h = headers.dup
+
+			h[:published_on] = h[:published_on].to_i
+
+			h[:ttl] = h[:ttl].to_i rescue -1
+			h[:ttl] = -1 if h[:ttl] == 0
+
+			h
+		end
+
+		def self.create_event_hash
+			s = Time.now.to_s + self.object_id.to_s + rand(100).to_s
+			'd' + Digest::MD5.hexdigest(s)
+		end
+
+		def self.extract_custom_headers(hash, opts={}, popts={})
+			popts[:headers] ||= {}
+			headers = popts[:headers]
+
+			headers[:published_on] ||= hash.delete('published_on') || opts[:published_on] || Time.now.getgm.to_i
+			headers[:ttl] ||= hash.delete('ttl') || (opts[:ttl] || Droid::DEFAULT_TTL).to_i
+			headers[:reply_to] ||= opts[:reply_to] if opts[:reply_to]
+
+			# this is the event hash that gets transferred through various publish/reply actions
+			headers[:event_hash] ||= hash.delete('event_hash') || opts[:event_hash] || create_event_hash
+
+			# this value should be unique for each published/received message pair
+			headers[:message_id] ||= create_event_hash
+
+			# some strange behavior with integers makes it better to
+			# convert all amqp headers to strings to avoid any problems
+			headers.each { |k,v| headers[k] = v.to_s }
+
+			[hash, headers]
+		end
+
+		def self.format_publish(data, opts={}, popts={})
+			raise Droid::BadPayload unless data.is_a?(Hash)
+
+			hash, headers = extract_custom_headers(data, opts, popts)
+
+			popts[:content_type] ||= 'application/json'
+
+			[hash.to_json, popts]
+		end
+
+		def self.generate_queue(exchange_name, second_name=nil)
+			second_name ||= $$
+			"#{generate_name_for_instance(exchange_name)}.#{second_name}"
+		end
+
+		def self.generate_name_for_instance(name)
+			"#{name}.#{Socket.gethostname}"
+		end
+
+		def self.generate_reply_to(name)
+			"temp.reply.#{name}.#{self.generate_sym}"
+		end
+
+		def self.generate_sym
+			values = [
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x0010000),
+				rand(0x1000000),
+				rand(0x1000000),
+			]
+			"%04x%04x%04x%04x%04x%06x%06x" % values
+		end
+
+		def self.data_summary(json)
+			return '-> (empty)' if json.empty?
+			summary = json.map do |k,v|
+				v = v.to_s
+				v = v[0..37] + '...' if v.size > 40
+				"#{k}=#{v}"
+			end.join(', ')
+			"-> #{summary}"
+		end
+
+		def self.format_data_summary(data, headers)
+			"(data) " + Droid::Utils.data_summary(data) + " (headers) " + Droid::Utils.data_summary(headers)
+		end
+	end
+
+	# Need this to be backwards compatible
+	def self.gen_queue(droid, name)
+		dn = droid
+		dn = dn.name if dn.respond_to?(:name)
+		dn ||= "d"
+		dn = dn.gsub(" ", "")
+		Droid::Utils.generate_queue(name, droid)
+	end
+end

data/lib/heroku_droid.rb

@@ -1,86 +1 @@
-require 'rubygems'
-require 'time'
-require 'uri'
-
-require File.dirname(__FILE__) + '/droid'
-require File.dirname(__FILE__) + '/local_stats'
-
-require File.dirname(__FILE__) + '/../vendor/logger_client/init'
-Log.configure do |c|
-	slot = LocalStats.slot
-	slot = 'railgun' if slot == 'userapps'
-	c.component = slot
-	c.instance_name = LocalStats.this_instance_name
-end
-
-class HerokuDroid < Droid
-	attr_reader :extended_stats
-	attr_reader :force_syslog
-
-	def initialize(name, options = {}, &block)
-		@name = name
-		@force_syslog = (options[:force_syslog] == true)
-		@extended_stats = !(options[:extended_stats] == false)
-		Droid.new(name, self.class.default_options) do |d|
-			standard_topics(d) unless options[:standard_topics] == false
-			LocalStats.attach
-			block.call(d)
-		end
-	end
-
-	def ruby_path
-		if File.exists?("/usr/ruby1.8.6/bin/ruby")
-			"/usr/ruby1.8.6/bin"
-		else
-			"/usr/local/bin"
-		end
-	end
-
-	def standard_topics(droid)
-		queue_name = Droid.gen_queue(droid, 'ping') + ".pid#{Process.pid}"
-		droid.listen4('ping', :queue => queue_name) do |msg|
-			Utilization.latency = (Time.now.to_f - msg['departed_at']).abs
-		end
-
-		blk = Proc.new do |d|
-			begin
-				t1 = Time.now
-				response = {}.merge(LocalStats.stats)
-
-				estats = nil
-				if self.extended_stats
-					estats = droid.call_stats
-					estats = { :notes => estats } unless estats.kind_of?(Hash)
-					estats[:notes] ||= estats.map { |k, v| "#{v} #{k}" }.join(", ")
-					estats.merge!(LocalStats.extended_stats)
-				end
-
-				response.merge!({
-					:extended_stats => estats,
-					:droid_name => name,
-					:utilization => Utilization.report,
-					:latency => Utilization.latency,
-				})
-
-				d.publish('pong', response.merge(:stat_collection => (Time.now - t1)))
-			rescue Object => e
-				Log.error "Ping Block Error: #{e.class} -> #{e.message}", :exception => e
-				STDERR.puts e.backtrace.join("\n") + "\n"
-			end
-		end
-
-		blk.call(droid)
-		EM.add_periodic_timer(50 + (rand*15).to_i) { blk.call(droid) }
-		EM.add_periodic_timer(360) { Utilization.reinit }
-
-		return unless LocalStats.slot == self.name.downcase or self.force_syslog
-	end
-
-	def self.publish_oneshot(topic, params={})
-		HerokuDroid.new('One-shot publish') do |droid|
-			droid.publish(topic, params)
-			HerokuDroid.stop_safe
-		end
-	end
-end
-
+require 'droid/heroku'

data/lib/local_stats.rb

@@ -1,143 +1 @@
-require 'rest_client'
-
-module LocalStats
-	extend self
-
-	def load_avg
-		File.read('/proc/loadavg').split(' ', 2).first.to_f
-	end
-
-	def memory_info
-		info = {}
-		File.read('/proc/meminfo').split("\n").each do |line|
-			name, value = line.split(/:\s+/, 2)
-			info[name] = value.to_i
-		end
-		info
-	end
-
-	def memory_use
-		info = memory_info
-		used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
-		(100 * used / info['MemTotal']).to_i
-	end
-
-	def swap_use
-		_, total, use, _ = `free | grep Swap:`.strip.split
-		return 0 if total.to_i == 0
-		(100 * use.to_i / total.to_i).to_i
-	end
-
-	def local_ip
-		@local_ip ||= fetch_local_ip
-	end
-
-	def fetch_local_ip
-		`/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
-	end
-
-	def public_ip
-		retries = 0
-		@public_ip ||= begin
-			RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").strip
-		rescue RestClient::RequestTimeout => e
-			retries += 1
-			raise if retries > 5
-			sleep 1
-			retry
-		end
-	end
-
-	def disk_stats
-		raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ##  " $5}'`.split("\n").collect { |d| d }
-		raw.shift
-		raw.collect do |d|
-			tokens = d.split("##").collect { |t| t.strip }
-			{
-				:device => tokens[0],
-				:path => tokens[1],
-				:usage => tokens[2].gsub('%','').to_i
-			}
-		end
-	end
-
-	def ion_instance_id
-		@ion_instnace_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
-	end
-
-	def slot
-		@slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
-	end
-
-	def this_instance_name
-		return nil if slot.nil? or ion_instance_id.nil?
-		"#{slot}.#{ion_instance_id}"
-	end
-
-	def stats
-		{
-			:slot            => slot,
-			:ion_id          => ion_instance_id,
-			:load_avg        => load_avg,
-			:memory_use      => memory_use,
-			:swap_use        => swap_use,
-			:local_ip        => local_ip,
-			:instance_name   => this_instance_name,
-			:public_ip       => public_ip,
-			:net_in_rate     => receive_rate,
-			:net_out_rate    => transmit_rate
-		}
-	end
-
-	def extended_stats
-		{
-			:disk_stats => disk_stats
-		}
-	end
-
-	# sample ifstat every 30 seconds
-	IFSTAT_INTERVAL = 30
-
-	# bytes received per second and bytes transmitted per
-	# second as a two tuple
-	def transfer_rates
-		[receive_rate, transmit_rate]
-	end
-
-	# bytes received per second
-	def receive_rate
-		@rxrate || 0
-	end
-
-	# bytes transmitted per second
-	def transmit_rate
-		@txrate || 0
-	end
-
-	# sample the current RX and TX bytes from ifconfig and
-	# return as a two-tuple.
-	def sample
-		data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
-		[data[1].to_i, data[2].to_i]
-	rescue => boom
-		Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
-	end
-
-	def update_counters
-		rx, tx = sample
-		@rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
-		@txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
-		@rx, @tx = rx, tx
-	end
-
-	# called when a droid starts up - setup timers and whatnot
-	def attach
-		@rx, @tx, @rxrate, @txrate = nil
-		update_counters
-		EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
-	end
-
-	def ifconfig
-		`/sbin/ifconfig eth0`
-	end
-end
+require 'droid/heroku/local_stats'