droid 0.9.0

data/lib/heroku_droid.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'time'
+require 'uri'
+
+require File.dirname(__FILE__) + '/droid'
+require File.dirname(__FILE__) + '/local_stats'
+
+require File.dirname(__FILE__) + '/../vendor/logger_client/init'
+Log.configure do |c|
+	slot = LocalStats.slot
+	slot = 'railgun' if slot == 'userapps'
+	c.component = slot
+	c.instance_name = LocalStats.this_instance_name
+end
+
+class HerokuDroid < Droid
+	attr_reader :extended_stats
+	attr_reader :force_syslog
+
+	def initialize(name, options = {}, &block)
+		@name = name
+		@force_syslog = (options[:force_syslog] == true)
+		@extended_stats = !(options[:extended_stats] == false)
+		Droid.new(name, self.class.default_options) do |d|
+			standard_topics(d) unless options[:standard_topics] == false
+			LocalStats.attach
+			block.call(d)
+		end
+	end
+
+	def ruby_path
+		if File.exists?("/usr/ruby1.8.6/bin/ruby")
+			"/usr/ruby1.8.6/bin"
+		else
+			"/usr/local/bin"
+		end
+	end
+
+	def standard_topics(droid)
+		queue_name = Droid.gen_queue(droid, 'ping') + ".pid#{Process.pid}"
+		droid.listen4('ping', :queue => queue_name) do |msg|
+			Utilization.latency = (Time.now.to_f - msg['departed_at']).abs
+		end
+
+		blk = Proc.new do |d|
+			begin
+				t1 = Time.now
+				response = {}.merge(LocalStats.stats)
+
+				estats = nil
+				if self.extended_stats
+					estats = droid.call_stats
+					estats = { :notes => estats } unless estats.kind_of?(Hash)
+					estats[:notes] ||= estats.map { |k, v| "#{v} #{k}" }.join(", ")
+					estats.merge!(LocalStats.extended_stats)
+				end
+
+				response.merge!({
+					:extended_stats => estats,
+					:droid_name => name,
+					:utilization => Utilization.report,
+					:latency => Utilization.latency,
+				})
+
+				d.publish('pong', response.merge(:stat_collection => (Time.now - t1)))
+			rescue Object => e
+				Log.error "Ping Block Error: #{e.class} -> #{e.message}", :exception => e
+				STDERR.puts e.backtrace.join("\n") + "\n"
+			end
+		end
+
+		blk.call(droid)
+		EM.add_periodic_timer(50 + (rand*15).to_i) { blk.call(droid) }
+		EM.add_periodic_timer(360) { Utilization.reinit }
+
+		return unless LocalStats.slot == self.name.downcase or self.force_syslog
+	end
+
+	def self.publish_oneshot(topic, params={})
+		HerokuDroid.new('One-shot publish') do |droid|
+			droid.publish(topic, params)
+			HerokuDroid.stop_safe
+		end
+	end
+end
+

data/lib/local_stats.rb

@@ -0,0 +1,143 @@
+require 'rest_client'
+
+module LocalStats
+	extend self
+
+	def load_avg
+		File.read('/proc/loadavg').split(' ', 2).first.to_f
+	end
+
+	def memory_info
+		info = {}
+		File.read('/proc/meminfo').split("\n").each do |line|
+			name, value = line.split(/:\s+/, 2)
+			info[name] = value.to_i
+		end
+		info
+	end
+
+	def memory_use
+		info = memory_info
+		used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
+		(100 * used / info['MemTotal']).to_i
+	end
+
+	def swap_use
+		_, total, use, _ = `free | grep Swap:`.strip.split
+		return 0 if total.to_i == 0
+		(100 * use.to_i / total.to_i).to_i
+	end
+
+	def local_ip
+		@local_ip ||= fetch_local_ip
+	end
+
+	def fetch_local_ip
+		`/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
+	end
+
+	def public_ip
+		retries = 0
+		@public_ip ||= begin
+			RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").strip
+		rescue RestClient::RequestTimeout => e
+			retries += 1
+			raise if retries > 5
+			sleep 1
+			retry
+		end
+	end
+
+	def disk_stats
+		raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ##  " $5}'`.split("\n").collect { |d| d }
+		raw.shift
+		raw.collect do |d|
+			tokens = d.split("##").collect { |t| t.strip }
+			{
+				:device => tokens[0],
+				:path => tokens[1],
+				:usage => tokens[2].gsub('%','').to_i
+			}
+		end
+	end
+
+	def ion_instance_id
+		@ion_instnace_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
+	end
+
+	def slot
+		@slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
+	end
+
+	def this_instance_name
+		return nil if slot.nil? or ion_instance_id.nil?
+		"#{slot}.#{ion_instance_id}"
+	end
+
+	def stats
+		{
+			:slot            => slot,
+			:ion_id          => ion_instance_id,
+			:load_avg        => load_avg,
+			:memory_use      => memory_use,
+			:swap_use        => swap_use,
+			:local_ip        => local_ip,
+			:instance_name   => this_instance_name,
+			:public_ip       => public_ip,
+			:net_in_rate     => receive_rate,
+			:net_out_rate    => transmit_rate
+		}
+	end
+
+	def extended_stats
+		{
+			:disk_stats => disk_stats
+		}
+	end
+
+	# sample ifstat every 30 seconds
+	IFSTAT_INTERVAL = 30
+
+	# bytes received per second and bytes transmitted per
+	# second as a two tuple
+	def transfer_rates
+		[receive_rate, transmit_rate]
+	end
+
+	# bytes received per second
+	def receive_rate
+		@rxrate || 0
+	end
+
+	# bytes transmitted per second
+	def transmit_rate
+		@txrate || 0
+	end
+
+	# sample the current RX and TX bytes from ifconfig and
+	# return as a two-tuple.
+	def sample
+		data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
+		[data[1].to_i, data[2].to_i]
+	rescue => boom
+		Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
+	end
+
+	def update_counters
+		rx, tx = sample
+		@rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
+		@txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
+		@rx, @tx = rx, tx
+	end
+
+	# called when a droid starts up - setup timers and whatnot
+	def attach
+		@rx, @tx, @rxrate, @txrate = nil
+		update_counters
+		EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
+	end
+
+	def ifconfig
+		`/sbin/ifconfig eth0`
+	end
+end

data/lib/memcache_cluster.rb

@@ -0,0 +1,127 @@
+require 'memcache'
+require 'yaml'
+require 'digest/sha1'
+
+# Manages a pool of memcache servers. This class should not be called
+# outside of the reactor - it does not account for asynchronous access
+# to the server list.
+module MemcacheCluster
+	extend self
+
+	# A MemCache object configured with heroku's internal memcache namespace.
+	def heroku
+		cache('0Xfa15837Z')
+	end
+
+	def cache_retry(prefix, opts={})
+		opts[:retries] ||= 5
+		opts[:delay] ||= 0.5
+
+		retried = 0
+		begin
+			c = cache(prefix)
+			yield c if block_given?
+		rescue MemCache::MemCacheError => e
+			Log.error "#{e.class} -> #{e.message}", :exception => e
+			raise if retried > opts[:retries]
+			retried += 1
+			sleep opts[:delay]
+			@caches = { }
+			retry
+		end
+	end
+
+	def set(prefix, *args)
+		res = nil
+		cache_retry(prefix) do |c|
+			res = c.set(*args)
+		end
+		res
+	end
+
+	def get(prefix, *args)
+		res = nil
+		cache_retry(prefix) do |c|
+			res = c.get(*args)
+		end
+		res
+	end
+
+	# Create listeners for standard memcache cluster related topics.
+	def attach(droid, file='memcached.yml')
+		load_from_file(file)
+
+		droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
+		droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
+		EM.add_timer(1) { droid.publish('memcache.needed', {}) }
+	end
+
+	# A MemCache object configured with the given prefix.
+	def cache(prefix, options={})
+		caches[prefix] ||=
+			MemCache.new(servers, options.merge(:namespace => prefix))
+	end
+
+	alias_method :[], :cache
+
+	def caches
+		reload_if_stale
+		@caches ||= {}
+	end
+
+	def servers
+		reload_if_stale
+		@servers ||= []
+	end
+
+	def add(ip, port)
+		host = [ip, port].join(':')
+		return if servers.include?(host)
+
+		log { "#{host} added" }
+		@servers.push host
+		@servers.sort!
+		@caches = {}
+		write_to_file
+		@last_read = Time.now
+	end
+
+	def remove(host)
+		if servers.reject!{ |s| s =~ /^#{host}/ }
+			log { "#{host} removed" }
+			caches.clear
+			write_to_file
+		end
+	end
+
+	def reload_if_stale
+		if @last_read &&
+			(Time.now - @last_read) > 5 &&
+			File.mtime(@file) > @last_read
+			log { "server list modified. reloading." }
+			load_from_file(@file)
+		end
+	rescue => boom
+		# ignore errors accessing/reading file.
+	end
+
+	def load_from_file(file)
+		@file = file
+		@last_read = Time.now
+		@servers = YAML.load(File.read(file)) rescue []
+		@caches = {}
+	end
+
+	def write_to_file
+		log { "writing server list: #{@file}" }
+		File.open(@file, 'w') do |f|
+			f.flock(File::LOCK_EX)
+			f.write YAML.dump(@servers)
+			f.flock(File::LOCK_UN)
+		end
+	end
+
+	def log(type=:debug)
+		Log.send(type, "memcached: #{yield}")
+	end
+end

data/lib/stats.rb

@@ -0,0 +1,30 @@
+module Stats
+	# The MemCache instance used to manipulate stats.
+	def cache
+		MemcacheCluster.cache("heroku:stats")
+	end
+
+	# Increment a stat counter. If the counter does not exist,
+	# yield to the block and use the result as the current counter
+	# value. With no block, the counter will be started at zero.
+	def increment(key, amount=1)
+		if (value = cache.incr(key, amount)).nil?
+			value = yield if block_given?
+			value = (value || 0) + amount
+			cache.add(key, value.to_s, 0, true)
+		end
+	rescue => boom
+		Log.default_error(boom)
+		nil
+	end
+
+	# Set the stat counter to a specific value.
+	def sample(key, value)
+		cache.set(key, value.to_s, 0, true)
+	rescue => boom
+		Log.default_error(boom)
+		nil
+	end
+
+	extend self
+end

data/lib/utilization.rb

@@ -0,0 +1,90 @@
+module Utilization
+	extend self
+
+	@latency = 0.0
+	def latency=(val); @latency = val; end
+	def latency; @latency; end
+
+	@@start = Time.now
+	@@data = { }
+
+	def reinit
+		@@start = Time.now
+		@@data = { }
+	end
+
+	def data(topic)
+		@@data[topic] ||= {
+			'msgs' => 0,
+			'time' => 0.0
+		}
+		@@data[topic]
+	end
+
+	def topics
+		@@data.keys
+	end
+
+	def record(topic, secs)
+		d = data(topic)
+		d['msgs'] += 1
+		d['time'] += secs
+	end
+
+	def calc_utilization(topic, t2=nil)
+		d = data(topic)
+		t1 = @@start
+		t2 ||= Time.now
+		secs = (t2 - t1)
+		secs = 1 if secs <= 0.0
+		if d['msgs'] == 0
+			avg = 0.0
+		else
+			avg = d['time'] / d['msgs']
+		end
+		utilization = d['time'] / secs
+		{
+			:avg => avg,
+			:secs => secs,
+			:utilization => utilization,
+			:msgs => d['msgs'],
+			:msgs_per_sec => d['msgs'] / secs
+		}
+	end
+
+	def monitor(topic, opts={})
+		topic = 'temporary' if opts[:temp]
+
+		t1 = Time.now
+		begin
+			yield if block_given?
+		ensure
+			t2 = Time.now
+			record(topic, t2 - t1)
+		end
+	end
+
+	def report
+		data = {}
+		t2 = Time.now
+		topics.each do |topic|
+			data[topic] = calc_utilization(topic, t2)
+		end
+
+		summary = { :avg => 0.0, :utilization => 0.0, :msgs => 0, :msgs_per_sec => 0.0, :secs => 0.0 }
+		data.each do |topic, d|
+			summary[:utilization] += d[:utilization]
+			summary[:msgs] += d[:msgs]
+			summary[:msgs_per_sec] += d[:msgs_per_sec]
+			summary[:avg] += d[:avg]
+			summary[:secs] += d[:secs]
+		end
+		if data.size < 1
+			summary[:avg] = 0.0
+		else
+			summary[:avg] /= data.size
+		end
+
+		{ :summary => summary, :data => data }
+	end
+end