droid 0.9.5 → 1.0.0

data/lib/droid/em.rb

@@ -0,0 +1,55 @@
+class Droid
+	module EMTimerUtils
+		def self.included(base)
+			base.extend(ClassMethods)
+		end
+
+		module ClassMethods
+			# Trap exceptions leaving the block and log them. Do not re-raise
+			def trap_exceptions
+				yield
+			rescue => e
+				em_exception(e)
+			end
+
+			def em_exception(e)
+				msg = format_em_exception(e)
+				log.error "[EM.timer] #{msg}", :exception => e
+			end
+
+			def format_em_exception(e)
+				# avoid backtrace in /usr or vendor if possible
+				system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
+				reordered_backtrace = app + system
+
+				# avoid "/" as the method name (we want the controller action)
+				row = 0
+				row = 1 if reordered_backtrace[row].match(/in `\/'$/)
+
+				# get file and method name
+				begin
+					file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
+					file.gsub!(/.*\//, '')
+					"#{e.class} in #{file} #{method}: #{e.message}"
+				rescue
+					"#{e.class} in #{e.backtrace.first}: #{e.message}"
+				end
+			end
+
+			# One-shot timer
+			def timer(duration, &blk)
+				EM.add_timer(duration) { trap_exceptions(&blk) }
+			end
+
+			# Add a periodic timer. If the now argument is true, run the block
+			# immediately in addition to scheduling the periodic timer.
+			def periodic_timer(duration, now=false, &blk)
+				timer(1, &blk) if now
+				EM.add_periodic_timer(duration) { trap_exceptions(&blk) }
+			end
+		end
+
+		def timer(*args, &blk); self.class.timer(*args, &blk); end
+		def periodic_timer(*args, &blk); self.class.periodic_timer(*args, &blk); end
+	end
+end

data/lib/droid/heroku.rb

@@ -0,0 +1,102 @@
+require 'droid'
+require 'droid/heroku/local_stats'
+require 'droid/heroku/logger_client'
+
+Log.configure do |c|
+	c.component = LocalStats.slot
+	c.instance_name = LocalStats.instance_name
+end
+
+Droid.log = Log
+
+class Droid
+	module Utils
+		def self.generate_name_for_instance(name)
+			"#{name}.#{LocalStats.slot}.#{LocalStats.ion_instance_id}"
+		end
+	end
+end
+
+begin
+	require 'droid/json_server'
+rescue LoadError => e
+	Droid.log.error "Could not load JSONServer", :exception => e
+end
+
+class HerokuDroid < Droid
+	attr_reader :extended_stats
+
+	def initialize(name, opts={}, &blk)
+		@extended_stats = !(opts[:extended_stats] == false)
+
+		super(name, opts) do |droid|
+			setup_standard_topics(self) unless opts[:standard_topics] == false
+			LocalStats.attach
+			blk.call(self)
+		end
+	end
+
+	def stats(&blk)
+		@stats = blk
+		out = call_stats
+		out = out.inspect unless out.kind_of?(String)
+		Log.notice out
+	end
+
+	def call_stats
+		@stats ? @stats.call : nil
+	end
+
+	def name
+		Droid.name
+	end
+
+	def ruby_path
+		if File.exists?("/usr/ruby1.8.6/bin/ruby")
+			"/usr/ruby1.8.6/bin"
+		else
+			"/usr/local/bin"
+		end
+	end
+
+	def setup_standard_topics(droid)
+		setup_ping_topic(droid)
+	end
+
+	def setup_ping_topic(droid)
+		require 'time'
+
+		droid.listener('ping').subscribe do |req|
+			Droid::Utilization.latency = (Time.now.to_f - req['departed_at']).abs
+		end
+
+		blk = Proc.new do |d|
+			begin
+				t1 = Time.now
+				response = {}.merge(LocalStats.stats)
+
+				estats = nil
+				if self.extended_stats
+					estats = droid.call_stats
+					estats = { :notes => estats } unless estats.kind_of?(Hash)
+					estats[:notes] ||= estats.map { |k, v| "#{v} #{k}" }.join(", ")
+					estats.merge!(LocalStats.extended_stats)
+				end
+
+				response.merge!({
+					:extended_stats => estats,
+					:droid_name => Droid.name,
+					:latency => Droid::Utilization.latency,
+				})
+
+				d.publish('pong', response.merge(:stat_collection => (Time.now - t1)))
+			rescue Object => e
+				log.error "Ping Block Error: #{e.class} -> #{e.message}", :exception => e
+			end
+		end
+
+		EM.add_timer(2) { blk.call(droid) }
+		EM.add_periodic_timer(50 + (rand*15).to_i) { blk.call(droid) }
+	end
+end
+

data/lib/droid/heroku/local_stats.rb

@@ -0,0 +1,145 @@
+require 'rest_client'
+
+module LocalStats
+	extend self
+
+	def load_avg
+		File.read('/proc/loadavg').split(' ', 2).first.to_f
+	end
+
+	def memory_info
+		info = {}
+		File.read('/proc/meminfo').split("\n").each do |line|
+			name, value = line.split(/:\s+/, 2)
+			info[name] = value.to_i
+		end
+		info
+	end
+
+	def memory_use
+		info = memory_info
+		used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
+		(100 * used / info['MemTotal']).to_i
+	end
+
+	def swap_use
+		_, total, use, _ = `free | grep Swap:`.strip.split
+		return 0 if total.to_i == 0
+		(100 * use.to_i / total.to_i).to_i
+	end
+
+	def local_ip
+		@local_ip ||= fetch_local_ip
+	end
+
+	def fetch_local_ip
+		`/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
+	end
+
+	def public_ip
+		retries = 0
+		@public_ip ||= begin
+			RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").to_s.strip
+		rescue RestClient::RequestTimeout => e
+			retries += 1
+			raise if retries > 5
+			sleep 1
+			retry
+		end
+	end
+
+	def disk_stats
+		raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ##  " $5}'`.split("\n").collect { |d| d }
+		raw.shift
+		raw.collect do |d|
+			tokens = d.split("##").collect { |t| t.strip }
+			{
+				:device => tokens[0],
+				:path => tokens[1],
+				:usage => tokens[2].gsub('%','').to_i
+			}
+		end
+	end
+
+	def ion_instance_id
+		@ion_instance_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
+	end
+
+	def slot
+		@slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
+	end
+
+	def instance_name
+		return nil if slot.nil? or ion_instance_id.nil?
+		"#{slot}.#{ion_instance_id}"
+	end
+
+	alias :this_instance_name :instance_name
+
+	def stats
+		{
+			:slot            => slot,
+			:ion_id          => ion_instance_id,
+			:load_avg        => load_avg,
+			:memory_use      => memory_use,
+			:swap_use        => swap_use,
+			:local_ip        => local_ip,
+			:instance_name   => this_instance_name,
+			:public_ip       => public_ip,
+			:net_in_rate     => receive_rate,
+			:net_out_rate    => transmit_rate
+		}
+	end
+
+	def extended_stats
+		{
+			:disk_stats => disk_stats
+		}
+	end
+
+	# sample ifstat every 30 seconds
+	IFSTAT_INTERVAL = 30
+
+	# bytes received per second and bytes transmitted per
+	# second as a two tuple
+	def transfer_rates
+		[receive_rate, transmit_rate]
+	end
+
+	# bytes received per second
+	def receive_rate
+		@rxrate || 0
+	end
+
+	# bytes transmitted per second
+	def transmit_rate
+		@txrate || 0
+	end
+
+	# sample the current RX and TX bytes from ifconfig and
+	# return as a two-tuple.
+	def sample
+		data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
+		[data[1].to_i, data[2].to_i]
+	rescue => boom
+		Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
+	end
+
+	def update_counters
+		rx, tx = sample
+		@rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
+		@txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
+		@rx, @tx = rx, tx
+	end
+
+	# called when a droid starts up - setup timers and whatnot
+	def attach
+		@rx, @tx, @rxrate, @txrate = nil
+		update_counters
+		EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
+	end
+
+	def ifconfig
+		`/sbin/ifconfig eth0`
+	end
+end

data/{vendor/logger_client/lib → lib/droid/heroku}/logger_client.rb

@@ -7,7 +7,6 @@
 #   config.failsafe = :file
 # end
 
-require 'rubygems'
 require 'time'
 require 'syslog'
 
@@ -21,7 +20,7 @@ end
 
 class Log::Config
 	def initialize
-		@contents = { :console_log => true }
+		@contents = { }
 	end
 
 	def method_missing(method, value)
@@ -53,6 +52,8 @@ module Log
 		log msg, options.merge(:level => 'notice')
 	end
 
+	alias :info :notice
+
 	def warning(msg, options={})
 		log msg, options.merge(:level => 'warning')
 	end
@@ -116,10 +117,13 @@ module Log
 		log "#{summary}\n#{body}", :level => 'error'
 	end
 
+	def console_puts(*args)
+		$stderr.puts(*args)
+	end
+
 	def exception(e)
 		msg = "Exception #{e.class} -> #{e.message}\n"
 		msg += filtered_backtrace(e.backtrace)
-		STDERR.puts msg
 		Log.error e.message, :exception => e
 	end
 
@@ -156,7 +160,7 @@ module Log
 	end
 
 	def default_options
-		@@default_options ||= {}
+		@@default_options ||= { :console_log => true }
 	end
 
 	def failsafe(params)
@@ -172,7 +176,7 @@ module Log
 	end
 
 	def console_log(msg)
-		STDERR.puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
+		console_puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
 	end
 
 	def syslog(msg, opts = {})

data/lib/droid/heroku/memcache_cluster.rb

@@ -0,0 +1,129 @@
+require 'memcache'
+require 'yaml'
+require 'digest/sha1'
+
+# Manages a pool of memcache servers. This class should not be called
+# outside of the reactor - it does not account for asynchronous access
+# to the server list.
+module MemcacheCluster
+	extend self
+
+	HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
+
+	# A MemCache object configured with heroku's internal memcache namespace.
+	def heroku
+		cache(HEROKU_NAMESPACE)
+	end
+
+	def cache_retry(prefix, opts={})
+		opts[:retries] ||= 5
+		opts[:delay] ||= 0.5
+
+		retried = 0
+		begin
+			c = cache(prefix)
+			yield c if block_given?
+		rescue MemCache::MemCacheError => e
+			Log.error "#{e.class} -> #{e.message}", :exception => e
+			raise if retried > opts[:retries]
+			retried += 1
+			sleep opts[:delay]
+			@caches = { }
+			retry
+		end
+	end
+
+	def set(prefix, *args)
+		res = nil
+		cache_retry(prefix) do |c|
+			res = c.set(*args)
+		end
+		res
+	end
+
+	def get(prefix, *args)
+		res = nil
+		cache_retry(prefix) do |c|
+			res = c.get(*args)
+		end
+		res
+	end
+
+	# Create listeners for standard memcache cluster related topics.
+	def attach(droid, file='memcached.yml')
+		load_from_file(file)
+
+		droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
+		droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
+		EM.add_timer(1) { droid.publish('memcache.needed', {}) }
+	end
+
+	# A MemCache object configured with the given prefix.
+	def cache(prefix, options={})
+		caches[prefix] ||=
+			MemCache.new(servers, options.merge(:namespace => prefix))
+	end
+
+	alias_method :[], :cache
+
+	def caches
+		reload_if_stale
+		@caches ||= {}
+	end
+
+	def servers
+		reload_if_stale
+		@servers ||= []
+	end
+
+	def add(ip, port)
+		host = [ip, port].join(':')
+		return if servers.include?(host)
+
+		log { "#{host} added" }
+		@servers.push host
+		@servers.sort!
+		@caches = {}
+		write_to_file
+		@last_read = Time.now
+	end
+
+	def remove(host)
+		if servers.reject!{ |s| s =~ /^#{host}/ }
+			log { "#{host} removed" }
+			caches.clear
+			write_to_file
+		end
+	end
+
+	def reload_if_stale
+		if @last_read &&
+			(Time.now - @last_read) > 5 &&
+			File.mtime(@file) > @last_read
+			log { "server list modified. reloading." }
+			load_from_file(@file)
+		end
+	rescue => boom
+		# ignore errors accessing/reading file.
+	end
+
+	def load_from_file(file)
+		@file = file
+		@last_read = Time.now
+		@servers = YAML.load(File.read(file)) rescue []
+		@caches = {}
+	end
+
+	def write_to_file
+		log { "writing server list: #{@file}" }
+		File.open(@file, 'w') do |f|
+			f.flock(File::LOCK_EX)
+			f.write YAML.dump(@servers)
+			f.flock(File::LOCK_UN)
+		end
+	end
+
+	def log(type=:debug)
+		Log.send(type, "memcached: #{yield}")
+	end
+end