droid19 1.0.2

data/lib/droid/heroku/memcache_cluster.rb

@@ -0,0 +1,152 @@
+require 'memcache'
+require 'yaml'
+require 'digest/sha1'
+
+# Manages a pool of memcache servers. This class should not be called
+# outside of the reactor - it does not account for asynchronous access
+# to the server list.
+module MemcacheCluster
+  extend self
+
+  HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
+
+  class Proxy
+    def initialize(name, options={})
+      @name = name
+      @options = options
+    end
+
+    def set(*args)
+      MemcacheCluster.cache_internal(@name, @options).set(*args)
+    end
+
+    def get(*args)
+      MemcacheCluster.cache_internal(@name, @options).get(*args)
+    end
+
+    def method_missing(method_name, *args)
+      MemcacheCluster.cache_internal(@name, @options).send(method_name, *args)
+    end
+  end
+
+  # A MemCache object configured with heroku's internal memcache namespace.
+  def heroku
+    cache(HEROKU_NAMESPACE)
+  end
+
+  def cache_retry(prefix, opts={})
+    opts[:retries] ||= 5
+    opts[:delay] ||= 0.5
+
+    retried = 0
+    begin
+      c = cache_internal(prefix)
+      yield c if block_given?
+    rescue MemCache::MemCacheError => e
+      Log.error "#{e.class} -> #{e.message}", :exception => e
+      raise if retried > opts[:retries]
+      retried += 1
+      sleep opts[:delay]
+      @caches = { }
+      retry
+    end
+  end
+
+  def set(prefix, *args)
+    res = nil
+    cache_retry(prefix) do |c|
+      res = c.set(*args)
+    end
+    res
+  end
+
+  def get(prefix, *args)
+    res = nil
+    cache_retry(prefix) do |c|
+      res = c.get(*args)
+    end
+    res
+  end
+
+  # Create listeners for standard memcache cluster related topics.
+  def attach(droid, file='memcached.yml')
+    load_from_file(file)
+
+    droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
+    droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
+    EM.add_timer(1) { droid.publish('memcache.needed', {}) }
+  end
+
+  # A MemCache object configured with the given prefix.
+  def cache_internal(prefix, options={})
+    caches[prefix] ||=
+      MemCache.new(servers, options.merge(:namespace => prefix))
+  end
+
+  def cache(prefix, options={})
+    Proxy.new(prefix, options)
+  end
+
+  alias_method :[], :cache
+
+  def caches
+    reload_if_stale
+    @caches ||= {}
+  end
+
+  def servers
+    reload_if_stale
+    @servers ||= []
+  end
+
+  def add(ip, port)
+    host = [ip, port].join(':')
+    return if servers.include?(host)
+
+    log { "action=added server=#{host}" }
+    @servers.push host
+    @servers.sort!
+    @caches = {}
+    write_to_file
+    @last_read = Time.now.to_i
+  end
+
+  def remove(host)
+    if servers.reject!{ |s| s =~ /^#{host}/ }
+      log { "action=remove server=#{host}" }
+      caches.clear
+      write_to_file
+    end
+  end
+
+  def reload_if_stale
+    if @last_read &&
+      (Time.now.to_i - @last_read) > 20 &&
+      File.mtime(@file).to_i > @last_read
+      load_from_file(@file)
+    end
+  rescue => e
+    log { "action=error file=#{@file} error_class='#{e.class}' error_message='#{e.message}'" }
+  end
+
+  def load_from_file(file)
+    @file = file
+    @last_read = Time.now.to_i
+    @servers = YAML.load(File.read(file)) rescue []
+    @caches = {}
+    log { "action=load file=#{@file} servers=#{@servers.join(',')}" }
+  end
+
+  def write_to_file
+    log { "action=write file=#{@file} servers=#{@servers.join(',')}" }
+    File.open(@file, 'w') do |f|
+      f.flock(File::LOCK_EX)
+      f.write YAML.dump(@servers)
+      f.flock(File::LOCK_UN)
+    end
+  end
+
+  def log(type=:notice)
+    Log.send(type, "memcache_cluster #{yield}")
+  end
+end

data/lib/droid/heroku/stats.rb

@@ -0,0 +1,14 @@
+# Deprecated
+module Stats
+  def cache
+    @@cache ||= MemcacheCluster.cache("heroku:stats")
+  end
+
+  def increment(key, amount=1)
+  end
+
+  def sample(key, value)
+  end
+
+  extend self
+end

data/lib/droid/json_server.rb

@@ -0,0 +1,109 @@
+require 'eventmachine'
+require 'evma_httpserver'
+
+class Droid
+	class JSONServer < ::EM::Connection
+		include ::EM::HttpServer
+
+		def post_init
+			super
+			no_environment_strings
+		end
+
+		def process_http_request
+			return not_found_response if @http_request_method != "GET"
+
+			if @http_request_uri == "/"
+				default_response
+			else
+				method_name = "get_#{@http_request_uri.split("/")[1]}".gsub(/[^\d\w_]/,'').downcase
+				if public_methods.include?(method_name)
+					generate_response(method_name)
+				else
+					not_found_response
+				end
+			end
+		end
+
+		def generate_response(method_name)
+			status, data, content_type = self.send(method_name)
+
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = status
+			response.content_type content_type
+			response.content = data
+			response.send_response
+		end
+
+		def default_response
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = 200
+			response.content_type 'application/json'
+			response.content = {"status" => "OK"}.to_json
+			response.send_response
+		end
+
+		def not_found_response
+			response = ::EM::DelegatedHttpResponse.new(self)
+			response.status = 404
+			response.content_type 'text/plain'
+			response.content = "Not Found"
+			response.send_response
+		end
+
+		def get_droid
+			report_data = Droid::Utilization.report_data
+
+			metrics = {}
+			report_data.each do |topic, data|
+				metrics[topic] = data['msgs']
+			end
+			metrics['latency'] = Droid::Utilization.latency
+
+			summary = Droid::Utilization.report_summary(report_data)
+
+			metrics['total_msgs'] = summary['msgs']
+
+			status = "AMQP: #{summary['msgs']} msgs processed since #{Droid::Utilization.start.utc}"
+
+			# reset metrics data
+			Droid::Utilization.reinit
+
+			data = {
+				'status' => status,
+				'state' => 'ok',
+				'metrics' => hash_to_metrics(metrics)
+			}
+			[200, data.to_json, "application/json"]
+		end
+
+		def hash_to_metrics(hash); self.class.hash_to_metrics(hash); end
+
+		# utility method to convert a ruby hash to a metrics format
+		# that can be consumed by cloudkick
+		def self.hash_to_metrics(hash)
+			hash.collect do |k,v|
+				name = k.to_s
+				value = v
+				type = if v.kind_of?(Integer)
+					'int'
+				elsif v.kind_of?(Float)
+					'float'
+				else
+					'string'
+				end
+
+				# bool -> int conversion
+				if [TrueClass, FalseClass].include?(v.class)
+					value = v ? 1 : 0
+					type = 'int'
+				end
+
+				# if type is really string then it should respond to .to_s
+				value = value.to_s if type == 'string'
+
+				{ 'name' => name, 'type' => type, 'value' => value }
+			end
+		end
+	end
+end

data/lib/droid/monkey.rb

@@ -0,0 +1,8 @@
+module AMQP
+	module Client
+		def reconnect(*args)
+			sleep 10
+			exit 1
+		end
+	end
+end

data/lib/droid/publish.rb

@@ -0,0 +1,31 @@
+require 'droid/utils'
+
+class Droid
+	# publish to queue directly
+	def self.publish_to_q(queue_name, data, opts={}, popts={})
+		q = ::MQ.queue(queue_name)
+		json, popts = Droid::Utils.format_publish(data, opts, popts)
+		q.publish(json, popts)
+		log.info "amqp_publish queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	def self.reply_to_q(queue_name, data, opts={}, popts={})
+		q = ::MQ.queue(queue_name, :auto_delete => true)
+		json, popts = Droid::Utils.format_publish(data, opts, popts)
+		q.publish(json, popts)
+		log.info "amqp_reply queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	# publish to exchange directly
+	def self.publish_to_ex(ex_name, data, opts={}, popts={})
+		ex = ::MQ.direct(ex_name)
+		json, popts = Droid::Utils.format_publish(data, opts, popts)
+		ex.publish(json, popts)
+		log.info "amqp_publish exchange=#{ex_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
+	end
+
+	# default is publish to exchange
+	def self.publish(ex_name, data, opts={}, popts={})
+		publish_to_ex(ex_name, data, opts, popts)
+	end
+end

data/lib/droid/queue.rb

@@ -0,0 +1,196 @@
+class Droid
+	class ExpiredMessage < RuntimeError; end
+
+	class BaseQueue
+		attr_reader :queue_name, :opts
+		attr_reader :q, :ex, :mq
+
+		def initialize(queue_name, opts={})
+			opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
+
+			@queue_name, @opts = queue_name, opts
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			# if we don't specify an exchange name it defaults to the queue_name
+			@ex = @mq.direct(opts[:exchange_name] || queue_name)
+		end
+
+		def temp?
+			false
+		end
+
+		def log
+			Droid.log
+		end
+
+		def tag
+			s = "queue=#{q.name}"
+			s += " exchange=#{ex.name}" if ex
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			setup
+
+			q.bind(ex) if ex
+			q.subscribe(amqp_opts) do |header, message|
+				Droid::Utilization.monitor(q.name, :temp => temp?) do
+					request = Droid::Request.new(self, header, message)
+					log.info "amqp_message #{tag} action=received ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+					begin
+						raise Droid::ExpiredMessage if request.expired?
+						yield request if block_given?
+						finished = Time.now.getgm.to_i
+						log.info "amqp_message action=processed #{tag} elapsed=#{finished-request.start} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+					rescue Droid::ExpiredMessage
+						log.info "amqp_message action=timeout #{tag} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
+						request.ack if amqp_opts[:ack]
+					rescue => e
+						request.ack if amqp_opts[:ack]
+						Droid.handle_error(e)
+					end
+				end
+			end
+			log.info "amqp_subscribe #{tag}"
+			self
+		end
+
+		def teardown
+			@q.unsubscribe
+			@mq.close
+			log.info "amqp_unsubscribe #{tag}"
+		end
+
+		def destroy
+			teardown
+		end
+	end
+
+	class WorkerQueue < BaseQueue
+		attr_reader :prefetch
+
+		def initialize(queue_name, opts={})
+			@prefetch = opts.delete(:prefetch) || 1
+			opts[:auto_delete] = false
+
+			super(queue_name, opts)
+		end
+
+		def setup
+			super
+			@mq.prefetch(self.prefetch)
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			amqp_opts[:ack] = true
+			super(amqp_opts, opts) do |request|
+				begin
+					yield request if block_given?
+				ensure
+					request.ack unless amqp_opts[:auto_ack] == false
+				end
+			end
+		end
+	end
+
+	class ListenQueue < BaseQueue
+		def initialize(exchange_name, opts={})
+			opts[:exchange_name] = exchange_name
+			queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
+			super(queue_name, opts)
+		end
+	end
+
+	class ReplyQueue < BaseQueue
+		def initialize(queue_name, opts={})
+			opts[:auto_delete] = true
+			super
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			@ex = nil
+		end
+
+		def temp?
+			true
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			super(amqp_opts, opts) do |request|
+				yield request if block_given?
+				self.destroy
+			end
+		end
+
+		def teardown
+			@q.delete
+			super
+		end
+	end
+
+	module QueueMethods
+		def worker(queue_name, opts={})
+			WorkerQueue.new(queue_name, opts)
+		end
+
+		def listener(exchange_name, opts={})
+			ListenQueue.new(exchange_name, opts)
+		end
+	end
+
+	class BackwardsCompatibleQueue < BaseQueue
+		def initialize(exchange_name, opts={})
+			opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
+			opts[:exchange_name] = exchange_name
+			queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
+			@queue_name, @opts = queue_name, opts
+		end
+
+		def setup
+			@mq = MQ.new
+			@q = @mq.queue(queue_name, opts)
+			@ex = @mq.direct(opts[:exchange_name])
+
+			@mq.prefetch(opts[:prefetch]) if opts[:prefetch]
+		end
+
+		def subscribe(amqp_opts={}, opts={})
+			super(amqp_opts, opts) do |request|
+				if block_given?
+					if opts[:detail]
+						yield request, request.header, request.raw_message if block_given?
+					else
+						yield request if block_given?
+					end
+				end
+			end
+
+			self
+		end
+	end
+
+	module BackwardsCompatibleMethods
+		def listen4(key, orig_opts={}, &block)
+			opts = {}
+			amqp_opts = {}
+			subscribe_opts = {}
+
+			if orig_opts[:prefetch] || orig_opts[:ack]
+				opts[:prefetch] = orig_opts[:prefetch] || 1
+				opts[:ack] = true
+			end
+			if orig_opts[:queue]
+				opts[:queue] = orig_opts[:queue]
+			end
+			if orig_opts[:detail]
+				subscribe_opts[:detail] = true
+			end
+
+			BackwardsCompatibleQueue.new(key, opts).subscribe(amqp_opts, subscribe_opts, &block)
+		end
+	end
+end