beetle 0.1

data/examples/multiple_queues.rb

@@ -0,0 +1,43 @@
+# multiple_queues.rb
+# this example shows how to route a message thru two queues to two handlers 
+# we're using the client.configure block to not duplicate our settings
+# 
+# 
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby multiple_queues.rb
+
+require "rubygems"
+require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+
+# this is our block configuration option, set options are used for all configs within it
+# in this example all items will use: exchange => foobar and key => foobar
+# so creating the two queues queue_1 and queue_2 will be bound to the same exchange
+# and same key, the message foobar will also use those setting
+client.configure :exchange => :foobar, :key => "foobar" do |config|
+  config.queue :queue_1
+  config.queue :queue_2
+  config.message :foobar
+  # different than other examples we use the option to configure a handler with a simple blocl
+  # rather than subclassing Beetle::Handler, this allow for very easy handlers to be created
+  # with a minimal amount of code
+  config.handler(:queue_1) {|message| puts "received message on queue 1: " + message.data}
+  # both queues will be getting the same messages ...
+  config.handler(:queue_2) {|message| puts "received message on queue 2: " + message.data}
+end
+
+# ... and publish a message, we expect both queue handlers to output the message
+client.publish(:foobar, "baz")
+
+# start the listen loop and stop listening after 0.1 seconds
+# this should be more than enough time to finish processing our messages
+client.listen do
+  EM.add_timer(0.1) { client.stop_listening }
+end
+

data/examples/redis_failover.rb

@@ -0,0 +1,65 @@
+# Testing redis failover functionality
+require "rubygems"
+require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
+
+Beetle.config.logger.level = Logger::INFO
+Beetle.config.redis_hosts = "localhost:6379, localhost:6380"
+Beetle.config.servers = "localhost:5672, localhost:5673"
+
+# instantiate a client
+client = Beetle::Client.new
+
+# register a durable queue named 'test'
+# this implicitly registers a durable topic exchange called 'test'
+client.register_queue(:test)
+client.purge(:test)
+client.register_message(:test, :redundant => true)
+
+# publish some test messages
+# at this point, the exchange will be created on the server and the queue will be bound to the exchange
+N = 10
+n = 0
+N.times do |i|
+  n += client.publish(:test, "Hello#{i+1}")
+end
+puts "published #{n} test messages"
+puts
+
+# check whether we were able to publish all messages
+if n != 2*N
+  puts "could not publish all messages"
+  exit 1
+end
+
+# register a handler for the test message, listing on queue "test"
+k = 0
+client.register_handler(:test) do |m|
+  k += 1
+  puts "Received test message from server #{m.server}"
+  puts "Message content: #{m.data}"
+  puts
+  sleep 1
+end
+
+# hack to switch redis programmatically
+class Beetle::DeduplicationStore
+  def switch_redis
+    slave = redis_instances.find{|r| r.server != redis.server}
+    redis.shutdown rescue nil
+    logger.info "Beetle: shut down master #{redis.server}"
+    slave.slaveof("no one")
+    logger.info "Beetle: enabled master mode on #{slave.server}"
+  end
+end
+
+# start listening
+# this starts the event machine loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+client.listen do
+  trap("INT") { client.stop_listening }
+  EM.add_timer(5) { client.deduplication_store.switch_redis }
+  EM.add_timer(11) { client.stop_listening }
+end
+
+puts "Received #{k} test messages"
+raise "Your setup is borked" if N != k

data/examples/redundant.rb

@@ -0,0 +1,65 @@
+# redundant.rb
+# 
+# 
+# 
+# 
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby redundant.rb
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+
+# use two servers
+Beetle.config.servers = "localhost:5672, localhost:5673"
+# instantiate a client
+client = Beetle::Client.new
+
+# register a durable queue named 'test'
+# this implicitly registers a durable topic exchange called 'test'
+client.register_queue(:test)
+client.purge(:test)
+client.register_message(:test, :redundant => true)
+
+# publish some test messages
+# at this point, the exchange will be created on the server and the queue will be bound to the exchange
+N = 3
+n = 0
+N.times do |i|
+  n += client.publish(:test, "Hello#{i+1}")
+end
+puts "published #{n} test messages"
+puts
+
+expected_publish_count = 2*N
+if n != expected_publish_count
+  puts "could not publish all messages"
+  exit 1
+end
+
+# register a handler for the test message, listing on queue "test"
+k = 0
+client.register_handler(:test) do |m|
+  k += 1
+  puts "Received test message from server #{m.server}"
+  puts m.msg_id
+  p m.header
+  puts "Message content: #{m.data}"
+  puts
+end
+
+# start listening
+# this starts the event machine event loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+client.listen do
+  EM.add_timer(0.1) { client.stop_listening }
+end
+
+puts "Received #{k} test messages"
+raise "Your setup is borked" if N != k

data/examples/rpc.rb

@@ -0,0 +1,45 @@
+# A simple usage example for Beetle
+require "rubygems"
+require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
+
+# suppress debug messages
+Beetle.config.logger.level = Logger::DEBUG
+
+# instantiate a client
+client = Beetle::Client.new(:servers => "localhost:5672, localhost:5673")
+
+# register a durable queue named 'test'
+# this implicitly registers a durable topic exchange called 'test'
+client.register_queue(:echo)
+client.register_message(:echo)
+
+if ARGV.include?("--server")
+  # register a handler for the test message, listing on queue "test"
+  # echoing all data sent to it
+  client.register_handler(:echo) do |m|
+    # send data back to publisher
+    m.data
+  end
+
+  # start the subscriber
+  client.listen do
+    puts "started echo server"
+    trap("INT") { puts "stopped echo server"; client.stop_listening }
+  end
+else
+  n = 100
+  ms = Benchmark.ms do
+    n.times do |i|
+      content = "Hello #{i}"
+      # puts "performing RPC with message content '#{content}'"
+      status, result = client.rpc(:echo, content)
+      # puts "status  #{status}"
+      # puts "result  #{result}"
+      # puts
+      $stderr.puts "processing failure for message '#{content}'" if result != content
+    end
+  end
+  printf "Runtime: %dms\n", ms
+  printf "Milliseconds per RPC: %.1f\n", ms/n
+end
+

data/examples/simple.rb

@@ -0,0 +1,39 @@
+# simple.rb
+# this example shows you a very basic message/handler setup
+#
+#
+#
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby multiple_exchanges.rb
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+client.register_queue(:test)
+client.register_message(:test)
+
+# purge the test queue
+client.purge(:test)
+
+# empty the dedup store
+client.deduplication_store.flushdb
+
+# register our handler to the message, check out the message.rb for more stuff you can get from the message object
+client.register_handler(:test) {|message| puts "got message: #{message.data}"}
+
+# publish our message
+client.publish(:test, 'bam')
+
+# start listening
+# this starts the event machine event loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+client.listen do
+  EM.add_timer(0.1) { client.stop_listening }
+end
+

data/lib/beetle.rb

@@ -0,0 +1,57 @@
+require 'amqp'
+require 'mq'
+require 'bunny'
+require 'uuid4r'
+require 'active_support'
+require 'redis'
+
+module Beetle
+
+  # abstract superclass for Beetle specific exceptions
+  class Error < StandardError; end
+  # raised when Beetle detects configuration errors
+  class ConfigurationError < Error; end
+  # raised when trying to access an unknown message
+  class UnknownMessage < Error; end
+  # raised when trying to access an unknown queue
+  class UnknownQueue < Error; end
+  # raised when no redis master server can be found
+  class NoRedisMaster < Error; end
+  # raised when two redis master servers are found
+  class TwoRedisMasters < Error; end
+
+  # AMQP options for exchange creation
+  EXCHANGE_CREATION_KEYS  = [:auto_delete, :durable, :internal, :nowait, :passive]
+  # AMQP options for queue creation
+  QUEUE_CREATION_KEYS     = [:passive, :durable, :exclusive, :auto_delete, :no_wait]
+  # AMQP options for queue bindings
+  QUEUE_BINDING_KEYS      = [:key, :no_wait]
+  # AMQP options for message publishing
+  PUBLISHING_KEYS         = [:key, :mandatory, :immediate, :persistent, :reply_to]
+  # AMQP options for subscribing to queues
+  SUBSCRIPTION_KEYS       = [:ack, :key]
+
+  # use ruby's autoload mechanism for loading beetle classes
+  lib_dir = File.expand_path(File.dirname(__FILE__) + '/beetle/')
+  Dir["#{lib_dir}/*.rb"].each do |libfile|
+    autoload File.basename(libfile)[/^(.*)\.rb$/, 1].classify, libfile
+  end
+
+  # returns the default configuration object and yields it if a block is given
+  def self.config
+    #:yields: config
+    @config ||= Configuration.new
+    block_given? ? yield(@config) : @config
+  end
+
+  # FIXME: there should be a better way to test
+  if defined?(Mocha)
+    def self.reraise_expectation_errors! #:nodoc:
+      raise if $!.is_a?(Mocha::ExpectationError)
+    end
+  else
+    def self.reraise_expectation_errors! #:nodoc:
+    end
+  end
+
+end

data/lib/beetle/base.rb

@@ -0,0 +1,78 @@
+module Beetle
+  # Abstract base class shared by Publisher and Subscriber
+  class Base
+
+    attr_accessor :options, :servers, :server  #:nodoc:
+
+    def initialize(client, options = {}) #:nodoc:
+      @options = options
+      @client = client
+      @servers = @client.servers.clone
+      @server = @servers[rand @servers.size]
+      @exchanges = {}
+      @queues = {}
+    end
+
+    private
+
+    def logger
+      self.class.logger
+    end
+
+    def self.logger
+      Beetle.config.logger
+    end
+
+    def error(text)
+      logger.error text
+      raise Error.new(text)
+    end
+
+    def current_host
+      @server.split(':').first
+    end
+
+    def current_port
+      @server =~ /:(\d+)$/ ? $1.to_i : 5672
+    end
+
+    def set_current_server(s)
+      @server = s
+    end
+
+    def each_server
+      @servers.each { |s| set_current_server(s); yield }
+    end
+
+    def exchanges
+      @exchanges[@server] ||= {}
+    end
+
+    def exchange(name)
+      exchanges[name] ||= create_exchange!(name, @client.exchanges[name])
+    end
+
+    def queues
+      @queues[@server] ||= {}
+    end
+
+    def queue(name)
+      queues[name] ||=
+        begin
+          opts = @client.queues[name]
+          error("You are trying to bind a queue #{name} which is not configured!") unless opts
+          logger.debug("Beetle: binding queue #{name} with internal name #{opts[:amqp_name]} on server #{@server}")
+          queue_name = opts[:amqp_name]
+          creation_options = opts.slice(*QUEUE_CREATION_KEYS)
+          the_queue = nil
+          @client.bindings[name].each do |binding_options|
+            exchange_name = binding_options[:exchange]
+            binding_options = binding_options.slice(*QUEUE_BINDING_KEYS)
+            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options)
+          end
+          the_queue
+        end
+    end
+
+  end
+end

data/lib/beetle/client.rb

@@ -0,0 +1,252 @@
+module Beetle
+  # This class provides the interface through which messaging is configured for both
+  # message producers and consumers. It keeps references to an instance of a
+  # Beetle::Subscriber, a Beetle::Publisher (both of which are instantiated on demand),
+  # and a reference to an instance of Beetle::DeduplicationStore.
+  #
+  # Configuration of exchanges, queues, messages, and message handlers is done by calls to
+  # corresponding register_ methods. Note that these methods just build up the
+  # configuration, they don't interact with the AMQP servers.
+  #
+  # On the publisher side, publishing a message will ensure that the exchange it will be
+  # sent to, and each of the queues bound to the exchange, will be created on demand. On
+  # the subscriber side, exchanges, queues, bindings and queue subscriptions will be
+  # created when the application calls the listen method. An application can decide to
+  # subscribe to only a subset of the configured queues by passing a list of queue names
+  # to the listen method.
+  #
+  # The net effect of this strategy is that producers and consumers can be started in any
+  # order, so that no message is lost if message producers are accidentally started before
+  # the corresponding consumers.
+  class Client
+    # the AMQP servers available for publishing
+    attr_reader :servers
+
+    # an options hash for the configured exchanges
+    attr_reader :exchanges
+
+    # an options hash for the configured queues
+    attr_reader :queues
+
+    # an options hash for the configured queue bindings
+    attr_reader :bindings
+
+    # an options hash for the configured messages
+    attr_reader :messages
+
+    # the deduplication store to use for this client
+    attr_reader :deduplication_store
+
+    # create a fresh Client instance from a given configuration object
+    def initialize(config = Beetle.config)
+      @servers = config.servers.split(/ *, */)
+      @exchanges = {}
+      @queues = {}
+      @messages = {}
+      @bindings = {}
+      @deduplication_store = DeduplicationStore.new(config.redis_hosts, config.redis_db)
+    end
+
+    # register an exchange with the given _name_ and a set of _options_:
+    # [<tt>:type</tt>]
+    #   the type option will be overwritten and always be <tt>:topic</tt>, beetle does not allow fanout exchanges
+    # [<tt>:durable</tt>]
+    #   the durable option will be overwritten and always be true. this is done to ensure that exchanges are never deleted
+
+    def register_exchange(name, options={})
+      name = name.to_s
+      raise ConfigurationError.new("exchange #{name} already configured") if exchanges.include?(name)
+      exchanges[name] = options.symbolize_keys.merge(:type => :topic, :durable => true)
+    end
+
+    # register a durable, non passive, non auto_deleted queue with the given _name_ and an _options_ hash:
+    # [<tt>:exchange</tt>]
+    #   the name of the exchange this queue will be bound to (defaults to the name of the queue)
+    # [<tt>:key</tt>]
+    #   the binding key (defaults to the name of the queue)
+    # automatically registers the specified exchange if it hasn't been registered yet
+
+    def register_queue(name, options={})
+      name = name.to_s
+      raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
+      opts = {:exchange => name, :key => name}.merge!(options.symbolize_keys)
+      opts.merge! :durable => true, :passive => false, :exclusive => false, :auto_delete => false, :amqp_name => name
+      exchange = opts.delete(:exchange).to_s
+      key = opts.delete(:key)
+      queues[name] = opts
+      register_binding(name, :exchange => exchange, :key => key)
+    end
+
+    # register an additional binding for an already configured queue _name_ and an _options_ hash:
+    # [<tt>:exchange</tt>]
+    #   the name of the exchange this queue will be bound to (defaults to the name of the queue)
+    # [<tt>:key</tt>]
+    #   the binding key (defaults to the name of the queue)
+    # automatically registers the specified exchange if it hasn't been registered yet
+
+    def register_binding(queue_name, options={})
+      name = queue_name.to_s
+      opts = options.symbolize_keys
+      exchange = (opts[:exchange] || name).to_s
+      key = (opts[:key] || name).to_s
+      (bindings[name] ||= []) << {:exchange => exchange, :key => key}
+      register_exchange(exchange) unless exchanges.include?(exchange)
+      queues = (exchanges[exchange][:queues] ||= [])
+      queues << name unless queues.include?(name)
+    end
+
+    # register a persistent message with a given _name_ and an _options_ hash:
+    # [<tt>:key</tt>]
+    #   specifies the routing key for message publishing (defaults to the name of the message)
+    # [<tt>:ttl</tt>]
+    #   specifies the time interval after which the message will be silently dropped (seconds).
+    #   defaults to Message::DEFAULT_TTL.
+    # [<tt>:redundant</tt>]
+    #   specifies whether the message should be published redundantly (defaults to false)
+
+    def register_message(message_name, options={})
+      name = message_name.to_s
+      raise ConfigurationError.new("message #{name} already configured") if messages.include?(name)
+      opts = {:exchange => name, :key => name}.merge!(options.symbolize_keys)
+      opts.merge! :persistent => true
+      opts[:exchange] = opts[:exchange].to_s
+      messages[name] = opts
+    end
+
+    # registers a handler for a list of queues (which must have been registered
+    # previously). The handler will be invoked when any messages arrive on the queue.
+    #
+    # Examples:
+    #   register_handler([:foo, :bar], :timeout => 10.seconds) { |message| puts "received #{message}" }
+    #
+    #   on_error   = lambda{ puts "something went wrong with baz" }
+    #   on_failure = lambda{ puts "baz has finally failed" }
+    #
+    #   register_handler(:baz, :exceptions => 1, :errback => on_error, :failback => on_failure) { puts "received baz" }
+    #
+    #   register_handler(:bar, BarHandler)
+    #
+    # For details on handler classes see class Beetle::Handler
+
+    def register_handler(queues, *args, &block)
+      queues = Array(queues).map(&:to_s)
+      queues.each {|q| raise UnknownQueue.new(q) unless self.queues.include?(q)}
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      handler = args.shift
+      raise ArgumentError.new("too many arguments for handler registration") unless args.empty?
+      subscriber.register_handler(queues, opts, handler, &block)
+    end
+
+    # this is a convenience method to configure exchanges, queues, messages and handlers
+    # with a common set of options. allows one to call all register methods without the
+    # register_ prefix.
+    #
+    # Example:
+    #  client.configure :exchange => :foobar do |config|
+    #    config.queue :q1, :key => "foo"
+    #    config.queue :q2, :key => "bar"
+    #    config.message :foo
+    #    config.message :bar
+    #    config.handler :q1 { puts "got foo"}
+    #    config.handler :q2 { puts "got bar"}
+    #  end
+    def configure(options={}) #:yields: config
+      yield Configurator.new(self, options)
+    end
+
+    # publishes a message. the given options hash is merged with options given on message registration.
+    def publish(message_name, data=nil, opts={})
+      message_name = message_name.to_s
+      raise UnknownMessage.new("unknown message #{message_name}") unless messages.include?(message_name)
+      publisher.publish(message_name, data, opts)
+    end
+
+    # sends the given message to one of the configured servers and returns the result of running the associated handler.
+    #
+    # unexpected behavior can ensue if the message gets routed to more than one recipient, so be careful.
+    def rpc(message_name, data=nil, opts={})
+      message_name = message_name.to_s
+      raise UnknownMessage.new("unknown message #{message_name}") unless messages.include?(message_name)
+      publisher.rpc(message_name, data, opts)
+    end
+
+    # purges the given queue on all configured servers
+    def purge(queue_name)
+      queue_name = queue_name.to_s
+      raise UnknownQueue.new("unknown queue #{queue_name}") unless queues.include?(queue_name)
+      publisher.purge(queue_name)
+    end
+
+    # start listening to a list of messages (default to all registered messages).
+    # runs the given block before entering the eventmachine loop.
+    def listen(messages=self.messages.keys, &block)
+      messages = messages.map(&:to_s)
+      messages.each{|m| raise UnknownMessage.new("unknown message #{m}") unless self.messages.include?(m)}
+      subscriber.listen(messages, &block)
+    end
+
+    # stops the eventmachine loop
+    def stop_listening
+      subscriber.stop!
+    end
+
+    # disconnects the publisher from all servers it's currently connected to
+    def stop_publishing
+      publisher.stop
+    end
+
+    # traces all messages received on all queues. useful for debugging message flow.
+    def trace(&block)
+      queues.each do |name, opts|
+        opts.merge! :durable => false, :auto_delete => true, :amqp_name => queue_name_for_tracing(name)
+      end
+      register_handler(queues.keys) do |msg|
+        puts "-----===== new message =====-----"
+        puts "SERVER: #{msg.server}"
+        puts "HEADER: #{msg.header.inspect}"
+        puts "MSGID: #{msg.msg_id}"
+        puts "DATA: #{msg.data}"
+      end
+      subscriber.listen(messages.keys, &block)
+    end
+
+    # evaluate the ruby files matching the given +glob+ pattern in the context of the client instance.
+    def load(glob)
+      b = binding
+      Dir[glob].each do |f|
+        eval(File.read(f), b, f)
+      end
+    end
+
+    # returns the configured Logger instance
+    def logger
+      @logger ||= Beetle.config.logger
+    end
+
+    private
+
+    class Configurator #:nodoc:all
+      def initialize(client, options={})
+        @client = client
+        @options = options
+      end
+      def method_missing(method, *args, &block)
+        super unless %w(exchange queue binding message handler).include?(method.to_s)
+        options = @options.merge(args.last.is_a?(Hash) ? args.pop : {})
+        @client.send("register_#{method}", *(args+[options]), &block)
+      end
+    end
+
+    def publisher
+      @publisher ||= Publisher.new(self)
+    end
+
+    def subscriber
+      @subscriber ||= Subscriber.new(self)
+    end
+
+    def queue_name_for_tracing(queue)
+      "trace-#{queue}-#{`hostname`.chomp}-#{$$}"
+    end
+  end
+end