beetle 0.1 → 0.2.1

data/README.rdoc

@@ -18,9 +18,9 @@ More information can be found on the {project website}[http://xing.github.com/be
 === Configuration
     # configure machines
 
-    Beetle.config do |c|
+    Beetle.config do |config|
       config.servers = "broker1:5672, broker2:5672"
-      config.redis_hosts = "redis1:6379, redis2:6379"
+      config.redis_server = "redis1:6379"
     end
 
     # instantiate a beetle client
@@ -41,7 +41,19 @@ More information can be found on the {project website}[http://xing.github.com/be
 === Subscribing
     b.listen
 
-:include: examples/README.rdoc
+=== Examples
+
+Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
+
+The top level Rakefile comes with targets to start several RabbitMQ and redis instances
+locally. Make sure the corresponding binaries are in your search path. Open four new shell
+windows and execute the following commands:
+
+    rake rabbit:start1
+    rake rabbit:start2
+    rake redis:start1
+    rake redis:start2
+
 
 == Prerequisites
 
@@ -68,8 +80,9 @@ For development, you'll need
 == Authors
 
 {Stefan Kaes}[http://github.com/skaes],
-{Pascal Friederich}[http://github.com/paukul] and
-{Ali Jelveh}[http://github.com/dudemeister].
+{Pascal Friederich}[http://github.com/paukul],
+{Ali Jelveh}[http://github.com/dudemeister] and
+{Sebastian Roebke}[http://github.com/boosty].
 
 You cand find out more about our work on our {dev blog}[http://devblog.xing.com].
 
@@ -77,6 +90,3 @@ Copyright (c) 2010 {XING AG}[http://www.xing.com/]
 
 Released under the MIT license. For full details see MIT-LICENSE included in this
 distribution.
-
-
-

data/beetle.gemspec

@@ -1,127 +1,43 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
-
 Gem::Specification.new do |s|
-  s.name = %q{beetle}
-  s.version = "0.1"
+  s.name    = "beetle"
+  s.version = "0.2.1"
+
+  s.required_rubygems_version = ">= 1.3.1"
+  s.authors            = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
+  s.date               = Time.now.strftime('%Y-%m-%d')
+  s.default_executable = "beetle"
+  s.description        = "A highly available, reliable messaging infrastructure"
+  s.summary            = "High Availability AMQP Messaging with Redundant Queues"
+  s.email              = "developers@xing.com"
+  s.executables        = ["beetle"]
+  s.extra_rdoc_files   = ["README.rdoc"]
+  s.files              = Dir['{examples,ext,lib}/**/*.rb'] + %w(beetle.gemspec examples/README.rdoc)
+  s.extensions         = 'ext/mkrf_conf.rb'
+  s.homepage           = "http://xing.github.com/beetle/"
+  s.rdoc_options       = ["--charset=UTF-8"]
+  s.require_paths      = ["lib"]
+  s.rubygems_version   = "1.3.7"
+  s.test_files         = Dir['test/**/*.rb']
+
+  s.post_install_message = <<-INFO
+  *********************************************************************************************
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh"]
-  s.date = %q{2010-04-14}
-  s.description = %q{A highly available, reliable messaging infrastructure}
-  s.email = %q{developers@xing.com}
-  s.extra_rdoc_files = [
-    "README.rdoc",
-     "TODO"
-  ]
-  s.files = [
-    ".gitignore",
-     "MIT-LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "TODO",
-     "beetle.gemspec",
-     "doc/redundant_queues.graffle",
-     "etc/redis-master.conf",
-     "etc/redis-slave.conf",
-     "examples/README.rdoc",
-     "examples/attempts.rb",
-     "examples/handler_class.rb",
-     "examples/handling_exceptions.rb",
-     "examples/multiple_exchanges.rb",
-     "examples/multiple_queues.rb",
-     "examples/redis_failover.rb",
-     "examples/redundant.rb",
-     "examples/rpc.rb",
-     "examples/simple.rb",
-     "lib/beetle.rb",
-     "lib/beetle/base.rb",
-     "lib/beetle/client.rb",
-     "lib/beetle/configuration.rb",
-     "lib/beetle/deduplication_store.rb",
-     "lib/beetle/handler.rb",
-     "lib/beetle/message.rb",
-     "lib/beetle/publisher.rb",
-     "lib/beetle/r_c.rb",
-     "lib/beetle/subscriber.rb",
-     "script/start_rabbit",
-     "snafu.rb",
-     "test/beetle.yml",
-     "test/beetle/base_test.rb",
-     "test/beetle/bla.rb",
-     "test/beetle/client_test.rb",
-     "test/beetle/configuration_test.rb",
-     "test/beetle/deduplication_store_test.rb",
-     "test/beetle/handler_test.rb",
-     "test/beetle/message_test.rb",
-     "test/beetle/publisher_test.rb",
-     "test/beetle/r_c_test.rb",
-     "test/beetle/subscriber_test.rb",
-     "test/beetle_test.rb",
-     "test/test_helper.rb",
-     "tmp/master/.gitignore",
-     "tmp/slave/.gitignore"
-  ]
-  s.homepage = %q{http://xing.github.com/beetle/}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
-  s.summary = %q{High Availability AMQP Messaging with Redundant Queues}
-  s.test_files = [
-    "test/beetle/base_test.rb",
-     "test/beetle/bla.rb",
-     "test/beetle/client_test.rb",
-     "test/beetle/configuration_test.rb",
-     "test/beetle/deduplication_store_test.rb",
-     "test/beetle/handler_test.rb",
-     "test/beetle/message_test.rb",
-     "test/beetle/publisher_test.rb",
-     "test/beetle/r_c_test.rb",
-     "test/beetle/subscriber_test.rb",
-     "test/beetle_test.rb",
-     "test/test_helper.rb",
-     "examples/attempts.rb",
-     "examples/handler_class.rb",
-     "examples/handling_exceptions.rb",
-     "examples/multiple_exchanges.rb",
-     "examples/multiple_queues.rb",
-     "examples/redis_failover.rb",
-     "examples/redundant.rb",
-     "examples/rpc.rb",
-     "examples/simple.rb"
-  ]
+    If you're running a ruby version < 1.9 we silently installed the SystemTimer gem for you.
+    See: http://ph7spot.com/musings/system-timer
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  *********************************************************************************************
+  INFO
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<uuid4r>, [">= 0.1.1"])
-      s.add_runtime_dependency(%q<bunny>, [">= 0.6.0"])
-      s.add_runtime_dependency(%q<redis>, [">= 0.1.2"])
-      s.add_runtime_dependency(%q<amqp>, [">= 0.6.7"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_development_dependency(%q<mocha>, [">= 0"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
-    else
-      s.add_dependency(%q<uuid4r>, [">= 0.1.1"])
-      s.add_dependency(%q<bunny>, [">= 0.6.0"])
-      s.add_dependency(%q<redis>, [">= 0.1.2"])
-      s.add_dependency(%q<amqp>, [">= 0.6.7"])
-      s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_dependency(%q<mocha>, [">= 0"])
-      s.add_dependency(%q<rcov>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<uuid4r>, [">= 0.1.1"])
-    s.add_dependency(%q<bunny>, [">= 0.6.0"])
-    s.add_dependency(%q<redis>, [">= 0.1.2"])
-    s.add_dependency(%q<amqp>, [">= 0.6.7"])
-    s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-    s.add_dependency(%q<mocha>, [">= 0"])
-    s.add_dependency(%q<rcov>, [">= 0"])
-  end
+  s.specification_version = 3
+  s.add_runtime_dependency("uuid4r",                [">= 0.1.1"])
+  s.add_runtime_dependency("bunny",                 [">= 0.6.0"])
+  s.add_runtime_dependency("redis",                 [">= 2.0.3"])
+  s.add_runtime_dependency("amqp",                  [">= 0.6.7"])
+  s.add_runtime_dependency("activesupport",         [">= 2.3.4"])
+  s.add_runtime_dependency("daemons",               [">= 1.0.10"])
+  s.add_development_dependency("mocha",             [">= 0"])
+  s.add_development_dependency("rcov",              [">= 0"])
+  s.add_development_dependency("cucumber",          [">= 0.7.2"])
+  s.add_development_dependency("daemon_controller", [">= 0"])
 end
 

data/bin/beetle

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'beetle/commands'
+rescue LoadError
+  beetle_path = File.expand_path("../../lib", __FILE__)
+  $:.unshift(beetle_path)
+  require 'beetle/commands'
+end

data/examples/README.rdoc

@@ -10,5 +10,3 @@ windows and execute the following commands:
     rake rabbit:start2
     rake redis:start1
     rake redis:start2
-
-After running the redis_failover.rb script you will need to restart both redis servers.

data/examples/rpc.rb

@@ -4,9 +4,10 @@ require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
 
 # suppress debug messages
 Beetle.config.logger.level = Logger::DEBUG
-
+Beetle.config.servers = "localhost:5672, localhost:5673"
 # instantiate a client
-client = Beetle::Client.new(:servers => "localhost:5672, localhost:5673")
+
+client = Beetle::Client.new
 
 # register a durable queue named 'test'
 # this implicitly registers a durable topic exchange called 'test'

data/ext/mkrf_conf.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'rubygems/command.rb'
+require 'rubygems/dependency_installer.rb' 
+begin
+ Gem::Command.build_args = ARGV
+ rescue NoMethodError
+end 
+inst = Gem::DependencyInstaller.new
+begin
+ if RUBY_VERSION < "1.9"
+   inst.install "SystemTimer", ">= 1.2"
+ end
+ rescue
+   exit(1)
+end
+
+f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w")   # create dummy rakefile to indicate success
+f.write("task :default\n")
+f.close

data/lib/beetle.rb

@@ -17,8 +17,6 @@ module Beetle
   class UnknownQueue < Error; end
   # raised when no redis master server can be found
   class NoRedisMaster < Error; end
-  # raised when two redis master servers are found
-  class TwoRedisMasters < Error; end
 
   # AMQP options for exchange creation
   EXCHANGE_CREATION_KEYS  = [:auto_delete, :durable, :internal, :nowait, :passive]
@@ -37,6 +35,8 @@ module Beetle
     autoload File.basename(libfile)[/^(.*)\.rb$/, 1].classify, libfile
   end
 
+  require "#{lib_dir}/redis_ext"
+
   # returns the default configuration object and yields it if a block is given
   def self.config
     #:yields: config

data/lib/beetle/base.rb

@@ -1,6 +1,7 @@
 module Beetle
   # Abstract base class shared by Publisher and Subscriber
   class Base
+    include Logging
 
     attr_accessor :options, :servers, :server  #:nodoc:
 
@@ -15,14 +16,6 @@ module Beetle
 
     private
 
-    def logger
-      self.class.logger
-    end
-
-    def self.logger
-      Beetle.config.logger
-    end
-
     def error(text)
       logger.error text
       raise Error.new(text)

data/lib/beetle/client.rb

@@ -19,6 +19,8 @@ module Beetle
   # order, so that no message is lost if message producers are accidentally started before
   # the corresponding consumers.
   class Client
+    include Logging
+
     # the AMQP servers available for publishing
     attr_reader :servers
 
@@ -37,14 +39,18 @@ module Beetle
     # the deduplication store to use for this client
     attr_reader :deduplication_store
 
+    # accessor for the beetle configuration
+    attr_reader :config
+
     # create a fresh Client instance from a given configuration object
     def initialize(config = Beetle.config)
+      @config  = config
       @servers = config.servers.split(/ *, */)
       @exchanges = {}
       @queues = {}
       @messages = {}
       @bindings = {}
-      @deduplication_store = DeduplicationStore.new(config.redis_hosts, config.redis_db)
+      @deduplication_store = DeduplicationStore.new(config)
     end
 
     # register an exchange with the given _name_ and a set of _options_:
@@ -69,8 +75,8 @@ module Beetle
     def register_queue(name, options={})
       name = name.to_s
       raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
-      opts = {:exchange => name, :key => name}.merge!(options.symbolize_keys)
-      opts.merge! :durable => true, :passive => false, :exclusive => false, :auto_delete => false, :amqp_name => name
+      opts = {:exchange => name, :key => name, :auto_delete => false, :amqp_name => name}.merge!(options.symbolize_keys)
+      opts.merge! :durable => true, :passive => false, :exclusive => false
       exchange = opts.delete(:exchange).to_s
       key = opts.delete(:key)
       queues[name] = opts
@@ -139,10 +145,10 @@ module Beetle
 
     # this is a convenience method to configure exchanges, queues, messages and handlers
     # with a common set of options. allows one to call all register methods without the
-    # register_ prefix.
+    # register_ prefix. returns self.
     #
     # Example:
-    #  client.configure :exchange => :foobar do |config|
+    #  client = Beetle.client.new.configure :exchange => :foobar do |config|
     #    config.queue :q1, :key => "foo"
     #    config.queue :q2, :key => "bar"
     #    config.message :foo
@@ -152,6 +158,7 @@ module Beetle
     #  end
     def configure(options={}) #:yields: config
       yield Configurator.new(self, options)
+      self
     end
 
     # publishes a message. the given options hash is merged with options given on message registration.
@@ -195,10 +202,10 @@ module Beetle
       publisher.stop
     end
 
-    # traces all messages received on all queues. useful for debugging message flow.
-    def trace(&block)
+    # traces messages without consuming them. useful for debugging message flow.
+    def trace(messages=self.messages.keys, &block)
       queues.each do |name, opts|
-        opts.merge! :durable => false, :auto_delete => true, :amqp_name => queue_name_for_tracing(name)
+        opts.merge! :durable => false, :auto_delete => true, :amqp_name => queue_name_for_tracing(opts[:amqp_name])
       end
       register_handler(queues.keys) do |msg|
         puts "-----===== new message =====-----"
@@ -207,7 +214,7 @@ module Beetle
         puts "MSGID: #{msg.msg_id}"
         puts "DATA: #{msg.data}"
       end
-      subscriber.listen(messages.keys, &block)
+      listen(messages, &block)
     end
 
     # evaluate the ruby files matching the given +glob+ pattern in the context of the client instance.
@@ -218,11 +225,6 @@ module Beetle
       end
     end
 
-    # returns the configured Logger instance
-    def logger
-      @logger ||= Beetle.config.logger
-    end
-
     private
 
     class Configurator #:nodoc:all

data/lib/beetle/commands.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'active_support'
+
+module Beetle
+  module Commands
+    # invokes given command by instantiating an appropriate command class
+    def self.execute(command)
+      if commands.include? command
+        require File.expand_path("../commands/#{command}", __FILE__)
+        "Beetle::Commands::#{command.classify}".constantize.execute
+      else
+        # me no likez no frikin heredocs
+        puts "\nCommand #{command} not known\n" if command
+        puts "Available commands are:"
+        puts
+        commands.each {|c| puts "\t #{c}"}
+        puts
+        exit 1
+      end
+    end
+
+    private
+    def self.commands
+      commands_dir = File.expand_path('../commands', __FILE__)
+      Dir[commands_dir + '/*.rb'].map {|f| File.basename(f)[0..-4]}
+    end
+  end
+end
+
+Beetle::Commands.execute(ARGV.shift)

data/lib/beetle/commands/configuration_client.rb

@@ -0,0 +1,73 @@
+require 'optparse'
+require 'daemons'
+require 'beetle'
+
+module Beetle
+  module Commands
+    # Command to start a RedisConfigurationClient daemon.
+    #
+    #   Usage: beetle configuration_client  [options] -- [client options]
+    #
+    #   client options:
+    #           --redis-master-file FILE     Write redis master server string to FILE
+    #           --id, --client-id ID         Set unique client id (default is minastirith.local)
+    #           --amqp-servers LIST          AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
+    #           --config-file PATH           Path to an external yaml config file
+    #           --pid-dir DIR                Write pid and log to DIR
+    #       -v, --verbose                    Set log level to DEBUG
+    #       -h, --help                       Show this message
+    #
+    class ConfigurationClient
+      # parses command line options and starts Beetle::RedisConfigurationClient as a daemon
+      def self.execute
+        command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
+
+        opts = OptionParser.new
+        opts.banner = "Usage: beetle configuration_client #{command} [options] -- [client options]"
+        opts.separator ""
+        opts.separator "client options:"
+
+        opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
+          Beetle.config.redis_server = val
+        end
+
+        client_id = nil
+        opts.on("--id ID", "--client-id ID", String, "Set unique client id (default is #{RedisConfigurationClient.new.id})") do |val|
+          client_id = val
+        end
+
+        opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
+          Beetle.config.servers = val
+        end
+
+        opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
+          Beetle.config.config_file = val
+        end
+
+        dir_mode = nil
+        dir = nil
+        opts.on("--pid-dir DIR", String, "Write pid and log to DIR") do |val|
+          dir_mode = :normal
+          dir = val
+        end
+
+        opts.on("-v", "--verbose", "Set log level to DEBUG") do |val|
+          Beetle.config.logger.level = Logger::DEBUG
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.parse!(app_options)
+
+        Daemons.run_proc("redis_configuration_client", :multiple => true, :log_output => true, :dir_mode => dir_mode, :dir => dir) do
+          client = Beetle::RedisConfigurationClient.new
+          client.id = client_id if client_id
+          client.start
+        end
+      end
+    end
+  end
+end