beetle 0.3.0.rc.2 → 0.3.0.rc.5

data/README.rdoc

@@ -58,7 +58,7 @@ windows and execute the following commands:
 
 To set up a redundant messaging system you will need
 * at least 2 AMQP servers (we use {RabbitMQ}[http://www.rabbitmq.com/])
-* at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup)
+* at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup, see REDIS_AUTO_FAILOVER.rdoc)
 
 == Gem Dependencies
 

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,10 @@
 = Release Notes
 
+== Version 0.3.0.rc.3
+
+* use hiredis as the redis backend, which overcomes lack of proper time-outs in the "generic" redis-rb
+  gem for Ruby 1.9
+
 == Version 0.2.9.8
 
 * since version 2.0, RabbitMQ supports Basic.reject(:requeue => true). we use it now too,

data/Rakefile

@@ -1,6 +1,8 @@
 require 'rake'
 require 'rake/testtask'
 require 'rcov/rcovtask'
+require 'bundler/gem_tasks'
+
 # rake 0.9.2 hack to supress deprecation warnings caused by cucumber
 include Rake::DSL if RAKEVERSION >= "0.9"
 require 'cucumber/rake/task'
@@ -30,7 +32,6 @@ namespace :test do
   end if RUBY_PLATFORM =~ /darwin/
 end
 
-
 namespace :beetle do
   task :test do
     Beetle::Client.new.test
@@ -108,8 +109,3 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('MIT-LICENSE')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
-
-desc "build the beetle gem"
-task :build do
-  system("gem build beetle.gemspec")
-end

data/beetle.gemspec

@@ -1,13 +1,17 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "beetle/version"
+
 Gem::Specification.new do |s|
   s.name    = "beetle"
-  s.version = "0.3.0.rc.2"
+  s.version = Beetle::VERSION
   s.required_rubygems_version = ">= 1.3.7"
-  s.authors            = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke"]
+  s.authors            = ["Stefan Kaes", "Pascal Friederich", "Ali Jelveh", "Sebastian Roebke", "Larry Baltz"]
   s.date               = Time.now.strftime('%Y-%m-%d')
   s.default_executable = "beetle"
   s.description        = "A highly available, reliable messaging infrastructure"
   s.summary            = "High Availability AMQP Messaging with Redundant Queues"
-  s.email              = "developers@xing.com"
+  s.email              = "opensource@xing.com"
   s.executables        = ["beetle"]
   s.extra_rdoc_files   = Dir['**/*.rdoc'] + %w(MIT-LICENSE)
   s.files              = Dir['{examples,lib}/**/*.rb'] + Dir['{features,script}/**/*'] + %w(beetle.gemspec Rakefile)
@@ -29,9 +33,12 @@ Gem::Specification.new do |s|
 
   s.specification_version = 3
   s.add_runtime_dependency("uuid4r",                [">= 0.1.2"])
-  s.add_runtime_dependency("bunny",                 ["~> 0.7.1"])
-  s.add_runtime_dependency("redis",                 ["= 2.0.4"])
-  s.add_runtime_dependency("amqp",                  ["~> 0.6.7"])
+  s.add_runtime_dependency("bunny",                 ["= 0.7.8"])
+  s.add_runtime_dependency("redis",                 ["= 2.2.0"])
+  s.add_runtime_dependency("hiredis",               ["= 0.3.2"])
+  s.add_runtime_dependency("amq-client",            ["= 0.8.3"])
+  s.add_runtime_dependency("amq-protocol",          ["= 0.8.1"])
+  s.add_runtime_dependency("amqp",                  ["= 0.8.0"])
   s.add_runtime_dependency("activesupport",         [">= 2.3.4"])
   s.add_runtime_dependency("daemons",               [">= 1.0.10"])
   s.add_development_dependency("rake",              [">= 0.8.7"])

data/examples/nonexistent_server.rb

@@ -0,0 +1,28 @@
+# nonexistent_server.rb
+# this example shows what happens when you try connect to a nonexistent server
+#
+# start it with ruby nonexistent_server.rb
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+Beetle.config.servers = "unknown.railsexpress.de:5672"
+
+# setup client
+client = Beetle::Client.new
+client.register_queue(:test)
+client.register_message(:test)
+
+# register our handler to the message, check out the message.rb for more stuff you can get from the message object
+client.register_handler(:test) {|message| puts "got message: #{message.data}"}
+
+# start listening
+# this starts the event machine event loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+client.listen do
+  EM.add_timer(10) { client.stop_listening }
+end
+

data/examples/redundant.rb

@@ -58,7 +58,7 @@ end
 # this starts the event machine event loop using EM.run
 # the block passed to listen will be yielded as the last step of the setup process
 client.listen do
-  EM.add_timer(0.1) { client.stop_listening }
+  EM.add_timer(0.2) { client.stop_listening }
 end
 
 puts "Received #{k} test messages"

data/examples/rpc.rb

@@ -3,7 +3,7 @@ require "rubygems"
 require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
 
 # suppress debug messages
-Beetle.config.logger.level = Logger::DEBUG
+Beetle.config.logger.level = Logger::INFO
 Beetle.config.servers = "localhost:5672, localhost:5673"
 # instantiate a client
 
@@ -28,12 +28,12 @@ if ARGV.include?("--server")
     trap("INT") { puts "stopped echo server"; client.stop_listening }
   end
 else
-  n = 100
+  n = 10000
   ms = Benchmark.ms do
     n.times do |i|
       content = "Hello #{i}"
       # puts "performing RPC with message content '#{content}'"
-      status, result = client.rpc(:echo, content)
+      status, result = client.rpc(:echo, content, :persistent => false)
       # puts "status  #{status}"
       # puts "result  #{result}"
       # puts

data/lib/beetle/base.rb

@@ -33,6 +33,10 @@ module Beetle
       @server = s
     end
 
+    def server_from_settings(settings)
+      settings.values_at(:host,:port).join(':')
+    end
+
     def each_server
       @servers.each { |s| set_current_server(s); yield }
     end

data/lib/beetle/client.rb

@@ -289,7 +289,7 @@ module Beetle
     end
 
     def queue_name_for_tracing(queue)
-      "trace-#{queue}-#{`hostname`.chomp}-#{$$}"
+      "trace-#{queue}-#{Beetle.hostname}-#{$$}"
     end
   end
 end

data/lib/beetle/message.rb

@@ -73,9 +73,10 @@ module Beetle
 
     # extracts various values form the AMQP header properties
     def decode #:nodoc:
-      amqp_headers = header.properties
+      # p header.attributes
+      amqp_headers = header.attributes
       @uuid = amqp_headers[:message_id]
-      headers = amqp_headers[:headers]
+      headers = amqp_headers[:headers].symbolize_keys
       @format_version = headers[:format_version].to_i
       @flags = headers[:flags].to_i
       @expires_at = headers[:expires_at].to_i

data/lib/beetle/redis_configuration_client.rb

@@ -25,7 +25,7 @@ module Beetle
 
     # Unique id for this instance (defaults to the fully qualified hostname)
     def id
-      @id ||= `hostname -f`.chomp
+      @id ||= Beetle.hostname
     end
 
     def initialize #:nodoc:

data/lib/beetle/redis_configuration_server.rb

@@ -1,8 +1,8 @@
 module Beetle
   # A RedisConfigurationServer is the supervisor part of beetle's
-  # redis failover solution
+  # redis failover solution.
   #
-  # An single instance of RedisConfigurationServer works as a supervisor for
+  # A single instance of RedisConfigurationServer works as a supervisor for
   # several RedisConfigurationClient instances. It is responsible for watching
   # the redis master and electing and publishing a new master in case of failure.
   #
@@ -75,6 +75,7 @@ module Beetle
       end
     end
 
+    # called by the message dispatcher when a "client_started" message from a RedisConfigurationClient is received
     def client_started(payload)
       id = payload["id"]
       if client_id_valid?(id)
@@ -186,7 +187,7 @@ module Beetle
       end
       known_client
     end
-    
+
     def client_id_valid?(client_id)
       @client_ids.include?(client_id)
     end

data/lib/beetle/subscriber.rb

@@ -1,5 +1,4 @@
 require 'amqp'
-require 'mq'
 
 module Beetle
   # Manages subscriptions and message processing on the receiver side of things.
@@ -10,9 +9,10 @@ module Beetle
       super
       @servers.concat @client.additional_subscription_servers
       @handlers = {}
-      @amqp_connections = {}
-      @mqs = {}
+      @connections = {}
+      @channels = {}
       @subscriptions = {}
+      @listened_queues = []
     end
 
     # the client calls this method to subscribe to a list of queues.
@@ -24,11 +24,12 @@ module Beetle
     #
     # yields before entering the eventmachine loop (if a block was given)
     def listen_queues(queues) #:nodoc:
+      @listened_queues = queues
+      @exchanges_for_queues = exchanges_for_queues(queues)
       EM.run do
-        exchanges = exchanges_for_queues(queues)
-        create_exchanges(exchanges)
-        bind_queues(queues)
-        subscribe_queues(queues)
+        each_server do
+          connect_server connection_settings
+        end
         yield if block_given?
       end
     end
@@ -47,10 +48,10 @@ module Beetle
 
     # closes all AMQP connections and stops the eventmachine loop
     def stop! #:nodoc:
-      if @amqp_connections.empty?
+      if @connections.empty?
         EM.stop_event_loop
       else
-        server, connection = @amqp_connections.shift
+        server, connection = @connections.shift
         logger.debug "Beetle: closing connection to #{server}"
         connection.close { stop! }
       end
@@ -74,30 +75,19 @@ module Beetle
     end
 
     def create_exchanges(exchanges)
-      each_server do
-        exchanges.each { |name| exchange(name) }
-      end
+      exchanges.each { |name| exchange(name) }
     end
 
     def bind_queues(queues)
-      each_server do
-        queues.each { |name| queue(name) }
-      end
+      queues.each { |name| queue(name) }
     end
 
     def subscribe_queues(queues)
-      each_server do
-        queues.each { |name| subscribe(name) if @handlers.include?(name) }
-      end
+      queues.each { |name| subscribe(name) if @handlers.include?(name) }
     end
 
-    # returns the mq object for the given server or returns a new one created with the
-    # prefetch(1) option. this tells it to just send one message to the receiving buffer
-    # (instead of filling it). this is necesssary to ensure that one subscriber always just
-    # handles one single message. we cannot ensure reliability if the buffer is filled with
-    # messages and crashes.
-    def mq(server=@server)
-      @mqs[server] ||= MQ.new(amqp_connection).prefetch(1)
+    def channel(server=@server)
+      @channels[server]
     end
 
     def subscriptions(server=@server)
@@ -116,12 +106,8 @@ module Beetle
       callback = create_subscription_callback(queue_name, amqp_queue_name, handler, opts)
       keys = opts.slice(*SUBSCRIPTION_KEYS).merge(:key => "#", :ack => true)
       logger.debug "Beetle: subscribing to queue #{amqp_queue_name} with key # on server #{@server}"
-      begin
-        queues[queue_name].subscribe(keys, &callback)
-        subscriptions[queue_name] = [keys, callback]
-      rescue MQ::Error
-        error("Beetle: binding multiple handlers for the same queue isn't possible.")
-      end
+      queues[queue_name].subscribe(keys, &callback)
+      subscriptions[queue_name] = [keys, callback]
     end
 
     def pause(queue_name)
@@ -147,13 +133,14 @@ module Beetle
           if result.reject?
             sleep 1
             header.reject(:requeue => true)
-          elsif reply_to = header.properties[:reply_to]
+          elsif reply_to = header.attributes[:reply_to]
+            # logger.info "Beetle: sending reply to queue #{reply_to}"
             # require 'ruby-debug'
             # Debugger.start
             # debugger
             status = result == Beetle::RC::OK ? "OK" : "FAILED"
-            exchange = MQ::Exchange.new(mq(server), :direct, "", :key => reply_to)
-            exchange.publish(m.handler_result.to_s, :headers => {:status => status})
+            exchange = AMQP::Exchange.new(channel(server), :direct, "")
+            exchange.publish(m.handler_result.to_s, :routing_key => reply_to, :persistent => false, :headers => {:status => status})
           end
           # logger.debug "Beetle: processed message"
         rescue Exception
@@ -168,27 +155,63 @@ module Beetle
     end
 
     def create_exchange!(name, opts)
-      mq.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
+      channel.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
     end
 
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      queue = mq.queue(queue_name, creation_keys)
+      queue = channel.queue(queue_name, creation_keys)
       exchange = exchange(exchange_name)
       queue.bind(exchange, binding_keys)
       queue
     end
 
-    def amqp_connection(server=@server)
-      @amqp_connections[server] ||= new_amqp_connection
+    def connection_settings
+      {
+        :host => current_host, :port => current_port, :logging => false,
+        :user => Beetle.config.user, :pass => Beetle.config.password, :vhost => Beetle.config.vhost,
+        :on_tcp_connection_failure => on_tcp_connection_failure
+      }
+    end
+
+    def on_tcp_connection_failure
+      Proc.new do |settings|
+        logger.warn "Beetle: connection failed: #{server_from_settings(settings)}"
+        EM::Timer.new(10) { connect_server(settings) }
+      end
     end
 
-    def new_amqp_connection
-      # FIXME: wtf, how to test that reconnection feature....
-      con = AMQP.connect(:host => current_host, :port => current_port, :logging => false,
-                         :user => Beetle.config.user, :pass => Beetle.config.password, :vhost => Beetle.config.vhost)
-      con.instance_variable_set("@on_disconnect", proc{ con.__send__(:reconnect) })
-      con
+    def on_tcp_connection_loss(connection, settings)
+      # reconnect in 10 seconds, without enforcement
+      logger.warn "Beetle: lost connection: #{server_from_settings(settings)}. reconnecting."
+      connection.reconnect(false, 10)
     end
 
+    def connect_server(settings)
+      server = server_from_settings settings
+      logger.info "Beetle: connecting to rabbit #{server}"
+      AMQP.connect(settings) do |connection|
+        connection.on_tcp_connection_loss(&method(:on_tcp_connection_loss))
+        @connections[server] = connection
+        open_channel_and_subscribe(connection, settings)
+      end
+    rescue EventMachine::ConnectionError => e
+      # something serious went wrong, for example DNS lookup failure
+      # in this case, the on_tcp_connection_failure callback is never called automatically
+      logger.error "Beetle: connection failed: #{e.class}(#{e})"
+      settings[:on_tcp_connection_failure].call(settings)
+    end
+
+    def open_channel_and_subscribe(connection, settings)
+      server = server_from_settings settings
+      AMQP::Channel.new(connection) do |channel|
+        channel.auto_recovery = true
+        channel.prefetch(1)
+        set_current_server server
+        @channels[server] = channel
+        create_exchanges(@exchanges_for_queues)
+        bind_queues(@listened_queues)
+        subscribe_queues(@listened_queues)
+      end
+    end
   end
 end

data/lib/beetle/version.rb

@@ -0,0 +1,3 @@
+module Beetle
+  VERSION = "0.3.0.rc.5"
+end

data/lib/beetle.rb

@@ -1,12 +1,27 @@
 $:.unshift(File.expand_path('..', __FILE__))
 require 'bunny'
 require 'uuid4r'
+require 'redis/connection/hiredis' # require *before* redis as specified in the redis-rb gem docs
+require 'redis'
 require 'active_support'
 require 'active_support/core_ext'
-require 'redis'
 require 'set'
+require 'socket'
 
 module Beetle
+  Timer = if RUBY_VERSION < "1.9"
+            begin
+              require 'system_timer'
+              SystemTimer
+            rescue Exception => e
+              warn "WARNING: It's highly recommended to install the SystemTimer gem: `gem install SystemTimer -v '=1.2.1'` See: http://ph7spot.com/musings/system-timer" if RUBY_VERSION < "1.9"
+              require 'timeout'
+              Timeout
+            end
+          else
+            require 'timeout'
+            Timeout
+          end
 
   # abstract superclass for Beetle specific exceptions
   class Error < StandardError; end
@@ -32,6 +47,13 @@ module Beetle
   # AMQP options for subscribing to queues
   SUBSCRIPTION_KEYS       = [:ack, :key]
 
+  # determine the fully qualified domainname of the host we're running on
+  def self.hostname
+    name = Socket.gethostname
+    parts = name.split('.')
+    parts.size > 1 ? name : Socket.gethostbyname(parts.first).first
+  end
+
   # use ruby's autoload mechanism for loading beetle classes
   lib_dir = File.expand_path(File.dirname(__FILE__) + '/beetle/')
   Dir["#{lib_dir}/*.rb"].each do |libfile|
@@ -56,11 +78,4 @@ module Beetle
     def self.reraise_expectation_errors! #:nodoc:
     end
   end
-
-  Timer = begin
-    RUBY_VERSION < "1.9" ? SystemTimer : Timeout
-  rescue NameError
-    warn "WARNING: It's highly recommended to install the SystemTimer gem: `gem install SystemTimer -v '=1.2.1'` See: http://ph7spot.com/musings/system-timer" if RUBY_VERSION < "1.9"
-    Timeout
-  end
 end

data/test/beetle/amqp_gem_behavior_test.rb

@@ -0,0 +1,30 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require 'eventmachine'
+require 'amqp'
+
+class AMQPGemBehaviorTest < Test::Unit::TestCase
+  test "subscribing twice to the same queue raises a RuntimeError which throws us out of the event loop" do
+    begin
+      @exception = nil
+      EM.run do
+        AMQP.start do |connection|
+          begin
+            EM::Timer.new(1){ connection.close { EM.stop }}
+            channel = AMQP::Channel.new(connection)
+            channel.on_error { puts "woot"}
+            exchange = channel.topic("beetle_tests")
+            queue = AMQP::Queue.new(channel)
+            queue.bind(exchange, :key => "#")
+            queue.subscribe { }
+            queue.subscribe { }
+          rescue
+            # we never get here, because the subscription is deferred
+            # the only known way to avoid this is to use the block version of AMQP::Queue.new
+          end
+        end
+      end
+    rescue Exception => @exception
+    end
+    assert @exception
+  end
+end

data/test/beetle/base_test.rb

@@ -48,5 +48,9 @@ module Beetle
       @bs.send(:set_current_server, "xxx:123")
       assert_equal "xxx:123", @bs.server
     end
+
+    test "server_from_settings should create a valid server string from an AMQP settings hash" do
+      assert_equal "goofy:123", @bs.send(:server_from_settings, {:host => "goofy", :port => 123})
+    end
   end
 end

data/test/beetle/deduplication_store_test.rb

@@ -9,8 +9,8 @@ module Beetle
     end
 
     test "trying to delete a non existent key doesn't throw an error" do
-      assert !@r.del("hahahaha")
       assert !@r.exists("hahahaha")
+      assert_equal 0, @r.del("hahahaha")
     end
 
     test "msetnx returns 0 or 1" do

data/test/beetle/redis_ext_test.rb

@@ -55,11 +55,9 @@ module Beetle
     end
   end
 
-  class RedisTimeoutTest < Test::Unit::TestCase
-    test "should use a timer" do
-      r = Redis.new(:host => "localhost", :port => 6390, :timeout => 1)
-      r.client.expects(:with_timeout).with(1).raises(Timeout::Error)
-      assert_equal({}, r.info_with_rescue)
+  class HiredisLoadedTest < Test::Unit::TestCase
+    test 'should be using hiredis instead of the redis ruby backend' do
+      assert defined?(Hiredis)
     end
   end
 end