beetle 0.3.14 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9af32086ee6f83824426aadee6665688b19de6f
-  data.tar.gz: f557cf6385121b1a6bcae4605d69b391d88e009b
+  metadata.gz: 4440f43f6c12064520bcf671c092da94206d83c4
+  data.tar.gz: 8f23d433a36db5942fe08b15a2024f8d979cc28b
 SHA512:
-  metadata.gz: 7dae19283de63eed536e58ff2378f48b0ee151dad24502668b9e6f062a801090c926736c546cb5a3ff377568ceb2dae671418c79329c1fbdd441b7d0706848be
-  data.tar.gz: c007937c6d910c75b16fad962e2cdf4bf8f11455ee5f68dd64004892740580fbc002fe4396aa4d4e3f1b103699d834de070b747ea3a4a467901224260fcb9f84
+  metadata.gz: 975fc5dbb57fdf533628a11ae7f77518945e548e7f4c041961663928efa438329d178b3fc7724af56f890542144578b4201318708352ab23418c44baa440e91b
+  data.tar.gz: 2b641c7cb51d17243454f5a52ace536825dda13f211f1e37660cb7d6f014d4cf6ba1e5d695de6ea126e2af795c78212d8afc3726c427d40361fcfc02553a3406

data/README.rdoc

@@ -99,7 +99,8 @@ For tests, you'll need
 
 {Stefan Kaes}[http://github.com/skaes],
 {Pascal Friederich}[http://github.com/paukul],
-{Ali Jelveh}[http://github.com/dudemeister] and
+{Ali Jelveh}[http://github.com/dudemeister],
+{Bjoern Rochel}[http://github.com/bjro] and
 {Sebastian Roebke}[http://github.com/boosty].
 
 You can find out more about our work on our {dev blog}[http://devblog.xing.com].

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,8 @@
 = Release Notes
 
+== Version 0.4.0
+* Added optional dead lettering feature to mimic RabbitMQ 2.x requeueing behaviour on RabbitMQ 3.x
+
 == Version 0.3.14
 * switched message id generation to use v4 uuids
 

data/beetle.gemspec

@@ -32,4 +32,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency("activesupport",           [">= 2.3.4"])
   s.add_runtime_dependency("eventmachine_httpserver", [">= 0.2.1"])
   s.add_runtime_dependency("daemons",                 [">= 1.0.10"])
+
+  s.add_development_dependency("webmock", [">= 1.21.0"])
 end

data/examples/attempts.rb

@@ -15,8 +15,11 @@ Beetle.config.logger.level = Logger::INFO
 
 # setup client
 $client = Beetle::Client.new
-$client.register_queue(:test)
-$client.register_message(:test)
+$client.config.dead_lettering_enabled = true
+$client.configure(:key => "my.test.message") do
+  message(:test)
+  queue(:test)
+end
 
 # purge the test queue
 $client.purge(:test)
@@ -35,6 +38,12 @@ class Handler < Beetle::Handler
 
   # called when the handler receives the message - fail everytime
   def process
+    logger.info "received message with routing key: #{message.routing_key}"
+    death = message.header.attributes[:headers]["x-death"]
+    if death
+      logger.info "X-DEATH: Message is coming back from dead letter queue (#{death.first["count"]})"
+      death.each {|d| logger.debug d}
+    end
     raise "failed #{$exceptions += 1} times"
   end
 

data/lib/beetle/base.rb

@@ -12,6 +12,7 @@ module Beetle
       @server = @servers[rand @servers.size]
       @exchanges = {}
       @queues = {}
+      @dead_lettering = DeadLettering.new(@client.config)
     end
 
     private

data/lib/beetle/configuration.rb

@@ -47,6 +47,31 @@ module Beetle
     attr_accessor :user
     # the password to use when connectiong to the AMQP servers (defaults to <tt>"guest"</tt>)
     attr_accessor :password
+    # the maximum permissible size of a frame (in bytes). Defaults to 128 KB
+    attr_accessor :frame_max
+
+    # In contrast to RabbitMQ 2.x, RabbitMQ 3.x preserves message order when requeing a message. This can lead to
+    # throughput degradation (when rejected messages block the processing of other messages
+    # at the head of the queue) in some cases.
+    #
+    # This setting enables the creation of dead letter queues that mimic the old beetle behaviour on RabbitMQ 3.x.
+    # Instead of rejecting messages with "requeue => true", beetle will setup dead letter queues for all queues, will
+    # reject messages with "requeue => false", where messages are temporarily moved to the side and are republished to
+    # the end of the original queue when they expire in the dead letter queue.
+    #
+    # By default this is turned off and needs to be explicitly enabled.
+    attr_accessor :dead_lettering_enabled
+    alias_method :dead_lettering_enabled?, :dead_lettering_enabled
+
+    # the time a message spends in the dead letter queue if dead lettering is enabled, before it is returned
+    # to the original queue
+    attr_accessor :dead_lettering_msg_ttl
+
+    # Read timeout for http requests to create dead letter bindings
+    attr_accessor :dead_lettering_read_timeout
+
+    # Returns the port on which the Rabbit API is hosted
+    attr_accessor :api_port
 
     # the socket timeout in seconds for message publishing (defaults to <tt>0</tt>).
     # consider this a highly experimental feature for now.
@@ -85,6 +110,12 @@ module Beetle
       self.vhost = "/"
       self.user = "guest"
       self.password = "guest"
+      self.api_port = 15672
+      self.frame_max = 131072
+
+      self.dead_lettering_enabled = false
+      self.dead_lettering_msg_ttl = 1000 #1 second
+      self.dead_lettering_read_timeout = 3 #3 seconds
 
       self.publishing_timeout = 0
       self.tmpdir = "/tmp"

data/lib/beetle/dead_lettering.rb

@@ -0,0 +1,94 @@
+require 'net/http'
+require 'json'
+
+module Beetle
+  class DeadLettering
+    class FailedRabbitRequest < StandardError; end
+
+    def initialize(config)
+      @config = config
+    end
+
+    def bind_dead_letter_queues!(channel, servers, target_queue, creation_keys = {})
+      return unless @config.dead_lettering_enabled?
+
+      dead_letter_queue_name = dead_letter_queue_name(target_queue)
+
+      logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
+      dead_letter_queue = channel.queue(dead_letter_queue_name, creation_keys)
+
+      logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
+      set_dead_letter_policies!(servers, target_queue)
+
+      logger.debug("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
+      set_dead_letter_policies!(
+        servers,
+        dead_letter_queue_name,
+        :message_ttl => @config.dead_lettering_msg_ttl,
+        :routing_key => target_queue
+      )
+    end
+
+    def set_dead_letter_policies!(servers, queue_name, options={})
+      servers.each { |server| set_dead_letter_policy!(server, queue_name, options) }
+    end
+
+    def set_dead_letter_policy!(server, queue_name, options={})
+      raise ArgumentError.new("server missing")     if server.blank?
+      raise ArgumentError.new("queue name missing") if queue_name.blank?
+
+      vhost = CGI.escape(@config.vhost)
+      request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
+      request = Net::HTTP::Put.new(request_url)
+
+      request_body = {
+        "pattern" => "^#{queue_name}$",
+        "priority" => 1,
+        "apply-to" => "queues",
+        "definition" => {
+          "dead-letter-routing-key" => dead_letter_routing_key(queue_name, options),
+          "dead-letter-exchange" => ""
+        }
+      }
+
+      request_body["definition"].merge!("message-ttl" => options[:message_ttl]) if options[:message_ttl]
+
+      response = run_rabbit_http_request(request_url, request) do |http|
+        http.request(request, request_body.to_json)
+      end
+
+      if response.code != "204"
+        log_error("Failed to create policy for queue #{queue_name}", response)
+        raise FailedRabbitRequest.new("Could not create policy")
+      end
+
+      :ok
+    end
+
+    def dead_letter_routing_key(queue_name, options)
+      options.fetch(:routing_key) { dead_letter_queue_name(queue_name) }
+    end
+
+    def dead_letter_queue_name(queue_name)
+      "#{queue_name}_dead_letter"
+    end
+
+    def run_rabbit_http_request(uri, request, &block)
+      request.basic_auth(@config.user, @config.password)
+      request["Content-Type"] = "application/json"
+      response = Net::HTTP.start(uri.hostname, @config.api_port, :read_timeout => @config.dead_lettering_read_timeout) do |http|
+        block.call(http) if block_given?
+      end
+    end
+
+    def log_error(msg, response)
+      logger.error(msg)
+      logger.error("Response code was #{response.code}")
+      logger.error(response.body)
+    end
+
+    def logger
+      @config.logger
+    end
+  end
+end

data/lib/beetle/message.rb

@@ -103,7 +103,11 @@ module Beetle
 
     # the routing key
     def routing_key
-      header.routing_key
+      @routing_key ||= if x_death = header.attributes[:headers]["x-death"]
+        x_death.last["routing-keys"].first
+      else
+        header.routing_key
+      end
     end
     alias_method :key, :routing_key
 

data/lib/beetle/publisher.rb

@@ -149,9 +149,15 @@ module Beetle
     end
 
     def new_bunny
-      b = Bunny.new(:host => current_host, :port => current_port, :logging => !!@options[:logging],
-                    :user => Beetle.config.user, :pass => Beetle.config.password, :vhost => Beetle.config.vhost,
-                    :socket_timeout => Beetle.config.publishing_timeout)
+      b = Bunny.new(
+        :host           => current_host,
+        :port           => current_port,
+        :logging        => !!@options[:logging],
+        :user           => @client.config.user,
+        :pass           => @client.config.password,
+        :vhost          => @client.config.vhost,
+        :frame_max      => @client.config.frame_max,
+        :socket_timeout => @client.config.publishing_timeout)
       b.start
       b
     end
@@ -195,10 +201,11 @@ module Beetle
       @exchanges_with_bound_queues[exchange_name] = true
     end
 
-    # TODO: Refactor, fethch the keys and stuff itself
+    # TODO: Refactor, fetch the keys and stuff itself
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
       logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
       queue = bunny.queue(queue_name, creation_keys)
+      @dead_lettering.bind_dead_letter_queues!(bunny, @client.servers, queue_name, creation_keys)
       logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
       queue.bind(exchange(exchange_name), binding_keys)
       queue

data/lib/beetle/subscriber.rb

@@ -144,8 +144,12 @@ module Beetle
           m = Message.new(amqp_queue_name, header, data, message_options)
           result = m.process(processor)
           if result.reject?
-            sleep 1
-            header.reject(:requeue => true)
+            if @client.config.dead_lettering_enabled?
+              header.reject(:requeue => false)
+            else
+              sleep 1
+              header.reject(:requeue => true)
+            end
           elsif reply_to = header.attributes[:reply_to]
             # logger.info "Beetle: sending reply to queue #{reply_to}"
             # require 'ruby-debug'
@@ -177,6 +181,8 @@ module Beetle
 
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
       queue = channel.queue(queue_name, creation_keys)
+      target_servers = @client.servers + @client.additional_subscription_servers
+      @dead_lettering.bind_dead_letter_queues!(channel, target_servers, queue_name, creation_keys)
       exchange = exchange(exchange_name)
       queue.bind(exchange, binding_keys)
       queue

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "0.3.14"
+  VERSION = "0.4.0"
 end

data/test/beetle/dead_lettering_test.rb

@@ -0,0 +1,130 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+  class SetDeadLetteringsTest < MiniTest::Unit::TestCase
+    def setup
+      @dead_lettering = DeadLettering.new(Configuration.new)
+    end
+
+    test "creates a dead letter queue for each server" do
+      servers = %w(a b)
+
+      @dead_lettering.expects(:set_dead_letter_policy!).
+        with("a", "QUEUE_NAME", :message_ttl => 10000)
+      @dead_lettering.expects(:set_dead_letter_policy!).
+        with("b", "QUEUE_NAME", :message_ttl => 10000)
+
+      @dead_lettering.set_dead_letter_policies!(servers, "QUEUE_NAME", :message_ttl => 10000)
+    end
+  end
+
+  class SetDeadLetterPolicyTest < MiniTest::Unit::TestCase
+    def setup
+      @server = "localhost:15672"
+      @queue_name = "QUEUE_NAME"
+      @config = Configuration.new
+      @config.logger = Logger.new("/dev/null")
+      @dead_lettering = DeadLettering.new(@config)
+    end
+
+    test "raises exception when queue name wasn't specified" do
+      assert_raises ArgumentError do
+        @dead_lettering.set_dead_letter_policy!(@server, "")
+      end
+    end
+
+    test "raises exception when no server was specified" do
+      assert_raises ArgumentError do
+        @dead_lettering.set_dead_letter_policy!("", @queue_name)
+      end
+    end
+
+    test "creates a policy by posting to the rabbitmq" do
+      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
+        with(:body => {
+        "pattern" => "^QUEUE_NAME$",
+        "priority" => 1,
+        "apply-to" => "queues",
+        "definition" => {
+          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+          "dead-letter-exchange" => ""
+        }}.to_json).
+        to_return(:status => 204)
+
+      @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+    end
+
+    test "raises exception when policy couldn't successfully be created" do
+      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
+        to_return(:status => [405])
+
+      assert_raises DeadLettering::FailedRabbitRequest do
+        @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+      end
+    end
+
+    test "can optionally specify a message ttl" do
+      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
+        with(:body => {
+        "pattern" => "^QUEUE_NAME$",
+        "priority" => 1,
+        "apply-to" => "queues",
+        "definition" => {
+          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+          "dead-letter-exchange" => "",
+          "message-ttl" => 10000
+        }}.to_json).
+        to_return(:status => 204)
+
+      @dead_lettering.set_dead_letter_policy!(@server, @queue_name, :message_ttl => 10000)
+    end
+
+    test "properly encodes the vhost from the configuration" do
+      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy").
+        with(:body => {
+        "pattern" => "^QUEUE_NAME$",
+        "priority" => 1,
+        "apply-to" => "queues",
+        "definition" => {
+          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+          "dead-letter-exchange" => ""
+        }}.to_json).
+        to_return(:status => 204)
+
+      @config.vhost = "foo/"
+
+      @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+    end
+  end
+
+  class BindDeadLetterQueuesTest < MiniTest::Unit::TestCase
+    def setup
+      @queue_name = "QUEUE_NAME"
+      @config = Configuration.new
+      @config.logger = Logger.new("/dev/null")
+      @dead_lettering = DeadLettering.new(@config)
+      @servers = ["localhost:55672"]
+    end
+
+    test "is turned off by default" do
+      channel = stub('channel')
+      @dead_lettering.expects(:set_dead_letter_policies!).never
+      @dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)
+    end
+
+    test "creates and connects the dead letter queue via policies when enabled" do
+      @config.dead_lettering_enabled = true
+
+      channel = stub('channel')
+
+      channel.expects(:queue).with("#{@queue_name}_dead_letter", {})
+      @dead_lettering.expects(:set_dead_letter_policies!).with(@servers, @queue_name)
+      @dead_lettering.expects(:set_dead_letter_policies!).with(@servers, "#{@queue_name}_dead_letter",
+        :routing_key => @queue_name,
+        :message_ttl => 1000
+      )
+
+      @dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)
+    end
+  end
+end

data/test/beetle/publisher_test.rb

@@ -19,7 +19,12 @@ module Beetle
       m = mock("dummy")
       expected_bunny_options = {
         :host => @pub.send(:current_host), :port => @pub.send(:current_port),
-        :logging => false, :user => "guest", :pass => "guest", :vhost => "/", :socket_timeout => 0
+        :logging => false,
+        :user => "guest",
+        :pass => "guest",
+        :vhost => "/",
+        :socket_timeout => 0,
+        :frame_max => 131072
       }
       Bunny.expects(:new).with(expected_bunny_options).returns(m)
       m.expects(:start)
@@ -230,7 +235,7 @@ module Beetle
       q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
       m = mock("Bunny")
       m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"foo" => "fighter"}).returns(q)
-      @pub.expects(:bunny).returns(m)
+      @pub.expects(:bunny).returns(m).twice
 
       @pub.send(:queue, "some_queue")
       assert_equal q, @pub.send(:queues)["some_queue"]

data/test/beetle/subscriber_test.rb

@@ -147,7 +147,7 @@ module Beetle
       q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
       m = mock("MQ")
       m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"schmu" => 5}).returns(q)
-      @sub.expects(:channel).returns(m)
+      @sub.expects(:channel).returns(m).twice
 
       @sub.send(:queue, "some_queue")
       assert_equal q, @sub.send(:queues)["some_queue"]

data/test/test_helper.rb

@@ -15,6 +15,8 @@ require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
 class MiniTest::Unit::TestCase
   require "active_support/testing/declarative"
   extend ActiveSupport::Testing::Declarative
+  require "webmock"
+  include WebMock::API
   def assert_nothing_raised(*)
     yield
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.4.0
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid4r
@@ -166,6 +166,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.10
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.21.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.21.0
 description: A highly available, reliable messaging infrastructure
 email: opensource@xing.com
 executables:
@@ -215,6 +229,7 @@ files:
 - lib/beetle/commands/configuration_server.rb
 - lib/beetle/commands/garbage_collect_deduplication_store.rb
 - lib/beetle/configuration.rb
+- lib/beetle/dead_lettering.rb
 - lib/beetle/deduplication_store.rb
 - lib/beetle/handler.rb
 - lib/beetle/logging.rb
@@ -236,6 +251,7 @@ files:
 - test/beetle/beetle_test.rb
 - test/beetle/client_test.rb
 - test/beetle/configuration_test.rb
+- test/beetle/dead_lettering_test.rb
 - test/beetle/deduplication_store_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/message_test.rb
@@ -278,6 +294,7 @@ test_files:
 - test/beetle/beetle_test.rb
 - test/beetle/client_test.rb
 - test/beetle/configuration_test.rb
+- test/beetle/dead_lettering_test.rb
 - test/beetle/deduplication_store_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/message_test.rb