beetle 0.3.0.rc.17 → 0.3.0.rc.18

data/README.rdoc

@@ -39,7 +39,7 @@ More information can be found on the {project website}[http://xing.github.com/be
     b.publish :test, "I'm a test message"
 
 === Subscribing
-    b.listen
+    b.listen_queues
 
 === Examples
 
@@ -65,16 +65,16 @@ To set up a redundant messaging system you will need
 At runtime, Beetle will use
 * {uuid4r}[http://github.com/skaes/uuid4r]
   (which needs ossp-uuid)
-* {bunny}[http://github.com/celldee/bunny]
-* {redis}[http://github.com/ezmobius/redis-rb]
-* {amqp}[http://github.com/tmm1/amqp]
+* {bunny}[http://github.com/ruby-amqp/bunny]
+* {redis}[http://github.com/redis/redis-rb]
+* {amqp}[http://github.com/ruby-amqp/amqp]
   (which is based on {eventmachine}[http://github.com/eventmachine/eventmachine])
 * {daemons}[http://daemons.rubyforge.org/]
 * activesupport
 
 For development, you'll need
 * {mocha}[http://github.com/floehopper/mocha]
-* {rcov}[http://github.com/relevance/rcov]
+* {rcov}[http://github.com/relevance/rcov] (for ruby 1.8.7)
 * {cucumber}[http://github.com/aslakhellesoy/cucumber]
 * {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
 

data/Rakefile

@@ -53,20 +53,35 @@ namespace :beetle do
 end
 
 namespace :rabbit do
-  def start(node_name, port)
+  def start(node_name, port, web_port)
     script = File.expand_path(File.dirname(__FILE__)+"/script/start_rabbit")
-    puts "starting rabbit #{node_name} on port #{port}"
+    # on my machine, the rabbitmq user is not be allowed to access my files.
+    # so we need to put the config file under /tmp
+    config_file = "/tmp/beetle-testing-rabbitmq-#{node_name}"
+
+    create_config_file config_file, web_port
+
+    puts "starting rabbit #{node_name} on port #{port}, web management port #{web_port}"
     puts "type ^C a RETURN to abort"
     sleep 1
-    exec "sudo #{script} #{node_name} #{port}"
+    exec "sudo #{script} #{node_name} #{port} #{config_file}"
+  end
+
+  def create_config_file(config_file, web_port)
+    File.open("#{config_file}.config",'w') do |f|
+      f.puts "["
+      f.puts "  {rabbitmq_management, [{listener, [{port, #{web_port}}]}]}"
+      f.puts "]."
+    end
   end
+
   desc "start rabbit instance 1"
   task :start1 do
-    start "rabbit1", 5672
+    start "rabbit1", 5672, 15672
   end
   desc "start rabbit instance 2"
   task :start2 do
-    start "rabbit2", 5673
+    start "rabbit2", 5673, 15673
   end
   desc "reset rabbit instances (deletes all data!)"
   task :reset do
@@ -109,7 +124,7 @@ end
 
 require 'rdoc/task'
 
-Rake::RDocTask.new do |rdoc|
+RDoc::Task.new do |rdoc|
   rdoc.rdoc_dir = 'site/rdoc'
   rdoc.title    = 'Beetle'
   rdoc.main     = 'README.rdoc'

data/beetle.gemspec

@@ -36,11 +36,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency("bunny",                   ["= 0.7.9"])
   s.add_runtime_dependency("redis",                   ["= 3.0.1"])
   s.add_runtime_dependency("hiredis",                 ["= 0.4.5"])
-  s.add_runtime_dependency("amq-client",              ["= 0.9.3"])
-  s.add_runtime_dependency("amq-protocol",            ["= 0.9.3"])
-  s.add_runtime_dependency("amqp",                    ["= 0.9.6"])
+  s.add_runtime_dependency("amq-client",              ["= 0.9.10"])
+  s.add_runtime_dependency("amq-protocol",            ["= 1.0.1"])
+  s.add_runtime_dependency("amqp",                    ["= 0.9.8"])
   s.add_runtime_dependency("activesupport",           [">= 2.3.4"])
   s.add_runtime_dependency("eventmachine_httpserver", [">= 0.2.1"])
   s.add_runtime_dependency("daemons",                 [">= 1.0.10"])
 end
-

data/features/support/test_daemons/redis.conf.erb

@@ -166,11 +166,6 @@ appendfsync everysec
 
 ############################### ADVANCED CONFIG ###############################
 
-# Glue small output buffers together in order to send small replies in a
-# single TCP packet. Uses a bit more CPU but most of the times it is a win
-# in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes
-
 # Use object sharing. Can save a lot of memory if you have many common
 # string in your dataset, but performs lookups against the shared objects
 # pool so it uses more CPU and can be a bit slower. Usually it's a good

data/lib/beetle.rb

@@ -1,5 +1,7 @@
 $:.unshift(File.expand_path('..', __FILE__))
-require 'bunny'
+require 'amq/client'               # defines AMQ::Client::Settings
+require 'bunny'                    # which bunny picks up
+require 'qrack/errors'             # needed by the publisher
 require 'uuid4r'
 require 'redis/connection/hiredis' # require *before* redis as specified in the redis-rb gem docs
 require 'redis'

data/lib/beetle/commands/garbage_collect_deduplication_store.rb

@@ -0,0 +1,50 @@
+require 'optparse'
+require 'beetle'
+
+module Beetle
+  module Commands
+    # Command to garbage collect the deduplication store
+    #
+    #   Usage: beetle garbage_collect_deduplication_store [options]
+    #
+    #   options:
+    #       --redis-servers LIST         Required (e.g. 192.168.0.1:6379,192.168.0.2:6379)
+    #       --config-file PATH           Path to an external yaml config file
+    #       -v, --verbose
+    #       -h, --help                   Show this message
+    #
+    class GarbageCollectDeduplicationStore
+      # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
+      def self.execute
+        opts = OptionParser.new
+        opts.banner = "Usage: beetle garbage_collect_deduplication_store [options]"
+        opts.separator ""
+
+        opts.on("--redis-servers LIST", Array, "Required (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
+          Beetle.config.redis_servers = val.join(",")
+        end
+
+        opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
+          Beetle.config.config_file = val
+        end
+
+        opts.on("--redis-db N", Integer, "Redis database used for GC") do |val|
+          Beetle.config.redis_db = val.to_i
+        end
+
+        opts.on("-v", "--verbose") do |val|
+          Beetle.config.logger.level = Logger::DEBUG
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.parse!(ARGV)
+
+        DeduplicationStore.new.garbage_collect_keys_using_master_and_slave
+      end
+    end
+  end
+end

data/lib/beetle/deduplication_store.rb

@@ -48,16 +48,69 @@ module Beetle
       key =~ /^(msgid:[^:]*:[-0-9a-f]*):.*$/ && $1
     end
 
+    # garbage collect using master and slave (and redis-cli)
+    def garbage_collect_keys_using_master_and_slave(now = Time.now.to_i)
+      logger.info "garbage collecting obsolete redis keys from #{@config.redis_servers}"
+      info = RedisServerInfo.new(@config, {})
+      info.refresh
+      unless connection = info.slaves.first
+        logger.warn "no slave available, falling back to master."
+        connection = redis
+      end
+      file = "/tmp/beetle_redis_expire_keys_#{$$}.txt"
+      cmd = "redis-cli -h #{connection.host} -p #{connection.port} -n #{@config.redis_db} keys 'msgid:*:expires' > #{file}"
+      logger.info "retrieving expire keys: '#{cmd}'"
+      if system(cmd)
+        garbage_collect_keys_from_file(file, now)
+      else
+        logger.error "could not retrieve expire keys"
+      end
+    rescue => e
+      logger.error "#{e.class}(#{e})"
+    ensure
+      system("rm -f #{file}")
+    end
+
     # garbage collect keys in Redis (always assume the worst!)
     def garbage_collect_keys(now = Time.now.to_i)
       keys = redis.keys("msgid:*:expires")
       threshold = now + @config.gc_threshold
       keys.each do |key|
-        expires_at = redis.get key
-        if expires_at && expires_at.to_i < threshold
+        gc_key(key, threshold)
+      end
+    end
+
+    # garbage collect keys from a given file
+    def garbage_collect_keys_from_file(file_name, now = Time.now.to_i)
+      threshold = now + @config.gc_threshold
+      expired = total = 0
+      File.foreach(file_name) do |line|
+        line.chomp!
+        if line =~ /^msgid:.*:expires$/
+          total += 1
+          expired += 1 if gc_key(line, threshold)
+        end
+      end
+      logger.info "expired #{expired} keys out of #{total}"
+    end
+
+    # garbage collect a single key if it's older than given threshold
+    def gc_key(key, threshold)
+      expires_at = redis.get key
+      if expires_at
+        if expires_at.to_i < threshold
           msg_id = msg_id(key)
+          # puts "expiring: #{msg_id}"
           redis.del(*keys(msg_id))
+          return true
+        else
+          # keys has future expiry date
+          # puts "#{msg_id} expires on #{Time.at(expires_at.to_i)}"
+          return false
         end
+      else
+        # key might have been deleted in the meantime
+        return false
       end
     end
 

data/lib/beetle/message.rb

@@ -263,10 +263,12 @@ module Beetle
       elsif !timed_out?
         RC::HandlerNotYetTimedOut
       elsif attempts_limit_reached?
+        completed!
         ack!
         logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
         RC::AttemptsLimitReached
       elsif exceptions_limit_reached?
+        completed!
         ack!
         logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"
         RC::ExceptionsLimitReached
@@ -306,10 +308,12 @@ module Beetle
     def handler_failed!(result)
       increment_exception_count!
       if attempts_limit_reached?
+        completed!
         ack!
         logger.debug "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
         RC::AttemptsLimitReached
       elsif exceptions_limit_reached?
+        completed!
         ack!
         logger.debug "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"
         RC::ExceptionsLimitReached

data/lib/beetle/redis_server_info.rb

@@ -14,7 +14,7 @@ module Beetle
       @instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
     end
 
-    # fetches the server from the insatnces whith the given <tt>server</tt> string
+    # fetches the server from the instances whith the given <tt>server</tt> string
     def find(server)
       instances.find{|r| r.server == server}
     end

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "0.3.0.rc.17"
+  VERSION = "0.3.0.rc.18"
 end

data/script/start_rabbit

@@ -21,9 +21,12 @@ export RABBITMQ_NODENAME=$1
 export RABBITMQ_NODE_PORT=$2
 # Defaults to 5672.
 
+export RABBITMQ_CONFIG_FILE=$3
 # RABBITMQ_CLUSTER_CONFIG_FILE
 # Defaults to /etc/rabbitmq/rabbitmq_cluster.config. If this file is present it
 # is used by the server to auto-configure a RabbitMQ cluster.  See the clustering
 # guide at <http://www.rabbitmq.com/clustering.html> for details.
 
 rabbitmq-server
+
+rm -f $RABBITMQ_CONFIG_FILE

data/test/beetle/deduplication_store_test.rb

@@ -111,4 +111,26 @@ module Beetle
     end
   end
 
+  class GarbageCollectionTest < Test::Unit::TestCase
+    def setup
+      @store = DeduplicationStore.new
+    end
+
+    test "never tries to delete message keys when expire key doesn not exist" do
+      key = "foo"
+      @store.redis.del key
+      assert !@store.gc_key(key, 0)
+    end
+
+    test "rescues exeptions and logs an error" do
+      RedisServerInfo.expects(:new).raises("foo")
+      assert_nothing_raised { @store.garbage_collect_keys_using_master_and_slave }
+    end
+
+    test "logs an error when system command fails" do
+      @store.stubs(:system).returns(false)
+      @store.logger.expects(:error)
+      @store.garbage_collect_keys_using_master_and_slave
+    end
+  end
 end

data/test/beetle/message_test.rb

@@ -93,6 +93,16 @@ module Beetle
       @store.garbage_collect_keys(Time.now.to_i+1)
     end
 
+    test "should be able to garbage collect expired keys using master and slave" do
+      Beetle.config.expects(:gc_threshold).returns(0)
+      header = header_with_params({:ttl => 0})
+      message = Message.new("somequeue", header, 'foo', :store => @store)
+      assert !message.key_exists?
+      assert message.key_exists?
+      @store.redis.expects(:del).with(*@store.keys(message.msg_id))
+      @store.garbage_collect_keys_using_master_and_slave(Time.now.to_i+1)
+    end
+
     test "should not garbage collect not yet expired keys" do
       Beetle.config.expects(:gc_threshold).returns(0)
       header = header_with_params({:ttl => 0})
@@ -348,7 +358,7 @@ module Beetle
 
       proc = lambda {|*args| raise "crash"}
       s = sequence("s")
-      message.expects(:completed!).never
+      message.expects(:completed!).once
       header.expects(:ack)
       assert_equal RC::ExceptionsLimitReached, message.__send__(:process_internal, proc)
     end
@@ -363,7 +373,7 @@ module Beetle
 
       proc = lambda {|*args| raise "crash"}
       s = sequence("s")
-      message.expects(:completed!).never
+      message.expects(:completed!).once
       header.expects(:ack)
       assert_equal RC::AttemptsLimitReached, message.__send__(:process_internal, proc)
     end
@@ -765,4 +775,3 @@ module Beetle
   end
 
 end
-

data/test/beetle/redis_configuration_server_test.rb

@@ -324,5 +324,12 @@ module Beetle
       @server.beetle.expects(:publish).with(:system_notification, ({:message => msg}).to_json)
       @server.client_started(payload)
     end
+
+    test "should not send a system notification when receiving a client started message from a known client" do
+      payload = {"id" => "rc-client-1"}
+      msg = "Received client_started message from unknown id 'unknown-client'"
+      @server.beetle.expects(:publish).never
+      @server.client_started(payload)
+    end
   end
 end

data/test/colorized_test_output.rb

@@ -22,4 +22,4 @@ begin
   end
 rescue LoadError => e
   # do nothing
-end unless ENV['TM_FILENAME']
+end if $stdout.tty?

data/test/test_helper.rb

@@ -12,17 +12,13 @@ require 'active_support/testing/declarative'
 require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
 require File.expand_path(File.dirname(__FILE__) + '/colorized_test_output')
 
-# we can remove this hack which is needed only for testing
-require 'qrack/errors'
-
-
 class Test::Unit::TestCase
   extend ActiveSupport::Testing::Declarative
 end
 
-
 Beetle.config.logger = Logger.new(File.dirname(__FILE__) + '/../test.log')
 Beetle.config.redis_server = "localhost:6379"
+Beetle.config.redis_servers = "localhost:6379,localhost:6380"
 
 
 def header_with_params(opts = {})

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 0.3.0.rc.17
+  version: 0.3.0.rc.18
   prerelease: 6
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-21 00:00:00.000000000 Z
+date: 2013-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid4r
@@ -86,7 +86,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.9.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -94,7 +94,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.9.10
 - !ruby/object:Gem::Dependency
   name: amq-protocol
   requirement: !ruby/object:Gem::Requirement
@@ -102,7 +102,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -110,7 +110,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
@@ -118,7 +118,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: 0.9.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -126,7 +126,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: 0.9.8
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -203,6 +203,7 @@ files:
 - lib/beetle/client.rb
 - lib/beetle/commands/configuration_client.rb
 - lib/beetle/commands/configuration_server.rb
+- lib/beetle/commands/garbage_collect_deduplication_store.rb
 - lib/beetle/commands.rb
 - lib/beetle/configuration.rb
 - lib/beetle/deduplication_store.rb
@@ -276,7 +277,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -36003570872479659
+      hash: -3293468122773492105
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: