beetle 1.0.3 → 2.0.0rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0abca6f6500fbad2151808a116640dec7f32a5e5
-  data.tar.gz: 42cdc9233665f3d358a2870ab8ddd2f8723c0a78
+  metadata.gz: 5648bfe9f8b8f76d05d04b360df2e160c7f9ab6b
+  data.tar.gz: 5fdb6ec4eea81aa6e7d79bbbdd9ed820711bdb2e
 SHA512:
-  metadata.gz: 6c1808d2633eb06a7e6b2b85f7f33a10f4d95c2efe2b3cfeedbeed736656ad009215d294f5505806fdd26e8ceefb8cd1d0d01b8469a3212b04eea026e2ef1900
-  data.tar.gz: bfb7831e79ed4f90e5cc3afd1ebad94b63cc9ed3c5dd4ba0f202ecd1a709a9ace1e5dcd4ed23ee648375779b3d61650335b8d189d9b49b3a3718b34705407e0b
+  metadata.gz: e5e5f2ba5cb31942fbf01584cae8b448cec7db14e260050e74fb5765c9d29a23c9f7d1cd8e93cf2be04ea74d46739e57c45149fe9c4557e3fce77936814d6ed5
+  data.tar.gz: 28b1e12e6fc4b9858c6184ff93dbc876cc51ced13cac9f31664547b07fbf9dfa5557c83cc7d42a609cda20bee8f00b85d6a9520aa28290ab7172a141eaa7a93f

data/README.rdoc

@@ -29,10 +29,10 @@ More information can be found on the {project website}[http://xing.github.com/be
 
     # configure exchanges, queues, bindings, messages and handlers
 
-    b.configure do |config|
-      config.queue :test
-      config.message :test
-      config.handler(:test) { |message| puts message.data }
+    b.configure do
+      queue :test
+      message :test
+      handler(:test) { |message| puts message.data }
     end
 
 === Publishing
@@ -72,6 +72,13 @@ You also need a Redis instance running. The default configuration of Redis will
 
     redis-server
 
+If you want to run the integration tests you need GO installed and you
+will need to build the beetle binary. We provide a Makefile for this
+purpose, so simply running
+
+    make
+
+should suffice.
 
 == Gem Dependencies
 
@@ -127,8 +134,9 @@ Update RELEASE_NOTES.rdoc!
 We use {semantic versioning}[http://semver.org/] and create a git tag
 for each release.
 
-Edit `lib/beetle/version.rb` to set the new version number
-(`Major.Minor.Patch`).
+Edit `lib/beetle/version.rb` and
+`go/src/github.com/xing/beetle/version.go` to set the new version
+number (`Major.Minor.Patch`).
 
 In short (see {semver.org}[http://semver.org] for details):
 
@@ -141,7 +149,7 @@ In short (see {semver.org}[http://semver.org] for details):
   are introduced.
 
 Then use `rake release` which will create the git tag and upload the
-gem to gems.xing.com:
+gem to github.com:
 
     $ bundle exec rake release
 

data/REDIS_AUTO_FAILOVER.rdoc

@@ -35,9 +35,9 @@ On every worker server runs another daemon, the Redis Configuration Client
 
 If the RCS finds another potential Redis Master, it sends out a message to see
 if all known RCCs are still available (once again to eliminate the risk of a
-partitioned network) and if they agree to the master switch. 
+partitioned network) and if they agree to the master switch.
 
-If all RCCs have answered to that message, the RCS sends out a message which 
+If all RCCs have answered to that message, the RCS sends out a message which
 tells the RCCs to invalidate the current master.
 
 This happens by deleting the contents of a special file which is used
@@ -53,7 +53,7 @@ to the RCCs, which then write that value into their redis master file.
 
 Additionally, the RCS sends reconfigure messages with the current Redis master
 periodically, to allow new RCCs to pick up the current master. Plus it turns
-all other redis servers into slaves of the current master. 
+all other redis servers into slaves of the current master.
 
 === Prerequisites
 
@@ -93,15 +93,11 @@ All command line options can also be given as a yaml configuration file via the
 
 Start the Redis Configuration Server:
 
-  beetle configuration_server start -- --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
-
-Get help for starting/stopping the server:
-
-  beetle configuration_server -h
+  beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
 
 Get help for server options:
 
-  beetle configuration_server start -- -h
+  beetle configuration_server -h
 
 === On every worker server
 
@@ -109,16 +105,12 @@ Start the Redis Configuration Client:
 
 On first worker server:
 
-  beetle configuration_client start -- --client-id rcc-1
+  beetle configuration_client --client-id rcc-1
 
 On second worker server:
 
-  beetle configuration_client start -- --client-id rcc-2
-
-Get help for starting/stopping the client:
-
-  beetle configuration_client -h
+  beetle configuration_client --client-id rcc-2
 
 Get help for client options:
 
-  beetle configuration_client start -- -h
+  beetle configuration_client -h

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,14 @@
 = Release Notes
 
+== Version 2.0.0rc1
+
+* beetle command has been rewritten in GO
+* garbage collecting redis keys now uses the redis SCAN command
+
+== Version 1.0.4
+
+* use amqp protocol version 0.9 by default for publishers
+
 == Version 1.0.3
 
 * fixed that publisher did not allow specifying message properties

data/beetle.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.description        = "A highly available, reliable messaging infrastructure"
   s.summary            = "High Availability AMQP Messaging with Redundant Queues"
   s.email              = "opensource@xing.com"
-  s.executables        = ["beetle"]
+  s.executables        = []
   s.extra_rdoc_files   = Dir['**/*.rdoc'] + %w(MIT-LICENSE)
   s.files              = Dir['{examples,lib}/**/*.rb'] + Dir['{features,script}/**/*'] + %w(beetle.gemspec Rakefile)
   s.homepage           = "http://xing.github.com/beetle/"
@@ -21,13 +21,25 @@ Gem::Specification.new do |s|
   s.test_files         = Dir['test/**/*.rb']
 
   s.specification_version = 3
-  s.add_runtime_dependency("uuid4r",                  [">= 0.1.2"])
-  s.add_runtime_dependency("bunny",                   ["~> 0.7.10"])
-  s.add_runtime_dependency("redis",                   [">= 2.2.2"])
-  s.add_runtime_dependency("hiredis",                 [">= 0.4.5"])
-  s.add_runtime_dependency("amq-protocol",            ["= 2.0.1"])
-  s.add_runtime_dependency("amqp",                    ["= 1.6.0"])
-  s.add_runtime_dependency("activesupport",           [">= 2.3.4"])
-  s.add_runtime_dependency("eventmachine_httpserver", [">= 0.2.1"])
-  s.add_runtime_dependency("daemons",                 [">= 1.2.0"])
+  s.add_runtime_dependency "uuid4r",                  ">= 0.1.2"
+  s.add_runtime_dependency "bunny",                   "~> 0.7.10"
+  s.add_runtime_dependency "redis",                   ">= 2.2.2"
+  s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
+  s.add_runtime_dependency "amq-protocol",            "= 2.0.1"
+  s.add_runtime_dependency "amqp",                    "= 1.6.0"
+  s.add_runtime_dependency "activesupport",           ">= 2.3.4"
+
+  s.add_development_dependency "activerecord",        "~> 5.0"
+  s.add_development_dependency "cucumber",            "~> 2.4.0"
+  s.add_development_dependency "daemon_controller",   "~> 1.2.0"
+  s.add_development_dependency "daemons",             ">= 1.2.0"
+  s.add_development_dependency "i18n"
+  s.add_development_dependency "minitest",            "~> 5.1"
+  s.add_development_dependency "mocha",               "~> 1.1.0"
+  s.add_development_dependency "mysql2",              "~> 0.4.4"
+  s.add_development_dependency "rake",                "~> 11.2"
+  s.add_development_dependency "rdoc",                "~> 4.0"
+  s.add_development_dependency "simplecov",           "~> 0.12.0"
+  s.add_development_dependency "webmock",             "~> 1.21.0"
+  s.add_development_dependency "websocket-eventmachine-client"
 end

data/features/support/system_notification_logger

@@ -2,30 +2,47 @@
 
 require "rubygems"
 require "daemons"
+require "eventmachine"
+require "websocket-eventmachine-client"
 require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
 
 tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
 system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
 
+DEBUG = true
+
 Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
   Beetle.config.servers = "127.0.0.1:5672" # rabbitmq
 
   # set Beetle log level to info, less noisy than debug
   Beetle.config.logger.level = Logger::DEBUG
 
-  client = Beetle::Client.new
-  client.configure :exchange => :system, :auto_delete => true do |config|
-    config.message  :system_notification
-    config.queue    :system_notification
-    config.handler  :system_notification do |message|
-      payload = ActiveSupport::JSON.decode(message.data)
-      text = payload["message"]
-      puts "Writing message to #{system_notification_log_file_path}: #{text}"
-      File.open(system_notification_log_file_path, "a+") do |f|
-        f << text
+  log_file = File.open(system_notification_log_file_path, "a+")
+  log_file.sync = true
+
+  @interrupted = false
+  def shutdown(ws)
+    @interrupted = true
+    ws.unbind
+  end
+
+  while !@interrupted
+    EventMachine.run do
+      ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://127.0.0.1:9650/notifications')
+      ws.onopen do
+        puts "established connection" if DEBUG
+      end
+      ws.onclose do
+        puts "server closed connection" if DEBUG && !@interrupted
+        EM.add_timer(1){ EM.stop_event_loop }
+      end
+      ws.onmessage do |text|
+        puts "writing message to #{system_notification_log_file_path}: #{text}"
+        log_file << text
       end
+      puts "Started system notification logger"
+      trap("INT") {  shutdown(ws) }
+      trap("TERM") { shutdown(ws) }
     end
   end
-  puts "Started system notification logger"
-  client.listen
 end

data/features/support/test_daemons/redis.rb

@@ -79,7 +79,7 @@ module TestDaemons
     end
 
     def running?
-      cmd = "ps aux | egrep 'redis-server.*#{port}' | grep -v grep"
+      cmd = "ps aux | fgrep 'redis-server \*:#{port}' | grep -v grep"
       res = `#{cmd}`
       x = res.chomp.split("\n")
       x.size == 1

data/features/support/test_daemons/redis_configuration_client.rb

@@ -36,11 +36,11 @@ module TestDaemons
     def daemon_controller
       @daemon_controller ||= DaemonController.new(
          :identifier    => "Redis configuration test client #{@name}",
-         :start_command => "ruby bin/beetle configuration_client start -- -v --multiple --redis-master-file #{redis_master_file} --id #{@name} --pid-dir #{tmp_path} --amqp-servers 127.0.0.1:5672",
+         :start_command => "./beetle configuration_client -v -d --redis-master-file #{redis_master_file} --id #{@name} --pid-file #{pid_file} --log-file #{log_file}",
          :ping_command  => lambda{ true },
          :pid_file      => pid_file,
          :log_file      => log_file,
-         :start_timeout => 5
+         :start_timeout => 5,
       )
     end
 
@@ -53,7 +53,7 @@ module TestDaemons
     end
 
     def log_file
-      "#{tmp_path}/redis_configuration_client.output"
+      "#{tmp_path}/redis_configuration_client_num#{@daemon_id}.output"
     end
 
     def tmp_path

data/features/support/test_daemons/redis_configuration_server.rb

@@ -25,7 +25,7 @@ module TestDaemons
       clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
       DaemonController.new(
          :identifier    => "Redis configuration test server",
-         :start_command => "ruby bin/beetle configuration_server start -- -v --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-retry-interval 1 --pid-dir #{tmp_path} --amqp-servers 127.0.0.1:5672",
+         :start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file}",
          :ping_command  => lambda{ answers_text_requests? },
          :pid_file      => pid_file,
          :log_file      => log_file,
@@ -74,7 +74,7 @@ module TestDaemons
       false
     end
 
-    HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
+    HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9650 : 8080
 
     def self.get_status(path, content_type)
       uri = URI.parse("http://127.0.0.1:#{HTTP_SERVER_PORT}#{path}")

data/lib/beetle/deduplication_store.rb

@@ -48,72 +48,6 @@ module Beetle
       key =~ /^(msgid:[^:]*:[-0-9a-f]*):.*$/ && $1
     end
 
-    # garbage collect using master and slave (and redis-cli)
-    def garbage_collect_keys_using_master_and_slave(now = Time.now.to_i)
-      logger.info "garbage collecting obsolete redis keys from #{@config.redis_servers}"
-      info = RedisServerInfo.new(@config, {})
-      info.refresh
-      unless connection = info.slaves.first
-        logger.warn "no slave available, falling back to master."
-        connection = redis
-      end
-      file = "#{@config.tmpdir}/beetle_redis_expire_keys_#{$$}.txt"
-      cmd = "redis-cli -h #{connection.host} -p #{connection.port} -n #{@config.redis_db} keys 'msgid:*:expires' > #{file}"
-      logger.info "retrieving expire keys: '#{cmd}'"
-      if system(cmd)
-        garbage_collect_keys_from_file(file, now)
-      else
-        logger.error "could not retrieve expire keys"
-      end
-    rescue => e
-      logger.error "#{e.class}(#{e})"
-    ensure
-      system("rm -f #{file}")
-    end
-
-    # garbage collect keys in Redis (always assume the worst!)
-    def garbage_collect_keys(now = Time.now.to_i)
-      keys = redis.keys("msgid:*:expires")
-      threshold = now - @config.gc_threshold
-      keys.each do |key|
-        gc_key(key, threshold)
-      end
-    end
-
-    # garbage collect keys from a given file
-    def garbage_collect_keys_from_file(file_name, now = Time.now.to_i)
-      threshold = now - @config.gc_threshold
-      expired = total = 0
-      File.foreach(file_name) do |line|
-        line.chomp!
-        if line =~ /^msgid:.*:expires$/
-          total += 1
-          expired += 1 if gc_key(line, threshold)
-        end
-      end
-      logger.info "expired #{expired} keys out of #{total}"
-    end
-
-    # garbage collect a single key if it's older than given threshold
-    def gc_key(key, threshold)
-      expires_at = redis.get key
-      if expires_at
-        if expires_at.to_i < threshold
-          msg_id = msg_id(key)
-          # puts "expiring: #{msg_id}"
-          redis.del(*keys(msg_id))
-          return true
-        else
-          # key has future expiry date
-          # puts "#{msg_id} expires on #{Time.at(expires_at.to_i)}"
-          return false
-        end
-      else
-        # key might have been deleted in the meantime
-        return false
-      end
-    end
-
     # unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
     def set(msg_id, suffix, value)
       with_failover { redis.set(key(msg_id, suffix), value) }

data/lib/beetle/message.rb

@@ -73,7 +73,7 @@ module Beetle
       @store            = opts[:store]
     end
 
-    # extracts various values form the AMQP header properties
+    # extracts various values from the AMQP header properties
     def decode #:nodoc:
       # p header.attributes
       amqp_headers = header.attributes

data/lib/beetle/publisher.rb

@@ -166,7 +166,8 @@ module Beetle
         :pass           => @client.config.password,
         :vhost          => @client.config.vhost,
         :frame_max      => @client.config.frame_max,
-        :socket_timeout => @client.config.publishing_timeout)
+        :socket_timeout => @client.config.publishing_timeout,
+        :spec => '09')
       b.start
       b
     end

data/lib/beetle/redis_master_file.rb

@@ -26,10 +26,5 @@ module Beetle
       config.redis_server
     end
 
-    def verify_redis_master_file_string
-      if master_file =~ /^[0-9a-z.]+:[0-9]+$/
-        raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
-      end
-    end
   end
 end

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "1.0.3"
+  VERSION = "2.0.0rc1"
 end

data/test/beetle/deduplication_store_test.rb

@@ -111,52 +111,4 @@ module Beetle
     end
   end
 
-  class GarbageCollectionTest < Minitest::Test
-    def setup
-      @store = DeduplicationStore.new
-      Beetle.config.stubs(:gc_threshold).returns(10)
-    end
-
-    test "never tries to delete message keys when expire key does not exist" do
-      key = "foo"
-      @store.redis.del key
-      @store.redis.expects(:del).never
-      assert !@store.gc_key(key, 0)
-    end
-
-    test "rescues exeptions and logs an error" do
-      RedisServerInfo.expects(:new).raises("foo")
-      assert_nothing_raised { @store.garbage_collect_keys_using_master_and_slave }
-    end
-
-    test "logs an error when system command fails" do
-      @store.stubs(:system).returns(false)
-      @store.logger.expects(:error)
-      @store.garbage_collect_keys_using_master_and_slave
-    end
-
-    test "garbage collects a key when it has expired" do
-      key = "foo"
-      t = Time.now.to_i
-      @store.redis.set(key, t)
-      @store.redis.expects(:del)
-      assert @store.gc_key(key, t+1)
-    end
-
-    test "does not garbage collect a key when it has not expired" do
-      key = "foo"
-      t = Time.now.to_i
-      @store.redis.set(key, t)
-      @store.redis.expects(:del).never
-      assert !@store.gc_key(key, t)
-    end
-
-    test "correctly sets threshold for garbage collection" do
-      t = Time.now.to_i
-      @store.redis.expects(:keys).returns(["foo"])
-      @store.expects(:gc_key).with("foo", t-10)
-      @store.garbage_collect_keys
-    end
-
-  end
 end