beetle 0.1 → 0.2.0

data/test/beetle/message_test.rb

@@ -4,6 +4,11 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
 module Beetle
 
   class EncodingTest < Test::Unit::TestCase
+    test "an exception during decoding should be stored in the exception attribute" do
+      header = stub_everything("raising header")
+      m = Message.new("queue", header, 'foo')
+      assert_instance_of NoMethodError, m.exception
+    end
 
     test "a message should encode/decode the message format version correctly" do
       header = header_with_params({})
@@ -371,6 +376,17 @@ module Beetle
       @store.flushdb
     end
 
+    test "a message with an exception set should not be processed at all, but it should be acked" do
+      header = {}
+      message = Message.new("somequeue", header, 'foo')
+      assert message.exception
+
+      proc = mock("proc")
+      proc.expects(:call).never
+      message.expects(:ack!)
+      assert_equal RC::DecodingError, message.__send__(:process_internal, proc)
+    end
+
     test "a completed existing message should be just acked and not run the handler" do
       header = header_with_params({})
       message = Message.new("somequeue", header, 'foo', :attempts => 2, :store => @store)
@@ -379,7 +395,6 @@ module Beetle
       assert message.completed?
 
       proc = mock("proc")
-      s = sequence("s")
       header.expects(:ack)
       proc.expects(:call).never
       assert_equal RC::OK, message.__send__(:process_internal, proc)
@@ -394,7 +409,6 @@ module Beetle
       assert message.delayed?
 
       proc = mock("proc")
-      s = sequence("s")
       header.expects(:ack).never
       proc.expects(:call).never
       assert_equal RC::Delayed, message.__send__(:process_internal, proc)
@@ -412,7 +426,6 @@ module Beetle
       assert !message.timed_out?
 
       proc = mock("proc")
-      s = sequence("s")
       header.expects(:ack).never
       proc.expects(:call).never
       assert_equal RC::HandlerNotYetTimedOut, message.__send__(:process_internal, proc)
@@ -577,7 +590,7 @@ module Beetle
       @store.flushdb
     end
 
-    test "a handler running longer than the specified timeout should be aborted" do
+    test "a handler running longer than the specified timeout should be aborted (when given a float timeout number)" do
       header = header_with_params({})
       header.expects(:ack)
       message = Message.new("somequeue", header, 'foo', :timeout => 0.1, :attempts => 2, :store => @store)
@@ -586,6 +599,17 @@ module Beetle
       result = message.process(handler)
       assert_equal RC::ExceptionsLimitReached, result
     end
+
+    test "a handler running longer than the specified timeout should be aborted (when using active_support seconds)" do
+      header = header_with_params({})
+      header.expects(:ack)
+      message = Message.new("somequeue", header, 'foo', :timeout => 1.seconds, :attempts => 2, :store => @store)
+      action = lambda{|*args| while true; end}
+      handler = Handler.create(action)
+      result = message.process(handler)
+      assert_equal RC::ExceptionsLimitReached, result
+    end
+
   end
 
   class SettingsTest < Test::Unit::TestCase

data/test/beetle/redis_configuration_client_test.rb

@@ -0,0 +1,97 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+  class RedisConfigurationClientTest < Test::Unit::TestCase
+    def setup
+      Beetle.config.redis_servers = "redis:0,redis:1"
+      @client = RedisConfigurationClient.new
+      Client.any_instance.stubs(:listen)
+      @client.stubs(:touch_master_file)
+      @client.stubs(:verify_redis_master_file_string)
+    end
+
+    test "config should return the beetle config" do
+      assert_equal Beetle.config, @client.config
+    end
+
+    test "ping message should answer with pong" do
+      @client.expects(:pong!)
+      @client.ping("token" => 1)
+    end
+
+    test "pong should publish a pong message" do
+      @client.beetle.expects(:publish)
+      @client.send(:pong!)
+    end
+
+    test "invalidation should send an invalidation message and clear the redis master file" do
+      @client.expects(:clear_redis_master_file)
+      @client.beetle.expects(:publish).with(:client_invalidated, anything)
+      @client.send(:invalidate!)
+    end
+
+    test "should ignore outdated invalidate messages" do
+      new_payload = {"token" => 2}
+      old_payload = {"token" => 1}
+
+      @client.expects(:invalidate!).once
+
+      @client.invalidate(new_payload)
+      @client.invalidate(old_payload)
+    end
+
+    test "should ignore invalidate messages when current master is still a master" do
+      @client.instance_variable_set :@current_master, stub(:master? => true)
+      @client.expects(:invalidate!).never
+      @client.invalidate("token" => 1)
+    end
+
+    test "should ignore outdated reconfigure messages" do
+      new_payload = {"token" => 2, "server" => "master:2"}
+      old_payload = {"token" => 1, "server" => "master:1"}
+      @client.stubs(:read_redis_master_file).returns("")
+
+      @client.expects(:write_redis_master_file).once
+
+      @client.reconfigure(new_payload)
+      @client.reconfigure(old_payload)
+    end
+
+    test "should clear redis master file if redis from master file is slave" do
+      @client.stubs(:redis_master_from_master_file).returns(stub(:master? => false))
+      @client.expects(:clear_redis_master_file)
+      @client.start
+    end
+
+    test "should clear redis master file if redis from master file is not available" do
+      @client.stubs(:redis_master_from_master_file).returns(nil)
+      @client.expects(:clear_redis_master_file)
+      @client.start
+    end
+
+    test "the dispatcher should just forward messages to the client" do
+      dispatcher_class = RedisConfigurationClient.class_eval "MessageDispatcher"
+      dispatcher_class.configuration_client = @client
+      dispatcher = dispatcher_class.new
+      payload = {"token" => 1}
+      dispatcher.stubs(:message).returns(stub(:data => payload.to_json, :header => stub(:routing_key=> "ping")))
+      @client.expects(:ping).with(payload)
+      dispatcher.send(:process)
+    end
+
+    test "determine_initial_master should return nil if there is no file" do
+      @client.expects(:master_file_exists?).returns(false)
+      assert_nil @client.send(:determine_initial_master)
+      assert_nil @client.current_master
+    end
+
+    test "determine_initial_master should instantiate a new redis if there is a file with content" do
+      @client.expects(:master_file_exists?).returns(true)
+      @client.expects(:read_redis_master_file).returns("localhost:6379")
+      master = @client.send(:determine_initial_master)
+      assert_equal "master", master.role
+      assert_equal master, @client.current_master
+    end
+
+  end
+end

data/test/beetle/redis_configuration_server_test.rb

@@ -0,0 +1,278 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+  class RedisConfigurationServerTest < Test::Unit::TestCase
+    def setup
+      Beetle.config.redis_configuration_client_ids = "rc-client-1,rc-client-2"
+      @server = RedisConfigurationServer.new
+      EventMachine.stubs(:add_timer).yields
+    end
+
+    test "should exit when started with less than two redis configured" do
+      Beetle.config.redis_servers = ""
+      assert_raise Beetle::ConfigurationError do
+        @server.start
+      end
+    end
+
+    test "should initialize the current token for messages to not reuse old tokens" do
+      sleep 0.1
+      later_server = RedisConfigurationServer.new
+      assert later_server.current_token > @server.current_token
+    end
+
+    test "should ignore outdated client_invalidated messages" do
+      @server.instance_variable_set(:@current_token, 2)
+      @server.client_invalidated("id" => "rc-client-1", "token" => 2)
+      old_token = 1.minute.ago.to_f
+      @server.client_invalidated("id" => "rc-client-2", "token" => 1)
+
+      assert_equal(["rc-client-1"].to_set, @server.instance_variable_get(:@client_invalidated_ids_received))
+    end
+
+    test "should ignore outdated pong messages" do
+      @server.instance_variable_set(:@current_token, 2)
+      @server.pong("id" => "rc-client-1", "token" => 2)
+      old_token = 1.minute.ago.to_f
+      @server.pong("id" => "rc-client-2", "token" => 1)
+
+      assert_equal(["rc-client-1"].to_set, @server.instance_variable_get(:@client_pong_ids_received))
+    end
+
+    test "the dispatcher should just forward messages to the server" do
+      dispatcher_class = RedisConfigurationServer.class_eval "MessageDispatcher"
+      dispatcher_class.configuration_server = @server
+      dispatcher = dispatcher_class.new
+      payload = {"token" => 1}
+      dispatcher.stubs(:message).returns(stub(:data => payload.to_json, :header => stub(:routing_key=> "pong")))
+      @server.expects(:pong).with(payload)
+      dispatcher.send(:process)
+    end
+
+    test "if a new master is available, it should be published and the available slaves should be configured" do
+      redis = Redis.new
+      other_master = Redis.new(:port => 6380)
+      other_master.expects(:slave_of!).with(redis.host, redis.port)
+      @server.stubs(:current_master).returns(redis)
+      @server.redis.instance_variable_set(:@server_info, {"master" => [redis, other_master], "slave" => [], "unknown" => []})
+      payload = @server.send(:payload_with_current_token, {"server" => redis.server})
+      @server.beetle.expects(:publish).with(:reconfigure, payload)
+      @server.master_available!
+    end
+  end
+
+  class RedisConfigurationServerInvalidationTest < Test::Unit::TestCase
+    def setup
+      Beetle.config.redis_configuration_client_ids = "rc-client-1,rc-client-2"
+      Beetle.config.redis_servers = "redis:0,redis:1"
+      @server = RedisConfigurationServer.new
+      @server.instance_variable_set(:@current_master, stub('redis stub', :server => 'stubbed_server', :available? => false))
+      @server.stubs(:verify_redis_master_file_string)
+      @server.beetle.stubs(:listen).yields
+      @server.beetle.stubs(:publish)
+      EM::Timer.stubs(:new).returns(true)
+      EventMachine.stubs(:add_periodic_timer).yields
+    end
+
+    test "should pause watching of the redis master when it becomes unavailable" do
+      @server.expects(:determine_initial_master)
+      EM.stubs(:add_periodic_timer).returns(stub("timer", :cancel => true))
+      @server.start
+      assert !@server.paused?
+      @server.master_unavailable!
+      assert @server.paused?
+    end
+
+    test "should setup an invalidation timeout" do
+      EM::Timer.expects(:new).yields
+      @server.expects(:cancel_invalidation)
+      @server.master_unavailable!
+    end
+
+    test "should continue watching after the invalidation timeout has expired" do
+      EM::Timer.expects(:new).yields
+      @server.master_unavailable!
+      assert !@server.paused?
+    end
+
+    test "should invalidate the current master after receiving all pong messages" do
+      EM::Timer.expects(:new).yields.returns(:timer)
+      @server.beetle.expects(:publish).with(:invalidate, anything)
+      @server.expects(:cancel_invalidation)
+      @server.expects(:redeem_token).with(1).twice.returns(true)
+      @server.pong("token" => 1, "id" => "rc-client-1")
+      @server.pong("token" => 1, "id" => "rc-client-2")
+    end
+
+    test "should switch the current master after receiving all client_invalidated messages" do
+      @server.expects(:redeem_token).with(1).twice.returns(true)
+      @server.expects(:switch_master)
+      @server.client_invalidated("token" => 1, "id" => "rc-client-1")
+      @server.client_invalidated("token" => 1, "id" => "rc-client-2")
+    end
+
+    test "should switch the current master immediately if there are no clients" do
+      @server.instance_variable_set :@client_ids, Set.new
+      @server.expects(:switch_master)
+      @server.master_unavailable!
+    end
+
+    test "switching the master should turn the new master candidate into a master" do
+      new_master = stub(:master! => nil, :server => "jo:6379")
+      @server.beetle.expects(:publish).with(:system_notification, anything)
+      @server.expects(:determine_new_master).returns(new_master)
+      @server.send :switch_master
+      assert_equal new_master, @server.current_master
+    end
+
+    test "switching the master should resort to the old master if no candidate can be found" do
+      old_master = @server.current_master
+      @server.beetle.expects(:publish).with(:system_notification, anything)
+      @server.expects(:determine_new_master).returns(nil)
+      @server.send :switch_master
+      assert_equal old_master, @server.current_master
+    end
+
+    test "checking the availability of redis servers should publish the available servers as long as the master is available" do
+      @server.expects(:master_available?).returns(true)
+      @server.expects(:master_available!)
+      @server.send(:master_watcher).send(:check_availability)
+    end
+
+    test "checking the availability of redis servers should call master_unavailable after trying the specified number of times" do
+      @server.stubs(:master_available?).returns(false)
+      @server.expects(:master_unavailable!)
+      watcher = @server.send(:master_watcher)
+      watcher.instance_variable_set :@master_retries, 0
+      watcher.send(:check_availability)
+    end
+  end
+
+  class RedisConfigurationServerInitialRedisMasterDeterminationTest < Test::Unit::TestCase
+    def setup
+      EM::Timer.stubs(:new).returns(true)
+      EventMachine.stubs(:add_periodic_timer).yields
+      @client = Client.new(Configuration.new)
+      @client.stubs(:listen).yields
+      @client.stubs(:publish)
+      @client.config.redis_configuration_client_ids = "rc-client-1,rc-client-2"
+      @server = RedisConfigurationServer.new
+      @server.stubs(:beetle).returns(@client)
+      @server.stubs(:write_redis_master_file)
+      @redis_master  = build_master_redis_stub
+      @redis_slave   = build_slave_redis_stub
+      @server.instance_variable_set(:@redis, build_redis_server_info(@redis_master, @redis_slave))
+    end
+
+    test "should not try to auto-detect if the master file contains a server string" do
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:read_redis_master_file).returns("foobar:0000")
+
+      @server.redis.expects(:auto_detect_master).never
+      @server.expects(:redis_master_from_master_file).returns(@redis_master)
+      @server.send(:determine_initial_master)
+    end
+
+    test "should try to auto-detect if the master file is empty" do
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:read_redis_master_file).returns("")
+
+      @server.redis.expects(:auto_detect_master).returns(@redis_master)
+      @server.send(:determine_initial_master)
+    end
+
+    test "should try to auto-detect if the master file is not present" do
+      @server.expects(:master_file_exists?).returns(false)
+
+      @server.redis.expects(:auto_detect_master).returns(@redis_master)
+      @server.send(:determine_initial_master)
+    end
+
+    test "should use redis master from successful auto-detection" do
+      @server.expects(:master_file_exists?).returns(false)
+
+      @server.expects(:write_redis_master_file).with(@redis_master.server)
+      @server.send(:determine_initial_master)
+      assert_equal @redis_master, @server.current_master
+    end
+
+    test "should use redis master if master in file is the only master" do
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:redis_master_from_master_file).returns(@redis_master)
+
+      @server.send(:determine_initial_master)
+      assert_equal @redis_master, @server.current_master
+    end
+
+    test "should start master switch if master in file is slave" do
+      @server.instance_variable_set(:@redis, build_redis_server_info(@redis_slave))
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:redis_master_from_master_file).returns(@redis_slave)
+
+      @server.expects(:master_unavailable!)
+      @server.send(:determine_initial_master)
+    end
+
+    test "should use master from master file if multiple masters are available" do
+      redis_master2 = build_master_redis_stub
+      @server.instance_variable_set(:@redis, build_redis_server_info(@redis_master, redis_master2))
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:redis_master_from_master_file).returns(@redis_master)
+
+      @server.send(:determine_initial_master)
+      assert_equal @redis_master, @server.current_master
+    end
+
+    test "should start master switch if master in file is not available" do
+      not_available_redis_master = build_unknown_redis_stub
+      @server.instance_variable_set(:@redis, build_redis_server_info(not_available_redis_master, @redis_slave))
+      @server.expects(:master_file_exists?).returns(true)
+      @server.stubs(:redis_master_from_master_file).returns(not_available_redis_master)
+
+      @server.expects(:master_unavailable!)
+      @server.send(:determine_initial_master)
+    end
+
+    test "should raise an exception if both master file and auto-detection fails" do
+      not_available_redis_master = build_unknown_redis_stub
+      not_available_redis_slave  = build_unknown_redis_stub
+      @server.instance_variable_set(:@redis, build_redis_server_info(not_available_redis_master, not_available_redis_slave))
+      @server.expects(:master_file_exists?).returns(true)
+      @server.expects(:read_redis_master_file).returns("")
+      @server.redis.expects(:auto_detect_master).returns(nil)
+
+      assert_raises Beetle::NoRedisMaster do
+        @server.send(:determine_initial_master)
+      end
+    end
+
+    test "should detect a new redis_master" do
+      not_available_redis_master = build_unknown_redis_stub
+      @redis_slave.expects(:slave_of?).returns(true)
+      @server.instance_variable_set(:@current_master, not_available_redis_master)
+      @server.instance_variable_set(:@redis, build_redis_server_info(@redis_slave, not_available_redis_master))
+      assert_equal @redis_slave, @server.send(:determine_new_master)
+    end
+
+    private
+
+    def build_master_redis_stub
+      stub("redis master", :host => "stubbed_master", :port => 0, :server => "stubbed_master:0", :available? => true, :master? => true, :slave? => false, :role => "master")
+    end
+
+    def build_slave_redis_stub
+      stub("redis slave", :host => "stubbed_slave", :port => 0, :server => "stubbed_slave:0",  :available? => true, :master? => false, :slave? => true, :role => "slave")
+    end
+
+    def build_unknown_redis_stub
+      stub("redis unknown", :host => "stubbed_unknown", :port => 0, :server => "stubbed_unknown:0",  :available? => false, :master? => false, :slave? => false, :role => "unknown")
+    end
+
+    def build_redis_server_info(*redis_instances)
+      info = RedisServerInfo.new(Beetle.config, :timeout => 1)
+      info.instance_variable_set :@instances, redis_instances
+      redis_instances.each{|redis| info.send("#{redis.role}s") << redis }
+      info
+    end
+  end
+end

data/test/beetle/redis_ext_test.rb

@@ -0,0 +1,71 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+
+  class NonExistentRedisTest < Test::Unit::TestCase
+    def setup
+      @r = Redis.new(:host => "localhost", :port => 6390)
+    end
+
+    test "should return an empty hash for the info_with_rescue call" do
+      assert_equal({}, @r.info_with_rescue)
+    end
+
+    test "should have a role of unknown" do
+      assert_equal "unknown", @r.role
+    end
+
+    test "should not be available" do
+      assert !@r.available?
+    end
+
+    test "should not be a master" do
+      assert !@r.master?
+    end
+
+    test "should not be a slave" do
+      assert !@r.slave?
+      assert !@r.slave_of?("localhost", 6379)
+    end
+
+    test "should not try toconnect to the redis server on inspect" do
+      assert_nothing_raised { @r.inspect }
+    end
+  end
+
+  class AddedRedisMethodsTest < Test::Unit::TestCase
+    def setup
+      @r = Redis.new(:host => "localhost", :port => 6390)
+    end
+
+    test "should return the host, port and server string" do
+      assert_equal "localhost", @r.host
+      assert_equal 6390, @r.port
+      assert_equal "localhost:6390", @r.server
+    end
+
+    test "should stop slavery" do
+      @r.expects(:slaveof).with("no", "one")
+      @r.master!
+    end
+
+    test "should support slavery" do
+      @r.expects(:slaveof).with("localhost", 6379)
+      @r.slave_of!("localhost", 6379)
+    end
+  end
+
+  class RedisTimeoutTest < Test::Unit::TestCase
+    test "should use Redis::Timer if timeout is greater 0" do
+      r = Redis.new(:host => "localhost", :port => 6390, :timeout => 1)
+      Redis::Timer.expects(:timeout).with(1).raises(Timeout::Error)
+      assert_equal({}, r.info_with_rescue)
+    end
+
+    test "should not use Redis::Timer if timeout 0" do
+      r = Redis.new(:host => "localhost", :port => 6390, :timeout => 0)
+      Redis::Timer.expects(:timeout).never
+      assert_equal({}, r.info_with_rescue)
+    end
+  end
+end

data/test/beetle/redis_master_file_test.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+
+  class RedisMasterFileTest < Test::Unit::TestCase
+    include Logging
+    include RedisMasterFile
+
+    def setup
+      File.open(master_file, "w"){|f| f.puts "localhost:6379"}
+    end
+
+    def teardown
+      File.unlink(master_file) if File.exist?(master_file)
+    end
+
+    test "should be able to check existence" do
+      assert master_file_exists?
+      File.unlink(master_file)
+      assert !master_file_exists?
+    end
+
+    test "should be able to read and write the master file"do
+      write_redis_master_file("localhost:6380")
+      assert_equal "localhost:6380", read_redis_master_file
+    end
+
+    test "should be able to clear the master file" do
+      logger.expects(:warn)
+      clear_redis_master_file
+      assert_equal "", read_redis_master_file
+    end
+
+    private
+    def master_file
+      "/tmp/mumu.txt"
+    end
+  end
+end

data/test/test_helper.rb

@@ -2,7 +2,10 @@ require 'rubygems'
 require 'active_support'
 require 'active_support/testing/declarative'
 require 'test/unit'
-require 'redgreen' unless ENV['TM_FILENAME'] rescue nil
+begin
+  require 'redgreen' unless ENV['TM_FILENAME'] 
+rescue MissingSourceFile
+end
 require 'mocha'
 require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
 
@@ -10,7 +13,9 @@ class Test::Unit::TestCase
   extend ActiveSupport::Testing::Declarative
 end
 
+
 Beetle.config.logger = Logger.new(File.dirname(__FILE__) + '/../test.log')
+Beetle.config.redis_server = "localhost:6379"
 
 def header_with_params(opts = {})
   beetle_headers = Beetle::Message.publishing_options(opts)
@@ -18,3 +23,10 @@ def header_with_params(opts = {})
   header.stubs(:properties).returns(beetle_headers)
   header
 end
+
+def redis_stub(name, opts = {})
+  default_port = opts['port'] || "1234"
+  default_host = opts['host'] || "foo"
+  opts = {'host' => default_host, 'port' => default_port, 'server' => "#{default_host}:#{default_port}"}.update(opts)
+  stub(name, opts)
+end