beetle 0.4.3 → 0.4.4

data/lib/beetle/commands.rb

@@ -27,4 +27,9 @@ module Beetle
   end
 end
 
-Beetle::Commands.execute(ARGV.shift)
+if ARGV.first.to_s.sub(/\A--/,'') == "version"
+  require 'beetle/version'
+  puts Beetle::VERSION
+else
+  Beetle::Commands.execute(ARGV.shift)
+end

data/lib/beetle/configuration.rb

@@ -27,6 +27,14 @@ module Beetle
     # handler timeout.
     attr_accessor :redis_failover_timeout
 
+    # how often heartbeat messages are exchanged between failover
+    # daemons. defaults to 10 seconds.
+    attr_accessor :redis_failover_client_heartbeat_interval
+
+    # how long to wait until a redis_failover client daeom can be considered
+    # dead. defaults to 60 seconds.
+    attr_accessor :redis_failover_client_dead_interval
+
     ## redis configuration server options
     # how often should the redis configuration server try to reach the redis master before nominating a new one (defaults to <tt>3</tt>)
     attr_accessor :redis_configuration_master_retries
@@ -99,6 +107,8 @@ module Beetle
       self.redis_servers = ""
       self.redis_db = 4
       self.redis_failover_timeout = 180.seconds
+      self.redis_failover_client_heartbeat_interval = 10.seconds
+      self.redis_failover_client_dead_interval = 60.seconds
 
       self.redis_configuration_master_retries = 3
       self.redis_configuration_master_retry_interval = 10.seconds

data/lib/beetle/redis_configuration_client.rb

@@ -42,7 +42,11 @@ module Beetle
       determine_initial_master
       clear_redis_master_file unless current_master.try(:master?)
       logger.info "Listening"
-      beetle.listen
+      beetle.listen do
+        EventMachine.add_periodic_timer(Beetle.config.redis_failover_client_heartbeat_interval) do
+          heartbeat!
+        end
+      end
     end
 
     # called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
@@ -99,6 +103,7 @@ module Beetle
         config.message :pong
         config.message :client_started
         config.message :client_invalidated
+        config.message :heartbeat
         # messages received
         config.message :ping
         config.message :invalidate
@@ -128,6 +133,11 @@ module Beetle
       beetle.publish(:client_started, {"id" => id}.to_json)
     end
 
+    def heartbeat!
+      logger.info "Sending heartbeat message with id '#{id}'"
+      beetle.publish(:heartbeat, {"id" => id}.to_json)
+    end
+
     def invalidate!
       @current_master = nil
       clear_redis_master_file

data/lib/beetle/redis_configuration_http_server.rb

@@ -77,7 +77,7 @@ module Beetle
       status.keys.sort_by{|k| k.to_s}.reverse.map do |k|
         name = k.to_s # .split('_').join(" ")
         if (value = status[k]).is_a?(Array)
-          value = value.join(", ")
+          value = value.empty? ? "none" : value.join(", ")
         end
         "#{name}: #{value}"
       end.join("\n")

data/lib/beetle/redis_configuration_server.rb

@@ -24,11 +24,23 @@ module Beetle
     # the list of known client ids
     attr_reader :client_ids
 
+    # the list of client ids we have received but we don't know about
+    attr_reader :unknown_client_ids
+
+    # clients last seen time stamps
+    attr_reader :clients_last_seen
+
+    # clients are presumed dead after this many seconds
+    attr_accessor :client_dead_threshold
+
     def initialize #:nodoc:
       @client_ids = Set.new(config.redis_configuration_client_ids.split(","))
+      @unknown_client_ids = Set.new
+      @clients_last_seen = {}
       @current_token = (Time.now.to_f * 1000).to_i
       @client_pong_ids_received = Set.new
       @client_invalidated_ids_received = Set.new
+      @client_dead_threshold = config.redis_failover_client_dead_interval
       MessageDispatcher.configuration_server = self
     end
 
@@ -57,9 +69,20 @@ module Beetle
         :redis_master_available? => master_available?,
         :redis_slaves_available => available_slaves.map(&:server),
         :switch_in_progress => paused?,
+        :unknown_client_ids => unknown_client_ids.to_a,
+        :unresponsive_clients => unresponsive_clients.map{|c,t| "#{c}:#{t.to_i}"},
       }
     end
 
+    # returns an array of [client_id, silent time in seconds] which haven't sent
+    # a message for more than <tt>client_dead_threshold</tt> seconds, sorted by
+    # silent time in descending order.
+    def unresponsive_clients
+      now = Time.now
+      threshold = now - client_dead_threshold
+      clients_last_seen.select{|_,t| t < threshold}.map{|c,t| [c, now - t]}.sort_by{|_,t| t}.reverse
+    end
+
     # start watching redis
     def start
       verify_redis_master_file_string
@@ -80,6 +103,7 @@ module Beetle
     # called by the message dispatcher when a "pong" message from a RedisConfigurationClient is received
     def pong(payload)
       id = payload["id"]
+      client_seen(id)
       token = payload["token"]
       return unless validate_pong_client_id(id)
       logger.info "Received pong message from id '#{id}' with token '#{token}'"
@@ -95,19 +119,37 @@ module Beetle
     # called by the message dispatcher when a "client_started" message from a RedisConfigurationClient is received
     def client_started(payload)
       id = payload["id"]
+      client_seen(id)
       if client_id_valid?(id)
         logger.info "Received client_started message from id '#{id}'"
       else
+        add_unknown_client_id(id)
         msg = "Received client_started message from unknown id '#{id}'"
         logger.error msg
         beetle.publish(:system_notification, {"message" => msg}.to_json)
       end
     end
 
+    # called by the message dispatcher when a "heartbeat" message from a RedisConfigurationClient is received
+    def heartbeat(payload)
+      id = payload["id"]
+      client_seen(id)
+      if client_id_valid?(id)
+        logger.info "Received heartbeat message from id '#{id}'"
+      else
+        add_unknown_client_id(id)
+        msg = "Received heartbeat message from unknown id '#{id}'"
+        logger.error msg
+        beetle.publish(:system_notification, {"message" => msg}.to_json)
+      end
+    end
+
     # called by the message dispatcher when a "client_invalidated" message from a RedisConfigurationClient is received
     def client_invalidated(payload)
       id = payload["id"]
+      client_seen(id)
       token = payload["token"]
+      add_unknown_client_id(id) unless client_id_valid?(id)
       logger.info "Received client_invalidated message from id '#{id}' with token '#{token}'"
       return unless redeem_token(token)
       @client_invalidated_ids_received << id
@@ -161,6 +203,23 @@ module Beetle
 
     private
 
+    # prevent memory overflows caused by evil or buggy clients
+    MAX_UNKNOWN_CLIENT_IDS = 20
+
+    def add_unknown_client_id(id)
+      ids = @unknown_client_ids
+      while ids.size >= MAX_UNKNOWN_CLIENT_IDS
+        old_id = ids.first
+        ids.delete(old_id)
+        clients_last_seen.delete(old_id)
+      end
+      ids << id
+    end
+
+    def client_seen(client_id)
+      clients_last_seen[client_id] = Time.now
+    end
+
     def check_redis_configuration
       raise ConfigurationError.new("Redis failover needs two or more redis servers") if redis.instances.size < 2
     end
@@ -185,10 +244,12 @@ module Beetle
         config.message :client_started
         config.message :pong
         config.message :client_invalidated
+        config.message :heartbeat
         # queue setup
         config.queue   :server, :key => 'pong', :amqp_name => "#{system}_configuration_server"
         config.binding :server, :key => 'client_started'
         config.binding :server, :key => 'client_invalidated'
+        config.binding :server, :key => 'heartbeat'
         config.handler :server, MessageDispatcher
       end
     end
@@ -216,6 +277,7 @@ module Beetle
 
     def validate_pong_client_id(client_id)
       unless known_client = client_id_valid?(client_id)
+        add_unknown_client_id(client_id)
         msg = "Received pong message from unknown id '#{client_id}'"
         logger.error msg
         logger.info "Sending system_notification message with text: #{msg}"

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "0.4.3"
+  VERSION = "0.4.4"
 end

data/test/beetle/redis_configuration_client_test.rb

@@ -5,7 +5,6 @@ module Beetle
     def setup
       Beetle.config.redis_servers = "redis:0,redis:1"
       @client = RedisConfigurationClient.new
-      Client.any_instance.stubs(:listen)
       @client.stubs(:touch_master_file)
       @client.stubs(:verify_redis_master_file_string)
     end
@@ -13,6 +12,16 @@ module Beetle
     test "should send a client_started message when started" do
       @client.stubs(:clear_redis_master_file)
       @client.beetle.expects(:publish).with(:client_started, {:id => @client.id}.to_json)
+      Client.any_instance.stubs(:listen)
+      @client.start
+    end
+
+    test "should install a periodic timer which sends a heartbeat message" do
+      @client.stubs(:clear_redis_master_file)
+      @client.beetle.expects(:publish).with(:client_started, {:id => @client.id}.to_json)
+      @client.beetle.expects(:listen).yields
+      EventMachine.expects(:add_periodic_timer).with(60).yields
+      @client.beetle.expects(:publish).with(:heartbeat, {:id => @client.id}.to_json)
       @client.start
     end
 
@@ -68,6 +77,7 @@ module Beetle
       Beetle::Client.any_instance.stubs(:publish)
       @client.expects(:clear_redis_master_file)
       @client.expects(:client_started!)
+      Client.any_instance.stubs(:listen)
       @client.start
     end
 
@@ -76,6 +86,7 @@ module Beetle
       Beetle::Client.any_instance.stubs(:publish)
       @client.expects(:clear_redis_master_file)
       @client.expects(:client_started!)
+      Client.any_instance.stubs(:listen)
       @client.start
     end
 

data/test/beetle/redis_configuration_server_test.rb

@@ -24,7 +24,6 @@ module Beetle
     test "should ignore outdated client_invalidated messages" do
       @server.instance_variable_set(:@current_token, 2)
       @server.client_invalidated("id" => "rc-client-1", "token" => 2)
-      old_token = 1.minute.ago.to_f
       @server.client_invalidated("id" => "rc-client-2", "token" => 1)
 
       assert_equal(["rc-client-1"].to_set, @server.instance_variable_get(:@client_invalidated_ids_received))
@@ -33,7 +32,6 @@ module Beetle
     test "should ignore outdated pong messages" do
       @server.instance_variable_set(:@current_token, 2)
       @server.pong("id" => "rc-client-1", "token" => 2)
-      old_token = 1.minute.ago.to_f
       @server.pong("id" => "rc-client-2", "token" => 1)
 
       assert_equal(["rc-client-1"].to_set, @server.instance_variable_get(:@client_pong_ids_received))
@@ -61,6 +59,8 @@ module Beetle
     end
 
     test "should be able to report current status" do
+      @server.expects(:unknown_client_ids).returns(Set.new ["x", "y"])
+      @server.expects(:unresponsive_clients).returns(["a", Time.now - 200])
       assert @server.status.is_a?(Hash)
     end
 
@@ -85,6 +85,24 @@ module Beetle
       assert @server.initiate_master_switch
     end
 
+    test "should put a limit on the number of stored unknown client ids" do
+      1000.times do |i|
+        id = i.to_s
+        @server.send(:client_seen, id)
+        @server.send(:add_unknown_client_id, id)
+      end
+      assert @server.unknown_client_ids.size < 100
+      assert_equal @server.unknown_client_ids.size, @server.clients_last_seen.size
+    end
+
+    test "should assume clients to be unresponsive after specified interval time" do
+      @server.send(:client_seen, "1")
+      @server.send(:client_seen, "2")
+      @server.client_dead_threshold = 0
+      assert_equal %w(1 2), @server.unresponsive_clients.map(&:first)
+      @server.client_dead_threshold = 10
+      assert_equal [], @server.unresponsive_clients
+    end
   end
 
   class RedisConfigurationServerInvalidationTest < MiniTest::Unit::TestCase
@@ -316,6 +334,7 @@ module Beetle
       msg = "Received pong message from unknown id 'unknown-client'"
       @server.beetle.expects(:publish).with(:system_notification, ({:message => msg}).to_json)
       @server.pong(payload)
+      assert @server.unknown_client_ids.include?("unknown-client")
     end
 
     test "should send a system notification when receiving client_started message from unknown client" do
@@ -323,13 +342,32 @@ module Beetle
       msg = "Received client_started message from unknown id 'unknown-client'"
       @server.beetle.expects(:publish).with(:system_notification, ({:message => msg}).to_json)
       @server.client_started(payload)
+      assert @server.unknown_client_ids.include?("unknown-client")
     end
 
     test "should not send a system notification when receiving a client started message from a known client" do
       payload = {"id" => "rc-client-1"}
-      msg = "Received client_started message from unknown id 'unknown-client'"
       @server.beetle.expects(:publish).never
+      @server.expects(:add_unknown_client_id).never
       @server.client_started(payload)
+      assert @server.clients_last_seen.include?("rc-client-1")
+    end
+
+    test "should send a system notification when receiving heartbeat message from unknown client" do
+      payload = {"id" => "unknown-client"}
+      msg = "Received heartbeat message from unknown id 'unknown-client'"
+      @server.beetle.expects(:publish).with(:system_notification, ({:message => msg}).to_json)
+      @server.heartbeat(payload)
+      assert @server.unknown_client_ids.include?("unknown-client")
     end
+
+    test "should not send a system notification when receiving a heartbeat message from a known client" do
+      payload = {"id" => "rc-client-1"}
+      @server.beetle.expects(:publish).never
+      @server.expects(:add_unknown_client_id).never
+      @server.heartbeat(payload)
+      assert @server.clients_last_seen.include?("rc-client-1")
+    end
+
   end
 end

data/test/test_helper.rb

@@ -11,6 +11,7 @@ require 'minitest/pride' if ENV['RAINBOW_COLORED_TESTS'] == "1" && $stdout.tty?
 require 'mocha/setup'
 
 require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
+require 'eventmachine'
 
 class MiniTest::Unit::TestCase
   require "active_support/testing/declarative"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-22 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid4r