beetle 0.1 → 0.2.0

data/lib/beetle/deduplication_store.rb

@@ -11,15 +11,23 @@ module Beetle
   #
   # It also provides a method to garbage collect keys for expired messages.
   class DeduplicationStore
-    # creates a new deduplication store
-    def initialize(hosts = "localhost:6379", db = 4)
-      @hosts = hosts
-      @db = db
+    include Logging
+
+    def initialize(config = Beetle.config)
+      @config = config
+      @current_master = nil
+      @last_time_master_file_changed = nil
     end
 
     # get the Redis instance
     def redis
-      @redis ||= find_redis_master
+      redis_master_source = @config.redis_server =~ /^\S+\:\d+$/ ? "server_string" : "master_file"
+      _eigenclass_.class_eval <<-EVALS, __FILE__, __LINE__
+        def redis
+          redis_master_from_#{redis_master_source}
+        end
+      EVALS
+      redis
     end
 
     # list of key suffixes to use for storing values in Redis.
@@ -43,7 +51,7 @@ module Beetle
     # garbage collect keys in Redis (always assume the worst!)
     def garbage_collect_keys(now = Time.now.to_i)
       keys = redis.keys("msgid:*:expires")
-      threshold = now + Beetle.config.gc_threshold
+      threshold = now + @config.gc_threshold
       keys.each do |key|
         expires_at = redis.get key
         if expires_at && expires_at.to_i < threshold
@@ -53,20 +61,20 @@ module Beetle
       end
     end
 
-    # unconditionally store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt>.
+    # unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
     def set(msg_id, suffix, value)
       with_failover { redis.set(key(msg_id, suffix), value) }
     end
 
-    # store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
+    # store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
     def setnx(msg_id, suffix, value)
       with_failover { redis.setnx(key(msg_id, suffix), value) }
     end
 
     # store some key/value pairs if none of the given keys exist.
     def msetnx(msg_id, values)
-      values = values.inject({}){|h,(k,v)| h[key(msg_id, k)] = v; h}
-      with_failover { redis.msetnx(values) }
+      values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
+      with_failover { redis.msetnx(*values) }
     end
 
     # increment counter for key with given <tt>suffix</tt> for given <tt>msg_id</tt>. returns an integer.
@@ -86,7 +94,7 @@ module Beetle
 
     # delete all keys associated with the given <tt>msg_id</tt>.
     def del_keys(msg_id)
-      with_failover { redis.del(keys(msg_id)) }
+      with_failover { redis.del(*keys(msg_id)) }
     end
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
@@ -101,16 +109,15 @@ module Beetle
 
     # performs redis operations by yielding a passed in block, waiting for a new master to
     # show up on the network if the operation throws an exception. if a new master doesn't
-    # appear after 120 seconds, we raise an exception.
+    # appear after the configured timeout interval, we raise an exception.
     def with_failover #:nodoc:
-      tries = 0
+      end_time = Time.now.to_i + @config.redis_failover_timeout.to_i
       begin
         yield
       rescue Exception => e
         Beetle::reraise_expectation_errors!
-        logger.error "Beetle: redis connection error '#{e}'"
-        if (tries+=1) < 120
-          @redis = nil
+        logger.error "Beetle: redis connection error #{e} #{@config.redis_server} (#{e.backtrace[0]})"
+        if Time.now.to_i < end_time
           sleep 1
           logger.info "Beetle: retrying redis operation"
           retry
@@ -120,33 +127,38 @@ module Beetle
       end
     end
 
-    # find the master redis instance
-    def find_redis_master
-      masters = []
-      redis_instances.each do |redis|
-        begin
-          masters << redis if redis.info[:role] == "master"
-        rescue Exception => e
-          logger.error "Beetle: could not determine status of redis instance #{redis.server}"
-        end
-      end
-      raise NoRedisMaster.new("unable to determine a new master redis instance") if masters.empty?
-      raise TwoRedisMasters.new("more than one redis master instances") if masters.size > 1
-      logger.info "Beetle: configured new redis master #{masters.first.server}"
-      masters.first
+    # set current redis master instance (as specified in the Beetle::Configuration)
+    def redis_master_from_server_string
+      @current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db)
     end
 
-    # returns the list of redis instances
-    def redis_instances
-      @redis_instances ||= @hosts.split(/ *, */).map{|s| s.split(':')}.map do |host, port|
-         Redis.new(:host => host, :port => port, :db => @db)
-      end
+    # set current redis master from master file
+    def redis_master_from_master_file
+      set_current_redis_master_from_master_file if redis_master_file_changed?
+      @current_master
+    rescue Errno::ENOENT
+      nil
     end
 
-    # returns the configured logger
-    def logger
-      Beetle.config.logger
+    # redis master file changed outside the running process?
+    def redis_master_file_changed?
+      @last_time_master_file_changed != File.mtime(@config.redis_server)
     end
 
+    # set current redis master from server:port string contained in the redis master file
+    def set_current_redis_master_from_master_file
+      @last_time_master_file_changed = File.mtime(@config.redis_server)
+      server_string = read_master_file
+      @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
+    end
+
+    # server:port string from the redis master file
+    def read_master_file
+      File.read(@config.redis_server).chomp
+    end
+
+    def _eigenclass_ #:nodoc:
+      class << self; self; end
+    end
   end
 end

data/lib/beetle/handler.rb

@@ -6,6 +6,8 @@ module Beetle
   # Most applications will define Handler subclasses and override the process, error and
   # failure methods.
   class Handler
+    include Logging
+
     # the Message instance which caused the handler to be created
     attr_reader :message
 
@@ -81,11 +83,6 @@ module Beetle
       logger.error "Beetle: handler has finally failed"
     end
 
-    # returns the configured Beetle logger
-    def logger
-      Beetle.config.logger
-    end
-
     # returns the configured Beetle logger
     def self.logger
       Beetle.config.logger

data/lib/beetle/logging.rb

@@ -0,0 +1,7 @@
+module Beetle
+  module Logging
+    def logger
+      Beetle.config.logger
+    end
+  end
+end

data/lib/beetle/message.rb

@@ -6,6 +6,8 @@ module Beetle
   # should retry executing the message handler after a handler has crashed (or forcefully
   # aborted).
   class Message
+    include Logging
+
     # current message format version
     FORMAT_VERSION = 1
     # flag for encoding redundant messages
@@ -14,7 +16,7 @@ module Beetle
     DEFAULT_TTL = 1.day
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
-    DEFAULT_HANDLER_TIMEOUT = 300.seconds
+    DEFAULT_HANDLER_TIMEOUT = 600.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -77,6 +79,8 @@ module Beetle
       @format_version = headers[:format_version].to_i
       @flags = headers[:flags].to_i
       @expires_at = headers[:expires_at].to_i
+    rescue Exception => @exception
+      logger.error "Could not decode message. #{self.inspect}"
     end
 
     # build hash with options for the publisher
@@ -193,7 +197,7 @@ module Beetle
     # have we already seen this message? if not, set the status to "incomplete" and store
     # the message exipration timestamp in the deduplication store.
     def key_exists?
-      old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at)
+      old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at, :timeout => now + timeout)
       if old_message
         logger.debug "Beetle: received duplicate message: #{msg_id} on queue: #{@queue}"
       end
@@ -236,7 +240,10 @@ module Beetle
     private
 
     def process_internal(handler)
-      if expired?
+      if @exception
+        ack!
+        RC::DecodingError
+      elsif expired?
         logger.warn "Beetle: ignored expired message (#{msg_id})!"
         ack!
         RC::Ancient
@@ -244,7 +251,6 @@ module Beetle
         ack!
         run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
       elsif !key_exists?
-        set_timeout!
         run_handler!(handler)
       elsif completed?
         ack!
@@ -273,7 +279,7 @@ module Beetle
     end
 
     def run_handler(handler)
-      Timeout::timeout(@timeout) { @handler_result = handler.call(self) }
+      Redis::Timer.timeout(@timeout.to_f) { @handler_result = handler.call(self) }
       RC::OK
     rescue Exception => @exception
       Beetle::reraise_expectation_errors!
@@ -313,14 +319,6 @@ module Beetle
       end
     end
 
-    def logger
-      @logger ||= self.class.logger
-    end
-
-    def self.logger
-      Beetle.config.logger
-    end
-
     # ack the message for rabbit. deletes all keys associated with this message in the
     # deduplication store if we are sure this is the last message with the given msg_id.
     def ack!

data/lib/beetle/publisher.rb

@@ -165,9 +165,9 @@ module Beetle
 
     # TODO: Refactor, fethch the keys and stuff itself
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      logger.debug("Creating queue with opts: #{creation_keys.inspect}")
+      logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
       queue = bunny.queue(queue_name, creation_keys)
-      logger.debug("Binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
+      logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
       queue.bind(exchange(exchange_name), binding_keys)
       queue
     end

data/lib/beetle/r_c.rb

@@ -27,7 +27,7 @@ module Beetle
     end
 
     rc :OK
-    rc :Ancient, :failure
+    rc :Ancient
     rc :AttemptsLimitReached, :failure
     rc :ExceptionsLimitReached, :failure
     rc :Delayed, :recover
@@ -35,6 +35,7 @@ module Beetle
     rc :HandlerNotYetTimedOut, :recover
     rc :MutexLocked, :recover
     rc :InternalError, :recover
+    rc :DecodingError, :failure
 
   end
 end

data/lib/beetle/redis_configuration_client.rb

@@ -0,0 +1,136 @@
+module Beetle
+  # A RedisConfigurationClient is the subordinate part of beetle's
+  # redis failover solution
+  #
+  # An instance of RedisConfigurationClient lives on every server that
+  # hosts message consumers (worker server).
+  #
+  # It is responsible for determining an initial redis master and reacting to redis master
+  # switches initiated by the RedisConfigurationServer.
+  #
+  # It will write the current redis master host:port string to a file specified via a
+  # Configuration, which is then read by DeduplicationStore on redis access.
+  #
+  # Usually started via <tt>beetle configuration_client</tt> command.
+  class RedisConfigurationClient
+    include Logging
+    include RedisMasterFile
+
+    # Set a custom unique id for this instance. Must match an entry in
+    # Configuration#redis_configuration_client_ids.
+    attr_writer :id
+
+    # The current redis master
+    attr_reader :current_master
+
+    # Unique id for this instance (defaults to the hostname)
+    def id
+      @id ||= `hostname`.chomp
+    end
+
+    def initialize #:nodoc:
+      @current_token = nil
+      MessageDispatcher.configuration_client = self
+    end
+
+    # determinines the initial redis master (if possible), then enters a messaging event
+    # loop, reacting to failover related messages sent by RedisConfigurationServer.
+    def start
+      verify_redis_master_file_string
+      logger.info "RedisConfigurationClient starting (client id: #{id})"
+      determine_initial_master
+      clear_redis_master_file unless current_master.try(:master?)
+      logger.info "Listening"
+      beetle.listen
+    end
+
+    # called by the message dispatcher when a "pong" message from RedisConfigurationServer is received
+    def ping(payload)
+      token = payload["token"]
+      logger.info "Received ping message with token '#{token}'"
+      pong! if redeem_token(token)
+    end
+
+    # called by the message dispatcher when a "invalidate" message from RedisConfigurationServer is received
+    def invalidate(payload)
+      token = payload["token"]
+      logger.info "Received invalidate message with token '#{token}'"
+      invalidate! if redeem_token(token) && !current_master.try(:master?)
+    end
+
+    # called by the message dispatcher when a "reconfigure"" message from RedisConfigurationServer is received
+    def reconfigure(payload)
+      server = payload["server"]
+      token = payload["token"]
+      logger.info "Received reconfigure message with server '#{server}' and token '#{token}'"
+      return unless redeem_token(token)
+      unless server == read_redis_master_file
+        new_master!(server)
+        write_redis_master_file(server)
+      end
+    end
+
+    # Beetle::Client instance for communication with the RedisConfigurationServer
+    def beetle
+      @beetle ||= build_beetle
+    end
+
+    def config #:nodoc:
+      beetle.config
+    end
+
+    private
+
+    def determine_initial_master
+      if master_file_exists? && server = read_redis_master_file
+        new_master!(server)
+      end
+    end
+
+    def new_master!(server)
+      @current_master = Redis.from_server_string(server, :timeout => 3)
+    end
+
+    def build_beetle
+      system = Beetle.config.system_name
+      Beetle::Client.new.configure :exchange => system, :auto_delete => true do |config|
+        config.message :ping
+        config.queue   :ping, :amqp_name => "#{system}_ping_#{id}"
+        config.message :pong
+        config.message :invalidate
+        config.queue   :invalidate, :amqp_name => "#{system}_invalidate_#{id}"
+        config.message :client_invalidated
+        config.message :reconfigure
+        config.queue   :reconfigure, :amqp_name => "#{system}_reconfigure_#{id}"
+
+        config.handler [:ping, :invalidate, :reconfigure], MessageDispatcher
+      end
+    end
+
+    def redeem_token(token)
+      @current_token = token if @current_token.nil? || token > @current_token
+      token_valid = token >= @current_token
+      logger.info "Ignored message (token was '#{token}', but expected to be >= '#{@current_token}')" unless token_valid
+      token_valid
+    end
+
+    def pong!
+      beetle.publish(:pong, {"id" => id, "token" => @current_token}.to_json)
+    end
+
+    def invalidate!
+      @current_master = nil
+      clear_redis_master_file
+      beetle.publish(:client_invalidated, {"id" => id, "token" => @current_token}.to_json)
+    end
+
+
+    # Dispatches messages from the queue to methods in RedisConfigurationClient
+    class MessageDispatcher < Beetle::Handler #:nodoc:
+      cattr_accessor :configuration_client
+      def process
+        @@configuration_client.__send__(message.header.routing_key, ActiveSupport::JSON.decode(message.data))
+      end
+    end
+  end
+end