beetle 0.1 → 0.2.1

data/lib/beetle/commands/configuration_server.rb

@@ -0,0 +1,85 @@
+require 'optparse'
+require 'daemons'
+require 'beetle'
+
+module Beetle
+  module Commands
+    # Command to start a RedisConfigurationServer daemon.
+    #
+    #   Usage: beetle configuration_server  [options] -- [server options]
+    #
+    #   server options:
+    #           --redis-servers LIST         Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)
+    #           --client-ids LIST            Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)
+    #           --redis-master-file FILE     Write redis master server string to FILE
+    #           --redis-retry-interval SEC   Number of seconds to wait between master checks
+    #           --amqp-servers LIST          AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
+    #           --config-file PATH           Path to an external yaml config file
+    #           --pid-dir DIR                Write pid and log to DIR
+    #       -v, --verbose
+    #       -h, --help                       Show this message
+    #
+    class ConfigurationServer
+      # parses command line options and starts Beetle::RedisConfigurationServer as a daemon
+      def self.execute
+        command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
+
+        opts = OptionParser.new
+        opts.banner = "Usage: beetle configuration_server #{command} [options] -- [server options]"
+        opts.separator ""
+        opts.separator "server options:"
+
+        opts.on("--redis-servers LIST", Array, "Required for start command (e.g. 192.168.0.1:6379,192.168.0.2:6379)") do |val|
+          Beetle.config.redis_servers = val.join(",")
+        end
+
+        opts.on("--client-ids LIST", "Clients that have to acknowledge on master switch (e.g. client-id1,client-id2)") do |val|
+          Beetle.config.redis_configuration_client_ids = val
+        end
+
+        opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
+          Beetle.config.redis_server = val
+        end
+
+        opts.on("--redis-retry-interval SEC", Integer, "Number of seconds to wait between master checks") do |val|
+          Beetle.config.redis_configuration_master_retry_interval = val
+        end
+
+        opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
+          Beetle.config.servers = val
+        end
+
+        opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
+          Beetle.config.config_file = val
+        end
+
+        dir_mode = nil
+        dir = nil
+        opts.on("--pid-dir DIR", String, "Write pid and log to DIR") do |val|
+          dir_mode = :normal
+          dir = val
+        end
+
+        opts.on("-v", "--verbose") do |val|
+          Beetle.config.logger.level = Logger::DEBUG
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.parse!(app_options)
+
+        if command =~ /start|run/ && Beetle.config.redis_servers.blank?
+          puts opts
+          exit
+        end
+
+        Daemons.run_proc("redis_configuration_server", :log_output => true, :dir_mode => dir_mode, :dir => dir) do
+          Beetle::RedisConfigurationServer.new.start
+        end
+      end
+    end
+  end
+end

data/lib/beetle/configuration.rb

@@ -1,31 +1,94 @@
+require 'erb'
+
 module Beetle
   class Configuration
+    # system name (used for redis cluster partitioning) (defaults to <tt>system</tt>)
+    attr_accessor :system_name
     # default logger (defaults to <tt>Logger.new(STDOUT)</tt>)
     attr_accessor :logger
     # number of seconds after which keys are removed form the message deduplication store (defaults to <tt>3.days</tt>)
     attr_accessor :gc_threshold
-    # the machines where the deduplication store lives (defaults to <tt>"localhost:6379"</tt>)
-    attr_accessor :redis_hosts
+    # the redis server to use for deduplication
+    # either a string like <tt>"localhost:6379"</tt> (default) or a file that contains the string.
+    # use a file if you are using a beetle configuration_client process to update it for automatic redis failover.
+    attr_accessor :redis_server
+    # comma separated list of redis servers available for master/slave switching
+    # e.g. "192.168.1.2:6379,192.168.1.3:6379"
+    attr_accessor :redis_servers
     # redis database number to use for the message deduplication store (defaults to <tt>4</tt>)
     attr_accessor :redis_db
+
+    # how long we should repeatedly retry a redis operation before giving up, with a one
+    # second sleep between retries (defaults to <tt>180.seconds</tt>). this value needs to be
+    # somewehere between the maximum time it takes to auto-switch redis and the smallest
+    # handler timeout.
+    attr_accessor :redis_failover_timeout
+
+    ## redis configuration server options
+    # how often should the redis configuration server try to reach the redis master before nominating a new one (defaults to <tt>3</tt>)
+    attr_accessor :redis_configuration_master_retries
+    # number of seconds to wait between retries (defaults to <tt>10</tt>)
+    attr_accessor :redis_configuration_master_retry_interval
+    # number of seconds the redis configuration server waits for answers from clients (defaults to <tt>5</tt>)
+    attr_accessor :redis_configuration_client_timeout
+    # the redis configuration client ids living on the worker machines taking part in the redis failover, separated by comma (defaults to <tt>""</tt>)
+    attr_accessor :redis_configuration_client_ids
+
     # list of amqp servers to use (defaults to <tt>"localhost:5672"</tt>)
     attr_accessor :servers
-    # the virtual host to use on the AMQP servers
+    # the virtual host to use on the AMQP servers (defaults to <tt>"/"</tt>)
     attr_accessor :vhost
-    # the AMQP user to use when connecting to the AMQP servers
+    # the AMQP user to use when connecting to the AMQP servers (defaults to <tt>"guest"</tt>)
     attr_accessor :user
-    # the password to use when connectiong to the AMQP servers
+    # the password to use when connectiong to the AMQP servers (defaults to <tt>"guest"</tt>)
     attr_accessor :password
 
+    # external config file (defaults to <tt>no file</tt>)
+    attr_reader :config_file
+
     def initialize #:nodoc:
-      self.logger = Logger.new(STDOUT)
+      self.system_name = "system"
+
+      self.logger = begin
+        logger = Logger.new(STDOUT)
+        logger.formatter = Logger::Formatter.new
+        logger.level = Logger::INFO
+        logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+        logger
+      end
+
       self.gc_threshold = 3.days
-      self.redis_hosts = "localhost:6379"
+      self.redis_server = "localhost:6379"
+      self.redis_servers = ""
       self.redis_db = 4
+      self.redis_failover_timeout = 180.seconds
+
+      self.redis_configuration_master_retries = 3
+      self.redis_configuration_master_retry_interval = 10.seconds
+      self.redis_configuration_client_timeout = 5.seconds
+      self.redis_configuration_client_ids = ""
+
       self.servers = "localhost:5672"
       self.vhost = "/"
       self.user = "guest"
       self.password = "guest"
     end
+
+    # setting the external config file will load it on assignment
+    def config_file=(file_name) #:nodoc:
+      @config_file = file_name
+      load_config
+    end
+
+    private
+    def load_config
+      hash = YAML::load(ERB.new(IO.read(config_file)).result)
+      hash.each do |key, value|
+        send("#{key}=", value)
+      end
+    rescue Exception
+      logger.error "Error loading beetle config file '#{config_file}': #{$!}"
+      raise
+    end
   end
 end

data/lib/beetle/deduplication_store.rb

@@ -11,15 +11,23 @@ module Beetle
   #
   # It also provides a method to garbage collect keys for expired messages.
   class DeduplicationStore
-    # creates a new deduplication store
-    def initialize(hosts = "localhost:6379", db = 4)
-      @hosts = hosts
-      @db = db
+    include Logging
+
+    def initialize(config = Beetle.config)
+      @config = config
+      @current_master = nil
+      @last_time_master_file_changed = nil
     end
 
     # get the Redis instance
     def redis
-      @redis ||= find_redis_master
+      redis_master_source = @config.redis_server =~ /^\S+\:\d+$/ ? "server_string" : "master_file"
+      _eigenclass_.class_eval <<-EVALS, __FILE__, __LINE__
+        def redis
+          redis_master_from_#{redis_master_source}
+        end
+      EVALS
+      redis
     end
 
     # list of key suffixes to use for storing values in Redis.
@@ -43,7 +51,7 @@ module Beetle
     # garbage collect keys in Redis (always assume the worst!)
     def garbage_collect_keys(now = Time.now.to_i)
       keys = redis.keys("msgid:*:expires")
-      threshold = now + Beetle.config.gc_threshold
+      threshold = now + @config.gc_threshold
       keys.each do |key|
         expires_at = redis.get key
         if expires_at && expires_at.to_i < threshold
@@ -53,20 +61,20 @@ module Beetle
       end
     end
 
-    # unconditionally store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt>.
+    # unconditionally store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt>.
     def set(msg_id, suffix, value)
       with_failover { redis.set(key(msg_id, suffix), value) }
     end
 
-    # store a key <tt>value></tt> with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
+    # store a (key,value) pair with given <tt>suffix</tt> for given <tt>msg_id</tt> if it doesn't exists yet.
     def setnx(msg_id, suffix, value)
       with_failover { redis.setnx(key(msg_id, suffix), value) }
     end
 
     # store some key/value pairs if none of the given keys exist.
     def msetnx(msg_id, values)
-      values = values.inject({}){|h,(k,v)| h[key(msg_id, k)] = v; h}
-      with_failover { redis.msetnx(values) }
+      values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
+      with_failover { redis.msetnx(*values) }
     end
 
     # increment counter for key with given <tt>suffix</tt> for given <tt>msg_id</tt>. returns an integer.
@@ -86,7 +94,7 @@ module Beetle
 
     # delete all keys associated with the given <tt>msg_id</tt>.
     def del_keys(msg_id)
-      with_failover { redis.del(keys(msg_id)) }
+      with_failover { redis.del(*keys(msg_id)) }
     end
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
@@ -101,16 +109,15 @@ module Beetle
 
     # performs redis operations by yielding a passed in block, waiting for a new master to
     # show up on the network if the operation throws an exception. if a new master doesn't
-    # appear after 120 seconds, we raise an exception.
+    # appear after the configured timeout interval, we raise an exception.
     def with_failover #:nodoc:
-      tries = 0
+      end_time = Time.now.to_i + @config.redis_failover_timeout.to_i
       begin
         yield
       rescue Exception => e
         Beetle::reraise_expectation_errors!
-        logger.error "Beetle: redis connection error '#{e}'"
-        if (tries+=1) < 120
-          @redis = nil
+        logger.error "Beetle: redis connection error #{e} #{@config.redis_server} (#{e.backtrace[0]})"
+        if Time.now.to_i < end_time
           sleep 1
           logger.info "Beetle: retrying redis operation"
           retry
@@ -120,33 +127,38 @@ module Beetle
       end
     end
 
-    # find the master redis instance
-    def find_redis_master
-      masters = []
-      redis_instances.each do |redis|
-        begin
-          masters << redis if redis.info[:role] == "master"
-        rescue Exception => e
-          logger.error "Beetle: could not determine status of redis instance #{redis.server}"
-        end
-      end
-      raise NoRedisMaster.new("unable to determine a new master redis instance") if masters.empty?
-      raise TwoRedisMasters.new("more than one redis master instances") if masters.size > 1
-      logger.info "Beetle: configured new redis master #{masters.first.server}"
-      masters.first
+    # set current redis master instance (as specified in the Beetle::Configuration)
+    def redis_master_from_server_string
+      @current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db)
     end
 
-    # returns the list of redis instances
-    def redis_instances
-      @redis_instances ||= @hosts.split(/ *, */).map{|s| s.split(':')}.map do |host, port|
-         Redis.new(:host => host, :port => port, :db => @db)
-      end
+    # set current redis master from master file
+    def redis_master_from_master_file
+      set_current_redis_master_from_master_file if redis_master_file_changed?
+      @current_master
+    rescue Errno::ENOENT
+      nil
     end
 
-    # returns the configured logger
-    def logger
-      Beetle.config.logger
+    # redis master file changed outside the running process?
+    def redis_master_file_changed?
+      @last_time_master_file_changed != File.mtime(@config.redis_server)
     end
 
+    # set current redis master from server:port string contained in the redis master file
+    def set_current_redis_master_from_master_file
+      @last_time_master_file_changed = File.mtime(@config.redis_server)
+      server_string = read_master_file
+      @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
+    end
+
+    # server:port string from the redis master file
+    def read_master_file
+      File.read(@config.redis_server).chomp
+    end
+
+    def _eigenclass_ #:nodoc:
+      class << self; self; end
+    end
   end
 end

data/lib/beetle/handler.rb

@@ -6,6 +6,8 @@ module Beetle
   # Most applications will define Handler subclasses and override the process, error and
   # failure methods.
   class Handler
+    include Logging
+
     # the Message instance which caused the handler to be created
     attr_reader :message
 
@@ -81,11 +83,6 @@ module Beetle
       logger.error "Beetle: handler has finally failed"
     end
 
-    # returns the configured Beetle logger
-    def logger
-      Beetle.config.logger
-    end
-
     # returns the configured Beetle logger
     def self.logger
       Beetle.config.logger

data/lib/beetle/logging.rb

@@ -0,0 +1,7 @@
+module Beetle
+  module Logging
+    def logger
+      Beetle.config.logger
+    end
+  end
+end

data/lib/beetle/message.rb

@@ -6,6 +6,8 @@ module Beetle
   # should retry executing the message handler after a handler has crashed (or forcefully
   # aborted).
   class Message
+    include Logging
+
     # current message format version
     FORMAT_VERSION = 1
     # flag for encoding redundant messages
@@ -14,7 +16,7 @@ module Beetle
     DEFAULT_TTL = 1.day
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
-    DEFAULT_HANDLER_TIMEOUT = 300.seconds
+    DEFAULT_HANDLER_TIMEOUT = 600.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -77,6 +79,8 @@ module Beetle
       @format_version = headers[:format_version].to_i
       @flags = headers[:flags].to_i
       @expires_at = headers[:expires_at].to_i
+    rescue Exception => @exception
+      logger.error "Could not decode message. #{self.inspect}"
     end
 
     # build hash with options for the publisher
@@ -193,7 +197,7 @@ module Beetle
     # have we already seen this message? if not, set the status to "incomplete" and store
     # the message exipration timestamp in the deduplication store.
     def key_exists?
-      old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at)
+      old_message = 0 == @store.msetnx(msg_id, :status =>"incomplete", :expires => @expires_at, :timeout => now + timeout)
       if old_message
         logger.debug "Beetle: received duplicate message: #{msg_id} on queue: #{@queue}"
       end
@@ -236,7 +240,10 @@ module Beetle
     private
 
     def process_internal(handler)
-      if expired?
+      if @exception
+        ack!
+        RC::DecodingError
+      elsif expired?
         logger.warn "Beetle: ignored expired message (#{msg_id})!"
         ack!
         RC::Ancient
@@ -244,7 +251,6 @@ module Beetle
         ack!
         run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
       elsif !key_exists?
-        set_timeout!
         run_handler!(handler)
       elsif completed?
         ack!
@@ -273,7 +279,7 @@ module Beetle
     end
 
     def run_handler(handler)
-      Timeout::timeout(@timeout) { @handler_result = handler.call(self) }
+      Redis::Timer.timeout(@timeout.to_f) { @handler_result = handler.call(self) }
       RC::OK
     rescue Exception => @exception
       Beetle::reraise_expectation_errors!
@@ -313,14 +319,6 @@ module Beetle
       end
     end
 
-    def logger
-      @logger ||= self.class.logger
-    end
-
-    def self.logger
-      Beetle.config.logger
-    end
-
     # ack the message for rabbit. deletes all keys associated with this message in the
     # deduplication store if we are sure this is the last message with the given msg_id.
     def ack!

data/lib/beetle/publisher.rb

@@ -132,11 +132,16 @@ module Beetle
       b
     end
 
+    # retry dead servers after ignoring them for 10.seconds
+    # if all servers are dead, retry the one which has been dead for the longest time
     def recycle_dead_servers
       recycle = []
       @dead_servers.each do |s, dead_since|
         recycle << s if dead_since < 10.seconds.ago
       end
+      if recycle.empty? && @servers.empty?
+        recycle << @dead_servers.keys.sort_by{|k| @dead_servers[k]}.first
+      end
       @servers.concat recycle
       recycle.each {|s| @dead_servers.delete(s)}
     end
@@ -149,8 +154,11 @@ module Beetle
     end
 
     def select_next_server
-      return logger.error("Beetle: message could not be delivered - no server available") && 0 if @servers.empty?
-      set_current_server(@servers[((@servers.index(@server) || 0)+1) % @servers.size])
+      if @servers.empty?
+        logger.error("Beetle: no server available")
+      else
+        set_current_server(@servers[((@servers.index(@server) || 0)+1) % @servers.size])
+      end
     end
 
     def create_exchange!(name, opts)
@@ -165,9 +173,9 @@ module Beetle
 
     # TODO: Refactor, fethch the keys and stuff itself
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      logger.debug("Creating queue with opts: #{creation_keys.inspect}")
+      logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
       queue = bunny.queue(queue_name, creation_keys)
-      logger.debug("Binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
+      logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
       queue.bind(exchange(exchange_name), binding_keys)
       queue
     end