beetle 0.2.3 → 0.2.5

data/features/step_definitions/redis_auto_failover_steps.rb

@@ -0,0 +1,133 @@
+Given /^a redis server "([^\"]*)" exists as master$/ do |redis_name|
+  TestDaemons::Redis[redis_name].start
+  TestDaemons::Redis[redis_name].master
+end
+
+Given /^a redis server "([^\"]*)" exists as slave of "([^\"]*)"$/ do |redis_name, redis_master_name|
+  TestDaemons::Redis[redis_name].start
+  Given "redis server \"#{redis_name}\" is slave of \"#{redis_master_name}\""
+end
+
+Given /^redis server "([^\"]*)" is master$/ do |redis_name|
+  TestDaemons::Redis[redis_name].master
+end
+
+Given /^redis server "([^\"]*)" is slave of "([^\"]*)"$/ do |redis_name, redis_master_name|
+  TestDaemons::Redis[redis_name].slave_of(TestDaemons::Redis[redis_master_name].port)
+  master = TestDaemons::Redis[redis_master_name].redis
+  slave = TestDaemons::Redis[redis_name].redis
+  begin
+    sleep 1
+  end while !slave.slave_of?(master.host, master.port)
+end
+
+Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" exists$/ do |redis_names, redis_configuration_client_names|
+  redis_servers = redis_names.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
+  TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names)
+end
+
+Given /^a redis configuration client "([^\"]*)" using redis servers "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_names|
+  redis_servers = redis_names.split(",").map do |redis_name|
+    TestDaemons::Redis[redis_name].ip_with_port
+  end
+  TestDaemons::RedisConfigurationClient[redis_configuration_client_name].start
+end
+
+Given /^redis server "([^\"]*)" is down$/ do |redis_name|
+  TestDaemons::Redis[redis_name].stop
+end
+
+Given /^the retry timeout for the redis master check is reached$/ do
+  basetime = Time.now
+  i = 0
+  while (i <= 10.0) do
+    break if TestDaemons::RedisConfigurationClient.instances.values.all? {|instance| File.mtime(instance.redis_master_file) > basetime rescue false}
+    i += 0.1
+    sleep(0.1)
+  end
+  sleep 1 # give it time to switch because the modified mtime might be because of the initial invalidation and not the switch
+end
+
+Given /^a beetle handler using the redis-master file from "([^\"]*)" exists$/ do |redis_configuration_client_name|
+  master_file = redis_master_file(redis_configuration_client_name)
+  `ruby features/support/beetle_handler start -- --redis-master-file=#{master_file}`
+  assert File.exist?(master_file), "file #{master_file} does not exist"
+end
+
+Given /^redis server "([^\"]*)" is down for less seconds than the retry timeout for the redis master check$/ do |redis_name|
+  TestDaemons::Redis[redis_name].restart(1)
+end
+
+Given /^the retry timeout for the redis master determination is reached$/ do
+  sleep 1
+end
+
+Given /^redis server "([^\"]*)" is coming back$/ do |redis_name|
+  TestDaemons::Redis[redis_name].start
+end
+
+Given /^an old redis master file for "([^\"]*)" with master "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_name|
+  master_file = redis_master_file(redis_configuration_client_name)
+  File.open(master_file, 'w') do |f|
+    f.puts TestDaemons::Redis[redis_name].ip_with_port
+  end
+end
+
+
+Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redis_name, role|
+  expected_role = false
+  10.times do
+    expected_role = true and break if TestDaemons::Redis[redis_name].__send__ "#{role}?"
+    sleep 1
+  end
+  assert expected_role, "#{redis_name} is not a #{role}"
+end
+
+Then /^the redis master of "([^\"]*)" should be "([^\"]*)"$/ do |redis_configuration_client_name, redis_name|
+  master_file = redis_master_file(redis_configuration_client_name)
+  master = false
+  server_info = nil
+  10.times do
+    server_info = File.read(master_file).chomp if File.exist?(master_file)
+    master = true and break if TestDaemons::Redis[redis_name].ip_with_port == server_info
+    sleep 1
+  end
+  assert master, "#{redis_name} is not master of #{redis_configuration_client_name}, master file content: #{server_info.inspect}"
+end
+
+Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configuration_client_name|
+  master_file = redis_master_file(redis_configuration_client_name)
+  empty = false
+  server_info = nil
+  10.times do
+    server_info = File.read(master_file).chomp if File.exist?(master_file)
+    empty = server_info == ""
+    break if empty
+    sleep 1
+  end
+  assert empty, "master file is not empty: #{server_info}"
+end
+
+Then /^the redis master of the beetle handler should be "([^\"]*)"$/ do |redis_name|
+  Beetle.config.servers = "localhost:5672" # rabbitmq
+  Beetle.config.logger.level = Logger::INFO
+  client = Beetle::Client.new
+  client.register_queue(:echo)
+  client.register_message(:echo)
+  assert_equal TestDaemons::Redis[redis_name].ip_with_port, client.rpc(:echo, 'nil').second
+end
+
+Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
+  text = "Redis master '#{TestDaemons::Redis[redis_name].ip_with_port}' not available"
+  assert_match /#{text}/, File.readlines(system_notification_log_path).last
+end
+
+Then /^a system notification for switching from "([^\"]*)" to "([^\"]*)" should be sent$/ do |old_redis_master_name, new_redis_master_name|
+  text = "Setting redis master to '#{TestDaemons::Redis[new_redis_master_name].ip_with_port}' (was '#{TestDaemons::Redis[old_redis_master_name].ip_with_port}')"
+  assert_match /#{Regexp.escape(text)}/, File.read(system_notification_log_path)
+end
+
+Then /^a system notification for no slave available to become new master should be sent$/ do
+  text = "Redis master could not be switched, no slave available to become new master"
+  assert_match /#{text}/, File.readlines(system_notification_log_path).last
+end

data/features/support/beetle_handler

@@ -0,0 +1,32 @@
+#! /usr/bin/env ruby
+
+require "rubygems"
+require "daemons"
+require "optparse"
+require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
+
+tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
+
+Daemons.run_proc("beetle_handler", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
+  opts = OptionParser.new
+
+  opts.on("-f", "--redis-master-file path", String) do |val|
+    Beetle.config.redis_server = val
+  end
+
+  opts.parse!(ARGV - ["start", "--"])
+
+  Beetle.config.servers = "localhost:5672" # rabbitmq
+
+  # set Beetle log level to info, less noisy than debug
+  Beetle.config.logger.level = Logger::INFO
+
+  client = Beetle::Client.new.configure :auto_delete => true do |config|
+    config.queue(:echo)
+    config.message(:echo)
+    config.handler(:echo) {|message| client.deduplication_store.redis.server rescue "no redis master"}
+  end
+  client.listen do
+    puts "Started beetle handler"
+  end
+end

data/features/support/env.rb

@@ -0,0 +1,48 @@
+require File.expand_path('../../../lib/beetle', __FILE__)
+
+# Allow using Test::Unit for step assertions
+# See http://wiki.github.com/aslakhellesoy/cucumber/using-testunit
+require 'test/unit/assertions'
+World(Test::Unit::Assertions)
+
+Before do
+  `ruby features/support/system_notification_logger start`
+end
+
+After do
+  cleanup_test_env
+end
+
+at_exit do
+  cleanup_test_env
+end
+
+def cleanup_test_env
+  TestDaemons::RedisConfigurationClient.stop_all
+  TestDaemons::RedisConfigurationServer.stop
+
+  `ruby features/support/beetle_handler stop`
+  redis_master_files = tmp_path + "/redis-master-*"
+  `rm -f #{redis_master_files}`
+
+  `ruby features/support/system_notification_logger stop`
+  `rm -f #{system_notification_log_path}`
+
+  TestDaemons::Redis.stop_all
+end
+
+def redis_master_file(client_name)
+  tmp_path + "/redis-master-#{client_name}"
+end
+
+def first_redis_configuration_client_pid
+  File.read("redis_configuration_client0.pid").chomp.to_i
+end
+
+def system_notification_log_path
+  tmp_path + "/system_notifications.log"
+end
+
+def tmp_path
+  File.expand_path("../../../tmp", __FILE__)
+end

data/features/support/system_notification_logger

@@ -0,0 +1,31 @@
+#! /usr/bin/env ruby
+
+require "rubygems"
+require "daemons"
+require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
+
+tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
+system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
+
+Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
+  Beetle.config.servers = "localhost:5672, localhost:5673" # rabbitmq
+
+  # set Beetle log level to info, less noisy than debug
+  Beetle.config.logger.level = Logger::DEBUG
+
+  client = Beetle::Client.new
+  client.configure :exchange => :system, :auto_delete => true do |config|
+    config.message  :system_notification
+    config.queue    :system_notification
+    config.handler  :system_notification do |message|
+      payload = ActiveSupport::JSON.decode(message.data)
+      text = payload["message"]
+      puts "Writing message to #{system_notification_log_file_path}: #{text}"
+      File.open(system_notification_log_file_path, "a+") do |f|
+        f << text
+      end
+    end
+  end
+  puts "Started system notification logger"
+  client.listen
+end

data/features/support/test_daemons/redis.conf.erb

@@ -0,0 +1,189 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile <%= pid_file %>
+
+# Accept connections on the specified port, default is 6379
+port <%= port %>
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile <%= log_file %>
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################ SNAPSHOTTING  #################################
+#
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# Compress string objects using LZF when dump .rdb databases?
+# For default that's set to 'yes' as it's almost always a win.
+# If you want to save some CPU in the saving child set it to 'no' but
+# the dataset will likely be bigger if you have compressible values or keys.
+rdbcompression yes
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir <%= dir %>
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+#
+# slaveof <masterip> <masterport>
+
+# If the master is password protected (using the "requirepass" configuration
+# directive below) it is possible to tell the slave to authenticate before
+# starting the replication synchronization process, otherwise the master will
+# refuse the slave request.
+#
+# masterauth <master-password>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+#
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+#
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+#
+# maxmemory <bytes>
+
+############################## APPEND ONLY MODE ###############################
+
+# By default Redis asynchronously dumps the dataset on disk. If you can live
+# with the idea that the latest records will be lost if something like a crash
+# happens this is the preferred way to run Redis. If instead you care a lot
+# about your data and don't want to that a single record can get lost you should
+# enable the append only mode: when this mode is enabled Redis will append
+# every write operation received in the file appendonly.log. This file will
+# be read on startup in order to rebuild the full dataset in memory.
+#
+# Note that you can have both the async dumps and the append only file if you
+# like (you have to comment the "save" statements above to disable the dumps).
+# Still if append only mode is enabled Redis will load the data from the
+# log file at startup ignoring the dump.rdb file.
+#
+# The name of the append only file is "appendonly.log"
+#
+# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
+# log file in background when it gets too big.
+
+appendonly yes
+
+# The fsync() call tells the Operating System to actually write data on disk
+# instead to wait for more data in the output buffer. Some OS will really flush 
+# data on disk, some other OS will just try to do it ASAP.
+#
+# Redis supports three different modes:
+#
+# no: don't fsync, just let the OS flush the data when it wants. Faster.
+# always: fsync after every write to the append only log . Slow, Safest.
+# everysec: fsync only if one second passed since the last fsync. Compromise.
+#
+# The default is "always" that's the safer of the options. It's up to you to
+# understand if you can relax this to "everysec" that will fsync every second
+# or to "no" that will let the operating system flush the output buffer when
+# it want, for better performances (but if you can live with the idea of
+# some data loss consider the default persistence mode that's snapshotting).
+
+# appendfsync always
+appendfsync everysec
+# appendfsync no
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.
+shareobjects no
+shareobjectspoolsize 1024

data/features/support/test_daemons/redis.rb

@@ -0,0 +1,186 @@
+require 'rubygems'
+require 'fileutils'
+require 'erb'
+require 'redis'
+require 'lib/beetle/redis_ext'
+require 'daemon_controller'
+
+module TestDaemons
+  class Redis
+
+    @@instances = {}
+    @@next_available_port = 6381
+
+    attr_reader :name, :port
+
+    def initialize(name)
+      @name = name
+      @port = @@next_available_port
+      
+      @@next_available_port += 1
+      @@instances[name] = self
+    end
+
+    class << self
+      def find_or_initialize_by_name(name)
+        @@instances[name] ||= new(name)
+      end
+      alias_method :[], :find_or_initialize_by_name
+
+      def stop_all
+        @@instances.values.each{|i| i.stop}
+      end
+    end
+
+    def start
+      create_dir
+      create_config
+      daemon_controller.start
+    end
+
+    def restart(delay=1)
+      redis.shutdown rescue Errno::ECONNREFUSED
+      sleep delay
+      `redis-server #{config_filename}`
+    end
+
+    def stop
+      # TODO: Might need to be moved into RedisConfigurationServer
+      #     10.times do
+      #       break if (redis.info["bgsave_in_progress"]) == 0 rescue false
+      #       sleep 1
+      #     end
+      daemon_controller.stop
+    ensure
+      cleanup
+    end
+
+    def cleanup
+      remove_dir
+      remove_config
+      remove_pid_file
+    end
+
+    # TODO: The retry logic must be moved into RedisConfigurationServer
+    def master
+      tries = 0
+      begin
+        redis.master!
+      rescue Errno::ECONNREFUSED, Errno::EAGAIN
+        puts "master role setting for #{name} failed: #{$!}"
+        sleep 1
+        retry if (tries+=1) > 5
+        raise "could not setup master #{name} #{$!}"
+      end
+    end
+
+    def running?
+      cmd = "ps aux | grep 'redis-server #{config_filename}' | grep -v grep"
+      res = `#{cmd}`
+      x = res.chomp.split("\n")
+      x.size == 1
+    end
+
+    def available?
+      redis.available?
+    end
+
+    def master?
+      redis.master?
+    end
+
+    def slave?
+      redis.slave?
+    end
+
+    # TODO: Move to redis_ext
+    def slave_of(master_port)
+      tries = 0
+      begin
+        redis.slave_of!("127.0.0.1", master_port)
+      rescue Errno::ECONNREFUSED, Errno::EAGAIN
+        puts "slave role setting for #{name} failed: #{$!}"
+        sleep 1
+        retry if (tries+=1) > 5
+        raise "could not setup slave #{name}: #{$!}"
+      end
+    end
+
+    def ip_with_port
+      "127.0.0.1:#{port}"
+    end
+  
+    def redis
+      @redis ||= ::Redis.new(:host => "127.0.0.1", :port => port)
+    end
+
+    private
+
+    def create_dir
+      FileUtils.mkdir(dir) unless File.exists?(dir)
+    end
+
+    def remove_dir
+      FileUtils.rm_r(dir) if File.exists?(dir)
+    end
+
+    def create_config
+      File.open(config_filename, "w") do |file|
+        file.puts config_content
+      end
+    end
+
+    def remove_config
+      FileUtils.rm(config_filename) if File.exists?(config_filename)
+    end
+
+    def remove_pid_file
+      FileUtils.rm(pid_file) if File.exists?(pid_file)
+    end
+
+    def tmp_path
+      File.expand_path(File.dirname(__FILE__) + "/../../../tmp")
+    end
+
+    def config_filename
+      tmp_path + "/redis-test-server-#{name}.conf"
+    end
+
+    def config_content
+      template = ERB.new(File.read(config_template_filename))
+      template.result(binding)
+    end
+
+    def config_template_filename
+      File.dirname(__FILE__) + "/redis.conf.erb"
+    end
+
+    def pid_file
+      tmp_path + "/redis-test-server-#{name}.pid"
+    end
+
+    def pid
+      File.read(pid_file).chomp.to_i
+    end
+
+    def log_file
+      tmp_path + "/redis-test-server-#{name}.log"
+    end
+
+    def dir
+      tmp_path + "/redis-test-server-#{name}/"
+    end
+
+    def daemon_controller
+      @daemon_controller ||= DaemonController.new(
+         :identifier    => "Redis test server",
+         :start_command => "redis-server #{config_filename}",
+         :ping_command  => lambda { running? && available? },
+         :pid_file      => pid_file,
+         :log_file      => log_file,
+         :start_timeout => 5
+      )
+    end
+  
+  end
+end