beetle 0.3.0.rc.8 → 0.3.0.rc.9

data/RELEASE_NOTES.rdoc

@@ -1,9 +1,15 @@
 = Release Notes
 
-== Version 0.3.0.rc.3
+== Version 0.3.0
 
+* allow accelerating master switch via POST to redis configuration server
+* embedded http server into the redis configuration server (port 8080)
+* fixed a problem with redis shutdown command
+* upgraded to redis 2.2.2
+* upgraded to amqp gem version 0.8 line
 * use hiredis as the redis backend, which overcomes lack of proper time-outs in the "generic" redis-rb
   gem for Ruby 1.9
+* use fully qualified hostnames to identify redis configuration clients
 
 == Version 0.2.9.8
 

data/beetle.gemspec

@@ -36,9 +36,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency("bunny",                   ["= 0.7.8"])
   s.add_runtime_dependency("redis",                   ["= 2.2.2"])
   s.add_runtime_dependency("hiredis",                 ["= 0.3.2"])
-  s.add_runtime_dependency("amq-client",              ["= 0.8.3"])
-  s.add_runtime_dependency("amq-protocol",            ["= 0.8.1"])
-  s.add_runtime_dependency("amqp",                    ["= 0.8.0"])
+  s.add_runtime_dependency("amq-client",              ["= 0.8.5"])
+  s.add_runtime_dependency("amq-protocol",            ["= 0.8.3"])
+  s.add_runtime_dependency("amqp",                    ["= 0.8.2"])
   s.add_runtime_dependency("activesupport",           [">= 2.3.4"])
   s.add_runtime_dependency("eventmachine_httpserver", [">= 0.2.1"])
   s.add_runtime_dependency("daemons",                 [">= 1.0.10"])

data/examples/test_publisher.rb

@@ -0,0 +1,32 @@
+# attempts.rb
+# this example shows you how to use the exception limiting feature of beetle
+# it allows you to control the number of retries your handler will go through
+# with one message before giving up on it
+#
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby attempts.rb
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+require "eventmachine"
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+
+# setup client
+client = Beetle::Client.new
+client.register_message(:test)
+
+n = 0
+EM.run do
+  EM.add_periodic_timer(0.1) do
+    data = (n+=1)
+    client.logger.info "publishing #{data}"
+    client.publish(:test, data)
+  end
+  trap("INT") do
+    client.stop_publishing
+    EM.stop_event_loop
+  end
+end

data/features/redis_auto_failover.feature

@@ -107,3 +107,42 @@ Feature: Redis auto failover
   Scenario: Redis configuation server should embed a http server
     Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
     Then the redis configuration server should answer http requests
+
+  Scenario: Accelerated redis master switch when master is down
+    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
+    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
+    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
+    And a beetle handler using the redis-master file from "rc-client-1" exists
+    And redis server "redis-1" is down
+    And an immediate master switch is initiated and responds with 201
+    Then a system notification for "redis-1" not being available should be sent
+    And the role of redis server "redis-2" should be "master"
+    And the redis master of "rc-client-1" should be "redis-2"
+    And the redis master of "rc-client-2" should be "redis-2"
+    And the redis master of the beetle handler should be "redis-2"
+    And a system notification for switching from "redis-1" to "redis-2" should be sent
+    Given a redis server "redis-1" exists as master
+    Then the role of redis server "redis-1" should be "slave"
+
+  Scenario: Accelerated redis master switch when master is up
+    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
+    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
+    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
+    And a beetle handler using the redis-master file from "rc-client-1" exists
+    And an immediate master switch is initiated and responds with 200
+    Then the role of redis server "redis-1" should be "master"
+    And the redis master of "rc-client-1" should be "redis-1"
+    And the redis master of "rc-client-2" should be "redis-1"
+    And the redis master of the beetle handler should be "redis-1"
+    And the role of redis server "redis-2" should be "slave"
+
+#   Scenario: Running the system for a few seconds to perform manual testing
+#     Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
+#     And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
+#     And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
+#     And a beetle handler using the redis-master file from "rc-client-1" exists
+#     And the redis master of "rc-client-1" should be "redis-1"
+#     And the redis master of "rc-client-2" should be "redis-1"
+#     And the redis master of the beetle handler should be "redis-1"
+#     And the role of redis server "redis-2" should be "slave"
+#     Then the system can run for a while without dying

data/features/step_definitions/redis_auto_failover_steps.rb

@@ -134,7 +134,17 @@ Then /^a system notification for no slave available to become new master should
 end
 
 Then /^the redis configuration server should answer http requests$/ do
-  TestDaemons::RedisConfigurationServer.answers_text_requests?
-  TestDaemons::RedisConfigurationServer.answers_html_requests?
-  TestDaemons::RedisConfigurationServer.answers_json_requests?
+  assert TestDaemons::RedisConfigurationServer.answers_text_requests?
+  assert TestDaemons::RedisConfigurationServer.answers_html_requests?
+  assert TestDaemons::RedisConfigurationServer.answers_json_requests?
+end
+
+Given /^an immediate master switch is initiated and responds with (\d+)$/ do |response_code|
+  response = TestDaemons::RedisConfigurationServer.initiate_master_switch
+  assert_equal response_code, response.code
+  sleep 1
+end
+
+Then /^the system can run for a while without dying$/ do
+  sleep 60
 end

data/features/support/test_daemons/redis_configuration_server.rb

@@ -66,9 +66,10 @@ module TestDaemons
     end
 
     def self.answers_html_requests?
-      response = get_status("/", "text/html")
-      response.code == '200' &&
-        response.content_type == "txt/html"
+      response1 = get_status("/", "text/html")
+      response2 = get_status("/.html", "text/html")
+      response1.code == '200' && response2.code == '200' &&
+        response1.content_type == "text/html" && response2.content_type == "text/html"
     rescue
       false
     end
@@ -79,9 +80,14 @@ module TestDaemons
       request = Net::HTTP::Get.new(uri.request_uri)
       request['Accept'] = content_type
       response = http.request(request)
-      # $stderr.puts response.content_type
-      # $stderr.puts response.body
       response
     end
+
+    def self.initiate_master_switch
+      http = Net::HTTP.new('127.0.0.1', 8080)
+      response = http.post '/initiate_master_switch', ''
+      response
+    end
+
   end
 end

data/lib/beetle/redis_configuration_http_server.rb

@@ -24,6 +24,7 @@ module Beetle
       #   @http_post_content
       #   @http_headers
       response = EM::DelegatedHttpResponse.new(self)
+      response.headers['Refresh'] = '3; url=/'
       # headers = @http_headers.split("\0").inject({}){|h, s| (s =~ /^([^:]+): (.*)$/ && (h[$1] = $2)); h }
 
       case @http_request_uri
@@ -36,6 +37,8 @@ module Beetle
       when "/.txt"
         response.content_type 'text/plain'
         server_status(response, "plain")
+      when '/initiate_master_switch'
+        initiate_master_switch(response)
       else
         not_found(response)
       end
@@ -74,7 +77,13 @@ module Beetle
         row =~/(^[^:]+): (.*)$/
         b << "<tr><td>#{$1}</td><td>#{$2}</td></tr>\n"
       end
-      b << "</table></body></html>"
+      b << "</table>"
+      unless status[:redis_master_available?]
+        b << "<form name='masterswitch' method='post' action='/initiate_master_switch'>"
+        b << "<a href='javascript: document.masterswitch.submit();'>Initiate master switch</a>"
+        b << "</form>"
+      end
+      b << "</body></html>"
     end
 
     def html_styles(status)
@@ -86,10 +95,32 @@ body { margin: 1em; }
 table tr:nth-child(2n+1){ background:#fff; }
 td { padding: 0.1em 0.2em; }
 h1 { color: #{warn_color}; margin-bottom: 0.2em;}
+a:link, a:visited {text-decoration:none; color:#A52A2A;}
+a:hover, a:active {text-decoration:none; color:#FF0000;}
+a {
+  font-size: 2em; padding: 10px; background: #cdcdcd;
+  -moz-border-radius: 5px;
+   border-radius: 5px;
+  -moz-box-shadow: 2px 2px 2px #bbb;
+  -webkit-box-shadow: 2px 2px 2px #bbb;
+  box-shadow: 2px 2px 2px #bbb;
+}
+form { margin-top: 1em; }
 </style>
 EOS
     end
 
+    def initiate_master_switch(response)
+      response.content_type 'text/plain'
+      if config_server.initiate_master_switch
+        response.status = 201
+        response.content = "Master switch initiated"
+      else
+        response.status = 200
+        response.content = "No master switch necessary"
+      end
+    end
+
     def not_found(response)
       response.content_type 'text/plain'
       response.status = 404

data/lib/beetle/redis_configuration_server.rb

@@ -56,6 +56,7 @@ module Beetle
         :redis_master => current_master.try(:server).to_s,
         :redis_master_available? => master_available?,
         :redis_slaves_available => available_slaves.map(&:server),
+        :switch_in_progress => paused?,
       }
     end
 
@@ -147,6 +148,17 @@ module Beetle
       redis.slaves
     end
 
+    # initiate a master switch if the current master is not available and no switch is in progress
+    def initiate_master_switch
+      redis.refresh
+      available, switch_in_progress = master_available?, paused?
+      logger.debug "initiating master switch: already in progress = #{switch_in_progress}"
+      unless available || switch_in_progress
+        master_unavailable!
+      end
+      !available || switch_in_progress
+    end
+
     private
 
     def check_redis_configuration

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "0.3.0.rc.8"
+  VERSION = "0.3.0.rc.9"
 end

data/test/beetle/redis_configuration_server_test.rb

@@ -63,6 +63,28 @@ module Beetle
     test "should be able to report current status" do
       assert @server.status.is_a?(Hash)
     end
+
+    test "should not execute a conditional master switch if the current master is available" do
+      @server.expects(:master_available?).returns(true)
+      @server.expects(:paused?).returns(false)
+      @server.expects(:master_unavailable!).never
+      assert !@server.initiate_master_switch
+    end
+
+    test "should not execute a conditional master switch if a switch is already in progress" do
+      @server.expects(:master_available?).returns(false)
+      @server.expects(:paused?).returns(true)
+      @server.expects(:master_unavailable!).never
+      assert @server.initiate_master_switch
+    end
+
+    test "should execute a conditional master switch if the current master is unavailable and no switch is in progress yet" do
+      @server.expects(:master_available?).returns(false)
+      @server.expects(:master_unavailable!).once
+      @server.expects(:paused?).returns(false)
+      assert @server.initiate_master_switch
+    end
+
   end
 
   class RedisConfigurationServerInvalidationTest < Test::Unit::TestCase

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: beetle
 version: !ruby/object:Gem::Version 
-  hash: 15424053
+  hash: 15424055
   prerelease: 6
   segments: 
   - 0
   - 3
   - 0
   - rc
-  - 8
-  version: 0.3.0.rc.8
+  - 9
+  version: 0.3.0.rc.9
 platform: ruby
 authors: 
 - Stefan Kaes
@@ -21,7 +21,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-29 00:00:00 +02:00
+date: 2011-10-31 00:00:00 +01:00
 default_executable: beetle
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -96,12 +96,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 57
+        hash: 53
         segments: 
         - 0
         - 8
-        - 3
-        version: 0.8.3
+        - 5
+        version: 0.8.5
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
@@ -112,12 +112,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 61
+        hash: 57
         segments: 
         - 0
         - 8
-        - 1
-        version: 0.8.1
+        - 3
+        version: 0.8.3
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
@@ -128,12 +128,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 63
+        hash: 59
         segments: 
         - 0
         - 8
-        - 0
-        version: 0.8.0
+        - 2
+        version: 0.8.2
   type: :runtime
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
@@ -282,6 +282,7 @@ files:
 - examples/redundant.rb
 - examples/rpc.rb
 - examples/simple.rb
+- examples/test_publisher.rb
 - lib/beetle/base.rb
 - lib/beetle/client.rb
 - lib/beetle/commands/configuration_client.rb
@@ -314,7 +315,6 @@ files:
 - features/support/test_daemons/redis_configuration_client.rb
 - features/support/test_daemons/redis_configuration_server.rb
 - script/console
-- script/console~
 - script/start_rabbit
 - beetle.gemspec
 - Rakefile

data/script/console~

@@ -1,2 +0,0 @@
-#!/usr/bin/env ruby
-$:.unshift(File.expand_path("../lib",__FILE__))