beetle 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07ab6ab204d65b6e29dab8620977174fde65fccf
-  data.tar.gz: 07c94854bc429eebe174d2d933449306b2cc7792
+  metadata.gz: f8b9df71eabc29dad2d131ff952029d8beffcf82
+  data.tar.gz: 0301abf7e282bbc202f14d8671dec32569ca6835
 SHA512:
-  metadata.gz: 5053cb17e558f284f871e086504c911fdc360b67de0d53838ea80decb3aaa90add2d46d9cc1f5f4aeac5e65b10fdaf417bafa4f7b6382faa9f7c83a9f0053a7c
-  data.tar.gz: e666edc5889731f1b65802927d440846d58e25c7eb56f7e7f6338710239908ff0449ad2b3b5e59c794a2f661e2aa752b5e06ec3cf1bb50ecdaa7741d669ffe97
+  metadata.gz: c88a91e53de54ef055ae3dc8e28d55395331a6ec9c798b39c86cb66f49156246a3bf2d8b21d6afd15f84f343b58f29e1238b35ffaab9b87621c8f769c0fadbda
+  data.tar.gz: a6bc8a6759b4dcc65bd9b095d80df03b748f1de0f2da8b6f1731778fc0d825eca33e13377ad43287bdce50674dca4c65d199738a0756a284320711ef6e0fd5f5

data/README.rdoc

@@ -151,6 +151,11 @@ In short (see {semver.org}[http://semver.org] for details):
 Then use `rake release` which will create the git tag and upload the
 gem to github.com:
 
-    $ bundle exec rake release
+    bundle exec rake release
 
 The generated gem is located in the `pkg/` directory.
+
+In order to build go binaries and upload the docker container with the
+beetle GO binary to docker hub, run
+
+    make release

data/RELEASE_NOTES.rdoc

@@ -1,8 +1,14 @@
 = Release Notes
 
+== Version 2.0.1
+
+* fix for beetle command not geting stuck when connecting to
+  configuration server
+* configuration server displays last seen times in a human readable format
+
 == Version 2.0.0
 
-* beetle command has been rewritten in GO
+* beetle command has been rewritten in Go
 * garbage collecting redis keys now uses the redis SCAN command
 
 == Version 1.0.4

data/Rakefile

@@ -85,6 +85,8 @@ Cucumber::Rake::Task.new(:cucumber) do |t|
   t.cucumber_opts = "features --format progress"
 end
 
+task :cucumber => :clean
+
 task :default do
   Rake::Task[:test].invoke
   Rake::Task[:cucumber].invoke

data/beetle.gemspec

@@ -35,11 +35,11 @@ Gem::Specification.new do |s|
   s.add_development_dependency "daemons",             ">= 1.2.0"
   s.add_development_dependency "i18n"
   s.add_development_dependency "minitest",            "~> 5.1"
-  s.add_development_dependency "mocha",               "~> 1.1.0"
+  s.add_development_dependency "mocha",               "~> 1.3.0"
   s.add_development_dependency "mysql2",              "~> 0.4.4"
   s.add_development_dependency "rake",                "~> 11.2"
   s.add_development_dependency "rdoc",                "~> 4.0"
-  s.add_development_dependency "simplecov",           "~> 0.12.0"
-  s.add_development_dependency "webmock",             "~> 1.21.0"
+  s.add_development_dependency "simplecov",           "~> 0.15"
+  s.add_development_dependency "webmock",             "~> 3.0"
   s.add_development_dependency "websocket-eventmachine-client"
 end

data/features/step_definitions/redis_auto_failover_steps.rb

@@ -126,17 +126,21 @@ end
 
 Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
   text = "Redis master '#{TestDaemons::Redis[redis_name].ip_with_port}' not available"
-  assert_match /#{text}/, File.readlines(system_notification_log_path).last
+  lines = File.readlines(system_notification_log_path)
+  tail = (["","",""]+lines)[-3..-1].join("\n")
+  assert_match /#{text}/, tail
 end
 
 Then /^a system notification for switching from "([^\"]*)" to "([^\"]*)" should be sent$/ do |old_redis_master_name, new_redis_master_name|
   text = "Setting redis master to '#{TestDaemons::Redis[new_redis_master_name].ip_with_port}' (was '#{TestDaemons::Redis[old_redis_master_name].ip_with_port}')"
-  assert_match /#{Regexp.escape(text)}/, File.read(system_notification_log_path)
+  lines = File.readlines(system_notification_log_path)
+  assert_match /#{Regexp.escape(text)}/, lines.last
 end
 
 Then /^a system notification for no slave available to become new master should be sent$/ do
   text = "Redis master could not be switched, no slave available to become new master"
-  assert_match /#{text}/, File.readlines(system_notification_log_path).last
+  lines = File.readlines(system_notification_log_path)
+  assert_match /#{text}/, lines.last
 end
 
 Then /^the redis configuration server should answer http requests$/ do

data/features/support/env.rb

@@ -27,7 +27,7 @@ def cleanup_test_env
   `rm -f #{redis_master_files}`
 
   `ruby features/support/system_notification_logger stop`
-  `rm -f #{system_notification_log_path}`
+  # `rm -f #{system_notification_log_path}`
 
   TestDaemons::Redis.stop_all
 end

data/features/support/system_notification_logger

@@ -9,7 +9,7 @@ require File.expand_path("../../lib/beetle", File.dirname(__FILE__))
 tmp_path = File.expand_path("../../tmp", File.dirname(__FILE__))
 system_notification_log_file_path = "#{tmp_path}/system_notifications.log"
 
-DEBUG = true
+DEBUG = false
 
 Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode => :normal, :dir => tmp_path) do
   Beetle.config.servers = "127.0.0.1:5672" # rabbitmq
@@ -34,11 +34,11 @@ Daemons.run_proc("system_notification_logger", :log_output => true, :dir_mode =>
       end
       ws.onclose do
         puts "server closed connection" if DEBUG && !@interrupted
-        EM.add_timer(1){ EM.stop_event_loop }
+        EM.add_timer(0){ EM.stop_event_loop }
       end
       ws.onmessage do |text|
         puts "writing message to #{system_notification_log_file_path}: #{text}"
-        log_file << text
+        log_file << (text + "\n")
       end
       puts "Started system notification logger"
       trap("INT") {  shutdown(ws) }

data/features/support/test_daemons/redis.conf.erb

@@ -425,7 +425,7 @@ slave-priority 100
 # Once the limit is reached Redis will close all the new connections sending
 # an error 'max number of clients reached'.
 #
-# maxclients 10000
+maxclients 100
 
 # Don't use more memory than the specified amount of bytes.
 # When the memory limit is reached Redis will try to remove keys

data/features/support/test_daemons/redis.rb

@@ -43,16 +43,26 @@ module TestDaemons
       create_config
       daemon_controller.stop if running?
       sleep delay
-      daemon_controller.start
+      tries = 3
+      begin
+        daemon_controller.start
+      rescue DaemonController::StartError => e
+        puts "?????????? redis-server failed to start: #{e}"
+        retry if (tries -= 1) > 0
+      rescue DaemonController::AlreadyStarted => e
+        puts "?????????? redis-server already startes: #{e}"
+      end
     end
 
     def stop
       return unless running?
       # TODO: Might need to be moved into RedisConfigurationServer
-      #     10.times do
-      #       break if (redis.info["bgsave_in_progress"]) == 0 rescue false
-      #       sleep 1
-      #     end
+      10.times do
+        rdb_bgsave_in_progress, aof_rewrite_in_progress = redis.info_with_rescue.values_at("rdb_bgsave_in_progress", "aof_rewrite_in_progress")
+        break if rdb_bgsave_in_progress == "0" && aof_rewrite_in_progress == "0"
+        puts "#{Time.now} redis #{name} is still saving to disk"
+        sleep 1
+      end
       daemon_controller.stop
       raise "FAILED TO STOP redis server on port #{port}" if running?
     ensure

data/features/support/test_daemons/redis_configuration_client.rb

@@ -40,7 +40,8 @@ module TestDaemons
          :ping_command  => lambda{ true },
          :pid_file      => pid_file,
          :log_file      => log_file,
-         :start_timeout => 5,
+         :start_timeout => 10,
+         :stop_timeout => 10,
       )
     end
 

data/features/support/test_daemons/redis_configuration_server.rb

@@ -29,7 +29,8 @@ module TestDaemons
          :ping_command  => lambda{ answers_text_requests? },
          :pid_file      => pid_file,
          :log_file      => log_file,
-         :start_timeout => 5
+         :start_timeout => 10,
+         :stop_timeout => 10,
       )
     end
 
@@ -74,10 +75,8 @@ module TestDaemons
       false
     end
 
-    HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9650 : 8080
-
     def self.get_status(path, content_type)
-      uri = URI.parse("http://127.0.0.1:#{HTTP_SERVER_PORT}#{path}")
+      uri = URI.parse("http://127.0.0.1:9650#{path}")
       http = Net::HTTP.new(uri.host, uri.port)
       request = Net::HTTP::Get.new(uri.request_uri)
       request['Accept'] = content_type
@@ -86,7 +85,7 @@ module TestDaemons
     end
 
     def self.initiate_master_switch
-      http = Net::HTTP.new('127.0.0.1', HTTP_SERVER_PORT)
+      http = Net::HTTP.new('127.0.0.1', 9650)
       response = http.post '/initiate_master_switch', ''
       response
     end

data/lib/beetle/redis_ext.rb

@@ -46,10 +46,11 @@ class Redis #:nodoc:
     info["role"] == "slave" && info["master_host"] == host && info["master_port"] == port.to_s
   end
 
-  # compatibility layer for redis 2.2.2
-  # remove this once all our apps have upgraded to 3.x
   if Redis::VERSION < "3.0"
 
+    # compatibility layer for redis 2.2.2
+    # remove this once all our apps have upgraded to 3.x
+
     # Redis 2 tries to establish a connection on inspect. this is evil!
     def inspect
       super
@@ -77,5 +78,19 @@ class Redis #:nodoc:
     def msetnx(*values)
       super != 0
     end
+
+  elsif Redis::VERSION >= "4.0.0"
+
+    # redis 4.0.0 has a shutdown method which raises if a connection to the redis server
+    # cannot be established.
+    module SaneShutdown
+      def shutdown
+        super
+      rescue CannotConnectError
+        nil
+      end
+    end
+    prepend SaneShutdown
+
   end
 end

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/test/beetle/dead_lettering_test.rb

@@ -40,23 +40,25 @@ module Beetle
     end
 
     test "creates a policy by posting to the rabbitmq" do
-      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
-        with(:body => {
-        "pattern" => "^QUEUE_NAME$",
-        "priority" => 1,
-        "apply-to" => "queues",
-        "definition" => {
-          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-          "dead-letter-exchange" => ""
-        }}.to_json).
-        to_return(:status => 204)
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                 "dead-letter-exchange" => ""
+               }}.to_json)
+        .to_return(:status => 204)
 
       @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
     end
 
     test "raises exception when policy couldn't successfully be created" do
-      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
-        to_return(:status => [405])
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => [405])
 
       assert_raises DeadLettering::FailedRabbitRequest do
         @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
@@ -64,32 +66,34 @@ module Beetle
     end
 
     test "can optionally specify a message ttl" do
-      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/%2F/QUEUE_NAME_policy").
-        with(:body => {
-        "pattern" => "^QUEUE_NAME$",
-        "priority" => 1,
-        "apply-to" => "queues",
-        "definition" => {
-          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-          "dead-letter-exchange" => "",
-          "message-ttl" => 10000
-        }}.to_json).
-        to_return(:status => 204)
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+                "pattern" => "^QUEUE_NAME$",
+                "priority" => 1,
+                "apply-to" => "queues",
+                "definition" => {
+                  "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                  "dead-letter-exchange" => "",
+                  "message-ttl" => 10000
+                }}.to_json)
+        .to_return(:status => 204)
 
       @dead_lettering.set_dead_letter_policy!(@server, @queue_name, :message_ttl => 10000)
     end
 
     test "properly encodes the vhost from the configuration" do
-      stub_request(:put, "http://guest:guest@localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy").
-        with(:body => {
-        "pattern" => "^QUEUE_NAME$",
-        "priority" => 1,
-        "apply-to" => "queues",
-        "definition" => {
-          "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-          "dead-letter-exchange" => ""
-        }}.to_json).
-        to_return(:status => 204)
+      stub_request(:put, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                 "dead-letter-exchange" => ""
+               }}.to_json)
+        .to_return(:status => 204)
 
       @config.vhost = "foo/"
 

data/test/beetle/redis_ext_test.rb

@@ -80,13 +80,20 @@ module Beetle
         @r.shutdown
       end
 
-    else
+    elsif Redis::VERSION < "4.0"
 
       test "redis shutdown implementation should call :shutdown and return nil" do
         @r.client.expects(:call).with([:shutdown]).once.raises(Redis::ConnectionError)
         assert_nil @r.shutdown
       end
 
+    else
+
+      test "patched redis shutdown implementation should not raise connection refused but return nil" do
+        # note that we do not have redis running on port 6390
+        assert_nil @r.shutdown
+      end
+
     end
   end
 

data/test/test_helper.rb

@@ -17,6 +17,7 @@ class Minitest::Test
   require "active_support/testing/declarative"
   extend ActiveSupport::Testing::Declarative
   require "webmock"
+  WebMock.enable!
   include WebMock::API
   def assert_nothing_raised(*)
     yield

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-28 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid4r
@@ -202,14 +202,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -258,28 +258,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '0.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.21.0
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.21.0
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: websocket-eventmachine-client
   requirement: !ruby/object:Gem::Requirement
@@ -389,7 +389,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.7
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 3
 summary: High Availability AMQP Messaging with Redundant Queues