beetle 3.5.1 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +8 -12
- data/beetle.gemspec +3 -1
- data/examples/redundant.rb +1 -1
- data/examples/rpc.rb +1 -1
- data/features/redis_auto_failover.feature +20 -0
- data/features/step_definitions/redis_auto_failover_steps.rb +28 -8
- data/features/support/test_daemons/redis_configuration_server.rb +2 -1
- data/lib/beetle/publisher.rb +3 -1
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/amqp_gem_behavior_test.rb +1 -1
- data/test/beetle/base_test.rb +1 -1
- data/test/beetle/client_test.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +1 -1
- data/test/beetle/publisher_test.rb +15 -6
- data/test/beetle/subscriber_test.rb +6 -2
- data/test/test_helper.rb +8 -7
- metadata +51 -31
- data/README.rdoc +0 -233
- data/REDIS_AUTO_FAILOVER.rdoc +0 -116
- data/RELEASE_NOTES.rdoc +0 -383
- data/examples/README.rdoc +0 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bce54bf1dc2290bb4c1598f6210b1c292d09691e91df2f67e913747535c90f2d
|
4
|
+
data.tar.gz: e6f510fb5b78a7ed0483f713920222ff1959e7a3e3073aac61abed25af56b0bd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8682a1720108f7e95ad9802bcfe5b30283c735cfaa67323e5aca6bca1e1c06d0b6bb58d8fd2b8f4fc1fb7032579832667b78bab0c0f7013a937556650ad14ad7
|
7
|
+
data.tar.gz: 255b3fd3827a1f33c1c244967649ab8eaa1adc8f6daa3a111b21e299f1bcc0341f324a1bee3f2c578f5c93b39eeb913caf2c3a6d8ea88d1e89c028c48576e1b0
|
data/Rakefile
CHANGED
@@ -115,18 +115,14 @@ Rake::TestTask.new do |t|
|
|
115
115
|
t.warning = false
|
116
116
|
end
|
117
117
|
|
118
|
-
|
119
|
-
|
120
|
-
RDoc::Task.new do |rdoc|
|
121
|
-
rdoc.rdoc_dir = 'site/rdoc'
|
122
|
-
rdoc.title = 'Beetle'
|
123
|
-
rdoc.main = 'README.rdoc'
|
124
|
-
rdoc.options << '--line-numbers' << '--inline-source' << '--quiet'
|
125
|
-
rdoc.rdoc_files.include('**/*.rdoc')
|
126
|
-
rdoc.rdoc_files.include('MIT-LICENSE')
|
127
|
-
rdoc.rdoc_files.include('lib/**/*.rb')
|
118
|
+
task :clean do
|
119
|
+
sh "rm -f tmp/*.output tmp/*.log tmp/master-dir/* tmp/slave-dir/* tmp/*lock tmp/*pid test.log" unless ENV['GITHUB_ACTIONS']
|
128
120
|
end
|
129
121
|
|
130
|
-
|
131
|
-
|
122
|
+
require 'yard'
|
123
|
+
|
124
|
+
YARD::Rake::YardocTask.new do |t|
|
125
|
+
OTHER_PATHS = %w()
|
126
|
+
t.files = ['lib/**/*.rb'] + OTHER_PATHS
|
127
|
+
t.options = %w(--markup-provider=redcarpet --markup=markdown --main=README.md --output-dir=site/yard)
|
132
128
|
end
|
data/beetle.gemspec
CHANGED
@@ -39,8 +39,10 @@ Gem::Specification.new do |s|
|
|
39
39
|
s.add_development_dependency "mocha", "~> 1.3.0"
|
40
40
|
s.add_development_dependency "mysql2", "~> 0.4.4"
|
41
41
|
s.add_development_dependency "rake", "~> 13.0"
|
42
|
-
s.add_development_dependency "rdoc", "~> 4.0"
|
43
42
|
s.add_development_dependency "simplecov", "~> 0.15"
|
44
43
|
s.add_development_dependency "webmock", "~> 3.0"
|
45
44
|
s.add_development_dependency "websocket-eventmachine-client"
|
45
|
+
s.add_development_dependency 'yard'
|
46
|
+
s.add_development_dependency 'redcarpet'
|
47
|
+
s.add_development_dependency 'github-markup'
|
46
48
|
end
|
data/examples/redundant.rb
CHANGED
@@ -17,7 +17,7 @@ Beetle.config.logger.level = Logger::INFO
|
|
17
17
|
client = Beetle::Client.new
|
18
18
|
|
19
19
|
# use two servers
|
20
|
-
Beetle.config.servers = "localhost:5672, localhost:5673"
|
20
|
+
Beetle.config.servers = ENV["RABBITMQ_SERVERS"] || "localhost:5672, localhost:5673"
|
21
21
|
# instantiate a client
|
22
22
|
client = Beetle::Client.new
|
23
23
|
|
data/examples/rpc.rb
CHANGED
@@ -4,7 +4,7 @@ require File.expand_path(File.dirname(__FILE__)+"/../lib/beetle")
|
|
4
4
|
|
5
5
|
# suppress debug messages
|
6
6
|
Beetle.config.logger.level = Logger::INFO
|
7
|
-
Beetle.config.servers = "localhost:5672, localhost:5673"
|
7
|
+
Beetle.config.servers = ENV["RABBITMQ_SERVERS"] || "localhost:5672, localhost:5673"
|
8
8
|
# instantiate a client
|
9
9
|
|
10
10
|
client = Beetle::Client.new
|
@@ -24,6 +24,26 @@ Feature: Redis auto failover
|
|
24
24
|
Given a redis server "redis-1" exists as master
|
25
25
|
Then the role of redis server "redis-1" should be "slave"
|
26
26
|
|
27
|
+
Scenario: Successful redis master switch with multiple slaves
|
28
|
+
And a redis server "redis-3" exists as slave of "redis-3"
|
29
|
+
Given a redis configuration server using redis servers "redis-1,redis-2,redis-3" with clients "rc-client-1,rc-client-2" exists
|
30
|
+
And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2,redis-3" exists
|
31
|
+
And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2,redis-3" exists
|
32
|
+
And a beetle handler using the redis-master file from "rc-client-1" exists
|
33
|
+
And redis server "redis-1" is down
|
34
|
+
And the retry timeout for the redis master check is reached
|
35
|
+
Then a system notification for "redis-1" not being available should be sent
|
36
|
+
And the role of redis server "redis-2" should be "master"
|
37
|
+
And the redis master file of the redis configuration server should contain "redis-2"
|
38
|
+
And the redis master of "rc-client-1" should be "redis-2"
|
39
|
+
And the redis master of "rc-client-2" should be "redis-2"
|
40
|
+
And the redis master of the beetle handler should be "redis-2"
|
41
|
+
And a system notification for switching from "redis-1" to "redis-2" should be sent
|
42
|
+
Given a redis server "redis-1" exists as master
|
43
|
+
Then the role of redis server "redis-1" should be "slave"
|
44
|
+
And the redis server "redis-1" is a slave of "redis-2"
|
45
|
+
And the redis server "redis-3" is a slave of "redis-2"
|
46
|
+
|
27
47
|
Scenario: Successful single redis master switch with multiple failover sets
|
28
48
|
Given a redis server "redis-3" exists as master
|
29
49
|
And a redis server "redis-4" exists as slave of "redis-3"
|
@@ -1,8 +1,9 @@
|
|
1
1
|
Given /^consul state has been cleared$/ do
|
2
|
+
consul_host = ENV["CONSUL_HOST"] || "localhost:8500"
|
2
3
|
system "killall beetle beetle_handler >/dev/null 2>/dev/null"
|
3
|
-
system "curl --silent --request PUT http
|
4
|
-
system "curl --silent --request PUT http
|
5
|
-
system "curl --silent --request DELETE http
|
4
|
+
system "curl --silent --request PUT http://#{consul_host}/v1/kv/apps/beetle/config/ >/dev/null"
|
5
|
+
system "curl --silent --request PUT http://#{consul_host}/v1/kv/shared/config/ >/dev/null"
|
6
|
+
system "curl --silent --request DELETE http://#{consul_host}/v1/kv/apps/beetle/state/redis_master_file_content >/dev/null"
|
6
7
|
end
|
7
8
|
|
8
9
|
Given /^a redis server "([^\"]*)" exists as master$/ do |redis_name|
|
@@ -93,7 +94,6 @@ Given /^an old redis master file for "([^\"]*)" with master "([^\"]*)" exists$/
|
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
96
|
-
|
97
97
|
Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redis_name, role|
|
98
98
|
expected_role = false
|
99
99
|
10.times do
|
@@ -103,6 +103,16 @@ Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redi
|
|
103
103
|
assert expected_role, "#{redis_name} is not a #{role}"
|
104
104
|
end
|
105
105
|
|
106
|
+
Then /^the redis server "([^\"]*)" is a slave of "([^\"]*)"$/ do |redis_name, redis_master_name|
|
107
|
+
master = TestDaemons::Redis[redis_master_name].redis
|
108
|
+
slave = TestDaemons::Redis[redis_name].redis
|
109
|
+
3.times do
|
110
|
+
sleep 1
|
111
|
+
break if slave.slave_of?(master.host, master.port)
|
112
|
+
end
|
113
|
+
assert slave.slave_of?(master.host, master.port)
|
114
|
+
end
|
115
|
+
|
106
116
|
Then /^the redis master of "([^\"]*)" (?:in system "([^"]*)" )?should be "([^\"]*)"$/ do |redis_configuration_client_name, system_name, redis_name|
|
107
117
|
system_name ||= "system"
|
108
118
|
master_file = redis_master_file(redis_configuration_client_name)
|
@@ -143,11 +153,21 @@ end
|
|
143
153
|
Then /^the redis master of the beetle handler should be "([^\"]*)"$/ do |redis_name|
|
144
154
|
Beetle.config.servers = "127.0.0.1:5672" # rabbitmq
|
145
155
|
Beetle.config.logger.level = Logger::INFO
|
146
|
-
|
147
|
-
|
148
|
-
|
156
|
+
redis_master = TestDaemons::Redis[redis_name].ip_with_port
|
157
|
+
response = nil
|
158
|
+
expected_response = ['OK', redis_master]
|
159
|
+
3.times do |i|
|
160
|
+
client = Beetle::Client.new.configure :auto_delete => true do |config|
|
161
|
+
config.queue(:echo, :lazy => true, :dead_lettering => true)
|
162
|
+
config.message(:echo)
|
163
|
+
end
|
164
|
+
t1 = Time.now
|
165
|
+
response = client.rpc(:echo, 'echo')
|
166
|
+
t2 = Time.now
|
167
|
+
# puts "OK,#{redis_master} =?= #{response.join(',')} after #{t2-t1}, attempt #{i+1}"
|
168
|
+
break if expected_response == response
|
149
169
|
end
|
150
|
-
|
170
|
+
assert_equal expected_response, response
|
151
171
|
end
|
152
172
|
|
153
173
|
Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
|
@@ -25,9 +25,10 @@ module TestDaemons
|
|
25
25
|
|
26
26
|
def self.daemon_controller
|
27
27
|
clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
|
28
|
+
consul_host = ENV["CONSUL_HOST"] || "localhost:8500"
|
28
29
|
DaemonController.new(
|
29
30
|
:identifier => "Redis configuration test server",
|
30
|
-
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level} --consul http
|
31
|
+
:start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level} --consul http://#{consul_host}",
|
31
32
|
:ping_command => lambda{ answers_text_requests? },
|
32
33
|
:pid_file => pid_file,
|
33
34
|
:log_file => log_file,
|
data/lib/beetle/publisher.rb
CHANGED
@@ -34,7 +34,7 @@ module Beetle
|
|
34
34
|
Bunny::ConnectionError, Bunny::ForcedChannelCloseError, Bunny::ForcedConnectionCloseError,
|
35
35
|
Bunny::MessageError, Bunny::ProtocolError, Bunny::ServerDownError, Bunny::UnsubscribeError,
|
36
36
|
Bunny::AcknowledgementError, Qrack::BufferOverflowError, Qrack::InvalidTypeError,
|
37
|
-
Errno::EHOSTUNREACH, Errno::ECONNRESET, Timeout::Error
|
37
|
+
Errno::EHOSTUNREACH, Errno::ECONNRESET, Errno::ETIMEDOUT, Timeout::Error
|
38
38
|
]
|
39
39
|
end
|
40
40
|
|
@@ -66,6 +66,7 @@ module Beetle
|
|
66
66
|
logger.debug "Beetle: message sent!"
|
67
67
|
published = 1
|
68
68
|
rescue *bunny_exceptions => e
|
69
|
+
logger.warn("Beetle: publishing exception #{e} #{e.backtrace[0..4].join("\n")}")
|
69
70
|
stop!(e)
|
70
71
|
tries -= 1
|
71
72
|
# retry same server on receiving the first exception for it (might have been a normal restart)
|
@@ -98,6 +99,7 @@ module Beetle
|
|
98
99
|
published << @server
|
99
100
|
logger.debug "Beetle: message sent (#{published})!"
|
100
101
|
rescue *bunny_exceptions => e
|
102
|
+
logger.warn("Beetle: publishing exception #{e} #{e.backtrace[0..4].join("\n")}")
|
101
103
|
stop!(e)
|
102
104
|
retry if (tries += 1) == 1
|
103
105
|
mark_server_dead
|
data/lib/beetle/version.rb
CHANGED
@@ -7,7 +7,7 @@ class AMQPGemBehaviorTest < Minitest::Test
|
|
7
7
|
begin
|
8
8
|
exception = nil
|
9
9
|
EM.run do
|
10
|
-
AMQP.start(:
|
10
|
+
AMQP.start(logging: false, host: ENV['RABBITMQ_SERVERS'] || 'localhost') do |connection|
|
11
11
|
EM::Timer.new(1){ connection.close { EM.stop }}
|
12
12
|
channel = AMQP::Channel.new(connection)
|
13
13
|
channel.on_error { puts "woot"}
|
data/test/beetle/base_test.rb
CHANGED
data/test/beetle/client_test.rb
CHANGED
@@ -87,7 +87,7 @@ module Beetle
|
|
87
87
|
private
|
88
88
|
def redis_test_master_file(server_string)
|
89
89
|
tmp_dir = File.expand_path("../../../tmp", __FILE__)
|
90
|
-
Dir.mkdir(tmp_dir) unless File.
|
90
|
+
Dir.mkdir(tmp_dir) unless File.exist?(tmp_dir)
|
91
91
|
path = tmp_dir + "/redis-master-for-unit-tests"
|
92
92
|
File.open(path, "w"){|f| f.puts server_string}
|
93
93
|
path
|
@@ -77,7 +77,9 @@ module Beetle
|
|
77
77
|
|
78
78
|
class PublisherPublishingTest < Minitest::Test
|
79
79
|
def setup
|
80
|
-
@
|
80
|
+
@config = Configuration.new
|
81
|
+
@config.servers = ENV['RABBITMQ_SERVERS'].split(',').first if ENV['RABBITMQ_SERVERS']
|
82
|
+
@client = Client.new(@config)
|
81
83
|
@pub = Publisher.new(@client)
|
82
84
|
@pub.stubs(:bind_queues_for_exchange)
|
83
85
|
@client.register_queue("mama", :exchange => "mama-exchange")
|
@@ -277,6 +279,7 @@ module Beetle
|
|
277
279
|
class PublisherQueueManagementTest < Minitest::Test
|
278
280
|
def setup
|
279
281
|
@config = Configuration.new
|
282
|
+
@config.servers = ENV['RABBITMQ_SERVERS'] if ENV['RABBITMQ_SERVERS']
|
280
283
|
@client = Client.new(@config)
|
281
284
|
@pub = Publisher.new(@client)
|
282
285
|
end
|
@@ -305,7 +308,9 @@ module Beetle
|
|
305
308
|
@client.register_queue('test_queue_2', :exchange => 'test_exchange')
|
306
309
|
@client.register_queue('test_queue_3', :exchange => 'test_exchange_2')
|
307
310
|
queue = mock("queue")
|
308
|
-
|
311
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "test_queue_1"}).once
|
312
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "test_queue_2"}).once
|
313
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange_2", {:key => "test_queue_3"}).once
|
309
314
|
@pub.expects(:declare_queue!).returns(queue).times(3)
|
310
315
|
@pub.send(:bind_queues_for_exchange, 'test_exchange')
|
311
316
|
@pub.send(:bind_queues_for_exchange, 'test_exchange_2')
|
@@ -315,18 +320,21 @@ module Beetle
|
|
315
320
|
@client.register_queue('test_queue_1', :exchange => 'test_exchange')
|
316
321
|
@client.register_queue('test_queue_2', :exchange => 'test_exchange')
|
317
322
|
queue = mock("queue")
|
318
|
-
queue.expects(:bind).twice
|
319
323
|
@pub.expects(:declare_queue!).returns(queue).twice
|
324
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "test_queue_1"}).once
|
325
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "test_queue_2"}).once
|
320
326
|
@pub.send(:bind_queues_for_exchange, 'test_exchange')
|
321
327
|
@pub.send(:bind_queues_for_exchange, 'test_exchange')
|
322
328
|
end
|
323
329
|
|
324
330
|
test "should declare queues only once even with many bindings" do
|
331
|
+
|
325
332
|
@client.register_queue('test_queue', :exchange => 'test_exchange')
|
326
333
|
@client.register_binding('test_queue', :exchange => 'test_exchange', :key => 'sir-message-a-lot')
|
327
334
|
queue = mock("queue")
|
328
|
-
queue.expects(:bind).twice
|
329
335
|
@pub.expects(:declare_queue!).returns(queue).once
|
336
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "test_queue"}).once
|
337
|
+
@pub.expects(:bind_queue!).with(queue, "test_exchange", {:key => "sir-message-a-lot"}).once
|
330
338
|
@pub.send(:bind_queues_for_exchange, 'test_exchange')
|
331
339
|
end
|
332
340
|
|
@@ -549,10 +557,11 @@ module Beetle
|
|
549
557
|
end
|
550
558
|
end
|
551
559
|
|
552
|
-
|
553
560
|
class RPCTest < Minitest::Test
|
554
561
|
def setup
|
555
|
-
@
|
562
|
+
@config = Configuration.new
|
563
|
+
@config.servers = ENV['RABBITMQ_SERVERS'].split(',').first if ENV['RABBITMQ_SERVERS']
|
564
|
+
@client = Client.new(@config)
|
556
565
|
@pub = Publisher.new(@client)
|
557
566
|
@client.register_message(:test, :exchange => :some_exchange)
|
558
567
|
end
|
@@ -52,7 +52,9 @@ module Beetle
|
|
52
52
|
|
53
53
|
class SubscriberPauseAndResumeTest < Minitest::Test
|
54
54
|
def setup
|
55
|
-
@
|
55
|
+
@config = Configuration.new
|
56
|
+
@config.servers = "localhost:5672"
|
57
|
+
@client = Client.new(@config)
|
56
58
|
@sub = @client.send(:subscriber)
|
57
59
|
@sub.servers << "localhost:7777"
|
58
60
|
@server1, @server2 = @sub.servers
|
@@ -342,7 +344,9 @@ module Beetle
|
|
342
344
|
|
343
345
|
class SubscriptionTest < Minitest::Test
|
344
346
|
def setup
|
345
|
-
@
|
347
|
+
@config = Configuration.new
|
348
|
+
@config.servers = "locahost:5672"
|
349
|
+
@client = Client.new(@config)
|
346
350
|
@sub = @client.send(:subscriber)
|
347
351
|
end
|
348
352
|
|
data/test/test_helper.rb
CHANGED
@@ -27,12 +27,14 @@ end
|
|
27
27
|
I18n.enforce_available_locales = false
|
28
28
|
|
29
29
|
Beetle.config.logger = Logger.new(File.dirname(__FILE__) + '/../test.log')
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
Beetle.config.servers = ENV["RABBITMQ_SERVERS"] || "localhost:5672"
|
31
|
+
|
32
|
+
if system('docker -v >/dev/null') && `docker inspect beetle-redis-master -f '{{.State.Status}}'`.chomp == "running"
|
33
|
+
Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6370"
|
34
|
+
Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6370,localhost:6380"
|
33
35
|
else
|
34
|
-
Beetle.config.redis_server = "localhost:6379"
|
35
|
-
Beetle.config.redis_servers = "localhost:6379,localhost:6380"
|
36
|
+
Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6379"
|
37
|
+
Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6379,localhost:6380"
|
36
38
|
end
|
37
39
|
|
38
40
|
def header_with_params(opts = {})
|
@@ -43,7 +45,6 @@ def header_with_params(opts = {})
|
|
43
45
|
header
|
44
46
|
end
|
45
47
|
|
46
|
-
|
47
48
|
def redis_stub(name, opts = {})
|
48
49
|
default_port = opts['port'] || "1234"
|
49
50
|
default_host = opts['host'] || "foo"
|
@@ -51,6 +52,6 @@ def redis_stub(name, opts = {})
|
|
51
52
|
stub(name, opts)
|
52
53
|
end
|
53
54
|
|
54
|
-
if `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
|
55
|
+
if system('docker -v >/dev/null') && `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
|
55
56
|
ENV['MYSQL_PORT'] = '6612'
|
56
57
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stefan Kaes
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date:
|
15
|
+
date: 2022-02-09 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -225,49 +225,77 @@ dependencies:
|
|
225
225
|
- !ruby/object:Gem::Version
|
226
226
|
version: '13.0'
|
227
227
|
- !ruby/object:Gem::Dependency
|
228
|
-
name:
|
228
|
+
name: simplecov
|
229
229
|
requirement: !ruby/object:Gem::Requirement
|
230
230
|
requirements:
|
231
231
|
- - "~>"
|
232
232
|
- !ruby/object:Gem::Version
|
233
|
-
version: '
|
233
|
+
version: '0.15'
|
234
234
|
type: :development
|
235
235
|
prerelease: false
|
236
236
|
version_requirements: !ruby/object:Gem::Requirement
|
237
237
|
requirements:
|
238
238
|
- - "~>"
|
239
239
|
- !ruby/object:Gem::Version
|
240
|
-
version: '
|
240
|
+
version: '0.15'
|
241
241
|
- !ruby/object:Gem::Dependency
|
242
|
-
name:
|
242
|
+
name: webmock
|
243
243
|
requirement: !ruby/object:Gem::Requirement
|
244
244
|
requirements:
|
245
245
|
- - "~>"
|
246
246
|
- !ruby/object:Gem::Version
|
247
|
-
version: '0
|
247
|
+
version: '3.0'
|
248
248
|
type: :development
|
249
249
|
prerelease: false
|
250
250
|
version_requirements: !ruby/object:Gem::Requirement
|
251
251
|
requirements:
|
252
252
|
- - "~>"
|
253
253
|
- !ruby/object:Gem::Version
|
254
|
-
version: '0
|
254
|
+
version: '3.0'
|
255
255
|
- !ruby/object:Gem::Dependency
|
256
|
-
name:
|
256
|
+
name: websocket-eventmachine-client
|
257
257
|
requirement: !ruby/object:Gem::Requirement
|
258
258
|
requirements:
|
259
|
-
- - "
|
259
|
+
- - ">="
|
260
260
|
- !ruby/object:Gem::Version
|
261
|
-
version: '
|
261
|
+
version: '0'
|
262
262
|
type: :development
|
263
263
|
prerelease: false
|
264
264
|
version_requirements: !ruby/object:Gem::Requirement
|
265
265
|
requirements:
|
266
|
-
- - "
|
266
|
+
- - ">="
|
267
267
|
- !ruby/object:Gem::Version
|
268
|
-
version: '
|
268
|
+
version: '0'
|
269
269
|
- !ruby/object:Gem::Dependency
|
270
|
-
name:
|
270
|
+
name: yard
|
271
|
+
requirement: !ruby/object:Gem::Requirement
|
272
|
+
requirements:
|
273
|
+
- - ">="
|
274
|
+
- !ruby/object:Gem::Version
|
275
|
+
version: '0'
|
276
|
+
type: :development
|
277
|
+
prerelease: false
|
278
|
+
version_requirements: !ruby/object:Gem::Requirement
|
279
|
+
requirements:
|
280
|
+
- - ">="
|
281
|
+
- !ruby/object:Gem::Version
|
282
|
+
version: '0'
|
283
|
+
- !ruby/object:Gem::Dependency
|
284
|
+
name: redcarpet
|
285
|
+
requirement: !ruby/object:Gem::Requirement
|
286
|
+
requirements:
|
287
|
+
- - ">="
|
288
|
+
- !ruby/object:Gem::Version
|
289
|
+
version: '0'
|
290
|
+
type: :development
|
291
|
+
prerelease: false
|
292
|
+
version_requirements: !ruby/object:Gem::Requirement
|
293
|
+
requirements:
|
294
|
+
- - ">="
|
295
|
+
- !ruby/object:Gem::Version
|
296
|
+
version: '0'
|
297
|
+
- !ruby/object:Gem::Dependency
|
298
|
+
name: github-markup
|
271
299
|
requirement: !ruby/object:Gem::Requirement
|
272
300
|
requirements:
|
273
301
|
- - ">="
|
@@ -285,19 +313,11 @@ email: opensource@xing.com
|
|
285
313
|
executables: []
|
286
314
|
extensions: []
|
287
315
|
extra_rdoc_files:
|
288
|
-
- RELEASE_NOTES.rdoc
|
289
|
-
- examples/README.rdoc
|
290
|
-
- REDIS_AUTO_FAILOVER.rdoc
|
291
|
-
- README.rdoc
|
292
316
|
- MIT-LICENSE
|
293
317
|
files:
|
294
318
|
- MIT-LICENSE
|
295
|
-
- README.rdoc
|
296
|
-
- REDIS_AUTO_FAILOVER.rdoc
|
297
|
-
- RELEASE_NOTES.rdoc
|
298
319
|
- Rakefile
|
299
320
|
- beetle.gemspec
|
300
|
-
- examples/README.rdoc
|
301
321
|
- examples/attempts.rb
|
302
322
|
- examples/attempts_with_dead_letter_and_exponential_backoff.rb
|
303
323
|
- examples/attempts_with_exponential_backoff.rb
|
@@ -378,24 +398,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
378
398
|
- !ruby/object:Gem::Version
|
379
399
|
version: 1.3.7
|
380
400
|
requirements: []
|
381
|
-
rubygems_version: 3.
|
401
|
+
rubygems_version: 3.3.4
|
382
402
|
signing_key:
|
383
403
|
specification_version: 3
|
384
404
|
summary: High Availability AMQP Messaging with Redundant Queues
|
385
405
|
test_files:
|
386
|
-
- test/beetle_test.rb
|
387
|
-
- test/beetle/client_test.rb
|
388
406
|
- test/beetle/amqp_gem_behavior_test.rb
|
389
|
-
- test/beetle/
|
390
|
-
- test/beetle/queue_properties_test.rb
|
391
|
-
- test/beetle/handler_test.rb
|
407
|
+
- test/beetle/base_test.rb
|
392
408
|
- test/beetle/beetle_test.rb
|
409
|
+
- test/beetle/client_test.rb
|
393
410
|
- test/beetle/configuration_test.rb
|
394
|
-
- test/beetle/
|
411
|
+
- test/beetle/deduplication_store_test.rb
|
412
|
+
- test/beetle/handler_test.rb
|
395
413
|
- test/beetle/message/settings_test.rb
|
396
|
-
- test/beetle/redis_ext_test.rb
|
397
414
|
- test/beetle/message_test.rb
|
398
415
|
- test/beetle/publisher_test.rb
|
416
|
+
- test/beetle/queue_properties_test.rb
|
399
417
|
- test/beetle/r_c_test.rb
|
400
|
-
- test/beetle/
|
418
|
+
- test/beetle/redis_ext_test.rb
|
419
|
+
- test/beetle/subscriber_test.rb
|
420
|
+
- test/beetle_test.rb
|
401
421
|
- test/test_helper.rb
|
data/README.rdoc
DELETED
@@ -1,233 +0,0 @@
|
|
1
|
-
= Beetle
|
2
|
-
|
3
|
-
High Availability AMQP Messaging with Redundant Queues
|
4
|
-
|
5
|
-
== About
|
6
|
-
|
7
|
-
Beetle grew out of a project to improve an existing ActiveMQ based messaging
|
8
|
-
infrastructure. It offers the following features:
|
9
|
-
|
10
|
-
* High Availability (by using multiple message broker instances)
|
11
|
-
* Redundancy (by replicating queues)
|
12
|
-
* Simple client API (by encapsulating the publishing/ deduplication logic)
|
13
|
-
|
14
|
-
More information can be found on the {project website}[http://xing.github.com/beetle].
|
15
|
-
|
16
|
-
== Release notes
|
17
|
-
|
18
|
-
See {RELEASE_NOTES.rdoc}[https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc]
|
19
|
-
|
20
|
-
== Usage
|
21
|
-
|
22
|
-
=== Configuration
|
23
|
-
# configure machines
|
24
|
-
|
25
|
-
Beetle.config do |config|
|
26
|
-
config.servers = "broker1:5672, broker2:5672"
|
27
|
-
config.redis_server = "redis1:6379"
|
28
|
-
end
|
29
|
-
|
30
|
-
# instantiate a beetle client
|
31
|
-
|
32
|
-
b = Beetle::Client.new
|
33
|
-
|
34
|
-
# configure exchanges, queues, bindings, messages and handlers
|
35
|
-
|
36
|
-
b.configure do
|
37
|
-
queue :test
|
38
|
-
message :test
|
39
|
-
handler(:test) { |message| puts message.data }
|
40
|
-
end
|
41
|
-
|
42
|
-
=== Publishing
|
43
|
-
b.publish :test, "I'm a test message"
|
44
|
-
|
45
|
-
=== Subscribing
|
46
|
-
b.listen_queues
|
47
|
-
|
48
|
-
=== Examples
|
49
|
-
|
50
|
-
Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
|
51
|
-
|
52
|
-
The top level Rakefile comes with targets to start several RabbitMQ and redis instances
|
53
|
-
locally. Make sure the corresponding binaries are in your search path. Open four new shell
|
54
|
-
windows and execute the following commands:
|
55
|
-
|
56
|
-
rake rabbit:start1
|
57
|
-
rake rabbit:start2
|
58
|
-
rake redis:start:master
|
59
|
-
rake redis:start:slave
|
60
|
-
|
61
|
-
== Prerequisites
|
62
|
-
|
63
|
-
To set up a redundant messaging system you will need
|
64
|
-
* at least 2 AMQP servers (we use {RabbitMQ}[http://www.rabbitmq.com/])
|
65
|
-
* at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup, see REDIS_AUTO_FAILOVER.rdoc)
|
66
|
-
|
67
|
-
== Test environment
|
68
|
-
|
69
|
-
For testing purposes, you will need a MySQL database with the database
|
70
|
-
`beetle_test` created. This is needed to test special cases in which
|
71
|
-
Beetle handles the connection with ActiveRecord:
|
72
|
-
|
73
|
-
mysql -e 'create database beetle_test;'
|
74
|
-
|
75
|
-
You also need a Redis instance running. The default configuration of Redis will work:
|
76
|
-
|
77
|
-
redis-server
|
78
|
-
|
79
|
-
If you want to run the integration tests you need GO installed and you
|
80
|
-
will need to build the beetle binary. We provide a Makefile for this
|
81
|
-
purpose, so simply running
|
82
|
-
|
83
|
-
make
|
84
|
-
|
85
|
-
should suffice.
|
86
|
-
|
87
|
-
== Gem Dependencies
|
88
|
-
|
89
|
-
At runtime, Beetle will use
|
90
|
-
* {bunny}[http://github.com/ruby-amqp/bunny]
|
91
|
-
* {redis}[http://github.com/redis/redis-rb]
|
92
|
-
* {amqp}[http://github.com/ruby-amqp/amqp]
|
93
|
-
(which is based on {eventmachine}[http://github.com/eventmachine/eventmachine])
|
94
|
-
* {daemons}[http://daemons.rubyforge.org/]
|
95
|
-
* {activesupport}[https://github.com/rails/rails/tree/master/activesupport]
|
96
|
-
|
97
|
-
For development, you'll need
|
98
|
-
* {mocha}[http://github.com/floehopper/mocha]
|
99
|
-
* {cucumber}[http://github.com/aslakhellesoy/cucumber]
|
100
|
-
* {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
|
101
|
-
* {consul}[https://www.consul.io/downloads.html]
|
102
|
-
|
103
|
-
For tests, you'll need
|
104
|
-
* {activerecord}[https://github.com/rails/rails/tree/master/activerecord]
|
105
|
-
* {mysql2}[https://github.com/brianmario/mysql2/]
|
106
|
-
|
107
|
-
Dependencies are managed by bundler.
|
108
|
-
|
109
|
-
== Authors
|
110
|
-
|
111
|
-
{Stefan Kaes}[http://github.com/skaes],
|
112
|
-
{Pascal Friederich}[http://github.com/paukul],
|
113
|
-
{Ali Jelveh}[http://github.com/dudemeister],
|
114
|
-
{Bjoern Rochel}[http://github.com/bjro] and
|
115
|
-
{Sebastian Roebke}[http://github.com/boosty].
|
116
|
-
|
117
|
-
You can find out more about our work on our {dev blog}[http://devblog.xing.com].
|
118
|
-
|
119
|
-
Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
|
120
|
-
|
121
|
-
Released under the MIT license. For full details see MIT-LICENSE included in this
|
122
|
-
distribution.
|
123
|
-
|
124
|
-
== Contributing
|
125
|
-
|
126
|
-
1. Fork it
|
127
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
128
|
-
3. Hack along and test your code.
|
129
|
-
4. Commit your changes (`git commit -am 'Add some feature'`)
|
130
|
-
5. Push to the branch (`git push origin my-new-feature`)
|
131
|
-
6. Create new Pull Request
|
132
|
-
|
133
|
-
Don't increase the gem version in your pull requests. It will be done after merging the request,
|
134
|
-
to allow merging of pull requests in a flexible order.
|
135
|
-
|
136
|
-
== Compiling beetle and running tests
|
137
|
-
|
138
|
-
In order to execute the unit tests, you need Ruby, a running rabbitmq server, a running
|
139
|
-
redis-server, a running mysql server and a runnning consul server.
|
140
|
-
|
141
|
-
In addition, beetle ships with a cucumber feature to test the automatic redis failover as
|
142
|
-
an integration test. For this you need a recent Go installation in order to compile the
|
143
|
-
beetle go binary. Just invoke `make` in the top level directory.
|
144
|
-
|
145
|
-
There are two ways to start the required test dependencies: using `docker-compose` or
|
146
|
-
starting the services manually.
|
147
|
-
|
148
|
-
=== Testing with docker-compose
|
149
|
-
|
150
|
-
Open a separate terminal window and run
|
151
|
-
|
152
|
-
docker-compose pull
|
153
|
-
|
154
|
-
followed by
|
155
|
-
|
156
|
-
docker-compose up
|
157
|
-
|
158
|
-
This will start mysql, two redis servers, two RabbitMQ instances and a single consul
|
159
|
-
development node.
|
160
|
-
|
161
|
-
Note: make sure to wait until all services are properly started.
|
162
|
-
|
163
|
-
|
164
|
-
== Tesing with locally installed services
|
165
|
-
|
166
|
-
The top level Rakefile comes with targets to start several RabbitMQ instances locally.
|
167
|
-
Make sure the corresponding binaries are in your search path. Open three shell windows and
|
168
|
-
execute the following command:
|
169
|
-
|
170
|
-
rake rabbit:start1
|
171
|
-
|
172
|
-
and
|
173
|
-
|
174
|
-
rake redis:start:master
|
175
|
-
|
176
|
-
as well as
|
177
|
-
|
178
|
-
rake consul:start
|
179
|
-
|
180
|
-
Then you can run the cucumber feature by running:
|
181
|
-
|
182
|
-
cucumber
|
183
|
-
|
184
|
-
or
|
185
|
-
|
186
|
-
rake cucumber
|
187
|
-
|
188
|
-
Note: Cucumber will automatically run after the unit tests when you run `rake` without
|
189
|
-
arguments.
|
190
|
-
|
191
|
-
|
192
|
-
== How to release a new gem version
|
193
|
-
|
194
|
-
Update RELEASE_NOTES.rdoc!
|
195
|
-
|
196
|
-
We use {semantic versioning}[http://semver.org/] and create a git tag
|
197
|
-
for each release.
|
198
|
-
|
199
|
-
Edit `lib/beetle/version.rb` and
|
200
|
-
`go/src/github.com/xing/beetle/version.go` to set the new version
|
201
|
-
number (`Major.Minor.Patch`).
|
202
|
-
|
203
|
-
In short (see {semver.org}[http://semver.org] for details):
|
204
|
-
|
205
|
-
* *Major* version MUST be incremented if any backwards incompatible changes
|
206
|
-
are introduced to the public API.
|
207
|
-
* *Minor* version MUST be incremented if new, backwards compatible functionality
|
208
|
-
is introduced to the public API. It MUST be incremented if any public API
|
209
|
-
functionality is marked as deprecated.
|
210
|
-
* *Patch* version MUST be incremented if only backwards compatible bug fixes
|
211
|
-
are introduced.
|
212
|
-
|
213
|
-
Then use `rake release` which will create the git tag and upload the
|
214
|
-
gem to github.com:
|
215
|
-
|
216
|
-
bundle exec rake release
|
217
|
-
|
218
|
-
The generated gem is located in the `pkg/` directory.
|
219
|
-
|
220
|
-
In order to build go binaries and upload the docker container with the
|
221
|
-
beetle GO binary to docker hub, run
|
222
|
-
|
223
|
-
make release
|
224
|
-
|
225
|
-
This will upload the go binaries to https://github.com/xing/beetle/
|
226
|
-
and push the beetle container to
|
227
|
-
https://hub.docker.com/r/xingarchitects/gobeetle/.
|
228
|
-
|
229
|
-
Run
|
230
|
-
|
231
|
-
make tag push TAG=X.X.X
|
232
|
-
|
233
|
-
to tag and push the container with a specific version number.
|
data/REDIS_AUTO_FAILOVER.rdoc
DELETED
@@ -1,116 +0,0 @@
|
|
1
|
-
= Automatic Redis Failover for Beetle
|
2
|
-
|
3
|
-
== Introduction
|
4
|
-
|
5
|
-
Redis is used as the persistence layer in the AMQP message deduplication
|
6
|
-
process. Because it is such a critical piece in our infrastructure, it is
|
7
|
-
essential that a failure of this service is as unlikely as possible. As our
|
8
|
-
AMQP workers are working in a highly distributed manner, all accessing the same
|
9
|
-
Redis server, a automatic failover to another Redis server has to be very
|
10
|
-
defensive and ensure that every worker in the system will switch to the new
|
11
|
-
server at the same time. If the new server would not get accepted from every
|
12
|
-
worker, a switch would not be possible. This ensures that even in the case of a
|
13
|
-
partitioned network it is impossible that two different workers use two
|
14
|
-
different Redis servers for message deduplication.
|
15
|
-
|
16
|
-
== Our goals
|
17
|
-
|
18
|
-
* opt-in, no need to use the redis-failover solution
|
19
|
-
* no single point of failure
|
20
|
-
* automatic switch in case of redis-master failure
|
21
|
-
* switch should not cause inconsistent data on the redis servers
|
22
|
-
* workers should be able to determine the current redis-master without asking
|
23
|
-
another process (as long as the redis servers are working)
|
24
|
-
|
25
|
-
== How it works
|
26
|
-
|
27
|
-
To ensure consistency, a service (the Redis Configuration Server - RCS) is
|
28
|
-
constantly checking the availability and configuration of the currently
|
29
|
-
configured Redis master server. If this service detects that the Redis master
|
30
|
-
is no longer available, it tries to find an alternative server (one of the
|
31
|
-
slaves) which could be promoted to be the new Redis master.
|
32
|
-
|
33
|
-
On every worker server runs another daemon, the Redis Configuration Client
|
34
|
-
(RCC) which listens to messages sent by the RCS.
|
35
|
-
|
36
|
-
If the RCS finds another potential Redis Master, it sends out a message to see
|
37
|
-
if all known RCCs are still available (once again to eliminate the risk of a
|
38
|
-
partitioned network) and if they agree to the master switch.
|
39
|
-
|
40
|
-
If all RCCs have answered to that message, the RCS sends out a message which
|
41
|
-
tells the RCCs to invalidate the current master.
|
42
|
-
|
43
|
-
This happens by deleting the contents of a special file which is used
|
44
|
-
by the workers to store the current Redis master (the content of that file is
|
45
|
-
the hostname:port of the currently active Redis master). By doing that, it is
|
46
|
-
ensured that no operations are done to the old Redis master server anymore, because the
|
47
|
-
AMQP workers check this file's mtime and reads its contents in case that the
|
48
|
-
file changed, before every Redis operation. When the file has been emptied, the
|
49
|
-
RCCs respond to the "invalidate" message of the RCS. When all RCCs have
|
50
|
-
responded, the RCS knows for sure that it is safe to switch the Redis master
|
51
|
-
now. It sends a "reconfigure" message with the new Redis master hostname:port
|
52
|
-
to the RCCs, which then write that value into their redis master file.
|
53
|
-
|
54
|
-
Additionally, the RCS sends reconfigure messages with the current Redis master
|
55
|
-
periodically, to allow new RCCs to pick up the current master. Plus it turns
|
56
|
-
all other redis servers into slaves of the current master.
|
57
|
-
|
58
|
-
=== Prerequisites
|
59
|
-
|
60
|
-
* one redis-configuration-server process ("RCS", on one server), one redis-configuration-client process ("RCC") on every worker server
|
61
|
-
* the RCS knows about all possible RCCs using a list of client ids
|
62
|
-
* the RCS and RCCs exchange messages via a "system queue"
|
63
|
-
|
64
|
-
=== Flow of actions
|
65
|
-
|
66
|
-
* on startup, an RCC can consult its redis master file to determine the current master without the help of the RCS by checking that it's still a master (or wait for the periodic reconfigure message with the current master from the RCS)
|
67
|
-
* when the RCS finds the master to be down, it will retry a couple of times before starting a reconfiguration round
|
68
|
-
* the RCS sends all RCCs a "ping" message to check if every client is there and able to answer
|
69
|
-
* the RCCs acknowledge via a "pong" message if they can confirm the current master to be unavailable
|
70
|
-
* the RCS waits for *all* RCCs to reply via pong
|
71
|
-
* the RCS tells all RCCs to stop using the master by sending an "invalidate" message
|
72
|
-
* the RCCs acknowledge via an "invalidated" message if they can still confirm the current master to be unavailable
|
73
|
-
* the RCS waits for *all* RCCs to acknowledge the invalidation
|
74
|
-
* the RCS promotes the former slave to become the new master (by sending SLAVEOF no one)
|
75
|
-
* the RCS sends a "reconfigure" message containing the new master to every RCC
|
76
|
-
* the RCCs write the new master to their redis master file
|
77
|
-
|
78
|
-
=== Configuration
|
79
|
-
|
80
|
-
See Beetle::Configuration for setting redis configuration server and client options.
|
81
|
-
|
82
|
-
Please note:
|
83
|
-
Beetle::Configuration#redis_server must be a file path (not a redis host:port string) to use the redis failover. The RCS and RCCs store the current redis master in that file, and the handlers read from it.
|
84
|
-
|
85
|
-
== How to use it
|
86
|
-
|
87
|
-
This example uses two worker servers, identified by rcc-1 and rcc-2.
|
88
|
-
|
89
|
-
Please note:
|
90
|
-
All command line options can also be given as a yaml configuration file via the --config-file option.
|
91
|
-
|
92
|
-
=== On one server
|
93
|
-
|
94
|
-
Start the Redis Configuration Server:
|
95
|
-
|
96
|
-
beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
|
97
|
-
|
98
|
-
Get help for server options:
|
99
|
-
|
100
|
-
beetle configuration_server -h
|
101
|
-
|
102
|
-
=== On every worker server
|
103
|
-
|
104
|
-
Start the Redis Configuration Client:
|
105
|
-
|
106
|
-
On first worker server:
|
107
|
-
|
108
|
-
beetle configuration_client --client-id rcc-1
|
109
|
-
|
110
|
-
On second worker server:
|
111
|
-
|
112
|
-
beetle configuration_client --client-id rcc-2
|
113
|
-
|
114
|
-
Get help for client options:
|
115
|
-
|
116
|
-
beetle configuration_client -h
|
data/RELEASE_NOTES.rdoc
DELETED
@@ -1,383 +0,0 @@
|
|
1
|
-
= Release Notes
|
2
|
-
|
3
|
-
== Version 3.5.1
|
4
|
-
* remove blank entries from server list strings
|
5
|
-
* make sure not to subscribe to any server twice
|
6
|
-
|
7
|
-
== Version 3.5.0
|
8
|
-
* expose publisher method to setup queues/policies ahead of use
|
9
|
-
|
10
|
-
== Version 3.4.3
|
11
|
-
* optimize declaration of queues with many bindings
|
12
|
-
|
13
|
-
== Version 3.4.2
|
14
|
-
* Updated amq-protocol gem to version 2.3.2.
|
15
|
-
* Fixed a rare race condition on message handler timeouts.
|
16
|
-
|
17
|
-
== Version 3.4.1
|
18
|
-
* Updated amq-protocol gem to version 2.3.1.
|
19
|
-
|
20
|
-
== Version 3.4.0
|
21
|
-
* Require redis gem version 4.2.1. This version changes the exists to check for the
|
22
|
-
existence of multiple keys, return the number of keys in the list that exist. This
|
23
|
-
requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
|
24
|
-
command.
|
25
|
-
|
26
|
-
== Version 3.3.12
|
27
|
-
* Support queue level declaration of dead letter queue message TTL.
|
28
|
-
|
29
|
-
== Version 3.3.11
|
30
|
-
* Fixed that dead lettering only works correctly with global config option.
|
31
|
-
|
32
|
-
== Version 3.3.10
|
33
|
-
* Support configuring RabbitMQ write timeout.
|
34
|
-
|
35
|
-
== Version 3.3.9
|
36
|
-
* Reduce the number of queue policies created on the servers by allowing
|
37
|
-
the spefication of a default broker policy. You need to install the
|
38
|
-
default policy with a priority of zero to match all queues ever
|
39
|
-
created. This feature is optional.
|
40
|
-
|
41
|
-
== Version 3.3.8
|
42
|
-
* Avoid needless put call when updating queue policies.
|
43
|
-
It seems the additional call to get the current definition of a policy
|
44
|
-
is to be preferred over relying on the idempotency of the PUT call.
|
45
|
-
This helps when adding a new fresh server to a beetle cluster: import
|
46
|
-
the definitions of one of the existing nodes on the fresh node before
|
47
|
-
actually adding it to the list of servers in the client's beetle config..
|
48
|
-
|
49
|
-
== Version 3.3.7
|
50
|
-
* Increased default http api read timeout to handle large server better.
|
51
|
-
|
52
|
-
== Version 3.3.6
|
53
|
-
* Fixed a redis connection leak in gobeetle.
|
54
|
-
|
55
|
-
== Version 3.3.5
|
56
|
-
* Support synchronous queue policy creation.
|
57
|
-
|
58
|
-
== Version 3.3.4
|
59
|
-
* Track publishing policy options metrics.
|
60
|
-
|
61
|
-
== Version 3.3.3
|
62
|
-
* Broken.
|
63
|
-
|
64
|
-
== Version 3.3.2
|
65
|
-
* Changed order of queue policy application to avoid ahead of line blocking in dead letter
|
66
|
-
queues.
|
67
|
-
|
68
|
-
== Version 3.3.1
|
69
|
-
* It seems that there is a certain preference on connection level when
|
70
|
-
selecting the next message to process. We try to protect against
|
71
|
-
such bias by connecting in random order.
|
72
|
-
|
73
|
-
== Version 3.3.0
|
74
|
-
* protect against duplicate handler execution by keeping the status of
|
75
|
-
handler exectution for both redundant and non-redundant messages in
|
76
|
-
the dedup store for a configurable time window. The config option
|
77
|
-
is named redis_status_key_expiry_interval. Note that this will
|
78
|
-
significantly increase the cpu load and memory usage of the Redis
|
79
|
-
server for applications using a large number of non-redundant
|
80
|
-
messages. This feature is turned off by default, but we will
|
81
|
-
activate it with the next major release (4.0.0).
|
82
|
-
|
83
|
-
== Version 3.2.0
|
84
|
-
* added currently processed message to the handler pre-processing step
|
85
|
-
|
86
|
-
== Version 3.1.0
|
87
|
-
* added more debug log statements
|
88
|
-
* added new callbacks on class Beetle::Handler:
|
89
|
-
* pre_process is called before any message processing commences
|
90
|
-
* post_process is called after all processing has been completed
|
91
|
-
these handlers can be used for logging purposes (such as logjam
|
92
|
-
integration)
|
93
|
-
|
94
|
-
== Version 3.0.0
|
95
|
-
|
96
|
-
* provide client method to setup queues and queue policies.
|
97
|
-
Setting up queue policies on demand in publisher and subscriber is OK
|
98
|
-
for a small number of queues, publishers and subscribers. But the HTTP
|
99
|
-
API of RabbitMQ doesn't scale all that well, so that a large number
|
100
|
-
of HTTP calls to set up queue policies can in fact crash a server.
|
101
|
-
* allow queue mode specification and dead lettering specification on a
|
102
|
-
per queue basis.
|
103
|
-
* change policy setup to be asynchronous, on demand in publisher and
|
104
|
-
consumer. Users have to run a processors to listen to messages and
|
105
|
-
call setup_queue_policies! with the parses JSON paylod of the
|
106
|
-
message.
|
107
|
-
* store redis gc stats in redis and display them in the configuration
|
108
|
-
server web UI
|
109
|
-
* config server: store current master server in consul, if consul
|
110
|
-
config has been provided.
|
111
|
-
* config server: server correctly updates configured client ids when
|
112
|
-
they change in consul.
|
113
|
-
* don't create dead letter queues when using the trace functionality
|
114
|
-
* make sure to clean dedup store when ack_count is larger than 2
|
115
|
-
* added dump_expiries command to beetle to dump dediplication store
|
116
|
-
expiry times.
|
117
|
-
* added delete_queue_keys command to beetle to allow deletion of
|
118
|
-
excess dedup store entries for a given queue.
|
119
|
-
|
120
|
-
== Version 2.3.2
|
121
|
-
* config server: fixed a race condition when accessing server state.
|
122
|
-
HTTP requests run in threads separate from the server dispatcher
|
123
|
-
thread and thus can cause race conditions/crashes when accessing
|
124
|
-
server state. Solved this by adding a closure evaluator to the
|
125
|
-
dispatcher.
|
126
|
-
|
127
|
-
== Version 2.3.1
|
128
|
-
* updated amqp and amq-protocol requirements in ruby client.
|
129
|
-
* fixed that a change in consul endpoint was not properly handled by
|
130
|
-
the beetle configuration client.
|
131
|
-
|
132
|
-
== Version 2.3.0
|
133
|
-
* redis failover: support multiple redis failover instances. This has
|
134
|
-
required a change in the redis master file format. The master file
|
135
|
-
content is now either the old format (host:port) for systems using a
|
136
|
-
single redis failover system or a mapping from system names to
|
137
|
-
host:port combinations. For example,
|
138
|
-
"system1/master1:6379\nsystem2/master" specifies to systems with
|
139
|
-
their corresponding redis masters. Beetle client uses the configured
|
140
|
-
system name to tind the master it should use.
|
141
|
-
* support lazy queues: setting :lazy_quques_enabled on the beetle
|
142
|
-
cofiguration will enable queue_mode: "lazy" for all queues of
|
143
|
-
declared on the beetle client.
|
144
|
-
* improved calculation of channel close an connection disconnect
|
145
|
-
timeouts for beetle publisher to avoid warnings in RabbitMQ logs.
|
146
|
-
* use SecureRandom.uuid instead of UUID4R::uuid(4) if UUID4R cannot
|
147
|
-
be loaded.
|
148
|
-
|
149
|
-
== Version 2.2.4
|
150
|
-
* redis failover: prevent starting a new master switch while one is running
|
151
|
-
|
152
|
-
== Version 2.2.3
|
153
|
-
* redis failover: server logs errors when redis oparations fail
|
154
|
-
|
155
|
-
== Version 2.2.2
|
156
|
-
|
157
|
-
* Reset redis configuration server state when master becomes available during
|
158
|
-
pinging or invalidating. Unlike the former ruby implementation, the go code
|
159
|
-
continues checking redis availability during pinging or invalidating. However,
|
160
|
-
the code did not reset state properly, leading the UI to display 'switch in
|
161
|
-
progress' when in fact there wasn't.
|
162
|
-
|
163
|
-
== Version 2.2.1
|
164
|
-
* Subscriber exits with meaningful error log on possible authentication failures.
|
165
|
-
|
166
|
-
== Version 2.2.0
|
167
|
-
|
168
|
-
* Support specifying a whitelist of retriable exceptions when registering a
|
169
|
-
message handler. Exceptions which are not on the list will be regarded as
|
170
|
-
irrecoverable failures.
|
171
|
-
|
172
|
-
== Version 2.1.2
|
173
|
-
|
174
|
-
* Fixed that redis key GC would never complete when a key scheduled
|
175
|
-
for GC was deleted by a consumer before the collector could retrieve
|
176
|
-
its expiry date.
|
177
|
-
* Fixed tha redis key GC would crash on malformed keys.
|
178
|
-
* Added method to collect keys specified in a file.
|
179
|
-
|
180
|
-
|
181
|
-
== Version 2.1.1
|
182
|
-
|
183
|
-
* Support redis failover with less than 100% acknowlegment from failover clients.
|
184
|
-
|
185
|
-
== Version 2.1.0
|
186
|
-
|
187
|
-
* Support exponential backoff when delaying messages using 'max_delay: int' option.
|
188
|
-
|
189
|
-
== Version 2.0.2
|
190
|
-
|
191
|
-
* fixed incorrect computation of responsiveness threshold in
|
192
|
-
configuration server
|
193
|
-
|
194
|
-
== Version 2.0.1
|
195
|
-
|
196
|
-
* fix for beetle command not geting stuck when connecting to
|
197
|
-
configuration server
|
198
|
-
* configuration server displays last seen times in a human readable format
|
199
|
-
|
200
|
-
== Version 2.0.0
|
201
|
-
|
202
|
-
* beetle command has been rewritten in Go
|
203
|
-
* garbage collecting redis keys now uses the redis SCAN command
|
204
|
-
|
205
|
-
== Version 1.0.4
|
206
|
-
|
207
|
-
* use amqp protocol version 0.9 by default for publishers
|
208
|
-
|
209
|
-
== Version 1.0.3
|
210
|
-
|
211
|
-
* fixed that publisher did not allow specifying message properties
|
212
|
-
|
213
|
-
== Version 1.0.2
|
214
|
-
|
215
|
-
* relax hiredis requirements to >= 0.4.5
|
216
|
-
|
217
|
-
== Version 1.0.1
|
218
|
-
|
219
|
-
* don't try to connect on publisher shutdown
|
220
|
-
|
221
|
-
== Version 1.0.0
|
222
|
-
|
223
|
-
* introduced semantic versioning
|
224
|
-
* upgraded gems used for devloping beetle
|
225
|
-
* upgraded amqp gem to version 1.6.0 and amq-protocol to 2.0.1
|
226
|
-
* relaxed requirements on redis and hiredis versions
|
227
|
-
* support setting prefetch count for subscriber
|
228
|
-
|
229
|
-
== Version 0.4.12
|
230
|
-
|
231
|
-
* Don't log warnings when publishing redundantly and only
|
232
|
-
one server has been configured
|
233
|
-
|
234
|
-
== Version 0.4.11
|
235
|
-
|
236
|
-
* Automatically close open publisher sockets at program exit
|
237
|
-
|
238
|
-
== Version 0.4.10
|
239
|
-
|
240
|
-
* Publisher handles nil and symbols as values in headers correctly
|
241
|
-
|
242
|
-
== Version 0.4.9
|
243
|
-
|
244
|
-
* Allow redis_configuration_client to run in the foreground (useful
|
245
|
-
for docker)
|
246
|
-
|
247
|
-
== Version 0.4.8
|
248
|
-
|
249
|
-
* unseen clients need to be an array
|
250
|
-
|
251
|
-
== Version 0.4.7
|
252
|
-
|
253
|
-
* list clients which have never sent a ping in the failover server UI
|
254
|
-
|
255
|
-
== Version 0.4.6
|
256
|
-
|
257
|
-
* Publish activesupport notifications to support performance measurements
|
258
|
-
|
259
|
-
== Version 0.4.5
|
260
|
-
|
261
|
-
* Starting mutliple redis failover clients is now prohibited by
|
262
|
-
default. This behavior can be overriden using
|
263
|
-
"beetle configuration_client start -- --multiple"
|
264
|
-
|
265
|
-
== Version 0.4.4
|
266
|
-
|
267
|
-
* added command to show beetle version: "beetle --version"
|
268
|
-
* configuration server tracks ids of unknown clients
|
269
|
-
* configuration clients now sends heartbeats
|
270
|
-
* configuration server tracks last seen times of clients, based on heartbeat
|
271
|
-
|
272
|
-
== Version 0.4.3
|
273
|
-
|
274
|
-
* fixed a race condition which could lead to duplicate message processing
|
275
|
-
* fixed eventmachine shutdown sequence problem, which led to ACKs
|
276
|
-
occasionally being lost due to writing to a closed socket, which in
|
277
|
-
turn caused messages to be processed twice
|
278
|
-
* stop_listening now always triggers the subscribe shutdown sequence
|
279
|
-
via a eventmachine timer callback, if the eventmachine reactor is running
|
280
|
-
|
281
|
-
== Version 0.4.2
|
282
|
-
|
283
|
-
* Fail hard on missing master file
|
284
|
-
* Set message timestamp header
|
285
|
-
|
286
|
-
== Version 0.4.1
|
287
|
-
|
288
|
-
* Require newer bunny version (0.7.10) to fix publishing of messages larger than frame_max
|
289
|
-
|
290
|
-
== Version 0.4.0
|
291
|
-
|
292
|
-
* Added optional dead lettering feature to mimic RabbitMQ 2.x requeueing behaviour on RabbitMQ 3.x
|
293
|
-
|
294
|
-
== Version 0.3.14
|
295
|
-
|
296
|
-
* switched message id generation to use v4 uuids
|
297
|
-
|
298
|
-
== Version 0.3.0
|
299
|
-
|
300
|
-
* redis master file contents now correctly reflects the state of the running configuration server
|
301
|
-
* allow accelerating master switch via POST to redis configuration server
|
302
|
-
* embedded http server into the redis configuration server (port 8080)
|
303
|
-
* fixed a problem with redis shutdown command
|
304
|
-
* upgraded to redis 2.2.2
|
305
|
-
* upgraded to amqp gem version 0.8 line
|
306
|
-
* use hiredis as the redis backend, which overcomes lack of proper time-outs in the "generic" redis-rb
|
307
|
-
gem for Ruby 1.9
|
308
|
-
* use fully qualified hostnames to identify redis configuration clients
|
309
|
-
|
310
|
-
== Version 0.2.9.8
|
311
|
-
|
312
|
-
* since version 2.0, RabbitMQ supports Basic.reject(:requeue => true). we use it now too,
|
313
|
-
because it enhances performance of message processors. this means of course, you can
|
314
|
-
only use beetle gem versions >= 0.2.9.8 if your rabbitmq brokers are at least version 2.0.
|
315
|
-
* publishing timeout defaults to 0 to avoid extreme message loss in some cases
|
316
|
-
|
317
|
-
|
318
|
-
== Version 0.2.9.7
|
319
|
-
|
320
|
-
* use new bunny_ext gem and allow specification of global publishing timeouts
|
321
|
-
* registering a message now automatically registers the corresponding exchange
|
322
|
-
* don't try to bind queues for an exchange hich has no queue
|
323
|
-
* ruby 1.9.2 compatibility fixes
|
324
|
-
|
325
|
-
== Version 0.2.9
|
326
|
-
|
327
|
-
* Beetle::Client now raises an exception when it fails to publish a message to at least 1 RabbitMQ server
|
328
|
-
* Subscribers are now stopped cleanly to avoid 'closed abruptly' messages in the RabbitMQ server log
|
329
|
-
|
330
|
-
== Version 0.2.6
|
331
|
-
|
332
|
-
* Set dependency on ActiveSupport to 2.3.x since it ain't compatible to version 3.x yet
|
333
|
-
* Publishers catch a wider range (all?) of possible exceptions when publishing messages
|
334
|
-
* Redis Configuration Servers detect and warn when unknown Redis Configuration Clients connect
|
335
|
-
|
336
|
-
== Version 0.2.5
|
337
|
-
|
338
|
-
Added missing files to gem and rdoc
|
339
|
-
|
340
|
-
== Version 0.2.4
|
341
|
-
|
342
|
-
Log and send a system notification when pong message from unknown client received.
|
343
|
-
|
344
|
-
== Version 0.2.2
|
345
|
-
|
346
|
-
Patch release which upgrades to redis-rb 2.0.4. This enables us to drop our redis monkey
|
347
|
-
patch which enabled connection timeouts for earlier redis versions. Note that earlier
|
348
|
-
Beetle versions are not compatible with redis 2.0.4.
|
349
|
-
|
350
|
-
== Version 0.2.1
|
351
|
-
|
352
|
-
Improved error message when no rabbitmq broker is available.
|
353
|
-
|
354
|
-
== Version 0.2
|
355
|
-
|
356
|
-
This version adds support for automatic redis deduplication store failover (see separate
|
357
|
-
file REDIS_AUTO_FAILOVER.rdoc).
|
358
|
-
|
359
|
-
=== User visible changes
|
360
|
-
|
361
|
-
* it's possible to register auto deleted queues and exchanges
|
362
|
-
* Beetle::Client#configure returns self in order to simplify client setup
|
363
|
-
* it's possible to trace specific messages (see Beetle::Client#trace)
|
364
|
-
* default message handler timeout is 10 minutes now
|
365
|
-
* system wide configuration values can be specified via a yml formatted configuration
|
366
|
-
file (Beetle::Configuration#config_file)
|
367
|
-
* the config value redis_server specifies either a single server or a file path (used
|
368
|
-
by the automatic redis failover logic)
|
369
|
-
|
370
|
-
=== Fugs Bixed
|
371
|
-
|
372
|
-
* handle active_support seconds notation for handler timeouts correctly
|
373
|
-
* error handler was erroneously called for expired messages
|
374
|
-
* subscribers would block when some non beetle process posts an undecodable message
|
375
|
-
|
376
|
-
=== Gem Dependency Changes
|
377
|
-
|
378
|
-
* redis needs to be at least version 2.0.3
|
379
|
-
* we make use of the SystemTimer gem for ruby 1.8.7
|
380
|
-
|
381
|
-
== Version 0.1
|
382
|
-
|
383
|
-
Initial Release
|
data/examples/README.rdoc
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
=== Examples
|
2
|
-
|
3
|
-
Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
|
4
|
-
|
5
|
-
The top level Rakefile comes with targets to start several RabbitMQ and redis instances
|
6
|
-
locally. Make sure the corresponding binaries are in your search path. Open four new shell
|
7
|
-
windows and execute the following commands:
|
8
|
-
|
9
|
-
rake rabbit:start1
|
10
|
-
rake rabbit:start2
|
11
|
-
rake redis:start:master
|
12
|
-
rake redis:start:slave
|