beetle 3.5.1 → 3.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +9 -13
- data/beetle.gemspec +9 -4
- data/examples/redundant.rb +1 -1
- data/examples/rpc.rb +1 -1
- data/features/redis_auto_failover.feature +20 -0
- data/features/step_definitions/redis_auto_failover_steps.rb +35 -11
- data/features/support/test_daemons/redis_configuration_server.rb +2 -1
- data/lib/beetle/configuration.rb +20 -0
- data/lib/beetle/deduplication_store.rb +4 -4
- data/lib/beetle/message.rb +6 -6
- data/lib/beetle/publisher.rb +3 -1
- data/lib/beetle/redis_ext.rb +14 -0
- data/lib/beetle/version.rb +1 -1
- data/lib/beetle.rb +9 -8
- data/test/beetle/amqp_gem_behavior_test.rb +1 -1
- data/test/beetle/base_test.rb +1 -1
- data/test/beetle/beetle_test.rb +7 -4
- data/test/beetle/client_test.rb +7 -3
- data/test/beetle/deduplication_store_test.rb +1 -1
- data/test/beetle/publisher_test.rb +15 -6
- data/test/beetle/redis_ext_test.rb +5 -1
- data/test/beetle/subscriber_test.rb +8 -4
- data/test/test_helper.rb +16 -9
- metadata +97 -35
- data/README.rdoc +0 -233
- data/REDIS_AUTO_FAILOVER.rdoc +0 -116
- data/RELEASE_NOTES.rdoc +0 -383
- data/examples/README.rdoc +0 -12
data/test/test_helper.rb
CHANGED
@@ -7,8 +7,14 @@ end
|
|
7
7
|
|
8
8
|
require 'minitest/autorun'
|
9
9
|
require 'minitest/unit'
|
10
|
-
require 'minitest
|
11
|
-
|
10
|
+
require 'mocha/minitest'
|
11
|
+
|
12
|
+
require 'minitest/reporters'
|
13
|
+
if ENV['MINITEST_REPORTER']
|
14
|
+
Minitest::Reporters.use!
|
15
|
+
else
|
16
|
+
Minitest::Reporters.use!([Minitest::Reporters::DefaultReporter.new])
|
17
|
+
end
|
12
18
|
|
13
19
|
require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
|
14
20
|
require 'eventmachine'
|
@@ -27,12 +33,14 @@ end
|
|
27
33
|
I18n.enforce_available_locales = false
|
28
34
|
|
29
35
|
Beetle.config.logger = Logger.new(File.dirname(__FILE__) + '/../test.log')
|
30
|
-
|
31
|
-
|
32
|
-
|
36
|
+
Beetle.config.servers = ENV["RABBITMQ_SERVERS"] || "localhost:5672"
|
37
|
+
|
38
|
+
if system('docker -v >/dev/null') && `docker inspect beetle-redis-master -f '{{.State.Status}}'`.chomp == "running"
|
39
|
+
Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6370"
|
40
|
+
Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6370,localhost:6380"
|
33
41
|
else
|
34
|
-
Beetle.config.redis_server = "localhost:6379"
|
35
|
-
Beetle.config.redis_servers = "localhost:6379,localhost:6380"
|
42
|
+
Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6379"
|
43
|
+
Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6379,localhost:6380"
|
36
44
|
end
|
37
45
|
|
38
46
|
def header_with_params(opts = {})
|
@@ -43,7 +51,6 @@ def header_with_params(opts = {})
|
|
43
51
|
header
|
44
52
|
end
|
45
53
|
|
46
|
-
|
47
54
|
def redis_stub(name, opts = {})
|
48
55
|
default_port = opts['port'] || "1234"
|
49
56
|
default_host = opts['host'] || "foo"
|
@@ -51,6 +58,6 @@ def redis_stub(name, opts = {})
|
|
51
58
|
stub(name, opts)
|
52
59
|
end
|
53
60
|
|
54
|
-
if `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
|
61
|
+
if system('docker -v >/dev/null') && `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
|
55
62
|
ENV['MYSQL_PORT'] = '6612'
|
56
63
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stefan Kaes
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date:
|
15
|
+
date: 2022-09-02 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -118,14 +118,14 @@ dependencies:
|
|
118
118
|
requirements:
|
119
119
|
- - "~>"
|
120
120
|
- !ruby/object:Gem::Version
|
121
|
-
version:
|
121
|
+
version: 8.0.0
|
122
122
|
type: :development
|
123
123
|
prerelease: false
|
124
124
|
version_requirements: !ruby/object:Gem::Requirement
|
125
125
|
requirements:
|
126
126
|
- - "~>"
|
127
127
|
- !ruby/object:Gem::Version
|
128
|
-
version:
|
128
|
+
version: 8.0.0
|
129
129
|
- !ruby/object:Gem::Dependency
|
130
130
|
name: daemon_controller
|
131
131
|
requirement: !ruby/object:Gem::Requirement
|
@@ -183,61 +183,61 @@ dependencies:
|
|
183
183
|
- !ruby/object:Gem::Version
|
184
184
|
version: '5.1'
|
185
185
|
- !ruby/object:Gem::Dependency
|
186
|
-
name:
|
186
|
+
name: minitest-reporters
|
187
187
|
requirement: !ruby/object:Gem::Requirement
|
188
188
|
requirements:
|
189
|
-
- - "
|
189
|
+
- - ">="
|
190
190
|
- !ruby/object:Gem::Version
|
191
|
-
version:
|
191
|
+
version: '0'
|
192
192
|
type: :development
|
193
193
|
prerelease: false
|
194
194
|
version_requirements: !ruby/object:Gem::Requirement
|
195
195
|
requirements:
|
196
|
-
- - "
|
196
|
+
- - ">="
|
197
197
|
- !ruby/object:Gem::Version
|
198
|
-
version:
|
198
|
+
version: '0'
|
199
199
|
- !ruby/object:Gem::Dependency
|
200
|
-
name:
|
200
|
+
name: mocha
|
201
201
|
requirement: !ruby/object:Gem::Requirement
|
202
202
|
requirements:
|
203
203
|
- - "~>"
|
204
204
|
- !ruby/object:Gem::Version
|
205
|
-
version:
|
205
|
+
version: '1.14'
|
206
206
|
type: :development
|
207
207
|
prerelease: false
|
208
208
|
version_requirements: !ruby/object:Gem::Requirement
|
209
209
|
requirements:
|
210
210
|
- - "~>"
|
211
211
|
- !ruby/object:Gem::Version
|
212
|
-
version:
|
212
|
+
version: '1.14'
|
213
213
|
- !ruby/object:Gem::Dependency
|
214
|
-
name:
|
214
|
+
name: mysql2
|
215
215
|
requirement: !ruby/object:Gem::Requirement
|
216
216
|
requirements:
|
217
217
|
- - "~>"
|
218
218
|
- !ruby/object:Gem::Version
|
219
|
-
version: '
|
219
|
+
version: '0.5'
|
220
220
|
type: :development
|
221
221
|
prerelease: false
|
222
222
|
version_requirements: !ruby/object:Gem::Requirement
|
223
223
|
requirements:
|
224
224
|
- - "~>"
|
225
225
|
- !ruby/object:Gem::Version
|
226
|
-
version: '
|
226
|
+
version: '0.5'
|
227
227
|
- !ruby/object:Gem::Dependency
|
228
|
-
name:
|
228
|
+
name: rake
|
229
229
|
requirement: !ruby/object:Gem::Requirement
|
230
230
|
requirements:
|
231
231
|
- - "~>"
|
232
232
|
- !ruby/object:Gem::Version
|
233
|
-
version: '
|
233
|
+
version: '13.0'
|
234
234
|
type: :development
|
235
235
|
prerelease: false
|
236
236
|
version_requirements: !ruby/object:Gem::Requirement
|
237
237
|
requirements:
|
238
238
|
- - "~>"
|
239
239
|
- !ruby/object:Gem::Version
|
240
|
-
version: '
|
240
|
+
version: '13.0'
|
241
241
|
- !ruby/object:Gem::Dependency
|
242
242
|
name: simplecov
|
243
243
|
requirement: !ruby/object:Gem::Requirement
|
@@ -280,24 +280,86 @@ dependencies:
|
|
280
280
|
- - ">="
|
281
281
|
- !ruby/object:Gem::Version
|
282
282
|
version: '0'
|
283
|
+
- !ruby/object:Gem::Dependency
|
284
|
+
name: yard
|
285
|
+
requirement: !ruby/object:Gem::Requirement
|
286
|
+
requirements:
|
287
|
+
- - ">="
|
288
|
+
- !ruby/object:Gem::Version
|
289
|
+
version: '0'
|
290
|
+
type: :development
|
291
|
+
prerelease: false
|
292
|
+
version_requirements: !ruby/object:Gem::Requirement
|
293
|
+
requirements:
|
294
|
+
- - ">="
|
295
|
+
- !ruby/object:Gem::Version
|
296
|
+
version: '0'
|
297
|
+
- !ruby/object:Gem::Dependency
|
298
|
+
name: redcarpet
|
299
|
+
requirement: !ruby/object:Gem::Requirement
|
300
|
+
requirements:
|
301
|
+
- - ">="
|
302
|
+
- !ruby/object:Gem::Version
|
303
|
+
version: '0'
|
304
|
+
type: :development
|
305
|
+
prerelease: false
|
306
|
+
version_requirements: !ruby/object:Gem::Requirement
|
307
|
+
requirements:
|
308
|
+
- - ">="
|
309
|
+
- !ruby/object:Gem::Version
|
310
|
+
version: '0'
|
311
|
+
- !ruby/object:Gem::Dependency
|
312
|
+
name: github-markup
|
313
|
+
requirement: !ruby/object:Gem::Requirement
|
314
|
+
requirements:
|
315
|
+
- - ">="
|
316
|
+
- !ruby/object:Gem::Version
|
317
|
+
version: '0'
|
318
|
+
type: :development
|
319
|
+
prerelease: false
|
320
|
+
version_requirements: !ruby/object:Gem::Requirement
|
321
|
+
requirements:
|
322
|
+
- - ">="
|
323
|
+
- !ruby/object:Gem::Version
|
324
|
+
version: '0'
|
325
|
+
- !ruby/object:Gem::Dependency
|
326
|
+
name: byebug
|
327
|
+
requirement: !ruby/object:Gem::Requirement
|
328
|
+
requirements:
|
329
|
+
- - ">="
|
330
|
+
- !ruby/object:Gem::Version
|
331
|
+
version: '0'
|
332
|
+
type: :development
|
333
|
+
prerelease: false
|
334
|
+
version_requirements: !ruby/object:Gem::Requirement
|
335
|
+
requirements:
|
336
|
+
- - ">="
|
337
|
+
- !ruby/object:Gem::Version
|
338
|
+
version: '0'
|
339
|
+
- !ruby/object:Gem::Dependency
|
340
|
+
name: appraisal
|
341
|
+
requirement: !ruby/object:Gem::Requirement
|
342
|
+
requirements:
|
343
|
+
- - ">="
|
344
|
+
- !ruby/object:Gem::Version
|
345
|
+
version: '0'
|
346
|
+
type: :development
|
347
|
+
prerelease: false
|
348
|
+
version_requirements: !ruby/object:Gem::Requirement
|
349
|
+
requirements:
|
350
|
+
- - ">="
|
351
|
+
- !ruby/object:Gem::Version
|
352
|
+
version: '0'
|
283
353
|
description: A highly available, reliable messaging infrastructure
|
284
354
|
email: opensource@xing.com
|
285
355
|
executables: []
|
286
356
|
extensions: []
|
287
357
|
extra_rdoc_files:
|
288
|
-
- RELEASE_NOTES.rdoc
|
289
|
-
- examples/README.rdoc
|
290
|
-
- REDIS_AUTO_FAILOVER.rdoc
|
291
|
-
- README.rdoc
|
292
358
|
- MIT-LICENSE
|
293
359
|
files:
|
294
360
|
- MIT-LICENSE
|
295
|
-
- README.rdoc
|
296
|
-
- REDIS_AUTO_FAILOVER.rdoc
|
297
|
-
- RELEASE_NOTES.rdoc
|
298
361
|
- Rakefile
|
299
362
|
- beetle.gemspec
|
300
|
-
- examples/README.rdoc
|
301
363
|
- examples/attempts.rb
|
302
364
|
- examples/attempts_with_dead_letter_and_exponential_backoff.rb
|
303
365
|
- examples/attempts_with_exponential_backoff.rb
|
@@ -378,24 +440,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
378
440
|
- !ruby/object:Gem::Version
|
379
441
|
version: 1.3.7
|
380
442
|
requirements: []
|
381
|
-
rubygems_version: 3.
|
443
|
+
rubygems_version: 3.3.19
|
382
444
|
signing_key:
|
383
445
|
specification_version: 3
|
384
446
|
summary: High Availability AMQP Messaging with Redundant Queues
|
385
447
|
test_files:
|
386
|
-
- test/beetle_test.rb
|
387
|
-
- test/beetle/client_test.rb
|
388
448
|
- test/beetle/amqp_gem_behavior_test.rb
|
389
|
-
- test/beetle/
|
390
|
-
- test/beetle/queue_properties_test.rb
|
391
|
-
- test/beetle/handler_test.rb
|
449
|
+
- test/beetle/base_test.rb
|
392
450
|
- test/beetle/beetle_test.rb
|
451
|
+
- test/beetle/client_test.rb
|
393
452
|
- test/beetle/configuration_test.rb
|
394
|
-
- test/beetle/
|
453
|
+
- test/beetle/deduplication_store_test.rb
|
454
|
+
- test/beetle/handler_test.rb
|
395
455
|
- test/beetle/message/settings_test.rb
|
396
|
-
- test/beetle/redis_ext_test.rb
|
397
456
|
- test/beetle/message_test.rb
|
398
457
|
- test/beetle/publisher_test.rb
|
458
|
+
- test/beetle/queue_properties_test.rb
|
399
459
|
- test/beetle/r_c_test.rb
|
400
|
-
- test/beetle/
|
460
|
+
- test/beetle/redis_ext_test.rb
|
461
|
+
- test/beetle/subscriber_test.rb
|
462
|
+
- test/beetle_test.rb
|
401
463
|
- test/test_helper.rb
|
data/README.rdoc
DELETED
@@ -1,233 +0,0 @@
|
|
1
|
-
= Beetle
|
2
|
-
|
3
|
-
High Availability AMQP Messaging with Redundant Queues
|
4
|
-
|
5
|
-
== About
|
6
|
-
|
7
|
-
Beetle grew out of a project to improve an existing ActiveMQ based messaging
|
8
|
-
infrastructure. It offers the following features:
|
9
|
-
|
10
|
-
* High Availability (by using multiple message broker instances)
|
11
|
-
* Redundancy (by replicating queues)
|
12
|
-
* Simple client API (by encapsulating the publishing/ deduplication logic)
|
13
|
-
|
14
|
-
More information can be found on the {project website}[http://xing.github.com/beetle].
|
15
|
-
|
16
|
-
== Release notes
|
17
|
-
|
18
|
-
See {RELEASE_NOTES.rdoc}[https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc]
|
19
|
-
|
20
|
-
== Usage
|
21
|
-
|
22
|
-
=== Configuration
|
23
|
-
# configure machines
|
24
|
-
|
25
|
-
Beetle.config do |config|
|
26
|
-
config.servers = "broker1:5672, broker2:5672"
|
27
|
-
config.redis_server = "redis1:6379"
|
28
|
-
end
|
29
|
-
|
30
|
-
# instantiate a beetle client
|
31
|
-
|
32
|
-
b = Beetle::Client.new
|
33
|
-
|
34
|
-
# configure exchanges, queues, bindings, messages and handlers
|
35
|
-
|
36
|
-
b.configure do
|
37
|
-
queue :test
|
38
|
-
message :test
|
39
|
-
handler(:test) { |message| puts message.data }
|
40
|
-
end
|
41
|
-
|
42
|
-
=== Publishing
|
43
|
-
b.publish :test, "I'm a test message"
|
44
|
-
|
45
|
-
=== Subscribing
|
46
|
-
b.listen_queues
|
47
|
-
|
48
|
-
=== Examples
|
49
|
-
|
50
|
-
Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
|
51
|
-
|
52
|
-
The top level Rakefile comes with targets to start several RabbitMQ and redis instances
|
53
|
-
locally. Make sure the corresponding binaries are in your search path. Open four new shell
|
54
|
-
windows and execute the following commands:
|
55
|
-
|
56
|
-
rake rabbit:start1
|
57
|
-
rake rabbit:start2
|
58
|
-
rake redis:start:master
|
59
|
-
rake redis:start:slave
|
60
|
-
|
61
|
-
== Prerequisites
|
62
|
-
|
63
|
-
To set up a redundant messaging system you will need
|
64
|
-
* at least 2 AMQP servers (we use {RabbitMQ}[http://www.rabbitmq.com/])
|
65
|
-
* at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup, see REDIS_AUTO_FAILOVER.rdoc)
|
66
|
-
|
67
|
-
== Test environment
|
68
|
-
|
69
|
-
For testing purposes, you will need a MySQL database with the database
|
70
|
-
`beetle_test` created. This is needed to test special cases in which
|
71
|
-
Beetle handles the connection with ActiveRecord:
|
72
|
-
|
73
|
-
mysql -e 'create database beetle_test;'
|
74
|
-
|
75
|
-
You also need a Redis instance running. The default configuration of Redis will work:
|
76
|
-
|
77
|
-
redis-server
|
78
|
-
|
79
|
-
If you want to run the integration tests you need GO installed and you
|
80
|
-
will need to build the beetle binary. We provide a Makefile for this
|
81
|
-
purpose, so simply running
|
82
|
-
|
83
|
-
make
|
84
|
-
|
85
|
-
should suffice.
|
86
|
-
|
87
|
-
== Gem Dependencies
|
88
|
-
|
89
|
-
At runtime, Beetle will use
|
90
|
-
* {bunny}[http://github.com/ruby-amqp/bunny]
|
91
|
-
* {redis}[http://github.com/redis/redis-rb]
|
92
|
-
* {amqp}[http://github.com/ruby-amqp/amqp]
|
93
|
-
(which is based on {eventmachine}[http://github.com/eventmachine/eventmachine])
|
94
|
-
* {daemons}[http://daemons.rubyforge.org/]
|
95
|
-
* {activesupport}[https://github.com/rails/rails/tree/master/activesupport]
|
96
|
-
|
97
|
-
For development, you'll need
|
98
|
-
* {mocha}[http://github.com/floehopper/mocha]
|
99
|
-
* {cucumber}[http://github.com/aslakhellesoy/cucumber]
|
100
|
-
* {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
|
101
|
-
* {consul}[https://www.consul.io/downloads.html]
|
102
|
-
|
103
|
-
For tests, you'll need
|
104
|
-
* {activerecord}[https://github.com/rails/rails/tree/master/activerecord]
|
105
|
-
* {mysql2}[https://github.com/brianmario/mysql2/]
|
106
|
-
|
107
|
-
Dependencies are managed by bundler.
|
108
|
-
|
109
|
-
== Authors
|
110
|
-
|
111
|
-
{Stefan Kaes}[http://github.com/skaes],
|
112
|
-
{Pascal Friederich}[http://github.com/paukul],
|
113
|
-
{Ali Jelveh}[http://github.com/dudemeister],
|
114
|
-
{Bjoern Rochel}[http://github.com/bjro] and
|
115
|
-
{Sebastian Roebke}[http://github.com/boosty].
|
116
|
-
|
117
|
-
You can find out more about our work on our {dev blog}[http://devblog.xing.com].
|
118
|
-
|
119
|
-
Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
|
120
|
-
|
121
|
-
Released under the MIT license. For full details see MIT-LICENSE included in this
|
122
|
-
distribution.
|
123
|
-
|
124
|
-
== Contributing
|
125
|
-
|
126
|
-
1. Fork it
|
127
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
128
|
-
3. Hack along and test your code.
|
129
|
-
4. Commit your changes (`git commit -am 'Add some feature'`)
|
130
|
-
5. Push to the branch (`git push origin my-new-feature`)
|
131
|
-
6. Create new Pull Request
|
132
|
-
|
133
|
-
Don't increase the gem version in your pull requests. It will be done after merging the request,
|
134
|
-
to allow merging of pull requests in a flexible order.
|
135
|
-
|
136
|
-
== Compiling beetle and running tests
|
137
|
-
|
138
|
-
In order to execute the unit tests, you need Ruby, a running rabbitmq server, a running
|
139
|
-
redis-server, a running mysql server and a runnning consul server.
|
140
|
-
|
141
|
-
In addition, beetle ships with a cucumber feature to test the automatic redis failover as
|
142
|
-
an integration test. For this you need a recent Go installation in order to compile the
|
143
|
-
beetle go binary. Just invoke `make` in the top level directory.
|
144
|
-
|
145
|
-
There are two ways to start the required test dependencies: using `docker-compose` or
|
146
|
-
starting the services manually.
|
147
|
-
|
148
|
-
=== Testing with docker-compose
|
149
|
-
|
150
|
-
Open a separate terminal window and run
|
151
|
-
|
152
|
-
docker-compose pull
|
153
|
-
|
154
|
-
followed by
|
155
|
-
|
156
|
-
docker-compose up
|
157
|
-
|
158
|
-
This will start mysql, two redis servers, two RabbitMQ instances and a single consul
|
159
|
-
development node.
|
160
|
-
|
161
|
-
Note: make sure to wait until all services are properly started.
|
162
|
-
|
163
|
-
|
164
|
-
== Tesing with locally installed services
|
165
|
-
|
166
|
-
The top level Rakefile comes with targets to start several RabbitMQ instances locally.
|
167
|
-
Make sure the corresponding binaries are in your search path. Open three shell windows and
|
168
|
-
execute the following command:
|
169
|
-
|
170
|
-
rake rabbit:start1
|
171
|
-
|
172
|
-
and
|
173
|
-
|
174
|
-
rake redis:start:master
|
175
|
-
|
176
|
-
as well as
|
177
|
-
|
178
|
-
rake consul:start
|
179
|
-
|
180
|
-
Then you can run the cucumber feature by running:
|
181
|
-
|
182
|
-
cucumber
|
183
|
-
|
184
|
-
or
|
185
|
-
|
186
|
-
rake cucumber
|
187
|
-
|
188
|
-
Note: Cucumber will automatically run after the unit tests when you run `rake` without
|
189
|
-
arguments.
|
190
|
-
|
191
|
-
|
192
|
-
== How to release a new gem version
|
193
|
-
|
194
|
-
Update RELEASE_NOTES.rdoc!
|
195
|
-
|
196
|
-
We use {semantic versioning}[http://semver.org/] and create a git tag
|
197
|
-
for each release.
|
198
|
-
|
199
|
-
Edit `lib/beetle/version.rb` and
|
200
|
-
`go/src/github.com/xing/beetle/version.go` to set the new version
|
201
|
-
number (`Major.Minor.Patch`).
|
202
|
-
|
203
|
-
In short (see {semver.org}[http://semver.org] for details):
|
204
|
-
|
205
|
-
* *Major* version MUST be incremented if any backwards incompatible changes
|
206
|
-
are introduced to the public API.
|
207
|
-
* *Minor* version MUST be incremented if new, backwards compatible functionality
|
208
|
-
is introduced to the public API. It MUST be incremented if any public API
|
209
|
-
functionality is marked as deprecated.
|
210
|
-
* *Patch* version MUST be incremented if only backwards compatible bug fixes
|
211
|
-
are introduced.
|
212
|
-
|
213
|
-
Then use `rake release` which will create the git tag and upload the
|
214
|
-
gem to github.com:
|
215
|
-
|
216
|
-
bundle exec rake release
|
217
|
-
|
218
|
-
The generated gem is located in the `pkg/` directory.
|
219
|
-
|
220
|
-
In order to build go binaries and upload the docker container with the
|
221
|
-
beetle GO binary to docker hub, run
|
222
|
-
|
223
|
-
make release
|
224
|
-
|
225
|
-
This will upload the go binaries to https://github.com/xing/beetle/
|
226
|
-
and push the beetle container to
|
227
|
-
https://hub.docker.com/r/xingarchitects/gobeetle/.
|
228
|
-
|
229
|
-
Run
|
230
|
-
|
231
|
-
make tag push TAG=X.X.X
|
232
|
-
|
233
|
-
to tag and push the container with a specific version number.
|
data/REDIS_AUTO_FAILOVER.rdoc
DELETED
@@ -1,116 +0,0 @@
|
|
1
|
-
= Automatic Redis Failover for Beetle
|
2
|
-
|
3
|
-
== Introduction
|
4
|
-
|
5
|
-
Redis is used as the persistence layer in the AMQP message deduplication
|
6
|
-
process. Because it is such a critical piece in our infrastructure, it is
|
7
|
-
essential that a failure of this service is as unlikely as possible. As our
|
8
|
-
AMQP workers are working in a highly distributed manner, all accessing the same
|
9
|
-
Redis server, a automatic failover to another Redis server has to be very
|
10
|
-
defensive and ensure that every worker in the system will switch to the new
|
11
|
-
server at the same time. If the new server would not get accepted from every
|
12
|
-
worker, a switch would not be possible. This ensures that even in the case of a
|
13
|
-
partitioned network it is impossible that two different workers use two
|
14
|
-
different Redis servers for message deduplication.
|
15
|
-
|
16
|
-
== Our goals
|
17
|
-
|
18
|
-
* opt-in, no need to use the redis-failover solution
|
19
|
-
* no single point of failure
|
20
|
-
* automatic switch in case of redis-master failure
|
21
|
-
* switch should not cause inconsistent data on the redis servers
|
22
|
-
* workers should be able to determine the current redis-master without asking
|
23
|
-
another process (as long as the redis servers are working)
|
24
|
-
|
25
|
-
== How it works
|
26
|
-
|
27
|
-
To ensure consistency, a service (the Redis Configuration Server - RCS) is
|
28
|
-
constantly checking the availability and configuration of the currently
|
29
|
-
configured Redis master server. If this service detects that the Redis master
|
30
|
-
is no longer available, it tries to find an alternative server (one of the
|
31
|
-
slaves) which could be promoted to be the new Redis master.
|
32
|
-
|
33
|
-
On every worker server runs another daemon, the Redis Configuration Client
|
34
|
-
(RCC) which listens to messages sent by the RCS.
|
35
|
-
|
36
|
-
If the RCS finds another potential Redis Master, it sends out a message to see
|
37
|
-
if all known RCCs are still available (once again to eliminate the risk of a
|
38
|
-
partitioned network) and if they agree to the master switch.
|
39
|
-
|
40
|
-
If all RCCs have answered to that message, the RCS sends out a message which
|
41
|
-
tells the RCCs to invalidate the current master.
|
42
|
-
|
43
|
-
This happens by deleting the contents of a special file which is used
|
44
|
-
by the workers to store the current Redis master (the content of that file is
|
45
|
-
the hostname:port of the currently active Redis master). By doing that, it is
|
46
|
-
ensured that no operations are done to the old Redis master server anymore, because the
|
47
|
-
AMQP workers check this file's mtime and reads its contents in case that the
|
48
|
-
file changed, before every Redis operation. When the file has been emptied, the
|
49
|
-
RCCs respond to the "invalidate" message of the RCS. When all RCCs have
|
50
|
-
responded, the RCS knows for sure that it is safe to switch the Redis master
|
51
|
-
now. It sends a "reconfigure" message with the new Redis master hostname:port
|
52
|
-
to the RCCs, which then write that value into their redis master file.
|
53
|
-
|
54
|
-
Additionally, the RCS sends reconfigure messages with the current Redis master
|
55
|
-
periodically, to allow new RCCs to pick up the current master. Plus it turns
|
56
|
-
all other redis servers into slaves of the current master.
|
57
|
-
|
58
|
-
=== Prerequisites
|
59
|
-
|
60
|
-
* one redis-configuration-server process ("RCS", on one server), one redis-configuration-client process ("RCC") on every worker server
|
61
|
-
* the RCS knows about all possible RCCs using a list of client ids
|
62
|
-
* the RCS and RCCs exchange messages via a "system queue"
|
63
|
-
|
64
|
-
=== Flow of actions
|
65
|
-
|
66
|
-
* on startup, an RCC can consult its redis master file to determine the current master without the help of the RCS by checking that it's still a master (or wait for the periodic reconfigure message with the current master from the RCS)
|
67
|
-
* when the RCS finds the master to be down, it will retry a couple of times before starting a reconfiguration round
|
68
|
-
* the RCS sends all RCCs a "ping" message to check if every client is there and able to answer
|
69
|
-
* the RCCs acknowledge via a "pong" message if they can confirm the current master to be unavailable
|
70
|
-
* the RCS waits for *all* RCCs to reply via pong
|
71
|
-
* the RCS tells all RCCs to stop using the master by sending an "invalidate" message
|
72
|
-
* the RCCs acknowledge via an "invalidated" message if they can still confirm the current master to be unavailable
|
73
|
-
* the RCS waits for *all* RCCs to acknowledge the invalidation
|
74
|
-
* the RCS promotes the former slave to become the new master (by sending SLAVEOF no one)
|
75
|
-
* the RCS sends a "reconfigure" message containing the new master to every RCC
|
76
|
-
* the RCCs write the new master to their redis master file
|
77
|
-
|
78
|
-
=== Configuration
|
79
|
-
|
80
|
-
See Beetle::Configuration for setting redis configuration server and client options.
|
81
|
-
|
82
|
-
Please note:
|
83
|
-
Beetle::Configuration#redis_server must be a file path (not a redis host:port string) to use the redis failover. The RCS and RCCs store the current redis master in that file, and the handlers read from it.
|
84
|
-
|
85
|
-
== How to use it
|
86
|
-
|
87
|
-
This example uses two worker servers, identified by rcc-1 and rcc-2.
|
88
|
-
|
89
|
-
Please note:
|
90
|
-
All command line options can also be given as a yaml configuration file via the --config-file option.
|
91
|
-
|
92
|
-
=== On one server
|
93
|
-
|
94
|
-
Start the Redis Configuration Server:
|
95
|
-
|
96
|
-
beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
|
97
|
-
|
98
|
-
Get help for server options:
|
99
|
-
|
100
|
-
beetle configuration_server -h
|
101
|
-
|
102
|
-
=== On every worker server
|
103
|
-
|
104
|
-
Start the Redis Configuration Client:
|
105
|
-
|
106
|
-
On first worker server:
|
107
|
-
|
108
|
-
beetle configuration_client --client-id rcc-1
|
109
|
-
|
110
|
-
On second worker server:
|
111
|
-
|
112
|
-
beetle configuration_client --client-id rcc-2
|
113
|
-
|
114
|
-
Get help for client options:
|
115
|
-
|
116
|
-
beetle configuration_client -h
|