beetle 3.5.0 → 3.5.4

Sign up to get free protection for your applications and to get access to all the features.
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: beetle
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.0
4
+ version: 3.5.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2021-01-22 00:00:00.000000000 Z
15
+ date: 2022-08-30 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bunny
@@ -35,6 +35,9 @@ dependencies:
35
35
  - - ">="
36
36
  - !ruby/object:Gem::Version
37
37
  version: 4.2.1
38
+ - - "<"
39
+ - !ruby/object:Gem::Version
40
+ version: '5.0'
38
41
  type: :runtime
39
42
  prerelease: false
40
43
  version_requirements: !ruby/object:Gem::Requirement
@@ -42,6 +45,9 @@ dependencies:
42
45
  - - ">="
43
46
  - !ruby/object:Gem::Version
44
47
  version: 4.2.1
48
+ - - "<"
49
+ - !ruby/object:Gem::Version
50
+ version: '5.0'
45
51
  - !ruby/object:Gem::Dependency
46
52
  name: hiredis
47
53
  requirement: !ruby/object:Gem::Requirement
@@ -118,14 +124,14 @@ dependencies:
118
124
  requirements:
119
125
  - - "~>"
120
126
  - !ruby/object:Gem::Version
121
- version: 2.4.0
127
+ version: 8.0.0
122
128
  type: :development
123
129
  prerelease: false
124
130
  version_requirements: !ruby/object:Gem::Requirement
125
131
  requirements:
126
132
  - - "~>"
127
133
  - !ruby/object:Gem::Version
128
- version: 2.4.0
134
+ version: 8.0.0
129
135
  - !ruby/object:Gem::Dependency
130
136
  name: daemon_controller
131
137
  requirement: !ruby/object:Gem::Requirement
@@ -188,28 +194,28 @@ dependencies:
188
194
  requirements:
189
195
  - - "~>"
190
196
  - !ruby/object:Gem::Version
191
- version: 1.3.0
197
+ version: '1.14'
192
198
  type: :development
193
199
  prerelease: false
194
200
  version_requirements: !ruby/object:Gem::Requirement
195
201
  requirements:
196
202
  - - "~>"
197
203
  - !ruby/object:Gem::Version
198
- version: 1.3.0
204
+ version: '1.14'
199
205
  - !ruby/object:Gem::Dependency
200
206
  name: mysql2
201
207
  requirement: !ruby/object:Gem::Requirement
202
208
  requirements:
203
209
  - - "~>"
204
210
  - !ruby/object:Gem::Version
205
- version: 0.4.4
211
+ version: '0.5'
206
212
  type: :development
207
213
  prerelease: false
208
214
  version_requirements: !ruby/object:Gem::Requirement
209
215
  requirements:
210
216
  - - "~>"
211
217
  - !ruby/object:Gem::Version
212
- version: 0.4.4
218
+ version: '0.5'
213
219
  - !ruby/object:Gem::Dependency
214
220
  name: rake
215
221
  requirement: !ruby/object:Gem::Requirement
@@ -225,49 +231,91 @@ dependencies:
225
231
  - !ruby/object:Gem::Version
226
232
  version: '13.0'
227
233
  - !ruby/object:Gem::Dependency
228
- name: rdoc
234
+ name: simplecov
229
235
  requirement: !ruby/object:Gem::Requirement
230
236
  requirements:
231
237
  - - "~>"
232
238
  - !ruby/object:Gem::Version
233
- version: '4.0'
239
+ version: '0.15'
234
240
  type: :development
235
241
  prerelease: false
236
242
  version_requirements: !ruby/object:Gem::Requirement
237
243
  requirements:
238
244
  - - "~>"
239
245
  - !ruby/object:Gem::Version
240
- version: '4.0'
246
+ version: '0.15'
241
247
  - !ruby/object:Gem::Dependency
242
- name: simplecov
248
+ name: webmock
243
249
  requirement: !ruby/object:Gem::Requirement
244
250
  requirements:
245
251
  - - "~>"
246
252
  - !ruby/object:Gem::Version
247
- version: '0.15'
253
+ version: '3.0'
248
254
  type: :development
249
255
  prerelease: false
250
256
  version_requirements: !ruby/object:Gem::Requirement
251
257
  requirements:
252
258
  - - "~>"
253
259
  - !ruby/object:Gem::Version
254
- version: '0.15'
260
+ version: '3.0'
255
261
  - !ruby/object:Gem::Dependency
256
- name: webmock
262
+ name: websocket-eventmachine-client
257
263
  requirement: !ruby/object:Gem::Requirement
258
264
  requirements:
259
- - - "~>"
265
+ - - ">="
260
266
  - !ruby/object:Gem::Version
261
- version: '3.0'
267
+ version: '0'
262
268
  type: :development
263
269
  prerelease: false
264
270
  version_requirements: !ruby/object:Gem::Requirement
265
271
  requirements:
266
- - - "~>"
272
+ - - ">="
267
273
  - !ruby/object:Gem::Version
268
- version: '3.0'
274
+ version: '0'
269
275
  - !ruby/object:Gem::Dependency
270
- name: websocket-eventmachine-client
276
+ name: yard
277
+ requirement: !ruby/object:Gem::Requirement
278
+ requirements:
279
+ - - ">="
280
+ - !ruby/object:Gem::Version
281
+ version: '0'
282
+ type: :development
283
+ prerelease: false
284
+ version_requirements: !ruby/object:Gem::Requirement
285
+ requirements:
286
+ - - ">="
287
+ - !ruby/object:Gem::Version
288
+ version: '0'
289
+ - !ruby/object:Gem::Dependency
290
+ name: redcarpet
291
+ requirement: !ruby/object:Gem::Requirement
292
+ requirements:
293
+ - - ">="
294
+ - !ruby/object:Gem::Version
295
+ version: '0'
296
+ type: :development
297
+ prerelease: false
298
+ version_requirements: !ruby/object:Gem::Requirement
299
+ requirements:
300
+ - - ">="
301
+ - !ruby/object:Gem::Version
302
+ version: '0'
303
+ - !ruby/object:Gem::Dependency
304
+ name: github-markup
305
+ requirement: !ruby/object:Gem::Requirement
306
+ requirements:
307
+ - - ">="
308
+ - !ruby/object:Gem::Version
309
+ version: '0'
310
+ type: :development
311
+ prerelease: false
312
+ version_requirements: !ruby/object:Gem::Requirement
313
+ requirements:
314
+ - - ">="
315
+ - !ruby/object:Gem::Version
316
+ version: '0'
317
+ - !ruby/object:Gem::Dependency
318
+ name: byebug
271
319
  requirement: !ruby/object:Gem::Requirement
272
320
  requirements:
273
321
  - - ">="
@@ -285,19 +333,11 @@ email: opensource@xing.com
285
333
  executables: []
286
334
  extensions: []
287
335
  extra_rdoc_files:
288
- - RELEASE_NOTES.rdoc
289
- - examples/README.rdoc
290
- - REDIS_AUTO_FAILOVER.rdoc
291
- - README.rdoc
292
336
  - MIT-LICENSE
293
337
  files:
294
338
  - MIT-LICENSE
295
- - README.rdoc
296
- - REDIS_AUTO_FAILOVER.rdoc
297
- - RELEASE_NOTES.rdoc
298
339
  - Rakefile
299
340
  - beetle.gemspec
300
- - examples/README.rdoc
301
341
  - examples/attempts.rb
302
342
  - examples/attempts_with_dead_letter_and_exponential_backoff.rb
303
343
  - examples/attempts_with_exponential_backoff.rb
@@ -378,24 +418,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
378
418
  - !ruby/object:Gem::Version
379
419
  version: 1.3.7
380
420
  requirements: []
381
- rubygems_version: 3.0.8
421
+ rubygems_version: 3.3.19
382
422
  signing_key:
383
423
  specification_version: 3
384
424
  summary: High Availability AMQP Messaging with Redundant Queues
385
425
  test_files:
386
- - test/beetle_test.rb
387
- - test/beetle/client_test.rb
388
426
  - test/beetle/amqp_gem_behavior_test.rb
389
- - test/beetle/deduplication_store_test.rb
390
- - test/beetle/queue_properties_test.rb
391
- - test/beetle/handler_test.rb
427
+ - test/beetle/base_test.rb
392
428
  - test/beetle/beetle_test.rb
429
+ - test/beetle/client_test.rb
393
430
  - test/beetle/configuration_test.rb
394
- - test/beetle/subscriber_test.rb
431
+ - test/beetle/deduplication_store_test.rb
432
+ - test/beetle/handler_test.rb
395
433
  - test/beetle/message/settings_test.rb
396
- - test/beetle/redis_ext_test.rb
397
434
  - test/beetle/message_test.rb
398
435
  - test/beetle/publisher_test.rb
436
+ - test/beetle/queue_properties_test.rb
399
437
  - test/beetle/r_c_test.rb
400
- - test/beetle/base_test.rb
438
+ - test/beetle/redis_ext_test.rb
439
+ - test/beetle/subscriber_test.rb
440
+ - test/beetle_test.rb
401
441
  - test/test_helper.rb
data/README.rdoc DELETED
@@ -1,233 +0,0 @@
1
- = Beetle
2
-
3
- High Availability AMQP Messaging with Redundant Queues
4
-
5
- == About
6
-
7
- Beetle grew out of a project to improve an existing ActiveMQ based messaging
8
- infrastructure. It offers the following features:
9
-
10
- * High Availability (by using multiple message broker instances)
11
- * Redundancy (by replicating queues)
12
- * Simple client API (by encapsulating the publishing/ deduplication logic)
13
-
14
- More information can be found on the {project website}[http://xing.github.com/beetle].
15
-
16
- == Release notes
17
-
18
- See {RELEASE_NOTES.rdoc}[https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc]
19
-
20
- == Usage
21
-
22
- === Configuration
23
- # configure machines
24
-
25
- Beetle.config do |config|
26
- config.servers = "broker1:5672, broker2:5672"
27
- config.redis_server = "redis1:6379"
28
- end
29
-
30
- # instantiate a beetle client
31
-
32
- b = Beetle::Client.new
33
-
34
- # configure exchanges, queues, bindings, messages and handlers
35
-
36
- b.configure do
37
- queue :test
38
- message :test
39
- handler(:test) { |message| puts message.data }
40
- end
41
-
42
- === Publishing
43
- b.publish :test, "I'm a test message"
44
-
45
- === Subscribing
46
- b.listen_queues
47
-
48
- === Examples
49
-
50
- Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
51
-
52
- The top level Rakefile comes with targets to start several RabbitMQ and redis instances
53
- locally. Make sure the corresponding binaries are in your search path. Open four new shell
54
- windows and execute the following commands:
55
-
56
- rake rabbit:start1
57
- rake rabbit:start2
58
- rake redis:start:master
59
- rake redis:start:slave
60
-
61
- == Prerequisites
62
-
63
- To set up a redundant messaging system you will need
64
- * at least 2 AMQP servers (we use {RabbitMQ}[http://www.rabbitmq.com/])
65
- * at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup, see REDIS_AUTO_FAILOVER.rdoc)
66
-
67
- == Test environment
68
-
69
- For testing purposes, you will need a MySQL database with the database
70
- `beetle_test` created. This is needed to test special cases in which
71
- Beetle handles the connection with ActiveRecord:
72
-
73
- mysql -e 'create database beetle_test;'
74
-
75
- You also need a Redis instance running. The default configuration of Redis will work:
76
-
77
- redis-server
78
-
79
- If you want to run the integration tests you need GO installed and you
80
- will need to build the beetle binary. We provide a Makefile for this
81
- purpose, so simply running
82
-
83
- make
84
-
85
- should suffice.
86
-
87
- == Gem Dependencies
88
-
89
- At runtime, Beetle will use
90
- * {bunny}[http://github.com/ruby-amqp/bunny]
91
- * {redis}[http://github.com/redis/redis-rb]
92
- * {amqp}[http://github.com/ruby-amqp/amqp]
93
- (which is based on {eventmachine}[http://github.com/eventmachine/eventmachine])
94
- * {daemons}[http://daemons.rubyforge.org/]
95
- * {activesupport}[https://github.com/rails/rails/tree/master/activesupport]
96
-
97
- For development, you'll need
98
- * {mocha}[http://github.com/floehopper/mocha]
99
- * {cucumber}[http://github.com/aslakhellesoy/cucumber]
100
- * {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
101
- * {consul}[https://www.consul.io/downloads.html]
102
-
103
- For tests, you'll need
104
- * {activerecord}[https://github.com/rails/rails/tree/master/activerecord]
105
- * {mysql2}[https://github.com/brianmario/mysql2/]
106
-
107
- Dependencies are managed by bundler.
108
-
109
- == Authors
110
-
111
- {Stefan Kaes}[http://github.com/skaes],
112
- {Pascal Friederich}[http://github.com/paukul],
113
- {Ali Jelveh}[http://github.com/dudemeister],
114
- {Bjoern Rochel}[http://github.com/bjro] and
115
- {Sebastian Roebke}[http://github.com/boosty].
116
-
117
- You can find out more about our work on our {dev blog}[http://devblog.xing.com].
118
-
119
- Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
120
-
121
- Released under the MIT license. For full details see MIT-LICENSE included in this
122
- distribution.
123
-
124
- == Contributing
125
-
126
- 1. Fork it
127
- 2. Create your feature branch (`git checkout -b my-new-feature`)
128
- 3. Hack along and test your code.
129
- 4. Commit your changes (`git commit -am 'Add some feature'`)
130
- 5. Push to the branch (`git push origin my-new-feature`)
131
- 6. Create new Pull Request
132
-
133
- Don't increase the gem version in your pull requests. It will be done after merging the request,
134
- to allow merging of pull requests in a flexible order.
135
-
136
- == Compiling beetle and running tests
137
-
138
- In order to execute the unit tests, you need Ruby, a running rabbitmq server, a running
139
- redis-server, a running mysql server and a runnning consul server.
140
-
141
- In addition, beetle ships with a cucumber feature to test the automatic redis failover as
142
- an integration test. For this you need a recent Go installation in order to compile the
143
- beetle go binary. Just invoke `make` in the top level directory.
144
-
145
- There are two ways to start the required test dependencies: using `docker-compose` or
146
- starting the services manually.
147
-
148
- === Testing with docker-compose
149
-
150
- Open a separate terminal window and run
151
-
152
- docker-compose pull
153
-
154
- followed by
155
-
156
- docker-compose up
157
-
158
- This will start mysql, two redis servers, two RabbitMQ instances and a single consul
159
- development node.
160
-
161
- Note: make sure to wait until all services are properly started.
162
-
163
-
164
- == Tesing with locally installed services
165
-
166
- The top level Rakefile comes with targets to start several RabbitMQ instances locally.
167
- Make sure the corresponding binaries are in your search path. Open three shell windows and
168
- execute the following command:
169
-
170
- rake rabbit:start1
171
-
172
- and
173
-
174
- rake redis:start:master
175
-
176
- as well as
177
-
178
- rake consul:start
179
-
180
- Then you can run the cucumber feature by running:
181
-
182
- cucumber
183
-
184
- or
185
-
186
- rake cucumber
187
-
188
- Note: Cucumber will automatically run after the unit tests when you run `rake` without
189
- arguments.
190
-
191
-
192
- == How to release a new gem version
193
-
194
- Update RELEASE_NOTES.rdoc!
195
-
196
- We use {semantic versioning}[http://semver.org/] and create a git tag
197
- for each release.
198
-
199
- Edit `lib/beetle/version.rb` and
200
- `go/src/github.com/xing/beetle/version.go` to set the new version
201
- number (`Major.Minor.Patch`).
202
-
203
- In short (see {semver.org}[http://semver.org] for details):
204
-
205
- * *Major* version MUST be incremented if any backwards incompatible changes
206
- are introduced to the public API.
207
- * *Minor* version MUST be incremented if new, backwards compatible functionality
208
- is introduced to the public API. It MUST be incremented if any public API
209
- functionality is marked as deprecated.
210
- * *Patch* version MUST be incremented if only backwards compatible bug fixes
211
- are introduced.
212
-
213
- Then use `rake release` which will create the git tag and upload the
214
- gem to github.com:
215
-
216
- bundle exec rake release
217
-
218
- The generated gem is located in the `pkg/` directory.
219
-
220
- In order to build go binaries and upload the docker container with the
221
- beetle GO binary to docker hub, run
222
-
223
- make release
224
-
225
- This will upload the go binaries to https://github.com/xing/beetle/
226
- and push the beetle container to
227
- https://hub.docker.com/r/xingarchitects/gobeetle/.
228
-
229
- Run
230
-
231
- make tag push TAG=X.X.X
232
-
233
- to tag and push the container with a specific version number.
@@ -1,116 +0,0 @@
1
- = Automatic Redis Failover for Beetle
2
-
3
- == Introduction
4
-
5
- Redis is used as the persistence layer in the AMQP message deduplication
6
- process. Because it is such a critical piece in our infrastructure, it is
7
- essential that a failure of this service is as unlikely as possible. As our
8
- AMQP workers are working in a highly distributed manner, all accessing the same
9
- Redis server, a automatic failover to another Redis server has to be very
10
- defensive and ensure that every worker in the system will switch to the new
11
- server at the same time. If the new server would not get accepted from every
12
- worker, a switch would not be possible. This ensures that even in the case of a
13
- partitioned network it is impossible that two different workers use two
14
- different Redis servers for message deduplication.
15
-
16
- == Our goals
17
-
18
- * opt-in, no need to use the redis-failover solution
19
- * no single point of failure
20
- * automatic switch in case of redis-master failure
21
- * switch should not cause inconsistent data on the redis servers
22
- * workers should be able to determine the current redis-master without asking
23
- another process (as long as the redis servers are working)
24
-
25
- == How it works
26
-
27
- To ensure consistency, a service (the Redis Configuration Server - RCS) is
28
- constantly checking the availability and configuration of the currently
29
- configured Redis master server. If this service detects that the Redis master
30
- is no longer available, it tries to find an alternative server (one of the
31
- slaves) which could be promoted to be the new Redis master.
32
-
33
- On every worker server runs another daemon, the Redis Configuration Client
34
- (RCC) which listens to messages sent by the RCS.
35
-
36
- If the RCS finds another potential Redis Master, it sends out a message to see
37
- if all known RCCs are still available (once again to eliminate the risk of a
38
- partitioned network) and if they agree to the master switch.
39
-
40
- If all RCCs have answered to that message, the RCS sends out a message which
41
- tells the RCCs to invalidate the current master.
42
-
43
- This happens by deleting the contents of a special file which is used
44
- by the workers to store the current Redis master (the content of that file is
45
- the hostname:port of the currently active Redis master). By doing that, it is
46
- ensured that no operations are done to the old Redis master server anymore, because the
47
- AMQP workers check this file's mtime and reads its contents in case that the
48
- file changed, before every Redis operation. When the file has been emptied, the
49
- RCCs respond to the "invalidate" message of the RCS. When all RCCs have
50
- responded, the RCS knows for sure that it is safe to switch the Redis master
51
- now. It sends a "reconfigure" message with the new Redis master hostname:port
52
- to the RCCs, which then write that value into their redis master file.
53
-
54
- Additionally, the RCS sends reconfigure messages with the current Redis master
55
- periodically, to allow new RCCs to pick up the current master. Plus it turns
56
- all other redis servers into slaves of the current master.
57
-
58
- === Prerequisites
59
-
60
- * one redis-configuration-server process ("RCS", on one server), one redis-configuration-client process ("RCC") on every worker server
61
- * the RCS knows about all possible RCCs using a list of client ids
62
- * the RCS and RCCs exchange messages via a "system queue"
63
-
64
- === Flow of actions
65
-
66
- * on startup, an RCC can consult its redis master file to determine the current master without the help of the RCS by checking that it's still a master (or wait for the periodic reconfigure message with the current master from the RCS)
67
- * when the RCS finds the master to be down, it will retry a couple of times before starting a reconfiguration round
68
- * the RCS sends all RCCs a "ping" message to check if every client is there and able to answer
69
- * the RCCs acknowledge via a "pong" message if they can confirm the current master to be unavailable
70
- * the RCS waits for *all* RCCs to reply via pong
71
- * the RCS tells all RCCs to stop using the master by sending an "invalidate" message
72
- * the RCCs acknowledge via an "invalidated" message if they can still confirm the current master to be unavailable
73
- * the RCS waits for *all* RCCs to acknowledge the invalidation
74
- * the RCS promotes the former slave to become the new master (by sending SLAVEOF no one)
75
- * the RCS sends a "reconfigure" message containing the new master to every RCC
76
- * the RCCs write the new master to their redis master file
77
-
78
- === Configuration
79
-
80
- See Beetle::Configuration for setting redis configuration server and client options.
81
-
82
- Please note:
83
- Beetle::Configuration#redis_server must be a file path (not a redis host:port string) to use the redis failover. The RCS and RCCs store the current redis master in that file, and the handlers read from it.
84
-
85
- == How to use it
86
-
87
- This example uses two worker servers, identified by rcc-1 and rcc-2.
88
-
89
- Please note:
90
- All command line options can also be given as a yaml configuration file via the --config-file option.
91
-
92
- === On one server
93
-
94
- Start the Redis Configuration Server:
95
-
96
- beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
97
-
98
- Get help for server options:
99
-
100
- beetle configuration_server -h
101
-
102
- === On every worker server
103
-
104
- Start the Redis Configuration Client:
105
-
106
- On first worker server:
107
-
108
- beetle configuration_client --client-id rcc-1
109
-
110
- On second worker server:
111
-
112
- beetle configuration_client --client-id rcc-2
113
-
114
- Get help for client options:
115
-
116
- beetle configuration_client -h