beetle 3.4.2 → 3.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/test/test_helper.rb CHANGED
@@ -27,12 +27,14 @@ end
27
27
  I18n.enforce_available_locales = false
28
28
 
29
29
  Beetle.config.logger = Logger.new(File.dirname(__FILE__) + '/../test.log')
30
- if `docker inspect beetle-redis-master -f '{{.State.Status}}'`.chomp == "running"
31
- Beetle.config.redis_server = "localhost:6370"
32
- Beetle.config.redis_servers = "localhost:6370,localhost:6380"
30
+ Beetle.config.servers = ENV["RABBITMQ_SERVERS"] || "localhost:5672"
31
+
32
+ if system('docker -v >/dev/null') && `docker inspect beetle-redis-master -f '{{.State.Status}}'`.chomp == "running"
33
+ Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6370"
34
+ Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6370,localhost:6380"
33
35
  else
34
- Beetle.config.redis_server = "localhost:6379"
35
- Beetle.config.redis_servers = "localhost:6379,localhost:6380"
36
+ Beetle.config.redis_server = ENV["REDIS_SERVER"] || "localhost:6379"
37
+ Beetle.config.redis_servers = ENV["REDIS_SERVERS"] || "localhost:6379,localhost:6380"
36
38
  end
37
39
 
38
40
  def header_with_params(opts = {})
@@ -43,7 +45,6 @@ def header_with_params(opts = {})
43
45
  header
44
46
  end
45
47
 
46
-
47
48
  def redis_stub(name, opts = {})
48
49
  default_port = opts['port'] || "1234"
49
50
  default_host = opts['host'] || "foo"
@@ -51,6 +52,6 @@ def redis_stub(name, opts = {})
51
52
  stub(name, opts)
52
53
  end
53
54
 
54
- if `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
55
+ if system('docker -v >/dev/null') && `docker inspect beetle-mysql -f '{{.State.Status}}'`.chomp == "running"
55
56
  ENV['MYSQL_PORT'] = '6612'
56
57
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: beetle
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.4.2
4
+ version: 3.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2020-12-01 00:00:00.000000000 Z
15
+ date: 2022-02-09 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bunny
@@ -225,49 +225,77 @@ dependencies:
225
225
  - !ruby/object:Gem::Version
226
226
  version: '13.0'
227
227
  - !ruby/object:Gem::Dependency
228
- name: rdoc
228
+ name: simplecov
229
229
  requirement: !ruby/object:Gem::Requirement
230
230
  requirements:
231
231
  - - "~>"
232
232
  - !ruby/object:Gem::Version
233
- version: '4.0'
233
+ version: '0.15'
234
234
  type: :development
235
235
  prerelease: false
236
236
  version_requirements: !ruby/object:Gem::Requirement
237
237
  requirements:
238
238
  - - "~>"
239
239
  - !ruby/object:Gem::Version
240
- version: '4.0'
240
+ version: '0.15'
241
241
  - !ruby/object:Gem::Dependency
242
- name: simplecov
242
+ name: webmock
243
243
  requirement: !ruby/object:Gem::Requirement
244
244
  requirements:
245
245
  - - "~>"
246
246
  - !ruby/object:Gem::Version
247
- version: '0.15'
247
+ version: '3.0'
248
248
  type: :development
249
249
  prerelease: false
250
250
  version_requirements: !ruby/object:Gem::Requirement
251
251
  requirements:
252
252
  - - "~>"
253
253
  - !ruby/object:Gem::Version
254
- version: '0.15'
254
+ version: '3.0'
255
255
  - !ruby/object:Gem::Dependency
256
- name: webmock
256
+ name: websocket-eventmachine-client
257
257
  requirement: !ruby/object:Gem::Requirement
258
258
  requirements:
259
- - - "~>"
259
+ - - ">="
260
260
  - !ruby/object:Gem::Version
261
- version: '3.0'
261
+ version: '0'
262
262
  type: :development
263
263
  prerelease: false
264
264
  version_requirements: !ruby/object:Gem::Requirement
265
265
  requirements:
266
- - - "~>"
266
+ - - ">="
267
267
  - !ruby/object:Gem::Version
268
- version: '3.0'
268
+ version: '0'
269
269
  - !ruby/object:Gem::Dependency
270
- name: websocket-eventmachine-client
270
+ name: yard
271
+ requirement: !ruby/object:Gem::Requirement
272
+ requirements:
273
+ - - ">="
274
+ - !ruby/object:Gem::Version
275
+ version: '0'
276
+ type: :development
277
+ prerelease: false
278
+ version_requirements: !ruby/object:Gem::Requirement
279
+ requirements:
280
+ - - ">="
281
+ - !ruby/object:Gem::Version
282
+ version: '0'
283
+ - !ruby/object:Gem::Dependency
284
+ name: redcarpet
285
+ requirement: !ruby/object:Gem::Requirement
286
+ requirements:
287
+ - - ">="
288
+ - !ruby/object:Gem::Version
289
+ version: '0'
290
+ type: :development
291
+ prerelease: false
292
+ version_requirements: !ruby/object:Gem::Requirement
293
+ requirements:
294
+ - - ">="
295
+ - !ruby/object:Gem::Version
296
+ version: '0'
297
+ - !ruby/object:Gem::Dependency
298
+ name: github-markup
271
299
  requirement: !ruby/object:Gem::Requirement
272
300
  requirements:
273
301
  - - ">="
@@ -285,19 +313,11 @@ email: opensource@xing.com
285
313
  executables: []
286
314
  extensions: []
287
315
  extra_rdoc_files:
288
- - RELEASE_NOTES.rdoc
289
- - examples/README.rdoc
290
- - REDIS_AUTO_FAILOVER.rdoc
291
- - README.rdoc
292
316
  - MIT-LICENSE
293
317
  files:
294
318
  - MIT-LICENSE
295
- - README.rdoc
296
- - REDIS_AUTO_FAILOVER.rdoc
297
- - RELEASE_NOTES.rdoc
298
319
  - Rakefile
299
320
  - beetle.gemspec
300
- - examples/README.rdoc
301
321
  - examples/attempts.rb
302
322
  - examples/attempts_with_dead_letter_and_exponential_backoff.rb
303
323
  - examples/attempts_with_exponential_backoff.rb
@@ -378,24 +398,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
378
398
  - !ruby/object:Gem::Version
379
399
  version: 1.3.7
380
400
  requirements: []
381
- rubygems_version: 3.0.8
401
+ rubygems_version: 3.3.4
382
402
  signing_key:
383
403
  specification_version: 3
384
404
  summary: High Availability AMQP Messaging with Redundant Queues
385
405
  test_files:
386
- - test/beetle_test.rb
387
- - test/beetle/client_test.rb
388
406
  - test/beetle/amqp_gem_behavior_test.rb
389
- - test/beetle/deduplication_store_test.rb
390
- - test/beetle/queue_properties_test.rb
391
- - test/beetle/handler_test.rb
407
+ - test/beetle/base_test.rb
392
408
  - test/beetle/beetle_test.rb
409
+ - test/beetle/client_test.rb
393
410
  - test/beetle/configuration_test.rb
394
- - test/beetle/subscriber_test.rb
411
+ - test/beetle/deduplication_store_test.rb
412
+ - test/beetle/handler_test.rb
395
413
  - test/beetle/message/settings_test.rb
396
- - test/beetle/redis_ext_test.rb
397
414
  - test/beetle/message_test.rb
398
415
  - test/beetle/publisher_test.rb
416
+ - test/beetle/queue_properties_test.rb
399
417
  - test/beetle/r_c_test.rb
400
- - test/beetle/base_test.rb
418
+ - test/beetle/redis_ext_test.rb
419
+ - test/beetle/subscriber_test.rb
420
+ - test/beetle_test.rb
401
421
  - test/test_helper.rb
data/README.rdoc DELETED
@@ -1,233 +0,0 @@
1
- = Beetle
2
-
3
- High Availability AMQP Messaging with Redundant Queues
4
-
5
- == About
6
-
7
- Beetle grew out of a project to improve an existing ActiveMQ based messaging
8
- infrastructure. It offers the following features:
9
-
10
- * High Availability (by using multiple message broker instances)
11
- * Redundancy (by replicating queues)
12
- * Simple client API (by encapsulating the publishing/ deduplication logic)
13
-
14
- More information can be found on the {project website}[http://xing.github.com/beetle].
15
-
16
- == Release notes
17
-
18
- See {RELEASE_NOTES.rdoc}[https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc]
19
-
20
- == Usage
21
-
22
- === Configuration
23
- # configure machines
24
-
25
- Beetle.config do |config|
26
- config.servers = "broker1:5672, broker2:5672"
27
- config.redis_server = "redis1:6379"
28
- end
29
-
30
- # instantiate a beetle client
31
-
32
- b = Beetle::Client.new
33
-
34
- # configure exchanges, queues, bindings, messages and handlers
35
-
36
- b.configure do
37
- queue :test
38
- message :test
39
- handler(:test) { |message| puts message.data }
40
- end
41
-
42
- === Publishing
43
- b.publish :test, "I'm a test message"
44
-
45
- === Subscribing
46
- b.listen_queues
47
-
48
- === Examples
49
-
50
- Beetle ships with a number of {example scripts}[http://github.com/xing/beetle/tree/master/examples/].
51
-
52
- The top level Rakefile comes with targets to start several RabbitMQ and redis instances
53
- locally. Make sure the corresponding binaries are in your search path. Open four new shell
54
- windows and execute the following commands:
55
-
56
- rake rabbit:start1
57
- rake rabbit:start2
58
- rake redis:start:master
59
- rake redis:start:slave
60
-
61
- == Prerequisites
62
-
63
- To set up a redundant messaging system you will need
64
- * at least 2 AMQP servers (we use {RabbitMQ}[http://www.rabbitmq.com/])
65
- * at least one {Redis}[http://github.com/antirez/redis] server (better are two in a master/slave setup, see REDIS_AUTO_FAILOVER.rdoc)
66
-
67
- == Test environment
68
-
69
- For testing purposes, you will need a MySQL database with the database
70
- `beetle_test` created. This is needed to test special cases in which
71
- Beetle handles the connection with ActiveRecord:
72
-
73
- mysql -e 'create database beetle_test;'
74
-
75
- You also need a Redis instance running. The default configuration of Redis will work:
76
-
77
- redis-server
78
-
79
- If you want to run the integration tests you need GO installed and you
80
- will need to build the beetle binary. We provide a Makefile for this
81
- purpose, so simply running
82
-
83
- make
84
-
85
- should suffice.
86
-
87
- == Gem Dependencies
88
-
89
- At runtime, Beetle will use
90
- * {bunny}[http://github.com/ruby-amqp/bunny]
91
- * {redis}[http://github.com/redis/redis-rb]
92
- * {amqp}[http://github.com/ruby-amqp/amqp]
93
- (which is based on {eventmachine}[http://github.com/eventmachine/eventmachine])
94
- * {daemons}[http://daemons.rubyforge.org/]
95
- * {activesupport}[https://github.com/rails/rails/tree/master/activesupport]
96
-
97
- For development, you'll need
98
- * {mocha}[http://github.com/floehopper/mocha]
99
- * {cucumber}[http://github.com/aslakhellesoy/cucumber]
100
- * {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
101
- * {consul}[https://www.consul.io/downloads.html]
102
-
103
- For tests, you'll need
104
- * {activerecord}[https://github.com/rails/rails/tree/master/activerecord]
105
- * {mysql2}[https://github.com/brianmario/mysql2/]
106
-
107
- Dependencies are managed by bundler.
108
-
109
- == Authors
110
-
111
- {Stefan Kaes}[http://github.com/skaes],
112
- {Pascal Friederich}[http://github.com/paukul],
113
- {Ali Jelveh}[http://github.com/dudemeister],
114
- {Bjoern Rochel}[http://github.com/bjro] and
115
- {Sebastian Roebke}[http://github.com/boosty].
116
-
117
- You can find out more about our work on our {dev blog}[http://devblog.xing.com].
118
-
119
- Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
120
-
121
- Released under the MIT license. For full details see MIT-LICENSE included in this
122
- distribution.
123
-
124
- == Contributing
125
-
126
- 1. Fork it
127
- 2. Create your feature branch (`git checkout -b my-new-feature`)
128
- 3. Hack along and test your code.
129
- 4. Commit your changes (`git commit -am 'Add some feature'`)
130
- 5. Push to the branch (`git push origin my-new-feature`)
131
- 6. Create new Pull Request
132
-
133
- Don't increase the gem version in your pull requests. It will be done after merging the request,
134
- to allow merging of pull requests in a flexible order.
135
-
136
- == Compiling beetle and running tests
137
-
138
- In order to execute the unit tests, you need Ruby, a running rabbitmq server, a running
139
- redis-server, a running mysql server and a runnning consul server.
140
-
141
- In addition, beetle ships with a cucumber feature to test the automatic redis failover as
142
- an integration test. For this you need a recent Go installation in order to compile the
143
- beetle go binary. Just invoke `make` in the top level directory.
144
-
145
- There are two ways to start the required test dependencies: using `docker-compose` or
146
- starting the services manually.
147
-
148
- === Testing with docker-compose
149
-
150
- Open a separate terminal window and run
151
-
152
- docker-compose pull
153
-
154
- followed by
155
-
156
- docker-compose up
157
-
158
- This will start mysql, two redis servers, two RabbitMQ instances and a single consul
159
- development node.
160
-
161
- Note: make sure to wait until all services are properly started.
162
-
163
-
164
- == Tesing with locally installed services
165
-
166
- The top level Rakefile comes with targets to start several RabbitMQ instances locally.
167
- Make sure the corresponding binaries are in your search path. Open three shell windows and
168
- execute the following command:
169
-
170
- rake rabbit:start1
171
-
172
- and
173
-
174
- rake redis:start:master
175
-
176
- as well as
177
-
178
- rake consul:start
179
-
180
- Then you can run the cucumber feature by running:
181
-
182
- cucumber
183
-
184
- or
185
-
186
- rake cucumber
187
-
188
- Note: Cucumber will automatically run after the unit tests when you run `rake` without
189
- arguments.
190
-
191
-
192
- == How to release a new gem version
193
-
194
- Update RELEASE_NOTES.rdoc!
195
-
196
- We use {semantic versioning}[http://semver.org/] and create a git tag
197
- for each release.
198
-
199
- Edit `lib/beetle/version.rb` and
200
- `go/src/github.com/xing/beetle/version.go` to set the new version
201
- number (`Major.Minor.Patch`).
202
-
203
- In short (see {semver.org}[http://semver.org] for details):
204
-
205
- * *Major* version MUST be incremented if any backwards incompatible changes
206
- are introduced to the public API.
207
- * *Minor* version MUST be incremented if new, backwards compatible functionality
208
- is introduced to the public API. It MUST be incremented if any public API
209
- functionality is marked as deprecated.
210
- * *Patch* version MUST be incremented if only backwards compatible bug fixes
211
- are introduced.
212
-
213
- Then use `rake release` which will create the git tag and upload the
214
- gem to github.com:
215
-
216
- bundle exec rake release
217
-
218
- The generated gem is located in the `pkg/` directory.
219
-
220
- In order to build go binaries and upload the docker container with the
221
- beetle GO binary to docker hub, run
222
-
223
- make release
224
-
225
- This will upload the go binaries to https://github.com/xing/beetle/
226
- and push the beetle container to
227
- https://hub.docker.com/r/xingarchitects/gobeetle/.
228
-
229
- Run
230
-
231
- make tag push TAG=X.X.X
232
-
233
- to tag and push the container with a specific version number.
@@ -1,116 +0,0 @@
1
- = Automatic Redis Failover for Beetle
2
-
3
- == Introduction
4
-
5
- Redis is used as the persistence layer in the AMQP message deduplication
6
- process. Because it is such a critical piece in our infrastructure, it is
7
- essential that a failure of this service is as unlikely as possible. As our
8
- AMQP workers are working in a highly distributed manner, all accessing the same
9
- Redis server, a automatic failover to another Redis server has to be very
10
- defensive and ensure that every worker in the system will switch to the new
11
- server at the same time. If the new server would not get accepted from every
12
- worker, a switch would not be possible. This ensures that even in the case of a
13
- partitioned network it is impossible that two different workers use two
14
- different Redis servers for message deduplication.
15
-
16
- == Our goals
17
-
18
- * opt-in, no need to use the redis-failover solution
19
- * no single point of failure
20
- * automatic switch in case of redis-master failure
21
- * switch should not cause inconsistent data on the redis servers
22
- * workers should be able to determine the current redis-master without asking
23
- another process (as long as the redis servers are working)
24
-
25
- == How it works
26
-
27
- To ensure consistency, a service (the Redis Configuration Server - RCS) is
28
- constantly checking the availability and configuration of the currently
29
- configured Redis master server. If this service detects that the Redis master
30
- is no longer available, it tries to find an alternative server (one of the
31
- slaves) which could be promoted to be the new Redis master.
32
-
33
- On every worker server runs another daemon, the Redis Configuration Client
34
- (RCC) which listens to messages sent by the RCS.
35
-
36
- If the RCS finds another potential Redis Master, it sends out a message to see
37
- if all known RCCs are still available (once again to eliminate the risk of a
38
- partitioned network) and if they agree to the master switch.
39
-
40
- If all RCCs have answered to that message, the RCS sends out a message which
41
- tells the RCCs to invalidate the current master.
42
-
43
- This happens by deleting the contents of a special file which is used
44
- by the workers to store the current Redis master (the content of that file is
45
- the hostname:port of the currently active Redis master). By doing that, it is
46
- ensured that no operations are done to the old Redis master server anymore, because the
47
- AMQP workers check this file's mtime and reads its contents in case that the
48
- file changed, before every Redis operation. When the file has been emptied, the
49
- RCCs respond to the "invalidate" message of the RCS. When all RCCs have
50
- responded, the RCS knows for sure that it is safe to switch the Redis master
51
- now. It sends a "reconfigure" message with the new Redis master hostname:port
52
- to the RCCs, which then write that value into their redis master file.
53
-
54
- Additionally, the RCS sends reconfigure messages with the current Redis master
55
- periodically, to allow new RCCs to pick up the current master. Plus it turns
56
- all other redis servers into slaves of the current master.
57
-
58
- === Prerequisites
59
-
60
- * one redis-configuration-server process ("RCS", on one server), one redis-configuration-client process ("RCC") on every worker server
61
- * the RCS knows about all possible RCCs using a list of client ids
62
- * the RCS and RCCs exchange messages via a "system queue"
63
-
64
- === Flow of actions
65
-
66
- * on startup, an RCC can consult its redis master file to determine the current master without the help of the RCS by checking that it's still a master (or wait for the periodic reconfigure message with the current master from the RCS)
67
- * when the RCS finds the master to be down, it will retry a couple of times before starting a reconfiguration round
68
- * the RCS sends all RCCs a "ping" message to check if every client is there and able to answer
69
- * the RCCs acknowledge via a "pong" message if they can confirm the current master to be unavailable
70
- * the RCS waits for *all* RCCs to reply via pong
71
- * the RCS tells all RCCs to stop using the master by sending an "invalidate" message
72
- * the RCCs acknowledge via an "invalidated" message if they can still confirm the current master to be unavailable
73
- * the RCS waits for *all* RCCs to acknowledge the invalidation
74
- * the RCS promotes the former slave to become the new master (by sending SLAVEOF no one)
75
- * the RCS sends a "reconfigure" message containing the new master to every RCC
76
- * the RCCs write the new master to their redis master file
77
-
78
- === Configuration
79
-
80
- See Beetle::Configuration for setting redis configuration server and client options.
81
-
82
- Please note:
83
- Beetle::Configuration#redis_server must be a file path (not a redis host:port string) to use the redis failover. The RCS and RCCs store the current redis master in that file, and the handlers read from it.
84
-
85
- == How to use it
86
-
87
- This example uses two worker servers, identified by rcc-1 and rcc-2.
88
-
89
- Please note:
90
- All command line options can also be given as a yaml configuration file via the --config-file option.
91
-
92
- === On one server
93
-
94
- Start the Redis Configuration Server:
95
-
96
- beetle configuration_server --redis-servers redis-1:6379,redis-2:6379 --client-ids rcc-1,rcc-2
97
-
98
- Get help for server options:
99
-
100
- beetle configuration_server -h
101
-
102
- === On every worker server
103
-
104
- Start the Redis Configuration Client:
105
-
106
- On first worker server:
107
-
108
- beetle configuration_client --client-id rcc-1
109
-
110
- On second worker server:
111
-
112
- beetle configuration_client --client-id rcc-2
113
-
114
- Get help for client options:
115
-
116
- beetle configuration_client -h