beetle 1.0.3 → 2.0.0rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +15 -7
- data/REDIS_AUTO_FAILOVER.rdoc +8 -16
- data/RELEASE_NOTES.rdoc +9 -0
- data/beetle.gemspec +22 -10
- data/features/support/system_notification_logger +29 -12
- data/features/support/test_daemons/redis.rb +1 -1
- data/features/support/test_daemons/redis_configuration_client.rb +3 -3
- data/features/support/test_daemons/redis_configuration_server.rb +2 -2
- data/lib/beetle/deduplication_store.rb +0 -66
- data/lib/beetle/message.rb +1 -1
- data/lib/beetle/publisher.rb +2 -1
- data/lib/beetle/redis_master_file.rb +0 -5
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +0 -48
- data/test/beetle/message_test.rb +1 -31
- data/test/beetle/publisher_test.rb +2 -1
- metadata +165 -26
- data/bin/beetle +0 -9
- data/lib/beetle/commands.rb +0 -35
- data/lib/beetle/commands/configuration_client.rb +0 -98
- data/lib/beetle/commands/configuration_server.rb +0 -98
- data/lib/beetle/commands/garbage_collect_deduplication_store.rb +0 -52
- data/lib/beetle/redis_configuration_client.rb +0 -157
- data/lib/beetle/redis_configuration_http_server.rb +0 -152
- data/lib/beetle/redis_configuration_server.rb +0 -438
- data/lib/beetle/redis_server_info.rb +0 -66
- data/script/docker-run-beetle-tests- +0 -5
- data/test/beetle/redis_configuration_client_test.rb +0 -118
- data/test/beetle/redis_configuration_server_test.rb +0 -381
data/test/beetle/message_test.rb
CHANGED
@@ -127,36 +127,6 @@ module Beetle
|
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
|
-
test "should be able to garbage collect expired keys" do
|
131
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
132
|
-
header = header_with_params({:ttl => 0})
|
133
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
134
|
-
assert !message.key_exists?
|
135
|
-
assert message.key_exists?
|
136
|
-
@store.redis.expects(:del).with(*@store.keys(message.msg_id))
|
137
|
-
@store.garbage_collect_keys(Time.now.to_i+11)
|
138
|
-
end
|
139
|
-
|
140
|
-
test "should be able to garbage collect expired keys using master and slave" do
|
141
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
142
|
-
header = header_with_params({:ttl => 0})
|
143
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
144
|
-
assert !message.key_exists?
|
145
|
-
assert message.key_exists?
|
146
|
-
@store.redis.expects(:del).with(*@store.keys(message.msg_id))
|
147
|
-
@store.garbage_collect_keys_using_master_and_slave(Time.now.to_i+11)
|
148
|
-
end
|
149
|
-
|
150
|
-
test "should not garbage collect not yet expired keys" do
|
151
|
-
Beetle.config.expects(:gc_threshold).returns(10)
|
152
|
-
header = header_with_params({:ttl => 0})
|
153
|
-
message = Message.new("somequeue", header, 'foo', :store => @store)
|
154
|
-
assert !message.key_exists?
|
155
|
-
assert message.key_exists?
|
156
|
-
@store.redis.expects(:del).never
|
157
|
-
@store.garbage_collect_keys(Time.now.to_i)
|
158
|
-
end
|
159
|
-
|
160
130
|
test "successful processing of a non redundant message should delete all keys from the database" do
|
161
131
|
header = header_with_params({})
|
162
132
|
header.expects(:ack)
|
@@ -263,7 +233,7 @@ module Beetle
|
|
263
233
|
header.expects(:ack)
|
264
234
|
message = Message.new("somequeue", header, 'foo', :store => @store)
|
265
235
|
|
266
|
-
|
236
|
+
assert_nil @store.get(message.msg_id, :ack_count)
|
267
237
|
message.process(lambda {|*args|})
|
268
238
|
assert message.redundant?
|
269
239
|
assert_equal "1", @store.get(message.msg_id, :ack_count)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stefan Kaes
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date:
|
15
|
+
date: 2017-08-04 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: uuid4r
|
@@ -113,19 +113,47 @@ dependencies:
|
|
113
113
|
- !ruby/object:Gem::Version
|
114
114
|
version: 2.3.4
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
|
-
name:
|
116
|
+
name: activerecord
|
117
117
|
requirement: !ruby/object:Gem::Requirement
|
118
118
|
requirements:
|
119
|
-
- - "
|
119
|
+
- - "~>"
|
120
120
|
- !ruby/object:Gem::Version
|
121
|
-
version: 0
|
122
|
-
type: :
|
121
|
+
version: '5.0'
|
122
|
+
type: :development
|
123
123
|
prerelease: false
|
124
124
|
version_requirements: !ruby/object:Gem::Requirement
|
125
125
|
requirements:
|
126
|
-
- - "
|
126
|
+
- - "~>"
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '5.0'
|
129
|
+
- !ruby/object:Gem::Dependency
|
130
|
+
name: cucumber
|
131
|
+
requirement: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - "~>"
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: 2.4.0
|
136
|
+
type: :development
|
137
|
+
prerelease: false
|
138
|
+
version_requirements: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - "~>"
|
127
141
|
- !ruby/object:Gem::Version
|
128
|
-
version:
|
142
|
+
version: 2.4.0
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: daemon_controller
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.2.0
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - "~>"
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 1.2.0
|
129
157
|
- !ruby/object:Gem::Dependency
|
130
158
|
name: daemons
|
131
159
|
requirement: !ruby/object:Gem::Requirement
|
@@ -133,17 +161,142 @@ dependencies:
|
|
133
161
|
- - ">="
|
134
162
|
- !ruby/object:Gem::Version
|
135
163
|
version: 1.2.0
|
136
|
-
type: :
|
164
|
+
type: :development
|
137
165
|
prerelease: false
|
138
166
|
version_requirements: !ruby/object:Gem::Requirement
|
139
167
|
requirements:
|
140
168
|
- - ">="
|
141
169
|
- !ruby/object:Gem::Version
|
142
170
|
version: 1.2.0
|
171
|
+
- !ruby/object:Gem::Dependency
|
172
|
+
name: i18n
|
173
|
+
requirement: !ruby/object:Gem::Requirement
|
174
|
+
requirements:
|
175
|
+
- - ">="
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '0'
|
178
|
+
type: :development
|
179
|
+
prerelease: false
|
180
|
+
version_requirements: !ruby/object:Gem::Requirement
|
181
|
+
requirements:
|
182
|
+
- - ">="
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
version: '0'
|
185
|
+
- !ruby/object:Gem::Dependency
|
186
|
+
name: minitest
|
187
|
+
requirement: !ruby/object:Gem::Requirement
|
188
|
+
requirements:
|
189
|
+
- - "~>"
|
190
|
+
- !ruby/object:Gem::Version
|
191
|
+
version: '5.1'
|
192
|
+
type: :development
|
193
|
+
prerelease: false
|
194
|
+
version_requirements: !ruby/object:Gem::Requirement
|
195
|
+
requirements:
|
196
|
+
- - "~>"
|
197
|
+
- !ruby/object:Gem::Version
|
198
|
+
version: '5.1'
|
199
|
+
- !ruby/object:Gem::Dependency
|
200
|
+
name: mocha
|
201
|
+
requirement: !ruby/object:Gem::Requirement
|
202
|
+
requirements:
|
203
|
+
- - "~>"
|
204
|
+
- !ruby/object:Gem::Version
|
205
|
+
version: 1.1.0
|
206
|
+
type: :development
|
207
|
+
prerelease: false
|
208
|
+
version_requirements: !ruby/object:Gem::Requirement
|
209
|
+
requirements:
|
210
|
+
- - "~>"
|
211
|
+
- !ruby/object:Gem::Version
|
212
|
+
version: 1.1.0
|
213
|
+
- !ruby/object:Gem::Dependency
|
214
|
+
name: mysql2
|
215
|
+
requirement: !ruby/object:Gem::Requirement
|
216
|
+
requirements:
|
217
|
+
- - "~>"
|
218
|
+
- !ruby/object:Gem::Version
|
219
|
+
version: 0.4.4
|
220
|
+
type: :development
|
221
|
+
prerelease: false
|
222
|
+
version_requirements: !ruby/object:Gem::Requirement
|
223
|
+
requirements:
|
224
|
+
- - "~>"
|
225
|
+
- !ruby/object:Gem::Version
|
226
|
+
version: 0.4.4
|
227
|
+
- !ruby/object:Gem::Dependency
|
228
|
+
name: rake
|
229
|
+
requirement: !ruby/object:Gem::Requirement
|
230
|
+
requirements:
|
231
|
+
- - "~>"
|
232
|
+
- !ruby/object:Gem::Version
|
233
|
+
version: '11.2'
|
234
|
+
type: :development
|
235
|
+
prerelease: false
|
236
|
+
version_requirements: !ruby/object:Gem::Requirement
|
237
|
+
requirements:
|
238
|
+
- - "~>"
|
239
|
+
- !ruby/object:Gem::Version
|
240
|
+
version: '11.2'
|
241
|
+
- !ruby/object:Gem::Dependency
|
242
|
+
name: rdoc
|
243
|
+
requirement: !ruby/object:Gem::Requirement
|
244
|
+
requirements:
|
245
|
+
- - "~>"
|
246
|
+
- !ruby/object:Gem::Version
|
247
|
+
version: '4.0'
|
248
|
+
type: :development
|
249
|
+
prerelease: false
|
250
|
+
version_requirements: !ruby/object:Gem::Requirement
|
251
|
+
requirements:
|
252
|
+
- - "~>"
|
253
|
+
- !ruby/object:Gem::Version
|
254
|
+
version: '4.0'
|
255
|
+
- !ruby/object:Gem::Dependency
|
256
|
+
name: simplecov
|
257
|
+
requirement: !ruby/object:Gem::Requirement
|
258
|
+
requirements:
|
259
|
+
- - "~>"
|
260
|
+
- !ruby/object:Gem::Version
|
261
|
+
version: 0.12.0
|
262
|
+
type: :development
|
263
|
+
prerelease: false
|
264
|
+
version_requirements: !ruby/object:Gem::Requirement
|
265
|
+
requirements:
|
266
|
+
- - "~>"
|
267
|
+
- !ruby/object:Gem::Version
|
268
|
+
version: 0.12.0
|
269
|
+
- !ruby/object:Gem::Dependency
|
270
|
+
name: webmock
|
271
|
+
requirement: !ruby/object:Gem::Requirement
|
272
|
+
requirements:
|
273
|
+
- - "~>"
|
274
|
+
- !ruby/object:Gem::Version
|
275
|
+
version: 1.21.0
|
276
|
+
type: :development
|
277
|
+
prerelease: false
|
278
|
+
version_requirements: !ruby/object:Gem::Requirement
|
279
|
+
requirements:
|
280
|
+
- - "~>"
|
281
|
+
- !ruby/object:Gem::Version
|
282
|
+
version: 1.21.0
|
283
|
+
- !ruby/object:Gem::Dependency
|
284
|
+
name: websocket-eventmachine-client
|
285
|
+
requirement: !ruby/object:Gem::Requirement
|
286
|
+
requirements:
|
287
|
+
- - ">="
|
288
|
+
- !ruby/object:Gem::Version
|
289
|
+
version: '0'
|
290
|
+
type: :development
|
291
|
+
prerelease: false
|
292
|
+
version_requirements: !ruby/object:Gem::Requirement
|
293
|
+
requirements:
|
294
|
+
- - ">="
|
295
|
+
- !ruby/object:Gem::Version
|
296
|
+
version: '0'
|
143
297
|
description: A highly available, reliable messaging infrastructure
|
144
298
|
email: opensource@xing.com
|
145
|
-
executables:
|
146
|
-
- beetle
|
299
|
+
executables: []
|
147
300
|
extensions: []
|
148
301
|
extra_rdoc_files:
|
149
302
|
- examples/README.rdoc
|
@@ -159,7 +312,6 @@ files:
|
|
159
312
|
- RELEASE_NOTES.rdoc
|
160
313
|
- Rakefile
|
161
314
|
- beetle.gemspec
|
162
|
-
- bin/beetle
|
163
315
|
- examples/README.rdoc
|
164
316
|
- examples/attempts.rb
|
165
317
|
- examples/consume_many_messages_and_shutdown_randomly.rb
|
@@ -187,10 +339,6 @@ files:
|
|
187
339
|
- lib/beetle.rb
|
188
340
|
- lib/beetle/base.rb
|
189
341
|
- lib/beetle/client.rb
|
190
|
-
- lib/beetle/commands.rb
|
191
|
-
- lib/beetle/commands/configuration_client.rb
|
192
|
-
- lib/beetle/commands/configuration_server.rb
|
193
|
-
- lib/beetle/commands/garbage_collect_deduplication_store.rb
|
194
342
|
- lib/beetle/configuration.rb
|
195
343
|
- lib/beetle/dead_lettering.rb
|
196
344
|
- lib/beetle/deduplication_store.rb
|
@@ -199,16 +347,11 @@ files:
|
|
199
347
|
- lib/beetle/message.rb
|
200
348
|
- lib/beetle/publisher.rb
|
201
349
|
- lib/beetle/r_c.rb
|
202
|
-
- lib/beetle/redis_configuration_client.rb
|
203
|
-
- lib/beetle/redis_configuration_http_server.rb
|
204
|
-
- lib/beetle/redis_configuration_server.rb
|
205
350
|
- lib/beetle/redis_ext.rb
|
206
351
|
- lib/beetle/redis_master_file.rb
|
207
|
-
- lib/beetle/redis_server_info.rb
|
208
352
|
- lib/beetle/subscriber.rb
|
209
353
|
- lib/beetle/version.rb
|
210
354
|
- script/console
|
211
|
-
- script/docker-run-beetle-tests-
|
212
355
|
- script/start_rabbit
|
213
356
|
- test/beetle/amqp_gem_behavior_test.rb
|
214
357
|
- test/beetle/base_test.rb
|
@@ -221,8 +364,6 @@ files:
|
|
221
364
|
- test/beetle/message_test.rb
|
222
365
|
- test/beetle/publisher_test.rb
|
223
366
|
- test/beetle/r_c_test.rb
|
224
|
-
- test/beetle/redis_configuration_client_test.rb
|
225
|
-
- test/beetle/redis_configuration_server_test.rb
|
226
367
|
- test/beetle/redis_ext_test.rb
|
227
368
|
- test/beetle/redis_master_file_test.rb
|
228
369
|
- test/beetle/subscriber_test.rb
|
@@ -248,7 +389,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
389
|
version: 1.3.7
|
249
390
|
requirements: []
|
250
391
|
rubyforge_project:
|
251
|
-
rubygems_version: 2.6.
|
392
|
+
rubygems_version: 2.6.12
|
252
393
|
signing_key:
|
253
394
|
specification_version: 3
|
254
395
|
summary: High Availability AMQP Messaging with Redundant Queues
|
@@ -264,8 +405,6 @@ test_files:
|
|
264
405
|
- test/beetle/message_test.rb
|
265
406
|
- test/beetle/publisher_test.rb
|
266
407
|
- test/beetle/r_c_test.rb
|
267
|
-
- test/beetle/redis_configuration_client_test.rb
|
268
|
-
- test/beetle/redis_configuration_server_test.rb
|
269
408
|
- test/beetle/redis_ext_test.rb
|
270
409
|
- test/beetle/redis_master_file_test.rb
|
271
410
|
- test/beetle/subscriber_test.rb
|
data/bin/beetle
DELETED
data/lib/beetle/commands.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'active_support'
|
3
|
-
|
4
|
-
module Beetle
|
5
|
-
module Commands
|
6
|
-
# invokes given command by instantiating an appropriate command class
|
7
|
-
def self.execute(command)
|
8
|
-
if commands.include? command
|
9
|
-
require File.expand_path("../commands/#{command}", __FILE__)
|
10
|
-
"Beetle::Commands::#{command.classify}".constantize.execute
|
11
|
-
else
|
12
|
-
# me no likez no frikin heredocs
|
13
|
-
puts "\nCommand #{command} not known\n" if command
|
14
|
-
puts "Available commands are:"
|
15
|
-
puts
|
16
|
-
commands.each {|c| puts "\t #{c}"}
|
17
|
-
puts
|
18
|
-
exit 1
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
def self.commands
|
24
|
-
commands_dir = File.expand_path('../commands', __FILE__)
|
25
|
-
Dir[commands_dir + '/*.rb'].map {|f| File.basename(f)[0..-4]}
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
if ARGV.first.to_s.sub(/\A--/,'') == "version"
|
31
|
-
require 'beetle/version'
|
32
|
-
puts Beetle::VERSION
|
33
|
-
else
|
34
|
-
Beetle::Commands.execute(ARGV.shift)
|
35
|
-
end
|
@@ -1,98 +0,0 @@
|
|
1
|
-
require 'optparse'
|
2
|
-
require 'daemons'
|
3
|
-
require 'beetle'
|
4
|
-
|
5
|
-
module Beetle
|
6
|
-
module Commands
|
7
|
-
# Command to start a RedisConfigurationClient daemon.
|
8
|
-
#
|
9
|
-
# Usage: beetle configuration_client [options] -- [client options]
|
10
|
-
#
|
11
|
-
# client options:
|
12
|
-
# --redis-master-file FILE Write redis master server string to FILE
|
13
|
-
# --id, --client-id ID Set unique client id (default is minastirith.local)
|
14
|
-
# --amqp-servers LIST AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)
|
15
|
-
# --config-file PATH Path to an external yaml config file
|
16
|
-
# --pid-dir DIR Write pid and log to DIR
|
17
|
-
# --multiple Allow multiple clients started in parallel (for testing only)
|
18
|
-
# -v, --verbose Set log level to DEBUG
|
19
|
-
# -h, --help Show this message
|
20
|
-
#
|
21
|
-
class ConfigurationClient
|
22
|
-
# parses command line options and starts Beetle::RedisConfigurationClient as a daemon
|
23
|
-
def self.execute
|
24
|
-
command, controller_options, app_options = Daemons::Controller.split_argv(ARGV)
|
25
|
-
|
26
|
-
opts = OptionParser.new
|
27
|
-
opts.banner = "Usage: beetle configuration_client #{command} [options] -- [client options]"
|
28
|
-
opts.separator ""
|
29
|
-
opts.separator "client options:"
|
30
|
-
|
31
|
-
opts.on("--redis-master-file FILE", String, "Write redis master server string to FILE") do |val|
|
32
|
-
Beetle.config.redis_server = val
|
33
|
-
end
|
34
|
-
|
35
|
-
client_id = nil
|
36
|
-
opts.on("--id ID", "--client-id ID", String, "Set unique client id (default is #{RedisConfigurationClient.new.id})") do |val|
|
37
|
-
client_id = val
|
38
|
-
end
|
39
|
-
|
40
|
-
opts.on("--amqp-servers LIST", String, "AMQP server list (e.g. 192.168.0.1:5672,192.168.0.2:5672)") do |val|
|
41
|
-
Beetle.config.servers = val
|
42
|
-
end
|
43
|
-
|
44
|
-
opts.on("--config-file PATH", String, "Path to an external yaml config file") do |val|
|
45
|
-
Beetle.config.config_file = val
|
46
|
-
end
|
47
|
-
|
48
|
-
dir_mode = nil
|
49
|
-
dir = nil
|
50
|
-
opts.on("--pid-dir DIR", String, "Write pid and output to DIR") do |val|
|
51
|
-
dir_mode = :normal
|
52
|
-
dir = val
|
53
|
-
end
|
54
|
-
|
55
|
-
multiple = false
|
56
|
-
opts.on("--multiple", "Allow multiple clients") do |val|
|
57
|
-
multiple = true
|
58
|
-
end
|
59
|
-
|
60
|
-
opts.on("-v", "--verbose", "Set log level to DEBUG") do |val|
|
61
|
-
Beetle.config.logger.level = Logger::DEBUG
|
62
|
-
end
|
63
|
-
|
64
|
-
foreground = false
|
65
|
-
opts.on("-f", "--foreground", "Run in foreground") do |val|
|
66
|
-
foreground = true
|
67
|
-
end
|
68
|
-
|
69
|
-
opts.on_tail("-h", "--help", "Show this message") do
|
70
|
-
puts opts
|
71
|
-
exit
|
72
|
-
end
|
73
|
-
|
74
|
-
opts.parse!(app_options)
|
75
|
-
|
76
|
-
if foreground
|
77
|
-
client = Beetle::RedisConfigurationClient.new
|
78
|
-
client.id = client_id if client_id
|
79
|
-
client.start
|
80
|
-
else
|
81
|
-
daemon_options = {
|
82
|
-
:multiple => multiple,
|
83
|
-
:log_output => true,
|
84
|
-
:dir_mode => dir_mode,
|
85
|
-
:dir => dir,
|
86
|
-
:force => true
|
87
|
-
}
|
88
|
-
|
89
|
-
Daemons.run_proc("redis_configuration_client", daemon_options) do
|
90
|
-
client = Beetle::RedisConfigurationClient.new
|
91
|
-
client.id = client_id if client_id
|
92
|
-
client.start
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|