beetle 0.3.0.rc.18 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/beetle.gemspec +4 -4
- data/features/support/test_daemons/redis_configuration_server.rb +4 -2
- data/lib/beetle/commands/configuration_server.rb +2 -1
- data/lib/beetle/redis_ext.rb +32 -0
- data/lib/beetle/subscriber.rb +20 -3
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/redis_ext_test.rb +22 -4
- data/test/beetle/subscriber_test.rb +22 -1
- data/test/test_helper.rb +2 -1
- metadata +54 -48
data/beetle.gemspec
CHANGED
@@ -34,11 +34,11 @@ Gem::Specification.new do |s|
|
|
34
34
|
s.specification_version = 3
|
35
35
|
s.add_runtime_dependency("uuid4r", [">= 0.1.2"])
|
36
36
|
s.add_runtime_dependency("bunny", ["= 0.7.9"])
|
37
|
-
s.add_runtime_dependency("redis", ["
|
37
|
+
s.add_runtime_dependency("redis", [">= 2.2.2", "< 3.1"])
|
38
38
|
s.add_runtime_dependency("hiredis", ["= 0.4.5"])
|
39
|
-
s.add_runtime_dependency("amq-client", ["= 0.9.
|
40
|
-
s.add_runtime_dependency("amq-protocol", ["= 1.0
|
41
|
-
s.add_runtime_dependency("amqp", ["= 0.9.
|
39
|
+
s.add_runtime_dependency("amq-client", ["= 0.9.12"])
|
40
|
+
s.add_runtime_dependency("amq-protocol", ["= 1.2.0"])
|
41
|
+
s.add_runtime_dependency("amqp", ["= 0.9.10"])
|
42
42
|
s.add_runtime_dependency("activesupport", [">= 2.3.4"])
|
43
43
|
s.add_runtime_dependency("eventmachine_httpserver", [">= 0.2.1"])
|
44
44
|
s.add_runtime_dependency("daemons", [">= 1.0.10"])
|
@@ -74,8 +74,10 @@ module TestDaemons
|
|
74
74
|
false
|
75
75
|
end
|
76
76
|
|
77
|
+
HTTP_SERVER_PORT = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
|
78
|
+
|
77
79
|
def self.get_status(path, content_type)
|
78
|
-
uri = URI.parse("http://127.0.0.1
|
80
|
+
uri = URI.parse("http://127.0.0.1:#{HTTP_SERVER_PORT}#{path}")
|
79
81
|
http = Net::HTTP.new(uri.host, uri.port)
|
80
82
|
request = Net::HTTP::Get.new(uri.request_uri)
|
81
83
|
request['Accept'] = content_type
|
@@ -84,7 +86,7 @@ module TestDaemons
|
|
84
86
|
end
|
85
87
|
|
86
88
|
def self.initiate_master_switch
|
87
|
-
http = Net::HTTP.new('127.0.0.1',
|
89
|
+
http = Net::HTTP.new('127.0.0.1', HTTP_SERVER_PORT)
|
88
90
|
response = http.post '/initiate_master_switch', ''
|
89
91
|
response
|
90
92
|
end
|
@@ -79,9 +79,10 @@ module Beetle
|
|
79
79
|
Daemons.run_proc("redis_configuration_server", :log_output => true, :dir_mode => dir_mode, :dir => dir) do
|
80
80
|
config_server = Beetle::RedisConfigurationServer.new
|
81
81
|
Beetle::RedisConfigurationHttpServer.config_server = config_server
|
82
|
+
http_server_port = RUBY_PLATFORM =~ /darwin/ ? 9080 : 8080
|
82
83
|
EM.run do
|
83
84
|
config_server.start
|
84
|
-
EM.start_server '0.0.0.0',
|
85
|
+
EM.start_server '0.0.0.0', http_server_port, Beetle::RedisConfigurationHttpServer
|
85
86
|
end
|
86
87
|
end
|
87
88
|
end
|
data/lib/beetle/redis_ext.rb
CHANGED
@@ -46,4 +46,36 @@ class Redis #:nodoc:
|
|
46
46
|
info["role"] == "slave" && info["master_host"] == host && info["master_port"] == port.to_s
|
47
47
|
end
|
48
48
|
|
49
|
+
# compatibility layer for redis 2.2.2
|
50
|
+
# remove this once all our apps have upgraded to 3.x
|
51
|
+
if Redis::VERSION < "3.0"
|
52
|
+
|
53
|
+
# Redis 2 tries to establish a connection on inspect. this is evil!
|
54
|
+
def inspect
|
55
|
+
super
|
56
|
+
end
|
57
|
+
|
58
|
+
# redis 2.2.2 shutdown implementation does not disconnect from the redis server.
|
59
|
+
# this leaves the connection in an inconsistent state and causes the next command to silently fail.
|
60
|
+
# this in turn breaks our cucumber test scenarios.
|
61
|
+
# fix this here, until a new version is released which fixes the problem.
|
62
|
+
|
63
|
+
alias_method :broken_shutdown, :shutdown
|
64
|
+
|
65
|
+
# Synchronously save the dataset to disk and then shut down the server.
|
66
|
+
def shutdown
|
67
|
+
synchronize do
|
68
|
+
begin
|
69
|
+
@client.call [:shutdown]
|
70
|
+
rescue Errno::ECONNREFUSED
|
71
|
+
ensure
|
72
|
+
@client.disconnect
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def msetnx(*values)
|
78
|
+
super != 0
|
79
|
+
end
|
80
|
+
end
|
49
81
|
end
|
data/lib/beetle/subscriber.rb
CHANGED
@@ -7,6 +7,8 @@ module Beetle
|
|
7
7
|
# create a new subscriber instance
|
8
8
|
def initialize(client, options = {}) #:nodoc:
|
9
9
|
super
|
10
|
+
@status = :idle
|
11
|
+
@request_stop = false
|
10
12
|
@servers.concat @client.additional_subscription_servers
|
11
13
|
@handlers = {}
|
12
14
|
@connections = {}
|
@@ -51,9 +53,19 @@ module Beetle
|
|
51
53
|
if @connections.empty?
|
52
54
|
EM.stop_event_loop
|
53
55
|
else
|
54
|
-
|
55
|
-
|
56
|
-
|
56
|
+
# Only kill connections if not currently processing a message
|
57
|
+
# otherwise messages can get ACKed after the connection is closed
|
58
|
+
# resulting in the ACK not being received and hence the
|
59
|
+
# message being re-delivered
|
60
|
+
if @status == :idle
|
61
|
+
server, connection = @connections.shift
|
62
|
+
logger.debug "Beetle: closing connection to #{server}"
|
63
|
+
connection.close { stop! }
|
64
|
+
else
|
65
|
+
# else ask for stop. After processing the current message the
|
66
|
+
# stop will be re-attempted
|
67
|
+
@request_stop = true
|
68
|
+
end
|
57
69
|
end
|
58
70
|
end
|
59
71
|
|
@@ -125,6 +137,7 @@ module Beetle
|
|
125
137
|
server = @server
|
126
138
|
lambda do |header, data|
|
127
139
|
begin
|
140
|
+
@status = :busy
|
128
141
|
# logger.debug "Beetle: received message"
|
129
142
|
processor = Handler.create(handler, opts)
|
130
143
|
message_options = opts.merge(:server => server, :store => @client.deduplication_store)
|
@@ -150,6 +163,10 @@ module Beetle
|
|
150
163
|
ensure
|
151
164
|
# processing_completed swallows all exceptions, so we don't need to protect this call
|
152
165
|
processor.processing_completed
|
166
|
+
@status = :idle
|
167
|
+
if @request_stop
|
168
|
+
stop!
|
169
|
+
end
|
153
170
|
end
|
154
171
|
end
|
155
172
|
end
|
data/lib/beetle/version.rb
CHANGED
@@ -56,7 +56,7 @@ module Beetle
|
|
56
56
|
end
|
57
57
|
|
58
58
|
class HiredisLoadedTest < Test::Unit::TestCase
|
59
|
-
test
|
59
|
+
test "should be using hiredis instead of the redis ruby backend" do
|
60
60
|
assert defined?(Hiredis)
|
61
61
|
end
|
62
62
|
end
|
@@ -66,9 +66,27 @@ module Beetle
|
|
66
66
|
@r = Redis.new(:host => "localhost", :port => 6390)
|
67
67
|
end
|
68
68
|
|
69
|
-
|
70
|
-
|
71
|
-
|
69
|
+
if Redis::VERSION < "3.0"
|
70
|
+
|
71
|
+
test "orginal redis shutdown implementation is broken" do
|
72
|
+
@r.client.expects(:call_without_reply).with([:shutdown]).once
|
73
|
+
@r.client.expects(:disconnect).never
|
74
|
+
@r.broken_shutdown
|
75
|
+
end
|
76
|
+
|
77
|
+
test "patched redis shutdown implementation should call :shutdown and rescue Errno::ECONNREFUSED" do
|
78
|
+
@r.client.expects(:call).with([:shutdown]).once.raises(Errno::ECONNREFUSED)
|
79
|
+
@r.client.expects(:disconnect).once
|
80
|
+
@r.shutdown
|
81
|
+
end
|
82
|
+
|
83
|
+
else
|
84
|
+
|
85
|
+
test "redis shutdown implementation should call :shutdown and return nil" do
|
86
|
+
@r.client.expects(:call).with([:shutdown]).once.raises(Redis::ConnectionError)
|
87
|
+
assert_nil @r.shutdown
|
88
|
+
end
|
89
|
+
|
72
90
|
end
|
73
91
|
end
|
74
92
|
|
@@ -22,7 +22,7 @@ module Beetle
|
|
22
22
|
assert_equal channel, @sub.send(:channel, "donald:1")
|
23
23
|
end
|
24
24
|
|
25
|
-
test "stop! should close all amqp connections and then stop the event loop" do
|
25
|
+
test "stop! should close all amqp connections and then stop the event loop if no handler is currently running" do
|
26
26
|
connection1 = mock('con1')
|
27
27
|
connection1.expects(:close).yields
|
28
28
|
connection2 = mock('con2')
|
@@ -30,6 +30,19 @@ module Beetle
|
|
30
30
|
@sub.instance_variable_set "@connections", [["server1", connection1], ["server2",connection2]]
|
31
31
|
EM.expects(:stop_event_loop)
|
32
32
|
@sub.send(:stop!)
|
33
|
+
assert !@sub.instance_variable_get("@request_stop")
|
34
|
+
end
|
35
|
+
|
36
|
+
test "stop! should not stop the event loop if a handler is currently running" do
|
37
|
+
@sub.instance_variable_set "@status", :busy
|
38
|
+
connection1 = mock('con1')
|
39
|
+
connection1.expects(:close).never
|
40
|
+
connection2 = mock('con2')
|
41
|
+
connection2.expects(:close).never
|
42
|
+
@sub.instance_variable_set "@connections", [["server1", connection1], ["server2",connection2]]
|
43
|
+
EM.expects(:stop_event_loop).never
|
44
|
+
@sub.send(:stop!)
|
45
|
+
assert @sub.instance_variable_get("@request_stop")
|
33
46
|
end
|
34
47
|
|
35
48
|
end
|
@@ -224,6 +237,14 @@ module Beetle
|
|
224
237
|
assert_nothing_raised { @callback.call(header, 'foo') }
|
225
238
|
end
|
226
239
|
|
240
|
+
test "should call stop! if @request_stop has been set" do
|
241
|
+
header = header_with_params({})
|
242
|
+
Message.any_instance.expects(:process).raises(Exception.new("don't worry"))
|
243
|
+
@sub.instance_variable_set("@request_stop", true)
|
244
|
+
@sub.expects(:stop!)
|
245
|
+
assert_nothing_raised { @callback.call(header, 'foo') }
|
246
|
+
end
|
247
|
+
|
227
248
|
test "should call reject on the message header when processing the handler returns true on reject?" do
|
228
249
|
header = header_with_params({})
|
229
250
|
result = mock("result")
|
data/test/test_helper.rb
CHANGED
@@ -3,10 +3,11 @@ if RUBY_VERSION >= "1.9"
|
|
3
3
|
require 'simplecov'
|
4
4
|
SimpleCov.start do
|
5
5
|
add_filter "/test/"
|
6
|
+
add_filter "/lib/beetle/redis_ext.rb"
|
6
7
|
end
|
7
8
|
end
|
8
9
|
require 'test/unit'
|
9
|
-
require 'mocha'
|
10
|
+
require 'mocha/setup'
|
10
11
|
require 'active_support/testing/declarative'
|
11
12
|
|
12
13
|
require File.expand_path(File.dirname(__FILE__) + '/../lib/beetle')
|
metadata
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
|
4
|
+
prerelease:
|
5
|
+
version: 0.3.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Stefan Kaes
|
@@ -13,168 +13,174 @@ authors:
|
|
13
13
|
autorequire:
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date: 2013-
|
16
|
+
date: 2013-03-12 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: uuid4r
|
20
20
|
requirement: !ruby/object:Gem::Requirement
|
21
|
-
none: false
|
22
21
|
requirements:
|
23
22
|
- - ! '>='
|
24
23
|
- !ruby/object:Gem::Version
|
25
24
|
version: 0.1.2
|
25
|
+
none: false
|
26
26
|
type: :runtime
|
27
|
-
prerelease: false
|
28
27
|
version_requirements: !ruby/object:Gem::Requirement
|
29
|
-
none: false
|
30
28
|
requirements:
|
31
29
|
- - ! '>='
|
32
30
|
- !ruby/object:Gem::Version
|
33
31
|
version: 0.1.2
|
32
|
+
none: false
|
33
|
+
prerelease: false
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: bunny
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
37
|
-
none: false
|
38
37
|
requirements:
|
39
38
|
- - '='
|
40
39
|
- !ruby/object:Gem::Version
|
41
40
|
version: 0.7.9
|
41
|
+
none: false
|
42
42
|
type: :runtime
|
43
|
-
prerelease: false
|
44
43
|
version_requirements: !ruby/object:Gem::Requirement
|
45
|
-
none: false
|
46
44
|
requirements:
|
47
45
|
- - '='
|
48
46
|
- !ruby/object:Gem::Version
|
49
47
|
version: 0.7.9
|
48
|
+
none: false
|
49
|
+
prerelease: false
|
50
50
|
- !ruby/object:Gem::Dependency
|
51
51
|
name: redis
|
52
52
|
requirement: !ruby/object:Gem::Requirement
|
53
|
-
none: false
|
54
53
|
requirements:
|
55
|
-
- - '
|
54
|
+
- - ! '>='
|
56
55
|
- !ruby/object:Gem::Version
|
57
|
-
version:
|
56
|
+
version: 2.2.2
|
57
|
+
- - <
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '3.1'
|
60
|
+
none: false
|
58
61
|
type: :runtime
|
59
|
-
prerelease: false
|
60
62
|
version_requirements: !ruby/object:Gem::Requirement
|
61
|
-
none: false
|
62
63
|
requirements:
|
63
|
-
- - '
|
64
|
+
- - ! '>='
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 2.2.2
|
67
|
+
- - <
|
64
68
|
- !ruby/object:Gem::Version
|
65
|
-
version: 3.
|
69
|
+
version: '3.1'
|
70
|
+
none: false
|
71
|
+
prerelease: false
|
66
72
|
- !ruby/object:Gem::Dependency
|
67
73
|
name: hiredis
|
68
74
|
requirement: !ruby/object:Gem::Requirement
|
69
|
-
none: false
|
70
75
|
requirements:
|
71
76
|
- - '='
|
72
77
|
- !ruby/object:Gem::Version
|
73
78
|
version: 0.4.5
|
79
|
+
none: false
|
74
80
|
type: :runtime
|
75
|
-
prerelease: false
|
76
81
|
version_requirements: !ruby/object:Gem::Requirement
|
77
|
-
none: false
|
78
82
|
requirements:
|
79
83
|
- - '='
|
80
84
|
- !ruby/object:Gem::Version
|
81
85
|
version: 0.4.5
|
86
|
+
none: false
|
87
|
+
prerelease: false
|
82
88
|
- !ruby/object:Gem::Dependency
|
83
89
|
name: amq-client
|
84
90
|
requirement: !ruby/object:Gem::Requirement
|
85
|
-
none: false
|
86
91
|
requirements:
|
87
92
|
- - '='
|
88
93
|
- !ruby/object:Gem::Version
|
89
|
-
version: 0.9.
|
94
|
+
version: 0.9.12
|
95
|
+
none: false
|
90
96
|
type: :runtime
|
91
|
-
prerelease: false
|
92
97
|
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
none: false
|
94
98
|
requirements:
|
95
99
|
- - '='
|
96
100
|
- !ruby/object:Gem::Version
|
97
|
-
version: 0.9.
|
101
|
+
version: 0.9.12
|
102
|
+
none: false
|
103
|
+
prerelease: false
|
98
104
|
- !ruby/object:Gem::Dependency
|
99
105
|
name: amq-protocol
|
100
106
|
requirement: !ruby/object:Gem::Requirement
|
101
|
-
none: false
|
102
107
|
requirements:
|
103
108
|
- - '='
|
104
109
|
- !ruby/object:Gem::Version
|
105
|
-
version: 1.0
|
110
|
+
version: 1.2.0
|
111
|
+
none: false
|
106
112
|
type: :runtime
|
107
|
-
prerelease: false
|
108
113
|
version_requirements: !ruby/object:Gem::Requirement
|
109
|
-
none: false
|
110
114
|
requirements:
|
111
115
|
- - '='
|
112
116
|
- !ruby/object:Gem::Version
|
113
|
-
version: 1.0
|
117
|
+
version: 1.2.0
|
118
|
+
none: false
|
119
|
+
prerelease: false
|
114
120
|
- !ruby/object:Gem::Dependency
|
115
121
|
name: amqp
|
116
122
|
requirement: !ruby/object:Gem::Requirement
|
117
|
-
none: false
|
118
123
|
requirements:
|
119
124
|
- - '='
|
120
125
|
- !ruby/object:Gem::Version
|
121
|
-
version: 0.9.
|
126
|
+
version: 0.9.10
|
127
|
+
none: false
|
122
128
|
type: :runtime
|
123
|
-
prerelease: false
|
124
129
|
version_requirements: !ruby/object:Gem::Requirement
|
125
|
-
none: false
|
126
130
|
requirements:
|
127
131
|
- - '='
|
128
132
|
- !ruby/object:Gem::Version
|
129
|
-
version: 0.9.
|
133
|
+
version: 0.9.10
|
134
|
+
none: false
|
135
|
+
prerelease: false
|
130
136
|
- !ruby/object:Gem::Dependency
|
131
137
|
name: activesupport
|
132
138
|
requirement: !ruby/object:Gem::Requirement
|
133
|
-
none: false
|
134
139
|
requirements:
|
135
140
|
- - ! '>='
|
136
141
|
- !ruby/object:Gem::Version
|
137
142
|
version: 2.3.4
|
143
|
+
none: false
|
138
144
|
type: :runtime
|
139
|
-
prerelease: false
|
140
145
|
version_requirements: !ruby/object:Gem::Requirement
|
141
|
-
none: false
|
142
146
|
requirements:
|
143
147
|
- - ! '>='
|
144
148
|
- !ruby/object:Gem::Version
|
145
149
|
version: 2.3.4
|
150
|
+
none: false
|
151
|
+
prerelease: false
|
146
152
|
- !ruby/object:Gem::Dependency
|
147
153
|
name: eventmachine_httpserver
|
148
154
|
requirement: !ruby/object:Gem::Requirement
|
149
|
-
none: false
|
150
155
|
requirements:
|
151
156
|
- - ! '>='
|
152
157
|
- !ruby/object:Gem::Version
|
153
158
|
version: 0.2.1
|
159
|
+
none: false
|
154
160
|
type: :runtime
|
155
|
-
prerelease: false
|
156
161
|
version_requirements: !ruby/object:Gem::Requirement
|
157
|
-
none: false
|
158
162
|
requirements:
|
159
163
|
- - ! '>='
|
160
164
|
- !ruby/object:Gem::Version
|
161
165
|
version: 0.2.1
|
166
|
+
none: false
|
167
|
+
prerelease: false
|
162
168
|
- !ruby/object:Gem::Dependency
|
163
169
|
name: daemons
|
164
170
|
requirement: !ruby/object:Gem::Requirement
|
165
|
-
none: false
|
166
171
|
requirements:
|
167
172
|
- - ! '>='
|
168
173
|
- !ruby/object:Gem::Version
|
169
174
|
version: 1.0.10
|
175
|
+
none: false
|
170
176
|
type: :runtime
|
171
|
-
prerelease: false
|
172
177
|
version_requirements: !ruby/object:Gem::Requirement
|
173
|
-
none: false
|
174
178
|
requirements:
|
175
179
|
- - ! '>='
|
176
180
|
- !ruby/object:Gem::Version
|
177
181
|
version: 1.0.10
|
182
|
+
none: false
|
183
|
+
prerelease: false
|
178
184
|
description: A highly available, reliable messaging infrastructure
|
179
185
|
email: opensource@xing.com
|
180
186
|
executables:
|
@@ -270,23 +276,23 @@ rdoc_options:
|
|
270
276
|
require_paths:
|
271
277
|
- lib
|
272
278
|
required_ruby_version: !ruby/object:Gem::Requirement
|
273
|
-
none: false
|
274
279
|
requirements:
|
275
280
|
- - ! '>='
|
276
281
|
- !ruby/object:Gem::Version
|
277
|
-
version: '0'
|
278
282
|
segments:
|
279
283
|
- 0
|
280
|
-
hash: -
|
281
|
-
|
284
|
+
hash: -863009242128424125
|
285
|
+
version: '0'
|
282
286
|
none: false
|
287
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
283
288
|
requirements:
|
284
289
|
- - ! '>='
|
285
290
|
- !ruby/object:Gem::Version
|
286
291
|
version: 1.3.7
|
292
|
+
none: false
|
287
293
|
requirements: []
|
288
294
|
rubyforge_project:
|
289
|
-
rubygems_version: 1.8.
|
295
|
+
rubygems_version: 1.8.25
|
290
296
|
signing_key:
|
291
297
|
specification_version: 3
|
292
298
|
summary: High Availability AMQP Messaging with Redundant Queues
|