mcollective-client 2.4.1 → 2.5.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mcollective-client might be problematic. Click here for more details.
- data/lib/mcollective.rb +1 -14
- data/lib/mcollective/config.rb +4 -1
- data/lib/mcollective/exceptions.rb +27 -0
- data/lib/mcollective/unix_daemon.rb +4 -1
- data/lib/mcollective/windows_daemon.rb +18 -9
- data/spec/unit/plugins/mcollective/connector/activemq_spec.rb +331 -228
- data/spec/unit/plugins/mcollective/connector/rabbitmq_spec.rb +332 -239
- data/spec/unit/runner_spec.rb +253 -0
- data/spec/unit/unix_daemon_spec.rb +2 -2
- data/spec/unit/windows_daemon_spec.rb +1 -7
- metadata +8 -23
- data/spec/unit/security/runner_spec.rb +0 -67
@@ -18,49 +18,80 @@ end
|
|
18
18
|
module MCollective
|
19
19
|
module Connector
|
20
20
|
describe Rabbitmq do
|
21
|
-
before do
|
22
|
-
unless ::Stomp::Error.constants.map{|c| c.to_s}.include?("NoCurrentConnection")
|
23
|
-
class ::Stomp::Error::NoCurrentConnection < RuntimeError ; end
|
24
|
-
end
|
25
21
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
22
|
+
let(:config) do
|
23
|
+
conf = mock
|
24
|
+
conf.stubs(:configured).returns(true)
|
25
|
+
conf.stubs(:identity).returns("rspec")
|
26
|
+
conf.stubs(:collectives).returns(["mcollective"])
|
27
|
+
conf
|
28
|
+
end
|
31
29
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
30
|
+
let(:logger) do
|
31
|
+
log = mock
|
32
|
+
log.stubs(:log)
|
33
|
+
log.stubs(:start)
|
34
|
+
Log.configure(log)
|
35
|
+
log
|
36
|
+
end
|
36
37
|
|
37
|
-
|
38
|
+
let(:msg) do
|
39
|
+
m = mock
|
40
|
+
m.stubs(:base64_encode!)
|
41
|
+
m.stubs(:payload).returns("msg")
|
42
|
+
m.stubs(:agent).returns("agent")
|
43
|
+
m.stubs(:type).returns(:reply)
|
44
|
+
m.stubs(:collective).returns("mcollective")
|
45
|
+
m
|
46
|
+
end
|
38
47
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
48
|
+
let(:subscription) do
|
49
|
+
sub = mock
|
50
|
+
sub.stubs("<<").returns(true)
|
51
|
+
sub.stubs("include?").returns(false)
|
52
|
+
sub.stubs("delete").returns(false)
|
53
|
+
sub
|
54
|
+
end
|
45
55
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
56
|
+
let(:connection) do
|
57
|
+
con = mock
|
58
|
+
con.stubs(:subscribe).returns(true)
|
59
|
+
con.stubs(:unsubscribe).returns(true)
|
60
|
+
con
|
61
|
+
end
|
62
|
+
|
63
|
+
let(:connector) do
|
64
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_exponential_back_off", "true").returns(true)
|
65
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.initial_reconnect_delay", 0.01).returns(0.01)
|
66
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.back_off_multiplier", 2).returns(2)
|
67
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.max_reconnect_delay", 30.0).returns(30.0)
|
68
|
+
c = Rabbitmq.new
|
69
|
+
c.instance_variable_set("@subscriptions", subscription)
|
70
|
+
c.instance_variable_set("@connection", connection)
|
71
|
+
c
|
72
|
+
end
|
50
73
|
|
51
|
-
|
52
|
-
|
53
|
-
|
74
|
+
before do
|
75
|
+
unless ::Stomp::Error.constants.map{|c| c.to_s}.include?("NoCurrentConnection")
|
76
|
+
class ::Stomp::Error::NoCurrentConnection < RuntimeError ; end
|
77
|
+
end
|
54
78
|
|
55
|
-
|
56
|
-
|
57
|
-
|
79
|
+
logger
|
80
|
+
Config.stubs(:instance).returns(config)
|
81
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_reply_exchange", false).returns(false)
|
58
82
|
end
|
59
83
|
|
60
84
|
describe "#initialize" do
|
85
|
+
before :each do
|
86
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_exponential_back_off", "true").returns(true)
|
87
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.initial_reconnect_delay", 0.01).returns(0.01)
|
88
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.back_off_multiplier", 2).returns(2)
|
89
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.max_reconnect_delay", 30.0).returns(30.0)
|
90
|
+
end
|
91
|
+
|
61
92
|
it "should set the @config variable" do
|
62
93
|
c = Rabbitmq.new
|
63
|
-
c.instance_variable_get("@config").should ==
|
94
|
+
c.instance_variable_get("@config").should == config
|
64
95
|
end
|
65
96
|
|
66
97
|
it "should set @subscriptions to an empty list" do
|
@@ -70,56 +101,57 @@ module MCollective
|
|
70
101
|
end
|
71
102
|
|
72
103
|
describe "#connect" do
|
104
|
+
before :each do
|
105
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(30)
|
106
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with('rabbitmq.stomp_1_0_fallback', true).returns(true)
|
107
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with('rabbitmq.base64', 'false').returns(false)
|
108
|
+
Rabbitmq.any_instance.stubs(:get_option).with('rabbitmq.vhost', '/').returns('rspec')
|
109
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.max_reconnect_attempts", 0).returns(5)
|
110
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.randomize", "false").returns(true)
|
111
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.backup", "false").returns(true)
|
112
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.timeout", -1).returns(1)
|
113
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.connect_timeout", 30).returns(5)
|
114
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.max_hbrlck_fails", 2).returns(2)
|
115
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.max_hbread_fails", 2).returns(2)
|
116
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.base64", 'false').returns(false)
|
117
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.size").returns(2)
|
118
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.host").returns("host1")
|
119
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.port", 61613).returns(6163)
|
120
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.pool.1.ssl", "false").returns(false)
|
121
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.2.host").returns("host2")
|
122
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.2.port", 61613).returns(6164)
|
123
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.pool.2.ssl", "false").returns(true)
|
124
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.pool.2.ssl.fallback", "false").returns(true)
|
125
|
+
Rabbitmq.any_instance.stubs(:get_env_or_option).with("STOMP_USER", "rabbitmq.pool.1.user").returns("user1")
|
126
|
+
Rabbitmq.any_instance.stubs(:get_env_or_option).with("STOMP_USER", "rabbitmq.pool.2.user").returns("user2")
|
127
|
+
Rabbitmq.any_instance.stubs(:get_env_or_option).with("STOMP_PASSWORD", "rabbitmq.pool.1.password").returns("password1")
|
128
|
+
Rabbitmq.any_instance.stubs(:get_env_or_option).with("STOMP_PASSWORD", "rabbitmq.pool.2.password").returns("password2")
|
129
|
+
Rabbitmq.any_instance.instance_variable_set("@subscriptions", subscription)
|
130
|
+
Rabbitmq.any_instance.instance_variable_set("@connection", connection)
|
131
|
+
end
|
132
|
+
|
73
133
|
it "should not try to reconnect if already connected" do
|
74
134
|
Log.expects(:debug).with("Already connection, not re-initializing connection").once
|
75
|
-
|
135
|
+
connector.connect
|
76
136
|
end
|
77
137
|
|
78
138
|
it "should support new style config" do
|
79
|
-
pluginconf = {"rabbitmq.pool.size" => "2",
|
80
|
-
"rabbitmq.pool.1.host" => "host1",
|
81
|
-
"rabbitmq.pool.1.port" => "6163",
|
82
|
-
"rabbitmq.pool.1.user" => "user1",
|
83
|
-
"rabbitmq.pool.1.password" => "password1",
|
84
|
-
"rabbitmq.pool.1.ssl" => "false",
|
85
|
-
"rabbitmq.pool.2.host" => "host2",
|
86
|
-
"rabbitmq.pool.2.port" => "6164",
|
87
|
-
"rabbitmq.pool.2.user" => "user2",
|
88
|
-
"rabbitmq.pool.2.password" => "password2",
|
89
|
-
"rabbitmq.pool.2.ssl" => "true",
|
90
|
-
"rabbitmq.pool.2.ssl.fallback" => "true",
|
91
|
-
"rabbitmq.initial_reconnect_delay" => "0.02",
|
92
|
-
"rabbitmq.max_reconnect_delay" => "40",
|
93
|
-
"rabbitmq.use_exponential_back_off" => "false",
|
94
|
-
"rabbitmq.back_off_multiplier" => "3",
|
95
|
-
"rabbitmq.max_reconnect_attempts" => "5",
|
96
|
-
"rabbitmq.randomize" => "true",
|
97
|
-
"rabbitmq.backup" => "true",
|
98
|
-
"rabbitmq.timeout" => "1",
|
99
|
-
"rabbitmq.vhost" => "mcollective",
|
100
|
-
"rabbitmq.max_hbrlck_fails" => 3,
|
101
|
-
"rabbitmq.max_hbread_fails" => 3,
|
102
|
-
"rabbitmq.connect_timeout" => "5"}
|
103
|
-
|
104
|
-
|
105
139
|
ENV.delete("STOMP_USER")
|
106
140
|
ENV.delete("STOMP_PASSWORD")
|
107
141
|
|
108
|
-
@config.expects(:pluginconf).returns(pluginconf).at_least_once
|
109
|
-
|
110
142
|
Rabbitmq::EventLogger.expects(:new).returns("logger")
|
111
143
|
|
112
|
-
|
113
|
-
|
114
|
-
:back_off_multiplier =>
|
115
|
-
:max_reconnect_delay =>
|
144
|
+
connector_obj = mock
|
145
|
+
connector_obj.expects(:new).with(:backup => true,
|
146
|
+
:back_off_multiplier => 2,
|
147
|
+
:max_reconnect_delay => 30.0,
|
116
148
|
:timeout => 1,
|
117
149
|
:connect_timeout => 5,
|
118
|
-
:use_exponential_back_off =>
|
150
|
+
:use_exponential_back_off => true,
|
119
151
|
:max_reconnect_attempts => 5,
|
120
|
-
:initial_reconnect_delay => 0.
|
121
|
-
:max_hbread_fails =>
|
122
|
-
:max_hbrlck_fails =>
|
152
|
+
:initial_reconnect_delay => 0.01,
|
153
|
+
:max_hbread_fails => 2,
|
154
|
+
:max_hbrlck_fails => 2,
|
123
155
|
:randomize => true,
|
124
156
|
:reliable => true,
|
125
157
|
:logger => "logger",
|
@@ -136,202 +168,257 @@ module MCollective
|
|
136
168
|
:login => 'user2'}
|
137
169
|
])
|
138
170
|
|
139
|
-
|
140
|
-
|
171
|
+
connector.expects(:ssl_parameters).with(2, true).returns(true)
|
172
|
+
connector.expects(:connection_headers).returns({})
|
141
173
|
|
142
|
-
|
143
|
-
|
174
|
+
connector.instance_variable_set("@connection", nil)
|
175
|
+
connector.connect(connector_obj)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
describe "#stomp_version_supports_heartbeat?" do
|
180
|
+
it "should not be supported with stomp 1.2.9" do
|
181
|
+
connector.stubs(:stomp_version).returns("1.2.9")
|
182
|
+
connector.stomp_version_supports_heartbeat? == false
|
183
|
+
end
|
184
|
+
|
185
|
+
it "should be supported with stomp 1.2.10" do
|
186
|
+
connector.stubs(:stomp_version).returns("1.2.10")
|
187
|
+
connector.stomp_version_supports_heartbeat? == true
|
144
188
|
end
|
145
189
|
end
|
146
190
|
|
147
191
|
describe "#connection_headers" do
|
148
192
|
before do
|
149
|
-
|
193
|
+
connector.stubs(:stomp_version).returns("1.2.10")
|
194
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(1)
|
195
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with('rabbitmq.stomp_1_0_fallback', true).returns(true)
|
196
|
+
Rabbitmq.any_instance.stubs(:get_option).with('rabbitmq.vhost', '/').returns('rspec')
|
150
197
|
end
|
151
198
|
|
152
|
-
it "should default to stomp 1.0
|
153
|
-
|
154
|
-
|
199
|
+
it "should default to stomp 1.0" do
|
200
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(0)
|
201
|
+
connector.connection_headers[:"accept-version"] == "1.0"
|
155
202
|
end
|
156
203
|
|
157
204
|
it "should support setting the vhost" do
|
158
|
-
|
159
|
-
|
205
|
+
connector.connection_headers[:host].should == "rspec"
|
206
|
+
end
|
207
|
+
|
208
|
+
it "should log an informational message about not using Stomp 1.1" do
|
209
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(0)
|
210
|
+
Log.expects(:info).with(regexp_matches(/without STOMP 1.1 heartbeats/))
|
211
|
+
connector.connection_headers
|
160
212
|
end
|
161
213
|
|
162
|
-
it "should log
|
163
|
-
|
164
|
-
|
165
|
-
|
214
|
+
it "should not log an informational message about not using Stomp 1.1 if the gem won't support it" do
|
215
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(0)
|
216
|
+
connector.stubs(:stomp_version).returns("1.0.0")
|
217
|
+
Log.expects(:info).with(regexp_matches(/without STOMP 1.1 heartbeats/)).never
|
218
|
+
connector.connection_headers
|
166
219
|
end
|
167
220
|
|
168
221
|
it "should not support stomp 1.1 with older versions of the stomp gem" do
|
169
|
-
|
170
|
-
|
171
|
-
expect {
|
222
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(30)
|
223
|
+
connector.expects(:stomp_version).returns("1.0.0").once
|
224
|
+
expect { connector.connection_headers }.to raise_error("Setting STOMP 1.1 properties like heartbeat intervals require at least version 1.2.10 of the STOMP gem")
|
172
225
|
end
|
173
226
|
|
174
227
|
it "should force the heartbeat to min 30 seconds" do
|
175
|
-
|
176
|
-
|
228
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(30)
|
229
|
+
connector.connection_headers[:"heart-beat"].should == "30500,29500"
|
177
230
|
end
|
178
231
|
|
179
232
|
it "should default to 1.0 and 1.1 support" do
|
180
|
-
|
181
|
-
|
233
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(30)
|
234
|
+
connector.connection_headers[:"accept-version"].should == "1.1,1.0"
|
182
235
|
end
|
183
236
|
|
184
237
|
it "should support stomp 1.1 only operation" do
|
185
|
-
|
186
|
-
|
238
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.heartbeat_interval", 0).returns(30)
|
239
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.stomp_1_0_fallback", true).returns(false)
|
240
|
+
connector.connection_headers[:"accept-version"].should == "1.1"
|
187
241
|
end
|
188
242
|
end
|
189
243
|
|
190
244
|
describe "#ssl_paramaters" do
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
245
|
+
before :each do
|
246
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.host").returns("host1")
|
247
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.port").returns("6164")
|
248
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.user").returns("user1")
|
249
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.password").returns("password1")
|
250
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.pool.1.ssl", false).returns(true)
|
251
|
+
end
|
198
252
|
|
199
|
-
|
253
|
+
it "should ensure all settings are provided" do
|
254
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.cert", false).returns("rspec")
|
255
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.key", false).returns(nil)
|
256
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.ca", false).returns(nil)
|
200
257
|
|
201
|
-
expect {
|
258
|
+
expect { connector.ssl_parameters(1, false) }.to raise_error("cert, key and ca has to be supplied for verified SSL mode")
|
202
259
|
end
|
203
260
|
|
204
261
|
it "should verify the ssl files exist" do
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
"rabbitmq.pool.1.ssl.key" => "rspec.key",
|
212
|
-
"rabbitmq.pool.1.ssl.ca" => "rspec1.ca,rspec2.ca"}
|
213
|
-
|
214
|
-
@config.expects(:pluginconf).returns(pluginconf).at_least_once
|
215
|
-
@c.expects(:get_key_file).returns("rspec.key").at_least_once
|
216
|
-
@c.expects(:get_cert_file).returns("rspec.cert").at_least_once
|
262
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.cert", false).returns("rspec")
|
263
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.key", false).returns('rspec.key')
|
264
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.ca", false).returns('rspec1.ca,rspec2.ca')
|
265
|
+
|
266
|
+
connector.expects(:get_key_file).returns("rspec.key").at_least_once
|
267
|
+
connector.expects(:get_cert_file).returns("rspec.cert").at_least_once
|
217
268
|
|
218
269
|
File.expects(:exist?).with("rspec.cert").twice.returns(true)
|
219
270
|
File.expects(:exist?).with("rspec.key").twice.returns(true)
|
220
271
|
File.expects(:exist?).with("rspec1.ca").twice.returns(true)
|
221
272
|
File.expects(:exist?).with("rspec2.ca").twice.returns(false)
|
222
273
|
|
223
|
-
expect {
|
274
|
+
expect { connector.ssl_parameters(1, false) }.to raise_error("Cannot find CA file rspec2.ca")
|
224
275
|
|
225
|
-
|
276
|
+
connector.ssl_parameters(1, true).should == true
|
226
277
|
end
|
227
278
|
|
228
279
|
it "should support fallback mode when there are errors" do
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
"rabbitmq.pool.1.password" => "password1",
|
233
|
-
"rabbitmq.pool.1.ssl" => "true"}
|
280
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.cert", false).returns("rspec")
|
281
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.key", false).returns('rspec.key')
|
282
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.ca", false).returns('rspec1.ca,rspec2.ca')
|
234
283
|
|
235
|
-
|
236
|
-
|
237
|
-
@c.ssl_parameters(1, true).should == true
|
284
|
+
connector.ssl_parameters(1, true).should == true
|
238
285
|
end
|
239
286
|
|
240
287
|
it "should fail if fallback isnt enabled" do
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
"rabbitmq.pool.1.password" => "password1",
|
245
|
-
"rabbitmq.pool.1.ssl" => "true"}
|
246
|
-
|
247
|
-
@config.expects(:pluginconf).returns(pluginconf).at_least_once
|
288
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.cert", false).returns("rspec")
|
289
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.key", false).returns('rspec.key')
|
290
|
+
Rabbitmq.any_instance.stubs(:get_option).with("rabbitmq.pool.1.ssl.ca", false).returns('rspec1.ca,rspec2.ca')
|
248
291
|
|
249
|
-
expect {
|
292
|
+
expect { connector.ssl_parameters(1, false) }.to raise_error
|
250
293
|
end
|
251
294
|
end
|
252
295
|
|
253
296
|
describe "#get_key_file" do
|
254
297
|
it "should return the filename from the environment variable" do
|
255
298
|
ENV["MCOLLECTIVE_RABBITMQ_POOL2_SSL_KEY"] = "/path/to/rspec/env"
|
256
|
-
|
299
|
+
connector.get_key_file(2).should == "/path/to/rspec/env"
|
257
300
|
end
|
258
301
|
|
259
302
|
it "should return the filename define in the config file if the environment variable doesn't exist" do
|
260
303
|
ENV.delete("MCOLLECTIVE_RABBITMQ_POOL2_SSL_KEY")
|
261
|
-
|
262
|
-
|
304
|
+
connector.expects(:get_option).with("rabbitmq.pool.2.ssl.key", false).returns("/path/to/rspec/conf")
|
305
|
+
connector.get_key_file(2).should == "/path/to/rspec/conf"
|
263
306
|
end
|
264
307
|
end
|
265
308
|
|
266
309
|
describe "#get_cert_file" do
|
267
310
|
it "shold return the filename from the environment variable" do
|
268
311
|
ENV["MCOLLECTIVE_RABBITMQ_POOL2_SSL_CERT"] = "/path/to/rspec/env"
|
269
|
-
|
312
|
+
connector.get_cert_file(2).should == "/path/to/rspec/env"
|
270
313
|
end
|
271
314
|
|
272
315
|
it "should return the filename defined in the config file if the environment variable doesn't exist" do
|
273
316
|
ENV.delete("MCOLLECTIVE_RABBITMQ_POOL2_SSL_CERT")
|
274
|
-
|
275
|
-
|
317
|
+
connector.expects(:get_option).with("rabbitmq.pool.2.ssl.cert", false).returns("/path/to/rspec/conf")
|
318
|
+
connector.get_cert_file(2).should == "/path/to/rspec/conf"
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
describe '#exponential_back_off' do
|
323
|
+
it "should not do anything when use_exponential_back_off is off" do
|
324
|
+
connector.instance_variable_set(:@use_exponential_back_off, false)
|
325
|
+
connector.exponential_back_off.should == nil
|
326
|
+
end
|
327
|
+
|
328
|
+
it 'should return values of the expected sequence on subsequent calls' do
|
329
|
+
connector.instance_variable_set(:@use_exponential_back_off, true)
|
330
|
+
connector.instance_variable_set(:@initial_reconnect_delay, 5.0)
|
331
|
+
connector.instance_variable_set(:@back_off_multiplier, 2)
|
332
|
+
connector.instance_variable_set(:@max_reconnect_delay, 30.0)
|
333
|
+
connector.instance_variable_set(:@reconnect_delay, 5.0)
|
334
|
+
|
335
|
+
connector.exponential_back_off.should == 5
|
336
|
+
connector.exponential_back_off.should == 10
|
337
|
+
connector.exponential_back_off.should == 20
|
338
|
+
connector.exponential_back_off.should == 30
|
339
|
+
connector.exponential_back_off.should == 30
|
276
340
|
end
|
277
341
|
end
|
278
342
|
|
279
343
|
describe "#receive" do
|
280
344
|
it "should receive from the middleware" do
|
281
345
|
payload = mock
|
346
|
+
payload.stubs(:command).returns("MESSAGE")
|
282
347
|
payload.stubs(:body).returns("msg")
|
283
348
|
payload.stubs(:headers).returns("headers")
|
284
349
|
|
285
|
-
|
350
|
+
connection.expects(:receive).returns(payload)
|
286
351
|
|
287
352
|
Message.expects(:new).with("msg", payload, :base64 => true, :headers => "headers").returns("message")
|
288
|
-
|
353
|
+
connector.instance_variable_set("@base64", true)
|
289
354
|
|
290
|
-
received =
|
355
|
+
received = connector.receive
|
291
356
|
received.should == "message"
|
292
357
|
end
|
293
358
|
|
294
359
|
it "should sleep and retry if recieving while disconnected" do
|
295
360
|
payload = mock
|
361
|
+
payload.stubs(:command).returns("MESSAGE")
|
296
362
|
payload.stubs(:body).returns("msg")
|
297
363
|
payload.stubs(:headers).returns("headers")
|
298
364
|
|
299
365
|
Message.stubs(:new).returns("rspec")
|
300
|
-
|
301
|
-
|
366
|
+
connection.expects(:receive).raises(::Stomp::Error::NoCurrentConnection).returns(payload).twice
|
367
|
+
connector.expects(:sleep).with(1)
|
368
|
+
|
369
|
+
connector.receive.should == "rspec"
|
370
|
+
end
|
302
371
|
|
303
|
-
|
372
|
+
it "should raise an error on failure to receive a frame" do
|
373
|
+
connection.expects(:receive).returns(nil)
|
374
|
+
|
375
|
+
expect { connector.receive }.to raise_error(MessageNotReceived, /No message received from RabbitMQ./)
|
376
|
+
end
|
377
|
+
|
378
|
+
it "should log and raise UnexpectedMessageType on non-MESSAGE frames" do
|
379
|
+
payload = mock
|
380
|
+
payload.stubs(:command).returns("ERROR")
|
381
|
+
payload.stubs(:body).returns("Out of cheese exception")
|
382
|
+
payload.stubs(:headers).returns("headers")
|
383
|
+
|
384
|
+
connection.expects(:receive).returns(payload)
|
385
|
+
|
386
|
+
Message.stubs(:new)
|
387
|
+
|
388
|
+
Log.expects(:debug).with('Waiting for a message from RabbitMQ')
|
389
|
+
Log.expects(:debug).with('Unexpected \'ERROR\' frame. Headers: "headers" Body: "Out of cheese exception"')
|
390
|
+
expect { connector.receive }.to raise_error(UnexpectedMessageType, /Received frame of type 'ERROR' expected 'MESSAGE'/)
|
304
391
|
end
|
305
392
|
end
|
306
393
|
|
307
394
|
describe "#publish" do
|
308
|
-
before do
|
309
|
-
|
395
|
+
before :each do
|
396
|
+
connection.stubs(:publish).with("test", "msg", {}).returns(true)
|
310
397
|
end
|
311
398
|
|
312
399
|
it "should base64 encode a message if configured to do so" do
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
400
|
+
connector.instance_variable_set("@base64", true)
|
401
|
+
connector.expects(:target_for).returns({:name => "test", :headers => {}})
|
402
|
+
connection.expects(:publish).with("test", "msg", {})
|
403
|
+
msg.expects(:base64_encode!)
|
317
404
|
|
318
|
-
|
405
|
+
connector.publish(msg)
|
319
406
|
end
|
320
407
|
|
321
408
|
it "should not base64 encode if not configured to do so" do
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
409
|
+
connector.instance_variable_set("@base64", false)
|
410
|
+
connector.expects(:target_for).returns({:name => "test", :headers => {}})
|
411
|
+
connection.expects(:publish).with("test", "msg", {})
|
412
|
+
msg.expects(:base64_encode!).never
|
326
413
|
|
327
|
-
|
414
|
+
connector.publish(msg)
|
328
415
|
end
|
329
416
|
|
330
417
|
it "should publish the correct message to the correct target with msgheaders" do
|
331
|
-
|
332
|
-
|
418
|
+
connection.expects(:publish).with("test", "msg", {}).once
|
419
|
+
connector.expects(:target_for).returns({:name => "test", :headers => {}})
|
333
420
|
|
334
|
-
|
421
|
+
connector.publish(msg)
|
335
422
|
end
|
336
423
|
|
337
424
|
it "should publish direct messages based on discovered_hosts" do
|
@@ -345,94 +432,90 @@ module MCollective
|
|
345
432
|
msg.stubs(:ttl).returns(60)
|
346
433
|
msg.expects(:discovered_hosts).returns(["one", "two"])
|
347
434
|
|
348
|
-
|
349
|
-
|
435
|
+
connection.expects(:publish).with('/exchange/mcollective_directed/one', 'msg', {'reply-to' => '/temp-queue/mcollective_reply_agent', 'expiration' => '70000'})
|
436
|
+
connection.expects(:publish).with('/exchange/mcollective_directed/two', 'msg', {'reply-to' => '/temp-queue/mcollective_reply_agent', 'expiration' => '70000'})
|
350
437
|
|
351
|
-
|
438
|
+
connector.publish(msg)
|
352
439
|
end
|
353
440
|
end
|
354
441
|
|
355
442
|
describe "#subscribe" do
|
356
443
|
it "should handle duplicate subscription errors" do
|
357
|
-
|
444
|
+
connection.expects(:subscribe).raises(::Stomp::Error::DuplicateSubscription)
|
358
445
|
Log.expects(:error).with(regexp_matches(/already had a matching subscription, ignoring/))
|
359
|
-
|
446
|
+
connector.subscribe("test", :broadcast, "mcollective")
|
360
447
|
end
|
361
448
|
|
362
449
|
it "should use the make_target correctly" do
|
363
|
-
|
364
|
-
|
450
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}})
|
451
|
+
connector.subscribe("test", :broadcast, "mcollective")
|
365
452
|
end
|
366
453
|
|
367
454
|
it "should check for existing subscriptions" do
|
368
|
-
|
369
|
-
|
370
|
-
|
455
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
456
|
+
subscription.expects("include?").with("rspec").returns(false)
|
457
|
+
connection.expects(:subscribe).never
|
371
458
|
|
372
|
-
|
459
|
+
connector.subscribe("test", :broadcast, "mcollective")
|
373
460
|
end
|
374
461
|
|
375
462
|
it "should subscribe to the middleware" do
|
376
|
-
|
377
|
-
|
378
|
-
|
463
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
464
|
+
connection.expects(:subscribe).with("test", {}, "rspec")
|
465
|
+
connector.subscribe("test", :broadcast, "mcollective")
|
379
466
|
end
|
380
467
|
|
381
468
|
it "should add to the list of subscriptions" do
|
382
|
-
|
383
|
-
|
384
|
-
|
469
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
470
|
+
subscription.expects("<<").with("rspec")
|
471
|
+
connector.subscribe("test", :broadcast, "mcollective")
|
385
472
|
end
|
386
473
|
|
387
474
|
it "should not normally subscribe to :reply messages" do
|
388
|
-
|
389
|
-
|
475
|
+
connection.expects(:subscribe).never
|
476
|
+
connector.subscribe("test", :reply, "mcollective")
|
390
477
|
end
|
391
478
|
|
392
479
|
it "should subscribe to :reply messages when use_reply_exchange is set" do
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
})
|
397
|
-
@connection.expects(:subscribe).with("test", {}, "rspec").once
|
480
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_reply_exchange", false).returns(true)
|
481
|
+
connector.expects("make_target").with("test", :reply, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
482
|
+
connection.expects(:subscribe).with("test", {}, "rspec").once
|
398
483
|
|
399
|
-
|
484
|
+
connector.subscribe("test", :reply, "mcollective")
|
400
485
|
end
|
401
486
|
end
|
402
487
|
|
403
488
|
describe "#unsubscribe" do
|
404
489
|
it "should use make_target correctly" do
|
405
|
-
|
406
|
-
|
490
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}})
|
491
|
+
connector.unsubscribe("test", :broadcast, "mcollective")
|
407
492
|
end
|
408
493
|
|
409
494
|
it "should unsubscribe from the target" do
|
410
|
-
|
411
|
-
|
495
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
496
|
+
connection.expects(:unsubscribe).with("test", {}, "rspec").once
|
412
497
|
|
413
|
-
|
498
|
+
connector.unsubscribe("test", :broadcast, "mcollective")
|
414
499
|
end
|
415
500
|
|
416
501
|
it "should delete the source from subscriptions" do
|
417
|
-
|
418
|
-
|
502
|
+
connector.expects("make_target").with("test", :broadcast, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
503
|
+
subscription.expects(:delete).with("rspec").once
|
419
504
|
|
420
|
-
|
505
|
+
connector.unsubscribe("test", :broadcast, "mcollective")
|
421
506
|
end
|
422
507
|
|
423
508
|
it "should not normally unsubscribe from :reply messages" do
|
424
|
-
|
425
|
-
|
509
|
+
connection.expects(:unsubscribe).never
|
510
|
+
connector.unsubscribe("test", :reply, "mcollective")
|
426
511
|
end
|
427
512
|
|
428
513
|
it "should unsubscribe from :reply messages when use_reply_exchange is set" do
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
})
|
433
|
-
@connection.expects(:unsubscribe).with("test", {}, "rspec").once
|
514
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_reply_exchange", false).returns(true)
|
515
|
+
connector.expects("make_target").with("test", :reply, "mcollective").returns({:name => "test", :headers => {}, :id => "rspec"})
|
516
|
+
connection.expects(:unsubscribe).with("test", {}, "rspec").once
|
434
517
|
|
435
|
-
|
518
|
+
connector.unsubscribe("test", :reply, "mcollective")
|
436
519
|
end
|
437
520
|
end
|
438
521
|
|
@@ -447,7 +530,7 @@ module MCollective
|
|
447
530
|
|
448
531
|
message.expects(:request).returns(request)
|
449
532
|
|
450
|
-
|
533
|
+
connector.target_for(message).should == {:name => "foo", :headers => {"expiration" => "70000"}, :id => ""}
|
451
534
|
end
|
452
535
|
|
453
536
|
it "should create new request targets" do
|
@@ -458,8 +541,8 @@ module MCollective
|
|
458
541
|
message.expects(:reply_to).returns("/topic/rspec")
|
459
542
|
message.expects(:ttl).returns(60)
|
460
543
|
|
461
|
-
|
462
|
-
|
544
|
+
connector.expects(:make_target).with("rspecagent", :request, "mcollective", "/topic/rspec", nil).returns({:name => "", :headers => {}, :id => nil})
|
545
|
+
connector.target_for(message)
|
463
546
|
end
|
464
547
|
|
465
548
|
it "should support direct requests" do
|
@@ -470,8 +553,8 @@ module MCollective
|
|
470
553
|
message.expects(:reply_to).returns("/topic/rspec")
|
471
554
|
message.expects(:ttl).returns(60)
|
472
555
|
|
473
|
-
|
474
|
-
|
556
|
+
connector.expects(:make_target).with("rspecagent", :direct_request, "mcollective", "/topic/rspec", nil).returns({:name => "", :headers => {}, :id => nil})
|
557
|
+
connector.target_for(message)
|
475
558
|
end
|
476
559
|
|
477
560
|
it "should fail for unknown message types" do
|
@@ -479,16 +562,16 @@ module MCollective
|
|
479
562
|
message.stubs(:type).returns(:fail)
|
480
563
|
|
481
564
|
expect {
|
482
|
-
|
565
|
+
connector.target_for(message)
|
483
566
|
}.to raise_error("Don't now how to create a target for message type fail")
|
484
567
|
end
|
485
568
|
end
|
486
569
|
|
487
570
|
describe "#disconnect" do
|
488
571
|
it "should disconnect from the stomp connection" do
|
489
|
-
|
490
|
-
|
491
|
-
|
572
|
+
connection.expects(:disconnect)
|
573
|
+
connector.disconnect
|
574
|
+
connector.connection.should == nil
|
492
575
|
end
|
493
576
|
end
|
494
577
|
|
@@ -496,32 +579,32 @@ module MCollective
|
|
496
579
|
context 'rabbitmq.use_reply_exchange' do
|
497
580
|
context 'default (false)' do
|
498
581
|
it "should create correct targets" do
|
499
|
-
|
582
|
+
connector.make_target("test", :reply, "mcollective").should eq({
|
500
583
|
:name => "/temp-queue/mcollective_reply_test",
|
501
584
|
:headers => {},
|
502
585
|
:id => "mcollective_test_replies",
|
503
586
|
})
|
504
|
-
|
587
|
+
connector.make_target("test", :broadcast, "mcollective").should eq({
|
505
588
|
:name => "/exchange/mcollective_broadcast/test",
|
506
589
|
:headers => { "reply-to" => "/temp-queue/mcollective_reply_test" },
|
507
590
|
:id => "mcollective_broadcast_test"
|
508
591
|
})
|
509
|
-
|
592
|
+
connector.make_target("test", :request, "mcollective").should eq({
|
510
593
|
:name => "/exchange/mcollective_broadcast/test",
|
511
594
|
:headers => { "reply-to" => "/temp-queue/mcollective_reply_test" },
|
512
595
|
:id => "mcollective_broadcast_test",
|
513
596
|
})
|
514
|
-
|
597
|
+
connector.make_target("test", :direct_request, "mcollective", nil, "rspec").should eq({
|
515
598
|
:headers => { "reply-to" => "/temp-queue/mcollective_reply_test" },
|
516
599
|
:name => "/exchange/mcollective_directed/rspec",
|
517
600
|
:id => nil
|
518
601
|
})
|
519
|
-
|
602
|
+
connector.make_target("test", :directed, "mcollective").should eq({
|
520
603
|
:name => "/exchange/mcollective_directed/rspec",
|
521
604
|
:headers => {},
|
522
605
|
:id => "mcollective_rspec_directed_to_identity",
|
523
606
|
})
|
524
|
-
|
607
|
+
connector.make_target("test", :request, "mcollective", "/topic/rspec", "rspec").should eq({
|
525
608
|
:headers => { "reply-to" => "/topic/rspec" },
|
526
609
|
:name => "/exchange/mcollective_broadcast/test",
|
527
610
|
:id => "mcollective_broadcast_test",
|
@@ -531,38 +614,36 @@ module MCollective
|
|
531
614
|
|
532
615
|
context 'true' do
|
533
616
|
before :each do
|
534
|
-
|
535
|
-
'rabbitmq.use_reply_exchange' => '1',
|
536
|
-
})
|
617
|
+
Rabbitmq.any_instance.stubs(:get_bool_option).with("rabbitmq.use_reply_exchange", false).returns(true)
|
537
618
|
end
|
538
619
|
|
539
620
|
it "should create correct targets" do
|
540
|
-
|
621
|
+
connector.make_target("test", :reply, "mcollective").should eq({
|
541
622
|
:name => "/exchange/mcollective_reply/rspec_#{$$}",
|
542
623
|
:headers => {},
|
543
624
|
:id => "mcollective_test_replies",
|
544
625
|
})
|
545
|
-
|
626
|
+
connector.make_target("test", :broadcast, "mcollective").should eq({
|
546
627
|
:name => "/exchange/mcollective_broadcast/test",
|
547
628
|
:headers => { "reply-to" => "/exchange/mcollective_reply/rspec_#{$$}" },
|
548
629
|
:id => "mcollective_broadcast_test"
|
549
630
|
})
|
550
|
-
|
631
|
+
connector.make_target("test", :request, "mcollective").should eq({
|
551
632
|
:name => "/exchange/mcollective_broadcast/test",
|
552
633
|
:headers => { "reply-to" => "/exchange/mcollective_reply/rspec_#{$$}" },
|
553
634
|
:id => "mcollective_broadcast_test",
|
554
635
|
})
|
555
|
-
|
636
|
+
connector.make_target("test", :direct_request, "mcollective", nil, "rspec").should eq({
|
556
637
|
:headers => { "reply-to" => "/exchange/mcollective_reply/rspec_#{$$}" },
|
557
638
|
:name => "/exchange/mcollective_directed/rspec",
|
558
639
|
:id => nil
|
559
640
|
})
|
560
|
-
|
641
|
+
connector.make_target("test", :directed, "mcollective").should eq({
|
561
642
|
:name => "/exchange/mcollective_directed/rspec",
|
562
643
|
:headers => {},
|
563
644
|
:id => "mcollective_rspec_directed_to_identity",
|
564
645
|
})
|
565
|
-
|
646
|
+
connector.make_target("test", :request, "mcollective", "/topic/rspec", "rspec").should eq({
|
566
647
|
:headers => { "reply-to" => "/topic/rspec" },
|
567
648
|
:name => "/exchange/mcollective_broadcast/test",
|
568
649
|
:id => "mcollective_broadcast_test",
|
@@ -573,13 +654,13 @@ module MCollective
|
|
573
654
|
|
574
655
|
it "should raise an error for unknown collectives" do
|
575
656
|
expect {
|
576
|
-
|
657
|
+
connector.make_target("test", :broadcast, "foo")
|
577
658
|
}.to raise_error("Unknown collective 'foo' known collectives are 'mcollective'")
|
578
659
|
end
|
579
660
|
|
580
661
|
it "should raise an error for unknown types" do
|
581
662
|
expect {
|
582
|
-
|
663
|
+
connector.make_target("test", :test, "mcollective")
|
583
664
|
}.to raise_error("Unknown target type test")
|
584
665
|
end
|
585
666
|
end
|
@@ -589,56 +670,68 @@ module MCollective
|
|
589
670
|
it "should return the environment variable if set" do
|
590
671
|
ENV["test"] = "rspec_env_test"
|
591
672
|
|
592
|
-
|
673
|
+
connector.get_env_or_option("test", nil, nil).should == "rspec_env_test"
|
593
674
|
|
594
675
|
ENV.delete("test")
|
595
676
|
end
|
596
677
|
|
597
678
|
it "should return the config option if set" do
|
598
|
-
|
599
|
-
|
679
|
+
config.expects(:pluginconf).returns({"test" => "rspec_test"}).twice
|
680
|
+
connector.get_env_or_option("test", "test", "test").should == "rspec_test"
|
600
681
|
end
|
601
682
|
|
602
683
|
it "should return default if nothing else matched" do
|
603
|
-
|
604
|
-
|
684
|
+
config.expects(:pluginconf).returns({}).once
|
685
|
+
connector.get_env_or_option("test", "test", "test").should == "test"
|
605
686
|
end
|
606
687
|
|
607
688
|
it "should raise an error if no default is supplied" do
|
608
|
-
|
689
|
+
config.expects(:pluginconf).returns({}).once
|
609
690
|
|
610
691
|
expect {
|
611
|
-
|
692
|
+
connector.get_env_or_option("test", "test")
|
612
693
|
}.to raise_error("No test environment or plugin.test configuration option given")
|
613
694
|
end
|
614
695
|
end
|
615
696
|
|
616
697
|
describe "#get_option" do
|
698
|
+
before :each do
|
699
|
+
# realize the connector let so that we can unstub it
|
700
|
+
connector
|
701
|
+
Rabbitmq.any_instance.unstub(:get_option)
|
702
|
+
end
|
703
|
+
|
617
704
|
it "should return the config option if set" do
|
618
|
-
|
619
|
-
|
705
|
+
config.expects(:pluginconf).returns({"test" => "rspec_test"}).twice
|
706
|
+
connector.get_option("test").should == "rspec_test"
|
620
707
|
end
|
621
708
|
|
622
709
|
it "should return default option was not found" do
|
623
|
-
|
624
|
-
|
710
|
+
config.expects(:pluginconf).returns({}).once
|
711
|
+
connector.get_option("test", "test").should == "test"
|
625
712
|
end
|
626
713
|
|
627
714
|
it "should raise an error if no default is supplied" do
|
628
|
-
|
715
|
+
config.expects(:pluginconf).returns({}).once
|
629
716
|
|
630
717
|
expect {
|
631
|
-
|
718
|
+
connector.get_option("test")
|
632
719
|
}.to raise_error("No plugin.test configuration option given")
|
633
720
|
end
|
634
721
|
end
|
635
722
|
|
636
723
|
describe "#get_bool_option" do
|
724
|
+
before :each do
|
725
|
+
# realize the connector let so that we can unstub it
|
726
|
+
connector
|
727
|
+
Rabbitmq.any_instance.unstub(:get_bool_option)
|
728
|
+
end
|
729
|
+
|
637
730
|
it "should use Util::str_to_bool to translate a boolean value found in the config" do
|
638
|
-
|
731
|
+
config.expects(:pluginconf).returns({"rspec" => "true"})
|
639
732
|
Util.expects(:str_to_bool).with("true").returns(true)
|
640
733
|
|
641
|
-
|
734
|
+
connector.get_bool_option("rspec", "true").should be_true
|
642
735
|
end
|
643
736
|
end
|
644
737
|
end
|