beetle 0.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +18 -8
- data/beetle.gemspec +37 -121
- data/bin/beetle +9 -0
- data/examples/README.rdoc +0 -2
- data/examples/rpc.rb +3 -2
- data/ext/mkrf_conf.rb +19 -0
- data/lib/beetle.rb +2 -2
- data/lib/beetle/base.rb +1 -8
- data/lib/beetle/client.rb +16 -14
- data/lib/beetle/commands.rb +30 -0
- data/lib/beetle/commands/configuration_client.rb +73 -0
- data/lib/beetle/commands/configuration_server.rb +85 -0
- data/lib/beetle/configuration.rb +70 -7
- data/lib/beetle/deduplication_store.rb +50 -38
- data/lib/beetle/handler.rb +2 -5
- data/lib/beetle/logging.rb +7 -0
- data/lib/beetle/message.rb +11 -13
- data/lib/beetle/publisher.rb +12 -4
- data/lib/beetle/r_c.rb +2 -1
- data/lib/beetle/redis_configuration_client.rb +136 -0
- data/lib/beetle/redis_configuration_server.rb +301 -0
- data/lib/beetle/redis_ext.rb +79 -0
- data/lib/beetle/redis_master_file.rb +35 -0
- data/lib/beetle/redis_server_info.rb +65 -0
- data/lib/beetle/subscriber.rb +4 -1
- data/test/beetle/configuration_test.rb +14 -2
- data/test/beetle/deduplication_store_test.rb +61 -43
- data/test/beetle/message_test.rb +28 -4
- data/test/beetle/publisher_test.rb +17 -3
- data/test/beetle/redis_configuration_client_test.rb +97 -0
- data/test/beetle/redis_configuration_server_test.rb +278 -0
- data/test/beetle/redis_ext_test.rb +71 -0
- data/test/beetle/redis_master_file_test.rb +39 -0
- data/test/test_helper.rb +13 -1
- metadata +162 -69
- data/.gitignore +0 -5
- data/MIT-LICENSE +0 -20
- data/Rakefile +0 -114
- data/TODO +0 -7
- data/etc/redis-master.conf +0 -189
- data/etc/redis-slave.conf +0 -189
- data/examples/redis_failover.rb +0 -65
- data/script/start_rabbit +0 -29
- data/snafu.rb +0 -55
- data/test/beetle.yml +0 -81
- data/test/beetle/bla.rb +0 -0
- data/tmp/master/.gitignore +0 -2
- data/tmp/slave/.gitignore +0 -3
@@ -0,0 +1,79 @@
|
|
1
|
+
# Redis convenience and compatibility layer
|
2
|
+
class Redis #:nodoc:
|
3
|
+
def self.from_server_string(server_string, options = {})
|
4
|
+
host, port = server_string.split(':')
|
5
|
+
options = {:host => host, :port => port}.update(options)
|
6
|
+
new(options)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Redis 2 removed some useful methods. add them back.
|
10
|
+
def host; @client.host; end
|
11
|
+
def port; @client.port; end
|
12
|
+
def server; "#{host}:#{port}"; end
|
13
|
+
|
14
|
+
def master!
|
15
|
+
slaveof("no", "one")
|
16
|
+
end
|
17
|
+
|
18
|
+
def slave_of!(host, port)
|
19
|
+
slaveof(host, port)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Redis 2 tries to establish a connection on inspect. this is evil!
|
23
|
+
def inspect
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
def info_with_rescue
|
28
|
+
info
|
29
|
+
rescue Exception
|
30
|
+
{}
|
31
|
+
end
|
32
|
+
|
33
|
+
def available?
|
34
|
+
info_with_rescue != {}
|
35
|
+
end
|
36
|
+
|
37
|
+
def role
|
38
|
+
info_with_rescue["role"] || "unknown"
|
39
|
+
end
|
40
|
+
|
41
|
+
def master?
|
42
|
+
role == "master"
|
43
|
+
end
|
44
|
+
|
45
|
+
def slave?
|
46
|
+
role == "slave"
|
47
|
+
end
|
48
|
+
|
49
|
+
def slave_of?(host, port)
|
50
|
+
info = info_with_rescue
|
51
|
+
info["role"] == "slave" && info["master_host"] == host && info["master_port"] == port.to_s
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class Redis::Client #:nodoc:
|
56
|
+
protected
|
57
|
+
def connect_to(host, port)
|
58
|
+
if @timeout != 0 and Redis::Timer
|
59
|
+
begin
|
60
|
+
Redis::Timer.timeout(@timeout){ @sock = TCPSocket.new(host, port) }
|
61
|
+
rescue Timeout::Error
|
62
|
+
@sock = nil
|
63
|
+
raise Timeout::Error, "Timeout connecting to the server"
|
64
|
+
end
|
65
|
+
else
|
66
|
+
@sock = TCPSocket.new(host, port)
|
67
|
+
end
|
68
|
+
|
69
|
+
@sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
|
70
|
+
|
71
|
+
# If the timeout is set we set the low level socket options in order
|
72
|
+
# to make sure a blocking read will return after the specified number
|
73
|
+
# of seconds. This hack is from memcached ruby client.
|
74
|
+
self.timeout = @timeout
|
75
|
+
|
76
|
+
rescue Errno::ECONNREFUSED
|
77
|
+
raise Errno::ECONNREFUSED, "Unable to connect to Redis on #{host}:#{port}"
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Beetle
|
2
|
+
module RedisMasterFile #:nodoc:
|
3
|
+
private
|
4
|
+
def master_file_exists?
|
5
|
+
File.exist?(master_file)
|
6
|
+
end
|
7
|
+
|
8
|
+
def redis_master_from_master_file
|
9
|
+
redis.find(read_redis_master_file)
|
10
|
+
end
|
11
|
+
|
12
|
+
def clear_redis_master_file
|
13
|
+
write_redis_master_file("")
|
14
|
+
end
|
15
|
+
|
16
|
+
def read_redis_master_file
|
17
|
+
File.read(master_file).chomp
|
18
|
+
end
|
19
|
+
|
20
|
+
def write_redis_master_file(redis_server_string)
|
21
|
+
logger.warn "Writing '#{redis_server_string}' to redis master file '#{master_file}'"
|
22
|
+
File.open(master_file, "w"){|f| f.puts redis_server_string }
|
23
|
+
end
|
24
|
+
|
25
|
+
def master_file
|
26
|
+
config.redis_server
|
27
|
+
end
|
28
|
+
|
29
|
+
def verify_redis_master_file_string
|
30
|
+
if master_file =~ /^[0-9a-z.]+:[0-9]+$/
|
31
|
+
raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Beetle
|
2
|
+
# class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
|
3
|
+
class RedisServerInfo
|
4
|
+
include Logging
|
5
|
+
|
6
|
+
def initialize(config, options) #:nodoc:
|
7
|
+
@config = config
|
8
|
+
@options = options
|
9
|
+
reset
|
10
|
+
end
|
11
|
+
|
12
|
+
# all configured redis servers
|
13
|
+
def instances
|
14
|
+
@instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
|
15
|
+
end
|
16
|
+
|
17
|
+
# fetches the server from the insatnces whith the given <tt>server</tt> string
|
18
|
+
def find(server)
|
19
|
+
instances.find{|r| r.server == server}
|
20
|
+
end
|
21
|
+
|
22
|
+
# refresh connectivity/role information
|
23
|
+
def refresh
|
24
|
+
logger.debug "Updating redis server info"
|
25
|
+
reset
|
26
|
+
instances.each {|r| @server_info[r.role] << r}
|
27
|
+
end
|
28
|
+
|
29
|
+
# subset of instances which are masters
|
30
|
+
def masters
|
31
|
+
@server_info["master"]
|
32
|
+
end
|
33
|
+
|
34
|
+
# subset of instances which are slaves
|
35
|
+
def slaves
|
36
|
+
@server_info["slave"]
|
37
|
+
end
|
38
|
+
|
39
|
+
# subset of instances which are not reachable
|
40
|
+
def unknowns
|
41
|
+
@server_info["unknown"]
|
42
|
+
end
|
43
|
+
|
44
|
+
# subset of instances which are set up as slaves of the given <tt>master</tt>
|
45
|
+
def slaves_of(master)
|
46
|
+
slaves.select{|r| r.slave_of?(master.host, master.port)}
|
47
|
+
end
|
48
|
+
|
49
|
+
# determine a master if we have one master and all other servers are slaves
|
50
|
+
def auto_detect_master
|
51
|
+
return nil unless master_and_slaves_reachable?
|
52
|
+
masters.first
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def master_and_slaves_reachable?
|
58
|
+
masters.size == 1 && slaves.size == instances.size - 1
|
59
|
+
end
|
60
|
+
|
61
|
+
def reset
|
62
|
+
@server_info = Hash.new {|h,k| h[k]= []}
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/beetle/subscriber.rb
CHANGED
@@ -10,7 +10,7 @@ module Beetle
|
|
10
10
|
@mqs = {}
|
11
11
|
end
|
12
12
|
|
13
|
-
# the client calls this method to
|
13
|
+
# the client calls this method to subscribe to all queues on all servers which have
|
14
14
|
# handlers registered for the given list of messages. this method does the following
|
15
15
|
# things:
|
16
16
|
#
|
@@ -105,6 +105,9 @@ module Beetle
|
|
105
105
|
sleep 1
|
106
106
|
mq(server).recover
|
107
107
|
elsif reply_to = header.properties[:reply_to]
|
108
|
+
# require 'ruby-debug'
|
109
|
+
# Debugger.start
|
110
|
+
# debugger
|
108
111
|
status = result == Beetle::RC::OK ? "OK" : "FAILED"
|
109
112
|
exchange = MQ::Exchange.new(mq(server), :direct, "", :key => reply_to)
|
110
113
|
exchange.publish(m.handler_result.to_s, :headers => {:status => status})
|
@@ -1,5 +1,17 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
2
|
-
|
2
|
+
require 'tempfile'
|
3
3
|
|
4
4
|
module Beetle
|
5
|
-
|
5
|
+
class ConfigurationTest < Test::Unit::TestCase
|
6
|
+
test "should load it's settings from a config file if that file exists" do
|
7
|
+
config = Configuration.new
|
8
|
+
old_value = config.gc_threshold
|
9
|
+
new_value = old_value + 1
|
10
|
+
config_file_content = "gc_threshold: #{new_value}\n"
|
11
|
+
IO.expects(:read).returns(config_file_content)
|
12
|
+
|
13
|
+
config.config_file = "some/path/to/a/file"
|
14
|
+
assert_equal new_value, config.gc_threshold
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -14,75 +14,93 @@ module Beetle
|
|
14
14
|
end
|
15
15
|
|
16
16
|
test "msetnx returns 0 or 1" do
|
17
|
-
assert_equal 1, @r.msetnx("a"
|
17
|
+
assert_equal 1, @r.msetnx("a", 1, "b", 2)
|
18
18
|
assert_equal "1", @r.get("a")
|
19
19
|
assert_equal "2", @r.get("b")
|
20
|
-
assert_equal 0, @r.msetnx("a"
|
20
|
+
assert_equal 0, @r.msetnx("a", 3, "b", 4)
|
21
21
|
assert_equal "1", @r.get("a")
|
22
22
|
assert_equal "2", @r.get("b")
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
class
|
26
|
+
class RedisServerStringTest < Test::Unit::TestCase
|
27
|
+
def setup
|
28
|
+
@original_redis_server = Beetle.config.redis_server
|
29
|
+
@store = DeduplicationStore.new
|
30
|
+
@server_string = "my_test_host_from_file:9999"
|
31
|
+
Beetle.config.redis_server = @server_string
|
32
|
+
end
|
33
|
+
|
34
|
+
def teardown
|
35
|
+
Beetle.config.redis_server = @original_redis_server
|
36
|
+
end
|
37
|
+
|
38
|
+
test "redis should match the redis server string" do
|
39
|
+
assert_equal @server_string, @store.redis.server
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
class RedisServerFileTest < Test::Unit::TestCase
|
27
44
|
def setup
|
28
|
-
@
|
45
|
+
@original_redis_server = Beetle.config.redis_server
|
46
|
+
@store = DeduplicationStore.new
|
47
|
+
@server_string = "my_test_host_from_file:6379"
|
48
|
+
Beetle.config.redis_server = redis_test_master_file(@server_string)
|
49
|
+
end
|
50
|
+
|
51
|
+
def teardown
|
52
|
+
Beetle.config.redis_server = @original_redis_server
|
53
|
+
end
|
54
|
+
|
55
|
+
test "redis should match the redis master file" do
|
56
|
+
assert_equal @server_string, @store.redis.server
|
29
57
|
end
|
30
58
|
|
31
|
-
test "redis
|
32
|
-
|
33
|
-
|
59
|
+
test "redis should be nil if the redis master file is blank" do
|
60
|
+
redis_test_master_file("")
|
61
|
+
assert_nil @store.redis
|
34
62
|
end
|
35
63
|
|
36
|
-
test "
|
37
|
-
|
38
|
-
|
39
|
-
instances.second.expects(:info).returns(:role => "master")
|
40
|
-
assert_equal instances.second, @store.redis
|
64
|
+
test "should keep using the current redis if the redis master file hasn't changed since the last request" do
|
65
|
+
@store.expects(:read_master_file).once.returns("localhost:1")
|
66
|
+
2.times { @store.redis }
|
41
67
|
end
|
42
68
|
|
43
|
-
test "
|
44
|
-
|
45
|
-
|
46
|
-
instances.second.expects(:info).returns(:role => "master")
|
47
|
-
assert_equal instances.second, @store.redis
|
69
|
+
test "should return nil if the master file doesn't exist" do
|
70
|
+
Beetle.config.redis_server = "/tmp/__i_don_not_exist__.txt"
|
71
|
+
assert_equal nil, @store.redis_master_from_master_file
|
48
72
|
end
|
49
73
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
74
|
+
private
|
75
|
+
def redis_test_master_file(server_string)
|
76
|
+
path = File.expand_path("../../../tmp/redis-master-for-unit-tests", __FILE__)
|
77
|
+
File.open(path, "w"){|f| f.puts server_string}
|
78
|
+
path
|
55
79
|
end
|
80
|
+
end
|
56
81
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
assert_raises(TwoRedisMasters) { @store.find_redis_master }
|
82
|
+
class RedisFailoverTest < Test::Unit::TestCase
|
83
|
+
def setup
|
84
|
+
@store = DeduplicationStore.new
|
85
|
+
Beetle.config.expects(:redis_failover_timeout).returns(1)
|
62
86
|
end
|
63
87
|
|
64
88
|
test "a redis operation protected with a redis failover block should succeed if it can find a new master" do
|
65
|
-
|
89
|
+
redis1 = stub("redis 1")
|
90
|
+
redis2 = stub("redis 2")
|
66
91
|
s = sequence("redis accesses")
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
instances.first.expects(:info).raises("disconnected").in_sequence(s)
|
72
|
-
instances.second.expects(:info).returns(:role => "master").in_sequence(s)
|
73
|
-
instances.second.expects(:get).with("foo:x").returns("42").in_sequence(s)
|
92
|
+
@store.expects(:redis).returns(redis1).in_sequence(s)
|
93
|
+
redis1.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
|
94
|
+
@store.expects(:redis).returns(redis2).in_sequence(s)
|
95
|
+
redis2.expects(:get).with("foo:x").returns("42").in_sequence(s)
|
74
96
|
assert_equal("42", @store.get("foo", "x"))
|
75
97
|
end
|
76
98
|
|
77
99
|
test "a redis operation protected with a redis failover block should fail if it cannot find a new master" do
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
instances.first.stubs(:get).with("foo:x").raises("disconnected")
|
83
|
-
instances.first.stubs(:info).raises("disconnected")
|
84
|
-
instances.second.stubs(:info).returns(:role => "slave")
|
85
|
-
@store.expects(:sleep).times(119)
|
100
|
+
redis1 = stub()
|
101
|
+
@store.stubs(:redis).returns(redis1)
|
102
|
+
redis1.stubs(:get).with("foo:x").raises("disconnected")
|
103
|
+
@store.stubs(:sleep)
|
86
104
|
assert_raises(NoRedisMaster) { @store.get("foo", "x") }
|
87
105
|
end
|
88
106
|
end
|
data/test/beetle/message_test.rb
CHANGED
@@ -4,6 +4,11 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
|
4
4
|
module Beetle
|
5
5
|
|
6
6
|
class EncodingTest < Test::Unit::TestCase
|
7
|
+
test "an exception during decoding should be stored in the exception attribute" do
|
8
|
+
header = stub_everything("raising header")
|
9
|
+
m = Message.new("queue", header, 'foo')
|
10
|
+
assert_instance_of NoMethodError, m.exception
|
11
|
+
end
|
7
12
|
|
8
13
|
test "a message should encode/decode the message format version correctly" do
|
9
14
|
header = header_with_params({})
|
@@ -371,6 +376,17 @@ module Beetle
|
|
371
376
|
@store.flushdb
|
372
377
|
end
|
373
378
|
|
379
|
+
test "a message with an exception set should not be processed at all, but it should be acked" do
|
380
|
+
header = {}
|
381
|
+
message = Message.new("somequeue", header, 'foo')
|
382
|
+
assert message.exception
|
383
|
+
|
384
|
+
proc = mock("proc")
|
385
|
+
proc.expects(:call).never
|
386
|
+
message.expects(:ack!)
|
387
|
+
assert_equal RC::DecodingError, message.__send__(:process_internal, proc)
|
388
|
+
end
|
389
|
+
|
374
390
|
test "a completed existing message should be just acked and not run the handler" do
|
375
391
|
header = header_with_params({})
|
376
392
|
message = Message.new("somequeue", header, 'foo', :attempts => 2, :store => @store)
|
@@ -379,7 +395,6 @@ module Beetle
|
|
379
395
|
assert message.completed?
|
380
396
|
|
381
397
|
proc = mock("proc")
|
382
|
-
s = sequence("s")
|
383
398
|
header.expects(:ack)
|
384
399
|
proc.expects(:call).never
|
385
400
|
assert_equal RC::OK, message.__send__(:process_internal, proc)
|
@@ -394,7 +409,6 @@ module Beetle
|
|
394
409
|
assert message.delayed?
|
395
410
|
|
396
411
|
proc = mock("proc")
|
397
|
-
s = sequence("s")
|
398
412
|
header.expects(:ack).never
|
399
413
|
proc.expects(:call).never
|
400
414
|
assert_equal RC::Delayed, message.__send__(:process_internal, proc)
|
@@ -412,7 +426,6 @@ module Beetle
|
|
412
426
|
assert !message.timed_out?
|
413
427
|
|
414
428
|
proc = mock("proc")
|
415
|
-
s = sequence("s")
|
416
429
|
header.expects(:ack).never
|
417
430
|
proc.expects(:call).never
|
418
431
|
assert_equal RC::HandlerNotYetTimedOut, message.__send__(:process_internal, proc)
|
@@ -577,7 +590,7 @@ module Beetle
|
|
577
590
|
@store.flushdb
|
578
591
|
end
|
579
592
|
|
580
|
-
test "a handler running longer than the specified timeout should be aborted" do
|
593
|
+
test "a handler running longer than the specified timeout should be aborted (when given a float timeout number)" do
|
581
594
|
header = header_with_params({})
|
582
595
|
header.expects(:ack)
|
583
596
|
message = Message.new("somequeue", header, 'foo', :timeout => 0.1, :attempts => 2, :store => @store)
|
@@ -586,6 +599,17 @@ module Beetle
|
|
586
599
|
result = message.process(handler)
|
587
600
|
assert_equal RC::ExceptionsLimitReached, result
|
588
601
|
end
|
602
|
+
|
603
|
+
test "a handler running longer than the specified timeout should be aborted (when using active_support seconds)" do
|
604
|
+
header = header_with_params({})
|
605
|
+
header.expects(:ack)
|
606
|
+
message = Message.new("somequeue", header, 'foo', :timeout => 1.seconds, :attempts => 2, :store => @store)
|
607
|
+
action = lambda{|*args| while true; end}
|
608
|
+
handler = Handler.create(action)
|
609
|
+
result = message.process(handler)
|
610
|
+
assert_equal RC::ExceptionsLimitReached, result
|
611
|
+
end
|
612
|
+
|
589
613
|
end
|
590
614
|
|
591
615
|
class SettingsTest < Test::Unit::TestCase
|
@@ -284,7 +284,7 @@ module Beetle
|
|
284
284
|
assert_kind_of Time, dead["localhost:2222"]
|
285
285
|
end
|
286
286
|
|
287
|
-
test "recycle_dead_servers should move servers from the dead server hash to the servers list only if
|
287
|
+
test "recycle_dead_servers should move servers from the dead server hash to the servers list only if they have been markd dead for longer than 10 seconds" do
|
288
288
|
@pub.servers = ["a:1", "b:2"]
|
289
289
|
@pub.send(:set_current_server, "a:1")
|
290
290
|
@pub.send(:mark_server_dead)
|
@@ -299,6 +299,19 @@ module Beetle
|
|
299
299
|
assert_equal({}, dead)
|
300
300
|
end
|
301
301
|
|
302
|
+
test "recycle_dead_servers should recycle the server which has been dead for the longest time if all servers are dead " do
|
303
|
+
@pub.servers = ["a:1", "b:2"]
|
304
|
+
@pub.send(:set_current_server, "a:1")
|
305
|
+
@pub.send(:mark_server_dead)
|
306
|
+
@pub.send(:mark_server_dead)
|
307
|
+
assert_equal [], @pub.servers
|
308
|
+
dead = @pub.instance_variable_get("@dead_servers")
|
309
|
+
assert_equal ["a:1", "b:2"], dead.keys.sort
|
310
|
+
@pub.send(:recycle_dead_servers)
|
311
|
+
assert_equal ["b:2"], dead.keys
|
312
|
+
assert_equal ["a:1"], @pub.servers
|
313
|
+
end
|
314
|
+
|
302
315
|
test "select_next_server should cycle through the list of all servers" do
|
303
316
|
@pub.servers = ["a:1", "b:2"]
|
304
317
|
@pub.send(:set_current_server, "a:1")
|
@@ -308,12 +321,13 @@ module Beetle
|
|
308
321
|
assert_equal "a:1", @pub.server
|
309
322
|
end
|
310
323
|
|
311
|
-
test "select_next_server should
|
324
|
+
test "select_next_server should log an error if there are no servers to publish to" do
|
312
325
|
@pub.servers = []
|
313
326
|
logger = mock('logger')
|
314
327
|
logger.expects(:error).returns(true)
|
315
328
|
@pub.expects(:logger).returns(logger)
|
316
|
-
|
329
|
+
@pub.expects(:set_current_server).never
|
330
|
+
@pub.send(:select_next_server)
|
317
331
|
end
|
318
332
|
|
319
333
|
test "stop should shut down all bunnies" do
|