mongo 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +50 -69
- data/docs/CREDITS.md +4 -0
- data/docs/HISTORY.md +9 -0
- data/docs/REPLICA_SETS.md +8 -10
- data/lib/mongo.rb +3 -1
- data/lib/mongo/collection.rb +2 -1
- data/lib/mongo/connection.rb +146 -314
- data/lib/mongo/db.rb +6 -2
- data/lib/mongo/gridfs/grid.rb +1 -1
- data/lib/mongo/gridfs/grid_io.rb +29 -5
- data/lib/mongo/repl_set_connection.rb +290 -0
- data/lib/mongo/util/pool.rb +6 -8
- data/lib/mongo/util/uri_parser.rb +71 -0
- data/mongo.gemspec +1 -2
- data/test/collection_test.rb +9 -7
- data/test/connection_test.rb +0 -6
- data/test/grid_file_system_test.rb +2 -2
- data/test/grid_io_test.rb +33 -1
- data/test/grid_test.rb +36 -6
- data/test/replica_sets/connect_test.rb +59 -21
- data/test/replica_sets/count_test.rb +9 -7
- data/test/replica_sets/insert_test.rb +11 -9
- data/test/replica_sets/pooled_insert_test.rb +12 -13
- data/test/replica_sets/query_secondaries.rb +48 -8
- data/test/replica_sets/query_test.rb +10 -9
- data/test/replica_sets/replication_ack_test.rb +15 -22
- data/test/replica_sets/rs_test_helper.rb +29 -0
- data/test/test_helper.rb +13 -20
- data/test/threading/{test_threading_large_pool.rb → threading_with_large_pool_test.rb} +1 -1
- data/test/tools/repl_set_manager.rb +241 -0
- data/test/tools/test.rb +13 -0
- data/test/unit/connection_test.rb +3 -85
- data/test/unit/repl_set_connection_test.rb +82 -0
- metadata +19 -21
- data/test/replica_pairs/count_test.rb +0 -34
- data/test/replica_pairs/insert_test.rb +0 -50
- data/test/replica_pairs/pooled_insert_test.rb +0 -54
- data/test/replica_pairs/query_test.rb +0 -39
- data/test/replica_sets/node_type_test.rb +0 -42
- data/test/rs.rb +0 -24
@@ -1,7 +1,5 @@
|
|
1
1
|
$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
-
require '
|
3
|
-
require 'test/unit'
|
4
|
-
require './test/test_helper'
|
2
|
+
require './test/replica_sets/rs_test_helper'
|
5
3
|
|
6
4
|
# NOTE: This test expects a replica set of three nodes to be running
|
7
5
|
# on the local host.
|
@@ -9,24 +7,27 @@ class ReplicaSetQueryTest < Test::Unit::TestCase
|
|
9
7
|
include Mongo
|
10
8
|
|
11
9
|
def setup
|
12
|
-
@conn =
|
10
|
+
@conn = ReplSetConnection.new([RS.host, RS.ports[0]])
|
13
11
|
@db = @conn.db(MONGO_TEST_DB)
|
14
12
|
@db.drop_collection("test-sets")
|
15
13
|
@coll = @db.collection("test-sets")
|
16
14
|
end
|
17
15
|
|
16
|
+
def teardown
|
17
|
+
RS.restart_killed_nodes
|
18
|
+
end
|
19
|
+
|
18
20
|
def test_query
|
19
|
-
@coll.save({:a => 20})
|
20
|
-
@coll.save({:a => 30})
|
21
|
-
@coll.save({:a => 40})
|
21
|
+
@coll.save({:a => 20}, :safe => {:w => 3})
|
22
|
+
@coll.save({:a => 30}, :safe => {:w => 3})
|
23
|
+
@coll.save({:a => 40}, :safe => {:w => 3})
|
22
24
|
results = []
|
23
25
|
@coll.find.each {|r| results << r}
|
24
26
|
[20, 30, 40].each do |a|
|
25
27
|
assert results.any? {|r| r['a'] == a}, "Could not find record for a => #{a}"
|
26
28
|
end
|
27
29
|
|
28
|
-
|
29
|
-
gets
|
30
|
+
RS.kill_primary
|
30
31
|
|
31
32
|
results = []
|
32
33
|
rescue_connection_failure do
|
@@ -1,20 +1,17 @@
|
|
1
1
|
$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
-
require '
|
3
|
-
require 'test/unit'
|
4
|
-
require './test/test_helper'
|
2
|
+
require './test/replica_sets/rs_test_helper'
|
5
3
|
|
6
4
|
# NOTE: This test expects a replica set of three nodes to be running on local host.
|
7
5
|
class ReplicaSetAckTest < Test::Unit::TestCase
|
8
6
|
include Mongo
|
9
7
|
|
10
8
|
def setup
|
11
|
-
|
9
|
+
RS.ensure_up
|
12
10
|
|
13
|
-
|
14
|
-
@slaves = @conn.nodes - master
|
11
|
+
@conn = ReplSetConnection.new([RS.host, RS.ports[0]])
|
15
12
|
|
16
|
-
@slave1 =
|
17
|
-
|
13
|
+
@slave1 = Connection.new(@conn.secondary_pools[0].host,
|
14
|
+
@conn.secondary_pools[0].port, :slave_ok => true)
|
18
15
|
|
19
16
|
@db = @conn.db(MONGO_TEST_DB)
|
20
17
|
@db.drop_collection("test-sets")
|
@@ -22,47 +19,43 @@ class ReplicaSetAckTest < Test::Unit::TestCase
|
|
22
19
|
end
|
23
20
|
|
24
21
|
def test_safe_mode_with_w_failure
|
25
|
-
assert_raise_error OperationFailure, "
|
22
|
+
assert_raise_error OperationFailure, "timeout" do
|
26
23
|
@col.insert({:foo => 1}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
|
27
24
|
end
|
28
|
-
assert_raise_error OperationFailure, "
|
25
|
+
assert_raise_error OperationFailure, "timeout" do
|
29
26
|
@col.update({:foo => 1}, {:foo => 2}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
|
30
27
|
end
|
31
|
-
assert_raise_error OperationFailure, "
|
28
|
+
assert_raise_error OperationFailure, "timeout" do
|
32
29
|
@col.remove({:foo => 2}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
|
33
30
|
end
|
34
31
|
end
|
35
32
|
|
36
33
|
def test_safe_mode_replication_ack
|
37
|
-
@col.insert({:baz => "bar"}, :safe => {:w =>
|
34
|
+
@col.insert({:baz => "bar"}, :safe => {:w => 2, :wtimeout => 5000})
|
38
35
|
|
39
|
-
assert @col.insert({:foo => "0" *
|
36
|
+
assert @col.insert({:foo => "0" * 5000}, :safe => {:w => 2, :wtimeout => 5000})
|
40
37
|
assert_equal 2, @slave1[MONGO_TEST_DB]["test-sets"].count
|
41
|
-
assert_equal 2, @slave2[MONGO_TEST_DB]["test-sets"].count
|
42
38
|
|
43
|
-
|
44
|
-
assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 3, :wtimeout => 1000})
|
39
|
+
assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 2, :wtimeout => 5000})
|
45
40
|
assert @slave1[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
|
46
|
-
assert @slave2[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
|
47
41
|
|
48
|
-
assert @col.remove({}, :safe => {:w =>
|
42
|
+
assert @col.remove({}, :safe => {:w => 2, :wtimeout => 5000})
|
49
43
|
assert_equal 0, @slave1[MONGO_TEST_DB]["test-sets"].count
|
50
|
-
assert_equal 0, @slave2[MONGO_TEST_DB]["test-sets"].count
|
51
44
|
end
|
52
45
|
|
53
46
|
def test_last_error_responses
|
54
47
|
20.times { @col.insert({:baz => "bar"}) }
|
55
|
-
response = @db.get_last_error(:w =>
|
48
|
+
response = @db.get_last_error(:w => 2, :wtimeout => 5000)
|
56
49
|
assert response['ok'] == 1
|
57
50
|
assert response['lastOp']
|
58
51
|
|
59
52
|
@col.update({}, {:baz => "foo"}, :multi => true)
|
60
|
-
response = @db.get_last_error(:w =>
|
53
|
+
response = @db.get_last_error(:w => 2, :wtimeout => 5000)
|
61
54
|
assert response['ok'] == 1
|
62
55
|
assert response['lastOp']
|
63
56
|
|
64
57
|
@col.remove({})
|
65
|
-
response = @db.get_last_error(:w =>
|
58
|
+
response = @db.get_last_error(:w => 2, :wtimeout => 5000)
|
66
59
|
assert response['ok'] == 1
|
67
60
|
assert response['n'] == 20
|
68
61
|
assert response['lastOp']
|
@@ -0,0 +1,29 @@
|
|
1
|
+
$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
require './test/test_helper'
|
3
|
+
require './test/tools/repl_set_manager'
|
4
|
+
|
5
|
+
unless defined? RS
|
6
|
+
RS = ReplSetManager.new
|
7
|
+
RS.start_set
|
8
|
+
end
|
9
|
+
|
10
|
+
class Test::Unit::TestCase
|
11
|
+
|
12
|
+
# Generic code for rescuing connection failures and retrying operations.
|
13
|
+
# This could be combined with some timeout functionality.
|
14
|
+
def rescue_connection_failure(max_retries=60)
|
15
|
+
success = false
|
16
|
+
tries = 0
|
17
|
+
while !success && tries < max_retries
|
18
|
+
begin
|
19
|
+
yield
|
20
|
+
success = true
|
21
|
+
rescue Mongo::ConnectionFailure
|
22
|
+
puts "Rescue attempt #{tries}\n"
|
23
|
+
tries += 1
|
24
|
+
sleep(1)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -22,7 +22,15 @@ end
|
|
22
22
|
require 'bson_ext/cbson' if !(RUBY_PLATFORM =~ /java/) && ENV['C_EXT']
|
23
23
|
|
24
24
|
unless defined? MONGO_TEST_DB
|
25
|
-
MONGO_TEST_DB = '
|
25
|
+
MONGO_TEST_DB = 'ruby-test-db'
|
26
|
+
end
|
27
|
+
|
28
|
+
unless defined? TEST_PORT
|
29
|
+
TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::Connection::DEFAULT_PORT
|
30
|
+
end
|
31
|
+
|
32
|
+
unless defined? TEST_HOST
|
33
|
+
TEST_HOST = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
|
26
34
|
end
|
27
35
|
|
28
36
|
class Test::Unit::TestCase
|
@@ -30,8 +38,7 @@ class Test::Unit::TestCase
|
|
30
38
|
include BSON
|
31
39
|
|
32
40
|
def self.standard_connection(options={})
|
33
|
-
Connection.new(
|
34
|
-
ENV['MONGO_RUBY_DRIVER_PORT'] || Connection::DEFAULT_PORT, options)
|
41
|
+
Connection.new(TEST_HOST, TEST_PORT, options)
|
35
42
|
end
|
36
43
|
|
37
44
|
def standard_connection(options={})
|
@@ -43,11 +50,11 @@ class Test::Unit::TestCase
|
|
43
50
|
end
|
44
51
|
|
45
52
|
def self.mongo_host
|
46
|
-
|
53
|
+
TEST_HOST
|
47
54
|
end
|
48
55
|
|
49
56
|
def self.mongo_port
|
50
|
-
|
57
|
+
TEST_PORT
|
51
58
|
end
|
52
59
|
|
53
60
|
def host_port
|
@@ -62,21 +69,7 @@ class Test::Unit::TestCase
|
|
62
69
|
self.class.mongo_port
|
63
70
|
end
|
64
71
|
|
65
|
-
|
66
|
-
# This could be combined with some timeout functionality.
|
67
|
-
def rescue_connection_failure
|
68
|
-
success = false
|
69
|
-
while !success
|
70
|
-
begin
|
71
|
-
yield
|
72
|
-
success = true
|
73
|
-
rescue Mongo::ConnectionFailure
|
74
|
-
puts "Rescuing"
|
75
|
-
sleep(1)
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
72
|
+
|
80
73
|
def assert_raise_error(klass, message)
|
81
74
|
begin
|
82
75
|
yield
|
@@ -0,0 +1,241 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
STDOUT.sync = true
|
4
|
+
|
5
|
+
unless defined? Mongo
|
6
|
+
require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
|
7
|
+
end
|
8
|
+
|
9
|
+
class ReplSetManager
|
10
|
+
|
11
|
+
attr_accessor :host, :start_port, :ports, :name, :mongods
|
12
|
+
|
13
|
+
def initialize(opts={})
|
14
|
+
@start_port = opts[:start_port] || 30000
|
15
|
+
@ports = []
|
16
|
+
@name = opts[:name] || 'replica-set-foo'
|
17
|
+
@host = opts[:host] || 'localhost'
|
18
|
+
@retries = opts[:retries] || 60
|
19
|
+
@config = {"_id" => @name, "members" => []}
|
20
|
+
@path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
|
21
|
+
|
22
|
+
@arbiter_count = opts[:arbiter_count] || 2
|
23
|
+
@secondary_count = opts[:secondary_count] || 1
|
24
|
+
@passive_count = opts[:passive_count] || 1
|
25
|
+
@primary_count = 1
|
26
|
+
|
27
|
+
@count = @primary_count + @passive_count + @arbiter_count + @secondary_count
|
28
|
+
if @count > 7
|
29
|
+
raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
|
30
|
+
end
|
31
|
+
|
32
|
+
@mongods = {}
|
33
|
+
end
|
34
|
+
|
35
|
+
def start_set
|
36
|
+
puts "** Starting a replica set with #{@count} nodes"
|
37
|
+
|
38
|
+
system("killall mongod")
|
39
|
+
|
40
|
+
n = 0
|
41
|
+
(@primary_count + @secondary_count).times do |n|
|
42
|
+
init_node(n)
|
43
|
+
n += 1
|
44
|
+
end
|
45
|
+
|
46
|
+
@passive_count.times do
|
47
|
+
init_node(n) do |attrs|
|
48
|
+
attrs['priority'] = 0
|
49
|
+
end
|
50
|
+
n += 1
|
51
|
+
end
|
52
|
+
|
53
|
+
@arbiter_count.times do
|
54
|
+
init_node(n) do |attrs|
|
55
|
+
attrs['arbiterOnly'] = true
|
56
|
+
end
|
57
|
+
n += 1
|
58
|
+
end
|
59
|
+
|
60
|
+
initiate
|
61
|
+
ensure_up
|
62
|
+
end
|
63
|
+
|
64
|
+
def init_node(n)
|
65
|
+
@mongods[n] ||= {}
|
66
|
+
port = @start_port + n
|
67
|
+
@ports << port
|
68
|
+
@mongods[n]['port'] = port
|
69
|
+
@mongods[n]['db_path'] = get_path("rs-#{port}")
|
70
|
+
@mongods[n]['log_path'] = get_path("log-#{port}")
|
71
|
+
system("rm -rf #{@mongods[n]['db_path']}")
|
72
|
+
system("mkdir -p #{@mongods[n]['db_path']}")
|
73
|
+
|
74
|
+
@mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
|
75
|
+
" --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
|
76
|
+
|
77
|
+
start(n)
|
78
|
+
|
79
|
+
member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
|
80
|
+
|
81
|
+
if block_given?
|
82
|
+
custom_attrs = {}
|
83
|
+
yield custom_attrs
|
84
|
+
member.merge!(custom_attrs)
|
85
|
+
@mongods[n].merge!(custom_attrs)
|
86
|
+
end
|
87
|
+
|
88
|
+
@config['members'] << member
|
89
|
+
end
|
90
|
+
|
91
|
+
def kill(node)
|
92
|
+
pid = @mongods[node]['pid']
|
93
|
+
puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
|
94
|
+
system("kill -2 #{@mongods[node]['pid']}")
|
95
|
+
@mongods[node]['up'] = false
|
96
|
+
sleep(1)
|
97
|
+
end
|
98
|
+
|
99
|
+
def kill_primary
|
100
|
+
node = get_node_with_state(1)
|
101
|
+
kill(node)
|
102
|
+
return node
|
103
|
+
end
|
104
|
+
|
105
|
+
# Note that we have to rescue a connection failure
|
106
|
+
# when we run the StepDown command because that
|
107
|
+
# command will close the connection.
|
108
|
+
def step_down_primary
|
109
|
+
primary = get_node_with_state(1)
|
110
|
+
con = get_connection(primary)
|
111
|
+
begin
|
112
|
+
con['admin'].command({'replSetStepDown' => 90})
|
113
|
+
rescue Mongo::ConnectionFailure
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def kill_secondary
|
118
|
+
node = get_node_with_state(2)
|
119
|
+
kill(node)
|
120
|
+
return node
|
121
|
+
end
|
122
|
+
|
123
|
+
def restart_killed_nodes
|
124
|
+
nodes = @mongods.keys.select do |key|
|
125
|
+
@mongods[key]['up'] == false
|
126
|
+
end
|
127
|
+
|
128
|
+
nodes.each do |node|
|
129
|
+
start(node)
|
130
|
+
end
|
131
|
+
|
132
|
+
ensure_up
|
133
|
+
end
|
134
|
+
|
135
|
+
def get_node_from_port(port)
|
136
|
+
@mongods.keys.detect { |key| @mongods[key]['port'] == port }
|
137
|
+
end
|
138
|
+
|
139
|
+
def start(node)
|
140
|
+
system(@mongods[node]['start'])
|
141
|
+
@mongods[node]['up'] = true
|
142
|
+
sleep(0.5)
|
143
|
+
@mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
|
144
|
+
end
|
145
|
+
alias :restart :start
|
146
|
+
|
147
|
+
def ensure_up
|
148
|
+
print "** Ensuring members are up..."
|
149
|
+
|
150
|
+
attempt do
|
151
|
+
con = get_connection
|
152
|
+
status = con['admin'].command({'replSetGetStatus' => 1})
|
153
|
+
print "."
|
154
|
+
if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
|
155
|
+
status['members'].any? { |m| m['state'] == 1 }
|
156
|
+
print "all members up!\n\n"
|
157
|
+
return status
|
158
|
+
else
|
159
|
+
raise Mongo::OperationFailure
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def primary
|
165
|
+
nodes = get_all_host_pairs_with_state(1)
|
166
|
+
nodes.empty? ? nil : nodes[0]
|
167
|
+
end
|
168
|
+
|
169
|
+
def secondaries
|
170
|
+
get_all_host_pairs_with_state(2)
|
171
|
+
end
|
172
|
+
|
173
|
+
def arbiters
|
174
|
+
get_all_host_pairs_with_state(7)
|
175
|
+
end
|
176
|
+
|
177
|
+
private
|
178
|
+
|
179
|
+
def initiate
|
180
|
+
con = get_connection
|
181
|
+
|
182
|
+
attempt do
|
183
|
+
con['admin'].command({'replSetInitiate' => @config})
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
def get_node_with_state(state)
|
188
|
+
status = ensure_up
|
189
|
+
node = status['members'].detect {|m| m['state'] == state}
|
190
|
+
if node
|
191
|
+
host_port = node['name'].split(':')
|
192
|
+
port = host_port[1] ? host_port[1].to_i : 27017
|
193
|
+
key = @mongods.keys.detect {|key| @mongods[key]['port'] == port}
|
194
|
+
return key
|
195
|
+
else
|
196
|
+
return false
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def get_all_host_pairs_with_state(state)
|
201
|
+
status = ensure_up
|
202
|
+
nodes = status['members'].select {|m| m['state'] == state}
|
203
|
+
nodes.map do |node|
|
204
|
+
host_port = node['name'].split(':')
|
205
|
+
port = host_port[1] ? host_port[1].to_i : 27017
|
206
|
+
[host, port]
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def get_connection(node=nil)
|
211
|
+
con = attempt do
|
212
|
+
if !node
|
213
|
+
node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
|
214
|
+
end
|
215
|
+
con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
|
216
|
+
end
|
217
|
+
|
218
|
+
return con
|
219
|
+
end
|
220
|
+
|
221
|
+
def get_path(name)
|
222
|
+
File.join(@path, name)
|
223
|
+
end
|
224
|
+
|
225
|
+
def attempt
|
226
|
+
raise "No block given!" unless block_given?
|
227
|
+
count = 0
|
228
|
+
|
229
|
+
while count < @retries do
|
230
|
+
begin
|
231
|
+
return yield
|
232
|
+
rescue Mongo::OperationFailure, Mongo::ConnectionFailure
|
233
|
+
sleep(1)
|
234
|
+
count += 1
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
raise exception
|
239
|
+
end
|
240
|
+
|
241
|
+
end
|