mongo 1.4.0 → 1.5.0.rc0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/docs/HISTORY.md +15 -0
- data/docs/REPLICA_SETS.md +19 -7
- data/lib/mongo.rb +1 -0
- data/lib/mongo/collection.rb +1 -1
- data/lib/mongo/connection.rb +29 -351
- data/lib/mongo/cursor.rb +88 -6
- data/lib/mongo/gridfs/grid.rb +4 -2
- data/lib/mongo/gridfs/grid_file_system.rb +4 -2
- data/lib/mongo/networking.rb +345 -0
- data/lib/mongo/repl_set_connection.rb +236 -191
- data/lib/mongo/util/core_ext.rb +45 -0
- data/lib/mongo/util/logging.rb +5 -0
- data/lib/mongo/util/node.rb +6 -4
- data/lib/mongo/util/pool.rb +73 -26
- data/lib/mongo/util/pool_manager.rb +100 -30
- data/lib/mongo/util/uri_parser.rb +29 -21
- data/lib/mongo/version.rb +1 -1
- data/test/bson/binary_test.rb +6 -8
- data/test/bson/bson_test.rb +1 -0
- data/test/bson/ordered_hash_test.rb +2 -0
- data/test/bson/test_helper.rb +0 -17
- data/test/collection_test.rb +22 -0
- data/test/connection_test.rb +1 -1
- data/test/cursor_test.rb +3 -3
- data/test/load/thin/load.rb +4 -7
- data/test/replica_sets/basic_test.rb +46 -0
- data/test/replica_sets/connect_test.rb +35 -58
- data/test/replica_sets/count_test.rb +15 -6
- data/test/replica_sets/insert_test.rb +6 -7
- data/test/replica_sets/query_test.rb +4 -6
- data/test/replica_sets/read_preference_test.rb +112 -8
- data/test/replica_sets/refresh_test.rb +66 -36
- data/test/replica_sets/refresh_with_threads_test.rb +55 -0
- data/test/replica_sets/replication_ack_test.rb +3 -6
- data/test/replica_sets/rs_test_helper.rb +12 -6
- data/test/replica_sets/threading_test.rb +111 -0
- data/test/test_helper.rb +9 -2
- data/test/threading_test.rb +14 -6
- data/test/tools/repl_set_manager.rb +55 -40
- data/test/unit/collection_test.rb +2 -1
- data/test/unit/connection_test.rb +8 -8
- data/test/unit/grid_test.rb +4 -2
- data/test/unit/pool_manager_test.rb +1 -0
- data/test/unit/read_test.rb +17 -5
- data/test/uri_test.rb +9 -4
- metadata +13 -28
- data/test/replica_sets/connection_string_test.rb +0 -29
- data/test/replica_sets/pooled_insert_test.rb +0 -58
- data/test/replica_sets/query_secondaries.rb +0 -109
@@ -2,12 +2,19 @@ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
2
2
|
require './test/test_helper'
|
3
3
|
require './test/tools/repl_set_manager'
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
module ReplicaSetTest
|
6
|
+
|
7
|
+
def self.rs
|
8
|
+
unless defined?(@rs)
|
9
|
+
@rs = ReplSetManager.new
|
10
|
+
@rs.start_set
|
11
|
+
end
|
12
|
+
@rs
|
13
|
+
end
|
9
14
|
|
10
|
-
|
15
|
+
def rs
|
16
|
+
ReplicaSetTest.rs
|
17
|
+
end
|
11
18
|
|
12
19
|
# Generic code for rescuing connection failures and retrying operations.
|
13
20
|
# This could be combined with some timeout functionality.
|
@@ -23,5 +30,4 @@ class Test::Unit::TestCase
|
|
23
30
|
retry
|
24
31
|
end
|
25
32
|
end
|
26
|
-
|
27
33
|
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
require './test/replica_sets/rs_test_helper'
|
3
|
+
|
4
|
+
class ReplicaSetThreadingTest < Test::Unit::TestCase
|
5
|
+
include ReplicaSetTest
|
6
|
+
|
7
|
+
def setup_safe_data
|
8
|
+
@conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
|
9
|
+
:pool_size => 100)
|
10
|
+
@db = @conn[MONGO_TEST_DB]
|
11
|
+
@coll = @db.collection('thread-test-collection')
|
12
|
+
@db.drop_collection('duplicate')
|
13
|
+
@db.drop_collection('unique')
|
14
|
+
@duplicate = @db.collection('duplicate')
|
15
|
+
@unique = @db.collection('unique')
|
16
|
+
|
17
|
+
@duplicate.insert("test" => "insert")
|
18
|
+
@duplicate.insert("test" => "update")
|
19
|
+
@unique.insert("test" => "insert")
|
20
|
+
@unique.insert("test" => "update")
|
21
|
+
@unique.create_index("test", :unique => true)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_safe_update
|
25
|
+
setup_safe_data
|
26
|
+
times = []
|
27
|
+
threads = []
|
28
|
+
100.times do |i|
|
29
|
+
threads[i] = Thread.new do
|
30
|
+
10.times do
|
31
|
+
if i % 2 == 0
|
32
|
+
assert_raise Mongo::OperationFailure do
|
33
|
+
t1 = Time.now
|
34
|
+
@unique.update({"test" => "insert"},
|
35
|
+
{"$set" => {"test" => "update"}}, :safe => true)
|
36
|
+
times << Time.now - t1
|
37
|
+
end
|
38
|
+
else
|
39
|
+
t1 = Time.now
|
40
|
+
@duplicate.update({"test" => "insert"},
|
41
|
+
{"$set" => {"test" => "update"}}, :safe => true)
|
42
|
+
times << Time.now - t1
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
100.times do |i|
|
49
|
+
threads[i].join
|
50
|
+
end
|
51
|
+
@conn.close
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_safe_insert
|
55
|
+
setup_safe_data
|
56
|
+
threads = []
|
57
|
+
100.times do |i|
|
58
|
+
threads[i] = Thread.new do
|
59
|
+
if i % 2 == 0
|
60
|
+
assert_raise Mongo::OperationFailure do
|
61
|
+
@unique.insert({"test" => "insert"}, :safe => true)
|
62
|
+
end
|
63
|
+
else
|
64
|
+
@duplicate.insert({"test" => "insert"}, :safe => true)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
100.times do |i|
|
70
|
+
threads[i].join
|
71
|
+
end
|
72
|
+
@conn.close
|
73
|
+
end
|
74
|
+
|
75
|
+
def setup_replica_set_connection
|
76
|
+
@conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
|
77
|
+
:pool_size => 100, :auto_refresh => :sync,
|
78
|
+
:refresh_interval => 2)
|
79
|
+
@db = @conn[MONGO_TEST_DB]
|
80
|
+
@coll = @db.collection('thread-test-collection')
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_threading_with_queries
|
84
|
+
setup_replica_set_connection
|
85
|
+
@coll.drop
|
86
|
+
@coll = @db.collection('thread-test-collection')
|
87
|
+
|
88
|
+
1000.times do |i|
|
89
|
+
@coll.insert("x" => i)
|
90
|
+
end
|
91
|
+
|
92
|
+
threads = []
|
93
|
+
|
94
|
+
10.times do |i|
|
95
|
+
threads[i] = Thread.new do
|
96
|
+
100.times do
|
97
|
+
sum = 0
|
98
|
+
@coll.find().each do |document|
|
99
|
+
sum += document["x"]
|
100
|
+
end
|
101
|
+
assert_equal 499500, sum
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
10.times do |i|
|
107
|
+
threads[i].join
|
108
|
+
end
|
109
|
+
@conn.close
|
110
|
+
end
|
111
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -82,6 +82,7 @@ class Test::Unit::TestCase
|
|
82
82
|
socket = Object.new
|
83
83
|
socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
|
84
84
|
socket.stubs(:close)
|
85
|
+
socket.stubs(:closed?)
|
85
86
|
socket
|
86
87
|
end
|
87
88
|
|
@@ -93,8 +94,14 @@ class Test::Unit::TestCase
|
|
93
94
|
begin
|
94
95
|
yield
|
95
96
|
rescue => e
|
96
|
-
|
97
|
-
|
97
|
+
if klass.to_s != e.class.to_s
|
98
|
+
flunk "Expected exception class #{klass} but got #{e.class}.\n #{e.backtrace}"
|
99
|
+
end
|
100
|
+
|
101
|
+
if !e.message.include?(message)
|
102
|
+
p e.backtrace
|
103
|
+
flunk "#{e.message} does not include #{message}.\n#{e.backtrace}"
|
104
|
+
end
|
98
105
|
else
|
99
106
|
flunk "Expected assertion #{klass} but none was raised."
|
100
107
|
end
|
data/test/threading_test.rb
CHANGED
@@ -4,7 +4,8 @@ class TestThreading < Test::Unit::TestCase
|
|
4
4
|
|
5
5
|
include Mongo
|
6
6
|
|
7
|
-
@@
|
7
|
+
@@con = standard_connection(:pool_size => 10, :timeout => 30)
|
8
|
+
@@db = @@con[MONGO_TEST_DB]
|
8
9
|
@@coll = @@db.collection('thread-test-collection')
|
9
10
|
|
10
11
|
def set_up_safe_data
|
@@ -21,16 +22,23 @@ class TestThreading < Test::Unit::TestCase
|
|
21
22
|
end
|
22
23
|
|
23
24
|
def test_safe_update
|
25
|
+
times = []
|
24
26
|
set_up_safe_data
|
25
27
|
threads = []
|
26
28
|
100.times do |i|
|
27
29
|
threads[i] = Thread.new do
|
28
|
-
|
29
|
-
|
30
|
-
|
30
|
+
100.times do
|
31
|
+
if i % 2 == 0
|
32
|
+
assert_raise Mongo::OperationFailure do
|
33
|
+
t1 = Time.now
|
34
|
+
@unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
|
35
|
+
times << Time.now - t1
|
36
|
+
end
|
37
|
+
else
|
38
|
+
t1 = Time.now
|
39
|
+
@duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
|
40
|
+
times << Time.now - t1
|
31
41
|
end
|
32
|
-
else
|
33
|
-
@duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
|
34
42
|
end
|
35
43
|
end
|
36
44
|
end
|
@@ -20,13 +20,14 @@ class ReplSetManager
|
|
20
20
|
@retries = opts[:retries] || 30
|
21
21
|
@config = {"_id" => @name, "members" => []}
|
22
22
|
@durable = opts.fetch(:durable, false)
|
23
|
+
@smallfiles = opts.fetch(:smallfiles, true)
|
23
24
|
@path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
|
24
|
-
@oplog_size = opts.fetch(:oplog_size,
|
25
|
+
@oplog_size = opts.fetch(:oplog_size, 16)
|
25
26
|
@tags = [{"dc" => "ny", "rack" => "a", "db" => "main"},
|
26
27
|
{"dc" => "ny", "rack" => "b", "db" => "main"},
|
27
28
|
{"dc" => "sf", "rack" => "a", "db" => "main"}]
|
28
29
|
|
29
|
-
@arbiter_count = opts[:arbiter_count] ||
|
30
|
+
@arbiter_count = opts[:arbiter_count] || 0
|
30
31
|
@secondary_count = opts[:secondary_count] || 2
|
31
32
|
@passive_count = opts[:passive_count] || 0
|
32
33
|
@primary_count = 1
|
@@ -43,19 +44,10 @@ class ReplSetManager
|
|
43
44
|
end
|
44
45
|
|
45
46
|
def start_set
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
if con && ensure_up(1, con)
|
52
|
-
should_start = false
|
53
|
-
puts "** Replica set already started."
|
54
|
-
else
|
55
|
-
should_start = true
|
56
|
-
system("killall mongod")
|
57
|
-
puts "** Starting a replica set with #{@count} nodes"
|
58
|
-
end
|
47
|
+
system("killall mongod")
|
48
|
+
sleep(1)
|
49
|
+
should_start = true
|
50
|
+
puts "** Starting a replica set with #{@count} nodes"
|
59
51
|
|
60
52
|
n = 0
|
61
53
|
(@primary_count + @secondary_count).times do
|
@@ -81,15 +73,8 @@ class ReplSetManager
|
|
81
73
|
n += 1
|
82
74
|
end
|
83
75
|
|
84
|
-
|
85
|
-
|
86
|
-
v['up'] = true
|
87
|
-
v['pid'] = File.open(File.join(v['db_path'], 'mongod.lock')).read.strip
|
88
|
-
end
|
89
|
-
else
|
90
|
-
initiate
|
91
|
-
ensure_up
|
92
|
-
end
|
76
|
+
initiate
|
77
|
+
ensure_up
|
93
78
|
end
|
94
79
|
|
95
80
|
def cleanup_set
|
@@ -142,6 +127,7 @@ class ReplSetManager
|
|
142
127
|
@mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
|
143
128
|
"--oplogSize #{@oplog_size} #{journal_switch} --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
|
144
129
|
@mongods[n]['start'] += " --dur" if @durable
|
130
|
+
@mongods[n]['start'] += " --smallfiles" if @smallfiles
|
145
131
|
@mongods[n]['start']
|
146
132
|
end
|
147
133
|
|
@@ -157,7 +143,7 @@ class ReplSetManager
|
|
157
143
|
@config['version'] = config['version'] + 1
|
158
144
|
|
159
145
|
begin
|
160
|
-
|
146
|
+
con['admin'].command({'replSetReconfig' => @config})
|
161
147
|
rescue Mongo::ConnectionFailure
|
162
148
|
end
|
163
149
|
|
@@ -169,8 +155,8 @@ class ReplSetManager
|
|
169
155
|
def add_node(n=nil)
|
170
156
|
primary = get_node_with_state(1)
|
171
157
|
con = get_connection(primary)
|
172
|
-
init_node(n || @mongods.length)
|
173
158
|
|
159
|
+
init_node(n || @mongods.length)
|
174
160
|
config = con['local']['system.replset'].find_one
|
175
161
|
@config['version'] = config['version'] + 1
|
176
162
|
|
@@ -187,9 +173,8 @@ class ReplSetManager
|
|
187
173
|
def kill(node, signal=2)
|
188
174
|
pid = @mongods[node]['pid']
|
189
175
|
puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
|
190
|
-
system("kill
|
176
|
+
system("kill #{pid}")
|
191
177
|
@mongods[node]['up'] = false
|
192
|
-
sleep(1)
|
193
178
|
end
|
194
179
|
|
195
180
|
def kill_primary(signal=2)
|
@@ -254,21 +239,49 @@ class ReplSetManager
|
|
254
239
|
print "** Ensuring members are up..."
|
255
240
|
|
256
241
|
attempt(n) do
|
257
|
-
con = connection || get_connection
|
258
|
-
status = con['admin'].command({'replSetGetStatus' => 1})
|
259
242
|
print "."
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
con.close
|
265
|
-
return status
|
266
|
-
else
|
243
|
+
con = connection || get_connection
|
244
|
+
begin
|
245
|
+
status = con['admin'].command({:replSetGetStatus => 1})
|
246
|
+
rescue Mongo::OperationFailure => ex
|
267
247
|
con.close
|
268
|
-
raise
|
248
|
+
raise ex
|
269
249
|
end
|
270
|
-
|
271
|
-
|
250
|
+
if status['members'].all? { |m| m['health'] == 1 &&
|
251
|
+
[1, 2, 7].include?(m['state']) } &&
|
252
|
+
status['members'].any? { |m| m['state'] == 1 }
|
253
|
+
|
254
|
+
connections = []
|
255
|
+
states = []
|
256
|
+
status['members'].each do |member|
|
257
|
+
begin
|
258
|
+
host, port = member['name'].split(':')
|
259
|
+
port = port.to_i
|
260
|
+
conn = Mongo::Connection.new(host, port, :slave_ok => true)
|
261
|
+
connections << conn
|
262
|
+
state = conn['admin'].command({:ismaster => 1})
|
263
|
+
states << state
|
264
|
+
rescue Mongo::ConnectionFailure
|
265
|
+
connections.each {|c| c.close }
|
266
|
+
con.close
|
267
|
+
raise Mongo::OperationFailure
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
if states.any? {|s| s['ismaster']}
|
272
|
+
print "all members up!\n\n"
|
273
|
+
connections.each {|c| c.close }
|
274
|
+
con.close
|
275
|
+
return status
|
276
|
+
else
|
277
|
+
con.close
|
278
|
+
raise Mongo::OperationFailure
|
279
|
+
end
|
280
|
+
else
|
281
|
+
con.close
|
282
|
+
raise Mongo::OperationFailure
|
283
|
+
end
|
284
|
+
end
|
272
285
|
return false
|
273
286
|
end
|
274
287
|
|
@@ -298,9 +311,11 @@ class ReplSetManager
|
|
298
311
|
private
|
299
312
|
|
300
313
|
def initiate
|
314
|
+
puts "Initiating replica set..."
|
301
315
|
con = get_connection
|
302
316
|
|
303
317
|
attempt do
|
318
|
+
con.object_id
|
304
319
|
con['admin'].command({'replSetInitiate' => @config})
|
305
320
|
end
|
306
321
|
|
@@ -5,7 +5,7 @@ class CollectionTest < Test::Unit::TestCase
|
|
5
5
|
context "Basic operations: " do
|
6
6
|
setup do
|
7
7
|
@logger = mock()
|
8
|
-
@logger.expects(:
|
8
|
+
@logger.expects(:warn)
|
9
9
|
end
|
10
10
|
|
11
11
|
should "send update message" do
|
@@ -36,6 +36,7 @@ class CollectionTest < Test::Unit::TestCase
|
|
36
36
|
@conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
|
37
37
|
@db = @conn['testing']
|
38
38
|
@coll = @db.collection('books')
|
39
|
+
@conn.expects(:checkout_writer).returns(mock())
|
39
40
|
@conn.expects(:receive_message).with do |op, msg, log, sock|
|
40
41
|
op == 2004
|
41
42
|
end.returns([[], 0, 0])
|
@@ -40,6 +40,12 @@ class ConnectionTest < Test::Unit::TestCase
|
|
40
40
|
assert_equal [host_name, 27017], @conn.host_to_try
|
41
41
|
end
|
42
42
|
|
43
|
+
should "allow db without username and password" do
|
44
|
+
host_name = "foo.bar-12345.org"
|
45
|
+
@conn = Connection.from_uri("mongodb://#{host_name}/foo", :connect => false)
|
46
|
+
assert_equal [host_name, 27017], @conn.host_to_try
|
47
|
+
end
|
48
|
+
|
43
49
|
should "parse a uri with a hyphen & underscore in the username or password" do
|
44
50
|
@conn = Connection.from_uri("mongodb://hyphen-user_name:p-s_s@localhost:27017/db", :connect => false)
|
45
51
|
assert_equal ['localhost', 27017], @conn.host_to_try
|
@@ -65,16 +71,10 @@ class ConnectionTest < Test::Unit::TestCase
|
|
65
71
|
assert_raise MongoArgumentError do
|
66
72
|
Connection.from_uri("mongodb://localhost:abc", :connect => false)
|
67
73
|
end
|
68
|
-
|
69
|
-
assert_raise MongoArgumentError do
|
70
|
-
Connection.from_uri("mongodb://localhost:27017, my.db.com:27018, ", :connect => false)
|
71
|
-
end
|
72
74
|
end
|
73
75
|
|
74
|
-
should "require all of username, password
|
75
|
-
|
76
|
-
Connection.from_uri("mongodb://localhost/db", :connect => false)
|
77
|
-
end
|
76
|
+
should "require all of username, if password and db are specified" do
|
77
|
+
assert Connection.from_uri("mongodb://kyle:jones@localhost/db", :connect => false)
|
78
78
|
|
79
79
|
assert_raise MongoArgumentError do
|
80
80
|
Connection.from_uri("mongodb://kyle:password@localhost", :connect => false)
|
data/test/unit/grid_test.rb
CHANGED
@@ -19,7 +19,8 @@ class GridTest < Test::Unit::TestCase
|
|
19
19
|
|
20
20
|
context "Grid classe with standard connections" do
|
21
21
|
setup do
|
22
|
-
@conn.expects(:
|
22
|
+
@conn.expects(:class).returns(Connection)
|
23
|
+
@conn.expects(:read_primary?).returns(true)
|
23
24
|
end
|
24
25
|
|
25
26
|
should "create indexes for Grid" do
|
@@ -36,7 +37,8 @@ class GridTest < Test::Unit::TestCase
|
|
36
37
|
|
37
38
|
context "Grid classes with slave connection" do
|
38
39
|
setup do
|
39
|
-
@conn.expects(:
|
40
|
+
@conn.expects(:class).twice.returns(Connection)
|
41
|
+
@conn.expects(:read_primary?).returns(false)
|
40
42
|
end
|
41
43
|
|
42
44
|
should "not create indexes for Grid" do
|
@@ -12,6 +12,7 @@ class PoolManagerTest < Test::Unit::TestCase
|
|
12
12
|
@connection = stub("Connection")
|
13
13
|
@connection.stubs(:connect_timeout).returns(5000)
|
14
14
|
@connection.stubs(:pool_size).returns(2)
|
15
|
+
@connection.stubs(:pool_timeout).returns(100)
|
15
16
|
@connection.stubs(:socket_class).returns(TCPSocket)
|
16
17
|
@connection.stubs(:[]).returns(@db)
|
17
18
|
|