mongo 1.4.0 → 1.5.0.rc0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/docs/HISTORY.md +15 -0
  2. data/docs/REPLICA_SETS.md +19 -7
  3. data/lib/mongo.rb +1 -0
  4. data/lib/mongo/collection.rb +1 -1
  5. data/lib/mongo/connection.rb +29 -351
  6. data/lib/mongo/cursor.rb +88 -6
  7. data/lib/mongo/gridfs/grid.rb +4 -2
  8. data/lib/mongo/gridfs/grid_file_system.rb +4 -2
  9. data/lib/mongo/networking.rb +345 -0
  10. data/lib/mongo/repl_set_connection.rb +236 -191
  11. data/lib/mongo/util/core_ext.rb +45 -0
  12. data/lib/mongo/util/logging.rb +5 -0
  13. data/lib/mongo/util/node.rb +6 -4
  14. data/lib/mongo/util/pool.rb +73 -26
  15. data/lib/mongo/util/pool_manager.rb +100 -30
  16. data/lib/mongo/util/uri_parser.rb +29 -21
  17. data/lib/mongo/version.rb +1 -1
  18. data/test/bson/binary_test.rb +6 -8
  19. data/test/bson/bson_test.rb +1 -0
  20. data/test/bson/ordered_hash_test.rb +2 -0
  21. data/test/bson/test_helper.rb +0 -17
  22. data/test/collection_test.rb +22 -0
  23. data/test/connection_test.rb +1 -1
  24. data/test/cursor_test.rb +3 -3
  25. data/test/load/thin/load.rb +4 -7
  26. data/test/replica_sets/basic_test.rb +46 -0
  27. data/test/replica_sets/connect_test.rb +35 -58
  28. data/test/replica_sets/count_test.rb +15 -6
  29. data/test/replica_sets/insert_test.rb +6 -7
  30. data/test/replica_sets/query_test.rb +4 -6
  31. data/test/replica_sets/read_preference_test.rb +112 -8
  32. data/test/replica_sets/refresh_test.rb +66 -36
  33. data/test/replica_sets/refresh_with_threads_test.rb +55 -0
  34. data/test/replica_sets/replication_ack_test.rb +3 -6
  35. data/test/replica_sets/rs_test_helper.rb +12 -6
  36. data/test/replica_sets/threading_test.rb +111 -0
  37. data/test/test_helper.rb +9 -2
  38. data/test/threading_test.rb +14 -6
  39. data/test/tools/repl_set_manager.rb +55 -40
  40. data/test/unit/collection_test.rb +2 -1
  41. data/test/unit/connection_test.rb +8 -8
  42. data/test/unit/grid_test.rb +4 -2
  43. data/test/unit/pool_manager_test.rb +1 -0
  44. data/test/unit/read_test.rb +17 -5
  45. data/test/uri_test.rb +9 -4
  46. metadata +13 -28
  47. data/test/replica_sets/connection_string_test.rb +0 -29
  48. data/test/replica_sets/pooled_insert_test.rb +0 -58
  49. data/test/replica_sets/query_secondaries.rb +0 -109
@@ -2,12 +2,19 @@ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/test_helper'
3
3
  require './test/tools/repl_set_manager'
4
4
 
5
- unless defined? RS
6
- RS = ReplSetManager.new
7
- RS.start_set
8
- end
5
+ module ReplicaSetTest
6
+
7
+ def self.rs
8
+ unless defined?(@rs)
9
+ @rs = ReplSetManager.new
10
+ @rs.start_set
11
+ end
12
+ @rs
13
+ end
9
14
 
10
- class Test::Unit::TestCase
15
+ def rs
16
+ ReplicaSetTest.rs
17
+ end
11
18
 
12
19
  # Generic code for rescuing connection failures and retrying operations.
13
20
  # This could be combined with some timeout functionality.
@@ -23,5 +30,4 @@ class Test::Unit::TestCase
23
30
  retry
24
31
  end
25
32
  end
26
-
27
33
  end
@@ -0,0 +1,111 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ require './test/replica_sets/rs_test_helper'
3
+
4
+ class ReplicaSetThreadingTest < Test::Unit::TestCase
5
+ include ReplicaSetTest
6
+
7
+ def setup_safe_data
8
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
9
+ :pool_size => 100)
10
+ @db = @conn[MONGO_TEST_DB]
11
+ @coll = @db.collection('thread-test-collection')
12
+ @db.drop_collection('duplicate')
13
+ @db.drop_collection('unique')
14
+ @duplicate = @db.collection('duplicate')
15
+ @unique = @db.collection('unique')
16
+
17
+ @duplicate.insert("test" => "insert")
18
+ @duplicate.insert("test" => "update")
19
+ @unique.insert("test" => "insert")
20
+ @unique.insert("test" => "update")
21
+ @unique.create_index("test", :unique => true)
22
+ end
23
+
24
+ def test_safe_update
25
+ setup_safe_data
26
+ times = []
27
+ threads = []
28
+ 100.times do |i|
29
+ threads[i] = Thread.new do
30
+ 10.times do
31
+ if i % 2 == 0
32
+ assert_raise Mongo::OperationFailure do
33
+ t1 = Time.now
34
+ @unique.update({"test" => "insert"},
35
+ {"$set" => {"test" => "update"}}, :safe => true)
36
+ times << Time.now - t1
37
+ end
38
+ else
39
+ t1 = Time.now
40
+ @duplicate.update({"test" => "insert"},
41
+ {"$set" => {"test" => "update"}}, :safe => true)
42
+ times << Time.now - t1
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ 100.times do |i|
49
+ threads[i].join
50
+ end
51
+ @conn.close
52
+ end
53
+
54
+ def test_safe_insert
55
+ setup_safe_data
56
+ threads = []
57
+ 100.times do |i|
58
+ threads[i] = Thread.new do
59
+ if i % 2 == 0
60
+ assert_raise Mongo::OperationFailure do
61
+ @unique.insert({"test" => "insert"}, :safe => true)
62
+ end
63
+ else
64
+ @duplicate.insert({"test" => "insert"}, :safe => true)
65
+ end
66
+ end
67
+ end
68
+
69
+ 100.times do |i|
70
+ threads[i].join
71
+ end
72
+ @conn.close
73
+ end
74
+
75
+ def setup_replica_set_connection
76
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
77
+ :pool_size => 100, :auto_refresh => :sync,
78
+ :refresh_interval => 2)
79
+ @db = @conn[MONGO_TEST_DB]
80
+ @coll = @db.collection('thread-test-collection')
81
+ end
82
+
83
+ def test_threading_with_queries
84
+ setup_replica_set_connection
85
+ @coll.drop
86
+ @coll = @db.collection('thread-test-collection')
87
+
88
+ 1000.times do |i|
89
+ @coll.insert("x" => i)
90
+ end
91
+
92
+ threads = []
93
+
94
+ 10.times do |i|
95
+ threads[i] = Thread.new do
96
+ 100.times do
97
+ sum = 0
98
+ @coll.find().each do |document|
99
+ sum += document["x"]
100
+ end
101
+ assert_equal 499500, sum
102
+ end
103
+ end
104
+ end
105
+
106
+ 10.times do |i|
107
+ threads[i].join
108
+ end
109
+ @conn.close
110
+ end
111
+ end
@@ -82,6 +82,7 @@ class Test::Unit::TestCase
82
82
  socket = Object.new
83
83
  socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
84
84
  socket.stubs(:close)
85
+ socket.stubs(:closed?)
85
86
  socket
86
87
  end
87
88
 
@@ -93,8 +94,14 @@ class Test::Unit::TestCase
93
94
  begin
94
95
  yield
95
96
  rescue => e
96
- assert_equal klass, e.class
97
- assert e.message.include?(message), "#{e.message} does not include #{message}."
97
+ if klass.to_s != e.class.to_s
98
+ flunk "Expected exception class #{klass} but got #{e.class}.\n #{e.backtrace}"
99
+ end
100
+
101
+ if !e.message.include?(message)
102
+ p e.backtrace
103
+ flunk "#{e.message} does not include #{message}.\n#{e.backtrace}"
104
+ end
98
105
  else
99
106
  flunk "Expected assertion #{klass} but none was raised."
100
107
  end
@@ -4,7 +4,8 @@ class TestThreading < Test::Unit::TestCase
4
4
 
5
5
  include Mongo
6
6
 
7
- @@db = standard_connection(:pool_size => 1, :timeout => 30).db(MONGO_TEST_DB)
7
+ @@con = standard_connection(:pool_size => 10, :timeout => 30)
8
+ @@db = @@con[MONGO_TEST_DB]
8
9
  @@coll = @@db.collection('thread-test-collection')
9
10
 
10
11
  def set_up_safe_data
@@ -21,16 +22,23 @@ class TestThreading < Test::Unit::TestCase
21
22
  end
22
23
 
23
24
  def test_safe_update
25
+ times = []
24
26
  set_up_safe_data
25
27
  threads = []
26
28
  100.times do |i|
27
29
  threads[i] = Thread.new do
28
- if i % 2 == 0
29
- assert_raise Mongo::OperationFailure do
30
- @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
30
+ 100.times do
31
+ if i % 2 == 0
32
+ assert_raise Mongo::OperationFailure do
33
+ t1 = Time.now
34
+ @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
35
+ times << Time.now - t1
36
+ end
37
+ else
38
+ t1 = Time.now
39
+ @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
40
+ times << Time.now - t1
31
41
  end
32
- else
33
- @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
34
42
  end
35
43
  end
36
44
  end
@@ -20,13 +20,14 @@ class ReplSetManager
20
20
  @retries = opts[:retries] || 30
21
21
  @config = {"_id" => @name, "members" => []}
22
22
  @durable = opts.fetch(:durable, false)
23
+ @smallfiles = opts.fetch(:smallfiles, true)
23
24
  @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
24
- @oplog_size = opts.fetch(:oplog_size, 32)
25
+ @oplog_size = opts.fetch(:oplog_size, 16)
25
26
  @tags = [{"dc" => "ny", "rack" => "a", "db" => "main"},
26
27
  {"dc" => "ny", "rack" => "b", "db" => "main"},
27
28
  {"dc" => "sf", "rack" => "a", "db" => "main"}]
28
29
 
29
- @arbiter_count = opts[:arbiter_count] || 2
30
+ @arbiter_count = opts[:arbiter_count] || 0
30
31
  @secondary_count = opts[:secondary_count] || 2
31
32
  @passive_count = opts[:passive_count] || 0
32
33
  @primary_count = 1
@@ -43,19 +44,10 @@ class ReplSetManager
43
44
  end
44
45
 
45
46
  def start_set
46
- begin
47
- con = Mongo::Connection.new(@host, @start_port)
48
- rescue Mongo::ConnectionFailure
49
- end
50
-
51
- if con && ensure_up(1, con)
52
- should_start = false
53
- puts "** Replica set already started."
54
- else
55
- should_start = true
56
- system("killall mongod")
57
- puts "** Starting a replica set with #{@count} nodes"
58
- end
47
+ system("killall mongod")
48
+ sleep(1)
49
+ should_start = true
50
+ puts "** Starting a replica set with #{@count} nodes"
59
51
 
60
52
  n = 0
61
53
  (@primary_count + @secondary_count).times do
@@ -81,15 +73,8 @@ class ReplSetManager
81
73
  n += 1
82
74
  end
83
75
 
84
- if con && ensure_up(1, con)
85
- @mongods.each do |k, v|
86
- v['up'] = true
87
- v['pid'] = File.open(File.join(v['db_path'], 'mongod.lock')).read.strip
88
- end
89
- else
90
- initiate
91
- ensure_up
92
- end
76
+ initiate
77
+ ensure_up
93
78
  end
94
79
 
95
80
  def cleanup_set
@@ -142,6 +127,7 @@ class ReplSetManager
142
127
  @mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
143
128
  "--oplogSize #{@oplog_size} #{journal_switch} --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
144
129
  @mongods[n]['start'] += " --dur" if @durable
130
+ @mongods[n]['start'] += " --smallfiles" if @smallfiles
145
131
  @mongods[n]['start']
146
132
  end
147
133
 
@@ -157,7 +143,7 @@ class ReplSetManager
157
143
  @config['version'] = config['version'] + 1
158
144
 
159
145
  begin
160
- con['admin'].command({'replSetReconfig' => @config})
146
+ con['admin'].command({'replSetReconfig' => @config})
161
147
  rescue Mongo::ConnectionFailure
162
148
  end
163
149
 
@@ -169,8 +155,8 @@ class ReplSetManager
169
155
  def add_node(n=nil)
170
156
  primary = get_node_with_state(1)
171
157
  con = get_connection(primary)
172
- init_node(n || @mongods.length)
173
158
 
159
+ init_node(n || @mongods.length)
174
160
  config = con['local']['system.replset'].find_one
175
161
  @config['version'] = config['version'] + 1
176
162
 
@@ -187,9 +173,8 @@ class ReplSetManager
187
173
  def kill(node, signal=2)
188
174
  pid = @mongods[node]['pid']
189
175
  puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
190
- system("kill -#{signal} #{@mongods[node]['pid']}")
176
+ system("kill #{pid}")
191
177
  @mongods[node]['up'] = false
192
- sleep(1)
193
178
  end
194
179
 
195
180
  def kill_primary(signal=2)
@@ -254,21 +239,49 @@ class ReplSetManager
254
239
  print "** Ensuring members are up..."
255
240
 
256
241
  attempt(n) do
257
- con = connection || get_connection
258
- status = con['admin'].command({'replSetGetStatus' => 1})
259
242
  print "."
260
- if status['members'].all? { |m| m['health'] == 1 &&
261
- [1, 2, 7].include?(m['state']) } &&
262
- status['members'].any? { |m| m['state'] == 1 }
263
- print "all members up!\n\n"
264
- con.close
265
- return status
266
- else
243
+ con = connection || get_connection
244
+ begin
245
+ status = con['admin'].command({:replSetGetStatus => 1})
246
+ rescue Mongo::OperationFailure => ex
267
247
  con.close
268
- raise Mongo::OperationFailure
248
+ raise ex
269
249
  end
270
- end
271
-
250
+ if status['members'].all? { |m| m['health'] == 1 &&
251
+ [1, 2, 7].include?(m['state']) } &&
252
+ status['members'].any? { |m| m['state'] == 1 }
253
+
254
+ connections = []
255
+ states = []
256
+ status['members'].each do |member|
257
+ begin
258
+ host, port = member['name'].split(':')
259
+ port = port.to_i
260
+ conn = Mongo::Connection.new(host, port, :slave_ok => true)
261
+ connections << conn
262
+ state = conn['admin'].command({:ismaster => 1})
263
+ states << state
264
+ rescue Mongo::ConnectionFailure
265
+ connections.each {|c| c.close }
266
+ con.close
267
+ raise Mongo::OperationFailure
268
+ end
269
+ end
270
+
271
+ if states.any? {|s| s['ismaster']}
272
+ print "all members up!\n\n"
273
+ connections.each {|c| c.close }
274
+ con.close
275
+ return status
276
+ else
277
+ con.close
278
+ raise Mongo::OperationFailure
279
+ end
280
+ else
281
+ con.close
282
+ raise Mongo::OperationFailure
283
+ end
284
+ end
272
285
  return false
273
286
  end
274
287
 
@@ -298,9 +311,11 @@ class ReplSetManager
298
311
  private
299
312
 
300
313
  def initiate
314
+ puts "Initiating replica set..."
301
315
  con = get_connection
302
316
 
303
317
  attempt do
318
+ con.object_id
304
319
  con['admin'].command({'replSetInitiate' => @config})
305
320
  end
306
321
 
@@ -5,7 +5,7 @@ class CollectionTest < Test::Unit::TestCase
5
5
  context "Basic operations: " do
6
6
  setup do
7
7
  @logger = mock()
8
- @logger.expects(:debug)
8
+ @logger.expects(:warn)
9
9
  end
10
10
 
11
11
  should "send update message" do
@@ -36,6 +36,7 @@ class CollectionTest < Test::Unit::TestCase
36
36
  @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
37
37
  @db = @conn['testing']
38
38
  @coll = @db.collection('books')
39
+ @conn.expects(:checkout_writer).returns(mock())
39
40
  @conn.expects(:receive_message).with do |op, msg, log, sock|
40
41
  op == 2004
41
42
  end.returns([[], 0, 0])
@@ -40,6 +40,12 @@ class ConnectionTest < Test::Unit::TestCase
40
40
  assert_equal [host_name, 27017], @conn.host_to_try
41
41
  end
42
42
 
43
+ should "allow db without username and password" do
44
+ host_name = "foo.bar-12345.org"
45
+ @conn = Connection.from_uri("mongodb://#{host_name}/foo", :connect => false)
46
+ assert_equal [host_name, 27017], @conn.host_to_try
47
+ end
48
+
43
49
  should "parse a uri with a hyphen & underscore in the username or password" do
44
50
  @conn = Connection.from_uri("mongodb://hyphen-user_name:p-s_s@localhost:27017/db", :connect => false)
45
51
  assert_equal ['localhost', 27017], @conn.host_to_try
@@ -65,16 +71,10 @@ class ConnectionTest < Test::Unit::TestCase
65
71
  assert_raise MongoArgumentError do
66
72
  Connection.from_uri("mongodb://localhost:abc", :connect => false)
67
73
  end
68
-
69
- assert_raise MongoArgumentError do
70
- Connection.from_uri("mongodb://localhost:27017, my.db.com:27018, ", :connect => false)
71
- end
72
74
  end
73
75
 
74
- should "require all of username, password, and database if any one is specified" do
75
- assert_raise MongoArgumentError do
76
- Connection.from_uri("mongodb://localhost/db", :connect => false)
77
- end
76
+ should "require all of username, if password and db are specified" do
77
+ assert Connection.from_uri("mongodb://kyle:jones@localhost/db", :connect => false)
78
78
 
79
79
  assert_raise MongoArgumentError do
80
80
  Connection.from_uri("mongodb://kyle:password@localhost", :connect => false)
@@ -19,7 +19,8 @@ class GridTest < Test::Unit::TestCase
19
19
 
20
20
  context "Grid classe with standard connections" do
21
21
  setup do
22
- @conn.expects(:slave_ok?).returns(false)
22
+ @conn.expects(:class).returns(Connection)
23
+ @conn.expects(:read_primary?).returns(true)
23
24
  end
24
25
 
25
26
  should "create indexes for Grid" do
@@ -36,7 +37,8 @@ class GridTest < Test::Unit::TestCase
36
37
 
37
38
  context "Grid classes with slave connection" do
38
39
  setup do
39
- @conn.expects(:slave_ok?).returns(true)
40
+ @conn.expects(:class).twice.returns(Connection)
41
+ @conn.expects(:read_primary?).returns(false)
40
42
  end
41
43
 
42
44
  should "not create indexes for Grid" do
@@ -12,6 +12,7 @@ class PoolManagerTest < Test::Unit::TestCase
12
12
  @connection = stub("Connection")
13
13
  @connection.stubs(:connect_timeout).returns(5000)
14
14
  @connection.stubs(:pool_size).returns(2)
15
+ @connection.stubs(:pool_timeout).returns(100)
15
16
  @connection.stubs(:socket_class).returns(TCPSocket)
16
17
  @connection.stubs(:[]).returns(@db)
17
18