mongo 1.4.0 → 1.5.0.rc0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/docs/HISTORY.md +15 -0
  2. data/docs/REPLICA_SETS.md +19 -7
  3. data/lib/mongo.rb +1 -0
  4. data/lib/mongo/collection.rb +1 -1
  5. data/lib/mongo/connection.rb +29 -351
  6. data/lib/mongo/cursor.rb +88 -6
  7. data/lib/mongo/gridfs/grid.rb +4 -2
  8. data/lib/mongo/gridfs/grid_file_system.rb +4 -2
  9. data/lib/mongo/networking.rb +345 -0
  10. data/lib/mongo/repl_set_connection.rb +236 -191
  11. data/lib/mongo/util/core_ext.rb +45 -0
  12. data/lib/mongo/util/logging.rb +5 -0
  13. data/lib/mongo/util/node.rb +6 -4
  14. data/lib/mongo/util/pool.rb +73 -26
  15. data/lib/mongo/util/pool_manager.rb +100 -30
  16. data/lib/mongo/util/uri_parser.rb +29 -21
  17. data/lib/mongo/version.rb +1 -1
  18. data/test/bson/binary_test.rb +6 -8
  19. data/test/bson/bson_test.rb +1 -0
  20. data/test/bson/ordered_hash_test.rb +2 -0
  21. data/test/bson/test_helper.rb +0 -17
  22. data/test/collection_test.rb +22 -0
  23. data/test/connection_test.rb +1 -1
  24. data/test/cursor_test.rb +3 -3
  25. data/test/load/thin/load.rb +4 -7
  26. data/test/replica_sets/basic_test.rb +46 -0
  27. data/test/replica_sets/connect_test.rb +35 -58
  28. data/test/replica_sets/count_test.rb +15 -6
  29. data/test/replica_sets/insert_test.rb +6 -7
  30. data/test/replica_sets/query_test.rb +4 -6
  31. data/test/replica_sets/read_preference_test.rb +112 -8
  32. data/test/replica_sets/refresh_test.rb +66 -36
  33. data/test/replica_sets/refresh_with_threads_test.rb +55 -0
  34. data/test/replica_sets/replication_ack_test.rb +3 -6
  35. data/test/replica_sets/rs_test_helper.rb +12 -6
  36. data/test/replica_sets/threading_test.rb +111 -0
  37. data/test/test_helper.rb +9 -2
  38. data/test/threading_test.rb +14 -6
  39. data/test/tools/repl_set_manager.rb +55 -40
  40. data/test/unit/collection_test.rb +2 -1
  41. data/test/unit/connection_test.rb +8 -8
  42. data/test/unit/grid_test.rb +4 -2
  43. data/test/unit/pool_manager_test.rb +1 -0
  44. data/test/unit/read_test.rb +17 -5
  45. data/test/uri_test.rb +9 -4
  46. metadata +13 -28
  47. data/test/replica_sets/connection_string_test.rb +0 -29
  48. data/test/replica_sets/pooled_insert_test.rb +0 -58
  49. data/test/replica_sets/query_secondaries.rb +0 -109
@@ -1,27 +1,26 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/replica_sets/rs_test_helper'
3
3
 
4
- # NOTE: This test expects a replica set of three nodes to be running
5
- # on the local host.
6
4
  class ReplicaSetInsertTest < Test::Unit::TestCase
7
- include Mongo
5
+ include ReplicaSetTest
8
6
 
9
7
  def setup
10
- @conn = ReplSetConnection.new([TEST_HOST, RS.ports[0]], [TEST_HOST, RS.ports[1]], [TEST_HOST, RS.ports[2]])
8
+ @conn = ReplSetConnection.new([TEST_HOST, self.rs.ports[0]],
9
+ [TEST_HOST, self.rs.ports[1]], [TEST_HOST, self.rs.ports[2]])
11
10
  @db = @conn.db(MONGO_TEST_DB)
12
11
  @db.drop_collection("test-sets")
13
12
  @coll = @db.collection("test-sets")
14
13
  end
15
14
 
16
15
  def teardown
17
- RS.restart_killed_nodes
16
+ self.rs.restart_killed_nodes
18
17
  @conn.close if @conn
19
18
  end
20
19
 
21
20
  def test_insert
22
21
  @coll.save({:a => 20}, :safe => true)
23
22
 
24
- RS.kill_primary
23
+ self.rs.kill_primary
25
24
 
26
25
  rescue_connection_failure do
27
26
  @coll.save({:a => 30}, :safe => true)
@@ -33,7 +32,7 @@ class ReplicaSetInsertTest < Test::Unit::TestCase
33
32
  @coll.save({:a => 70}, :safe => true)
34
33
 
35
34
  # Restart the old master and wait for sync
36
- RS.restart_killed_nodes
35
+ self.rs.restart_killed_nodes
37
36
  sleep(1)
38
37
  results = []
39
38
 
@@ -1,20 +1,18 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/replica_sets/rs_test_helper'
3
3
 
4
- # NOTE: This test expects a replica set of three nodes to be running
5
- # on the local host.
6
4
  class ReplicaSetQueryTest < Test::Unit::TestCase
7
- include Mongo
5
+ include ReplicaSetTest
8
6
 
9
7
  def setup
10
- @conn = ReplSetConnection.new([RS.host, RS.ports[0], RS.ports[1]])
8
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0], self.rs.ports[1]])
11
9
  @db = @conn.db(MONGO_TEST_DB)
12
10
  @db.drop_collection("test-sets")
13
11
  @coll = @db.collection("test-sets")
14
12
  end
15
13
 
16
14
  def teardown
17
- RS.restart_killed_nodes
15
+ self.rs.restart_killed_nodes
18
16
  @conn.close if @conn
19
17
  end
20
18
 
@@ -30,7 +28,7 @@ class ReplicaSetQueryTest < Test::Unit::TestCase
30
28
 
31
29
  puts "Benchmark before failover: #{benchmark_queries}"
32
30
 
33
- RS.kill_primary
31
+ self.rs.kill_primary
34
32
 
35
33
  results = []
36
34
  rescue_connection_failure do
@@ -1,15 +1,119 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/replica_sets/rs_test_helper'
3
+ require 'logger'
3
4
 
4
- # TODO: enable this once we enable reads from tags.
5
5
  class ReadPreferenceTest < Test::Unit::TestCase
6
- include Mongo
6
+ include ReplicaSetTest
7
7
 
8
- #def setup
9
- # @conn = ReplSetConnection.new([RS.host, RS.ports[0], RS.host, RS.ports[1]], :read => :secondary, :pool_size => 50)
10
- # @db = @conn.db(MONGO_TEST_DB)
11
- # @db.drop_collection("test-sets")
12
- #end
8
+ def setup
9
+ log = Logger.new("test.log")
10
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
11
+ [self.rs.host, self.rs.ports[1]],
12
+ :read => :secondary, :pool_size => 50,
13
+ :refresh_mode => false, :refresh_interval => 5, :logger => log)
14
+ @db = @conn.db(MONGO_TEST_DB)
15
+ @db.drop_collection("test-sets")
16
+ col = @db['mongo-test']
17
+ end
18
+
19
+ def teardown
20
+ self.rs.restart_killed_nodes
21
+ end
22
+
23
+ def test_read_primary
24
+ rescue_connection_failure do
25
+ assert !@conn.read_primary?
26
+ assert !@conn.primary?
27
+ end
28
+ end
29
+
30
+ def test_con
31
+ assert @conn.primary_pool, "No primary pool!"
32
+ assert @conn.read_pool, "No read pool!"
33
+ assert @conn.primary_pool.port != @conn.read_pool.port,
34
+ "Primary port and read port at the same!"
35
+ end
36
+
37
+ def test_query_secondaries
38
+ @secondary = Connection.new(self.rs.host, @conn.read_pool.port, :slave_ok => true)
39
+ @coll = @db.collection("test-sets", :safe => {:w => 3, :wtimeout => 20000})
40
+ @coll.save({:a => 20})
41
+ @coll.save({:a => 30})
42
+ @coll.save({:a => 40})
43
+ results = []
44
+ queries_before = @secondary['admin'].command({:serverStatus => 1})['opcounters']['query']
45
+ @coll.find.each {|r| results << r["a"]}
46
+ queries_after = @secondary['admin'].command({:serverStatus => 1})['opcounters']['query']
47
+ assert_equal 1, queries_after - queries_before
48
+ assert results.include?(20)
49
+ assert results.include?(30)
50
+ assert results.include?(40)
51
+
52
+ self.rs.kill_primary
53
+
54
+ results = []
55
+ rescue_connection_failure do
56
+ puts "@coll.find().each"
57
+ @coll.find.each {|r| results << r}
58
+ [20, 30, 40].each do |a|
59
+ assert results.any? {|r| r['a'] == a}, "Could not find record for a => #{a}"
60
+ end
61
+ end
62
+ end
63
+
64
+ def test_kill_primary
65
+ @coll = @db.collection("test-sets", :safe => {:w => 3, :wtimeout => 10000})
66
+ @coll.save({:a => 20})
67
+ @coll.save({:a => 30})
68
+ assert_equal 2, @coll.find.to_a.length
69
+
70
+ # Should still be able to read immediately after killing master node
71
+ self.rs.kill_primary
72
+ assert_equal 2, @coll.find.to_a.length
73
+ rescue_connection_failure do
74
+ puts "@coll.save()"
75
+ @coll.save({:a => 50}, :safe => {:w => 2, :wtimeout => 10000})
76
+ end
77
+ self.rs.restart_killed_nodes
78
+ @coll.save({:a => 50}, :safe => {:w => 2, :wtimeout => 10000})
79
+ assert_equal 4, @coll.find.to_a.length
80
+ end
81
+
82
+ def test_kill_secondary
83
+ @coll = @db.collection("test-sets", {:safe => {:w => 3, :wtimeout => 20000}})
84
+ @coll.save({:a => 20})
85
+ @coll.save({:a => 30})
86
+ assert_equal 2, @coll.find.to_a.length
87
+
88
+ read_node = self.rs.get_node_from_port(@conn.read_pool.port)
89
+ self.rs.kill(read_node)
90
+
91
+ # Should fail immediately on next read
92
+ old_read_pool_port = @conn.read_pool.port
93
+ assert_raise ConnectionFailure do
94
+ @coll.find.to_a.length
95
+ end
96
+
97
+ # Should eventually reconnect and be able to read
98
+ rescue_connection_failure do
99
+ length = @coll.find.to_a.length
100
+ assert_equal 2, length
101
+ end
102
+ new_read_pool_port = @conn.read_pool.port
103
+ assert old_read_pool_port != new_read_pool_port
104
+ end
105
+
106
+ def test_write_lots_of_data
107
+ @coll = @db.collection("test-sets", {:safe => {:w => 2}})
108
+
109
+ 6000.times do |n|
110
+ @coll.save({:a => n})
111
+ end
112
+
113
+ cursor = @coll.find()
114
+ cursor.next
115
+ cursor.close
116
+ end
13
117
 
14
118
  # TODO: enable this once we enable reads from tags.
15
119
  # def test_query_tagged
@@ -37,7 +141,7 @@ class ReadPreferenceTest < Test::Unit::TestCase
37
141
  # end
38
142
 
39
143
  #def teardown
40
- # RS.restart_killed_nodes
144
+ # self.rs.restart_killed_nodes
41
145
  #end
42
146
 
43
147
  end
@@ -2,16 +2,15 @@ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/replica_sets/rs_test_helper'
3
3
  require 'benchmark'
4
4
 
5
- # on ports TEST_PORT, RS.ports[1], and TEST + 2.
6
5
  class ReplicaSetRefreshTest < Test::Unit::TestCase
7
- include Mongo
6
+ include ReplicaSetTest
8
7
 
9
8
  def setup
10
9
  @conn = nil
11
10
  end
12
11
 
13
12
  def teardown
14
- RS.restart_killed_nodes
13
+ self.rs.restart_killed_nodes
15
14
  @conn.close if @conn
16
15
  end
17
16
 
@@ -19,13 +18,17 @@ class ReplicaSetRefreshTest < Test::Unit::TestCase
19
18
  Benchmark.bm do |x|
20
19
  x.report("Connect") do
21
20
  10.times do
22
- ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
23
- [RS.host, RS.ports[2]], :background_refresh => false)
21
+ ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
22
+ [self.rs.host, self.rs.ports[1]],
23
+ [self.rs.host, self.rs.ports[2]],
24
+ :refresh_mode => false)
24
25
  end
25
26
  end
26
27
 
27
- @con = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
28
- [RS.host, RS.ports[2]], :background_refresh => false)
28
+ @con = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
29
+ [self.rs.host, self.rs.ports[1]],
30
+ [self.rs.host, self.rs.ports[2]],
31
+ :refresh_mode => false)
29
32
 
30
33
  x.report("manager") do
31
34
  man = Mongo::PoolManager.new(@con, @con.seeds)
@@ -37,11 +40,13 @@ class ReplicaSetRefreshTest < Test::Unit::TestCase
37
40
  end
38
41
 
39
42
  def test_connect_and_manual_refresh_with_secondaries_down
40
- RS.kill_all_secondaries
43
+ self.rs.kill_all_secondaries
41
44
 
42
45
  rescue_connection_failure do
43
- @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
44
- [RS.host, RS.ports[2]], :background_refresh => false)
46
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
47
+ [self.rs.host, self.rs.ports[1]],
48
+ [self.rs.host, self.rs.ports[2]],
49
+ :refresh_mode => false)
45
50
  end
46
51
 
47
52
  assert_equal [], @conn.secondaries
@@ -54,7 +59,7 @@ class ReplicaSetRefreshTest < Test::Unit::TestCase
54
59
  assert @conn.connected?
55
60
  assert_equal @conn.read_pool, @conn.primary_pool
56
61
 
57
- RS.restart_killed_nodes
62
+ self.rs.restart_killed_nodes
58
63
  assert_equal [], @conn.secondaries
59
64
  assert @conn.connected?
60
65
  assert_equal @conn.read_pool, @conn.primary_pool
@@ -66,57 +71,82 @@ class ReplicaSetRefreshTest < Test::Unit::TestCase
66
71
  end
67
72
 
68
73
  def test_automated_refresh_with_secondaries_down
69
- RS.kill_all_secondaries
74
+ self.rs.kill_all_secondaries
70
75
 
71
76
  rescue_connection_failure do
72
- @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
73
- [RS.host, RS.ports[2]], :refresh_interval => 2, :background_refresh => true)
77
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
78
+ [self.rs.host, self.rs.ports[1]],
79
+ [self.rs.host, self.rs.ports[2]],
80
+ :refresh_interval => 2,
81
+ :refresh_mode => :sync)
74
82
  end
75
83
 
76
84
  assert_equal [], @conn.secondaries
77
85
  assert @conn.connected?
78
86
  assert_equal @conn.read_pool, @conn.primary_pool
87
+ old_refresh_version = @conn.refresh_version
79
88
 
80
- RS.restart_killed_nodes
81
-
82
- sleep(3)
83
-
84
- assert @conn.read_pool != @conn.primary_pool, "Read pool and primary pool are identical."
85
- assert @conn.secondaries.length > 0, "No secondaries have been added."
89
+ self.rs.restart_killed_nodes
90
+ sleep(4)
91
+ @conn['foo']['bar'].find_one
92
+ @conn['foo']['bar'].insert({:a => 1})
93
+
94
+ assert @conn.refresh_version > old_refresh_version,
95
+ "Refresh version hasn't changed."
96
+ assert @conn.secondaries.length > 0,
97
+ "No secondaries have been added."
98
+ assert @conn.read_pool != @conn.primary_pool,
99
+ "Read pool and primary pool are identical."
86
100
  end
87
101
 
88
102
  def test_automated_refresh_with_removed_node
89
- @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
90
- [RS.host, RS.ports[2]], :refresh_interval => 2, :background_refresh => true)
103
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
104
+ [self.rs.host, self.rs.ports[1]],
105
+ [self.rs.host, self.rs.ports[2]],
106
+ :refresh_interval => 2,
107
+ :refresh_mode => :sync)
91
108
 
92
- assert_equal 2, @conn.secondary_pools.length
93
- assert_equal 2, @conn.secondaries.length
109
+ num_secondaries = @conn.secondary_pools.length
110
+ old_refresh_version = @conn.refresh_version
94
111
 
95
- n = RS.remove_secondary_node
112
+ n = self.rs.remove_secondary_node
96
113
  sleep(4)
114
+ @conn['foo']['bar'].find_one
97
115
 
98
- assert_equal 1, @conn.secondaries.length
99
- assert_equal 1, @conn.secondary_pools.length
116
+ assert @conn.refresh_version > old_refresh_version,
117
+ "Refresh version hasn't changed."
118
+ assert_equal num_secondaries - 1, @conn.secondaries.length
119
+ assert_equal num_secondaries - 1, @conn.secondary_pools.length
100
120
 
101
- RS.add_node(n)
121
+ self.rs.add_node(n)
102
122
  end
103
123
 
104
124
  def test_adding_and_removing_nodes
105
- @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
106
- [RS.host, RS.ports[2]], :refresh_interval => 2, :background_refresh => true)
125
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
126
+ [self.rs.host, self.rs.ports[1]],
127
+ [self.rs.host, self.rs.ports[2]],
128
+ :refresh_interval => 2, :refresh_mode => :sync)
107
129
 
108
- RS.add_node
109
- sleep(5)
130
+ self.rs.add_node
131
+ sleep(4)
132
+ @conn['foo']['bar'].find_one
110
133
 
111
- @conn2 = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
112
- [RS.host, RS.ports[2]], :refresh_interval => 2, :background_refresh => true)
134
+ @conn2 = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
135
+ [self.rs.host, self.rs.ports[1]],
136
+ [self.rs.host, self.rs.ports[2]],
137
+ :refresh_interval => 2, :refresh_mode => :sync)
113
138
 
114
- assert @conn2.secondaries == @conn.secondaries
139
+ assert @conn2.secondaries.sort == @conn.secondaries.sort,
140
+ "Second connection secondaries not equal to first."
115
141
  assert_equal 3, @conn.secondary_pools.length
116
142
  assert_equal 3, @conn.secondaries.length
117
143
 
118
- RS.remove_secondary_node
144
+ config = @conn['admin'].command({:ismaster => 1})
145
+
146
+ self.rs.remove_secondary_node
119
147
  sleep(4)
148
+ config = @conn['admin'].command({:ismaster => 1})
149
+
120
150
  assert_equal 2, @conn.secondary_pools.length
121
151
  assert_equal 2, @conn.secondaries.length
122
152
  end
@@ -0,0 +1,55 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ require './test/replica_sets/rs_test_helper'
3
+ require 'benchmark'
4
+
5
+ class ReplicaSetRefreshWithThreadsTest < Test::Unit::TestCase
6
+ include ReplicaSetTest
7
+
8
+ def setup
9
+ @conn = nil
10
+ end
11
+
12
+ def teardown
13
+ self.rs.restart_killed_nodes
14
+ @conn.close if @conn
15
+ end
16
+
17
+ def test_read_write_load_with_added_nodes
18
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]],
19
+ [self.rs.host, self.rs.ports[1]],
20
+ [self.rs.host, self.rs.ports[2]],
21
+ :refresh_interval => 5,
22
+ :refresh_mode => :sync,
23
+ :read => :secondary)
24
+ @duplicate = @conn[MONGO_TEST_DB]['duplicate']
25
+ @unique = @conn[MONGO_TEST_DB]['unique']
26
+ @duplicate.insert("test" => "insert")
27
+ @duplicate.insert("test" => "update")
28
+ @unique.insert("test" => "insert")
29
+ @unique.insert("test" => "update")
30
+ @unique.create_index("test", :unique => true)
31
+
32
+ threads = []
33
+ 10.times do
34
+ threads << Thread.new do
35
+ 1000.times do |i|
36
+ if i % 2 == 0
37
+ assert_raise Mongo::OperationFailure do
38
+ @unique.insert({"test" => "insert"}, :safe => true)
39
+ end
40
+ else
41
+ @duplicate.insert({"test" => "insert"}, :safe => true)
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ self.rs.add_node
48
+ threads.each {|t| t.join }
49
+
50
+ config = @conn['admin'].command({:ismaster => 1})
51
+
52
+ assert_equal 3, @conn.secondary_pools.length
53
+ assert_equal 3, @conn.secondaries.length
54
+ end
55
+ end
@@ -1,14 +1,11 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
2
  require './test/replica_sets/rs_test_helper'
3
3
 
4
- # NOTE: This test expects a replica set of three nodes to be running on local host.
5
4
  class ReplicaSetAckTest < Test::Unit::TestCase
6
- include Mongo
5
+ include ReplicaSetTest
7
6
 
8
7
  def setup
9
- RS.ensure_up
10
-
11
- @conn = ReplSetConnection.new([RS.host, RS.ports[0]])
8
+ @conn = ReplSetConnection.new([self.rs.host, self.rs.ports[0]])
12
9
 
13
10
  @slave1 = Connection.new(@conn.secondary_pools[0].host,
14
11
  @conn.secondary_pools[0].port, :slave_ok => true)
@@ -21,7 +18,7 @@ class ReplicaSetAckTest < Test::Unit::TestCase
21
18
  end
22
19
 
23
20
  def teardown
24
- RS.restart_killed_nodes
21
+ self.rs.restart_killed_nodes
25
22
  @conn.close if @conn
26
23
  end
27
24