mongo 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/Rakefile +50 -69
  2. data/docs/CREDITS.md +4 -0
  3. data/docs/HISTORY.md +9 -0
  4. data/docs/REPLICA_SETS.md +8 -10
  5. data/lib/mongo.rb +3 -1
  6. data/lib/mongo/collection.rb +2 -1
  7. data/lib/mongo/connection.rb +146 -314
  8. data/lib/mongo/db.rb +6 -2
  9. data/lib/mongo/gridfs/grid.rb +1 -1
  10. data/lib/mongo/gridfs/grid_io.rb +29 -5
  11. data/lib/mongo/repl_set_connection.rb +290 -0
  12. data/lib/mongo/util/pool.rb +6 -8
  13. data/lib/mongo/util/uri_parser.rb +71 -0
  14. data/mongo.gemspec +1 -2
  15. data/test/collection_test.rb +9 -7
  16. data/test/connection_test.rb +0 -6
  17. data/test/grid_file_system_test.rb +2 -2
  18. data/test/grid_io_test.rb +33 -1
  19. data/test/grid_test.rb +36 -6
  20. data/test/replica_sets/connect_test.rb +59 -21
  21. data/test/replica_sets/count_test.rb +9 -7
  22. data/test/replica_sets/insert_test.rb +11 -9
  23. data/test/replica_sets/pooled_insert_test.rb +12 -13
  24. data/test/replica_sets/query_secondaries.rb +48 -8
  25. data/test/replica_sets/query_test.rb +10 -9
  26. data/test/replica_sets/replication_ack_test.rb +15 -22
  27. data/test/replica_sets/rs_test_helper.rb +29 -0
  28. data/test/test_helper.rb +13 -20
  29. data/test/threading/{test_threading_large_pool.rb → threading_with_large_pool_test.rb} +1 -1
  30. data/test/tools/repl_set_manager.rb +241 -0
  31. data/test/tools/test.rb +13 -0
  32. data/test/unit/connection_test.rb +3 -85
  33. data/test/unit/repl_set_connection_test.rb +82 -0
  34. metadata +19 -21
  35. data/test/replica_pairs/count_test.rb +0 -34
  36. data/test/replica_pairs/insert_test.rb +0 -50
  37. data/test/replica_pairs/pooled_insert_test.rb +0 -54
  38. data/test/replica_pairs/query_test.rb +0 -39
  39. data/test/replica_sets/node_type_test.rb +0 -42
  40. data/test/rs.rb +0 -24
@@ -1,7 +1,5 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
- require 'mongo'
3
- require 'test/unit'
4
- require './test/test_helper'
2
+ require './test/replica_sets/rs_test_helper'
5
3
 
6
4
  # NOTE: This test expects a replica set of three nodes to be running
7
5
  # on the local host.
@@ -9,24 +7,27 @@ class ReplicaSetQueryTest < Test::Unit::TestCase
9
7
  include Mongo
10
8
 
11
9
  def setup
12
- @conn = Mongo::Connection.multi([['localhost', 27017], ['localhost', 27018], ['localhost', 27019]])
10
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]])
13
11
  @db = @conn.db(MONGO_TEST_DB)
14
12
  @db.drop_collection("test-sets")
15
13
  @coll = @db.collection("test-sets")
16
14
  end
17
15
 
16
+ def teardown
17
+ RS.restart_killed_nodes
18
+ end
19
+
18
20
  def test_query
19
- @coll.save({:a => 20})
20
- @coll.save({:a => 30})
21
- @coll.save({:a => 40})
21
+ @coll.save({:a => 20}, :safe => {:w => 3})
22
+ @coll.save({:a => 30}, :safe => {:w => 3})
23
+ @coll.save({:a => 40}, :safe => {:w => 3})
22
24
  results = []
23
25
  @coll.find.each {|r| results << r}
24
26
  [20, 30, 40].each do |a|
25
27
  assert results.any? {|r| r['a'] == a}, "Could not find record for a => #{a}"
26
28
  end
27
29
 
28
- puts "Please disconnect the current master."
29
- gets
30
+ RS.kill_primary
30
31
 
31
32
  results = []
32
33
  rescue_connection_failure do
@@ -1,20 +1,17 @@
1
1
  $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
- require 'mongo'
3
- require 'test/unit'
4
- require './test/test_helper'
2
+ require './test/replica_sets/rs_test_helper'
5
3
 
6
4
  # NOTE: This test expects a replica set of three nodes to be running on local host.
7
5
  class ReplicaSetAckTest < Test::Unit::TestCase
8
6
  include Mongo
9
7
 
10
8
  def setup
11
- @conn = Mongo::Connection.multi([['localhost', 27017], ['localhost', 27018], ['localhost', 27019]])
9
+ RS.ensure_up
12
10
 
13
- master = [@conn.host, @conn.port]
14
- @slaves = @conn.nodes - master
11
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]])
15
12
 
16
- @slave1 = Mongo::Connection.new(@slaves[0][0], @slaves[0][1], :slave_ok => true)
17
- @slave2 = Mongo::Connection.new(@slaves[1][0], @slaves[1][1], :slave_ok => true)
13
+ @slave1 = Connection.new(@conn.secondary_pools[0].host,
14
+ @conn.secondary_pools[0].port, :slave_ok => true)
18
15
 
19
16
  @db = @conn.db(MONGO_TEST_DB)
20
17
  @db.drop_collection("test-sets")
@@ -22,47 +19,43 @@ class ReplicaSetAckTest < Test::Unit::TestCase
22
19
  end
23
20
 
24
21
  def test_safe_mode_with_w_failure
25
- assert_raise_error OperationFailure, "timed out waiting for slaves" do
22
+ assert_raise_error OperationFailure, "timeout" do
26
23
  @col.insert({:foo => 1}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
27
24
  end
28
- assert_raise_error OperationFailure, "timed out waiting for slaves" do
25
+ assert_raise_error OperationFailure, "timeout" do
29
26
  @col.update({:foo => 1}, {:foo => 2}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
30
27
  end
31
- assert_raise_error OperationFailure, "timed out waiting for slaves" do
28
+ assert_raise_error OperationFailure, "timeout" do
32
29
  @col.remove({:foo => 2}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})
33
30
  end
34
31
  end
35
32
 
36
33
  def test_safe_mode_replication_ack
37
- @col.insert({:baz => "bar"}, :safe => {:w => 3, :wtimeout => 1000})
34
+ @col.insert({:baz => "bar"}, :safe => {:w => 2, :wtimeout => 5000})
38
35
 
39
- assert @col.insert({:foo => "0" * 10000}, :safe => {:w => 3, :wtimeout => 1000})
36
+ assert @col.insert({:foo => "0" * 5000}, :safe => {:w => 2, :wtimeout => 5000})
40
37
  assert_equal 2, @slave1[MONGO_TEST_DB]["test-sets"].count
41
- assert_equal 2, @slave2[MONGO_TEST_DB]["test-sets"].count
42
38
 
43
-
44
- assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 3, :wtimeout => 1000})
39
+ assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 2, :wtimeout => 5000})
45
40
  assert @slave1[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
46
- assert @slave2[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
47
41
 
48
- assert @col.remove({}, :safe => {:w => 3, :wtimeout => 1000})
42
+ assert @col.remove({}, :safe => {:w => 2, :wtimeout => 5000})
49
43
  assert_equal 0, @slave1[MONGO_TEST_DB]["test-sets"].count
50
- assert_equal 0, @slave2[MONGO_TEST_DB]["test-sets"].count
51
44
  end
52
45
 
53
46
  def test_last_error_responses
54
47
  20.times { @col.insert({:baz => "bar"}) }
55
- response = @db.get_last_error(:w => 3, :wtimeout => 10000)
48
+ response = @db.get_last_error(:w => 2, :wtimeout => 5000)
56
49
  assert response['ok'] == 1
57
50
  assert response['lastOp']
58
51
 
59
52
  @col.update({}, {:baz => "foo"}, :multi => true)
60
- response = @db.get_last_error(:w => 3, :wtimeout => 1000)
53
+ response = @db.get_last_error(:w => 2, :wtimeout => 5000)
61
54
  assert response['ok'] == 1
62
55
  assert response['lastOp']
63
56
 
64
57
  @col.remove({})
65
- response = @db.get_last_error(:w => 3, :wtimeout => 1000)
58
+ response = @db.get_last_error(:w => 2, :wtimeout => 5000)
66
59
  assert response['ok'] == 1
67
60
  assert response['n'] == 20
68
61
  assert response['lastOp']
@@ -0,0 +1,29 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ require './test/test_helper'
3
+ require './test/tools/repl_set_manager'
4
+
5
+ unless defined? RS
6
+ RS = ReplSetManager.new
7
+ RS.start_set
8
+ end
9
+
10
+ class Test::Unit::TestCase
11
+
12
+ # Generic code for rescuing connection failures and retrying operations.
13
+ # This could be combined with some timeout functionality.
14
+ def rescue_connection_failure(max_retries=60)
15
+ success = false
16
+ tries = 0
17
+ while !success && tries < max_retries
18
+ begin
19
+ yield
20
+ success = true
21
+ rescue Mongo::ConnectionFailure
22
+ puts "Rescue attempt #{tries}\n"
23
+ tries += 1
24
+ sleep(1)
25
+ end
26
+ end
27
+ end
28
+
29
+ end
@@ -22,7 +22,15 @@ end
22
22
  require 'bson_ext/cbson' if !(RUBY_PLATFORM =~ /java/) && ENV['C_EXT']
23
23
 
24
24
  unless defined? MONGO_TEST_DB
25
- MONGO_TEST_DB = 'mongo-ruby-test'
25
+ MONGO_TEST_DB = 'ruby-test-db'
26
+ end
27
+
28
+ unless defined? TEST_PORT
29
+ TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::Connection::DEFAULT_PORT
30
+ end
31
+
32
+ unless defined? TEST_HOST
33
+ TEST_HOST = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
26
34
  end
27
35
 
28
36
  class Test::Unit::TestCase
@@ -30,8 +38,7 @@ class Test::Unit::TestCase
30
38
  include BSON
31
39
 
32
40
  def self.standard_connection(options={})
33
- Connection.new(ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost',
34
- ENV['MONGO_RUBY_DRIVER_PORT'] || Connection::DEFAULT_PORT, options)
41
+ Connection.new(TEST_HOST, TEST_PORT, options)
35
42
  end
36
43
 
37
44
  def standard_connection(options={})
@@ -43,11 +50,11 @@ class Test::Unit::TestCase
43
50
  end
44
51
 
45
52
  def self.mongo_host
46
- ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
53
+ TEST_HOST
47
54
  end
48
55
 
49
56
  def self.mongo_port
50
- ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : 27017
57
+ TEST_PORT
51
58
  end
52
59
 
53
60
  def host_port
@@ -62,21 +69,7 @@ class Test::Unit::TestCase
62
69
  self.class.mongo_port
63
70
  end
64
71
 
65
- # Generic code for rescuing connection failures and retrying operations.
66
- # This could be combined with some timeout functionality.
67
- def rescue_connection_failure
68
- success = false
69
- while !success
70
- begin
71
- yield
72
- success = true
73
- rescue Mongo::ConnectionFailure
74
- puts "Rescuing"
75
- sleep(1)
76
- end
77
- end
78
- end
79
-
72
+
80
73
  def assert_raise_error(klass, message)
81
74
  begin
82
75
  yield
@@ -1,4 +1,4 @@
1
- require 'test/test_helper'
1
+ require './test/test_helper'
2
2
 
3
3
  # Essentialy the same as test_threading.rb but with an expanded pool for
4
4
  # testing multiple connections.
@@ -0,0 +1,241 @@
1
+ #!/usr/bin/ruby
2
+
3
+ STDOUT.sync = true
4
+
5
+ unless defined? Mongo
6
+ require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
7
+ end
8
+
9
+ class ReplSetManager
10
+
11
+ attr_accessor :host, :start_port, :ports, :name, :mongods
12
+
13
+ def initialize(opts={})
14
+ @start_port = opts[:start_port] || 30000
15
+ @ports = []
16
+ @name = opts[:name] || 'replica-set-foo'
17
+ @host = opts[:host] || 'localhost'
18
+ @retries = opts[:retries] || 60
19
+ @config = {"_id" => @name, "members" => []}
20
+ @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
21
+
22
+ @arbiter_count = opts[:arbiter_count] || 2
23
+ @secondary_count = opts[:secondary_count] || 1
24
+ @passive_count = opts[:passive_count] || 1
25
+ @primary_count = 1
26
+
27
+ @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
28
+ if @count > 7
29
+ raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
30
+ end
31
+
32
+ @mongods = {}
33
+ end
34
+
35
+ def start_set
36
+ puts "** Starting a replica set with #{@count} nodes"
37
+
38
+ system("killall mongod")
39
+
40
+ n = 0
41
+ (@primary_count + @secondary_count).times do |n|
42
+ init_node(n)
43
+ n += 1
44
+ end
45
+
46
+ @passive_count.times do
47
+ init_node(n) do |attrs|
48
+ attrs['priority'] = 0
49
+ end
50
+ n += 1
51
+ end
52
+
53
+ @arbiter_count.times do
54
+ init_node(n) do |attrs|
55
+ attrs['arbiterOnly'] = true
56
+ end
57
+ n += 1
58
+ end
59
+
60
+ initiate
61
+ ensure_up
62
+ end
63
+
64
+ def init_node(n)
65
+ @mongods[n] ||= {}
66
+ port = @start_port + n
67
+ @ports << port
68
+ @mongods[n]['port'] = port
69
+ @mongods[n]['db_path'] = get_path("rs-#{port}")
70
+ @mongods[n]['log_path'] = get_path("log-#{port}")
71
+ system("rm -rf #{@mongods[n]['db_path']}")
72
+ system("mkdir -p #{@mongods[n]['db_path']}")
73
+
74
+ @mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
75
+ " --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
76
+
77
+ start(n)
78
+
79
+ member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
80
+
81
+ if block_given?
82
+ custom_attrs = {}
83
+ yield custom_attrs
84
+ member.merge!(custom_attrs)
85
+ @mongods[n].merge!(custom_attrs)
86
+ end
87
+
88
+ @config['members'] << member
89
+ end
90
+
91
+ def kill(node)
92
+ pid = @mongods[node]['pid']
93
+ puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
94
+ system("kill -2 #{@mongods[node]['pid']}")
95
+ @mongods[node]['up'] = false
96
+ sleep(1)
97
+ end
98
+
99
+ def kill_primary
100
+ node = get_node_with_state(1)
101
+ kill(node)
102
+ return node
103
+ end
104
+
105
+ # Note that we have to rescue a connection failure
106
+ # when we run the StepDown command because that
107
+ # command will close the connection.
108
+ def step_down_primary
109
+ primary = get_node_with_state(1)
110
+ con = get_connection(primary)
111
+ begin
112
+ con['admin'].command({'replSetStepDown' => 90})
113
+ rescue Mongo::ConnectionFailure
114
+ end
115
+ end
116
+
117
+ def kill_secondary
118
+ node = get_node_with_state(2)
119
+ kill(node)
120
+ return node
121
+ end
122
+
123
+ def restart_killed_nodes
124
+ nodes = @mongods.keys.select do |key|
125
+ @mongods[key]['up'] == false
126
+ end
127
+
128
+ nodes.each do |node|
129
+ start(node)
130
+ end
131
+
132
+ ensure_up
133
+ end
134
+
135
+ def get_node_from_port(port)
136
+ @mongods.keys.detect { |key| @mongods[key]['port'] == port }
137
+ end
138
+
139
+ def start(node)
140
+ system(@mongods[node]['start'])
141
+ @mongods[node]['up'] = true
142
+ sleep(0.5)
143
+ @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
144
+ end
145
+ alias :restart :start
146
+
147
+ def ensure_up
148
+ print "** Ensuring members are up..."
149
+
150
+ attempt do
151
+ con = get_connection
152
+ status = con['admin'].command({'replSetGetStatus' => 1})
153
+ print "."
154
+ if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
155
+ status['members'].any? { |m| m['state'] == 1 }
156
+ print "all members up!\n\n"
157
+ return status
158
+ else
159
+ raise Mongo::OperationFailure
160
+ end
161
+ end
162
+ end
163
+
164
+ def primary
165
+ nodes = get_all_host_pairs_with_state(1)
166
+ nodes.empty? ? nil : nodes[0]
167
+ end
168
+
169
+ def secondaries
170
+ get_all_host_pairs_with_state(2)
171
+ end
172
+
173
+ def arbiters
174
+ get_all_host_pairs_with_state(7)
175
+ end
176
+
177
+ private
178
+
179
+ def initiate
180
+ con = get_connection
181
+
182
+ attempt do
183
+ con['admin'].command({'replSetInitiate' => @config})
184
+ end
185
+ end
186
+
187
+ def get_node_with_state(state)
188
+ status = ensure_up
189
+ node = status['members'].detect {|m| m['state'] == state}
190
+ if node
191
+ host_port = node['name'].split(':')
192
+ port = host_port[1] ? host_port[1].to_i : 27017
193
+ key = @mongods.keys.detect {|key| @mongods[key]['port'] == port}
194
+ return key
195
+ else
196
+ return false
197
+ end
198
+ end
199
+
200
+ def get_all_host_pairs_with_state(state)
201
+ status = ensure_up
202
+ nodes = status['members'].select {|m| m['state'] == state}
203
+ nodes.map do |node|
204
+ host_port = node['name'].split(':')
205
+ port = host_port[1] ? host_port[1].to_i : 27017
206
+ [host, port]
207
+ end
208
+ end
209
+
210
+ def get_connection(node=nil)
211
+ con = attempt do
212
+ if !node
213
+ node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
214
+ end
215
+ con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
216
+ end
217
+
218
+ return con
219
+ end
220
+
221
+ def get_path(name)
222
+ File.join(@path, name)
223
+ end
224
+
225
+ def attempt
226
+ raise "No block given!" unless block_given?
227
+ count = 0
228
+
229
+ while count < @retries do
230
+ begin
231
+ return yield
232
+ rescue Mongo::OperationFailure, Mongo::ConnectionFailure
233
+ sleep(1)
234
+ count += 1
235
+ end
236
+ end
237
+
238
+ raise exception
239
+ end
240
+
241
+ end