mongo 1.4.0 → 1.5.0.rc0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/docs/HISTORY.md +15 -0
  2. data/docs/REPLICA_SETS.md +19 -7
  3. data/lib/mongo.rb +1 -0
  4. data/lib/mongo/collection.rb +1 -1
  5. data/lib/mongo/connection.rb +29 -351
  6. data/lib/mongo/cursor.rb +88 -6
  7. data/lib/mongo/gridfs/grid.rb +4 -2
  8. data/lib/mongo/gridfs/grid_file_system.rb +4 -2
  9. data/lib/mongo/networking.rb +345 -0
  10. data/lib/mongo/repl_set_connection.rb +236 -191
  11. data/lib/mongo/util/core_ext.rb +45 -0
  12. data/lib/mongo/util/logging.rb +5 -0
  13. data/lib/mongo/util/node.rb +6 -4
  14. data/lib/mongo/util/pool.rb +73 -26
  15. data/lib/mongo/util/pool_manager.rb +100 -30
  16. data/lib/mongo/util/uri_parser.rb +29 -21
  17. data/lib/mongo/version.rb +1 -1
  18. data/test/bson/binary_test.rb +6 -8
  19. data/test/bson/bson_test.rb +1 -0
  20. data/test/bson/ordered_hash_test.rb +2 -0
  21. data/test/bson/test_helper.rb +0 -17
  22. data/test/collection_test.rb +22 -0
  23. data/test/connection_test.rb +1 -1
  24. data/test/cursor_test.rb +3 -3
  25. data/test/load/thin/load.rb +4 -7
  26. data/test/replica_sets/basic_test.rb +46 -0
  27. data/test/replica_sets/connect_test.rb +35 -58
  28. data/test/replica_sets/count_test.rb +15 -6
  29. data/test/replica_sets/insert_test.rb +6 -7
  30. data/test/replica_sets/query_test.rb +4 -6
  31. data/test/replica_sets/read_preference_test.rb +112 -8
  32. data/test/replica_sets/refresh_test.rb +66 -36
  33. data/test/replica_sets/refresh_with_threads_test.rb +55 -0
  34. data/test/replica_sets/replication_ack_test.rb +3 -6
  35. data/test/replica_sets/rs_test_helper.rb +12 -6
  36. data/test/replica_sets/threading_test.rb +111 -0
  37. data/test/test_helper.rb +9 -2
  38. data/test/threading_test.rb +14 -6
  39. data/test/tools/repl_set_manager.rb +55 -40
  40. data/test/unit/collection_test.rb +2 -1
  41. data/test/unit/connection_test.rb +8 -8
  42. data/test/unit/grid_test.rb +4 -2
  43. data/test/unit/pool_manager_test.rb +1 -0
  44. data/test/unit/read_test.rb +17 -5
  45. data/test/uri_test.rb +9 -4
  46. metadata +13 -28
  47. data/test/replica_sets/connection_string_test.rb +0 -29
  48. data/test/replica_sets/pooled_insert_test.rb +0 -58
  49. data/test/replica_sets/query_secondaries.rb +0 -109
@@ -16,15 +16,14 @@
16
16
  # limitations under the License.
17
17
  # ++
18
18
 
19
- require 'sync'
20
-
21
19
  module Mongo
22
20
 
23
21
  # Instantiates and manages connections to a MongoDB replica set.
24
22
  class ReplSetConnection < Connection
25
- attr_reader :nodes, :secondaries, :arbiters, :secondary_pools,
26
- :replica_set_name, :read_pool, :seeds, :tags_to_pools,
27
- :refresh_interval, :refresh_mode
23
+ CLEANUP_INTERVAL = 300
24
+
25
+ attr_reader :replica_set_name, :seeds, :refresh_interval, :refresh_mode,
26
+ :refresh_version
28
27
 
29
28
  # Create a connection to a MongoDB replica set.
30
29
  #
@@ -57,11 +56,10 @@ module Mongo
57
56
  # @option opts [Float] :connect_timeout (nil) The number of seconds to wait before timing out a
58
57
  # connection attempt.
59
58
  # @option opts [Boolean] :ssl (false) If true, create the connection to the server using SSL.
60
- # @option opts [Boolean] :refresh_mode (:sync) Set this to :async to enable a background thread that
61
- # periodically updates the state of the connection. If, for example, you initially connect while a secondary
62
- # is down, this will reconnect to that secondary behind the scenes to
63
- # prevent you from having to reconnect manually. If set to :sync, refresh will happen
64
- # synchronously. If +false+, no automatic refresh will occur unless there's a connection failure.
59
+ # @option opts [Boolean] :refresh_mode (false) Set this to :sync to periodically update the
60
+ # state of the connection every :refresh_interval seconds. Replica set connection failures
61
+ # will always trigger a complete refresh. This option is useful when you want to add new nodes
62
+ # or remove replica set nodes not currently in use by the driver.
65
63
  # @option opts [Integer] :refresh_interval (90) If :refresh_mode is enabled, this is the number of seconds
66
64
  # between calls to check the replica set's state.
67
65
  # @option opts [Boolean] :require_primary (true) If true, require a primary node for the connection
@@ -83,8 +81,6 @@ module Mongo
83
81
  # @raise [ReplicaSetConnectionError] This is raised if a replica set name is specified and the
84
82
  # driver fails to connect to a replica set with that name.
85
83
  def initialize(*args)
86
- extend Sync_m
87
-
88
84
  if args.last.is_a?(Hash)
89
85
  opts = args.pop
90
86
  else
@@ -101,31 +97,21 @@ module Mongo
101
97
  # TODO: get rid of this
102
98
  @nodes = @seeds.dup
103
99
 
104
- # The members of the replica set, stored as instances of Mongo::Node.
105
- @members = []
106
-
107
- # Connection pool for primary node
108
- @primary = nil
109
- @primary_pool = nil
110
-
111
- # Connection pools for each secondary node
112
- @secondaries = []
113
- @secondary_pools = []
114
-
115
- # The secondary pool to which we'll be sending reads.
116
- # This may be identical to the primary pool.
117
- @read_pool = nil
118
-
119
- # A list of arbiter addresses (for client information only)
120
- @arbiters = []
121
-
122
100
  # Refresh
123
- @refresh_mode = opts.fetch(:refresh_mode, :sync)
101
+ @refresh_mode = opts.fetch(:refresh_mode, false)
124
102
  @refresh_interval = opts[:refresh_interval] || 90
103
+ @last_refresh = Time.now
125
104
 
126
- if ![:sync, :async, false].include?(@refresh_mode)
105
+ # No connection manager by default.
106
+ @manager = nil
107
+ @pool_mutex = Mutex.new
108
+
109
+ if @refresh_mode == :async
110
+ warn ":async refresh mode has been deprecated. Refresh
111
+ mode will be disabled."
112
+ elsif ![:sync, false].include?(@refresh_mode)
127
113
  raise MongoArgumentError,
128
- "Refresh mode must be one of :sync, :async, or false."
114
+ "Refresh mode must be either :sync or false."
129
115
  end
130
116
 
131
117
  # Are we allowing reads from secondaries?
@@ -140,13 +126,7 @@ module Mongo
140
126
  end
141
127
 
142
128
  @connected = false
143
-
144
- # Store the refresher thread
145
- @refresh_thread = nil
146
-
147
- # Maps
148
- @sockets_to_pools = {}
149
- @tags_to_pools = {}
129
+ @refresh_version = 0
150
130
 
151
131
  # Replica set name
152
132
  if opts[:rs_name]
@@ -164,68 +144,75 @@ module Mongo
164
144
  end
165
145
 
166
146
  def inspect
167
- "<Mongo::ReplSetConnection:0x#{self.object_id.to_s(16)} @seeds=#{@seeds} " +
147
+ "<Mongo::ReplSetConnection:0x#{self.object_id.to_s(16)} @seeds=#{@seeds.inspect} " +
168
148
  "@connected=#{@connected}>"
169
149
  end
170
150
 
171
151
  # Initiate a connection to the replica set.
172
152
  def connect
173
153
  log(:info, "Connecting...")
174
- sync_synchronize(:EX) do
175
- return if @connected
176
- manager = PoolManager.new(self, @seeds)
177
- manager.connect
178
-
179
- update_config(manager)
180
- initiate_refresh_mode
181
-
182
- if @require_primary && @primary.nil? #TODO: in v2.0, we'll let this be optional and do a lazy connect.
183
- raise ConnectionFailure, "Failed to connect to primary node."
184
- elsif !@read_pool
185
- raise ConnectionFailure, "Failed to connect to any node."
186
- else
187
- @connected = true
188
- end
154
+ return if @connected
155
+ manager = PoolManager.new(self, @seeds)
156
+ manager.connect
157
+
158
+ update_config(manager)
159
+
160
+ if @require_primary && self.primary.nil? #TODO: in v2.0, we'll let this be optional and do a lazy connect.
161
+ close
162
+ raise ConnectionFailure, "Failed to connect to primary node."
163
+ elsif self.read_pool.nil?
164
+ close
165
+ raise ConnectionFailure, "Failed to connect to any node."
166
+ else
167
+ @connected = true
189
168
  end
190
169
  end
191
170
 
192
- # Note: this method must be called from within
193
- # an exclusive lock.
194
- def update_config(manager)
195
- @arbiters = manager.arbiters.nil? ? [] : manager.arbiters.dup
196
- @primary = manager.primary.nil? ? nil : manager.primary.dup
197
- @secondaries = manager.secondaries.dup
198
- @hosts = manager.hosts.dup
199
-
200
- @primary_pool = manager.primary_pool
201
- @read_pool = manager.read_pool
202
- @secondary_pools = manager.secondary_pools
203
- @tags_to_pools = manager.tags_to_pools
204
- @seeds = manager.seeds
205
- @manager = manager
206
- @nodes = manager.nodes
207
- @max_bson_size = manager.max_bson_size
208
- end
209
-
210
- # Refresh the current replica set configuration.
171
+ # Determine whether a replica set refresh is
172
+ # required. If so, run a hard refresh. You can
173
+ # force a hard refresh by running
174
+ # ReplSetConnection#hard_refresh!
175
+ #
176
+ # @return [Boolean] +true+ unless a hard refresh
177
+ # is run and the refresh lock can't be acquired.
211
178
  def refresh(opts={})
212
- return false if !connected?
179
+ if !connected?
180
+ log(:info, "Trying to check replica set health but not " +
181
+ "connected...")
182
+ return hard_refresh!
183
+ end
213
184
 
214
- # Return if another thread is already in the process of refreshing.
215
- return if sync_exclusive?
185
+ log(:debug, "Checking replica set connection health...")
186
+ @manager.check_connection_health
216
187
 
217
- sync_synchronize(:EX) do
218
- log(:info, "Refreshing...")
219
- @background_manager ||= PoolManager.new(self, @seeds)
220
- @background_manager.connect
221
- update_config(@background_manager)
188
+ if @manager.refresh_required?
189
+ return hard_refresh!
222
190
  end
223
191
 
224
192
  return true
225
193
  end
226
194
 
195
+ # Force a hard refresh of this connection's view
196
+ # of the replica set.
197
+ #
198
+ # @return [Boolean] +true+ if hard refresh
199
+ # occurred. +false+ is returned when unable
200
+ # to get the refresh lock.
201
+ def hard_refresh!
202
+ log(:info, "Initiating hard refresh...")
203
+ background_manager = PoolManager.new(self, @seeds)
204
+ background_manager.connect
205
+
206
+ # TODO: make sure that connect has succeeded
207
+ old_manager = @manager
208
+ update_config(background_manager)
209
+ old_manager.close(:soft => true)
210
+
211
+ return true
212
+ end
213
+
227
214
  def connected?
228
- !@primary_pool.nil? || !@read_pool.nil?
215
+ self.primary_pool || self.read_pool
229
216
  end
230
217
 
231
218
  # @deprecated
@@ -238,14 +225,14 @@ module Mongo
238
225
  #
239
226
  # @return [String]
240
227
  def host
241
- super
228
+ self.primary_pool.host
242
229
  end
243
230
 
244
231
  # The replica set primary's port.
245
232
  #
246
233
  # @return [Integer]
247
234
  def port
248
- super
235
+ self.primary_pool.port
249
236
  end
250
237
 
251
238
  def nodes
@@ -259,9 +246,7 @@ module Mongo
259
246
  #
260
247
  # @return [Boolean]
261
248
  def read_primary?
262
- sync_synchronize(:SH) do
263
- @read_pool == @primary_pool
264
- end
249
+ self.read_pool == self.primary_pool
265
250
  end
266
251
  alias :primary? :read_primary?
267
252
 
@@ -271,36 +256,8 @@ module Mongo
271
256
 
272
257
  # Close the connection to the database.
273
258
  def close
274
- sync_synchronize(:EX) do
275
- @connected = false
276
- super
277
-
278
- if @refresh_thread
279
- @refresh_thread.kill
280
- @refresh_thread = nil
281
- end
282
-
283
- if @nodes
284
- @nodes.each do |member|
285
- member.close
286
- end
287
- end
288
-
289
- @nodes = []
290
- @read_pool = nil
291
-
292
- if @secondary_pools
293
- @secondary_pools.each do |pool|
294
- pool.close
295
- end
296
- end
297
-
298
- @secondaries = []
299
- @secondary_pools = []
300
- @arbiters = []
301
- @tags_to_pools.clear
302
- @sockets_to_pools.clear
303
- end
259
+ @connected = false
260
+ @manager.close(:soft => true) if @manager
304
261
  end
305
262
 
306
263
  # If a ConnectionFailure is raised, this method will be called
@@ -324,45 +281,29 @@ module Mongo
324
281
  end
325
282
 
326
283
  def authenticate_pools
327
- super
328
- @secondary_pools.each do |pool|
284
+ self.primary_pool.authenticate_existing
285
+ self.secondary_pools.each do |pool|
329
286
  pool.authenticate_existing
330
287
  end
331
288
  end
332
289
 
333
290
  def logout_pools(db)
334
- super
335
- @secondary_pools.each do |pool|
291
+ self.primary_pool.logout_existing(db)
292
+ self.secondary_pools.each do |pool|
336
293
  pool.logout_existing(db)
337
294
  end
338
295
  end
339
296
 
340
- private
341
-
342
- def initiate_refresh_mode
343
- if @refresh_mode == :async
344
- return if @refresh_thread && @refresh_thread.alive?
345
- @refresh_thread = Thread.new do
346
- while true do
347
- sleep(@refresh_interval)
348
- refresh
349
- end
350
- end
351
- end
352
-
353
- @last_refresh = Time.now
354
- end
355
-
356
297
  # Checkout a socket for reading (i.e., a secondary node).
357
298
  # Note that @read_pool might point to the primary pool
358
299
  # if no read pool has been defined.
359
300
  def checkout_reader
360
301
  connect unless connected?
361
- socket = get_socket_from_pool(@read_pool)
302
+ socket = get_socket_from_pool(self.read_pool)
362
303
 
363
304
  if !socket
364
- refresh
365
- socket = get_socket_from_pool(@primary_pool)
305
+ connect
306
+ socket = get_socket_from_pool(self.primary_pool)
366
307
  end
367
308
 
368
309
  if socket
@@ -372,35 +313,14 @@ module Mongo
372
313
  end
373
314
  end
374
315
 
375
- # Checkout a socket connected to a node with one of
376
- # the provided tags. If no such node exists, raise
377
- # an exception.
378
- #
379
- # NOTE: will be available in driver release v2.0.
380
- def checkout_tagged(tags)
381
- sync_synchronize(:SH) do
382
- tags.each do |k, v|
383
- pool = @tags_to_pools[{k.to_s => v}]
384
- if pool
385
- socket = pool.checkout
386
- @sockets_to_pools[socket] = pool
387
- return socket
388
- end
389
- end
390
- end
391
-
392
- raise NodeWithTagsNotFound,
393
- "Could not find a connection tagged with #{tags}."
394
- end
395
-
396
316
  # Checkout a socket for writing (i.e., a primary node).
397
317
  def checkout_writer
398
318
  connect unless connected?
399
- socket = get_socket_from_pool(@primary_pool)
319
+ socket = get_socket_from_pool(self.primary_pool)
400
320
 
401
321
  if !socket
402
- refresh
403
- socket = get_socket_from_pool(@primary_pool)
322
+ connect
323
+ socket = get_socket_from_pool(self.primary_pool)
404
324
  end
405
325
 
406
326
  if socket
@@ -410,49 +330,174 @@ module Mongo
410
330
  end
411
331
  end
412
332
 
333
+ # Checkin a socket used for reading.
334
+ def checkin_reader(socket)
335
+ if !self.read_pool.checkin(socket) &&
336
+ !self.primary_pool.checkin(socket)
337
+ close_socket(socket)
338
+ end
339
+ sync_refresh
340
+ end
341
+
342
+ # Checkin a socket used for writing.
343
+ def checkin_writer(socket)
344
+ if !self.primary_pool.checkin(socket)
345
+ close_socket(socket)
346
+ end
347
+ sync_refresh
348
+ end
349
+
350
+ def close_socket(socket)
351
+ begin
352
+ socket.close
353
+ rescue IOError
354
+ log(:info, "Tried to close socket #{socket} but already closed.")
355
+ end
356
+ end
357
+
413
358
  def get_socket_from_pool(pool)
414
359
  begin
415
- sync_synchronize(:SH) do
416
- if pool
417
- socket = pool.checkout
418
- @sockets_to_pools[socket] = pool
419
- socket
420
- end
360
+ if pool
361
+ socket = pool.checkout
362
+ socket
421
363
  end
422
-
423
364
  rescue ConnectionFailure => ex
424
365
  log(:info, "Failed to checkout from #{pool} with #{ex.class}; #{ex.message}")
425
366
  return nil
426
367
  end
427
368
  end
428
369
 
429
- # Checkin a socket used for reading.
430
- def checkin_reader(socket)
431
- warn "ReplSetConnection#checkin_writer is deprecated and will be removed " +
432
- "in driver v2.0. Use ReplSetConnection#checkin instead."
433
- checkin(socket)
370
+ def arbiters
371
+ @manager.arbiters.nil? ? [] : @manager.arbiters
434
372
  end
435
373
 
436
- # Checkin a socket used for writing.
437
- def checkin_writer(socket)
438
- warn "ReplSetConnection#checkin_writer is deprecated and will be removed " +
439
- "in driver v2.0. Use ReplSetConnection#checkin instead."
440
- checkin(socket)
374
+ def primary
375
+ @manager.primary
376
+ end
377
+
378
+ # Note: might want to freeze these after connecting.
379
+ def secondaries
380
+ @manager.secondaries
381
+ end
382
+
383
+ def hosts
384
+ @manager.hosts
385
+ end
386
+
387
+ def primary_pool
388
+ @manager.primary_pool
389
+ end
390
+
391
+ def read_pool
392
+ @manager.read_pool
393
+ end
394
+
395
+ def secondary_pools
396
+ @manager.secondary_pools
397
+ end
398
+
399
+ def tag_map
400
+ @manager.tag_map
401
+ end
402
+
403
+ def max_bson_size
404
+ @manager.max_bson_size
405
+ end
406
+
407
+ private
408
+
409
+ # Generic initialization code.
410
+ def setup(opts)
411
+ # Default maximum BSON object size
412
+ @max_bson_size = Mongo::DEFAULT_MAX_BSON_SIZE
413
+
414
+ @safe_mutex_lock = Mutex.new
415
+ @safe_mutexes = Hash.new {|hash, key| hash[key] = Mutex.new}
416
+
417
+ # Determine whether to use SSL.
418
+ @ssl = opts.fetch(:ssl, false)
419
+ if @ssl
420
+ @socket_class = Mongo::SSLSocket
421
+ else
422
+ @socket_class = ::TCPSocket
423
+ end
424
+
425
+ # Authentication objects
426
+ @auths = opts.fetch(:auths, [])
427
+
428
+ # Lock for request ids.
429
+ @id_lock = Mutex.new
430
+
431
+ # Pool size and timeout.
432
+ @pool_size = opts[:pool_size] || 1
433
+ if opts[:timeout]
434
+ warn "The :timeout option has been deprecated " +
435
+ "and will be removed in the 2.0 release. Use :pool_timeout instead."
436
+ end
437
+ @pool_timeout = opts[:pool_timeout] || opts[:timeout] || 5.0
438
+
439
+ # Timeout on socket read operation.
440
+ @op_timeout = opts[:op_timeout] || nil
441
+
442
+ # Timeout on socket connect.
443
+ @connect_timeout = opts[:connect_timeout] || nil
444
+
445
+ # Mutex for synchronizing pool access
446
+ # TODO: remove this.
447
+ @connection_mutex = Mutex.new
448
+
449
+ # Global safe option. This is false by default.
450
+ @safe = opts[:safe] || false
451
+
452
+ # Condition variable for signal and wait
453
+ @queue = ConditionVariable.new
454
+
455
+ @logger = opts[:logger] || nil
456
+
457
+ # Clean up connections to dead threads.
458
+ @last_cleanup = Time.now
459
+ @cleanup_lock = Mutex.new
460
+
461
+ if @logger
462
+ write_logging_startup_message
463
+ end
464
+
465
+ @last_refresh = Time.now
466
+
467
+ should_connect = opts.fetch(:connect, true)
468
+ connect if should_connect
441
469
  end
442
470
 
443
- def checkin(socket)
444
- sync_synchronize(:SH) do
445
- if pool = @sockets_to_pools[socket]
446
- pool.checkin(socket)
447
- elsif socket
448
- socket.close
471
+ # Given a pool manager, update this connection's
472
+ # view of the replica set.
473
+ def update_config(new_manager)
474
+ @manager = new_manager
475
+ @seeds = @manager.seeds.dup
476
+ @refresh_version += 1
477
+ end
478
+
479
+ # Checkout a socket connected to a node with one of
480
+ # the provided tags. If no such node exists, raise
481
+ # an exception.
482
+ #
483
+ # NOTE: will be available in driver release v2.0.
484
+ def checkout_tagged(tags)
485
+ tags.each do |k, v|
486
+ pool = self.tag_map[{k.to_s => v}]
487
+ if pool
488
+ socket = pool.checkout
489
+ return socket
449
490
  end
450
491
  end
451
492
 
452
- # Refresh synchronously every @refresh_interval seconds
453
- # if synchronous refresh mode is enabled.
493
+ raise NodeWithTagsNotFound,
494
+ "Could not find a connection tagged with #{tags}."
495
+ end
496
+
497
+ def sync_refresh
454
498
  if @refresh_mode == :sync &&
455
499
  ((Time.now - @last_refresh) > @refresh_interval)
500
+ @last_refresh = Time.now
456
501
  refresh
457
502
  end
458
503
  end