mongo 1.8.0 → 1.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/README.md +14 -29
  2. data/VERSION +1 -1
  3. data/lib/mongo.rb +3 -0
  4. data/lib/mongo/collection.rb +99 -49
  5. data/lib/mongo/cursor.rb +17 -17
  6. data/lib/mongo/db.rb +30 -14
  7. data/lib/mongo/gridfs/grid.rb +5 -3
  8. data/lib/mongo/gridfs/grid_file_system.rb +5 -3
  9. data/lib/mongo/gridfs/grid_io.rb +5 -3
  10. data/lib/mongo/legacy.rb +9 -2
  11. data/lib/mongo/mongo_client.rb +100 -72
  12. data/lib/mongo/mongo_replica_set_client.rb +46 -60
  13. data/lib/mongo/mongo_sharded_client.rb +5 -66
  14. data/lib/mongo/networking.rb +2 -1
  15. data/lib/mongo/util/node.rb +41 -42
  16. data/lib/mongo/util/pool.rb +15 -43
  17. data/lib/mongo/util/pool_manager.rb +16 -65
  18. data/lib/mongo/util/read_preference.rb +82 -0
  19. data/lib/mongo/util/sharding_pool_manager.rb +0 -86
  20. data/lib/mongo/util/ssl_socket.rb +2 -1
  21. data/lib/mongo/util/support.rb +8 -18
  22. data/lib/mongo/util/tcp_socket.rb +5 -4
  23. data/lib/mongo/util/thread_local_variable_manager.rb +29 -0
  24. data/lib/mongo/util/unix_socket.rb +23 -0
  25. data/lib/mongo/util/uri_parser.rb +31 -18
  26. data/lib/mongo/util/write_concern.rb +7 -2
  27. data/mongo.gemspec +1 -1
  28. data/test/auxillary/repl_set_auth_test.rb +2 -2
  29. data/test/bson/bson_test.rb +1 -1
  30. data/test/bson/byte_buffer_test.rb +24 -26
  31. data/test/bson/hash_with_indifferent_access_test.rb +11 -1
  32. data/test/functional/collection_test.rb +16 -16
  33. data/test/functional/connection_test.rb +1 -4
  34. data/test/functional/db_api_test.rb +14 -10
  35. data/test/functional/pool_test.rb +23 -31
  36. data/test/functional/timeout_test.rb +3 -5
  37. data/test/functional/uri_test.rb +10 -5
  38. data/test/replica_set/basic_test.rb +3 -8
  39. data/test/replica_set/client_test.rb +47 -31
  40. data/test/replica_set/complex_connect_test.rb +12 -10
  41. data/test/replica_set/connection_test.rb +8 -151
  42. data/test/replica_set/count_test.rb +9 -5
  43. data/test/replica_set/cursor_test.rb +17 -27
  44. data/test/replica_set/insert_test.rb +5 -10
  45. data/test/replica_set/query_test.rb +4 -9
  46. data/test/replica_set/read_preference_test.rb +200 -0
  47. data/test/replica_set/refresh_test.rb +54 -65
  48. data/test/replica_set/replication_ack_test.rb +16 -14
  49. data/test/sharded_cluster/basic_test.rb +30 -0
  50. data/test/test_helper.rb +33 -15
  51. data/test/threading/basic_test.rb +79 -0
  52. data/test/tools/mongo_config.rb +62 -22
  53. data/test/unit/client_test.rb +36 -14
  54. data/test/unit/collection_test.rb +23 -0
  55. data/test/unit/connection_test.rb +30 -14
  56. data/test/unit/cursor_test.rb +137 -7
  57. data/test/unit/db_test.rb +17 -4
  58. data/test/unit/grid_test.rb +2 -2
  59. data/test/unit/node_test.rb +2 -1
  60. data/test/unit/pool_manager_test.rb +29 -1
  61. data/test/unit/read_test.rb +15 -15
  62. data/test/unit/safe_test.rb +4 -4
  63. data/test/unit/write_concern_test.rb +4 -4
  64. metadata +134 -143
  65. data/examples/admin.rb +0 -43
  66. data/examples/capped.rb +0 -22
  67. data/examples/cursor.rb +0 -48
  68. data/examples/gridfs.rb +0 -44
  69. data/examples/index_test.rb +0 -126
  70. data/examples/info.rb +0 -31
  71. data/examples/queries.rb +0 -74
  72. data/examples/replica_set.rb +0 -26
  73. data/examples/simple.rb +0 -25
  74. data/examples/strict.rb +0 -35
  75. data/examples/types.rb +0 -36
  76. data/examples/web/thin/load.rb +0 -23
  77. data/examples/web/unicorn/load.rb +0 -25
  78. data/test/support/hash_with_indifferent_access.rb +0 -186
  79. data/test/support/keys.rb +0 -45
  80. data/test/threading/threading_with_large_pool_test.rb +0 -90
@@ -1,5 +1,3 @@
1
- # encoding: UTF-8
2
-
3
1
  # --
4
2
  # Copyright (C) 2008-2012 10gen Inc.
5
3
  #
@@ -19,7 +17,7 @@ module Mongo
19
17
  class Pool
20
18
  PING_ATTEMPTS = 6
21
19
  MAX_PING_TIME = 1_000_000
22
- PRUNE_INTERVAL = 10_000
20
+ include ThreadLocalVariableManager
23
21
 
24
22
  attr_accessor :host,
25
23
  :port,
@@ -58,12 +56,10 @@ module Mongo
58
56
  @socket_ops = Hash.new { |h, k| h[k] = [] }
59
57
 
60
58
  @sockets = []
61
- @pids = {}
62
59
  @checked_out = []
63
60
  @ping_time = nil
64
61
  @last_ping = nil
65
62
  @closed = false
66
- @threads_to_sockets = {}
67
63
  @checkout_counter = 0
68
64
  end
69
65
 
@@ -205,9 +201,9 @@ module Mongo
205
201
  @client.apply_saved_authentication(:socket => socket)
206
202
 
207
203
  @sockets << socket
208
- @pids[socket] = Process.pid
209
204
  @checked_out << socket
210
- @threads_to_sockets[Thread.current] = socket
205
+
206
+ thread_local[:sockets][self.object_id] = socket
211
207
  socket
212
208
  end
213
209
 
@@ -249,8 +245,7 @@ module Mongo
249
245
  socket = (@sockets - @checked_out).first
250
246
  end
251
247
 
252
- if @pids[socket] != Process.pid
253
- @pids[socket] = nil
248
+ if socket.pid != Process.pid
254
249
  @sockets.delete(socket)
255
250
  if socket
256
251
  socket.close unless socket.closed?
@@ -258,19 +253,11 @@ module Mongo
258
253
  checkout_new_socket
259
254
  else
260
255
  @checked_out << socket
261
- @threads_to_sockets[Thread.current] = socket
256
+ thread_local[:sockets][self.object_id] = socket
262
257
  socket
263
258
  end
264
259
  end
265
260
 
266
- def prune_thread_socket_hash
267
- current_threads = Set[*Thread.list]
268
-
269
- @threads_to_sockets.delete_if do |thread, socket|
270
- !current_threads.include?(thread)
271
- end
272
- end
273
-
274
261
  # Check out an existing socket or create a new socket if the maximum
275
262
  # pool size has not been exceeded. Otherwise, wait for the next
276
263
  # available socket.
@@ -278,32 +265,16 @@ module Mongo
278
265
  @client.connect if !@client.connected?
279
266
  start_time = Time.now
280
267
  loop do
281
- if (Time.now - start_time) > @timeout
282
- raise ConnectionTimeoutError, "could not obtain connection within " +
283
- "#{@timeout} seconds. The max pool size is currently #{@size}; " +
284
- "consider increasing the pool size or timeout."
285
- end
286
-
287
268
  @connection_mutex.synchronize do
288
- if @checkout_counter > PRUNE_INTERVAL
289
- @checkout_counter = 0
290
- prune_thread_socket_hash
291
- else
292
- @checkout_counter += 1
293
- end
294
-
295
- if socket_for_thread = @threads_to_sockets[Thread.current]
269
+ if socket_for_thread = thread_local[:sockets][self.object_id]
296
270
  if !@checked_out.include?(socket_for_thread)
297
271
  socket = checkout_existing_socket(socket_for_thread)
298
272
  end
299
- else # First checkout for this thread
300
- thread_length = @threads_to_sockets.keys.length
301
- if (thread_length <= @sockets.size) && (@sockets.size < @size)
273
+ else
274
+ if @sockets.size < @size
302
275
  socket = checkout_new_socket
303
276
  elsif @checked_out.size < @sockets.size
304
277
  socket = checkout_existing_socket
305
- elsif @sockets.size < @size
306
- socket = checkout_new_socket
307
278
  end
308
279
  end
309
280
 
@@ -318,12 +289,7 @@ module Mongo
318
289
  if socket.closed?
319
290
  @checked_out.delete(socket)
320
291
  @sockets.delete(socket)
321
- @threads_to_sockets.each do |k,v|
322
- if v == socket
323
- @threads_to_sockets.delete(k)
324
- end
325
- end
326
-
292
+ thread_local[:sockets].delete self.object_id
327
293
  socket = checkout_new_socket
328
294
  end
329
295
 
@@ -333,6 +299,12 @@ module Mongo
333
299
  @queue.wait(@connection_mutex)
334
300
  end
335
301
  end
302
+
303
+ if (Time.now - start_time) > @timeout
304
+ raise ConnectionTimeoutError, "could not obtain connection within " +
305
+ "#{@timeout} seconds. The max pool size is currently #{@size}; " +
306
+ "consider increasing the pool size or timeout."
307
+ end
336
308
  end
337
309
  end
338
310
 
@@ -1,12 +1,12 @@
1
1
  module Mongo
2
2
  class PoolManager
3
+ include Mongo::ReadPreference
4
+ include ThreadLocalVariableManager
3
5
 
4
6
  attr_reader :client, :arbiters, :primary, :secondaries, :primary_pool,
5
7
  :secondary_pool, :secondary_pools, :hosts, :nodes, :members, :seeds,
6
8
  :max_bson_size
7
9
 
8
- attr_accessor :pinned_pools
9
-
10
10
  # Create a new set of connection pools.
11
11
  #
12
12
  # The pool manager will by default use the original seed list passed
@@ -15,7 +15,6 @@ module Mongo
15
15
  # time. The union of these lists will be used when attempting to connect,
16
16
  # with the newly-discovered nodes being used first.
17
17
  def initialize(client, seeds=[])
18
- @pinned_pools = {}
19
18
  @client = client
20
19
  @seeds = seeds
21
20
  @previously_connected = false
@@ -49,8 +48,8 @@ module Mongo
49
48
  return
50
49
  end
51
50
 
52
- config = seed.set_config
53
- if !config
51
+ config = seed.config
52
+ if config
54
53
  @refresh_required = true
55
54
  seed.close
56
55
  return
@@ -99,30 +98,16 @@ module Mongo
99
98
  read_pool.host_port
100
99
  end
101
100
 
102
- def read_pool(mode=@client.read_preference,
101
+ def read_pool(mode=@client.read,
103
102
  tags=@client.tag_sets,
104
103
  acceptable_latency=@client.acceptable_latency)
105
104
 
106
- if mode == :primary && !tags.empty?
107
- raise MongoArgumentError, "Read preferecy :primary cannot be combined with tags"
108
- end
105
+ pinned = thread_local[:pinned_pools][self.object_id]
109
106
 
110
- pinned = @pinned_pools[Thread.current]
111
107
  if pinned && pinned.matches_mode(mode) && pinned.matches_tag_sets(tags) && pinned.up?
112
108
  pool = pinned
113
109
  else
114
- pool = case mode
115
- when :primary
116
- @primary_pool
117
- when :primary_preferred
118
- @primary_pool || select_pool(@secondary_pools, tags, acceptable_latency)
119
- when :secondary
120
- select_pool(@secondary_pools, tags, acceptable_latency)
121
- when :secondary_preferred
122
- select_pool(@secondary_pools, tags, acceptable_latency) || @primary_pool
123
- when :nearest
124
- select_pool(pools, tags, acceptable_latency)
125
- end
110
+ pool = select_pool(mode, tags, acceptable_latency)
126
111
  end
127
112
 
128
113
  unless pool
@@ -140,8 +125,8 @@ module Mongo
140
125
  private
141
126
 
142
127
  def validate_existing_member(member)
143
- config = member.set_config
144
- if !config
128
+ config = member.config
129
+ if config
145
130
  return false
146
131
  else
147
132
  if member.primary?
@@ -165,13 +150,12 @@ module Mongo
165
150
  @read = nil
166
151
  @read_pool = nil
167
152
  @arbiters = []
168
- @secondaries = []
153
+ @secondaries = Set.new
169
154
  @secondary_pool = nil
170
155
  @secondary_pools = []
171
156
  @hosts = Set.new
172
157
  @members = Set.new
173
158
  @refresh_required = false
174
- @pinned_pools = {}
175
159
  end
176
160
 
177
161
  # Connect to each member of the replica set
@@ -184,9 +168,8 @@ module Mongo
184
168
 
185
169
  seed.node_list.each do |host|
186
170
  node = Mongo::Node.new(self.client, host)
187
- if node.connect && node.set_config && node.healthy?
188
- members << node
189
- end
171
+ node.connect
172
+ members << node if node.healthy?
190
173
  end
191
174
  seed.close
192
175
 
@@ -201,10 +184,10 @@ module Mongo
201
184
  def initialize_pools(members)
202
185
  members.each do |member|
203
186
  @hosts << member.host_string
204
-
205
187
  if member.primary?
206
188
  assign_primary(member)
207
- elsif member.secondary? && !@secondaries.include?(member.host_port)
189
+ elsif member.secondary?
190
+ # member could be not primary but secondary still is false
208
191
  assign_secondary(member)
209
192
  end
210
193
  end
@@ -235,33 +218,6 @@ module Mongo
235
218
  @secondary_pools << pool
236
219
  end
237
220
 
238
- def select_pool(candidates, tag_sets, acceptable_latency)
239
- tag_sets = [tag_sets] unless tag_sets.is_a?(Array)
240
-
241
- if !tag_sets.empty?
242
- matches = []
243
- tag_sets.detect do |tag_set|
244
- matches = candidates.select do |candidate|
245
- tag_set.none? { |k,v| candidate.tags[k.to_s] != v } &&
246
- candidate.ping_time
247
- end
248
- !matches.empty?
249
- end
250
- else
251
- matches = candidates
252
- end
253
-
254
- matches.empty? ? nil : near_pool(matches, acceptable_latency)
255
- end
256
-
257
- def near_pool(pool_set, acceptable_latency)
258
- nearest_pool = pool_set.min_by { |pool| pool.ping_time }
259
- near_pools = pool_set.select do |pool|
260
- (pool.ping_time - nearest_pool.ping_time) <= acceptable_latency
261
- end
262
- near_pools[ rand(near_pools.length) ]
263
- end
264
-
265
221
  # Iterate through the list of provided seed
266
222
  # nodes until we've gotten a response from the
267
223
  # replica set we're trying to connect to.
@@ -270,13 +226,8 @@ module Mongo
270
226
  def get_valid_seed_node
271
227
  @seeds.each do |seed|
272
228
  node = Mongo::Node.new(self.client, seed)
273
- if !node.connect
274
- next
275
- elsif node.set_config && node.healthy?
276
- return node
277
- else
278
- node.close
279
- end
229
+ node.connect
230
+ return node if node.healthy?
280
231
  end
281
232
 
282
233
  raise ConnectionFailure, "Cannot connect to a replica set using seeds " +
@@ -0,0 +1,82 @@
1
+ module Mongo
2
+ module ReadPreference
3
+ READ_PREFERENCES = [
4
+ :primary,
5
+ :primary_preferred,
6
+ :secondary,
7
+ :secondary_preferred,
8
+ :nearest
9
+ ]
10
+
11
+ MONGOS_MODES = {
12
+ :primary => :primary,
13
+ :primary_preferred => :primaryPreferred,
14
+ :secondary => :secondary,
15
+ :secondary_preferred => :secondaryPreferred,
16
+ :nearest => :nearest
17
+ }
18
+
19
+ def self.mongos(mode, tag_sets)
20
+ if mode != :secondary_preferred || !tag_sets.empty?
21
+ mongos_read_preference = BSON::OrderedHash[:mode => MONGOS_MODES[mode]]
22
+ mongos_read_preference[:tags] = tag_sets if !tag_sets.empty?
23
+ end
24
+ mongos_read_preference
25
+ end
26
+
27
+ def self.validate(value)
28
+ if READ_PREFERENCES.include?(value)
29
+ return true
30
+ else
31
+ raise MongoArgumentError, "#{value} is not a valid read preference. " +
32
+ "Please specify one of the following read preferences as a symbol: #{READ_PREFERENCES}"
33
+ end
34
+ end
35
+
36
+ def select_pool(mode, tags, latency)
37
+ if mode == :primary && !tags.empty?
38
+ raise MongoArgumentError, "Read preferecy :primary cannot be combined with tags"
39
+ end
40
+
41
+ case mode
42
+ when :primary
43
+ primary_pool
44
+ when :primary_preferred
45
+ primary_pool || select_secondary_pool(secondary_pools, tags, latency)
46
+ when :secondary
47
+ select_secondary_pool(secondary_pools, tags, latency)
48
+ when :secondary_preferred
49
+ select_secondary_pool(secondary_pools, tags, latency) || primary_pool
50
+ when :nearest
51
+ select_secondary_pool(pools, tags, latency)
52
+ end
53
+ end
54
+
55
+ def select_secondary_pool(candidates, tag_sets, latency)
56
+ tag_sets = [tag_sets] unless tag_sets.is_a?(Array)
57
+
58
+ if !tag_sets.empty?
59
+ matches = []
60
+ tag_sets.detect do |tag_set|
61
+ matches = candidates.select do |candidate|
62
+ tag_set.none? { |k,v| candidate.tags[k.to_s] != v } &&
63
+ candidate.ping_time
64
+ end
65
+ !matches.empty?
66
+ end
67
+ else
68
+ matches = candidates
69
+ end
70
+
71
+ matches.empty? ? nil : select_near_pool(matches, latency)
72
+ end
73
+
74
+ def select_near_pool(candidates, latency)
75
+ nearest_pool = candidates.min_by { |candidate| candidate.ping_time }
76
+ near_pools = candidates.select do |candidate|
77
+ (candidate.ping_time - nearest_pool.ping_time) <= latency
78
+ end
79
+ near_pools[ rand(near_pools.length) ]
80
+ end
81
+ end
82
+ end
@@ -1,42 +1,5 @@
1
1
 
2
2
  module Mongo
3
- module ShardingNode
4
- def set_config
5
- begin
6
- @config = @client['admin'].command({:ismaster => 1}, :socket => @socket)
7
-
8
- # warning: instance variable @logger not initialized
9
- #if @config['msg'] && @logger
10
- # @client.log(:warn, "#{config['msg']}")
11
- #end
12
-
13
- rescue ConnectionFailure, OperationFailure, OperationTimeout, SocketError, SystemCallError, IOError => ex
14
- @client.log(:warn, "Attempted connection to node #{host_string} raised " +
15
- "#{ex.class}: #{ex.message}")
16
-
17
- # Socket may already be nil from issuing command
18
- if @socket && !@socket.closed?
19
- @socket.close
20
- end
21
-
22
- return nil
23
- end
24
-
25
- @config
26
- end
27
-
28
- # Return a list of sharded cluster nodes from the config - currently just the current node.
29
- def node_list
30
- connect unless connected?
31
- set_config unless @config
32
-
33
- return [] unless config
34
-
35
- ["#{@host}:#{@port}"]
36
- end
37
-
38
- end
39
-
40
3
  class ShardingPoolManager < PoolManager
41
4
 
42
5
  attr_reader :client, :primary, :primary_pool, :hosts, :nodes,
@@ -90,54 +53,5 @@ module Mongo
90
53
  @refresh_required = true
91
54
  end
92
55
  end
93
-
94
- private
95
-
96
- # Connect to each member of the sharded cluster
97
- # as reported by the given seed node, and return
98
- # as a list of Mongo::Node objects.
99
- def connect_to_members
100
- members = []
101
-
102
- seed = get_valid_seed_node
103
-
104
- seed.node_list.each do |host|
105
- node = Mongo::Node.new(self.client, host)
106
- node.extend ShardingNode
107
- if node.connect && node.set_config
108
- members << node
109
- end
110
- end
111
- seed.close
112
-
113
- if members.empty?
114
- raise ConnectionFailure, "Failed to connect to any given member."
115
- end
116
-
117
- members
118
- end
119
-
120
- # Iterate through the list of provided seed
121
- # nodes until we've gotten a response from the
122
- # sharded cluster we're trying to connect to.
123
- #
124
- # If we don't get a response, raise an exception.
125
- def get_valid_seed_node
126
- @seeds.each do |seed|
127
- node = Mongo::Node.new(self.client, seed)
128
- node.extend ShardingNode
129
- if !node.connect
130
- next
131
- elsif node.set_config && node.healthy?
132
- return node
133
- else
134
- node.close
135
- end
136
- end
137
-
138
- raise ConnectionFailure, "Cannot connect to a sharded cluster using seeds " +
139
- "#{@seeds.map {|s| "#{s[0]}:#{s[1]}" }.join(', ')}"
140
- end
141
-
142
56
  end
143
57
  end