mongo 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/Rakefile +50 -69
  2. data/docs/CREDITS.md +4 -0
  3. data/docs/HISTORY.md +9 -0
  4. data/docs/REPLICA_SETS.md +8 -10
  5. data/lib/mongo.rb +3 -1
  6. data/lib/mongo/collection.rb +2 -1
  7. data/lib/mongo/connection.rb +146 -314
  8. data/lib/mongo/db.rb +6 -2
  9. data/lib/mongo/gridfs/grid.rb +1 -1
  10. data/lib/mongo/gridfs/grid_io.rb +29 -5
  11. data/lib/mongo/repl_set_connection.rb +290 -0
  12. data/lib/mongo/util/pool.rb +6 -8
  13. data/lib/mongo/util/uri_parser.rb +71 -0
  14. data/mongo.gemspec +1 -2
  15. data/test/collection_test.rb +9 -7
  16. data/test/connection_test.rb +0 -6
  17. data/test/grid_file_system_test.rb +2 -2
  18. data/test/grid_io_test.rb +33 -1
  19. data/test/grid_test.rb +36 -6
  20. data/test/replica_sets/connect_test.rb +59 -21
  21. data/test/replica_sets/count_test.rb +9 -7
  22. data/test/replica_sets/insert_test.rb +11 -9
  23. data/test/replica_sets/pooled_insert_test.rb +12 -13
  24. data/test/replica_sets/query_secondaries.rb +48 -8
  25. data/test/replica_sets/query_test.rb +10 -9
  26. data/test/replica_sets/replication_ack_test.rb +15 -22
  27. data/test/replica_sets/rs_test_helper.rb +29 -0
  28. data/test/test_helper.rb +13 -20
  29. data/test/threading/{test_threading_large_pool.rb → threading_with_large_pool_test.rb} +1 -1
  30. data/test/tools/repl_set_manager.rb +241 -0
  31. data/test/tools/test.rb +13 -0
  32. data/test/unit/connection_test.rb +3 -85
  33. data/test/unit/repl_set_connection_test.rb +82 -0
  34. metadata +19 -21
  35. data/test/replica_pairs/count_test.rb +0 -34
  36. data/test/replica_pairs/insert_test.rb +0 -50
  37. data/test/replica_pairs/pooled_insert_test.rb +0 -54
  38. data/test/replica_pairs/query_test.rb +0 -39
  39. data/test/replica_sets/node_type_test.rb +0 -42
  40. data/test/rs.rb +0 -24
@@ -468,8 +468,12 @@ module Mongo
468
468
  raise MongoArgumentError, "DB#command requires an OrderedHash when hash contains multiple keys"
469
469
  end
470
470
 
471
- result = Cursor.new(system_command_collection,
472
- :limit => -1, :selector => selector, :socket => sock).next_document
471
+ begin
472
+ result = Cursor.new(system_command_collection,
473
+ :limit => -1, :selector => selector, :socket => sock).next_document
474
+ rescue OperationFailure => ex
475
+ raise OperationFailure, "Database command '#{selector.keys.first}' failed: #{ex.message}"
476
+ end
473
477
 
474
478
  if result.nil?
475
479
  raise OperationFailure, "Database command '#{selector.keys.first}' failed: returned null."
@@ -45,7 +45,7 @@ module Mongo
45
45
  end
46
46
 
47
47
  # Store a file in the file store. This method is designed only for writing new files;
48
- # if you need to update a given file, first delete it using #Grid#delete.
48
+ # if you need to update a given file, first delete it using Grid#delete.
49
49
  #
50
50
  # Note that arbitary metadata attributes can be saved to the file by passing
51
51
  # them in as options.
@@ -32,7 +32,7 @@ module Mongo
32
32
  PROTECTED_ATTRS = [:files_id, :file_length, :client_md5, :server_md5]
33
33
 
34
34
  attr_reader :content_type, :chunk_size, :upload_date, :files_id, :filename,
35
- :metadata, :server_md5, :client_md5, :file_length
35
+ :metadata, :server_md5, :client_md5, :file_length, :file_position
36
36
 
37
37
  # Create a new GridIO object. Note that most users will not need to use this class directly;
38
38
  # the Grid and GridFileSystem classes will instantiate this class
@@ -192,6 +192,23 @@ module Mongo
192
192
  id
193
193
  end
194
194
 
195
+ # Read a chunk of the data from the file and yield it to the given
196
+ # block.
197
+ #
198
+ # Note that this method reads from the current file position.
199
+ #
200
+ # @yield Yields on chunk per iteration as defined by this file's
201
+ # chunk size.
202
+ #
203
+ # @return [Mongo::GridIO] self
204
+ def each
205
+ return read_all unless block_given?
206
+ while chunk = read(chunk_size)
207
+ yield chunk
208
+ end
209
+ self
210
+ end
211
+
195
212
  def inspect
196
213
  "#<GridIO _id: #{@files_id}>"
197
214
  end
@@ -225,11 +242,14 @@ module Mongo
225
242
  # Read a file in its entirety.
226
243
  def read_all
227
244
  buf = ''
228
- while true
245
+ if @current_chunk
229
246
  buf << @current_chunk['data'].to_s
230
- @current_chunk = get_chunk(@current_chunk['n'] + 1)
231
- break unless @current_chunk
247
+ while chunk = get_chunk(@current_chunk['n'] + 1)
248
+ buf << chunk['data'].to_s
249
+ @current_chunk = chunk
250
+ end
232
251
  end
252
+ @file_position = @file_length
233
253
  buf
234
254
  end
235
255
 
@@ -237,7 +257,11 @@ module Mongo
237
257
  def read_length(length)
238
258
  cache_chunk_data
239
259
  remaining = (@file_length - @file_position)
240
- to_read = length > remaining ? remaining : length
260
+ if length.nil?
261
+ to_read = remaining
262
+ else
263
+ to_read = length > remaining ? remaining : length
264
+ end
241
265
  return nil unless remaining > 0
242
266
 
243
267
  buf = ''
@@ -0,0 +1,290 @@
1
+ # encoding: UTF-8
2
+
3
+ # --
4
+ # Copyright (C) 2008-2010 10gen Inc.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ # ++
18
+
19
+ module Mongo
20
+
21
+ # Instantiates and manages connections to a MongoDB replica set.
22
+ class ReplSetConnection < Connection
23
+ attr_reader :nodes, :secondaries, :arbiters, :read_pool, :secondary_pools
24
+
25
+ # Create a connection to a MongoDB replica set.
26
+ #
27
+ # Once connected to a replica set, you can find out which nodes are primary, secondary, and
28
+ # arbiters with the corresponding accessors: Connection#primary, Connection#secondaries, and
29
+ # Connection#arbiters. This is useful if your application needs to connect manually to nodes other
30
+ # than the primary.
31
+ #
32
+ # @param [Array] args A list of host-port pairs ending with a hash containing any options. See
33
+ # the examples below for exactly how to use the constructor.
34
+ #
35
+ # @option options [Boolean, Hash] :safe (false) Set the default safe-mode options
36
+ # propogated to DB objects instantiated off of this Connection. This
37
+ # default can be overridden upon instantiation of any DB by explicity setting a :safe value
38
+ # on initialization.
39
+ # @option options [Boolean] :read_secondary(false) If true, a random secondary node will be chosen,
40
+ # and all reads will be directed to that node.
41
+ # @option options [Logger, #debug] :logger (nil) Logger instance to receive driver operation log.
42
+ # @option options [Integer] :pool_size (1) The maximum number of socket connections allowed per
43
+ # connection pool. Note: this setting is relevant only for multi-threaded applications.
44
+ # @option options [Float] :timeout (5.0) When all of the connections a pool are checked out,
45
+ # this is the number of seconds to wait for a new connection to be released before throwing an exception.
46
+ # Note: this setting is relevant only for multi-threaded applications.
47
+ #
48
+ # @example Connect to a replica set and provide two seed nodes:
49
+ # ReplSetConnection.new(['localhost', 30000], ['localhost', 30001])
50
+ #
51
+ # @example Connect to a replica set providing two seed nodes and allowing reads from a
52
+ # secondary node:
53
+ # ReplSetConnection.new(['localhost', 30000], ['localhost', 30001], :read_secondary => true)
54
+ #
55
+ # @see http://api.mongodb.org/ruby/current/file.REPLICA_SETS.html Replica sets in Ruby
56
+ #
57
+ # @raise [ReplicaSetConnectionError] This is raised if a replica set name is specified and the
58
+ # driver fails to connect to a replica set with that name.
59
+ def initialize(*args)
60
+ if args.last.is_a?(Hash)
61
+ opts = args.pop
62
+ else
63
+ opts = {}
64
+ end
65
+
66
+ unless args.length > 0
67
+ raise MongoArgumentError, "A ReplSetConnection requires at least one node."
68
+ end
69
+
70
+ # Get seed nodes
71
+ @nodes = args
72
+
73
+ # Replica set name
74
+ @replica_set = opts[:rs_name]
75
+
76
+ # Cache the various node types when connecting to a replica set.
77
+ @secondaries = []
78
+ @arbiters = []
79
+
80
+ # Connection pools for each secondary node
81
+ @secondary_pools = []
82
+ @read_pool = nil
83
+
84
+ # Are we allowing reads from secondaries?
85
+ @read_secondary = opts.fetch(:read_secondary, false)
86
+
87
+ setup(opts)
88
+ end
89
+
90
+ # Create a new socket and attempt to connect to master.
91
+ # If successful, sets host and port to master and returns the socket.
92
+ #
93
+ # If connecting to a replica set, this method will replace the
94
+ # initially-provided seed list with any nodes known to the set.
95
+ #
96
+ # @raise [ConnectionFailure] if unable to connect to any host or port.
97
+ def connect
98
+ reset_connection
99
+ @nodes_to_try = @nodes.clone
100
+
101
+ while connecting?
102
+ node = @nodes_to_try.shift
103
+ config = check_is_master(node)
104
+
105
+ if is_primary?(config)
106
+ set_primary(node)
107
+ else
108
+ set_auxillary(node, config)
109
+ end
110
+ end
111
+
112
+ pick_secondary_for_read if @read_secondary
113
+
114
+ if !connected?
115
+ if @secondary_pools.empty?
116
+ raise ConnectionFailure, "Failed to connect any given host:port"
117
+ else
118
+ raise ConnectionFailure, "Failed to connect to primary node."
119
+ end
120
+ end
121
+ end
122
+
123
+ def connecting?
124
+ @nodes_to_try.length > 0
125
+ end
126
+
127
+ # Close the connection to the database.
128
+ def close
129
+ super
130
+ @read_pool = nil
131
+ @secondary_pools.each do |pool|
132
+ pool.close
133
+ end
134
+ end
135
+
136
+ # If a ConnectionFailure is raised, this method will be called
137
+ # to close the connection and reset connection values.
138
+ # TODO: what's the point of this method?
139
+ def reset_connection
140
+ super
141
+ @secondaries = []
142
+ @secondary_pools = []
143
+ @arbiters = []
144
+ @nodes_tried = []
145
+ @nodes_to_try = []
146
+ end
147
+
148
+ # Is it okay to connect to a slave?
149
+ #
150
+ # @return [Boolean]
151
+ def slave_ok?
152
+ @read_secondary || @slave_ok
153
+ end
154
+
155
+ private
156
+
157
+ def check_is_master(node)
158
+ begin
159
+ host, port = *node
160
+ socket = TCPSocket.new(host, port)
161
+ socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
162
+
163
+ config = self['admin'].command({:ismaster => 1}, :sock => socket)
164
+
165
+ check_set_name(config, socket)
166
+ rescue OperationFailure, SocketError, SystemCallError, IOError => ex
167
+ close unless connected?
168
+ ensure
169
+ @nodes_tried << node
170
+ if config
171
+ nodes = []
172
+ nodes += config['hosts'] if config['hosts']
173
+ nodes += config['arbiters'] if config['arbiters']
174
+ nodes += config['passives'] if config['passives']
175
+ update_node_list(nodes)
176
+
177
+ if config['msg'] && @logger
178
+ @logger.warn("MONGODB #{config['msg']}")
179
+ end
180
+ end
181
+
182
+ socket.close if socket
183
+ end
184
+
185
+ config
186
+ end
187
+
188
+ # Primary, when connecting to a replica can, can only be a true primary node.
189
+ # (And not a slave, which is possible when connecting with the standard
190
+ # Connection class.
191
+ def is_primary?(config)
192
+ config && (config['ismaster'] == 1 || config['ismaster'] == true)
193
+ end
194
+
195
+ # Pick a node randomly from the set of possible secondaries.
196
+ def pick_secondary_for_read
197
+ if (size = @secondary_pools.size) > 0
198
+ @read_pool = @secondary_pools[rand(size)]
199
+ end
200
+ end
201
+
202
+ # Make sure that we're connected to the expected replica set.
203
+ def check_set_name(config, socket)
204
+ if @replica_set
205
+ config = self['admin'].command({:replSetGetStatus => 1},
206
+ :sock => socket, :check_response => false)
207
+
208
+ if !Mongo::Support.ok?(config)
209
+ raise ReplicaSetConnectionError, config['errmsg']
210
+ elsif config['set'] != @replica_set
211
+ raise ReplicaSetConnectionError,
212
+ "Attempting to connect to replica set '#{config['set']}' but expected '#{@replica_set}'"
213
+ end
214
+ end
215
+ end
216
+
217
+ # Determines what kind of node we have and caches its host
218
+ # and port so that users can easily connect manually.
219
+ def set_auxillary(node, config)
220
+ if config
221
+ if config['secondary']
222
+ host, port = *node
223
+ @secondaries << node unless @secondaries.include?(node)
224
+ @secondary_pools << Pool.new(self, host, port, :size => @pool_size, :timeout => @timeout)
225
+ elsif config['arbiterOnly']
226
+ @arbiters << node unless @arbiters.include?(node)
227
+ end
228
+ end
229
+ end
230
+
231
+ # Update the list of known nodes. Only applies to replica sets,
232
+ # where the response to the ismaster command will return a list
233
+ # of known hosts.
234
+ #
235
+ # @param hosts [Array] a list of hosts, specified as string-encoded
236
+ # host-port values. Example: ["myserver-1.org:27017", "myserver-1.org:27017"]
237
+ #
238
+ # @return [Array] the updated list of nodes
239
+ def update_node_list(hosts)
240
+ new_nodes = hosts.map do |host|
241
+ if !host.respond_to?(:split)
242
+ warn "Could not parse host #{host.inspect}."
243
+ next
244
+ end
245
+
246
+ host, port = host.split(':')
247
+ [host, port ? port.to_i : Connection::DEFAULT_PORT]
248
+ end
249
+
250
+ # Replace the list of seed nodes with the canonical list.
251
+ @nodes = new_nodes.clone
252
+
253
+ @nodes_to_try = new_nodes - @nodes_tried
254
+ end
255
+
256
+ # Checkout a socket for reading (i.e., a secondary node).
257
+ def checkout_reader
258
+ connect unless connected?
259
+
260
+ if @read_pool
261
+ @read_pool.checkout
262
+ else
263
+ checkout_writer
264
+ end
265
+ end
266
+
267
+ # Checkout a socket for writing (i.e., a primary node).
268
+ def checkout_writer
269
+ connect unless connected?
270
+
271
+ @primary_pool.checkout
272
+ end
273
+
274
+ # Checkin a socket used for reading.
275
+ def checkin_reader(socket)
276
+ if @read_pool
277
+ @read_pool.checkin(socket)
278
+ else
279
+ checkin_writer(socket)
280
+ end
281
+ end
282
+
283
+ # Checkin a socket used for writing.
284
+ def checkin_writer(socket)
285
+ if @primary_pool
286
+ @primary_pool.checkin(socket)
287
+ end
288
+ end
289
+ end
290
+ end
@@ -34,13 +34,6 @@ module Mongo
34
34
  # Mutex for synchronizing pool access
35
35
  @connection_mutex = Mutex.new
36
36
 
37
- # Global safe option. This is false by default.
38
- @safe = options[:safe] || false
39
-
40
- # Create a mutex when a new key, in this case a socket,
41
- # is added to the hash.
42
- @safe_mutexes = Hash.new { |h, k| h[k] = Mutex.new }
43
-
44
37
  # Condition variable for signal and wait
45
38
  @queue = ConditionVariable.new
46
39
 
@@ -50,7 +43,12 @@ module Mongo
50
43
 
51
44
  def close
52
45
  @sockets.each do |sock|
53
- sock.close
46
+ begin
47
+ sock.close
48
+ rescue IOError => ex
49
+ warn "IOError when attempting to close socket connected "
50
+ + "to #{@host}:#{@port}: #{ex.inspect}"
51
+ end
54
52
  end
55
53
  @host = @port = nil
56
54
  @sockets.clear
@@ -0,0 +1,71 @@
1
+ # encoding: UTF-8
2
+
3
+ # --
4
+ # Copyright (C) 2008-2010 10gen Inc.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ # ++
18
+
19
+ module Mongo
20
+ module URIParser
21
+
22
+ DEFAULT_PORT = 27017
23
+ MONGODB_URI_MATCHER = /(([-_.\w\d]+):([-_\w\d]+)@)?([-.\w\d]+)(:([\w\d]+))?(\/([-\d\w]+))?/
24
+ MONGODB_URI_SPEC = "mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/database]"
25
+
26
+ extend self
27
+
28
+ # Parse a MongoDB URI. This method is used by Connection.from_uri.
29
+ # Returns an array of nodes and an array of db authorizations, if applicable.
30
+ #
31
+ # @private
32
+ def parse(string)
33
+ if string =~ /^mongodb:\/\//
34
+ string = string[10..-1]
35
+ else
36
+ raise MongoArgumentError, "MongoDB URI must match this spec: #{MONGODB_URI_SPEC}"
37
+ end
38
+
39
+ nodes = []
40
+ auths = []
41
+ specs = string.split(',')
42
+ specs.each do |spec|
43
+ matches = MONGODB_URI_MATCHER.match(spec)
44
+ if !matches
45
+ raise MongoArgumentError, "MongoDB URI must match this spec: #{MONGODB_URI_SPEC}"
46
+ end
47
+
48
+ uname = matches[2]
49
+ pwd = matches[3]
50
+ host = matches[4]
51
+ port = matches[6] || DEFAULT_PORT
52
+ if !(port.to_s =~ /^\d+$/)
53
+ raise MongoArgumentError, "Invalid port #{port}; port must be specified as digits."
54
+ end
55
+ port = port.to_i
56
+ db = matches[8]
57
+
58
+ if uname && pwd && db
59
+ auths << {'db_name' => db, 'username' => uname, 'password' => pwd}
60
+ elsif uname || pwd || db
61
+ raise MongoArgumentError, "MongoDB URI must include all three of username, password, " +
62
+ "and db if any one of these is specified."
63
+ end
64
+
65
+ nodes << [host, port]
66
+ end
67
+
68
+ [nodes, auths]
69
+ end
70
+ end
71
+ end