mongo 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +50 -69
- data/docs/CREDITS.md +4 -0
- data/docs/HISTORY.md +9 -0
- data/docs/REPLICA_SETS.md +8 -10
- data/lib/mongo.rb +3 -1
- data/lib/mongo/collection.rb +2 -1
- data/lib/mongo/connection.rb +146 -314
- data/lib/mongo/db.rb +6 -2
- data/lib/mongo/gridfs/grid.rb +1 -1
- data/lib/mongo/gridfs/grid_io.rb +29 -5
- data/lib/mongo/repl_set_connection.rb +290 -0
- data/lib/mongo/util/pool.rb +6 -8
- data/lib/mongo/util/uri_parser.rb +71 -0
- data/mongo.gemspec +1 -2
- data/test/collection_test.rb +9 -7
- data/test/connection_test.rb +0 -6
- data/test/grid_file_system_test.rb +2 -2
- data/test/grid_io_test.rb +33 -1
- data/test/grid_test.rb +36 -6
- data/test/replica_sets/connect_test.rb +59 -21
- data/test/replica_sets/count_test.rb +9 -7
- data/test/replica_sets/insert_test.rb +11 -9
- data/test/replica_sets/pooled_insert_test.rb +12 -13
- data/test/replica_sets/query_secondaries.rb +48 -8
- data/test/replica_sets/query_test.rb +10 -9
- data/test/replica_sets/replication_ack_test.rb +15 -22
- data/test/replica_sets/rs_test_helper.rb +29 -0
- data/test/test_helper.rb +13 -20
- data/test/threading/{test_threading_large_pool.rb → threading_with_large_pool_test.rb} +1 -1
- data/test/tools/repl_set_manager.rb +241 -0
- data/test/tools/test.rb +13 -0
- data/test/unit/connection_test.rb +3 -85
- data/test/unit/repl_set_connection_test.rb +82 -0
- metadata +19 -21
- data/test/replica_pairs/count_test.rb +0 -34
- data/test/replica_pairs/insert_test.rb +0 -50
- data/test/replica_pairs/pooled_insert_test.rb +0 -54
- data/test/replica_pairs/query_test.rb +0 -39
- data/test/replica_sets/node_type_test.rb +0 -42
- data/test/rs.rb +0 -24
data/lib/mongo/db.rb
CHANGED
@@ -468,8 +468,12 @@ module Mongo
|
|
468
468
|
raise MongoArgumentError, "DB#command requires an OrderedHash when hash contains multiple keys"
|
469
469
|
end
|
470
470
|
|
471
|
-
|
472
|
-
|
471
|
+
begin
|
472
|
+
result = Cursor.new(system_command_collection,
|
473
|
+
:limit => -1, :selector => selector, :socket => sock).next_document
|
474
|
+
rescue OperationFailure => ex
|
475
|
+
raise OperationFailure, "Database command '#{selector.keys.first}' failed: #{ex.message}"
|
476
|
+
end
|
473
477
|
|
474
478
|
if result.nil?
|
475
479
|
raise OperationFailure, "Database command '#{selector.keys.first}' failed: returned null."
|
data/lib/mongo/gridfs/grid.rb
CHANGED
@@ -45,7 +45,7 @@ module Mongo
|
|
45
45
|
end
|
46
46
|
|
47
47
|
# Store a file in the file store. This method is designed only for writing new files;
|
48
|
-
# if you need to update a given file, first delete it using
|
48
|
+
# if you need to update a given file, first delete it using Grid#delete.
|
49
49
|
#
|
50
50
|
# Note that arbitary metadata attributes can be saved to the file by passing
|
51
51
|
# them in as options.
|
data/lib/mongo/gridfs/grid_io.rb
CHANGED
@@ -32,7 +32,7 @@ module Mongo
|
|
32
32
|
PROTECTED_ATTRS = [:files_id, :file_length, :client_md5, :server_md5]
|
33
33
|
|
34
34
|
attr_reader :content_type, :chunk_size, :upload_date, :files_id, :filename,
|
35
|
-
:metadata, :server_md5, :client_md5, :file_length
|
35
|
+
:metadata, :server_md5, :client_md5, :file_length, :file_position
|
36
36
|
|
37
37
|
# Create a new GridIO object. Note that most users will not need to use this class directly;
|
38
38
|
# the Grid and GridFileSystem classes will instantiate this class
|
@@ -192,6 +192,23 @@ module Mongo
|
|
192
192
|
id
|
193
193
|
end
|
194
194
|
|
195
|
+
# Read a chunk of the data from the file and yield it to the given
|
196
|
+
# block.
|
197
|
+
#
|
198
|
+
# Note that this method reads from the current file position.
|
199
|
+
#
|
200
|
+
# @yield Yields on chunk per iteration as defined by this file's
|
201
|
+
# chunk size.
|
202
|
+
#
|
203
|
+
# @return [Mongo::GridIO] self
|
204
|
+
def each
|
205
|
+
return read_all unless block_given?
|
206
|
+
while chunk = read(chunk_size)
|
207
|
+
yield chunk
|
208
|
+
end
|
209
|
+
self
|
210
|
+
end
|
211
|
+
|
195
212
|
def inspect
|
196
213
|
"#<GridIO _id: #{@files_id}>"
|
197
214
|
end
|
@@ -225,11 +242,14 @@ module Mongo
|
|
225
242
|
# Read a file in its entirety.
|
226
243
|
def read_all
|
227
244
|
buf = ''
|
228
|
-
|
245
|
+
if @current_chunk
|
229
246
|
buf << @current_chunk['data'].to_s
|
230
|
-
|
231
|
-
|
247
|
+
while chunk = get_chunk(@current_chunk['n'] + 1)
|
248
|
+
buf << chunk['data'].to_s
|
249
|
+
@current_chunk = chunk
|
250
|
+
end
|
232
251
|
end
|
252
|
+
@file_position = @file_length
|
233
253
|
buf
|
234
254
|
end
|
235
255
|
|
@@ -237,7 +257,11 @@ module Mongo
|
|
237
257
|
def read_length(length)
|
238
258
|
cache_chunk_data
|
239
259
|
remaining = (@file_length - @file_position)
|
240
|
-
|
260
|
+
if length.nil?
|
261
|
+
to_read = remaining
|
262
|
+
else
|
263
|
+
to_read = length > remaining ? remaining : length
|
264
|
+
end
|
241
265
|
return nil unless remaining > 0
|
242
266
|
|
243
267
|
buf = ''
|
@@ -0,0 +1,290 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2010 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
module Mongo
|
20
|
+
|
21
|
+
# Instantiates and manages connections to a MongoDB replica set.
|
22
|
+
class ReplSetConnection < Connection
|
23
|
+
attr_reader :nodes, :secondaries, :arbiters, :read_pool, :secondary_pools
|
24
|
+
|
25
|
+
# Create a connection to a MongoDB replica set.
|
26
|
+
#
|
27
|
+
# Once connected to a replica set, you can find out which nodes are primary, secondary, and
|
28
|
+
# arbiters with the corresponding accessors: Connection#primary, Connection#secondaries, and
|
29
|
+
# Connection#arbiters. This is useful if your application needs to connect manually to nodes other
|
30
|
+
# than the primary.
|
31
|
+
#
|
32
|
+
# @param [Array] args A list of host-port pairs ending with a hash containing any options. See
|
33
|
+
# the examples below for exactly how to use the constructor.
|
34
|
+
#
|
35
|
+
# @option options [Boolean, Hash] :safe (false) Set the default safe-mode options
|
36
|
+
# propogated to DB objects instantiated off of this Connection. This
|
37
|
+
# default can be overridden upon instantiation of any DB by explicity setting a :safe value
|
38
|
+
# on initialization.
|
39
|
+
# @option options [Boolean] :read_secondary(false) If true, a random secondary node will be chosen,
|
40
|
+
# and all reads will be directed to that node.
|
41
|
+
# @option options [Logger, #debug] :logger (nil) Logger instance to receive driver operation log.
|
42
|
+
# @option options [Integer] :pool_size (1) The maximum number of socket connections allowed per
|
43
|
+
# connection pool. Note: this setting is relevant only for multi-threaded applications.
|
44
|
+
# @option options [Float] :timeout (5.0) When all of the connections a pool are checked out,
|
45
|
+
# this is the number of seconds to wait for a new connection to be released before throwing an exception.
|
46
|
+
# Note: this setting is relevant only for multi-threaded applications.
|
47
|
+
#
|
48
|
+
# @example Connect to a replica set and provide two seed nodes:
|
49
|
+
# ReplSetConnection.new(['localhost', 30000], ['localhost', 30001])
|
50
|
+
#
|
51
|
+
# @example Connect to a replica set providing two seed nodes and allowing reads from a
|
52
|
+
# secondary node:
|
53
|
+
# ReplSetConnection.new(['localhost', 30000], ['localhost', 30001], :read_secondary => true)
|
54
|
+
#
|
55
|
+
# @see http://api.mongodb.org/ruby/current/file.REPLICA_SETS.html Replica sets in Ruby
|
56
|
+
#
|
57
|
+
# @raise [ReplicaSetConnectionError] This is raised if a replica set name is specified and the
|
58
|
+
# driver fails to connect to a replica set with that name.
|
59
|
+
def initialize(*args)
|
60
|
+
if args.last.is_a?(Hash)
|
61
|
+
opts = args.pop
|
62
|
+
else
|
63
|
+
opts = {}
|
64
|
+
end
|
65
|
+
|
66
|
+
unless args.length > 0
|
67
|
+
raise MongoArgumentError, "A ReplSetConnection requires at least one node."
|
68
|
+
end
|
69
|
+
|
70
|
+
# Get seed nodes
|
71
|
+
@nodes = args
|
72
|
+
|
73
|
+
# Replica set name
|
74
|
+
@replica_set = opts[:rs_name]
|
75
|
+
|
76
|
+
# Cache the various node types when connecting to a replica set.
|
77
|
+
@secondaries = []
|
78
|
+
@arbiters = []
|
79
|
+
|
80
|
+
# Connection pools for each secondary node
|
81
|
+
@secondary_pools = []
|
82
|
+
@read_pool = nil
|
83
|
+
|
84
|
+
# Are we allowing reads from secondaries?
|
85
|
+
@read_secondary = opts.fetch(:read_secondary, false)
|
86
|
+
|
87
|
+
setup(opts)
|
88
|
+
end
|
89
|
+
|
90
|
+
# Create a new socket and attempt to connect to master.
|
91
|
+
# If successful, sets host and port to master and returns the socket.
|
92
|
+
#
|
93
|
+
# If connecting to a replica set, this method will replace the
|
94
|
+
# initially-provided seed list with any nodes known to the set.
|
95
|
+
#
|
96
|
+
# @raise [ConnectionFailure] if unable to connect to any host or port.
|
97
|
+
def connect
|
98
|
+
reset_connection
|
99
|
+
@nodes_to_try = @nodes.clone
|
100
|
+
|
101
|
+
while connecting?
|
102
|
+
node = @nodes_to_try.shift
|
103
|
+
config = check_is_master(node)
|
104
|
+
|
105
|
+
if is_primary?(config)
|
106
|
+
set_primary(node)
|
107
|
+
else
|
108
|
+
set_auxillary(node, config)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
pick_secondary_for_read if @read_secondary
|
113
|
+
|
114
|
+
if !connected?
|
115
|
+
if @secondary_pools.empty?
|
116
|
+
raise ConnectionFailure, "Failed to connect any given host:port"
|
117
|
+
else
|
118
|
+
raise ConnectionFailure, "Failed to connect to primary node."
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def connecting?
|
124
|
+
@nodes_to_try.length > 0
|
125
|
+
end
|
126
|
+
|
127
|
+
# Close the connection to the database.
|
128
|
+
def close
|
129
|
+
super
|
130
|
+
@read_pool = nil
|
131
|
+
@secondary_pools.each do |pool|
|
132
|
+
pool.close
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# If a ConnectionFailure is raised, this method will be called
|
137
|
+
# to close the connection and reset connection values.
|
138
|
+
# TODO: what's the point of this method?
|
139
|
+
def reset_connection
|
140
|
+
super
|
141
|
+
@secondaries = []
|
142
|
+
@secondary_pools = []
|
143
|
+
@arbiters = []
|
144
|
+
@nodes_tried = []
|
145
|
+
@nodes_to_try = []
|
146
|
+
end
|
147
|
+
|
148
|
+
# Is it okay to connect to a slave?
|
149
|
+
#
|
150
|
+
# @return [Boolean]
|
151
|
+
def slave_ok?
|
152
|
+
@read_secondary || @slave_ok
|
153
|
+
end
|
154
|
+
|
155
|
+
private
|
156
|
+
|
157
|
+
def check_is_master(node)
|
158
|
+
begin
|
159
|
+
host, port = *node
|
160
|
+
socket = TCPSocket.new(host, port)
|
161
|
+
socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
|
162
|
+
|
163
|
+
config = self['admin'].command({:ismaster => 1}, :sock => socket)
|
164
|
+
|
165
|
+
check_set_name(config, socket)
|
166
|
+
rescue OperationFailure, SocketError, SystemCallError, IOError => ex
|
167
|
+
close unless connected?
|
168
|
+
ensure
|
169
|
+
@nodes_tried << node
|
170
|
+
if config
|
171
|
+
nodes = []
|
172
|
+
nodes += config['hosts'] if config['hosts']
|
173
|
+
nodes += config['arbiters'] if config['arbiters']
|
174
|
+
nodes += config['passives'] if config['passives']
|
175
|
+
update_node_list(nodes)
|
176
|
+
|
177
|
+
if config['msg'] && @logger
|
178
|
+
@logger.warn("MONGODB #{config['msg']}")
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
socket.close if socket
|
183
|
+
end
|
184
|
+
|
185
|
+
config
|
186
|
+
end
|
187
|
+
|
188
|
+
# Primary, when connecting to a replica can, can only be a true primary node.
|
189
|
+
# (And not a slave, which is possible when connecting with the standard
|
190
|
+
# Connection class.
|
191
|
+
def is_primary?(config)
|
192
|
+
config && (config['ismaster'] == 1 || config['ismaster'] == true)
|
193
|
+
end
|
194
|
+
|
195
|
+
# Pick a node randomly from the set of possible secondaries.
|
196
|
+
def pick_secondary_for_read
|
197
|
+
if (size = @secondary_pools.size) > 0
|
198
|
+
@read_pool = @secondary_pools[rand(size)]
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Make sure that we're connected to the expected replica set.
|
203
|
+
def check_set_name(config, socket)
|
204
|
+
if @replica_set
|
205
|
+
config = self['admin'].command({:replSetGetStatus => 1},
|
206
|
+
:sock => socket, :check_response => false)
|
207
|
+
|
208
|
+
if !Mongo::Support.ok?(config)
|
209
|
+
raise ReplicaSetConnectionError, config['errmsg']
|
210
|
+
elsif config['set'] != @replica_set
|
211
|
+
raise ReplicaSetConnectionError,
|
212
|
+
"Attempting to connect to replica set '#{config['set']}' but expected '#{@replica_set}'"
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
# Determines what kind of node we have and caches its host
|
218
|
+
# and port so that users can easily connect manually.
|
219
|
+
def set_auxillary(node, config)
|
220
|
+
if config
|
221
|
+
if config['secondary']
|
222
|
+
host, port = *node
|
223
|
+
@secondaries << node unless @secondaries.include?(node)
|
224
|
+
@secondary_pools << Pool.new(self, host, port, :size => @pool_size, :timeout => @timeout)
|
225
|
+
elsif config['arbiterOnly']
|
226
|
+
@arbiters << node unless @arbiters.include?(node)
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
# Update the list of known nodes. Only applies to replica sets,
|
232
|
+
# where the response to the ismaster command will return a list
|
233
|
+
# of known hosts.
|
234
|
+
#
|
235
|
+
# @param hosts [Array] a list of hosts, specified as string-encoded
|
236
|
+
# host-port values. Example: ["myserver-1.org:27017", "myserver-1.org:27017"]
|
237
|
+
#
|
238
|
+
# @return [Array] the updated list of nodes
|
239
|
+
def update_node_list(hosts)
|
240
|
+
new_nodes = hosts.map do |host|
|
241
|
+
if !host.respond_to?(:split)
|
242
|
+
warn "Could not parse host #{host.inspect}."
|
243
|
+
next
|
244
|
+
end
|
245
|
+
|
246
|
+
host, port = host.split(':')
|
247
|
+
[host, port ? port.to_i : Connection::DEFAULT_PORT]
|
248
|
+
end
|
249
|
+
|
250
|
+
# Replace the list of seed nodes with the canonical list.
|
251
|
+
@nodes = new_nodes.clone
|
252
|
+
|
253
|
+
@nodes_to_try = new_nodes - @nodes_tried
|
254
|
+
end
|
255
|
+
|
256
|
+
# Checkout a socket for reading (i.e., a secondary node).
|
257
|
+
def checkout_reader
|
258
|
+
connect unless connected?
|
259
|
+
|
260
|
+
if @read_pool
|
261
|
+
@read_pool.checkout
|
262
|
+
else
|
263
|
+
checkout_writer
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
# Checkout a socket for writing (i.e., a primary node).
|
268
|
+
def checkout_writer
|
269
|
+
connect unless connected?
|
270
|
+
|
271
|
+
@primary_pool.checkout
|
272
|
+
end
|
273
|
+
|
274
|
+
# Checkin a socket used for reading.
|
275
|
+
def checkin_reader(socket)
|
276
|
+
if @read_pool
|
277
|
+
@read_pool.checkin(socket)
|
278
|
+
else
|
279
|
+
checkin_writer(socket)
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
# Checkin a socket used for writing.
|
284
|
+
def checkin_writer(socket)
|
285
|
+
if @primary_pool
|
286
|
+
@primary_pool.checkin(socket)
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
end
|
data/lib/mongo/util/pool.rb
CHANGED
@@ -34,13 +34,6 @@ module Mongo
|
|
34
34
|
# Mutex for synchronizing pool access
|
35
35
|
@connection_mutex = Mutex.new
|
36
36
|
|
37
|
-
# Global safe option. This is false by default.
|
38
|
-
@safe = options[:safe] || false
|
39
|
-
|
40
|
-
# Create a mutex when a new key, in this case a socket,
|
41
|
-
# is added to the hash.
|
42
|
-
@safe_mutexes = Hash.new { |h, k| h[k] = Mutex.new }
|
43
|
-
|
44
37
|
# Condition variable for signal and wait
|
45
38
|
@queue = ConditionVariable.new
|
46
39
|
|
@@ -50,7 +43,12 @@ module Mongo
|
|
50
43
|
|
51
44
|
def close
|
52
45
|
@sockets.each do |sock|
|
53
|
-
|
46
|
+
begin
|
47
|
+
sock.close
|
48
|
+
rescue IOError => ex
|
49
|
+
warn "IOError when attempting to close socket connected "
|
50
|
+
+ "to #{@host}:#{@port}: #{ex.inspect}"
|
51
|
+
end
|
54
52
|
end
|
55
53
|
@host = @port = nil
|
56
54
|
@sockets.clear
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2010 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
module Mongo
|
20
|
+
module URIParser
|
21
|
+
|
22
|
+
DEFAULT_PORT = 27017
|
23
|
+
MONGODB_URI_MATCHER = /(([-_.\w\d]+):([-_\w\d]+)@)?([-.\w\d]+)(:([\w\d]+))?(\/([-\d\w]+))?/
|
24
|
+
MONGODB_URI_SPEC = "mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/database]"
|
25
|
+
|
26
|
+
extend self
|
27
|
+
|
28
|
+
# Parse a MongoDB URI. This method is used by Connection.from_uri.
|
29
|
+
# Returns an array of nodes and an array of db authorizations, if applicable.
|
30
|
+
#
|
31
|
+
# @private
|
32
|
+
def parse(string)
|
33
|
+
if string =~ /^mongodb:\/\//
|
34
|
+
string = string[10..-1]
|
35
|
+
else
|
36
|
+
raise MongoArgumentError, "MongoDB URI must match this spec: #{MONGODB_URI_SPEC}"
|
37
|
+
end
|
38
|
+
|
39
|
+
nodes = []
|
40
|
+
auths = []
|
41
|
+
specs = string.split(',')
|
42
|
+
specs.each do |spec|
|
43
|
+
matches = MONGODB_URI_MATCHER.match(spec)
|
44
|
+
if !matches
|
45
|
+
raise MongoArgumentError, "MongoDB URI must match this spec: #{MONGODB_URI_SPEC}"
|
46
|
+
end
|
47
|
+
|
48
|
+
uname = matches[2]
|
49
|
+
pwd = matches[3]
|
50
|
+
host = matches[4]
|
51
|
+
port = matches[6] || DEFAULT_PORT
|
52
|
+
if !(port.to_s =~ /^\d+$/)
|
53
|
+
raise MongoArgumentError, "Invalid port #{port}; port must be specified as digits."
|
54
|
+
end
|
55
|
+
port = port.to_i
|
56
|
+
db = matches[8]
|
57
|
+
|
58
|
+
if uname && pwd && db
|
59
|
+
auths << {'db_name' => db, 'username' => uname, 'password' => pwd}
|
60
|
+
elsif uname || pwd || db
|
61
|
+
raise MongoArgumentError, "MongoDB URI must include all three of username, password, " +
|
62
|
+
"and db if any one of these is specified."
|
63
|
+
end
|
64
|
+
|
65
|
+
nodes << [host, port]
|
66
|
+
end
|
67
|
+
|
68
|
+
[nodes, auths]
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|