mongo 1.7.1 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. data/{LICENSE.txt → LICENSE} +0 -0
  2. data/README.md +124 -111
  3. data/Rakefile +9 -325
  4. data/VERSION +1 -0
  5. data/bin/mongo_console +4 -4
  6. data/examples/admin.rb +43 -0
  7. data/examples/capped.rb +22 -0
  8. data/examples/cursor.rb +48 -0
  9. data/examples/gridfs.rb +44 -0
  10. data/examples/index_test.rb +126 -0
  11. data/examples/info.rb +31 -0
  12. data/examples/queries.rb +74 -0
  13. data/examples/replica_set.rb +26 -0
  14. data/examples/simple.rb +25 -0
  15. data/examples/strict.rb +35 -0
  16. data/examples/types.rb +36 -0
  17. data/{test/load → examples/web}/thin/load.rb +3 -1
  18. data/{test/load → examples/web}/unicorn/load.rb +5 -3
  19. data/lib/mongo.rb +8 -10
  20. data/lib/mongo/collection.rb +134 -114
  21. data/lib/mongo/cursor.rb +21 -14
  22. data/lib/mongo/db.rb +30 -28
  23. data/lib/mongo/exceptions.rb +1 -1
  24. data/lib/mongo/gridfs/grid.rb +8 -7
  25. data/lib/mongo/gridfs/grid_ext.rb +1 -1
  26. data/lib/mongo/gridfs/grid_file_system.rb +6 -5
  27. data/lib/mongo/gridfs/grid_io.rb +22 -19
  28. data/lib/mongo/legacy.rb +82 -0
  29. data/lib/mongo/{connection.rb → mongo_client.rb} +82 -61
  30. data/lib/mongo/{repl_set_connection.rb → mongo_replica_set_client.rb} +54 -39
  31. data/lib/mongo/{sharded_connection.rb → mongo_sharded_client.rb} +9 -9
  32. data/lib/mongo/networking.rb +25 -20
  33. data/lib/mongo/util/conversions.rb +1 -1
  34. data/lib/mongo/util/core_ext.rb +1 -1
  35. data/lib/mongo/util/logging.rb +20 -4
  36. data/lib/mongo/util/node.rb +16 -16
  37. data/lib/mongo/util/pool.rb +56 -27
  38. data/lib/mongo/util/pool_manager.rb +28 -27
  39. data/lib/mongo/util/server_version.rb +1 -1
  40. data/lib/mongo/util/sharding_pool_manager.rb +8 -8
  41. data/lib/mongo/util/ssl_socket.rb +1 -5
  42. data/lib/mongo/util/support.rb +24 -8
  43. data/lib/mongo/util/tcp_socket.rb +0 -4
  44. data/lib/mongo/util/uri_parser.rb +54 -38
  45. data/lib/mongo/util/write_concern.rb +67 -0
  46. data/mongo.gemspec +21 -32
  47. data/test/auxillary/{1.4_features.rb → 1.4_feature_test.rb} +4 -5
  48. data/test/auxillary/authentication_test.rb +18 -20
  49. data/test/auxillary/autoreconnect_test.rb +3 -5
  50. data/test/auxillary/fork_test.rb +5 -7
  51. data/test/auxillary/repl_set_auth_test.rb +13 -15
  52. data/test/auxillary/slave_connection_test.rb +8 -7
  53. data/test/auxillary/threaded_authentication_test.rb +15 -17
  54. data/test/bson/binary_test.rb +1 -1
  55. data/test/bson/bson_test.rb +60 -36
  56. data/test/bson/byte_buffer_test.rb +1 -1
  57. data/test/bson/hash_with_indifferent_access_test.rb +2 -2
  58. data/test/bson/json_test.rb +1 -2
  59. data/test/bson/object_id_test.rb +1 -2
  60. data/test/bson/ordered_hash_test.rb +1 -1
  61. data/test/bson/timestamp_test.rb +1 -1
  62. data/test/{collection_test.rb → functional/collection_test.rb} +57 -57
  63. data/test/{connection_test.rb → functional/connection_test.rb} +75 -89
  64. data/test/{conversions_test.rb → functional/conversions_test.rb} +1 -1
  65. data/test/{cursor_fail_test.rb → functional/cursor_fail_test.rb} +3 -29
  66. data/test/{cursor_message_test.rb → functional/cursor_message_test.rb} +1 -1
  67. data/test/{cursor_test.rb → functional/cursor_test.rb} +5 -1
  68. data/test/{db_api_test.rb → functional/db_api_test.rb} +8 -9
  69. data/test/{db_connection_test.rb → functional/db_connection_test.rb} +3 -5
  70. data/test/{db_test.rb → functional/db_test.rb} +13 -13
  71. data/test/{grid_file_system_test.rb → functional/grid_file_system_test.rb} +2 -2
  72. data/test/{grid_io_test.rb → functional/grid_io_test.rb} +6 -6
  73. data/test/{grid_test.rb → functional/grid_test.rb} +4 -10
  74. data/test/{pool_test.rb → functional/pool_test.rb} +1 -1
  75. data/test/functional/safe_test.rb +84 -0
  76. data/test/{support_test.rb → functional/support_test.rb} +1 -1
  77. data/test/{threading_test.rb → functional/threading_test.rb} +9 -9
  78. data/test/{timeout_test.rb → functional/timeout_test.rb} +1 -1
  79. data/test/{uri_test.rb → functional/uri_test.rb} +1 -1
  80. data/test/functional/write_concern_test.rb +104 -0
  81. data/test/replica_set/basic_test.rb +139 -0
  82. data/test/replica_set/client_test.rb +255 -0
  83. data/test/replica_set/complex_connect_test.rb +62 -0
  84. data/test/replica_set/connection_test.rb +255 -0
  85. data/test/{replica_sets → replica_set}/count_test.rb +17 -14
  86. data/test/replica_set/cursor_test.rb +75 -0
  87. data/test/{replica_sets → replica_set}/insert_test.rb +19 -16
  88. data/test/replica_set/query_test.rb +64 -0
  89. data/test/replica_set/refresh_test.rb +153 -0
  90. data/test/{replica_sets → replica_set}/replication_ack_test.rb +21 -17
  91. data/test/sharded_cluster/basic_test.rb +31 -50
  92. data/test/support/hash_with_indifferent_access.rb +1 -1
  93. data/test/test_helper.rb +56 -9
  94. data/test/threading/threading_with_large_pool_test.rb +8 -8
  95. data/test/tools/mongo_config.rb +270 -58
  96. data/test/tools/mongo_config_test.rb +146 -0
  97. data/test/unit/client_test.rb +230 -0
  98. data/test/unit/collection_test.rb +45 -32
  99. data/test/unit/connection_test.rb +82 -74
  100. data/test/unit/cursor_test.rb +14 -6
  101. data/test/unit/db_test.rb +8 -8
  102. data/test/unit/grid_test.rb +11 -11
  103. data/test/unit/node_test.rb +24 -24
  104. data/test/unit/pool_manager_test.rb +13 -13
  105. data/test/unit/pool_test.rb +1 -1
  106. data/test/unit/read_test.rb +21 -26
  107. data/test/unit/safe_test.rb +52 -33
  108. data/test/unit/util_test.rb +55 -0
  109. data/test/unit/write_concern_test.rb +161 -0
  110. metadata +158 -171
  111. data/docs/CREDITS.md +0 -123
  112. data/docs/FAQ.md +0 -116
  113. data/docs/GRID_FS.md +0 -158
  114. data/docs/HISTORY.md +0 -392
  115. data/docs/READ_PREFERENCE.md +0 -99
  116. data/docs/RELEASES.md +0 -54
  117. data/docs/REPLICA_SETS.md +0 -113
  118. data/docs/TAILABLE_CURSORS.md +0 -51
  119. data/docs/TUTORIAL.md +0 -356
  120. data/docs/WRITE_CONCERN.md +0 -31
  121. data/lib/mongo/gridfs/grid_io_fix.rb +0 -38
  122. data/lib/mongo/version.rb +0 -3
  123. data/test/bson/test_helper.rb +0 -30
  124. data/test/replica_sets/basic_test.rb +0 -119
  125. data/test/replica_sets/complex_connect_test.rb +0 -57
  126. data/test/replica_sets/complex_read_preference_test.rb +0 -237
  127. data/test/replica_sets/connect_test.rb +0 -156
  128. data/test/replica_sets/cursor_test.rb +0 -70
  129. data/test/replica_sets/pooled_insert_test.rb +0 -57
  130. data/test/replica_sets/query_test.rb +0 -50
  131. data/test/replica_sets/read_preference_test.rb +0 -234
  132. data/test/replica_sets/refresh_test.rb +0 -156
  133. data/test/replica_sets/refresh_with_threads_test.rb +0 -60
  134. data/test/replica_sets/rs_test_helper.rb +0 -39
  135. data/test/safe_test.rb +0 -68
  136. data/test/sharded_cluster/mongo_config_test.rb +0 -126
  137. data/test/sharded_cluster/sc_test_helper.rb +0 -39
  138. data/test/tools/repl_set_manager.rb +0 -418
@@ -1,39 +0,0 @@
1
- $:.unshift(File.expand_path('../../lib', File.dirname(__FILE__)))
2
- require File.expand_path("../../test_helper", __FILE__)
3
- require 'test/tools/mongo_config'
4
-
5
- class Test::Unit::TestCase
6
- # Ensure sharded cluster is available as an instance variable and that
7
- # a new set is spun up for each TestCase class
8
- def ensure_sc
9
- if defined?(@@current_class) and @@current_class == self.class
10
- @@sc.start
11
- else
12
- @@current_class = self.class
13
- dbpath = 'sc'
14
- opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE.merge(:dbpath => dbpath).merge(:routers => 4)
15
- #debug 1, opts
16
- config = Mongo::Config.cluster(opts)
17
- #debug 1, config
18
- @@sc = Mongo::Config::ClusterManager.new(config)
19
- @@sc.start
20
- end
21
- @sc = @@sc
22
- end
23
-
24
- # Generic code for rescuing connection failures and retrying operations.
25
- # This could be combined with some timeout functionality.
26
- def rescue_connection_failure(max_retries=30)
27
- retries = 0
28
- begin
29
- yield
30
- rescue Mongo::ConnectionFailure => ex
31
- #puts "Rescue attempt #{retries}: from #{ex}"
32
- retries += 1
33
- raise ex if retries > max_retries
34
- sleep(2)
35
- retry
36
- end
37
- end
38
-
39
- end
@@ -1,418 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- require 'thread'
4
-
5
- STDOUT.sync = true
6
-
7
- unless defined? Mongo
8
- require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
9
- end
10
-
11
- class ReplSetManager
12
-
13
- attr_accessor :host, :start_port, :ports, :name, :mongods, :tags, :version
14
-
15
- def initialize(opts={})
16
- @mongod = ENV['mongod'] || 'mongod'
17
- @start_port = opts[:start_port] || 30000
18
- @ports = []
19
- @name = opts[:name] || 'replica-set-foo'
20
- @host = opts[:host] || 'localhost'
21
- @retries = opts[:retries] || 30
22
- @config = {"_id" => @name, "members" => []}
23
- @durable = opts.fetch(:durable, false)
24
- @smallfiles = opts.fetch(:smallfiles, true)
25
- @prealloc = opts.fetch(:prealloc, false)
26
- @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
27
- @oplog_size = opts.fetch(:oplog_size, 16)
28
- @tags = [{"dc" => "ny", "rack" => "a", "db" => "main"},
29
- {"dc" => "ny", "rack" => "b", "db" => "main"},
30
- {"dc" => "sf", "rack" => "a", "db" => "main"}]
31
-
32
- @arbiter_count = opts[:arbiter_count] || 0
33
- @secondary_count = opts[:secondary_count] || 2
34
- @passive_count = opts[:passive_count] || 0
35
- @primary_count = 1
36
-
37
- @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
38
- if @count > 7
39
- raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
40
- end
41
-
42
- @mongods = {}
43
- version_string = `#{@mongod} --version`
44
- version_string =~ /(\d\.\d\.\d)/
45
- @version = $1
46
- end
47
-
48
- def start_set
49
- system("killall mongod > /dev/null 2> /dev/null")
50
- sleep(1)
51
- should_start = true
52
- #puts "** Starting a replica set with #{@count} nodes"
53
-
54
- n = 0
55
- (@primary_count + @secondary_count).times do
56
- init_node(n, should_start) do |attrs|
57
- if @version >= "2"
58
- attrs['tags'] = @tags[n % @tags.size]
59
- end
60
- end
61
- n += 1
62
- end
63
-
64
- @passive_count.times do
65
- init_node(n, should_start) do |attrs|
66
- attrs['priority'] = 0
67
- end
68
- n += 1
69
- end
70
-
71
- @arbiter_count.times do
72
- init_node(n, should_start) do |attrs|
73
- attrs['arbiterOnly'] = true
74
- end
75
- n += 1
76
- end
77
-
78
- initiate
79
- ensure_up
80
- end
81
-
82
- def cleanup_set
83
- system("killall mongod")
84
- @count.times do |n|
85
- system("rm -rf #{@mongods[n]['db_path']}")
86
- end
87
- end
88
-
89
- def init_node(n, should_start=true)
90
- @mongods[n] ||= {}
91
- port = @start_port + n
92
- @ports << port
93
- @mongods[n]['port'] = port
94
- @mongods[n]['db_path'] = get_path("rs-#{port}")
95
- @mongods[n]['log_path'] = get_path("log-#{port}")
96
- @mongods[n]['start'] = start_cmd(n)
97
-
98
- if should_start
99
- system("rm -rf #{@mongods[n]['db_path']}")
100
- system("mkdir -p #{@mongods[n]['db_path']}")
101
- start(n)
102
- end
103
-
104
- member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
105
-
106
- if block_given?
107
- custom_attrs = {}
108
- yield custom_attrs
109
- member.merge!(custom_attrs)
110
- @mongods[n].merge!(custom_attrs)
111
- end
112
-
113
- @config['members'] << member
114
- end
115
-
116
- def journal_switch
117
- if @version >= "2"
118
- if @durable
119
- "--journal"
120
- else
121
- "--nojournal"
122
- end
123
- elsif @durable
124
- "--journal"
125
- end
126
- end
127
-
128
- def start_cmd(n)
129
- @mongods[n]['start'] = "#{@mongod} --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
130
- "--oplogSize #{@oplog_size} #{journal_switch} --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
131
- @mongods[n]['start'] += " --dur" if @durable
132
- @mongods[n]['start'] += " --smallfiles" if @smallfiles
133
- @mongods[n]['start'] += " --noprealloc" unless @prealloc
134
- @mongods[n]['start'] += "> /dev/null 2>/dev/null"
135
- @mongods[n]['start']
136
- end
137
-
138
- def remove_secondary_node
139
- primary = get_node_with_state(1)
140
- con = get_connection(primary)
141
- @config = con['local']['system.replset'].find_one
142
- secondary = get_node_with_state(2)
143
- host_port = "#{@host}:#{@mongods[secondary]['port']}"
144
- kill(secondary)
145
- @mongods.delete(secondary)
146
- @config['members'].reject! {|m| m['host'] == host_port}
147
- @config['version'] += 1
148
-
149
- begin
150
- con['admin'].command({'replSetReconfig' => @config})
151
- rescue Mongo::ConnectionFailure
152
- end
153
-
154
- con.close
155
- ensure_up
156
-
157
- return secondary
158
- end
159
-
160
- def add_node(n=nil, &block)
161
- primary = get_node_with_state(1)
162
- con = get_connection(primary)
163
-
164
- @config = con['local']['system.replset'].find_one
165
- init_node(n || @mongods.length, &block)
166
- @config['version'] += 1
167
-
168
- # We expect a connection failure on reconfigure here.
169
- begin
170
- con['admin'].command({'replSetReconfig' => @config})
171
- rescue Mongo::ConnectionFailure
172
- end
173
-
174
- con.close
175
- ensure_up
176
- end
177
-
178
- def add_arbiter(n=nil)
179
- add_node(n) do |attrs|
180
- attrs['arbiterOnly'] = true
181
- end
182
- end
183
-
184
- def wait_for_death(pid)
185
- @retries.times do
186
- if `ps a | grep mongod`.include?("#{pid}")
187
- #puts "waiting for mongod @ pid #{pid} to die..."
188
- sleep(1)
189
- else
190
- #puts "mongod @ pid #{pid} was killed successfully"
191
- return true
192
- end
193
- end
194
- #puts "mongod never died"
195
- return false
196
- end
197
-
198
- def kill(node, signal=2)
199
- pid = @mongods[node]['pid']
200
- #puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
201
- system("kill #{pid}")
202
- dead = wait_for_death(pid)
203
- @mongods[node]['up'] = false if dead
204
- end
205
-
206
- def kill_primary(signal=2)
207
- node = get_node_with_state(1)
208
- kill(node, signal)
209
- return node
210
- end
211
-
212
- # Note that we have to rescue a connection failure
213
- # when we run the StepDown command because that
214
- # command will close the connection.
215
- def step_down_primary
216
- primary = get_node_with_state(1)
217
- con = get_connection(primary)
218
- begin
219
- con['admin'].command({'replSetStepDown' => 90})
220
- rescue Mongo::ConnectionFailure
221
- end
222
- con.close
223
- end
224
-
225
- def kill_secondary
226
- node = get_node_with_state(2)
227
- kill(node)
228
- return node
229
- end
230
-
231
- def kill_all_secondaries
232
- nodes = get_all_nodes_with_state(2)
233
- if nodes
234
- nodes.each do |n|
235
- kill(n)
236
- end
237
- end
238
- end
239
-
240
- def restart_killed_nodes
241
- nodes = @mongods.keys.select do |key|
242
- @mongods[key]['up'] == false
243
- end
244
-
245
- nodes.each do |node|
246
- start(node)
247
- end
248
-
249
- ensure_up
250
- end
251
-
252
- def get_node_from_port(port)
253
- @mongods.keys.detect { |key| @mongods[key]['port'] == port }
254
- end
255
-
256
- def start(node)
257
- system(@mongods[node]['start'])
258
- @mongods[node]['up'] = true
259
- sleep(0.5)
260
- @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
261
- end
262
- alias :restart :start
263
-
264
- def ensure_up(n=nil, connection=nil)
265
- #print "** Ensuring members are up..."
266
-
267
- attempt(n) do
268
- #print "."
269
- con = connection || get_connection
270
- begin
271
- status = con['admin'].command({:replSetGetStatus => 1})
272
- rescue Mongo::OperationFailure => ex
273
- con.close
274
- raise ex
275
- end
276
- if status['members'].all? { |m| m['health'] == 1 &&
277
- [1, 2, 7].include?(m['state']) } &&
278
- status['members'].any? { |m| m['state'] == 1 }
279
-
280
- connections = []
281
- states = []
282
- status['members'].each do |member|
283
- begin
284
- host, port = member['name'].split(':')
285
- port = port.to_i
286
- conn = Mongo::Connection.new(host, port, :slave_ok => true)
287
- connections << conn
288
- state = conn['admin'].command({:ismaster => 1})
289
- states << state
290
- rescue Mongo::ConnectionFailure
291
- connections.each {|c| c.close }
292
- con.close
293
- raise Mongo::OperationFailure
294
- end
295
- end
296
-
297
- if states.any? {|s| s['ismaster']}
298
- #puts "all members up!"
299
- connections.each {|c| c.close }
300
- con.close
301
- return status
302
- else
303
- con.close
304
- raise Mongo::OperationFailure
305
- end
306
- else
307
- con.close
308
- raise Mongo::OperationFailure
309
- end
310
- end
311
- return false
312
- end
313
-
314
- def primary
315
- nodes = get_all_host_pairs_with_state(1)
316
- nodes.empty? ? nil : nodes[0]
317
- end
318
-
319
- def secondaries
320
- get_all_host_pairs_with_state(2)
321
- end
322
-
323
- def arbiters
324
- get_all_host_pairs_with_state(7)
325
- end
326
-
327
- # String used for adding a shard via mongos
328
- # using the addshard command.
329
- def shard_string
330
- str = "#{@name}/"
331
- str << @mongods.map do |k, mongod|
332
- "#{@host}:#{mongod['port']}"
333
- end.join(',')
334
- str
335
- end
336
-
337
- #private
338
-
339
- def initiate
340
- #puts "Initiating replica set..."
341
- con = get_connection
342
-
343
- attempt do
344
- con.object_id
345
- con['admin'].command({'replSetInitiate' => @config})
346
- end
347
-
348
- con.close
349
- end
350
-
351
- def get_all_nodes_with_state(state)
352
- status = ensure_up
353
- nodes = status['members'].select {|m| m['state'] == state}
354
- nodes = nodes.map do |node|
355
- host_port = node['name'].split(':')
356
- port = host_port[1] ? host_port[1].to_i : 27017
357
- @mongods.keys.detect {|key| @mongods[key]['port'] == port}
358
- end
359
-
360
- nodes == [] ? false : nodes
361
- end
362
-
363
- def get_node_with_state(state)
364
- status = ensure_up
365
- node = status['members'].detect {|m| m['state'] == state}
366
- if node
367
- host_port = node['name'].split(':')
368
- port = host_port[1] ? host_port[1].to_i : 27017
369
- key = @mongods.keys.detect {|n| @mongods[n]['port'] == port}
370
- return key
371
- else
372
- return false
373
- end
374
- end
375
-
376
- def get_all_host_pairs_with_state(state)
377
- status = ensure_up
378
- nodes = status['members'].select {|m| m['state'] == state}
379
- nodes.map do |node|
380
- host_port = node['name'].split(':')
381
- port = host_port[1] ? host_port[1].to_i : 27017
382
- [host, port]
383
- end
384
- end
385
-
386
- def get_connection(node=nil)
387
- con = attempt do
388
- if !node
389
- node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
390
- end
391
- con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
392
- end
393
-
394
- return con
395
- end
396
-
397
- def get_path(name)
398
- File.join(@path, name)
399
- end
400
-
401
- def attempt(retries=nil)
402
- raise "No block given!" unless block_given?
403
- count = 0
404
-
405
- while count < (retries || @retries) do
406
- begin
407
- return yield
408
- rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
409
- sleep(2)
410
- count += 1
411
- end
412
- end
413
-
414
- puts "NO MORE ATTEMPTS"
415
- raise ex
416
- end
417
-
418
- end