mongo 1.8.0 → 1.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/README.md +14 -29
  2. data/VERSION +1 -1
  3. data/lib/mongo.rb +3 -0
  4. data/lib/mongo/collection.rb +99 -49
  5. data/lib/mongo/cursor.rb +17 -17
  6. data/lib/mongo/db.rb +30 -14
  7. data/lib/mongo/gridfs/grid.rb +5 -3
  8. data/lib/mongo/gridfs/grid_file_system.rb +5 -3
  9. data/lib/mongo/gridfs/grid_io.rb +5 -3
  10. data/lib/mongo/legacy.rb +9 -2
  11. data/lib/mongo/mongo_client.rb +100 -72
  12. data/lib/mongo/mongo_replica_set_client.rb +46 -60
  13. data/lib/mongo/mongo_sharded_client.rb +5 -66
  14. data/lib/mongo/networking.rb +2 -1
  15. data/lib/mongo/util/node.rb +41 -42
  16. data/lib/mongo/util/pool.rb +15 -43
  17. data/lib/mongo/util/pool_manager.rb +16 -65
  18. data/lib/mongo/util/read_preference.rb +82 -0
  19. data/lib/mongo/util/sharding_pool_manager.rb +0 -86
  20. data/lib/mongo/util/ssl_socket.rb +2 -1
  21. data/lib/mongo/util/support.rb +8 -18
  22. data/lib/mongo/util/tcp_socket.rb +5 -4
  23. data/lib/mongo/util/thread_local_variable_manager.rb +29 -0
  24. data/lib/mongo/util/unix_socket.rb +23 -0
  25. data/lib/mongo/util/uri_parser.rb +31 -18
  26. data/lib/mongo/util/write_concern.rb +7 -2
  27. data/mongo.gemspec +1 -1
  28. data/test/auxillary/repl_set_auth_test.rb +2 -2
  29. data/test/bson/bson_test.rb +1 -1
  30. data/test/bson/byte_buffer_test.rb +24 -26
  31. data/test/bson/hash_with_indifferent_access_test.rb +11 -1
  32. data/test/functional/collection_test.rb +16 -16
  33. data/test/functional/connection_test.rb +1 -4
  34. data/test/functional/db_api_test.rb +14 -10
  35. data/test/functional/pool_test.rb +23 -31
  36. data/test/functional/timeout_test.rb +3 -5
  37. data/test/functional/uri_test.rb +10 -5
  38. data/test/replica_set/basic_test.rb +3 -8
  39. data/test/replica_set/client_test.rb +47 -31
  40. data/test/replica_set/complex_connect_test.rb +12 -10
  41. data/test/replica_set/connection_test.rb +8 -151
  42. data/test/replica_set/count_test.rb +9 -5
  43. data/test/replica_set/cursor_test.rb +17 -27
  44. data/test/replica_set/insert_test.rb +5 -10
  45. data/test/replica_set/query_test.rb +4 -9
  46. data/test/replica_set/read_preference_test.rb +200 -0
  47. data/test/replica_set/refresh_test.rb +54 -65
  48. data/test/replica_set/replication_ack_test.rb +16 -14
  49. data/test/sharded_cluster/basic_test.rb +30 -0
  50. data/test/test_helper.rb +33 -15
  51. data/test/threading/basic_test.rb +79 -0
  52. data/test/tools/mongo_config.rb +62 -22
  53. data/test/unit/client_test.rb +36 -14
  54. data/test/unit/collection_test.rb +23 -0
  55. data/test/unit/connection_test.rb +30 -14
  56. data/test/unit/cursor_test.rb +137 -7
  57. data/test/unit/db_test.rb +17 -4
  58. data/test/unit/grid_test.rb +2 -2
  59. data/test/unit/node_test.rb +2 -1
  60. data/test/unit/pool_manager_test.rb +29 -1
  61. data/test/unit/read_test.rb +15 -15
  62. data/test/unit/safe_test.rb +4 -4
  63. data/test/unit/write_concern_test.rb +4 -4
  64. metadata +134 -143
  65. data/examples/admin.rb +0 -43
  66. data/examples/capped.rb +0 -22
  67. data/examples/cursor.rb +0 -48
  68. data/examples/gridfs.rb +0 -44
  69. data/examples/index_test.rb +0 -126
  70. data/examples/info.rb +0 -31
  71. data/examples/queries.rb +0 -74
  72. data/examples/replica_set.rb +0 -26
  73. data/examples/simple.rb +0 -25
  74. data/examples/strict.rb +0 -35
  75. data/examples/types.rb +0 -36
  76. data/examples/web/thin/load.rb +0 -23
  77. data/examples/web/unicorn/load.rb +0 -25
  78. data/test/support/hash_with_indifferent_access.rb +0 -186
  79. data/test/support/keys.rb +0 -45
  80. data/test/threading/threading_with_large_pool_test.rb +0 -90
@@ -7,8 +7,8 @@ class ReplicaSetAckTest < Test::Unit::TestCase
7
7
  @client = MongoReplicaSetClient.new(@rs.repl_set_seeds)
8
8
 
9
9
  @slave1 = MongoClient.new(
10
- @client.secondary_pools[0].host,
11
- @client.secondary_pools[0].port, :slave_ok => true)
10
+ @client.secondary_pools.first.host,
11
+ @client.secondary_pools.first.port, :slave_ok => true)
12
12
 
13
13
  assert !@slave1.read_primary?
14
14
 
@@ -21,33 +21,35 @@ class ReplicaSetAckTest < Test::Unit::TestCase
21
21
  @client.close if @conn
22
22
  end
23
23
 
24
- def self.shutdown
25
- @@cluster.stop
26
- @@cluster.clobber
27
- end
28
-
29
24
  def test_safe_mode_with_w_failure
30
25
  assert_raise_error OperationFailure, "timeout" do
31
- @col.insert({:foo => 1}, :w => 4, :wtimeout => 1, :fsync => true)
26
+ @col.insert({:foo => 1}, :w => 3, :wtimeout => 1, :fsync => true)
32
27
  end
33
28
  assert_raise_error OperationFailure, "timeout" do
34
- @col.update({:foo => 1}, {:foo => 2}, :w => 4, :wtimeout => 1, :fsync => true)
29
+ @col.update({:foo => 1}, {:foo => 2}, :w => 3, :wtimeout => 1, :fsync => true)
35
30
  end
36
31
  assert_raise_error OperationFailure, "timeout" do
37
- @col.remove({:foo => 2}, :w => 4, :wtimeout => 1, :fsync => true)
32
+ @col.remove({:foo => 2}, :w => 3, :wtimeout => 1, :fsync => true)
33
+ end
34
+ assert_raise_error OperationFailure do
35
+ @col.insert({:foo => 3}, :w => "test-tag")
38
36
  end
39
37
  end
40
38
 
41
39
  def test_safe_mode_replication_ack
42
- @col.insert({:baz => "bar"}, :w => 3, :wtimeout => 5000)
40
+ @col.insert({:baz => "bar"}, :w => 2, :wtimeout => 5000)
43
41
 
44
- assert @col.insert({:foo => "0" * 5000}, :w => 3, :wtimeout => 5000)
42
+ assert @col.insert({:foo => "0" * 5000}, :w => 2, :wtimeout => 5000)
45
43
  assert_equal 2, @slave1[MONGO_TEST_DB]["test-sets"].count
46
44
 
47
- assert @col.update({:baz => "bar"}, {:baz => "foo"}, :w => 3, :wtimeout => 5000)
45
+ assert @col.update({:baz => "bar"}, {:baz => "foo"}, :w => 2, :wtimeout => 5000)
48
46
  assert @slave1[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
49
47
 
50
- assert @col.remove({}, :w => 3, :wtimeout => 5000)
48
+ assert @col.insert({:foo => "bar"}, :w => "majority")
49
+
50
+ assert @col.insert({:bar => "baz"}, :w => :majority)
51
+
52
+ assert @col.remove({}, :w => 2, :wtimeout => 5000)
51
53
  assert_equal 0, @slave1[MONGO_TEST_DB]["test-sets"].count
52
54
  end
53
55
 
@@ -1,6 +1,10 @@
1
1
  require 'test_helper'
2
2
  include Mongo
3
3
 
4
+ class Cursor
5
+ public :construct_query_spec
6
+ end
7
+
4
8
  class BasicTest < Test::Unit::TestCase
5
9
 
6
10
  def setup
@@ -22,6 +26,32 @@ class BasicTest < Test::Unit::TestCase
22
26
  @client.close
23
27
  end
24
28
 
29
+ def test_connect_from_standard_client
30
+ mongos = @sc.mongos_seeds.first
31
+ @client = MongoClient.new(*mongos.split(':'))
32
+ assert @client.connected?
33
+ assert @client.mongos?
34
+ @client.close
35
+ end
36
+
37
+ def test_read_from_client
38
+ host, port = @sc.mongos_seeds.first.split(':')
39
+ tags = [{:dc => "mongolia"}]
40
+ @client = MongoClient.new(host, port, {:read => :secondary, :tag_sets => tags})
41
+ assert @client.connected?
42
+ cursor = Cursor.new(@client[MONGO_TEST_DB]['whatever'], {})
43
+ assert_equal cursor.construct_query_spec['$readPreference'], {:mode => :secondary, :tags => tags}
44
+ end
45
+
46
+ def test_read_from_sharded_client
47
+ seeds = @sc.mongos_seeds
48
+ tags = [{:dc => "mongolia"}]
49
+ @client = MongoShardedClient.new(seeds, {:read => :secondary, :tag_sets => tags})
50
+ assert @client.connected?
51
+ cursor = Cursor.new(@client[MONGO_TEST_DB]['whatever'], {})
52
+ assert_equal cursor.construct_query_spec['$readPreference'], {:mode => :secondary, :tags => tags}
53
+ end
54
+
25
55
  def test_hard_refresh
26
56
  seeds = @sc.mongos_seeds
27
57
  @client = MongoShardedClient.new(seeds)
@@ -6,29 +6,28 @@ class Test::Unit::TestCase
6
6
 
7
7
  TEST_DATA = File.join(File.dirname(__FILE__), 'data')
8
8
 
9
- def ensure_cluster(kind=nil)
10
- if defined?(@@current_class) and @@current_class == self.class
11
- @@cluster.start
12
- else
13
- @@current_class = self.class
9
+ def ensure_cluster(kind=nil, opts={})
10
+ @@cluster ||= false
14
11
 
12
+ unless @@cluster
15
13
  if kind == :rs
16
- opts = Mongo::Config::DEFAULT_REPLICA_SET
17
- opts.merge!(:arbiters => 2)
14
+ cluster_opts = Mongo::Config::DEFAULT_REPLICA_SET.dup
18
15
  else
19
- opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE
20
- opts.merge!(:routers => 4)
16
+ cluster_opts = Mongo::Config::DEFAULT_SHARDED_SIMPLE.dup
21
17
  end
22
18
 
19
+ cluster_opts.merge!(opts)
20
+
23
21
  dbpath = ENV['DBPATH'] || 'data'
24
- opts.merge!(:dbpath => dbpath)
22
+ cluster_opts.merge!(:dbpath => dbpath)
25
23
 
26
24
  #debug 1, opts
27
- config = Mongo::Config.cluster(opts)
25
+ config = Mongo::Config.cluster(cluster_opts)
28
26
  #debug 1, config
29
27
  @@cluster = Mongo::Config::ClusterManager.new(config)
30
- @@cluster.start
31
28
  end
29
+
30
+ @@cluster.start
32
31
  instance_variable_set("@#{kind}", @@cluster)
33
32
  end
34
33
 
@@ -61,7 +60,7 @@ end
61
60
 
62
61
  begin
63
62
  silently { require 'shoulda' }
64
- silently { require 'mocha' }
63
+ silently { require 'mocha/setup' }
65
64
  rescue LoadError
66
65
  puts <<MSG
67
66
 
@@ -74,8 +73,6 @@ MSG
74
73
  exit
75
74
  end
76
75
 
77
- require 'bson_ext/cbson' if !(RUBY_PLATFORM =~ /java/) && ENV['C_EXT']
78
-
79
76
  unless defined? MONGO_TEST_DB
80
77
  MONGO_TEST_DB = 'ruby-test-db'
81
78
  end
@@ -128,6 +125,19 @@ class Test::Unit::TestCase
128
125
  self.class.mongo_port
129
126
  end
130
127
 
128
+ def method_name
129
+ caller[0]=~/`(.*?)'/
130
+ $1
131
+ end
132
+
133
+ def step_down_command
134
+ # Adding force=true to avoid 'no secondaries within 10 seconds of my optime' errors
135
+ step_down_command = BSON::OrderedHash.new
136
+ step_down_command[:replSetStepDown] = 5
137
+ step_down_command[:force] = true
138
+ step_down_command
139
+ end
140
+
131
141
  def new_mock_socket(host='localhost', port=27017)
132
142
  socket = Object.new
133
143
  socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
@@ -136,6 +146,14 @@ class Test::Unit::TestCase
136
146
  socket
137
147
  end
138
148
 
149
+ def new_mock_unix_socket(sockfile='/tmp/mongod.sock')
150
+ socket = Object.new
151
+ socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP)
152
+ socket.stubs(:close)
153
+ socket.stubs(:closed?)
154
+ socket
155
+ end
156
+
139
157
  def new_mock_db
140
158
  Object.new
141
159
  end
@@ -0,0 +1,79 @@
1
+ require 'test_helper'
2
+
3
+ class TestThreading < Test::Unit::TestCase
4
+
5
+ include Mongo
6
+
7
+ def setup
8
+ @client = standard_connection(:pool_size => 50, :pool_timeout => 60)
9
+ @db = @client.db(MONGO_TEST_DB)
10
+ @coll = @db.collection('thread-test-collection')
11
+ @coll.drop
12
+
13
+ collections = ['duplicate', 'unique']
14
+
15
+ collections.each do |coll_name|
16
+ coll = @db.collection(coll_name)
17
+ coll.drop
18
+ coll.insert("test" => "insert")
19
+ coll.insert("test" => "update")
20
+ instance_variable_set("@#{coll_name}", coll)
21
+ end
22
+
23
+ @unique.create_index("test", :unique => true)
24
+ end
25
+
26
+ def test_safe_update
27
+ threads = []
28
+ 300.times do |i|
29
+ threads << Thread.new do
30
+ if i % 2 == 0
31
+ assert_raise Mongo::OperationFailure do
32
+ @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}})
33
+ end
34
+ else
35
+ @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}})
36
+ @duplicate.update({"test" => "update"}, {"$set" => {"test" => "insert"}})
37
+ end
38
+ end
39
+ end
40
+
41
+ threads.each {|thread| thread.join}
42
+ end
43
+
44
+ def test_safe_insert
45
+ threads = []
46
+ 300.times do |i|
47
+ threads << Thread.new do
48
+ if i % 2 == 0
49
+ assert_raise Mongo::OperationFailure do
50
+ @unique.insert({"test" => "insert"})
51
+ end
52
+ else
53
+ @duplicate.insert({"test" => "insert"})
54
+ end
55
+ end
56
+ end
57
+
58
+ threads.each {|thread| thread.join}
59
+ end
60
+
61
+ def test_count
62
+ 1000.times do |i|
63
+ @coll.insert({ "x" => i })
64
+ end
65
+
66
+ threads = []
67
+ 10.times do |i|
68
+ threads << Thread.new do
69
+ sum = 0
70
+ @coll.find().each do |document|
71
+ sum += document["x"]
72
+ end
73
+ assert_equal 499500, sum
74
+ end
75
+ end
76
+
77
+ threads.each {|thread| thread.join}
78
+ end
79
+ end
@@ -25,8 +25,8 @@ end
25
25
  module Mongo
26
26
  class Config
27
27
  DEFAULT_BASE_OPTS = { :host => 'localhost', :dbpath => 'data', :logpath => 'data/log'}
28
- DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 3, :arbiters => 0 )
29
- DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 2 )
28
+ DEFAULT_REPLICA_SET = DEFAULT_BASE_OPTS.merge( :replicas => 2, :arbiters => 1 )
29
+ DEFAULT_SHARDED_SIMPLE = DEFAULT_BASE_OPTS.merge( :shards => 2, :configs => 1, :routers => 4 )
30
30
  DEFAULT_SHARDED_REPLICA = DEFAULT_SHARDED_SIMPLE.merge( :replicas => 3, :arbiters => 0)
31
31
 
32
32
  IGNORE_KEYS = [:host, :command, :_id]
@@ -35,7 +35,7 @@ module Mongo
35
35
  MONGODS_OPT_KEYS = [:mongods]
36
36
  CLUSTER_OPT_KEYS = SHARDING_OPT_KEYS + REPLICA_OPT_KEYS + MONGODS_OPT_KEYS
37
37
 
38
- FLAGS = [:noprealloc, :smallfiles, :logappend, :configsvr, :shardsvr, :quiet]
38
+ FLAGS = [:noprealloc, :smallfiles, :logappend, :configsvr, :shardsvr, :quiet, :fastsync]
39
39
 
40
40
  DEFAULT_VERIFIES = 60
41
41
  BASE_PORT = 3000
@@ -102,13 +102,15 @@ module Mongo
102
102
  noprealloc = opts[:noprealloc] || true
103
103
  smallfiles = opts[:smallfiles] || true
104
104
  quiet = opts[:quiet] || true
105
+ fast_sync = opts[:fastsync] || false
105
106
 
106
107
  params.merge(
107
108
  :command => mongod,
108
109
  :dbpath => path,
109
110
  :smallfiles => smallfiles,
110
111
  :noprealloc => noprealloc,
111
- :quiet => quiet
112
+ :quiet => quiet,
113
+ :fastsync => fast_sync
112
114
  )
113
115
  end
114
116
 
@@ -116,7 +118,7 @@ module Mongo
116
118
  params = make_mongod('replicas', opts)
117
119
 
118
120
  replSet = opts[:replSet] || 'ruby-driver-test'
119
- oplog_size = opts[:oplog_size] || 10
121
+ oplog_size = opts[:oplog_size] || 5
120
122
 
121
123
  params.merge(
122
124
  :_id => count,
@@ -185,8 +187,13 @@ module Mongo
185
187
  clear_zombie
186
188
  return @pid if running?
187
189
  begin
188
- @pid = Process.spawn(*@cmd) # redirection not supported in jruby
189
- sleep 1
190
+ # redirection not supported in jruby
191
+ if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
192
+ @pid = Process.spawn(*@cmd)
193
+ else
194
+ cmd_and_opts = [@cmd, {:out => :close}].flatten
195
+ @pid = Process.spawn(*cmd_and_opts)
196
+ end
190
197
  verify(verifies) if verifies > 0
191
198
  @pid
192
199
  end
@@ -203,6 +210,8 @@ module Mongo
203
210
  rescue Errno::ESRCH
204
211
  false
205
212
  end
213
+ # cleanup lock if unclean shutdown
214
+ File.delete(File.join(@config[:dbpath], 'mongod.lock')) if @config[:dbpath]
206
215
  end
207
216
 
208
217
  def wait
@@ -260,9 +269,9 @@ module Mongo
260
269
  params = @config.reject{|k,v| IGNORE_KEYS.include?(k)}
261
270
  arguments = params.sort{|a, b| a[0].to_s <=> b[0].to_s}.collect do |arg, value| # sort block is needed for 1.8.7 which lacks Symbol#<=>
262
271
  argument = '--' + arg.to_s
263
- if FLAGS.member?(arg)
272
+ if FLAGS.member?(arg) && value == true
264
273
  [argument]
265
- else
274
+ elsif !FLAGS.member?(arg)
266
275
  [argument, value.to_s]
267
276
  end
268
277
  end
@@ -330,7 +339,7 @@ module Mongo
330
339
  end
331
340
 
332
341
  def repl_set_get_status
333
- command( @config[:replicas].first, 'admin', { :replSetGetStatus => 1 }, {:check_response => false } )
342
+ command( @config[:replicas], 'admin', { :replSetGetStatus => 1 }, {:check_response => false } )
334
343
  end
335
344
 
336
345
  def repl_set_get_config
@@ -354,16 +363,27 @@ module Mongo
354
363
  end
355
364
 
356
365
  def repl_set_startup
357
- response = nil
358
- 60.times do |i|
359
- response = repl_set_get_status
360
- members = response['members']
361
- if response['ok'] == 1.0 && members.collect{|m| m['state']}.all?{|state| [1,2,7].index(state)}
362
- return response if members.any?{|m| m['state'] == 1}
366
+ states = nil
367
+ 60.times do
368
+ states = repl_set_get_status.zip(repl_set_is_master)
369
+ healthy = states.all? do |status, is_master|
370
+ members = status['members']
371
+ if status['ok'] == 1.0 && members.collect{|m| m['state']}.all?{|state| [1,2,7].index(state)}
372
+ members.any?{|m| m['state'] == 1} &&
373
+ case status['myState']
374
+ when 1
375
+ is_master['ismaster'] == true && is_master['secondary'] == false
376
+ when 2
377
+ is_master['ismaster'] == false && is_master['secondary'] == true
378
+ when 7
379
+ is_master['ismaster'] == false && is_master['secondary'] == false
380
+ end
381
+ end
363
382
  end
364
- sleep 1
383
+ return true if healthy
384
+ sleep(1)
365
385
  end
366
- raise Mongo::OperationFailure, "replSet startup failed - status: #{response.inspect}"
386
+ raise Mongo::OperationFailure, "replSet startup failed - status: #{states.inspect}"
367
387
  end
368
388
 
369
389
  def repl_set_seeds
@@ -374,13 +394,17 @@ module Mongo
374
394
  @config[:replicas].collect{|node| [node[:host], node[:port]]}
375
395
  end
376
396
 
397
+ def repl_set_seeds_uri
398
+ repl_set_seeds.join(',')
399
+ end
400
+
377
401
  def repl_set_name
378
402
  @config[:replicas].first[:replSet]
379
403
  end
380
404
 
381
405
  def member_names_by_state(state)
382
406
  states = Array(state)
383
- status = repl_set_get_status
407
+ status = repl_set_get_status.first
384
408
  status['members'].find_all{|member| states.index(member['state']) }.collect{|member| member['name']}
385
409
  end
386
410
 
@@ -414,6 +438,14 @@ module Mongo
414
438
  members_by_name(secondary_names)
415
439
  end
416
440
 
441
+ def kill_primary
442
+ primary.kill
443
+ end
444
+
445
+ def kill_secondary
446
+ secondaries[rand(secondaries.length)].kill
447
+ end
448
+
417
449
  def replicas
418
450
  members_by_name(replica_names)
419
451
  end
@@ -459,8 +491,16 @@ module Mongo
459
491
  config_names_by_kind(:routers)
460
492
  end
461
493
 
462
- def ismaster
463
- command( @config[:routers], 'admin', { :ismaster => 1 } )
494
+ def ismaster(servers)
495
+ command( servers, 'admin', { :ismaster => 1 } )
496
+ end
497
+
498
+ def sharded_cluster_is_master
499
+ ismaster(@config[:routers])
500
+ end
501
+
502
+ def repl_set_is_master
503
+ ismaster(@config[:replicas])
464
504
  end
465
505
 
466
506
  def addshards(shards = @config[:shards])
@@ -494,7 +534,7 @@ module Mongo
494
534
  servers.each{|server| server.start}
495
535
  # TODO - sharded replica sets - pending
496
536
  if @config[:replicas]
497
- repl_set_initiate if repl_set_get_status['startupStatus'] == 3
537
+ repl_set_initiate if repl_set_get_status.first['startupStatus'] == 3
498
538
  repl_set_startup
499
539
  end
500
540
  if @config[:routers]