jonbell-mongo 1.3.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. data/LICENSE.txt +190 -0
  2. data/README.md +333 -0
  3. data/Rakefile +215 -0
  4. data/bin/mongo_console +21 -0
  5. data/docs/CREDITS.md +123 -0
  6. data/docs/FAQ.md +116 -0
  7. data/docs/GridFS.md +158 -0
  8. data/docs/HISTORY.md +263 -0
  9. data/docs/RELEASES.md +33 -0
  10. data/docs/REPLICA_SETS.md +72 -0
  11. data/docs/TUTORIAL.md +247 -0
  12. data/docs/WRITE_CONCERN.md +28 -0
  13. data/lib/mongo.rb +97 -0
  14. data/lib/mongo/collection.rb +895 -0
  15. data/lib/mongo/connection.rb +926 -0
  16. data/lib/mongo/cursor.rb +474 -0
  17. data/lib/mongo/db.rb +617 -0
  18. data/lib/mongo/exceptions.rb +71 -0
  19. data/lib/mongo/gridfs/grid.rb +107 -0
  20. data/lib/mongo/gridfs/grid_ext.rb +57 -0
  21. data/lib/mongo/gridfs/grid_file_system.rb +146 -0
  22. data/lib/mongo/gridfs/grid_io.rb +485 -0
  23. data/lib/mongo/gridfs/grid_io_fix.rb +38 -0
  24. data/lib/mongo/repl_set_connection.rb +356 -0
  25. data/lib/mongo/util/conversions.rb +89 -0
  26. data/lib/mongo/util/core_ext.rb +60 -0
  27. data/lib/mongo/util/pool.rb +177 -0
  28. data/lib/mongo/util/server_version.rb +71 -0
  29. data/lib/mongo/util/support.rb +82 -0
  30. data/lib/mongo/util/uri_parser.rb +185 -0
  31. data/mongo.gemspec +34 -0
  32. data/test/auxillary/1.4_features.rb +166 -0
  33. data/test/auxillary/authentication_test.rb +68 -0
  34. data/test/auxillary/autoreconnect_test.rb +41 -0
  35. data/test/auxillary/fork_test.rb +30 -0
  36. data/test/auxillary/repl_set_auth_test.rb +58 -0
  37. data/test/auxillary/slave_connection_test.rb +36 -0
  38. data/test/auxillary/threaded_authentication_test.rb +101 -0
  39. data/test/bson/binary_test.rb +15 -0
  40. data/test/bson/bson_test.rb +654 -0
  41. data/test/bson/byte_buffer_test.rb +208 -0
  42. data/test/bson/hash_with_indifferent_access_test.rb +38 -0
  43. data/test/bson/json_test.rb +17 -0
  44. data/test/bson/object_id_test.rb +154 -0
  45. data/test/bson/ordered_hash_test.rb +210 -0
  46. data/test/bson/timestamp_test.rb +24 -0
  47. data/test/collection_test.rb +910 -0
  48. data/test/connection_test.rb +324 -0
  49. data/test/conversions_test.rb +119 -0
  50. data/test/cursor_fail_test.rb +75 -0
  51. data/test/cursor_message_test.rb +43 -0
  52. data/test/cursor_test.rb +483 -0
  53. data/test/db_api_test.rb +738 -0
  54. data/test/db_connection_test.rb +15 -0
  55. data/test/db_test.rb +315 -0
  56. data/test/grid_file_system_test.rb +259 -0
  57. data/test/grid_io_test.rb +209 -0
  58. data/test/grid_test.rb +258 -0
  59. data/test/load/thin/load.rb +24 -0
  60. data/test/load/unicorn/load.rb +23 -0
  61. data/test/replica_sets/connect_test.rb +112 -0
  62. data/test/replica_sets/connection_string_test.rb +32 -0
  63. data/test/replica_sets/count_test.rb +35 -0
  64. data/test/replica_sets/insert_test.rb +53 -0
  65. data/test/replica_sets/pooled_insert_test.rb +55 -0
  66. data/test/replica_sets/query_secondaries.rb +108 -0
  67. data/test/replica_sets/query_test.rb +51 -0
  68. data/test/replica_sets/replication_ack_test.rb +66 -0
  69. data/test/replica_sets/rs_test_helper.rb +27 -0
  70. data/test/safe_test.rb +68 -0
  71. data/test/support/hash_with_indifferent_access.rb +186 -0
  72. data/test/support/keys.rb +45 -0
  73. data/test/support_test.rb +18 -0
  74. data/test/test_helper.rb +102 -0
  75. data/test/threading/threading_with_large_pool_test.rb +90 -0
  76. data/test/threading_test.rb +87 -0
  77. data/test/tools/auth_repl_set_manager.rb +14 -0
  78. data/test/tools/repl_set_manager.rb +266 -0
  79. data/test/unit/collection_test.rb +130 -0
  80. data/test/unit/connection_test.rb +85 -0
  81. data/test/unit/cursor_test.rb +109 -0
  82. data/test/unit/db_test.rb +94 -0
  83. data/test/unit/grid_test.rb +49 -0
  84. data/test/unit/pool_test.rb +9 -0
  85. data/test/unit/repl_set_connection_test.rb +59 -0
  86. data/test/unit/safe_test.rb +125 -0
  87. data/test/uri_test.rb +91 -0
  88. metadata +224 -0
@@ -0,0 +1,18 @@
1
+ require './test/test_helper'
2
+
3
+ class SupportTest < Test::Unit::TestCase
4
+
5
+ def test_command_response_succeeds
6
+ assert Support.ok?('ok' => 1)
7
+ assert Support.ok?('ok' => 1.0)
8
+ assert Support.ok?('ok' => true)
9
+ end
10
+
11
+ def test_command_response_fails
12
+ assert !Support.ok?('ok' => 0)
13
+ assert !Support.ok?('ok' => 0.0)
14
+ assert !Support.ok?('ok' => 0.0)
15
+ assert !Support.ok?('ok' => 'str')
16
+ assert !Support.ok?('ok' => false)
17
+ end
18
+ end
@@ -0,0 +1,102 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ require 'rubygems' if RUBY_VERSION < '1.9.0' && ENV['C_EXT']
3
+ require 'mongo'
4
+ require 'test/unit'
5
+
6
+ def silently
7
+ warn_level = $VERBOSE
8
+ $VERBOSE = nil
9
+ result = yield
10
+ $VERBOSE = warn_level
11
+ result
12
+ end
13
+
14
+ begin
15
+ require 'rubygems' if RUBY_VERSION < "1.9.0" && !ENV['C_EXT']
16
+ silently { require 'shoulda' }
17
+ silently { require 'mocha' }
18
+ rescue LoadError
19
+ puts <<MSG
20
+
21
+ This test suite requires shoulda and mocha.
22
+ You can install them as follows:
23
+ gem install shoulda
24
+ gem install mocha
25
+
26
+ MSG
27
+
28
+ exit
29
+ end
30
+
31
+ require 'bson_ext/cbson' if !(RUBY_PLATFORM =~ /java/) && ENV['C_EXT']
32
+
33
+ unless defined? MONGO_TEST_DB
34
+ MONGO_TEST_DB = 'ruby-test-db'
35
+ end
36
+
37
+ unless defined? TEST_PORT
38
+ TEST_PORT = ENV['MONGO_RUBY_DRIVER_PORT'] ? ENV['MONGO_RUBY_DRIVER_PORT'].to_i : Mongo::Connection::DEFAULT_PORT
39
+ end
40
+
41
+ unless defined? TEST_HOST
42
+ TEST_HOST = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
43
+ end
44
+
45
+ class Test::Unit::TestCase
46
+ include Mongo
47
+ include BSON
48
+
49
+ def self.standard_connection(options={})
50
+ Connection.new(TEST_HOST, TEST_PORT, options)
51
+ end
52
+
53
+ def standard_connection(options={})
54
+ self.class.standard_connection(options)
55
+ end
56
+
57
+ def self.host_port
58
+ "#{mongo_host}:#{mongo_port}"
59
+ end
60
+
61
+ def self.mongo_host
62
+ TEST_HOST
63
+ end
64
+
65
+ def self.mongo_port
66
+ TEST_PORT
67
+ end
68
+
69
+ def host_port
70
+ self.class.host_port
71
+ end
72
+
73
+ def mongo_host
74
+ self.class.mongo_host
75
+ end
76
+
77
+ def mongo_port
78
+ self.class.mongo_port
79
+ end
80
+
81
+ def new_mock_socket(host='localhost', port=27017)
82
+ socket = Object.new
83
+ socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
84
+ socket.stubs(:close)
85
+ socket
86
+ end
87
+
88
+ def new_mock_db
89
+ db = Object.new
90
+ end
91
+
92
+ def assert_raise_error(klass, message)
93
+ begin
94
+ yield
95
+ rescue => e
96
+ assert_equal klass, e.class
97
+ assert e.message.include?(message), "#{e.message} does not include #{message}."
98
+ else
99
+ flunk "Expected assertion #{klass} but none was raised."
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,90 @@
1
+ require './test/test_helper'
2
+
3
+ # Essentialy the same as test_threading.rb but with an expanded pool for
4
+ # testing multiple connections.
5
+ class TestThreadingLargePool < Test::Unit::TestCase
6
+
7
+ include Mongo
8
+
9
+ @@db = standard_connection(:pool_size => 50, :timeout => 60).db(MONGO_TEST_DB)
10
+ @@coll = @@db.collection('thread-test-collection')
11
+
12
+ def set_up_safe_data
13
+ @@db.drop_collection('duplicate')
14
+ @@db.drop_collection('unique')
15
+ @duplicate = @@db.collection('duplicate')
16
+ @unique = @@db.collection('unique')
17
+
18
+ @duplicate.insert("test" => "insert")
19
+ @duplicate.insert("test" => "update")
20
+ @unique.insert("test" => "insert")
21
+ @unique.insert("test" => "update")
22
+ @unique.create_index("test", :unique => true)
23
+ end
24
+
25
+ def test_safe_update
26
+ set_up_safe_data
27
+ threads = []
28
+ 300.times do |i|
29
+ threads[i] = Thread.new do
30
+ if i % 2 == 0
31
+ assert_raise Mongo::OperationFailure do
32
+ @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
33
+ end
34
+ else
35
+ @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
36
+ end
37
+ end
38
+ end
39
+
40
+ 300.times do |i|
41
+ threads[i].join
42
+ end
43
+ end
44
+
45
+ def test_safe_insert
46
+ set_up_safe_data
47
+ threads = []
48
+ 300.times do |i|
49
+ threads[i] = Thread.new do
50
+ if i % 2 == 0
51
+ assert_raise Mongo::OperationFailure do
52
+ @unique.insert({"test" => "insert"}, :safe => true)
53
+ end
54
+ else
55
+ @duplicate.insert({"test" => "insert"}, :safe => true)
56
+ end
57
+ end
58
+ end
59
+
60
+ 300.times do |i|
61
+ threads[i].join
62
+ end
63
+ end
64
+
65
+ def test_threading
66
+ @@coll.drop
67
+ @@coll = @@db.collection('thread-test-collection')
68
+
69
+ 1000.times do |i|
70
+ @@coll.insert("x" => i)
71
+ end
72
+
73
+ threads = []
74
+
75
+ 10.times do |i|
76
+ threads[i] = Thread.new do
77
+ sum = 0
78
+ @@coll.find().each do |document|
79
+ sum += document["x"]
80
+ end
81
+ assert_equal 499500, sum
82
+ end
83
+ end
84
+
85
+ 10.times do |i|
86
+ threads[i].join
87
+ end
88
+ end
89
+
90
+ end
@@ -0,0 +1,87 @@
1
+ require './test/test_helper'
2
+
3
+ class TestThreading < Test::Unit::TestCase
4
+
5
+ include Mongo
6
+
7
+ @@db = standard_connection(:pool_size => 1, :timeout => 30).db(MONGO_TEST_DB)
8
+ @@coll = @@db.collection('thread-test-collection')
9
+
10
+ def set_up_safe_data
11
+ @@db.drop_collection('duplicate')
12
+ @@db.drop_collection('unique')
13
+ @duplicate = @@db.collection('duplicate')
14
+ @unique = @@db.collection('unique')
15
+
16
+ @duplicate.insert("test" => "insert")
17
+ @duplicate.insert("test" => "update")
18
+ @unique.insert("test" => "insert")
19
+ @unique.insert("test" => "update")
20
+ @unique.create_index("test", :unique => true)
21
+ end
22
+
23
+ def test_safe_update
24
+ set_up_safe_data
25
+ threads = []
26
+ 100.times do |i|
27
+ threads[i] = Thread.new do
28
+ if i % 2 == 0
29
+ assert_raise Mongo::OperationFailure do
30
+ @unique.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
31
+ end
32
+ else
33
+ @duplicate.update({"test" => "insert"}, {"$set" => {"test" => "update"}}, :safe => true)
34
+ end
35
+ end
36
+ end
37
+
38
+ 100.times do |i|
39
+ threads[i].join
40
+ end
41
+ end
42
+
43
+ def test_safe_insert
44
+ set_up_safe_data
45
+ threads = []
46
+ 100.times do |i|
47
+ threads[i] = Thread.new do
48
+ if i % 2 == 0
49
+ assert_raise Mongo::OperationFailure do
50
+ @unique.insert({"test" => "insert"}, :safe => true)
51
+ end
52
+ else
53
+ @duplicate.insert({"test" => "insert"}, :safe => true)
54
+ end
55
+ end
56
+ end
57
+
58
+ 100.times do |i|
59
+ threads[i].join
60
+ end
61
+ end
62
+
63
+ def test_threading
64
+ @@coll.drop
65
+ @@coll = @@db.collection('thread-test-collection')
66
+
67
+ 1000.times do |i|
68
+ @@coll.insert("x" => i)
69
+ end
70
+
71
+ threads = []
72
+
73
+ 10.times do |i|
74
+ threads[i] = Thread.new do
75
+ sum = 0
76
+ @@coll.find().each do |document|
77
+ sum += document["x"]
78
+ end
79
+ assert_equal 499500, sum
80
+ end
81
+ end
82
+
83
+ 10.times do |i|
84
+ threads[i].join
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,14 @@
1
+ require File.join((File.expand_path(File.dirname(__FILE__))), 'repl_set_manager')
2
+
3
+ class AuthReplSetManager < ReplSetManager
4
+ def initialize(opts={})
5
+ super(opts)
6
+
7
+ @key_path = opts[:key_path] || File.join(File.expand_path(File.dirname(__FILE__)), "keyfile.txt")
8
+ system("chmod 600 #{@key_path}")
9
+ end
10
+
11
+ def start_cmd(n)
12
+ super + " --keyFile #{@key_path}"
13
+ end
14
+ end
@@ -0,0 +1,266 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require 'thread'
4
+
5
+ STDOUT.sync = true
6
+
7
+ unless defined? Mongo
8
+ require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
9
+ end
10
+
11
+ class ReplSetManager
12
+
13
+ attr_accessor :host, :start_port, :ports, :name, :mongods
14
+
15
+ def initialize(opts={})
16
+ @start_port = opts[:start_port] || 30000
17
+ @ports = []
18
+ @name = opts[:name] || 'replica-set-foo'
19
+ @host = opts[:host] || 'localhost'
20
+ @retries = opts[:retries] || 60
21
+ @config = {"_id" => @name, "members" => []}
22
+ @durable = opts.fetch(:durable, false)
23
+ @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
24
+
25
+ @arbiter_count = opts[:arbiter_count] || 2
26
+ @secondary_count = opts[:secondary_count] || 2
27
+ @passive_count = opts[:passive_count] || 0
28
+ @primary_count = 1
29
+
30
+ @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
31
+ if @count > 7
32
+ raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
33
+ end
34
+
35
+ @mongods = {}
36
+ end
37
+
38
+ def start_set
39
+ puts "** Starting a replica set with #{@count} nodes"
40
+
41
+ system("killall mongod")
42
+
43
+ n = 0
44
+ (@primary_count + @secondary_count).times do
45
+ init_node(n)
46
+ n += 1
47
+ end
48
+
49
+ @passive_count.times do
50
+ init_node(n) do |attrs|
51
+ attrs['priority'] = 0
52
+ end
53
+ n += 1
54
+ end
55
+
56
+ @arbiter_count.times do
57
+ init_node(n) do |attrs|
58
+ attrs['arbiterOnly'] = true
59
+ end
60
+ n += 1
61
+ end
62
+
63
+ initiate
64
+ ensure_up
65
+ end
66
+
67
+ def cleanup_set
68
+ system("killall mongod")
69
+ @count.times do |n|
70
+ system("rm -rf #{@mongods[n]['db_path']}")
71
+ end
72
+ end
73
+
74
+ def init_node(n)
75
+ @mongods[n] ||= {}
76
+ port = @start_port + n
77
+ @ports << port
78
+ @mongods[n]['port'] = port
79
+ @mongods[n]['db_path'] = get_path("rs-#{port}")
80
+ @mongods[n]['log_path'] = get_path("log-#{port}")
81
+ system("rm -rf #{@mongods[n]['db_path']}")
82
+ system("mkdir -p #{@mongods[n]['db_path']}")
83
+
84
+ @mongods[n]['start'] = start_cmd(n)
85
+ start(n)
86
+
87
+ member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
88
+
89
+ if block_given?
90
+ custom_attrs = {}
91
+ yield custom_attrs
92
+ member.merge!(custom_attrs)
93
+ @mongods[n].merge!(custom_attrs)
94
+ end
95
+
96
+ @config['members'] << member
97
+ end
98
+
99
+ def start_cmd(n)
100
+ @mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
101
+ " --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
102
+ @mongods[n]['start'] += " --dur" if @durable
103
+ @mongods[n]['start']
104
+ end
105
+
106
+ def kill(node, signal=2)
107
+ pid = @mongods[node]['pid']
108
+ puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
109
+ system("kill -#{signal} #{@mongods[node]['pid']}")
110
+ @mongods[node]['up'] = false
111
+ sleep(1)
112
+ end
113
+
114
+ def kill_primary(signal=2)
115
+ node = get_node_with_state(1)
116
+ kill(node, signal)
117
+ return node
118
+ end
119
+
120
+ # Note that we have to rescue a connection failure
121
+ # when we run the StepDown command because that
122
+ # command will close the connection.
123
+ def step_down_primary
124
+ primary = get_node_with_state(1)
125
+ con = get_connection(primary)
126
+ begin
127
+ con['admin'].command({'replSetStepDown' => 90})
128
+ rescue Mongo::ConnectionFailure
129
+ end
130
+ end
131
+
132
+ def kill_secondary
133
+ node = get_node_with_state(2)
134
+ kill(node)
135
+ return node
136
+ end
137
+
138
+ def restart_killed_nodes
139
+ nodes = @mongods.keys.select do |key|
140
+ @mongods[key]['up'] == false
141
+ end
142
+
143
+ nodes.each do |node|
144
+ start(node)
145
+ end
146
+
147
+ ensure_up
148
+ end
149
+
150
+ def get_node_from_port(port)
151
+ @mongods.keys.detect { |key| @mongods[key]['port'] == port }
152
+ end
153
+
154
+ def start(node)
155
+ system(@mongods[node]['start'])
156
+ @mongods[node]['up'] = true
157
+ sleep(0.5)
158
+ @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
159
+ end
160
+ alias :restart :start
161
+
162
+ def ensure_up
163
+ print "** Ensuring members are up..."
164
+
165
+ attempt do
166
+ con = get_connection
167
+ status = con['admin'].command({'replSetGetStatus' => 1})
168
+ print "."
169
+ if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
170
+ status['members'].any? { |m| m['state'] == 1 }
171
+ print "all members up!\n\n"
172
+ return status
173
+ else
174
+ raise Mongo::OperationFailure
175
+ end
176
+ end
177
+ end
178
+
179
+ def primary
180
+ nodes = get_all_host_pairs_with_state(1)
181
+ nodes.empty? ? nil : nodes[0]
182
+ end
183
+
184
+ def secondaries
185
+ get_all_host_pairs_with_state(2)
186
+ end
187
+
188
+ def arbiters
189
+ get_all_host_pairs_with_state(7)
190
+ end
191
+
192
+ # String used for adding a shard via mongos
193
+ # using the addshard command.
194
+ def shard_string
195
+ str = "#{@name}/"
196
+ str << @mongods.map do |k, mongod|
197
+ "#{@host}:#{mongod['port']}"
198
+ end.join(',')
199
+ str
200
+ end
201
+
202
+ private
203
+
204
+ def initiate
205
+ con = get_connection
206
+
207
+ attempt do
208
+ con['admin'].command({'replSetInitiate' => @config})
209
+ end
210
+ end
211
+
212
+ def get_node_with_state(state)
213
+ status = ensure_up
214
+ node = status['members'].detect {|m| m['state'] == state}
215
+ if node
216
+ host_port = node['name'].split(':')
217
+ port = host_port[1] ? host_port[1].to_i : 27017
218
+ key = @mongods.keys.detect {|key| @mongods[key]['port'] == port}
219
+ return key
220
+ else
221
+ return false
222
+ end
223
+ end
224
+
225
+ def get_all_host_pairs_with_state(state)
226
+ status = ensure_up
227
+ nodes = status['members'].select {|m| m['state'] == state}
228
+ nodes.map do |node|
229
+ host_port = node['name'].split(':')
230
+ port = host_port[1] ? host_port[1].to_i : 27017
231
+ [host, port]
232
+ end
233
+ end
234
+
235
+ def get_connection(node=nil)
236
+ con = attempt do
237
+ if !node
238
+ node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
239
+ end
240
+ con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
241
+ end
242
+
243
+ return con
244
+ end
245
+
246
+ def get_path(name)
247
+ File.join(@path, name)
248
+ end
249
+
250
+ def attempt
251
+ raise "No block given!" unless block_given?
252
+ count = 0
253
+
254
+ while count < @retries do
255
+ begin
256
+ return yield
257
+ rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
258
+ sleep(1)
259
+ count += 1
260
+ end
261
+ end
262
+
263
+ raise ex
264
+ end
265
+
266
+ end