mongo-lyon 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. data/LICENSE.txt +190 -0
  2. data/README.md +344 -0
  3. data/Rakefile +202 -0
  4. data/bin/mongo_console +34 -0
  5. data/docs/1.0_UPGRADE.md +21 -0
  6. data/docs/CREDITS.md +123 -0
  7. data/docs/FAQ.md +116 -0
  8. data/docs/GridFS.md +158 -0
  9. data/docs/HISTORY.md +225 -0
  10. data/docs/REPLICA_SETS.md +72 -0
  11. data/docs/TUTORIAL.md +247 -0
  12. data/docs/WRITE_CONCERN.md +28 -0
  13. data/lib/mongo.rb +77 -0
  14. data/lib/mongo/collection.rb +872 -0
  15. data/lib/mongo/connection.rb +875 -0
  16. data/lib/mongo/cursor.rb +449 -0
  17. data/lib/mongo/db.rb +607 -0
  18. data/lib/mongo/exceptions.rb +68 -0
  19. data/lib/mongo/gridfs/grid.rb +106 -0
  20. data/lib/mongo/gridfs/grid_ext.rb +57 -0
  21. data/lib/mongo/gridfs/grid_file_system.rb +145 -0
  22. data/lib/mongo/gridfs/grid_io.rb +394 -0
  23. data/lib/mongo/gridfs/grid_io_fix.rb +38 -0
  24. data/lib/mongo/repl_set_connection.rb +342 -0
  25. data/lib/mongo/util/conversions.rb +89 -0
  26. data/lib/mongo/util/core_ext.rb +60 -0
  27. data/lib/mongo/util/pool.rb +185 -0
  28. data/lib/mongo/util/server_version.rb +71 -0
  29. data/lib/mongo/util/support.rb +82 -0
  30. data/lib/mongo/util/uri_parser.rb +181 -0
  31. data/lib/mongo/version.rb +3 -0
  32. data/mongo.gemspec +34 -0
  33. data/test/auxillary/1.4_features.rb +166 -0
  34. data/test/auxillary/authentication_test.rb +68 -0
  35. data/test/auxillary/autoreconnect_test.rb +41 -0
  36. data/test/auxillary/repl_set_auth_test.rb +58 -0
  37. data/test/auxillary/slave_connection_test.rb +36 -0
  38. data/test/auxillary/threaded_authentication_test.rb +101 -0
  39. data/test/bson/binary_test.rb +15 -0
  40. data/test/bson/bson_test.rb +614 -0
  41. data/test/bson/byte_buffer_test.rb +190 -0
  42. data/test/bson/hash_with_indifferent_access_test.rb +38 -0
  43. data/test/bson/json_test.rb +17 -0
  44. data/test/bson/object_id_test.rb +154 -0
  45. data/test/bson/ordered_hash_test.rb +197 -0
  46. data/test/collection_test.rb +893 -0
  47. data/test/connection_test.rb +303 -0
  48. data/test/conversions_test.rb +120 -0
  49. data/test/cursor_fail_test.rb +75 -0
  50. data/test/cursor_message_test.rb +43 -0
  51. data/test/cursor_test.rb +457 -0
  52. data/test/db_api_test.rb +715 -0
  53. data/test/db_connection_test.rb +15 -0
  54. data/test/db_test.rb +287 -0
  55. data/test/grid_file_system_test.rb +244 -0
  56. data/test/grid_io_test.rb +120 -0
  57. data/test/grid_test.rb +200 -0
  58. data/test/load/thin/load.rb +24 -0
  59. data/test/load/unicorn/load.rb +23 -0
  60. data/test/replica_sets/connect_test.rb +86 -0
  61. data/test/replica_sets/connection_string_test.rb +32 -0
  62. data/test/replica_sets/count_test.rb +35 -0
  63. data/test/replica_sets/insert_test.rb +53 -0
  64. data/test/replica_sets/pooled_insert_test.rb +55 -0
  65. data/test/replica_sets/query_secondaries.rb +96 -0
  66. data/test/replica_sets/query_test.rb +51 -0
  67. data/test/replica_sets/replication_ack_test.rb +66 -0
  68. data/test/replica_sets/rs_test_helper.rb +27 -0
  69. data/test/safe_test.rb +68 -0
  70. data/test/support/hash_with_indifferent_access.rb +199 -0
  71. data/test/support/keys.rb +45 -0
  72. data/test/support_test.rb +19 -0
  73. data/test/test_helper.rb +83 -0
  74. data/test/threading/threading_with_large_pool_test.rb +90 -0
  75. data/test/threading_test.rb +87 -0
  76. data/test/tools/auth_repl_set_manager.rb +14 -0
  77. data/test/tools/repl_set_manager.rb +266 -0
  78. data/test/unit/collection_test.rb +130 -0
  79. data/test/unit/connection_test.rb +98 -0
  80. data/test/unit/cursor_test.rb +99 -0
  81. data/test/unit/db_test.rb +96 -0
  82. data/test/unit/grid_test.rb +49 -0
  83. data/test/unit/pool_test.rb +9 -0
  84. data/test/unit/repl_set_connection_test.rb +72 -0
  85. data/test/unit/safe_test.rb +125 -0
  86. data/test/uri_test.rb +91 -0
  87. metadata +202 -0
@@ -0,0 +1,14 @@
1
+ require File.join((File.expand_path(File.dirname(__FILE__))), 'repl_set_manager')
2
+
3
+ class AuthReplSetManager < ReplSetManager
4
+ def initialize(opts={})
5
+ super(opts)
6
+
7
+ @key_path = opts[:key_path] || File.join(File.expand_path(File.dirname(__FILE__)), "keyfile.txt")
8
+ system("chmod 600 #{@key_path}")
9
+ end
10
+
11
+ def start_cmd(n)
12
+ super + " --keyFile #{@key_path}"
13
+ end
14
+ end
@@ -0,0 +1,266 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require 'thread'
4
+
5
+ STDOUT.sync = true
6
+
7
+ unless defined? Mongo
8
+ require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
9
+ end
10
+
11
+ class ReplSetManager
12
+
13
+ attr_accessor :host, :start_port, :ports, :name, :mongods
14
+
15
+ def initialize(opts={})
16
+ @start_port = opts[:start_port] || 30000
17
+ @ports = []
18
+ @name = opts[:name] || 'replica-set-foo'
19
+ @host = opts[:host] || 'localhost'
20
+ @retries = opts[:retries] || 60
21
+ @config = {"_id" => @name, "members" => []}
22
+ @durable = opts.fetch(:durable, false)
23
+ @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
24
+
25
+ @arbiter_count = opts[:arbiter_count] || 2
26
+ @secondary_count = opts[:secondary_count] || 1
27
+ @passive_count = opts[:passive_count] || 1
28
+ @primary_count = 1
29
+
30
+ @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
31
+ if @count > 7
32
+ raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
33
+ end
34
+
35
+ @mongods = {}
36
+ end
37
+
38
+ def start_set
39
+ puts "** Starting a replica set with #{@count} nodes"
40
+
41
+ system("killall mongod")
42
+
43
+ n = 0
44
+ (@primary_count + @secondary_count).times do |n|
45
+ init_node(n)
46
+ n += 1
47
+ end
48
+
49
+ @passive_count.times do
50
+ init_node(n) do |attrs|
51
+ attrs['priority'] = 0
52
+ end
53
+ n += 1
54
+ end
55
+
56
+ @arbiter_count.times do
57
+ init_node(n) do |attrs|
58
+ attrs['arbiterOnly'] = true
59
+ end
60
+ n += 1
61
+ end
62
+
63
+ initiate
64
+ ensure_up
65
+ end
66
+
67
+ def cleanup_set
68
+ system("killall mongod")
69
+ @count.times do |n|
70
+ system("rm -rf #{@mongods[n]['db_path']}")
71
+ end
72
+ end
73
+
74
+ def init_node(n)
75
+ @mongods[n] ||= {}
76
+ port = @start_port + n
77
+ @ports << port
78
+ @mongods[n]['port'] = port
79
+ @mongods[n]['db_path'] = get_path("rs-#{port}")
80
+ @mongods[n]['log_path'] = get_path("log-#{port}")
81
+ system("rm -rf #{@mongods[n]['db_path']}")
82
+ system("mkdir -p #{@mongods[n]['db_path']}")
83
+
84
+ @mongods[n]['start'] = start_cmd(n)
85
+ start(n)
86
+
87
+ member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
88
+
89
+ if block_given?
90
+ custom_attrs = {}
91
+ yield custom_attrs
92
+ member.merge!(custom_attrs)
93
+ @mongods[n].merge!(custom_attrs)
94
+ end
95
+
96
+ @config['members'] << member
97
+ end
98
+
99
+ def start_cmd(n)
100
+ @mongods[n]['start'] = "mongod --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
101
+ " --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
102
+ @mongods[n]['start'] += " --dur" if @durable
103
+ @mongods[n]['start']
104
+ end
105
+
106
+ def kill(node, signal=2)
107
+ pid = @mongods[node]['pid']
108
+ puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
109
+ system("kill -#{signal} #{@mongods[node]['pid']}")
110
+ @mongods[node]['up'] = false
111
+ sleep(1)
112
+ end
113
+
114
+ def kill_primary(signal=2)
115
+ node = get_node_with_state(1)
116
+ kill(node, signal)
117
+ return node
118
+ end
119
+
120
+ # Note that we have to rescue a connection failure
121
+ # when we run the StepDown command because that
122
+ # command will close the connection.
123
+ def step_down_primary
124
+ primary = get_node_with_state(1)
125
+ con = get_connection(primary)
126
+ begin
127
+ con['admin'].command({'replSetStepDown' => 90})
128
+ rescue Mongo::ConnectionFailure
129
+ end
130
+ end
131
+
132
+ def kill_secondary
133
+ node = get_node_with_state(2)
134
+ kill(node)
135
+ return node
136
+ end
137
+
138
+ def restart_killed_nodes
139
+ nodes = @mongods.keys.select do |key|
140
+ @mongods[key]['up'] == false
141
+ end
142
+
143
+ nodes.each do |node|
144
+ start(node)
145
+ end
146
+
147
+ ensure_up
148
+ end
149
+
150
+ def get_node_from_port(port)
151
+ @mongods.keys.detect { |key| @mongods[key]['port'] == port }
152
+ end
153
+
154
+ def start(node)
155
+ system(@mongods[node]['start'])
156
+ @mongods[node]['up'] = true
157
+ sleep(0.5)
158
+ @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
159
+ end
160
+ alias :restart :start
161
+
162
+ def ensure_up
163
+ print "** Ensuring members are up..."
164
+
165
+ attempt do
166
+ con = get_connection
167
+ status = con['admin'].command({'replSetGetStatus' => 1})
168
+ print "."
169
+ if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
170
+ status['members'].any? { |m| m['state'] == 1 }
171
+ print "all members up!\n\n"
172
+ return status
173
+ else
174
+ raise Mongo::OperationFailure
175
+ end
176
+ end
177
+ end
178
+
179
+ def primary
180
+ nodes = get_all_host_pairs_with_state(1)
181
+ nodes.empty? ? nil : nodes[0]
182
+ end
183
+
184
+ def secondaries
185
+ get_all_host_pairs_with_state(2)
186
+ end
187
+
188
+ def arbiters
189
+ get_all_host_pairs_with_state(7)
190
+ end
191
+
192
+ # String used for adding a shard via mongos
193
+ # using the addshard command.
194
+ def shard_string
195
+ str = "#{@name}/"
196
+ str << @mongods.map do |k, mongod|
197
+ "#{@host}:#{mongod['port']}"
198
+ end.join(',')
199
+ str
200
+ end
201
+
202
+ private
203
+
204
+ def initiate
205
+ con = get_connection
206
+
207
+ attempt do
208
+ con['admin'].command({'replSetInitiate' => @config})
209
+ end
210
+ end
211
+
212
+ def get_node_with_state(state)
213
+ status = ensure_up
214
+ node = status['members'].detect {|m| m['state'] == state}
215
+ if node
216
+ host_port = node['name'].split(':')
217
+ port = host_port[1] ? host_port[1].to_i : 27017
218
+ key = @mongods.keys.detect {|key| @mongods[key]['port'] == port}
219
+ return key
220
+ else
221
+ return false
222
+ end
223
+ end
224
+
225
+ def get_all_host_pairs_with_state(state)
226
+ status = ensure_up
227
+ nodes = status['members'].select {|m| m['state'] == state}
228
+ nodes.map do |node|
229
+ host_port = node['name'].split(':')
230
+ port = host_port[1] ? host_port[1].to_i : 27017
231
+ [host, port]
232
+ end
233
+ end
234
+
235
+ def get_connection(node=nil)
236
+ con = attempt do
237
+ if !node
238
+ node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
239
+ end
240
+ con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
241
+ end
242
+
243
+ return con
244
+ end
245
+
246
+ def get_path(name)
247
+ File.join(@path, name)
248
+ end
249
+
250
+ def attempt
251
+ raise "No block given!" unless block_given?
252
+ count = 0
253
+
254
+ while count < @retries do
255
+ begin
256
+ return yield
257
+ rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
258
+ sleep(1)
259
+ count += 1
260
+ end
261
+ end
262
+
263
+ raise ex
264
+ end
265
+
266
+ end
@@ -0,0 +1,130 @@
1
+ require File.expand_path('./test/test_helper.rb')
2
+
3
+ class CollectionTest < Test::Unit::TestCase
4
+
5
+ context "Basic operations: " do
6
+ setup do
7
+ @logger = mock()
8
+ @logger.expects(:debug)
9
+ end
10
+
11
+ should "send update message" do
12
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
13
+ @db = @conn['testing']
14
+ @coll = @db.collection('books')
15
+ @conn.expects(:send_message).with do |op, msg, log|
16
+ op == 2001
17
+ end
18
+ @conn.stubs(:log_operation)
19
+ @coll.update({}, {:title => 'Moby Dick'})
20
+ end
21
+
22
+ should "send insert message" do
23
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
24
+ @db = @conn['testing']
25
+ @coll = @db.collection('books')
26
+ @conn.expects(:send_message).with do |op, msg, log|
27
+ op == 2002
28
+ end
29
+ @conn.expects(:log_operation).with do |name, payload|
30
+ (name == :insert) && payload[:documents][0][:title].include?('Moby')
31
+ end
32
+ @coll.insert({:title => 'Moby Dick'})
33
+ end
34
+
35
+ should "send sort data" do
36
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
37
+ @db = @conn['testing']
38
+ @coll = @db.collection('books')
39
+ @conn.expects(:receive_message).with do |op, msg, log, sock|
40
+ op == 2004
41
+ end.returns([[], 0, 0])
42
+ @conn.expects(:log_operation).with do |name, payload|
43
+ (name == :find) && payload[:selector][:title].include?('Moby')
44
+ end
45
+ @coll.find({:title => 'Moby Dick'}).sort([['title', 1], ['author', 1]]).next_document
46
+ end
47
+
48
+ should "not log binary data" do
49
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
50
+ @db = @conn['testing']
51
+ @coll = @db.collection('books')
52
+ data = BSON::Binary.new(("BINARY " * 1000).unpack("c*"))
53
+ @conn.expects(:send_message).with do |op, msg, log|
54
+ op == 2002
55
+ end
56
+ @conn.expects(:log_operation).with do |name, payload|
57
+ (name == :insert) && payload[:documents][0][:data].inspect.include?('Binary')
58
+ end
59
+ @coll.insert({:data => data})
60
+ end
61
+
62
+ should "send safe update message" do
63
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
64
+ @db = @conn['testing']
65
+ @coll = @db.collection('books')
66
+ @conn.expects(:send_message_with_safe_check).with do |op, msg, db_name, log|
67
+ op == 2001
68
+ end
69
+ @conn.expects(:log_operation).with do |name, payload|
70
+ (name == :update) && payload[:document][:title].include?('Moby')
71
+ end
72
+ @coll.update({}, {:title => 'Moby Dick'}, :safe => true)
73
+ end
74
+
75
+ should "send safe insert message" do
76
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
77
+ @db = @conn['testing']
78
+ @coll = @db.collection('books')
79
+ @conn.expects(:send_message_with_safe_check).with do |op, msg, db_name, log|
80
+ op == 2001
81
+ end
82
+ @conn.stubs(:log_operation)
83
+ @coll.update({}, {:title => 'Moby Dick'}, :safe => true)
84
+ end
85
+
86
+ should "not call insert for each ensure_index call" do
87
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
88
+ @db = @conn['testing']
89
+ @coll = @db.collection('books')
90
+ @coll.expects(:generate_indexes).once
91
+
92
+ @coll.ensure_index [["x", Mongo::DESCENDING]]
93
+ @coll.ensure_index [["x", Mongo::DESCENDING]]
94
+ end
95
+
96
+ should "call generate_indexes for a new direction on the same field for ensure_index" do
97
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
98
+ @db = @conn['testing']
99
+ @coll = @db.collection('books')
100
+ @coll.expects(:generate_indexes).twice
101
+
102
+ @coll.ensure_index [["x", Mongo::DESCENDING]]
103
+ @coll.ensure_index [["x", Mongo::ASCENDING]]
104
+
105
+ end
106
+
107
+ should "call generate_indexes twice because the cache time is 0 seconds" do
108
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
109
+ @db = @conn['testing']
110
+ @db.cache_time = 0
111
+ @coll = @db.collection('books')
112
+ @coll.expects(:generate_indexes).twice
113
+
114
+ @coll.ensure_index [["x", Mongo::DESCENDING]]
115
+ @coll.ensure_index [["x", Mongo::DESCENDING]]
116
+ end
117
+
118
+ should "call generate_indexes for each key when calling ensure_indexes" do
119
+ @conn = Connection.new('localhost', 27017, :logger => @logger, :connect => false)
120
+ @db = @conn['testing']
121
+ @db.cache_time = 300
122
+ @coll = @db.collection('books')
123
+ @coll.expects(:generate_indexes).once.with do |a, b, c|
124
+ a == {"x"=>-1, "y"=>-1}
125
+ end
126
+
127
+ @coll.ensure_index [["x", Mongo::DESCENDING], ["y", Mongo::DESCENDING]]
128
+ end
129
+ end
130
+ end
@@ -0,0 +1,98 @@
1
+ require './test/test_helper'
2
+ include Mongo
3
+
4
+ class ConnectionTest < Test::Unit::TestCase
5
+ context "Initialization: " do
6
+ setup do
7
+ def new_mock_socket(host='localhost', port=27017)
8
+ socket = Object.new
9
+ socket.stubs(:setsockopt).with(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
10
+ socket.stubs(:close)
11
+ socket
12
+ end
13
+
14
+ def new_mock_db
15
+ db = Object.new
16
+ end
17
+ end
18
+
19
+ context "given a single node" do
20
+ setup do
21
+ @conn = Connection.new('localhost', 27017, :connect => false)
22
+ TCPSocket.stubs(:new).returns(new_mock_socket)
23
+
24
+ admin_db = new_mock_db
25
+ admin_db.expects(:command).returns({'ok' => 1, 'ismaster' => 1}).twice
26
+ @conn.expects(:[]).with('admin').returns(admin_db).twice
27
+ @conn.connect
28
+ end
29
+
30
+ should "set localhost and port to master" do
31
+ assert_equal 'localhost', @conn.primary_pool.host
32
+ assert_equal 27017, @conn.primary_pool.port
33
+ end
34
+
35
+ should "set connection pool to 1" do
36
+ assert_equal 1, @conn.primary_pool.size
37
+ end
38
+
39
+ should "default slave_ok to false" do
40
+ assert !@conn.slave_ok?
41
+ end
42
+ end
43
+
44
+ context "initializing with a mongodb uri" do
45
+ should "parse a simple uri" do
46
+ @conn = Connection.from_uri("mongodb://localhost", :connect => false)
47
+ assert_equal ['localhost', 27017], @conn.host_to_try
48
+ end
49
+
50
+ should "allow a complex host names" do
51
+ host_name = "foo.bar-12345.org"
52
+ @conn = Connection.from_uri("mongodb://#{host_name}", :connect => false)
53
+ assert_equal [host_name, 27017], @conn.host_to_try
54
+ end
55
+
56
+ should "parse a uri with a hyphen & underscore in the username or password" do
57
+ @conn = Connection.from_uri("mongodb://hyphen-user_name:p-s_s@localhost:27017/db", :connect => false)
58
+ assert_equal ['localhost', 27017], @conn.host_to_try
59
+ auth_hash = { 'db_name' => 'db', 'username' => 'hyphen-user_name', "password" => 'p-s_s' }
60
+ assert_equal auth_hash, @conn.auths[0]
61
+ end
62
+
63
+ should "attempt to connect" do
64
+ TCPSocket.stubs(:new).returns(new_mock_socket)
65
+ @conn = Connection.from_uri("mongodb://localhost", :connect => false)
66
+
67
+ admin_db = new_mock_db
68
+ admin_db.expects(:command).returns({'ok' => 1, 'ismaster' => 1}).twice
69
+ @conn.expects(:[]).with('admin').returns(admin_db).twice
70
+ @conn.connect
71
+ end
72
+
73
+ should "raise an error on invalid uris" do
74
+ assert_raise MongoArgumentError do
75
+ Connection.from_uri("mongo://localhost", :connect => false)
76
+ end
77
+
78
+ assert_raise MongoArgumentError do
79
+ Connection.from_uri("mongodb://localhost:abc", :connect => false)
80
+ end
81
+
82
+ assert_raise MongoArgumentError do
83
+ Connection.from_uri("mongodb://localhost:27017, my.db.com:27018, ", :connect => false)
84
+ end
85
+ end
86
+
87
+ should "require all of username, password, and database if any one is specified" do
88
+ assert_raise MongoArgumentError do
89
+ Connection.from_uri("mongodb://localhost/db", :connect => false)
90
+ end
91
+
92
+ assert_raise MongoArgumentError do
93
+ Connection.from_uri("mongodb://kyle:password@localhost", :connect => false)
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end