mongo-lyon 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. data/LICENSE.txt +190 -0
  2. data/README.md +344 -0
  3. data/Rakefile +202 -0
  4. data/bin/mongo_console +34 -0
  5. data/docs/1.0_UPGRADE.md +21 -0
  6. data/docs/CREDITS.md +123 -0
  7. data/docs/FAQ.md +116 -0
  8. data/docs/GridFS.md +158 -0
  9. data/docs/HISTORY.md +225 -0
  10. data/docs/REPLICA_SETS.md +72 -0
  11. data/docs/TUTORIAL.md +247 -0
  12. data/docs/WRITE_CONCERN.md +28 -0
  13. data/lib/mongo.rb +77 -0
  14. data/lib/mongo/collection.rb +872 -0
  15. data/lib/mongo/connection.rb +875 -0
  16. data/lib/mongo/cursor.rb +449 -0
  17. data/lib/mongo/db.rb +607 -0
  18. data/lib/mongo/exceptions.rb +68 -0
  19. data/lib/mongo/gridfs/grid.rb +106 -0
  20. data/lib/mongo/gridfs/grid_ext.rb +57 -0
  21. data/lib/mongo/gridfs/grid_file_system.rb +145 -0
  22. data/lib/mongo/gridfs/grid_io.rb +394 -0
  23. data/lib/mongo/gridfs/grid_io_fix.rb +38 -0
  24. data/lib/mongo/repl_set_connection.rb +342 -0
  25. data/lib/mongo/util/conversions.rb +89 -0
  26. data/lib/mongo/util/core_ext.rb +60 -0
  27. data/lib/mongo/util/pool.rb +185 -0
  28. data/lib/mongo/util/server_version.rb +71 -0
  29. data/lib/mongo/util/support.rb +82 -0
  30. data/lib/mongo/util/uri_parser.rb +181 -0
  31. data/lib/mongo/version.rb +3 -0
  32. data/mongo.gemspec +34 -0
  33. data/test/auxillary/1.4_features.rb +166 -0
  34. data/test/auxillary/authentication_test.rb +68 -0
  35. data/test/auxillary/autoreconnect_test.rb +41 -0
  36. data/test/auxillary/repl_set_auth_test.rb +58 -0
  37. data/test/auxillary/slave_connection_test.rb +36 -0
  38. data/test/auxillary/threaded_authentication_test.rb +101 -0
  39. data/test/bson/binary_test.rb +15 -0
  40. data/test/bson/bson_test.rb +614 -0
  41. data/test/bson/byte_buffer_test.rb +190 -0
  42. data/test/bson/hash_with_indifferent_access_test.rb +38 -0
  43. data/test/bson/json_test.rb +17 -0
  44. data/test/bson/object_id_test.rb +154 -0
  45. data/test/bson/ordered_hash_test.rb +197 -0
  46. data/test/collection_test.rb +893 -0
  47. data/test/connection_test.rb +303 -0
  48. data/test/conversions_test.rb +120 -0
  49. data/test/cursor_fail_test.rb +75 -0
  50. data/test/cursor_message_test.rb +43 -0
  51. data/test/cursor_test.rb +457 -0
  52. data/test/db_api_test.rb +715 -0
  53. data/test/db_connection_test.rb +15 -0
  54. data/test/db_test.rb +287 -0
  55. data/test/grid_file_system_test.rb +244 -0
  56. data/test/grid_io_test.rb +120 -0
  57. data/test/grid_test.rb +200 -0
  58. data/test/load/thin/load.rb +24 -0
  59. data/test/load/unicorn/load.rb +23 -0
  60. data/test/replica_sets/connect_test.rb +86 -0
  61. data/test/replica_sets/connection_string_test.rb +32 -0
  62. data/test/replica_sets/count_test.rb +35 -0
  63. data/test/replica_sets/insert_test.rb +53 -0
  64. data/test/replica_sets/pooled_insert_test.rb +55 -0
  65. data/test/replica_sets/query_secondaries.rb +96 -0
  66. data/test/replica_sets/query_test.rb +51 -0
  67. data/test/replica_sets/replication_ack_test.rb +66 -0
  68. data/test/replica_sets/rs_test_helper.rb +27 -0
  69. data/test/safe_test.rb +68 -0
  70. data/test/support/hash_with_indifferent_access.rb +199 -0
  71. data/test/support/keys.rb +45 -0
  72. data/test/support_test.rb +19 -0
  73. data/test/test_helper.rb +83 -0
  74. data/test/threading/threading_with_large_pool_test.rb +90 -0
  75. data/test/threading_test.rb +87 -0
  76. data/test/tools/auth_repl_set_manager.rb +14 -0
  77. data/test/tools/repl_set_manager.rb +266 -0
  78. data/test/unit/collection_test.rb +130 -0
  79. data/test/unit/connection_test.rb +98 -0
  80. data/test/unit/cursor_test.rb +99 -0
  81. data/test/unit/db_test.rb +96 -0
  82. data/test/unit/grid_test.rb +49 -0
  83. data/test/unit/pool_test.rb +9 -0
  84. data/test/unit/repl_set_connection_test.rb +72 -0
  85. data/test/unit/safe_test.rb +125 -0
  86. data/test/uri_test.rb +91 -0
  87. metadata +202 -0
@@ -0,0 +1,120 @@
1
+ require './test/test_helper'
2
+ include Mongo
3
+
4
+ class GridIOTest < Test::Unit::TestCase
5
+
6
+ context "GridIO" do
7
+ setup do
8
+ @db = standard_connection.db(MONGO_TEST_DB)
9
+ @files = @db.collection('fs.files')
10
+ @chunks = @db.collection('fs.chunks')
11
+ @chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]])
12
+ end
13
+
14
+ teardown do
15
+ @files.remove
16
+ @chunks.remove
17
+ end
18
+
19
+ context "Options" do
20
+ setup do
21
+ @filename = 'test'
22
+ @mode = 'w'
23
+ end
24
+
25
+ should "set default 256k chunk size" do
26
+ file = GridIO.new(@files, @chunks, @filename, @mode)
27
+ assert_equal 256 * 1024, file.chunk_size
28
+ end
29
+
30
+ should "set chunk size" do
31
+ file = GridIO.new(@files, @chunks, @filename, @mode, :chunk_size => 1000)
32
+ assert_equal 1000, file.chunk_size
33
+ end
34
+ end
35
+
36
+ context "Seeking" do
37
+ setup do
38
+ @filename = 'test'
39
+ @mode = 'w'
40
+ @data = "1" * 1024 * 1024
41
+ @file = GridIO.new(@files, @chunks, @filename, @mode)
42
+ @file.write(@data)
43
+ @file.close
44
+ end
45
+
46
+ should "read all data using read_length and then be able to seek" do
47
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
48
+ assert_equal @data, file.read(1024 * 1024)
49
+ file.seek(0)
50
+ assert_equal @data, file.read
51
+ end
52
+
53
+ should "read all data using read_all and then be able to seek" do
54
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
55
+ assert_equal @data, file.read
56
+ file.seek(0)
57
+ assert_equal @data, file.read
58
+ file.seek(1024 * 512)
59
+ assert_equal 524288, file.file_position
60
+ assert_equal @data.length / 2, file.read.length
61
+ assert_equal 1048576, file.file_position
62
+ assert_nil file.read
63
+ file.seek(1024 * 512)
64
+ assert_equal 524288, file.file_position
65
+ end
66
+
67
+ end
68
+
69
+ context "Grid MD5 check" do
70
+ should "run in safe mode" do
71
+ file = GridIO.new(@files, @chunks, 'smallfile', 'w', :safe => true)
72
+ file.write("DATA" * 100)
73
+ assert file.close
74
+ assert_equal file.server_md5, file.client_md5
75
+ end
76
+
77
+ should "validate with a large file" do
78
+ io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
79
+ file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
80
+ file.write(io)
81
+ assert file.close
82
+ assert_equal file.server_md5, file.client_md5
83
+ end
84
+
85
+ should "raise an exception when check fails" do
86
+ io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
87
+ @db.stubs(:command).returns({'md5' => '12345'})
88
+ file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
89
+ file.write(io)
90
+ assert_raise GridMD5Failure do
91
+ assert file.close
92
+ end
93
+ assert_not_equal file.server_md5, file.client_md5
94
+ end
95
+ end
96
+
97
+ context "Content types" do
98
+ if defined?(MIME)
99
+ should "determine common content types from the extension" do
100
+ file = GridIO.new(@files, @chunks, 'sample.pdf', 'w')
101
+ assert_equal 'application/pdf', file.content_type
102
+
103
+ file = GridIO.new(@files, @chunks, 'sample.txt', 'w')
104
+ assert_equal 'text/plain', file.content_type
105
+ end
106
+ end
107
+
108
+ should "default to binary/octet-stream when type is unknown" do
109
+ file = GridIO.new(@files, @chunks, 'sample.l33t', 'w')
110
+ assert_equal 'binary/octet-stream', file.content_type
111
+ end
112
+
113
+ should "use any provided content type by default" do
114
+ file = GridIO.new(@files, @chunks, 'sample.l33t', 'w', :content_type => 'image/jpg')
115
+ assert_equal 'image/jpg', file.content_type
116
+ end
117
+ end
118
+ end
119
+
120
+ end
data/test/grid_test.rb ADDED
@@ -0,0 +1,200 @@
1
+ require './test/test_helper'
2
+ include Mongo
3
+
4
+ class GridTest < Test::Unit::TestCase
5
+ context "Tests:" do
6
+ setup do
7
+ @db = standard_connection.db(MONGO_TEST_DB)
8
+ @files = @db.collection('test-fs.files')
9
+ @chunks = @db.collection('test-fs.chunks')
10
+ end
11
+
12
+ teardown do
13
+ @files.remove
14
+ @chunks.remove
15
+ end
16
+
17
+ context "A basic grid-stored file" do
18
+ setup do
19
+ @data = "GRIDDATA" * 50000
20
+ @grid = Grid.new(@db, 'test-fs')
21
+ @id = @grid.put(@data, :filename => 'sample',
22
+ :metadata => {'app' => 'photos'})
23
+ end
24
+
25
+ should "check existence" do
26
+ file = @grid.exist?(:filename => 'sample')
27
+ assert_equal 'sample', file['filename']
28
+ end
29
+
30
+ should "not be able to overwrite an exising file" do
31
+ assert_raise GridError do
32
+ @grid.put(@data, :filename => 'sample', :_id => @id, :safe => true)
33
+ end
34
+ end
35
+
36
+ should "return nil if it doesn't exist" do
37
+ assert_nil @grid.exist?(:metadata => 'foo')
38
+ end
39
+
40
+ should "retrieve the stored data" do
41
+ data = @grid.get(@id).data
42
+ assert_equal @data, data
43
+ end
44
+
45
+ should "have a unique index on chunks" do
46
+ assert @chunks.index_information['files_id_1_n_1']['unique']
47
+ end
48
+
49
+ should "store the filename" do
50
+ file = @grid.get(@id)
51
+ assert_equal 'sample', file.filename
52
+ end
53
+
54
+ should "store any relevant metadata" do
55
+ file = @grid.get(@id)
56
+ assert_equal 'photos', file.metadata['app']
57
+ end
58
+
59
+ should "delete the file and any chunks" do
60
+ @grid.delete(@id)
61
+ assert_raise GridFileNotFound do
62
+ @grid.get(@id)
63
+ end
64
+ assert_equal nil, @db['test-fs']['chunks'].find_one({:files_id => @id})
65
+ end
66
+ end
67
+
68
+ context "Filename not required" do
69
+ setup do
70
+ @data = "GRIDDATA" * 50000
71
+ @grid = Grid.new(@db, 'test-fs')
72
+ @metadata = {'app' => 'photos'}
73
+ end
74
+
75
+ should "store the file with the old filename api" do
76
+ id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
77
+ file = @grid.get(id)
78
+ assert_equal 'sample', file.filename
79
+ assert_equal @metadata, file.metadata
80
+ end
81
+
82
+ should "store without a filename" do
83
+ id = @grid.put(@data, :metadata => @metadata)
84
+ file = @grid.get(id)
85
+ assert_nil file.filename
86
+ file_doc = @files.find_one({'_id' => id})
87
+ assert !file_doc.has_key?('filename')
88
+ assert_equal @metadata, file.metadata
89
+ end
90
+
91
+ should "store with filename and metadata with the new api" do
92
+ id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
93
+ file = @grid.get(id)
94
+ assert_equal 'sample', file.filename
95
+ assert_equal @metadata, file.metadata
96
+ end
97
+ end
98
+
99
+ context "Writing arbitrary data fields" do
100
+ setup do
101
+ @data = "GRIDDATA" * 50000
102
+ @grid = Grid.new(@db, 'test-fs')
103
+ end
104
+
105
+ should "write random keys to the files collection" do
106
+ id = @grid.put(@data, :phrases => ["blimey", "ahoy!"])
107
+ file = @grid.get(id)
108
+
109
+ assert_equal ["blimey", "ahoy!"], file['phrases']
110
+ end
111
+
112
+ should "ignore special keys" do
113
+ id = @grid.put(@data, :file_length => 100, :phrase => "blimey")
114
+ file = @grid.get(id)
115
+
116
+ assert_equal "blimey", file['phrase']
117
+ assert_equal 400_000, file.file_length
118
+ end
119
+ end
120
+
121
+ context "Storing data with a length of zero" do
122
+ setup do
123
+ @grid = Grid.new(@db, 'test-fs')
124
+ @id = @grid.put('', :filename => 'sample',
125
+ :metadata => {'app' => 'photos'})
126
+ end
127
+
128
+ should "return the zero length" do
129
+ data = @grid.get(@id)
130
+ assert_equal 0, data.read.length
131
+ end
132
+ end
133
+
134
+ context "Grid streaming: " do
135
+ setup do
136
+ @grid = Grid.new(@db, 'test-fs')
137
+ filename = 'sample_data'
138
+ @io = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
139
+ id = @grid.put(@io, :filename => filename)
140
+ @file = @grid.get(id)
141
+ @io.rewind
142
+ @data = @io.read
143
+ if @data.respond_to?(:force_encoding)
144
+ @data.force_encoding("binary")
145
+ end
146
+ end
147
+
148
+ should "read the file" do
149
+ read_data = ""
150
+ @file.each do |chunk|
151
+ read_data << chunk
152
+ end
153
+ assert_equal @data.length, read_data.length
154
+ end
155
+
156
+ should "read the file if no block is given" do
157
+ read_data = @file.each
158
+ assert_equal @data.length, read_data.length
159
+ end
160
+ end
161
+
162
+ context "Streaming: " do || {}
163
+ setup do
164
+ def read_and_write_stream(filename, read_length, opts={})
165
+ io = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
166
+ id = @grid.put(io, opts.merge!(:filename => filename + read_length.to_s))
167
+ file = @grid.get(id)
168
+ io.rewind
169
+ data = io.read
170
+ if data.respond_to?(:force_encoding)
171
+ data.force_encoding("binary")
172
+ end
173
+ read_data = ""
174
+ while(chunk = file.read(read_length))
175
+ read_data << chunk
176
+ end
177
+ assert_equal data.length, read_data.length
178
+ end
179
+
180
+ @grid = Grid.new(@db, 'test-fs')
181
+ end
182
+
183
+ should "put and get a small io object with a small chunk size" do
184
+ read_and_write_stream('small_data.txt', 1, :chunk_size => 2)
185
+ end
186
+
187
+ should "put and get a small io object" do
188
+ read_and_write_stream('small_data.txt', 1)
189
+ end
190
+
191
+ should "put and get a large io object if reading less than the chunk size" do
192
+ read_and_write_stream('sample_data', 256 * 1024)
193
+ end
194
+
195
+ should "put and get a large io object if reading more than the chunk size" do
196
+ read_and_write_stream('sample_data', 300 * 1024)
197
+ end
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,24 @@
1
+ require File.join(File.dirname(__FILE__), '..', '..', '..', 'lib', 'mongo')
2
+ require 'logger'
3
+
4
+ $con = Mongo::Connection.new
5
+ $db = $con['foo']
6
+
7
+ class Load < Sinatra::Base
8
+
9
+ configure do
10
+ LOGGER = Logger.new("sinatra.log")
11
+ enable :logging, :dump_errors
12
+ set :raise_errors, true
13
+ end
14
+
15
+ get '/' do
16
+ 3.times do |n|
17
+ if (v=$db.eval("1 + #{n}")) != 1 + n
18
+ STDERR << "#{1 + n} expected but got #{v}"
19
+ raise StandardError, "#{1 + n} expected but got #{v}"
20
+ end
21
+ end
22
+ end
23
+
24
+ end
@@ -0,0 +1,23 @@
1
+ require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
2
+
3
+ $con = Mongo::Connection.new
4
+ $db = $con['foo']
5
+
6
+ class Load < Sinatra::Base
7
+
8
+ configure do
9
+ LOGGER = Logger.new("sinatra.log")
10
+ enable :logging, :dump_errors
11
+ set :raise_errors, true
12
+ end
13
+
14
+ get '/' do
15
+ 3.times do |n|
16
+ if (v=$db.eval("1 + #{n}")) != 1 + n
17
+ STDERR << "#{1 + n} expected but got #{v}"
18
+ raise StandardError, "#{1 + n} expected but got #{v}"
19
+ end
20
+ end
21
+ end
22
+
23
+ end
@@ -0,0 +1,86 @@
1
+ $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ require './test/replica_sets/rs_test_helper'
3
+
4
+ # NOTE: This test expects a replica set of three nodes to be running on RS.host,
5
+ # on ports TEST_PORT, RS.ports[1], and TEST + 2.
6
+ class ConnectTest < Test::Unit::TestCase
7
+ include Mongo
8
+
9
+ def setup
10
+ RS.restart_killed_nodes
11
+ end
12
+
13
+ def teardown
14
+ RS.restart_killed_nodes
15
+ end
16
+
17
+ def test_connect_with_deprecated_multi
18
+ @conn = Connection.multi([[RS.host, RS.ports[0]], [RS.host, RS.ports[1]]], :name => RS.name)
19
+ assert @conn.is_a?(ReplSetConnection)
20
+ assert @conn.connected?
21
+ end
22
+
23
+ def test_connect_bad_name
24
+ assert_raise_error(ReplicaSetConnectionError, "-wrong") do
25
+ ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
26
+ [RS.host, RS.ports[2]], :rs_name => RS.name + "-wrong")
27
+ end
28
+ end
29
+
30
+ def test_connect
31
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
32
+ [RS.host, RS.ports[2]], :name => RS.name)
33
+ assert @conn.connected?
34
+ assert @conn.read_primary?
35
+ assert @conn.primary?
36
+
37
+ assert_equal RS.primary, @conn.primary
38
+ assert_equal RS.secondaries.sort, @conn.secondaries.sort
39
+ assert_equal RS.arbiters.sort, @conn.arbiters.sort
40
+ end
41
+
42
+ def test_connect_with_primary_node_killed
43
+ node = RS.kill_primary
44
+
45
+ # Becuase we're killing the primary and trying to connect right away,
46
+ # this is going to fail right away.
47
+ assert_raise_error(ConnectionFailure, "Failed to connect to primary node") do
48
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
49
+ [RS.host, RS.ports[2]])
50
+ end
51
+
52
+ # This allows the secondary to come up as a primary
53
+ rescue_connection_failure do
54
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
55
+ [RS.host, RS.ports[2]])
56
+ end
57
+ assert @conn.connected?
58
+ end
59
+
60
+ def test_connect_with_secondary_node_killed
61
+ node = RS.kill_secondary
62
+
63
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
64
+ [RS.host, RS.ports[2]])
65
+ assert @conn.connected?
66
+ end
67
+
68
+ def test_connect_with_third_node_killed
69
+ RS.kill(RS.get_node_from_port(RS.ports[2]))
70
+
71
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
72
+ [RS.host, RS.ports[2]])
73
+ assert @conn.connected?
74
+ end
75
+
76
+ def test_connect_with_primary_stepped_down
77
+ RS.step_down_primary
78
+
79
+ rescue_connection_failure do
80
+ @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
81
+ [RS.host, RS.ports[2]])
82
+ end
83
+ assert @conn.connected?
84
+ end
85
+
86
+ end