jonbell-mongo 1.3.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. data/LICENSE.txt +190 -0
  2. data/README.md +333 -0
  3. data/Rakefile +215 -0
  4. data/bin/mongo_console +21 -0
  5. data/docs/CREDITS.md +123 -0
  6. data/docs/FAQ.md +116 -0
  7. data/docs/GridFS.md +158 -0
  8. data/docs/HISTORY.md +263 -0
  9. data/docs/RELEASES.md +33 -0
  10. data/docs/REPLICA_SETS.md +72 -0
  11. data/docs/TUTORIAL.md +247 -0
  12. data/docs/WRITE_CONCERN.md +28 -0
  13. data/lib/mongo.rb +97 -0
  14. data/lib/mongo/collection.rb +895 -0
  15. data/lib/mongo/connection.rb +926 -0
  16. data/lib/mongo/cursor.rb +474 -0
  17. data/lib/mongo/db.rb +617 -0
  18. data/lib/mongo/exceptions.rb +71 -0
  19. data/lib/mongo/gridfs/grid.rb +107 -0
  20. data/lib/mongo/gridfs/grid_ext.rb +57 -0
  21. data/lib/mongo/gridfs/grid_file_system.rb +146 -0
  22. data/lib/mongo/gridfs/grid_io.rb +485 -0
  23. data/lib/mongo/gridfs/grid_io_fix.rb +38 -0
  24. data/lib/mongo/repl_set_connection.rb +356 -0
  25. data/lib/mongo/util/conversions.rb +89 -0
  26. data/lib/mongo/util/core_ext.rb +60 -0
  27. data/lib/mongo/util/pool.rb +177 -0
  28. data/lib/mongo/util/server_version.rb +71 -0
  29. data/lib/mongo/util/support.rb +82 -0
  30. data/lib/mongo/util/uri_parser.rb +185 -0
  31. data/mongo.gemspec +34 -0
  32. data/test/auxillary/1.4_features.rb +166 -0
  33. data/test/auxillary/authentication_test.rb +68 -0
  34. data/test/auxillary/autoreconnect_test.rb +41 -0
  35. data/test/auxillary/fork_test.rb +30 -0
  36. data/test/auxillary/repl_set_auth_test.rb +58 -0
  37. data/test/auxillary/slave_connection_test.rb +36 -0
  38. data/test/auxillary/threaded_authentication_test.rb +101 -0
  39. data/test/bson/binary_test.rb +15 -0
  40. data/test/bson/bson_test.rb +654 -0
  41. data/test/bson/byte_buffer_test.rb +208 -0
  42. data/test/bson/hash_with_indifferent_access_test.rb +38 -0
  43. data/test/bson/json_test.rb +17 -0
  44. data/test/bson/object_id_test.rb +154 -0
  45. data/test/bson/ordered_hash_test.rb +210 -0
  46. data/test/bson/timestamp_test.rb +24 -0
  47. data/test/collection_test.rb +910 -0
  48. data/test/connection_test.rb +324 -0
  49. data/test/conversions_test.rb +119 -0
  50. data/test/cursor_fail_test.rb +75 -0
  51. data/test/cursor_message_test.rb +43 -0
  52. data/test/cursor_test.rb +483 -0
  53. data/test/db_api_test.rb +738 -0
  54. data/test/db_connection_test.rb +15 -0
  55. data/test/db_test.rb +315 -0
  56. data/test/grid_file_system_test.rb +259 -0
  57. data/test/grid_io_test.rb +209 -0
  58. data/test/grid_test.rb +258 -0
  59. data/test/load/thin/load.rb +24 -0
  60. data/test/load/unicorn/load.rb +23 -0
  61. data/test/replica_sets/connect_test.rb +112 -0
  62. data/test/replica_sets/connection_string_test.rb +32 -0
  63. data/test/replica_sets/count_test.rb +35 -0
  64. data/test/replica_sets/insert_test.rb +53 -0
  65. data/test/replica_sets/pooled_insert_test.rb +55 -0
  66. data/test/replica_sets/query_secondaries.rb +108 -0
  67. data/test/replica_sets/query_test.rb +51 -0
  68. data/test/replica_sets/replication_ack_test.rb +66 -0
  69. data/test/replica_sets/rs_test_helper.rb +27 -0
  70. data/test/safe_test.rb +68 -0
  71. data/test/support/hash_with_indifferent_access.rb +186 -0
  72. data/test/support/keys.rb +45 -0
  73. data/test/support_test.rb +18 -0
  74. data/test/test_helper.rb +102 -0
  75. data/test/threading/threading_with_large_pool_test.rb +90 -0
  76. data/test/threading_test.rb +87 -0
  77. data/test/tools/auth_repl_set_manager.rb +14 -0
  78. data/test/tools/repl_set_manager.rb +266 -0
  79. data/test/unit/collection_test.rb +130 -0
  80. data/test/unit/connection_test.rb +85 -0
  81. data/test/unit/cursor_test.rb +109 -0
  82. data/test/unit/db_test.rb +94 -0
  83. data/test/unit/grid_test.rb +49 -0
  84. data/test/unit/pool_test.rb +9 -0
  85. data/test/unit/repl_set_connection_test.rb +59 -0
  86. data/test/unit/safe_test.rb +125 -0
  87. data/test/uri_test.rb +91 -0
  88. metadata +224 -0
@@ -0,0 +1,209 @@
1
+ require './test/test_helper'
2
+
3
+ class GridIOTest < Test::Unit::TestCase
4
+
5
+ context "GridIO" do
6
+ setup do
7
+ @db = standard_connection.db(MONGO_TEST_DB)
8
+ @files = @db.collection('fs.files')
9
+ @chunks = @db.collection('fs.chunks')
10
+ @chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]])
11
+ end
12
+
13
+ teardown do
14
+ @files.remove
15
+ @chunks.remove
16
+ end
17
+
18
+ context "Options" do
19
+ setup do
20
+ @filename = 'test'
21
+ @mode = 'w'
22
+ end
23
+
24
+ should "set default 256k chunk size" do
25
+ file = GridIO.new(@files, @chunks, @filename, @mode)
26
+ assert_equal 256 * 1024, file.chunk_size
27
+ end
28
+
29
+ should "set chunk size" do
30
+ file = GridIO.new(@files, @chunks, @filename, @mode, :chunk_size => 1000)
31
+ assert_equal 1000, file.chunk_size
32
+ end
33
+ end
34
+
35
+ context "StringIO methods" do
36
+ setup do
37
+ @filename = 'test'
38
+ @mode = 'w'
39
+ @data = "012345678\n" * 100000
40
+ @file = GridIO.new(@files, @chunks, @filename, @mode)
41
+ @file.write(@data)
42
+ @file.close
43
+ end
44
+
45
+ should "read data character by character using" do
46
+ bytes = 0
47
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
48
+ while char = file.getc
49
+ bytes += 1
50
+ end
51
+ assert_equal bytes, 1_000_000
52
+ end
53
+
54
+ should "read length is a length is given" do
55
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
56
+ string = file.gets(1000)
57
+ assert_equal string.length, 1000
58
+ bytes = 0
59
+ bytes += string.length
60
+ while string = file.gets(1000)
61
+ bytes += string.length
62
+ end
63
+ assert_equal bytes, 1_000_000
64
+ end
65
+
66
+ should "read to the end of the line by default and assign to $_" do
67
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
68
+ string = file.gets
69
+ assert_equal 10, string.length
70
+ end
71
+
72
+ should "read to the end of the file one line at a time" do
73
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
74
+ bytes = 0
75
+ while string = file.gets
76
+ bytes += string.length
77
+ end
78
+ assert_equal 1_000_000, bytes
79
+ end
80
+
81
+ should "read to the end of the file one multi-character separator at a time" do
82
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
83
+ bytes = 0
84
+ while string = file.gets("45")
85
+ bytes += string.length
86
+ end
87
+ assert_equal 1_000_000, bytes
88
+ end
89
+
90
+ should "read to a given separator" do
91
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
92
+ string = file.gets("5")
93
+ assert_equal 6, string.length
94
+ end
95
+
96
+ should "read a multi-character separator" do
97
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
98
+ string = file.gets("45")
99
+ assert_equal 6, string.length
100
+ string = file.gets("45")
101
+ assert_equal "678\n012345", string
102
+ string = file.gets("\n01")
103
+ assert_equal "678\n01", string
104
+ end
105
+
106
+ should "read a mult-character separator with a length" do
107
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
108
+ string = file.gets("45", 3)
109
+ assert_equal 3, string.length
110
+ end
111
+
112
+ should "tell position, eof, and rewind" do
113
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
114
+ string = file.read(1000)
115
+ assert_equal 1000, file.pos
116
+ assert !file.eof?
117
+ file.read
118
+ assert file.eof?
119
+ file.rewind
120
+ assert_equal 0, file.pos
121
+ assert_equal 1_000_000, file.read.length
122
+ end
123
+ end
124
+
125
+ context "Seeking" do
126
+ setup do
127
+ @filename = 'test'
128
+ @mode = 'w'
129
+ @data = "1" * 1024 * 1024
130
+ @file = GridIO.new(@files, @chunks, @filename, @mode)
131
+ @file.write(@data)
132
+ @file.close
133
+ end
134
+
135
+ should "read all data using read_length and then be able to seek" do
136
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
137
+ assert_equal @data, file.read(1024 * 1024)
138
+ file.seek(0)
139
+ assert_equal @data, file.read
140
+ end
141
+
142
+ should "read all data using read_all and then be able to seek" do
143
+ file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
144
+ assert_equal @data, file.read
145
+ file.seek(0)
146
+ assert_equal @data, file.read
147
+ file.seek(1024 * 512)
148
+ assert_equal 524288, file.file_position
149
+ assert_equal @data.length / 2, file.read.length
150
+ assert_equal 1048576, file.file_position
151
+ assert_nil file.read
152
+ file.seek(1024 * 512)
153
+ assert_equal 524288, file.file_position
154
+ end
155
+
156
+ end
157
+
158
+ context "Grid MD5 check" do
159
+ should "run in safe mode" do
160
+ file = GridIO.new(@files, @chunks, 'smallfile', 'w', :safe => true)
161
+ file.write("DATA" * 100)
162
+ assert file.close
163
+ assert_equal file.server_md5, file.client_md5
164
+ end
165
+
166
+ should "validate with a large file" do
167
+ io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
168
+ file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
169
+ file.write(io)
170
+ assert file.close
171
+ assert_equal file.server_md5, file.client_md5
172
+ end
173
+
174
+ should "raise an exception when check fails" do
175
+ io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
176
+ @db.stubs(:command).returns({'md5' => '12345'})
177
+ file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
178
+ file.write(io)
179
+ assert_raise GridMD5Failure do
180
+ assert file.close
181
+ end
182
+ assert_not_equal file.server_md5, file.client_md5
183
+ end
184
+ end
185
+
186
+ context "Content types" do
187
+ if defined?(MIME)
188
+ should "determine common content types from the extension" do
189
+ file = GridIO.new(@files, @chunks, 'sample.pdf', 'w')
190
+ assert_equal 'application/pdf', file.content_type
191
+
192
+ file = GridIO.new(@files, @chunks, 'sample.txt', 'w')
193
+ assert_equal 'text/plain', file.content_type
194
+ end
195
+ end
196
+
197
+ should "default to binary/octet-stream when type is unknown" do
198
+ file = GridIO.new(@files, @chunks, 'sample.l33t', 'w')
199
+ assert_equal 'binary/octet-stream', file.content_type
200
+ end
201
+
202
+ should "use any provided content type by default" do
203
+ file = GridIO.new(@files, @chunks, 'sample.l33t', 'w', :content_type => 'image/jpg')
204
+ assert_equal 'image/jpg', file.content_type
205
+ end
206
+ end
207
+ end
208
+
209
+ end
data/test/grid_test.rb ADDED
@@ -0,0 +1,258 @@
1
+ require './test/test_helper'
2
+ include Mongo
3
+
4
+ def read_and_write_stream(filename, read_length, opts={})
5
+ io = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
6
+ id = @grid.put(io, opts.merge!(:filename => filename + read_length.to_s))
7
+ file = @grid.get(id)
8
+ io.rewind
9
+ data = io.read
10
+ if data.respond_to?(:force_encoding)
11
+ data.force_encoding("binary")
12
+ end
13
+ read_data = ""
14
+ while(chunk = file.read(read_length))
15
+ read_data << chunk
16
+ break if chunk.empty?
17
+ end
18
+ assert_equal data.length, read_data.length
19
+ end
20
+
21
+ class GridTest < Test::Unit::TestCase
22
+ context "Tests:" do
23
+ setup do
24
+ @db = standard_connection.db(MONGO_TEST_DB)
25
+ @files = @db.collection('test-fs.files')
26
+ @chunks = @db.collection('test-fs.chunks')
27
+ end
28
+
29
+ teardown do
30
+ @files.remove
31
+ @chunks.remove
32
+ end
33
+
34
+ context "A one-chunk grid-stored file" do
35
+ setup do
36
+ @data = "GRIDDATA" * 5
37
+ @grid = Grid.new(@db, 'test-fs')
38
+ @id = @grid.put(@data, :filename => 'sample',
39
+ :metadata => {'app' => 'photos'})
40
+ end
41
+
42
+ should "retrieve the file" do
43
+ data = @grid.get(@id).data
44
+ assert_equal @data, data
45
+ end
46
+
47
+ end
48
+
49
+ context "A basic grid-stored file" do
50
+ setup do
51
+ @data = "GRIDDATA" * 50000
52
+ @grid = Grid.new(@db, 'test-fs')
53
+ @id = @grid.put(@data, :filename => 'sample',
54
+ :metadata => {'app' => 'photos'})
55
+ end
56
+
57
+ should "check existence" do
58
+ file = @grid.exist?(:filename => 'sample')
59
+ assert_equal 'sample', file['filename']
60
+ end
61
+
62
+ should "not be able to overwrite an exising file" do
63
+ assert_raise GridError do
64
+ @grid.put(@data, :filename => 'sample', :_id => @id, :safe => true)
65
+ end
66
+ end
67
+
68
+ should "return nil if it doesn't exist" do
69
+ assert_nil @grid.exist?(:metadata => 'foo')
70
+ end
71
+
72
+ should "retrieve the stored data" do
73
+ data = @grid.get(@id).data
74
+ assert_equal @data.length, data.length
75
+ end
76
+
77
+ should "have a unique index on chunks" do
78
+ assert @chunks.index_information['files_id_1_n_1']['unique']
79
+ end
80
+
81
+ should "store the filename" do
82
+ file = @grid.get(@id)
83
+ assert_equal 'sample', file.filename
84
+ end
85
+
86
+ should "store any relevant metadata" do
87
+ file = @grid.get(@id)
88
+ assert_equal 'photos', file.metadata['app']
89
+ end
90
+
91
+ should "delete the file and any chunks" do
92
+ @grid.delete(@id)
93
+ assert_raise GridFileNotFound do
94
+ @grid.get(@id)
95
+ end
96
+ assert_equal nil, @db['test-fs']['chunks'].find_one({:files_id => @id})
97
+ end
98
+ end
99
+
100
+ context "Filename not required" do
101
+ setup do
102
+ @data = "GRIDDATA" * 50000
103
+ @grid = Grid.new(@db, 'test-fs')
104
+ @metadata = {'app' => 'photos'}
105
+ end
106
+
107
+ should "store the file with the old filename api" do
108
+ id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
109
+ file = @grid.get(id)
110
+ assert_equal 'sample', file.filename
111
+ assert_equal @metadata, file.metadata
112
+ end
113
+
114
+ should "store without a filename" do
115
+ id = @grid.put(@data, :metadata => @metadata)
116
+ file = @grid.get(id)
117
+ assert_nil file.filename
118
+ file_doc = @files.find_one({'_id' => id})
119
+ assert !file_doc.has_key?('filename')
120
+ assert_equal @metadata, file.metadata
121
+ end
122
+
123
+ should "store with filename and metadata with the new api" do
124
+ id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
125
+ file = @grid.get(id)
126
+ assert_equal 'sample', file.filename
127
+ assert_equal @metadata, file.metadata
128
+ end
129
+ end
130
+
131
+ context "Writing arbitrary data fields" do
132
+ setup do
133
+ @data = "GRIDDATA" * 50000
134
+ @grid = Grid.new(@db, 'test-fs')
135
+ end
136
+
137
+ should "write random keys to the files collection" do
138
+ id = @grid.put(@data, :phrases => ["blimey", "ahoy!"])
139
+ file = @grid.get(id)
140
+
141
+ assert_equal ["blimey", "ahoy!"], file['phrases']
142
+ end
143
+
144
+ should "ignore special keys" do
145
+ id = @grid.put(@data, :file_length => 100, :phrase => "blimey")
146
+ file = @grid.get(id)
147
+
148
+ assert_equal "blimey", file['phrase']
149
+ assert_equal 400_000, file.file_length
150
+ end
151
+ end
152
+
153
+ context "Storing data with a length of zero" do
154
+ setup do
155
+ @grid = Grid.new(@db, 'test-fs')
156
+ @id = @grid.put('', :filename => 'sample',
157
+ :metadata => {'app' => 'photos'})
158
+ end
159
+
160
+ should "return the zero length" do
161
+ data = @grid.get(@id)
162
+ assert_equal 0, data.read.length
163
+ end
164
+ end
165
+
166
+ context "Grid streaming: " do
167
+ setup do
168
+ @grid = Grid.new(@db, 'test-fs')
169
+ filename = 'sample_data'
170
+ @io = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
171
+ id = @grid.put(@io, :filename => filename)
172
+ @file = @grid.get(id)
173
+ @io.rewind
174
+ @data = @io.read
175
+ if @data.respond_to?(:force_encoding)
176
+ @data.force_encoding("binary")
177
+ end
178
+ end
179
+
180
+ should "be equal in length" do
181
+ @io.rewind
182
+ assert_equal @io.read.length, @file.read.length
183
+ end
184
+
185
+ should "read the file" do
186
+ read_data = ""
187
+ @file.each do |chunk|
188
+ read_data << chunk
189
+ end
190
+ assert_equal @data.length, read_data.length
191
+ end
192
+
193
+ should "read the file if no block is given" do
194
+ read_data = @file.each
195
+ assert_equal @data.length, read_data.length
196
+ end
197
+ end
198
+
199
+ context "Grid streaming an empty file: " do
200
+ setup do
201
+ @grid = Grid.new(@db, 'test-fs')
202
+ filename = 'empty_data'
203
+ @io = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
204
+ id = @grid.put(@io, :filename => filename)
205
+ @file = @grid.get(id)
206
+ @io.rewind
207
+ @data = @io.read
208
+ if @data.respond_to?(:force_encoding)
209
+ @data.force_encoding("binary")
210
+ end
211
+ end
212
+
213
+ should "be equal in length" do
214
+ @io.rewind
215
+ assert_equal @io.read.length, @file.read.length
216
+ end
217
+
218
+ should "read the file" do
219
+ read_data = ""
220
+ @file.each do |chunk|
221
+ read_data << chunk
222
+ end
223
+ assert_equal @data.length, read_data.length
224
+ end
225
+
226
+ should "read the file if no block is given" do
227
+ read_data = @file.each
228
+ assert_equal @data.length, read_data.length
229
+ end
230
+ end
231
+
232
+ context "Streaming: " do || {}
233
+ setup do
234
+ @grid = Grid.new(@db, 'test-fs')
235
+ end
236
+
237
+ should "put and get a small io object with a small chunk size" do
238
+ read_and_write_stream('small_data.txt', 1, :chunk_size => 2)
239
+ end
240
+
241
+ should "put and get an empty io object" do
242
+ read_and_write_stream('empty_data', 1)
243
+ end
244
+
245
+ should "put and get a small io object" do
246
+ read_and_write_stream('small_data.txt', 1)
247
+ end
248
+
249
+ should "put and get a large io object if reading less than the chunk size" do
250
+ read_and_write_stream('sample_data', 256 * 1024)
251
+ end
252
+
253
+ should "put and get a large io object if reading more than the chunk size" do
254
+ read_and_write_stream('sample_data', 300 * 1024)
255
+ end
256
+ end
257
+ end
258
+ end