kbaum-mongo 0.18.3p

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. data/LICENSE.txt +202 -0
  2. data/README.rdoc +339 -0
  3. data/Rakefile +138 -0
  4. data/bin/bson_benchmark.rb +59 -0
  5. data/bin/fail_if_no_c.rb +11 -0
  6. data/examples/admin.rb +42 -0
  7. data/examples/capped.rb +22 -0
  8. data/examples/cursor.rb +48 -0
  9. data/examples/gridfs.rb +88 -0
  10. data/examples/index_test.rb +126 -0
  11. data/examples/info.rb +31 -0
  12. data/examples/queries.rb +70 -0
  13. data/examples/simple.rb +24 -0
  14. data/examples/strict.rb +35 -0
  15. data/examples/types.rb +36 -0
  16. data/lib/mongo/collection.rb +609 -0
  17. data/lib/mongo/connection.rb +672 -0
  18. data/lib/mongo/cursor.rb +403 -0
  19. data/lib/mongo/db.rb +555 -0
  20. data/lib/mongo/exceptions.rb +66 -0
  21. data/lib/mongo/gridfs/chunk.rb +91 -0
  22. data/lib/mongo/gridfs/grid.rb +79 -0
  23. data/lib/mongo/gridfs/grid_file_system.rb +101 -0
  24. data/lib/mongo/gridfs/grid_io.rb +338 -0
  25. data/lib/mongo/gridfs/grid_store.rb +580 -0
  26. data/lib/mongo/gridfs.rb +25 -0
  27. data/lib/mongo/types/binary.rb +52 -0
  28. data/lib/mongo/types/code.rb +36 -0
  29. data/lib/mongo/types/dbref.rb +40 -0
  30. data/lib/mongo/types/min_max_keys.rb +58 -0
  31. data/lib/mongo/types/objectid.rb +180 -0
  32. data/lib/mongo/types/regexp_of_holding.rb +45 -0
  33. data/lib/mongo/util/bson_c.rb +18 -0
  34. data/lib/mongo/util/bson_ruby.rb +606 -0
  35. data/lib/mongo/util/byte_buffer.rb +222 -0
  36. data/lib/mongo/util/conversions.rb +87 -0
  37. data/lib/mongo/util/ordered_hash.rb +140 -0
  38. data/lib/mongo/util/server_version.rb +69 -0
  39. data/lib/mongo/util/support.rb +26 -0
  40. data/lib/mongo.rb +63 -0
  41. data/mongo-ruby-driver.gemspec +28 -0
  42. data/test/auxillary/autoreconnect_test.rb +42 -0
  43. data/test/binary_test.rb +15 -0
  44. data/test/bson_test.rb +427 -0
  45. data/test/byte_buffer_test.rb +81 -0
  46. data/test/chunk_test.rb +82 -0
  47. data/test/collection_test.rb +515 -0
  48. data/test/connection_test.rb +160 -0
  49. data/test/conversions_test.rb +120 -0
  50. data/test/cursor_test.rb +379 -0
  51. data/test/db_api_test.rb +780 -0
  52. data/test/db_connection_test.rb +16 -0
  53. data/test/db_test.rb +272 -0
  54. data/test/grid_file_system_test.rb +210 -0
  55. data/test/grid_io_test.rb +78 -0
  56. data/test/grid_store_test.rb +334 -0
  57. data/test/grid_test.rb +87 -0
  58. data/test/objectid_test.rb +125 -0
  59. data/test/ordered_hash_test.rb +172 -0
  60. data/test/replica/count_test.rb +34 -0
  61. data/test/replica/insert_test.rb +50 -0
  62. data/test/replica/pooled_insert_test.rb +54 -0
  63. data/test/replica/query_test.rb +39 -0
  64. data/test/slave_connection_test.rb +36 -0
  65. data/test/test_helper.rb +42 -0
  66. data/test/threading/test_threading_large_pool.rb +90 -0
  67. data/test/threading_test.rb +87 -0
  68. data/test/unit/collection_test.rb +61 -0
  69. data/test/unit/connection_test.rb +117 -0
  70. data/test/unit/cursor_test.rb +93 -0
  71. data/test/unit/db_test.rb +98 -0
  72. metadata +127 -0
@@ -0,0 +1,79 @@
1
+ # --
2
+ # Copyright (C) 2008-2010 10gen Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ # ++
16
+
17
+ module Mongo
18
+
19
+ # Implementation of the MongoDB GridFS specification. A file store.
20
+ class Grid
21
+ DEFAULT_FS_NAME = 'fs'
22
+
23
+ # Initialize a new Grid instance, consisting of a MongoDB database
24
+ # and a filesystem prefix if not using the default.
25
+ #
26
+ # @core gridfs
27
+ # @see GridFileSystem
28
+ def initialize(db, fs_name=DEFAULT_FS_NAME)
29
+ raise MongoArgumentError, "db must be a Mongo::DB." unless db.is_a?(Mongo::DB)
30
+
31
+ @db = db
32
+ @files = @db["#{fs_name}.files"]
33
+ @chunks = @db["#{fs_name}.chunks"]
34
+ @fs_name = fs_name
35
+
36
+ @chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]])
37
+ end
38
+
39
+ # Store a file in the file store.
40
+ #
41
+ # @param [String, #read] data a string or io-like object to store.
42
+ # @param [String] filename a name for the file.
43
+ #
44
+ # @return [Mongo::ObjectID] the file's id.
45
+ def put(data, filename, opts={})
46
+ opts.merge!(default_grid_io_opts)
47
+ file = GridIO.new(@files, @chunks, filename, 'w', opts=opts)
48
+ file.write(data)
49
+ file.close
50
+ file.files_id
51
+ end
52
+
53
+ # Read a file from the file store.
54
+ #
55
+ # @param [] id the file's unique id.
56
+ #
57
+ # @return [Mongo::GridIO]
58
+ def get(id)
59
+ opts = {:query => {'_id' => id}}.merge!(default_grid_io_opts)
60
+ GridIO.new(@files, @chunks, nil, 'r', opts)
61
+ end
62
+
63
+ # Delete a file from the store.
64
+ #
65
+ # @param [] id
66
+ #
67
+ # @return [Boolean]
68
+ def delete(id)
69
+ @files.remove({"_id" => id})
70
+ @chunks.remove({"_id" => id})
71
+ end
72
+
73
+ private
74
+
75
+ def default_grid_io_opts
76
+ {:fs_name => @fs_name}
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,101 @@
1
+ # --
2
+ # Copyright (C) 2008-2010 10gen Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ # ++
16
+
17
+ module Mongo
18
+
19
+ # A file store built on the GridFS specification featuring
20
+ # an API and behavior similar to that of a traditional file system.
21
+ class GridFileSystem
22
+
23
+ # Initialize a new Grid instance, consisting of a MongoDB database
24
+ # and a filesystem prefix if not using the default.
25
+ #
26
+ # @param [Mongo::DB] db a MongoDB database.
27
+ # @param [String] fs_name A name for the file system. The default name, based on
28
+ # the specification, is 'fs'.
29
+ def initialize(db, fs_name=Grid::DEFAULT_FS_NAME)
30
+ raise MongoArgumentError, "db must be a Mongo::DB." unless db.is_a?(Mongo::DB)
31
+
32
+ @db = db
33
+ @files = @db["#{fs_name}.files"]
34
+ @chunks = @db["#{fs_name}.chunks"]
35
+ @fs_name = fs_name
36
+
37
+ @files.create_index([['filename', 1], ['uploadDate', -1]])
38
+ @default_query_opts = {:sort => [['filename', 1], ['uploadDate', -1]], :limit => 1}
39
+ end
40
+
41
+ # Open a file for reading or writing.
42
+ #
43
+ # @param [String] filename the name of the file.
44
+ # @param [String] mode either 'r' or 'w' for reading from
45
+ # or writing to the file.
46
+ #
47
+ # @example
48
+ #
49
+ # # Store the text "Hello, world!" in the grid file system.
50
+ # @grid = GridFileSystem.new(@db)
51
+ # @grid.open('filename', 'w') do |f|
52
+ # f.write "Hello, world!"
53
+ # end
54
+ #
55
+ # # Output "Hello, world!"
56
+ # @grid = GridFileSystem.new(@db)
57
+ # @grid.open('filename', 'r') do |f|
58
+ # puts f.read
59
+ # end
60
+ #
61
+ # # Write a file on disk to the GridFileSystem
62
+ # @file = File.open('image.jpg')
63
+ # @grid = GridFileSystem.new(@db)
64
+ # @grid.open('image.jpg, 'w') do |f|
65
+ # f.write @file
66
+ # end
67
+ def open(filename, mode, opts={})
68
+ opts.merge!(default_grid_io_opts(filename))
69
+ file = GridIO.new(@files, @chunks, filename, mode, opts)
70
+ return file unless block_given?
71
+ result = nil
72
+ begin
73
+ result = yield file
74
+ ensure
75
+ file.close
76
+ end
77
+ result
78
+ end
79
+
80
+ # Delete the file with the given filename. Note that this will delete
81
+ # all versions of the file.
82
+ #
83
+ # @param [String] filename
84
+ #
85
+ # @return [Boolean]
86
+ def delete(filename)
87
+ files = @files.find({'filename' => filename}, :fields => ['_id'])
88
+ files.each do |file|
89
+ @files.remove({'_id' => file['_id']})
90
+ @chunks.remove({'files_id' => file['_id']})
91
+ end
92
+ end
93
+ alias_method :unlink, :delete
94
+
95
+ private
96
+
97
+ def default_grid_io_opts(filename=nil)
98
+ {:fs_name => @fs_name, :query => {'filename' => filename}, :query_opts => @default_query_opts}
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,338 @@
1
+ # --
2
+ # Copyright (C) 2008-2010 10gen Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ # ++
16
+
17
+ require 'digest/md5'
18
+ begin
19
+ require 'mime/types'
20
+ rescue LoadError
21
+ end
22
+
23
+ module Mongo
24
+
25
+ # GridIO objects represent files in the GridFS specification. This class
26
+ # manages the reading and writing of file chunks and metadata.
27
+ class GridIO
28
+ DEFAULT_CHUNK_SIZE = 256 * 1024
29
+ DEFAULT_CONTENT_TYPE = 'binary/octet-stream'
30
+
31
+ attr_reader :content_type, :chunk_size, :upload_date, :files_id, :filename,
32
+ :metadata, :server_md5, :client_md5
33
+
34
+ # Create a new GridIO object. Note that most users will not need to use this class directly;
35
+ # the Grid and GridFileSystem classes will instantiate this class
36
+ #
37
+ # @param [Mongo::Collection] files a collection for storing file metadata.
38
+ # @param [Mongo::Collection] chunks a collection for storing file chunks.
39
+ # @param [String] filename the name of the file to open or write.
40
+ # @param [String] mode 'r' or 'w' or reading or creating a file.
41
+ #
42
+ # @option opts [Hash] :query a query selector used when opening the file in 'r' mode.
43
+ # @option opts [Hash] :query_opts any query options to be used when opening the file in 'r' mode.
44
+ # @option opts [String] :fs_name the file system prefix.
45
+ # @options opts [Integer] (262144) :chunk_size size of file chunks in bytes.
46
+ # @options opts [Hash] :metadata ({}) any additional data to store with the file.
47
+ # @options opts [ObjectID] :_id (ObjectID) a unique id for
48
+ # the file to be use in lieu of an automatically generated one.
49
+ # @options opts [String] :content_type ('binary/octet-stream') If no content type is specified,
50
+ # the content type will may be inferred from the filename extension if the mime-types gem can be
51
+ # loaded. Otherwise, the content type 'binary/octet-stream' will be used.
52
+ # @options opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
53
+ # will be validated using an md5 hash. If validation fails, an exception will be raised.
54
+ def initialize(files, chunks, filename, mode, opts={})
55
+ @files = files
56
+ @chunks = chunks
57
+ @filename = filename
58
+ @mode = mode
59
+ @query = opts[:query] || {}
60
+ @query_opts = opts[:query_opts] || {}
61
+ @fs_name = opts[:fs_name] || Grid::DEFAULT_FS_NAME
62
+ @safe = opts[:safe] || false
63
+ @local_md5 = Digest::MD5.new if @safe
64
+
65
+ case @mode
66
+ when 'r' then init_read
67
+ when 'w' then init_write(opts)
68
+ else
69
+ raise GridError, "Invalid file mode #{@mode}. Mode should be 'r' or 'w'."
70
+ end
71
+ end
72
+
73
+ # Read the data from the file. If a length if specified, will read from the
74
+ # current file position.
75
+ #
76
+ # @param [Integer] length
77
+ #
78
+ # @return [String]
79
+ # the data in the file
80
+ def read(length=nil)
81
+ if length == 0
82
+ return ''
83
+ elsif length.nil? && @file_position.zero?
84
+ read_all
85
+ else
86
+ read_length(length)
87
+ end
88
+ end
89
+ alias_method :data, :read
90
+
91
+ # Write the given string (binary) data to the file.
92
+ #
93
+ # @param [String] string
94
+ # the data to write
95
+ #
96
+ # @return [Integer]
97
+ # the number of bytes written.
98
+ def write(io)
99
+ raise GridError, "file not opened for write" unless @mode[0] == ?w
100
+ if io.is_a? String
101
+ if @safe
102
+ @local_md5.update(io)
103
+ end
104
+ write_string(io)
105
+ else
106
+ length = 0
107
+ if @safe
108
+ while(string = io.read(@chunk_size))
109
+ @local_md5.update(string)
110
+ length += write_string(string)
111
+ end
112
+ else
113
+ while(string = io.read(@chunk_size))
114
+ length += write_string(string)
115
+ end
116
+ end
117
+ length
118
+ end
119
+ end
120
+
121
+ # Position the file pointer at the provided location.
122
+ #
123
+ # @param [Integer] pos
124
+ # the number of bytes to advance the file pointer. this can be a negative
125
+ # number.
126
+ # @param [Integer] whence
127
+ # one of IO::SEEK_CUR, IO::SEEK_END, or IO::SEEK_SET
128
+ #
129
+ # @return [Integer] the new file position
130
+ def seek(pos, whence=IO::SEEK_SET)
131
+ raise GridError, "Seek is only allowed in read mode." unless @mode == 'r'
132
+ target_pos = case whence
133
+ when IO::SEEK_CUR
134
+ @file_position + pos
135
+ when IO::SEEK_END
136
+ @file_length + pos
137
+ when IO::SEEK_SET
138
+ pos
139
+ end
140
+
141
+ new_chunk_number = (target_pos / @chunk_size).to_i
142
+ if new_chunk_number != @current_chunk['n']
143
+ save_chunk(@current_chunk) if @mode[0] == ?w
144
+ @current_chunk = get_chunk(new_chunk_number)
145
+ end
146
+ @file_position = target_pos
147
+ @chunk_position = @file_position % @chunk_size
148
+ @file_position
149
+ end
150
+
151
+ # The current position of the file.
152
+ #
153
+ # @return [Integer]
154
+ def tell
155
+ @file_position
156
+ end
157
+
158
+ # Creates or updates the document from the files collection that
159
+ # stores the chunks' metadata. The file becomes available only after
160
+ # this method has been called.
161
+ #
162
+ # This method will be invoked automatically when
163
+ # on GridIO#open is passed a block. Otherwise, it must be called manually.
164
+ #
165
+ # @return [True]
166
+ def close
167
+ if @mode[0] == ?w
168
+ @upload_date = Time.now.utc
169
+ @files.insert(to_mongo_object)
170
+ end
171
+ true
172
+ end
173
+
174
+ def inspect
175
+ "#<GridIO _id: #{@files_id}>"
176
+ end
177
+
178
+ private
179
+
180
+ def create_chunk(n)
181
+ chunk = OrderedHash.new
182
+ chunk['_id'] = Mongo::ObjectID.new
183
+ chunk['n'] = n
184
+ chunk['files_id'] = @files_id
185
+ chunk['data'] = ''
186
+ @chunk_position = 0
187
+ chunk
188
+ end
189
+
190
+ def save_chunk(chunk)
191
+ @chunks.insert(chunk)
192
+ end
193
+
194
+ def get_chunk(n)
195
+ chunk = @chunks.find({'files_id' => @files_id, 'n' => n}).next_document
196
+ @chunk_position = 0
197
+ chunk
198
+ end
199
+
200
+ def last_chunk_number
201
+ (@file_length / @chunk_size).to_i
202
+ end
203
+
204
+ # Read a file in its entirety.
205
+ def read_all
206
+ buf = ''
207
+ while true
208
+ buf << @current_chunk['data'].to_s
209
+ @current_chunk = get_chunk(@current_chunk['n'] + 1)
210
+ break unless @current_chunk
211
+ end
212
+ buf
213
+ end
214
+
215
+ # Read a file incrementally.
216
+ def read_length(length)
217
+ cache_chunk_data
218
+ remaining = (@file_length - @file_position)
219
+ to_read = length > remaining ? remaining : length
220
+ return nil unless remaining > 0
221
+
222
+ buf = ''
223
+ while to_read > 0
224
+ if @chunk_position == @chunk_data_length
225
+ @current_chunk = get_chunk(@current_chunk['n'] + 1)
226
+ cache_chunk_data
227
+ end
228
+ chunk_remainder = @chunk_data_length - @chunk_position
229
+ size = (to_read >= chunk_remainder) ? chunk_remainder : to_read
230
+ buf << @current_chunk_data[@chunk_position, size]
231
+ to_read -= size
232
+ @chunk_position += size
233
+ @file_position += size
234
+ end
235
+ buf
236
+ end
237
+
238
+ def cache_chunk_data
239
+ @current_chunk_data = @current_chunk['data'].to_s
240
+ if @current_chunk_data.respond_to?(:force_encoding)
241
+ @current_chunk_data.force_encoding("binary")
242
+ end
243
+ @chunk_data_length = @current_chunk['data'].length
244
+ end
245
+
246
+ def write_string(string)
247
+ # Since Ruby 1.9.1 doesn't necessarily store one character per byte.
248
+ if string.respond_to?(:force_encoding)
249
+ string.force_encoding("binary")
250
+ end
251
+
252
+ to_write = string.length
253
+ while (to_write > 0) do
254
+ if @current_chunk && @chunk_position == @chunk_size
255
+ next_chunk_number = @current_chunk['n'] + 1
256
+ @current_chunk = create_chunk(next_chunk_number)
257
+ end
258
+ chunk_available = @chunk_size - @chunk_position
259
+ step_size = (to_write > chunk_available) ? chunk_available : to_write
260
+ @current_chunk['data'] = Binary.new((@current_chunk['data'].to_s << string[-to_write, step_size]).unpack("c*"))
261
+ @chunk_position += step_size
262
+ to_write -= step_size
263
+ save_chunk(@current_chunk)
264
+ end
265
+ string.length - to_write
266
+ end
267
+
268
+ # Initialize the class for reading a file.
269
+ def init_read
270
+ doc = @files.find(@query, @query_opts).next_document
271
+ raise GridError, "Could not open file matching #{@query.inspect} #{@query_opts.inspect}" unless doc
272
+
273
+ @files_id = doc['_id']
274
+ @content_type = doc['contentType']
275
+ @chunk_size = doc['chunkSize']
276
+ @upload_date = doc['uploadDate']
277
+ @aliases = doc['aliases']
278
+ @file_length = doc['length']
279
+ @metadata = doc['metadata']
280
+ @md5 = doc['md5']
281
+ @filename = doc['filename']
282
+
283
+ @current_chunk = get_chunk(0)
284
+ @file_position = 0
285
+ end
286
+
287
+ # Initialize the class for writing a file.
288
+ def init_write(opts)
289
+ @files_id = opts[:_id] || Mongo::ObjectID.new
290
+ @content_type = opts[:content_type] || (defined? MIME) && get_content_type || DEFAULT_CONTENT_TYPE
291
+ @chunk_size = opts[:chunk_size] || DEFAULT_CHUNK_SIZE
292
+ @file_length = 0
293
+ @metadata = opts[:metadata] if opts[:metadata]
294
+
295
+ @current_chunk = create_chunk(0)
296
+ @file_position = 0
297
+ end
298
+
299
+ def to_mongo_object
300
+ h = OrderedHash.new
301
+ h['_id'] = @files_id
302
+ h['filename'] = @filename
303
+ h['contentType'] = @content_type
304
+ h['length'] = @current_chunk ? @current_chunk['n'] * @chunk_size + @chunk_position : 0
305
+ h['chunkSize'] = @chunk_size
306
+ h['uploadDate'] = @upload_date
307
+ h['aliases'] = @aliases
308
+ h['metadata'] = @metadata
309
+ h['md5'] = get_md5
310
+ h
311
+ end
312
+
313
+ # Get a server-side md5 and validate against the client if running in safe mode.
314
+ def get_md5
315
+ md5_command = OrderedHash.new
316
+ md5_command['filemd5'] = @files_id
317
+ md5_command['root'] = @fs_name
318
+ @server_md5 = @files.db.command(md5_command)['md5']
319
+ if @safe
320
+ @client_md5 = @local_md5.hexdigest
321
+ if @local_md5 != @server_md5
322
+ raise @local_md5 != @server_md5, "File on server failed MD5 check"
323
+ end
324
+ else
325
+ @server_md5
326
+ end
327
+ end
328
+
329
+ # Determine the content type based on the filename.
330
+ def get_content_type
331
+ if @filename
332
+ if types = MIME::Types.type_for(@filename)
333
+ types.first.simplified unless types.empty?
334
+ end
335
+ end
336
+ end
337
+ end
338
+ end