mongo-lyon 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE.txt +190 -0
- data/README.md +344 -0
- data/Rakefile +202 -0
- data/bin/mongo_console +34 -0
- data/docs/1.0_UPGRADE.md +21 -0
- data/docs/CREDITS.md +123 -0
- data/docs/FAQ.md +116 -0
- data/docs/GridFS.md +158 -0
- data/docs/HISTORY.md +225 -0
- data/docs/REPLICA_SETS.md +72 -0
- data/docs/TUTORIAL.md +247 -0
- data/docs/WRITE_CONCERN.md +28 -0
- data/lib/mongo.rb +77 -0
- data/lib/mongo/collection.rb +872 -0
- data/lib/mongo/connection.rb +875 -0
- data/lib/mongo/cursor.rb +449 -0
- data/lib/mongo/db.rb +607 -0
- data/lib/mongo/exceptions.rb +68 -0
- data/lib/mongo/gridfs/grid.rb +106 -0
- data/lib/mongo/gridfs/grid_ext.rb +57 -0
- data/lib/mongo/gridfs/grid_file_system.rb +145 -0
- data/lib/mongo/gridfs/grid_io.rb +394 -0
- data/lib/mongo/gridfs/grid_io_fix.rb +38 -0
- data/lib/mongo/repl_set_connection.rb +342 -0
- data/lib/mongo/util/conversions.rb +89 -0
- data/lib/mongo/util/core_ext.rb +60 -0
- data/lib/mongo/util/pool.rb +185 -0
- data/lib/mongo/util/server_version.rb +71 -0
- data/lib/mongo/util/support.rb +82 -0
- data/lib/mongo/util/uri_parser.rb +181 -0
- data/lib/mongo/version.rb +3 -0
- data/mongo.gemspec +34 -0
- data/test/auxillary/1.4_features.rb +166 -0
- data/test/auxillary/authentication_test.rb +68 -0
- data/test/auxillary/autoreconnect_test.rb +41 -0
- data/test/auxillary/repl_set_auth_test.rb +58 -0
- data/test/auxillary/slave_connection_test.rb +36 -0
- data/test/auxillary/threaded_authentication_test.rb +101 -0
- data/test/bson/binary_test.rb +15 -0
- data/test/bson/bson_test.rb +614 -0
- data/test/bson/byte_buffer_test.rb +190 -0
- data/test/bson/hash_with_indifferent_access_test.rb +38 -0
- data/test/bson/json_test.rb +17 -0
- data/test/bson/object_id_test.rb +154 -0
- data/test/bson/ordered_hash_test.rb +197 -0
- data/test/collection_test.rb +893 -0
- data/test/connection_test.rb +303 -0
- data/test/conversions_test.rb +120 -0
- data/test/cursor_fail_test.rb +75 -0
- data/test/cursor_message_test.rb +43 -0
- data/test/cursor_test.rb +457 -0
- data/test/db_api_test.rb +715 -0
- data/test/db_connection_test.rb +15 -0
- data/test/db_test.rb +287 -0
- data/test/grid_file_system_test.rb +244 -0
- data/test/grid_io_test.rb +120 -0
- data/test/grid_test.rb +200 -0
- data/test/load/thin/load.rb +24 -0
- data/test/load/unicorn/load.rb +23 -0
- data/test/replica_sets/connect_test.rb +86 -0
- data/test/replica_sets/connection_string_test.rb +32 -0
- data/test/replica_sets/count_test.rb +35 -0
- data/test/replica_sets/insert_test.rb +53 -0
- data/test/replica_sets/pooled_insert_test.rb +55 -0
- data/test/replica_sets/query_secondaries.rb +96 -0
- data/test/replica_sets/query_test.rb +51 -0
- data/test/replica_sets/replication_ack_test.rb +66 -0
- data/test/replica_sets/rs_test_helper.rb +27 -0
- data/test/safe_test.rb +68 -0
- data/test/support/hash_with_indifferent_access.rb +199 -0
- data/test/support/keys.rb +45 -0
- data/test/support_test.rb +19 -0
- data/test/test_helper.rb +83 -0
- data/test/threading/threading_with_large_pool_test.rb +90 -0
- data/test/threading_test.rb +87 -0
- data/test/tools/auth_repl_set_manager.rb +14 -0
- data/test/tools/repl_set_manager.rb +266 -0
- data/test/unit/collection_test.rb +130 -0
- data/test/unit/connection_test.rb +98 -0
- data/test/unit/cursor_test.rb +99 -0
- data/test/unit/db_test.rb +96 -0
- data/test/unit/grid_test.rb +49 -0
- data/test/unit/pool_test.rb +9 -0
- data/test/unit/repl_set_connection_test.rb +72 -0
- data/test/unit/safe_test.rb +125 -0
- data/test/uri_test.rb +91 -0
- metadata +202 -0
@@ -0,0 +1,68 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
#
|
4
|
+
# --
|
5
|
+
# Copyright (C) 2008-2011 10gen Inc.
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
# ++
|
19
|
+
|
20
|
+
module Mongo
|
21
|
+
# Generic Mongo Ruby Driver exception class.
|
22
|
+
class MongoRubyError < StandardError; end
|
23
|
+
|
24
|
+
# Raised when MongoDB itself has returned an error.
|
25
|
+
class MongoDBError < RuntimeError; end
|
26
|
+
|
27
|
+
# Raised when configuration options cause connections, queries, etc., to fail.
|
28
|
+
class ConfigurationError < MongoRubyError; end
|
29
|
+
|
30
|
+
# Raised on fatal errors to GridFS.
|
31
|
+
class GridError < MongoRubyError; end
|
32
|
+
|
33
|
+
# Raised on fatal errors to GridFS.
|
34
|
+
class GridFileNotFound < GridError; end
|
35
|
+
|
36
|
+
# Raised on fatal errors to GridFS.
|
37
|
+
class GridMD5Failure < GridError; end
|
38
|
+
|
39
|
+
# Raised when invalid arguments are sent to Mongo Ruby methods.
|
40
|
+
class MongoArgumentError < MongoRubyError; end
|
41
|
+
|
42
|
+
# Raised on failures in connection to the database server.
|
43
|
+
class ConnectionError < MongoRubyError; end
|
44
|
+
|
45
|
+
# Raised on failures in connection to the database server.
|
46
|
+
class ReplicaSetConnectionError < ConnectionError; end
|
47
|
+
|
48
|
+
# Raised on failures in connection to the database server.
|
49
|
+
class ConnectionTimeoutError < MongoRubyError; end
|
50
|
+
|
51
|
+
# Raised when a connection operation fails.
|
52
|
+
class ConnectionFailure < MongoDBError; end
|
53
|
+
|
54
|
+
# Raised when authentication fails.
|
55
|
+
class AuthenticationError < MongoDBError; end
|
56
|
+
|
57
|
+
# Raised when a database operation fails.
|
58
|
+
class OperationFailure < MongoDBError; end
|
59
|
+
|
60
|
+
# Raised when a client attempts to perform an invalid operation.
|
61
|
+
class InvalidOperation < MongoDBError; end
|
62
|
+
|
63
|
+
# Raised when an invalid collection or database name is used (invalid namespace name).
|
64
|
+
class InvalidNSName < RuntimeError; end
|
65
|
+
|
66
|
+
# Raised when the client supplies an invalid value to sort by.
|
67
|
+
class InvalidSortValueError < MongoRubyError; end
|
68
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2011 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
module Mongo
|
20
|
+
|
21
|
+
# Implementation of the MongoDB GridFS specification. A file store.
|
22
|
+
class Grid
|
23
|
+
include GridExt::InstanceMethods
|
24
|
+
|
25
|
+
DEFAULT_FS_NAME = 'fs'
|
26
|
+
|
27
|
+
# Initialize a new Grid instance, consisting of a MongoDB database
|
28
|
+
# and a filesystem prefix if not using the default.
|
29
|
+
#
|
30
|
+
# @core gridfs
|
31
|
+
#
|
32
|
+
# @see GridFileSystem
|
33
|
+
def initialize(db, fs_name=DEFAULT_FS_NAME)
|
34
|
+
raise MongoArgumentError, "db must be a Mongo::DB." unless db.is_a?(Mongo::DB)
|
35
|
+
|
36
|
+
@db = db
|
37
|
+
@files = @db["#{fs_name}.files"]
|
38
|
+
@chunks = @db["#{fs_name}.chunks"]
|
39
|
+
@fs_name = fs_name
|
40
|
+
|
41
|
+
# Ensure indexes only if not connected to slave.
|
42
|
+
unless db.connection.slave_ok?
|
43
|
+
@chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]], :unique => true)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Store a file in the file store. This method is designed only for writing new files;
|
48
|
+
# if you need to update a given file, first delete it using Grid#delete.
|
49
|
+
#
|
50
|
+
# Note that arbitary metadata attributes can be saved to the file by passing
|
51
|
+
# them in as options.
|
52
|
+
#
|
53
|
+
# @param [String, #read] data a string or io-like object to store.
|
54
|
+
#
|
55
|
+
# @option opts [String] :filename (nil) a name for the file.
|
56
|
+
# @option opts [Hash] :metadata ({}) any additional data to store with the file.
|
57
|
+
# @option opts [ObjectId] :_id (ObjectId) a unique id for
|
58
|
+
# the file to be use in lieu of an automatically generated one.
|
59
|
+
# @option opts [String] :content_type ('binary/octet-stream') If no content type is specified,
|
60
|
+
# the content type will may be inferred from the filename extension if the mime-types gem can be
|
61
|
+
# loaded. Otherwise, the content type 'binary/octet-stream' will be used.
|
62
|
+
# @option opts [Integer] (262144) :chunk_size size of file chunks in bytes.
|
63
|
+
# @option opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
|
64
|
+
# will be validated using an md5 hash. If validation fails, an exception will be raised.
|
65
|
+
#
|
66
|
+
# @return [Mongo::ObjectId] the file's id.
|
67
|
+
def put(data, opts={})
|
68
|
+
filename = opts.delete :filename
|
69
|
+
opts.merge!(default_grid_io_opts)
|
70
|
+
file = GridIO.new(@files, @chunks, filename, 'w', opts=opts)
|
71
|
+
file.write(data)
|
72
|
+
file.close
|
73
|
+
file.files_id
|
74
|
+
end
|
75
|
+
|
76
|
+
# Read a file from the file store.
|
77
|
+
#
|
78
|
+
# @param [] id the file's unique id.
|
79
|
+
#
|
80
|
+
# @return [Mongo::GridIO]
|
81
|
+
def get(id)
|
82
|
+
opts = {:query => {'_id' => id}}.merge!(default_grid_io_opts)
|
83
|
+
GridIO.new(@files, @chunks, nil, 'r', opts)
|
84
|
+
end
|
85
|
+
|
86
|
+
# Delete a file from the store.
|
87
|
+
#
|
88
|
+
# Note that deleting a GridFS file can result in read errors if another process
|
89
|
+
# is attempting to read a file while it's being deleted. While the odds for this
|
90
|
+
# kind of race condition are small, it's important to be aware of.
|
91
|
+
#
|
92
|
+
# @param [] id
|
93
|
+
#
|
94
|
+
# @return [Boolean]
|
95
|
+
def delete(id)
|
96
|
+
@files.remove({"_id" => id})
|
97
|
+
@chunks.remove({"files_id" => id})
|
98
|
+
end
|
99
|
+
|
100
|
+
private
|
101
|
+
|
102
|
+
def default_grid_io_opts
|
103
|
+
{:fs_name => @fs_name}
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2011 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
module Mongo
|
20
|
+
module GridExt
|
21
|
+
module InstanceMethods
|
22
|
+
|
23
|
+
# Check the existence of a file matching the given query selector.
|
24
|
+
#
|
25
|
+
# Note that this method can be used with both the Grid and GridFileSystem classes. Also
|
26
|
+
# keep in mind that if you're going to be performing lots of existence checks, you should
|
27
|
+
# keep an instance of Grid or GridFileSystem handy rather than instantiating for each existence
|
28
|
+
# check. Alternatively, simply keep a reference to the proper files collection and query that
|
29
|
+
# as needed. That's exactly how this methods works.
|
30
|
+
#
|
31
|
+
# @param [Hash] selector a query selector.
|
32
|
+
#
|
33
|
+
# @example
|
34
|
+
#
|
35
|
+
# # Check for the existence of a given filename
|
36
|
+
# @grid = GridFileSystem.new(@db)
|
37
|
+
# @grid.exist?(:filename => 'foo.txt')
|
38
|
+
#
|
39
|
+
# # Check for existence filename and content type
|
40
|
+
# @grid = GridFileSystem.new(@db)
|
41
|
+
# @grid.exist?(:filename => 'foo.txt', :content_type => 'image/jpg')
|
42
|
+
#
|
43
|
+
# # Check for existence by _id
|
44
|
+
# @grid = Grid.new(@db)
|
45
|
+
# @grid.exist?(:_id => BSON::ObjectId.from_string('4bddcd24beffd95a7db9b8c8'))
|
46
|
+
#
|
47
|
+
# # Check for existence by an arbitrary attribute.
|
48
|
+
# @grid = Grid.new(@db)
|
49
|
+
# @grid.exist?(:tags => {'$in' => ['nature', 'zen', 'photography']})
|
50
|
+
#
|
51
|
+
# @return [nil, Hash] either nil for the file's metadata as a hash.
|
52
|
+
def exist?(selector)
|
53
|
+
@files.find_one(selector)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2011 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
module Mongo
|
20
|
+
|
21
|
+
# A file store built on the GridFS specification featuring
|
22
|
+
# an API and behavior similar to that of a traditional file system.
|
23
|
+
class GridFileSystem
|
24
|
+
include GridExt::InstanceMethods
|
25
|
+
|
26
|
+
# Initialize a new GridFileSystem instance, consisting of a MongoDB database
|
27
|
+
# and a filesystem prefix if not using the default.
|
28
|
+
#
|
29
|
+
# @param [Mongo::DB] db a MongoDB database.
|
30
|
+
# @param [String] fs_name A name for the file system. The default name, based on
|
31
|
+
# the specification, is 'fs'.
|
32
|
+
def initialize(db, fs_name=Grid::DEFAULT_FS_NAME)
|
33
|
+
raise MongoArgumentError, "db must be a Mongo::DB." unless db.is_a?(Mongo::DB)
|
34
|
+
|
35
|
+
@db = db
|
36
|
+
@files = @db["#{fs_name}.files"]
|
37
|
+
@chunks = @db["#{fs_name}.chunks"]
|
38
|
+
@fs_name = fs_name
|
39
|
+
|
40
|
+
@default_query_opts = {:sort => [['filename', 1], ['uploadDate', -1]], :limit => 1}
|
41
|
+
|
42
|
+
# Ensure indexes only if not connected to slave.
|
43
|
+
unless db.connection.slave_ok?
|
44
|
+
@files.create_index([['filename', 1], ['uploadDate', -1]])
|
45
|
+
@chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]], :unique => true)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# Open a file for reading or writing. Note that the options for this method only apply
|
50
|
+
# when opening in 'w' mode.
|
51
|
+
#
|
52
|
+
# Note that arbitary metadata attributes can be saved to the file by passing
|
53
|
+
# them is as options.
|
54
|
+
#
|
55
|
+
# @param [String] filename the name of the file.
|
56
|
+
# @param [String] mode either 'r' or 'w' for reading from
|
57
|
+
# or writing to the file.
|
58
|
+
# @param [Hash] opts see GridIO#new
|
59
|
+
#
|
60
|
+
# @option opts [Hash] :metadata ({}) any additional data to store with the file.
|
61
|
+
# @option opts [ObjectId] :_id (ObjectId) a unique id for
|
62
|
+
# the file to be use in lieu of an automatically generated one.
|
63
|
+
# @option opts [String] :content_type ('binary/octet-stream') If no content type is specified,
|
64
|
+
# the content type will may be inferred from the filename extension if the mime-types gem can be
|
65
|
+
# loaded. Otherwise, the content type 'binary/octet-stream' will be used.
|
66
|
+
# @option opts [Integer] (262144) :chunk_size size of file chunks in bytes.
|
67
|
+
# @option opts [Boolean] :delete_old (false) ensure that old versions of the file are deleted. This option
|
68
|
+
# only work in 'w' mode. Certain precautions must be taken when deleting GridFS files. See the notes under
|
69
|
+
# GridFileSystem#delete.
|
70
|
+
# @option opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
|
71
|
+
# will be validated using an md5 hash. If validation fails, an exception will be raised.
|
72
|
+
#
|
73
|
+
# @example
|
74
|
+
#
|
75
|
+
# # Store the text "Hello, world!" in the grid file system.
|
76
|
+
# @grid = GridFileSystem.new(@db)
|
77
|
+
# @grid.open('filename', 'w') do |f|
|
78
|
+
# f.write "Hello, world!"
|
79
|
+
# end
|
80
|
+
#
|
81
|
+
# # Output "Hello, world!"
|
82
|
+
# @grid = GridFileSystem.new(@db)
|
83
|
+
# @grid.open('filename', 'r') do |f|
|
84
|
+
# puts f.read
|
85
|
+
# end
|
86
|
+
#
|
87
|
+
# # Write a file on disk to the GridFileSystem
|
88
|
+
# @file = File.open('image.jpg')
|
89
|
+
# @grid = GridFileSystem.new(@db)
|
90
|
+
# @grid.open('image.jpg, 'w') do |f|
|
91
|
+
# f.write @file
|
92
|
+
# end
|
93
|
+
#
|
94
|
+
# @return [Mongo::GridIO]
|
95
|
+
def open(filename, mode, opts={})
|
96
|
+
opts.merge!(default_grid_io_opts(filename))
|
97
|
+
del = opts.delete(:delete_old) && mode == 'w'
|
98
|
+
file = GridIO.new(@files, @chunks, filename, mode, opts)
|
99
|
+
return file unless block_given?
|
100
|
+
result = nil
|
101
|
+
begin
|
102
|
+
result = yield file
|
103
|
+
ensure
|
104
|
+
id = file.close
|
105
|
+
if del
|
106
|
+
self.delete do
|
107
|
+
@files.find({'filename' => filename, '_id' => {'$ne' => id}}, :fields => ['_id'])
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
result
|
112
|
+
end
|
113
|
+
|
114
|
+
# Delete the file with the given filename. Note that this will delete
|
115
|
+
# all versions of the file.
|
116
|
+
#
|
117
|
+
# Be careful with this. Deleting a GridFS file can result in read errors if another process
|
118
|
+
# is attempting to read a file while it's being deleted. While the odds for this
|
119
|
+
# kind of race condition are small, it's important to be aware of.
|
120
|
+
#
|
121
|
+
# @param [String] filename
|
122
|
+
#
|
123
|
+
# @yield [] pass a block that returns an array of documents to be deleted.
|
124
|
+
#
|
125
|
+
# @return [Boolean]
|
126
|
+
def delete(filename=nil)
|
127
|
+
if block_given?
|
128
|
+
files = yield
|
129
|
+
else
|
130
|
+
files = @files.find({'filename' => filename}, :fields => ['_id'])
|
131
|
+
end
|
132
|
+
files.each do |file|
|
133
|
+
@files.remove({'_id' => file['_id']})
|
134
|
+
@chunks.remove({'files_id' => file['_id']})
|
135
|
+
end
|
136
|
+
end
|
137
|
+
alias_method :unlink, :delete
|
138
|
+
|
139
|
+
private
|
140
|
+
|
141
|
+
def default_grid_io_opts(filename=nil)
|
142
|
+
{:fs_name => @fs_name, :query => {'filename' => filename}, :query_opts => @default_query_opts}
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,394 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
# --
|
4
|
+
# Copyright (C) 2008-2011 10gen Inc.
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
# ++
|
18
|
+
|
19
|
+
require 'digest/md5'
|
20
|
+
begin
|
21
|
+
require 'mime/types'
|
22
|
+
rescue LoadError
|
23
|
+
end
|
24
|
+
|
25
|
+
module Mongo
|
26
|
+
|
27
|
+
# GridIO objects represent files in the GridFS specification. This class
|
28
|
+
# manages the reading and writing of file chunks and metadata.
|
29
|
+
class GridIO
|
30
|
+
DEFAULT_CHUNK_SIZE = 256 * 1024
|
31
|
+
DEFAULT_CONTENT_TYPE = 'binary/octet-stream'
|
32
|
+
PROTECTED_ATTRS = [:files_id, :file_length, :client_md5, :server_md5]
|
33
|
+
|
34
|
+
attr_reader :content_type, :chunk_size, :upload_date, :files_id, :filename,
|
35
|
+
:metadata, :server_md5, :client_md5, :file_length, :file_position
|
36
|
+
|
37
|
+
# Create a new GridIO object. Note that most users will not need to use this class directly;
|
38
|
+
# the Grid and GridFileSystem classes will instantiate this class
|
39
|
+
#
|
40
|
+
# @param [Mongo::Collection] files a collection for storing file metadata.
|
41
|
+
# @param [Mongo::Collection] chunks a collection for storing file chunks.
|
42
|
+
# @param [String] filename the name of the file to open or write.
|
43
|
+
# @param [String] mode 'r' or 'w' or reading or creating a file.
|
44
|
+
#
|
45
|
+
# @option opts [Hash] :query a query selector used when opening the file in 'r' mode.
|
46
|
+
# @option opts [Hash] :query_opts any query options to be used when opening the file in 'r' mode.
|
47
|
+
# @option opts [String] :fs_name the file system prefix.
|
48
|
+
# @option opts [Integer] (262144) :chunk_size size of file chunks in bytes.
|
49
|
+
# @option opts [Hash] :metadata ({}) any additional data to store with the file.
|
50
|
+
# @option opts [ObjectId] :_id (ObjectId) a unique id for
|
51
|
+
# the file to be use in lieu of an automatically generated one.
|
52
|
+
# @option opts [String] :content_type ('binary/octet-stream') If no content type is specified,
|
53
|
+
# the content type will may be inferred from the filename extension if the mime-types gem can be
|
54
|
+
# loaded. Otherwise, the content type 'binary/octet-stream' will be used.
|
55
|
+
# @option opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
|
56
|
+
# will be validated using an md5 hash. If validation fails, an exception will be raised.
|
57
|
+
def initialize(files, chunks, filename, mode, opts={})
|
58
|
+
@files = files
|
59
|
+
@chunks = chunks
|
60
|
+
@filename = filename
|
61
|
+
@mode = mode
|
62
|
+
@query = opts.delete(:query) || {}
|
63
|
+
@query_opts = opts.delete(:query_opts) || {}
|
64
|
+
@fs_name = opts.delete(:fs_name) || Grid::DEFAULT_FS_NAME
|
65
|
+
@safe = opts.delete(:safe) || false
|
66
|
+
@local_md5 = Digest::MD5.new if @safe
|
67
|
+
@custom_attrs = {}
|
68
|
+
|
69
|
+
case @mode
|
70
|
+
when 'r' then init_read
|
71
|
+
when 'w' then init_write(opts)
|
72
|
+
else
|
73
|
+
raise GridError, "Invalid file mode #{@mode}. Mode should be 'r' or 'w'."
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def [](key)
|
78
|
+
@custom_attrs[key] || instance_variable_get("@#{key.to_s}")
|
79
|
+
end
|
80
|
+
|
81
|
+
def []=(key, value)
|
82
|
+
if PROTECTED_ATTRS.include?(key.to_sym)
|
83
|
+
warn "Attempting to overwrite protected value."
|
84
|
+
return nil
|
85
|
+
else
|
86
|
+
@custom_attrs[key] = value
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# Read the data from the file. If a length if specified, will read from the
|
91
|
+
# current file position.
|
92
|
+
#
|
93
|
+
# @param [Integer] length
|
94
|
+
#
|
95
|
+
# @return [String]
|
96
|
+
# the data in the file
|
97
|
+
def read(length=nil)
|
98
|
+
return '' if @file_length.zero?
|
99
|
+
if length == 0
|
100
|
+
return ''
|
101
|
+
elsif length.nil? && @file_position.zero?
|
102
|
+
read_all
|
103
|
+
else
|
104
|
+
read_length(length)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
alias_method :data, :read
|
108
|
+
|
109
|
+
# Write the given string (binary) data to the file.
|
110
|
+
#
|
111
|
+
# @param [String] string
|
112
|
+
# the data to write
|
113
|
+
#
|
114
|
+
# @return [Integer]
|
115
|
+
# the number of bytes written.
|
116
|
+
def write(io)
|
117
|
+
raise GridError, "file not opened for write" unless @mode[0] == ?w
|
118
|
+
if io.is_a? String
|
119
|
+
if @safe
|
120
|
+
@local_md5.update(io)
|
121
|
+
end
|
122
|
+
write_string(io)
|
123
|
+
else
|
124
|
+
length = 0
|
125
|
+
if @safe
|
126
|
+
while(string = io.read(@chunk_size))
|
127
|
+
@local_md5.update(string)
|
128
|
+
length += write_string(string)
|
129
|
+
end
|
130
|
+
else
|
131
|
+
while(string = io.read(@chunk_size))
|
132
|
+
length += write_string(string)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
length
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
# Position the file pointer at the provided location.
|
140
|
+
#
|
141
|
+
# @param [Integer] pos
|
142
|
+
# the number of bytes to advance the file pointer. this can be a negative
|
143
|
+
# number.
|
144
|
+
# @param [Integer] whence
|
145
|
+
# one of IO::SEEK_CUR, IO::SEEK_END, or IO::SEEK_SET
|
146
|
+
#
|
147
|
+
# @return [Integer] the new file position
|
148
|
+
def seek(pos, whence=IO::SEEK_SET)
|
149
|
+
raise GridError, "Seek is only allowed in read mode." unless @mode == 'r'
|
150
|
+
target_pos = case whence
|
151
|
+
when IO::SEEK_CUR
|
152
|
+
@file_position + pos
|
153
|
+
when IO::SEEK_END
|
154
|
+
@file_length + pos
|
155
|
+
when IO::SEEK_SET
|
156
|
+
pos
|
157
|
+
end
|
158
|
+
|
159
|
+
new_chunk_number = (target_pos / @chunk_size).to_i
|
160
|
+
if new_chunk_number != @current_chunk['n']
|
161
|
+
save_chunk(@current_chunk) if @mode[0] == ?w
|
162
|
+
@current_chunk = get_chunk(new_chunk_number)
|
163
|
+
end
|
164
|
+
@file_position = target_pos
|
165
|
+
@chunk_position = @file_position % @chunk_size
|
166
|
+
@file_position
|
167
|
+
end
|
168
|
+
|
169
|
+
# The current position of the file.
|
170
|
+
#
|
171
|
+
# @return [Integer]
|
172
|
+
def tell
|
173
|
+
@file_position
|
174
|
+
end
|
175
|
+
|
176
|
+
# Creates or updates the document from the files collection that
|
177
|
+
# stores the chunks' metadata. The file becomes available only after
|
178
|
+
# this method has been called.
|
179
|
+
#
|
180
|
+
# This method will be invoked automatically when
|
181
|
+
# on GridIO#open is passed a block. Otherwise, it must be called manually.
|
182
|
+
#
|
183
|
+
# @return [BSON::ObjectId]
|
184
|
+
def close
|
185
|
+
if @mode[0] == ?w
|
186
|
+
if @current_chunk['n'].zero? && @chunk_position.zero?
|
187
|
+
warn "Warning: Storing a file with zero length."
|
188
|
+
end
|
189
|
+
@upload_date = Time.now.utc
|
190
|
+
id = @files.insert(to_mongo_object)
|
191
|
+
end
|
192
|
+
id
|
193
|
+
end
|
194
|
+
|
195
|
+
# Read a chunk of the data from the file and yield it to the given
|
196
|
+
# block.
|
197
|
+
#
|
198
|
+
# Note that this method reads from the current file position.
|
199
|
+
#
|
200
|
+
# @yield Yields on chunk per iteration as defined by this file's
|
201
|
+
# chunk size.
|
202
|
+
#
|
203
|
+
# @return [Mongo::GridIO] self
|
204
|
+
def each
|
205
|
+
return read_all unless block_given?
|
206
|
+
while chunk = read(chunk_size)
|
207
|
+
yield chunk
|
208
|
+
end
|
209
|
+
self
|
210
|
+
end
|
211
|
+
|
212
|
+
def inspect
|
213
|
+
"#<GridIO _id: #{@files_id}>"
|
214
|
+
end
|
215
|
+
|
216
|
+
private
|
217
|
+
|
218
|
+
def create_chunk(n)
|
219
|
+
chunk = BSON::OrderedHash.new
|
220
|
+
chunk['_id'] = BSON::ObjectId.new
|
221
|
+
chunk['n'] = n
|
222
|
+
chunk['files_id'] = @files_id
|
223
|
+
chunk['data'] = ''
|
224
|
+
@chunk_position = 0
|
225
|
+
chunk
|
226
|
+
end
|
227
|
+
|
228
|
+
def save_chunk(chunk)
|
229
|
+
@chunks.insert(chunk)
|
230
|
+
end
|
231
|
+
|
232
|
+
def get_chunk(n)
|
233
|
+
chunk = @chunks.find({'files_id' => @files_id, 'n' => n}).next_document
|
234
|
+
@chunk_position = 0
|
235
|
+
chunk
|
236
|
+
end
|
237
|
+
|
238
|
+
def last_chunk_number
|
239
|
+
(@file_length / @chunk_size).to_i
|
240
|
+
end
|
241
|
+
|
242
|
+
# Read a file in its entirety.
|
243
|
+
def read_all
|
244
|
+
buf = ''
|
245
|
+
if @current_chunk
|
246
|
+
buf << @current_chunk['data'].to_s
|
247
|
+
while chunk = get_chunk(@current_chunk['n'] + 1)
|
248
|
+
buf << chunk['data'].to_s
|
249
|
+
@current_chunk = chunk
|
250
|
+
end
|
251
|
+
end
|
252
|
+
@file_position = @file_length
|
253
|
+
buf
|
254
|
+
end
|
255
|
+
|
256
|
+
# Read a file incrementally.
|
257
|
+
def read_length(length)
|
258
|
+
cache_chunk_data
|
259
|
+
remaining = (@file_length - @file_position)
|
260
|
+
if length.nil?
|
261
|
+
to_read = remaining
|
262
|
+
else
|
263
|
+
to_read = length > remaining ? remaining : length
|
264
|
+
end
|
265
|
+
return nil unless remaining > 0
|
266
|
+
|
267
|
+
buf = ''
|
268
|
+
while to_read > 0
|
269
|
+
if @chunk_position == @chunk_data_length
|
270
|
+
@current_chunk = get_chunk(@current_chunk['n'] + 1)
|
271
|
+
cache_chunk_data
|
272
|
+
end
|
273
|
+
chunk_remainder = @chunk_data_length - @chunk_position
|
274
|
+
size = (to_read >= chunk_remainder) ? chunk_remainder : to_read
|
275
|
+
buf << @current_chunk_data[@chunk_position, size]
|
276
|
+
to_read -= size
|
277
|
+
@chunk_position += size
|
278
|
+
@file_position += size
|
279
|
+
end
|
280
|
+
buf
|
281
|
+
end
|
282
|
+
|
283
|
+
def cache_chunk_data
|
284
|
+
@current_chunk_data = @current_chunk['data'].to_s
|
285
|
+
if @current_chunk_data.respond_to?(:force_encoding)
|
286
|
+
@current_chunk_data.force_encoding("binary")
|
287
|
+
end
|
288
|
+
@chunk_data_length = @current_chunk['data'].length
|
289
|
+
end
|
290
|
+
|
291
|
+
def write_string(string)
|
292
|
+
# Since Ruby 1.9.1 doesn't necessarily store one character per byte.
|
293
|
+
if string.respond_to?(:force_encoding)
|
294
|
+
string.force_encoding("binary")
|
295
|
+
end
|
296
|
+
|
297
|
+
to_write = string.length
|
298
|
+
while (to_write > 0) do
|
299
|
+
if @current_chunk && @chunk_position == @chunk_size
|
300
|
+
next_chunk_number = @current_chunk['n'] + 1
|
301
|
+
@current_chunk = create_chunk(next_chunk_number)
|
302
|
+
end
|
303
|
+
chunk_available = @chunk_size - @chunk_position
|
304
|
+
step_size = (to_write > chunk_available) ? chunk_available : to_write
|
305
|
+
@current_chunk['data'] = BSON::Binary.new((@current_chunk['data'].to_s << string[-to_write, step_size]).unpack("c*"))
|
306
|
+
@chunk_position += step_size
|
307
|
+
to_write -= step_size
|
308
|
+
save_chunk(@current_chunk)
|
309
|
+
end
|
310
|
+
string.length - to_write
|
311
|
+
end
|
312
|
+
|
313
|
+
# Initialize the class for reading a file.
|
314
|
+
def init_read
|
315
|
+
doc = @files.find(@query, @query_opts).next_document
|
316
|
+
raise GridFileNotFound, "Could not open file matching #{@query.inspect} #{@query_opts.inspect}" unless doc
|
317
|
+
|
318
|
+
@files_id = doc['_id']
|
319
|
+
@content_type = doc['contentType']
|
320
|
+
@chunk_size = doc['chunkSize']
|
321
|
+
@upload_date = doc['uploadDate']
|
322
|
+
@aliases = doc['aliases']
|
323
|
+
@file_length = doc['length']
|
324
|
+
@metadata = doc['metadata']
|
325
|
+
@md5 = doc['md5']
|
326
|
+
@filename = doc['filename']
|
327
|
+
@custom_attrs = doc
|
328
|
+
|
329
|
+
@current_chunk = get_chunk(0)
|
330
|
+
@file_position = 0
|
331
|
+
end
|
332
|
+
|
333
|
+
# Initialize the class for writing a file.
|
334
|
+
def init_write(opts)
|
335
|
+
@files_id = opts.delete(:_id) || BSON::ObjectId.new
|
336
|
+
@content_type = opts.delete(:content_type) || (defined? MIME) && get_content_type || DEFAULT_CONTENT_TYPE
|
337
|
+
@chunk_size = opts.delete(:chunk_size) || DEFAULT_CHUNK_SIZE
|
338
|
+
@metadata = opts.delete(:metadata) if opts[:metadata]
|
339
|
+
@aliases = opts.delete(:aliases) if opts[:aliases]
|
340
|
+
@file_length = 0
|
341
|
+
opts.each {|k, v| self[k] = v}
|
342
|
+
check_existing_file if @safe
|
343
|
+
|
344
|
+
@current_chunk = create_chunk(0)
|
345
|
+
@file_position = 0
|
346
|
+
end
|
347
|
+
|
348
|
+
def check_existing_file
|
349
|
+
if @files.find_one('_id' => @files_id)
|
350
|
+
raise GridError, "Attempting to overwrite with Grid#put. You must delete the file first."
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
def to_mongo_object
|
355
|
+
h = BSON::OrderedHash.new
|
356
|
+
h['_id'] = @files_id
|
357
|
+
h['filename'] = @filename if @filename
|
358
|
+
h['contentType'] = @content_type
|
359
|
+
h['length'] = @current_chunk ? @current_chunk['n'] * @chunk_size + @chunk_position : 0
|
360
|
+
h['chunkSize'] = @chunk_size
|
361
|
+
h['uploadDate'] = @upload_date
|
362
|
+
h['aliases'] = @aliases if @aliases
|
363
|
+
h['metadata'] = @metadata if @metadata
|
364
|
+
h['md5'] = get_md5
|
365
|
+
h.merge!(@custom_attrs)
|
366
|
+
h
|
367
|
+
end
|
368
|
+
|
369
|
+
# Get a server-side md5 and validate against the client if running in safe mode.
|
370
|
+
def get_md5
|
371
|
+
md5_command = BSON::OrderedHash.new
|
372
|
+
md5_command['filemd5'] = @files_id
|
373
|
+
md5_command['root'] = @fs_name
|
374
|
+
@server_md5 = @files.db.command(md5_command)['md5']
|
375
|
+
if @safe
|
376
|
+
@client_md5 = @local_md5.hexdigest
|
377
|
+
if @local_md5 != @server_md5
|
378
|
+
raise GridMD5Failure, "File on server failed MD5 check"
|
379
|
+
end
|
380
|
+
else
|
381
|
+
@server_md5
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
# Determine the content type based on the filename.
|
386
|
+
def get_content_type
|
387
|
+
if @filename
|
388
|
+
if types = MIME::Types.type_for(@filename)
|
389
|
+
types.first.simplified unless types.empty?
|
390
|
+
end
|
391
|
+
end
|
392
|
+
end
|
393
|
+
end
|
394
|
+
end
|