shrine-gridfs 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b919da6ec801c8888bf1fbeb877cbdc8ad251d43
-  data.tar.gz: a3cbea2effb35eaee531176df0bcd533749bae62
+  metadata.gz: 32f189d0c100716ffe7658f1bf04e84f6ed05415
+  data.tar.gz: 2ede8fa5db17c83fff263cd33e7d1434cad9da96
 SHA512:
-  metadata.gz: cf9beeb12753001aaf4366ecc0507ea450accd740994b9d28d9c31f4e94dba1c02f71668ebeb946692e5b29ad76feedb38b72ad53443c687fc5c1d87d24f569a
-  data.tar.gz: f668ac2435e9e46dc46b2e28eaa2fe37aa30cf80a716331901096f0175e71455e3b4773f6a74369c176a88c079ba3969f3e7c575bdde00bc8a617cf2989d7933
+  metadata.gz: e0cd1b8dfbfaaf44644777f45b087b49485dd19528c4a2b1eff61a0fb1e932b891f7e4be4150cc134deea9940045fb4784ed1ca08e13c3ebb437bf28b0378384
+  data.tar.gz: b8394324dea43b926e78ae961cfaecb949d98983c5afde5a0c0752c8074fd5b1ea3c70be7a6058c44f035cd3cef801af8a0e14aa003a750ab8af75bf17acbb20

data/README.md

@@ -40,6 +40,15 @@ change that via `:chunk_size`:
 Shrine::Storage::Gridfs.new(client: client, chunk_size: 1*1024*1024) # 1MB
 ```
 
+### Batch size
+
+When Gridfs storage uploads or copies a file, it will ensure that no more
+than `:batch_size` of data is loaded into memory at once (defaults to 5MB).
+
+```rb
+Shrine::Storage::Gridfs.new(client: client, batch_size: 10*1024*1024) # 10MB
+```
+
 ### URLs
 
 You can generate URLs through which the GridFS files will be streamed with the

data/lib/shrine/storage/gridfs.rb

@@ -8,53 +8,26 @@ class Shrine
     class Gridfs
       attr_reader :client, :prefix, :bucket, :chunk_size
 
-      BATCH_SIZE = 5 * 1024 * 1024
-
-      def initialize(client:, prefix: "fs", chunk_size: 256*1024, **options)
+      def initialize(client:, prefix: "fs", chunk_size: 256*1024, batch_size: 5 * 1024*1024, **options)
         @client     = client
         @prefix     = prefix
         @chunk_size = chunk_size
+        @batch_size = batch_size
         @bucket     = @client.database.fs(bucket_name: @prefix)
 
         @bucket.send(:ensure_indexes!)
       end
 
       def upload(io, id, shrine_metadata: {}, **)
-        file = create_file(id, shrine_metadata: shrine_metadata)
-
-        until io.eof?
-          chunk = io.read([BATCH_SIZE, chunk_size].max, buffer ||= "")
-          grid_chunks = Mongo::Grid::File::Chunk.split(chunk, file.info, offset ||= 0)
-
-          chunks_collection.insert_many(grid_chunks)
-
-          offset += grid_chunks.count
-          grid_chunks.each { |grid_chunk| grid_chunk.data.data.clear } # deallocate strings
-          chunk.clear # deallocate string
+        if copyable?(io, id)
+          copy(io, id, shrine_metadata: shrine_metadata)
+        else
+          create(io, id, shrine_metadata: shrine_metadata)
         end
-
-        files_collection.find(_id: file.id).update_one(
-          "$set" => {
-            length:     io.size,
-            uploadDate: Time.now.utc,
-            md5:        file.info.md5.hexdigest,
-          }
-        )
-      end
-
-      def move(io, id, shrine_metadata: {}, **)
-        file = create_file(id, shrine_metadata: shrine_metadata)
-
-        chunks_collection.find(files_id: bson_id(io.id)).update_many("$set" => {files_id: file.id})
-        files_collection.delete_one(_id: bson_id(io.id))
-      end
-
-      def movable?(io, id)
-        io.is_a?(UploadedFile) && io.storage.is_a?(Storage::Gridfs)
       end
 
       def open(id)
-        content_length = bucket.find(_id: bson_id(id)).limit(1).first[:length]
+        content_length = file_info(id)[:length]
         stream = bucket.open_download_stream(bson_id(id))
 
         Down::ChunkedIO.new(
@@ -65,7 +38,7 @@ class Shrine
       end
 
       def exists?(id)
-        !!bucket.find(_id: bson_id(id)).first
+        !!file_info(id)
       end
 
       def delete(id)
@@ -87,29 +60,90 @@ class Shrine
         chunks_collection.find.delete_many
       end
 
+      protected
+
+      def file_info(id)
+        bucket.find(_id: bson_id(id)).limit(1).first
+      end
+
+      def files_collection
+        bucket.files_collection
+      end
+
+      def chunks_collection
+        bucket.chunks_collection
+      end
+
       private
 
+      def create(io, id, shrine_metadata: {})
+        file = create_file(id, shrine_metadata: shrine_metadata)
+
+        until io.eof?
+          chunk = io.read([@batch_size, chunk_size].max, buffer ||= "")
+          grid_chunks = Mongo::Grid::File::Chunk.split(chunk, file.info, offset ||= 0)
+
+          chunks_collection.insert_many(grid_chunks)
+
+          offset += grid_chunks.count
+          grid_chunks.each { |grid_chunk| grid_chunk.data.data.clear } # deallocate strings
+          chunk.clear # deallocate string
+        end
+
+        files_collection.find(_id: file.id).update_one("$set" => {
+          length:     io.size,
+          uploadDate: Time.now.utc,
+          md5:        file.info.md5.hexdigest,
+        })
+      end
+
+      def copy(io, id, shrine_metadata: {})
+        source_storage = io.storage
+        source_info    = source_storage.file_info(io.id)
+        dest_info      = source_info.merge(_id: BSON::ObjectId.new)
+
+        batch_size = (@batch_size.to_f / chunk_size).ceil
+        chunk_batches = source_storage.chunks_collection
+          .find(files_id: source_info[:_id])
+          .batch_size(batch_size).each_slice(batch_size)
+
+        chunk_batches.each do |chunks|
+          chunks.each do |chunk|
+            chunk[:_id] = BSON::ObjectId.new
+            chunk[:files_id] = dest_info[:_id]
+          end
+
+          chunks_collection.insert_many(chunks)
+
+          chunks.each do |chunk|
+            chunk[:data].data.clear # deallocate strings
+          end
+        end
+
+        dest_info[:uploadDate] = Time.now.utc
+        dest_info[:filename]   = shrine_metadata["filename"] || id
+        files_collection.insert_one(dest_info)
+        id.replace(dest_info[:_id].to_s + File.extname(id))
+      end
+
+      def copyable?(io, id)
+        io.is_a?(UploadedFile) && io.storage.is_a?(Storage::Gridfs)
+      end
+
       def create_file(id, shrine_metadata: {})
         file = Mongo::Grid::File.new("",
           filename:     shrine_metadata["filename"] || id,
-          content_type: shrine_metadata["mime_type"],
+          content_type: shrine_metadata["mime_type"] || "application/octet-stream",
           metadata:     shrine_metadata,
           chunk_size:   chunk_size,
         )
-        id.replace(file.id.to_s + File.extname(id))
 
         bucket.insert_one(file)
 
+        id.replace(file.id.to_s + File.extname(id))
         file.info.document[:md5] = Digest::MD5.new
-        file
-      end
-
-      def files_collection
-        bucket.files_collection
-      end
 
-      def chunks_collection
-        bucket.chunks_collection
+        file
       end
 
       def bson_id(id)

data/shrine-gridfs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = "shrine-gridfs"
-  gem.version       = "0.3.1"
+  gem.version       = "0.3.2"
 
   gem.required_ruby_version = ">= 2.1"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shrine-gridfs
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Janko Marohnić