moped-gridfs 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a366fb5dfef7a6f12ee74c4e5e4d1bb39d810a0
+  data.tar.gz: b20b96931842cfb6a35ee986a1fa7e327d9882f2
+SHA512:
+  metadata.gz: 7921ced6e4e20baae692c7a7255e1f4dc1ab9afa8cf743660fe4d4bc1968ebf4184e191c878026e8f0c88bfe40a5c80ceee07165b3e40a191631ca667c021318
+  data.tar.gz: afd679c13f190f0e4a40ce2a533cdf5563e58b4580e20a24f2ede733cc59025dd70fafad3cbf1f359eaa8ffdb9bae92f57c405d601f40700a6eea8a81d018828

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.DS_Store

data/.rspec

@@ -0,0 +1 @@
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+moped-gridfs

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.0
+  - jruby
+gemfile:
+  - Gemfile
+  - gemfiles/moped15.gemfile
+services:
+  - mongodb

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 topac
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# moped-gridfs
+
+Add [GridFS support](http://docs.mongodb.org/manual/core/gridfs) to the [Moped driver](https://github.com/mongoid/moped).
+
+Moped is a fast MongoDB driver for Ruby,
+but it does not implement the GridFS specifications
+(while [mongo-ruby-driver](https://github.com/mongodb/mongo-ruby-driver) does).
+
+## Bucket
+
+GridFS places the collections in a common bucket by prefixing each with the bucket name.
+By default, GridFS uses two collections with names prefixed by "fs" bucket: _fs.files_ and _fs.chunks_.
+
+You can choose a different bucket name than "fs", and create multiple buckets in a single database.  
+Access the default bucket (named "fs") this way:
+
+```ruby
+  require 'moped'
+  require 'moped/gridfs'
+
+  session = Moped::Session.new(["127.0.0.1:27017"])
+  session.use("test")
+  bucket = session.bucket #<Moped::GridFS::Bucket:7ffbdbd4e160 name=fs>
+```
+or
+```ruby
+  bucket = Moped::GridFS::Bucket.new(session) #<Moped::GridFS::Bucket:7fc06db72c00 name=fs>
+```
+
+A list of all the buckets can be retrieved with `Session#buckets`.
+For example, you can access the _photos_ bucket with `session.buckets['photos']`.
+
+To open a file call `Bucket#open`, with the filename (or the _id) and the open mode.
+A more generic selector can be also given instead of the filename.
+
+## File
+
+The GridFS::File class exposes an API similar to the ruby File class.
+
+```ruby
+  file = bucket.open("myfile", "w+") #<Moped::GridFS::File:7f88599a58b0 bucket=fs _id=539c532ddb13a973ed000001 mode=w+ filename=myfile length=0>
+  file.write("foobar") # 6
+  file.seek(3) # 3
+  file.read # "bar"
+```
+
+All the [open modes](http://www.ruby-doc.org/core-2.1.2/IO.html#method-c-new-label-IO+Open+Mode) are supported:
+r, r+, w, w+, a and a+.
+
+GridFS::File attributes are: _id, length, chunk_size, filename, content_type, md5, aliases, metadata and uploadDate.
+Some of them may be changed if the file is opened in write/append mode.
+
+```ruby
+  file.content_type # "application/octet-stream"
+  file.md5 # "3858f62230ac3c915f300c664312c63f"
+  file.filename = "test"
+  file.filename # "test"
+```
+
+## Thread safe?
+It depends on what you're doing.  
+You may face race conditions if many threads are writing on the same file a buffer that have to be splitted onto multiple chunks. This is due to how the GridFS specs have been designed: [read this](https://jira.mongodb.org/browse/NODE-157).
+
+## Performance
+
+Are pretty much the same of mongo-ruby-driver (run the script perf/compare.rb).
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'moped-gridfs'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install moped-gridfs
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/gemfiles/moped15.gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+gemspec path: '..'
+
+gem "moped",  "~> 1.5.0"

data/lib/moped/gridfs.rb

@@ -0,0 +1,2 @@
+require "moped/gridfs/version"
+require "moped/gridfs/extensions/session"

data/lib/moped/gridfs/access_modes.rb

@@ -0,0 +1,48 @@
+module Moped
+  module GridFS
+    module AccessModes
+
+      attr_reader :mode
+
+      ACCESS_MODES = %w[r r+ w w+ a a+]
+
+      def readable?
+        mode =~ /r|\+/
+      end
+
+      def writable?
+        mode =~ /w|\+|a/
+      end
+
+      def append?
+        mode =~ /a/
+      end
+
+      def read_only?
+        mode == 'r'
+      end
+
+      def write_only?
+        mode == 'w' or mode == 'a'
+      end
+
+      def read_write?
+        mode =~ /\+/
+      end
+
+      private
+
+      def append_only?
+        mode == 'a'
+      end
+
+      def need_file?
+        mode == 'r' or mode == 'r+'
+      end
+
+      def truncate?
+        mode == 'w' or mode == 'w+'
+      end
+    end
+  end
+end

data/lib/moped/gridfs/bucket.rb

@@ -0,0 +1,66 @@
+require "moped/gridfs/files"
+require "moped/gridfs/file"
+require "moped/gridfs/inspectable"
+require "moped/gridfs/bucketable"
+
+module Moped
+  module GridFS
+    class Bucket
+      include Bucketable
+      include Inspectable
+
+      attr_reader :name, :session
+
+      DEFAULT_NAME = 'fs'
+
+      def initialize(session, name = DEFAULT_NAME)
+        @name = name.to_s.strip
+        @session = session
+
+        raise ArgumentError.new("Bucket name cannot be empty") if @name.empty?
+      end
+
+      def open(selector, mode)
+        ensure_indexes
+        file = File.new(self, mode, selector)
+        block_given? ? yield(file) : file
+      end
+
+      def ensure_indexes
+        @indexes_ensured ||= begin
+          chunks_collection.indexes.create(files_id: 1, n: 1)
+          # Optional index on filename
+          files_collection.indexes.create({filename: 1}, {background: true})
+          true
+        end
+      end
+
+      def files
+        Files.new(self)
+      end
+
+      def md5(file_id)
+        session.command(filemd5: file_id, root: name)['md5']
+      end
+
+      def delete(selector)
+        document = files_collection.find(parse_selector(selector)).first
+        return unless document
+        chunks_collection.find(files_id: document['_id']).remove_all
+        files_collection.find(_id: document['_id']).remove_all
+        true
+      end
+
+      alias :remove :delete
+
+      def drop
+        [files_collection, chunks_collection].map(&:drop)
+        @indexes_ensured = false
+      end
+
+      def inspect
+        build_inspect_string(name: name)
+      end
+    end
+  end
+end

data/lib/moped/gridfs/bucketable.rb

@@ -0,0 +1,33 @@
+module Moped
+  module GridFS
+    module Bucketable
+      def files_collection
+        if self.respond_to?(:session)
+          session[:"#{name}.files"]
+        else
+          bucket.files_collection
+        end
+      end
+
+      def chunks_collection
+        if self.respond_to?(:session)
+          session[:"#{name}.chunks"]
+        else
+          bucket.chunks_collection
+        end
+      end
+
+      private
+
+      def parse_selector(selector)
+        if selector.kind_of?(String)
+          {filename: selector}
+        elsif selector.kind_of?(BSON::ObjectId)
+          {_id: selector}
+        else
+          selector
+        end
+      end
+    end
+  end
+end

data/lib/moped/gridfs/buckets.rb

@@ -0,0 +1,37 @@
+require "moped/gridfs/bucket"
+
+module Moped
+  module GridFS
+    class Buckets
+      include Enumerable
+
+      attr_reader :session
+
+      def initialize(session)
+        @session = session
+      end
+
+      def names
+        collections.map { |collection| collection.name.gsub('.files', '') }
+      end
+
+      def count
+        collections.size
+      end
+
+      def [](name)
+        Bucket.new(session, name)
+      end
+
+      def each(&block)
+        names.each { |name| yield(self[name]) }
+      end
+
+      private
+
+      def collections
+        session.collections.select { |collection| collection.name =~ /.+\.files\z/ }
+      end
+    end
+  end
+end

data/lib/moped/gridfs/extensions/session.rb

@@ -0,0 +1,21 @@
+require "moped"
+require "moped/gridfs/bucket"
+require "moped/gridfs/buckets"
+
+module Moped
+  module GridFS
+    module Extensions
+      module Session
+        def bucket
+          Bucket.new(self)
+        end
+
+        def buckets
+          Buckets.new(self)
+        end
+      end
+    end
+  end
+end
+
+Moped::Session.__send__(:include, Moped::GridFS::Extensions::Session)

data/lib/moped/gridfs/file.rb

@@ -0,0 +1,233 @@
+require "moped/gridfs/inspectable"
+require "moped/gridfs/bucketable"
+require "moped/gridfs/access_modes"
+
+module Moped
+  module GridFS
+    class File
+      include Bucketable
+      include Inspectable
+      include AccessModes
+
+      attr_reader :mode, :attributes, :bucket, :pos
+
+      def initialize(bucket, mode, selector)
+        selector = parse_selector(selector)
+        @mode = mode
+        @bucket = bucket
+        @cached_chunk = nil
+        @pos = 0
+
+        raise ArgumentError.new("Invalid access mode #{mode}") unless ACCESS_MODES.include?(mode)
+
+        document = files_collection.find(selector).first
+
+        raise "No such file" if need_file? and !document
+
+        if document and truncate?
+          chunks_collection.find(files_id: document['_id']).remove_all
+          files_collection.find(_id: document['_id']).remove_all
+
+          @attributes = normalize_attributes(selector)
+        else
+          @attributes = normalize_attributes(document || selector)
+          @attributes.freeze if read_only?
+        end
+
+        define_dynamic_accessors
+
+        file_query.upsert(attributes) if writable?
+
+        @pos = length if append_only?
+      end
+
+      alias :tell :pos
+
+      def pos=(value)
+        check_negative_value(value)
+
+        @pos = (append_only? and value < length) ? length : value
+      end
+
+      alias :seek :pos=
+
+      def rewind
+        self.pos = 0
+      end
+
+      def eof?
+        raise "Not opened for reading" if write_only?
+
+        pos >= length
+      end
+
+      EMPTINESS = ''.force_encoding('BINARY')
+
+      def read(size = length)
+        raise "Not opened for reading" if write_only?
+
+        check_negative_value(size)
+
+        chunk_number = pos / chunk_size
+        chunk_offset = pos % chunk_size
+
+        data = EMPTINESS
+
+        loop do
+          break if data.size >= size
+          break unless read_chunk(chunk_number)
+          buffer = @cached_chunk[:data][chunk_offset..-1]
+          data.empty? ? (data = buffer) : (data << buffer)
+          chunk_number += 1
+          chunk_offset = 0
+        end
+
+        data = data[0..size - 1]
+        @pos += data.size
+        data
+      end
+
+      def write(data)
+        raise "Not opened for writing" if read_only?
+
+        data.force_encoding('BINARY') if data.respond_to?(:force_encoding)
+
+        @pos = length if @pos > length
+        @pos = length if append?
+
+        chunk_number = pos / chunk_size
+        chunk_offset = pos % chunk_size
+        written = data.size
+        new_length = 0
+
+        loop do
+          if buffer = read_chunk(chunk_number)
+            data = (chunk_offset.zero? ? EMPTINESS : buffer[0..chunk_offset - 1]) + data + (buffer[chunk_offset + data.size..-1] || EMPTINESS)
+          end
+
+          to_write = data[0..chunk_size - 1] || EMPTINESS
+
+          break if to_write.empty?
+
+          new_length = chunk_number * chunk_size + write_chunk(chunk_number, to_write)
+
+          data = data[chunk_size..-1] || EMPTINESS
+
+          break if data.empty?
+
+          chunk_number += 1
+          chunk_offset = 0
+        end
+
+        # Update internal position
+        @pos += written
+
+        # Calculate new md5 (if needed)
+        md5 = bucket.md5(@attributes[:_id]) if written > 0
+
+        # Update if something changed
+        updates = {}
+        updates[:md5] = md5 if md5
+        updates[:length] = new_length if new_length > length
+        change_attributes(updates) if updates.any?
+
+        written
+      end
+
+      DEFAULT_CHUNK_SIZE = 255 * 1024
+
+      def default_chunk_size
+        DEFAULT_CHUNK_SIZE
+      end
+
+      def inspect
+        build_inspect_string(bucket: bucket.name, _id: _id, mode: mode, filename: filename, length: length)
+      end
+
+      private
+
+      def normalize_attributes(provided)
+        provided.keys.each do |key|
+          provided[key.to_sym] = provided.delete(key)
+        end
+
+        attrs = {}
+        attrs[:_id]         = provided[:_id] ? BSON::ObjectId.from_string(provided[:_id]) : BSON::ObjectId.new
+        attrs[:length]      = (provided[:length] || 0).to_i
+        attrs[:chunkSize]   = (provided[:chunkSize] || provided[:chunk_size] || default_chunk_size).to_i
+        attrs[:filename]    = provided[:filename] || attrs[:_id].to_s
+        attrs[:contentType] = provided[:content_type] || provided[:contentType] || 'application/octet-stream'
+        attrs[:md5]         = provided[:md5]
+        attrs[:aliases]     = provided[:aliases] || []
+        attrs[:metadata]    = provided[:metadata] || {}
+        attrs[:uploadDate]  = provided[:upload_date] || provided[:uploadDate] || Time.now.utc
+        attrs
+      end
+
+      PROTECTED_ATTRIBUTES = [:_id, :length, :chunkSize, :md5]
+
+      def define_dynamic_accessors
+        attributes.keys.each do |attrname|
+          method_name = underscorize(attrname)
+
+          __send__(:define_singleton_method, method_name) { attributes[attrname] }
+
+          if writable? and !PROTECTED_ATTRIBUTES.include?(attrname)
+            __send__(:define_singleton_method, :"#{method_name}=") do |value|
+              change_attributes(:"#{attrname}" => value)
+            end
+          end
+        end
+      end
+
+      def underscorize(name)
+        name.to_s.gsub(/([A-Z])/, '_\1').downcase.to_sym
+      end
+
+      def check_negative_value(value)
+        raise ArgumentError.new("negative value #{value} given") if value < 0
+      end
+
+      def change_attributes(hash)
+        file_query.update('$set' => hash)
+        @attributes.merge!(hash)
+      end
+
+      def file_query
+        files_collection.find(_id: attributes[:_id])
+      end
+
+      def chunk_query(n)
+        chunks_collection.find(files_id: attributes[:_id], n: n)
+      end
+
+      def chunk(n)
+        chunk_query(n).first
+      end
+
+      def write_chunk(n, data)
+        chunk_query(n).upsert('$set' => {data: binarize(data)})
+        @cached_chunk = {n: n, data: data}
+        data.size
+      end
+
+      def read_chunk(n)
+        return @cached_chunk[:data] if @cached_chunk and @cached_chunk[:n] == n
+        readed = chunk(n)
+        return unless readed
+        @cached_chunk = {n: readed['n'], data: readed['data'].data}
+        @cached_chunk[:data]
+      end
+
+      if Moped::VERSION < '2.0.0'
+        def binarize(data)
+          BSON::Binary.new(:generic, data)
+        end
+      else
+        def binarize(data)
+          BSON::Binary.new(data, :generic)
+        end
+      end
+    end
+  end
+end