mongodb-mongo 0.4.4 → 0.5.0

data/README.rdoc

@@ -121,13 +121,31 @@ Mongo as-is. If the string is ASCII all is well, because ASCII is a subset of
 UTF-8. If the string is not ASCII then it may not be a well-formed UTF-8
 string.
 
-== The DB class
+== Primary Keys
+
+The field _id is a primary key. It is treated specially by the database, and
+its use makes many operations more efficient.
+
+The value of an _id may be of any type. (Older versions of Mongo required that
+they be XGen::Mongo::Driver::ObjectID instances.)
+
+The database itself inserts an _id value if none is specified when a record is
+inserted.
+
+The driver automatically sends the _id field to the database first, which is
+how Mongo likes it. You don't have to worry about where the _id field is in
+your hash record, or worry if you are using an OrderedHash or not.
 
 === Primary Key Factories
 
-A basic Mongo driver is not responsible for creating primary keys or knowing
-how to interpret them. You can tell the Ruby Mongo driver how to create
-primary keys by passing in the :pk option to the Mongo#db method.
+A primary key factory is a class you supply to a DB object that knows how to
+generate _id values. Primary key factories are no longer necessary because
+Mongo now inserts an _id value for every record that does not already have
+one. However, if you want to control _id values or even their types, using a
+PK factory lets you do so.
+
+You can tell the Ruby Mongo driver how to create primary keys by passing in
+the :pk option to the Mongo#db method.
 
   include XGen::Mongo::Driver
   db = Mongo.new.db('dbname', :pk => MyPKFactory.new)
@@ -163,10 +181,17 @@ ActiveRecord-like framework for non-Rails apps) and the AR Mongo adapter code
     end
   end
 
-A database's PK factory object may be set after you obtain it, but only once.
-The only reason it is changeable is so that libraries such as MongoRecord that
-use this driver can set the PK factory after obtaining the database but before
-using it for the first time.
+A database's PK factory object may be set either when a DB object is created
+or immediately after you obtain it, but only once. The only reason it is
+changeable at all is so that libraries such as MongoRecord that use this
+driver can set the PK factory after obtaining the database but before using it
+for the first time.
+
+== The DB Class
+
+=== Primary Key factories
+
+See the section on "Primary Keys" above.
 
 === Strict mode
 

data/bin/run_test_script

@@ -0,0 +1,15 @@
+#!/bin/bash
+# usage: run_test_script test_name output_file
+#
+# See http://mongodb.onconfluence.com/display/DOCS/Using+the+Framework+(for+Driver+Developers)
+
+HERE=`dirname $0`
+
+begintime=`date`
+ruby $HERE/../tests/mongo-qa/$1
+exitval=$?
+endtime=`date`
+
+echo "begintime:$begintime" >> $2
+echo "endtime:$endtime" >> $2
+echo "exit_code:$exitval" >> $2

data/lib/mongo/gridfs/chunk.rb

@@ -0,0 +1,82 @@
+require 'mongo/types/objectid'
+require 'mongo/util/byte_buffer'
+require 'mongo/util/ordered_hash'
+
+
+module XGen
+  module Mongo
+    module GridFS
+
+      # A chunk stores a portion of GridStore data.
+      #
+      # TODO: user-defined chunk size
+      class Chunk
+
+        DEFAULT_CHUNK_SIZE = 1024 * 256
+
+        attr_reader :object_id, :chunk_number
+        attr_accessor :data
+
+        def initialize(file, mongo_object={})
+          @file = file
+          @object_id = mongo_object['_id'] || XGen::Mongo::Driver::ObjectID.new
+          @chunk_number = mongo_object['n'] || 0
+
+          @data = ByteBuffer.new
+          case mongo_object['data']
+          when String
+            mongo_object['data'].each_byte { |b| @data.put(b) }
+          when ByteBuffer
+            @data.put_array(mongo_object['data'].to_a)
+          when Array
+            @data.put_array(mongo_object['data'])
+          when nil
+          else
+            raise "illegal chunk format; data is #{mongo_object['data'] ? (' ' + mongo_object['data'].class.name) : 'nil'}"
+          end
+          @data.rewind
+        end
+
+        def pos; @data.position; end
+        def pos=(pos); @data.position = pos; end
+        def eof?; !@data.more?; end
+
+        def size; @data.size; end
+        alias_method :length, :size
+
+        # Erase all data after current position.
+        def truncate
+          if @data.position < @data.length
+            curr_data = @data
+            @data = ByteBuffer.new
+            @data.put_array(curr_data.to_a[0...curr_data.position])
+          end
+        end
+
+        def getc
+          @data.more? ? @data.get : nil
+        end
+
+        def putc(byte)
+          @data.put(byte)
+        end
+
+        def save
+          coll = @file.chunk_collection
+          coll.remove({'_id' => @object_id})
+          coll.insert(to_mongo_object)
+        end
+
+        def to_mongo_object
+          h = OrderedHash.new
+          h['_id'] = @object_id
+          h['files_id'] = @file.files_id
+          h['n'] = @chunk_number
+          h['data'] = data
+          h
+        end
+
+      end
+    end
+  end
+end

data/lib/mongo/gridfs/grid_store.rb

@@ -0,0 +1,433 @@
+require 'mongo/types/objectid'
+require 'mongo/util/ordered_hash'
+require 'mongo/gridfs/chunk'
+
+module XGen
+  module Mongo
+    module GridFS
+
+      # GridStore is an IO-like object that provides input and output for
+      # streams of data to Mongo. See Mongo's documentation about GridFS for
+      # storage implementation details.
+      #
+      # Example code:
+      #
+      #   require 'mongo/gridfs'
+      #   GridStore.open(database, 'filename', 'w') { |f|
+      #     f.puts "Hello, world!"
+      #   }
+      #   GridStore.open(database, 'filename, 'r') { |f|
+      #     puts f.read         # => Hello, world!\n
+      #   }
+      #   GridStore.open(database, 'filename', 'w+') { |f|
+      #     f.puts "But wait, there's more!"
+      #   }
+      #   GridStore.open(database, 'filename, 'r') { |f|
+      #     puts f.read         # => Hello, world!\nBut wait, there's more!\n
+      #   }
+      class GridStore
+
+        DEFAULT_ROOT_COLLECTION = 'gridfs'
+        DEFAULT_CONTENT_TYPE = 'text/plain'
+
+        include Enumerable
+
+        attr_accessor :filename
+
+        # Array of strings; may be +nil+
+        attr_accessor :aliases
+
+        # Default is DEFAULT_CONTENT_TYPE
+        attr_accessor :content_type
+
+        attr_accessor :metadata
+
+        attr_reader :files_id
+
+        # Time that the file was first saved.
+        attr_reader :upload_date
+
+        attr_reader :chunk_size
+
+        attr_accessor :lineno
+
+        class << self
+
+          def exist?(db, name, root_collection=DEFAULT_ROOT_COLLECTION)
+            db.collection("#{root_collection}.files").find({'filename' => name}).next_object != nil
+          end
+
+          def open(db, name, mode, options={})
+            gs = self.new(db, name, mode, options)
+            result = nil
+            begin
+              result = yield gs if block_given?
+            ensure
+              gs.close
+            end
+            result
+          end
+
+          def read(db, name, length=nil, offset=nil)
+            GridStore.open(db, name, 'r') { |gs|
+              gs.seek(offset) if offset
+              gs.read(length)
+            }
+          end
+
+          def readlines(db, name, separator=$/)
+            GridStore.open(db, name, 'r') { |gs|
+              gs.readlines(separator)
+            }
+          end
+
+          def unlink(db, *names)
+            names.each { |name|
+              gs = GridStore.new(db, name)
+              gs.send(:delete_chunks)
+              gs.collection.remove('_id' => gs.files_id)
+            }
+          end
+          alias_method :delete, :unlink
+
+        end
+
+        #---
+        # ================================================================
+        #+++
+
+        # Mode may only be 'r', 'w', or 'w+'.
+        #
+        # Options. Descriptions start with a list of the modes for which that
+        # option is legitimate.
+        #
+        # :root :: (r, w, w+) Name of root collection to use, instead of
+        #          DEFAULT_ROOT_COLLECTION.
+        #
+        # :metadata:: (w, w+) A hash containing any data you want persisted as
+        #                     this file's metadata. See also metadata=
+        #
+        # :chunk_size :: (w) Sets chunk size for files opened for writing
+        #                See also chunk_size= which may only be called before
+        #                any data is written.
+        #
+        # :content_type :: (w) Default value is DEFAULT_CONTENT_TYPE. See
+        #                  also #content_type=
+        def initialize(db, name, mode='r', options={})
+          @db, @filename, @mode = db, name, mode
+          @root = options[:root] || DEFAULT_ROOT_COLLECTION
+
+          doc = collection.find({'filename' => @filename}).next_object
+          if doc
+            @files_id = doc['_id']
+            @content_type = doc['contentType']
+            @chunk_size = doc['chunkSize']
+            @upload_date = doc['uploadDate']
+            @aliases = doc['aliases']
+            @length = doc['length']
+            @metadata = doc['metadata']
+          else
+            @files_id = XGen::Mongo::Driver::ObjectID.new
+            @content_type = DEFAULT_CONTENT_TYPE
+            @chunk_size = Chunk::DEFAULT_CHUNK_SIZE
+            @length = 0
+          end
+
+          case mode
+          when 'r'
+            @curr_chunk = nth_chunk(0)
+            @position = 0
+          when 'w'
+            chunk_collection.create_index("chunk_index", ['files_id', 'n'])
+            delete_chunks
+            @curr_chunk = Chunk.new(self, 'n' => 0)
+            @content_type = options[:content_type] if options[:content_type]
+            @chunk_size = options[:chunk_size] if options[:chunk_size]
+            @metadata = options[:metadata] if options[:metadata]
+            @position = 0
+          when 'w+'
+            chunk_collection.create_index("chunk_index", ['files_id', 'n'])
+            @curr_chunk = nth_chunk(last_chunk_number) || Chunk.new(self, 'n' => 0) # might be empty
+            @curr_chunk.pos = @curr_chunk.data.length if @curr_chunk
+            @metadata = options[:metadata] if options[:metadata]
+            @position = @length
+          else
+            raise "error: illegal mode #{mode}"
+          end
+
+          @lineno = 0
+          @pushback_byte = nil
+        end
+
+        def collection
+          @db.collection("#{@root}.files")
+        end
+
+        # Returns collection used for storing chunks. Depends on value of
+        # @root.
+        def chunk_collection
+          @db.collection("#{@root}.chunks")
+        end
+
+        # Change chunk size. Can only change if the file is opened for write
+        # and no data has yet been written.
+        def chunk_size=(size)
+          unless @mode[0] == ?w && @position == 0 && @upload_date == nil
+            raise "error: can only change chunk size if open for write and no data written."
+          end
+          @chunk_size = size
+        end
+
+        #---
+        # ================ reading ================
+        #+++
+
+        def getc
+          if @pushback_byte
+            byte = @pushback_byte
+            @pushback_byte = nil
+            @position += 1
+            byte
+          elsif eof?
+            nil
+          else
+            if @curr_chunk.eof?
+              @curr_chunk = nth_chunk(@curr_chunk.chunk_number + 1)
+            end
+            @position += 1
+            @curr_chunk.getc
+          end
+        end
+
+        def gets(separator=$/)
+          str = ''
+          byte = getc
+          return nil if byte == nil # EOF
+          while byte != nil
+            s = byte.chr
+            str << s
+            break if s == separator
+            byte = getc
+          end
+          @lineno += 1
+          str
+        end
+
+        def read(len=nil, buf=nil)
+          buf ||= ''
+          byte = getc
+          while byte != nil && (len == nil || len > 0)
+            buf << byte.chr
+            len -= 1 if len
+            byte = getc if (len == nil || len > 0)
+          end
+          buf
+        end
+
+        def readchar
+          byte = getc
+          raise EOFError.new if byte == nil
+          byte
+        end
+
+        def readline(separator=$/)
+          line = gets
+          raise EOFError.new if line == nil
+          line
+        end
+
+        def readlines(separator=$/)
+          read.split(separator).collect { |line| "#{line}#{separator}" }
+        end
+
+        def each
+          line = gets
+          while line
+            yield line
+            line = gets
+          end
+        end
+        alias_method :each_line, :each
+
+        def each_byte
+          byte = getc
+          while byte
+            yield byte
+            byte = getc
+          end
+        end
+
+        def ungetc(byte)
+          @pushback_byte = byte
+          @position -= 1
+        end
+
+        #---
+        # ================ writing ================
+        #+++
+
+        def putc(byte)
+          if @curr_chunk.pos == @chunk_size
+            prev_chunk_number = @curr_chunk.chunk_number
+            @curr_chunk.save
+            @curr_chunk = Chunk.new(self, 'n' => prev_chunk_number + 1)
+          end
+          @position += 1
+          @curr_chunk.putc(byte)
+        end
+
+        def print(*objs)
+          objs = [$_] if objs == nil || objs.empty?
+          objs.each { |obj|
+            str = obj.to_s
+            str.each_byte { |byte| putc(byte) }
+          }
+          nil
+        end
+
+        def puts(*objs)
+          if objs == nil || objs.empty?
+            putc(10)
+          else
+            print(*objs.collect{ |obj|
+                    str = obj.to_s
+                    str << "\n" unless str =~ /\n$/
+                    str
+                  })
+          end
+          nil
+        end
+
+        def <<(obj)
+          write(obj.to_s)
+        end
+
+        # Writes +string+ as bytes and returns the number of bytes written.
+        def write(string)
+          raise "#@filename not opened for write" unless @mode[0] == ?w
+          count = 0
+          string.each_byte { |byte|
+            putc byte
+            count += 1
+          }
+          count
+        end
+
+        # A no-op.
+        def flush
+        end
+
+        #---
+        # ================ status ================
+        #+++
+
+        def eof
+          raise IOError.new("stream not open for reading") unless @mode[0] == ?r
+          @position >= @length
+        end
+        alias_method :eof?, :eof
+
+        #---
+        # ================ positioning ================
+        #+++
+
+        def rewind
+          if @curr_chunk.chunk_number != 0
+            if @mode[0] == ?w
+              delete_chunks
+              @curr_chunk = Chunk.new(self, 'n' => 0)
+            else
+              @curr_chunk == nth_chunk(0)
+            end
+          end
+          @curr_chunk.pos = 0
+          @lineno = 0
+          @position = 0
+        end
+
+        def seek(pos, whence=IO::SEEK_SET)
+          target_pos = case whence
+                       when IO::SEEK_CUR
+                         @position + pos
+                       when IO::SEEK_END
+                         @length - pos
+                       when IO::SEEK_SET
+                         pos
+                       end
+                         
+          new_chunk_number = (target_pos / @chunk_size).to_i
+          if new_chunk_number != @curr_chunk.chunk_number
+            @curr_chunk.save if @mode[0] == ?w
+            @curr_chunk = nth_chunk(new_chunk_number)
+          end
+          @position = target_pos
+          @curr_chunk.pos = @position % @chunk_size
+          0
+        end
+
+        def tell
+          @position
+         end
+
+        #---
+        # ================ closing ================
+        #+++
+
+        def close
+          if @mode[0] == ?w
+            if @curr_chunk
+              @curr_chunk.truncate
+              @curr_chunk.save if @curr_chunk.pos > 0
+            end
+            files = collection
+            if @upload_date
+              files.remove('_id' => @files_id)
+            else
+              @upload_date = Time.now
+            end
+            files.insert(to_mongo_object)
+          end
+          @db = nil
+        end
+          
+        def closed?
+          @db == nil
+        end
+
+        #---
+        # ================ protected ================
+        #+++
+
+        protected
+
+        def to_mongo_object
+          h = OrderedHash.new
+          h['_id'] = @files_id
+          h['filename'] = @filename
+          h['contentType'] = @content_type
+          h['length'] = @curr_chunk ? @curr_chunk.chunk_number * @chunk_size + @curr_chunk.pos : 0
+          h['chunkSize'] = @chunk_size
+          h['uploadDate'] = @upload_date
+          h['aliases'] = @aliases
+          h['metadata'] = @metadata
+          h
+        end
+
+        def delete_chunks
+          chunk_collection.remove({'files_id' => @files_id}) if @files_id
+          @curr_chunk = nil
+        end
+
+        def nth_chunk(n)
+          mongo_chunk = chunk_collection.find({'files_id' => @files_id, 'n' => n}).next_object
+          Chunk.new(self, mongo_chunk || {})
+        end
+
+        def last_chunk_number
+          (@length / @chunk_size).to_i
+        end
+
+      end
+    end
+  end
+end

data/lib/mongo/gridfs.rb

@@ -0,0 +1 @@
+require 'mongo/gridfs/grid_store'

data/lib/mongo/types/binary.rb

@@ -14,21 +14,29 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 # ++
 
+require 'mongo/util/byte_buffer'
+
 module XGen
   module Mongo
     module Driver
 
-      # An array of binary bytes. The only reason this exists is so that the
-      # BSON encoder will know to output the Mongo BINARY type.
-      class Binary < String; end
+      # An array of binary bytes with a Mongo subtype value.
+      class Binary < ByteBuffer
 
-    end
-  end
-end
+        SUBTYPE_BYTES = 0x02
+        SUBTYPE_UUID = 0x03
+        SUBTYPE_MD5 = 0x05
+        SUBTYPE_USER_DEFINED = 0x80
 
-class String
-  # Convert a string into a XGen::Mongo::Driver::Binary
-  def to_mongo_binary
-    XGen::Mongo::Driver::Binary.new(self)
+        # One of the SUBTYPE_* constants. Default is SUBTYPE_BYTES.
+        attr_accessor :subtype
+
+        def initialize(initial_data=[], subtype=SUBTYPE_BYTES)
+          super(initial_data)
+          @subtype = subtype
+        end
+
+      end
+    end
   end
 end

data/lib/mongo/util/bson.rb

@@ -26,6 +26,8 @@ require 'mongo/types/undefined'
 # A BSON seralizer/deserializer.
 class BSON
 
+  include XGen::Mongo::Driver
+
   MINKEY = -1
   EOO = 0
   NUMBER = 1
@@ -46,8 +48,6 @@ class BSON
   NUMBER_INT = 16
   MAXKEY = 127
 
-  BYTE_TYPE = 2
-
   if RUBY_VERSION >= '1.9'
     def self.to_utf8(str)
       str.encode("utf-8")
@@ -169,7 +169,7 @@ class BSON
         doc[key] = nil
       when UNDEFINED
         key = deserialize_cstr(@buf)
-        doc[key] = XGen::Mongo::Driver::Undefined.new
+        doc[key] = Undefined.new
       when REF
         key = deserialize_cstr(@buf)
         doc[key] = deserialize_dbref_data(@buf, key, parent)
@@ -242,7 +242,7 @@ class BSON
     options |= Regexp::MULTILINE if options_str.include?('m')
     options |= Regexp::EXTENDED if options_str.include?('x')
     options_str.gsub!(/[imx]/, '') # Now remove the three we understand
-    XGen::Mongo::Driver::RegexpOfHolding.new(str, options, options_str)
+    RegexpOfHolding.new(str, options, options_str)
   end
 
   def deserialize_string_data(buf)
@@ -256,23 +256,20 @@ class BSON
   end
 
   def deserialize_oid_data(buf)
-    XGen::Mongo::Driver::ObjectID.new(buf.get(12))
+    ObjectID.new(buf.get(12))
   end
 
   def deserialize_dbref_data(buf, key, parent)
     ns = deserialize_string_data(buf)
     oid = deserialize_oid_data(buf)
-    XGen::Mongo::Driver::DBRef.new(parent, key, @db, ns, oid)
+    DBRef.new(parent, key, @db, ns, oid)
   end
 
   def deserialize_binary_data(buf)
-    buf.get_int                 # length + 4; ignored
-    buf.get                     # byte type; ignored
     len = buf.get_int
-    bytes = buf.get(len)
-    str = ''
-    bytes.each { |c| str << c.chr }
-    str.to_mongo_binary
+    type = buf.get
+    len = buf.get_int if type == Binary::SUBTYPE_BYTES
+    Binary.new(buf.get(len), type)
   end
 
   def serialize_eoo_element(buf)
@@ -293,24 +290,19 @@ class BSON
     buf.put(BINARY)
     self.class.serialize_cstr(buf, key)
 
-    bytes = case val
-            when ByteBuffer
-              val.to_a
-            else
-              if RUBY_VERSION >= '1.9'
-                val.bytes.to_a
-              else
-                a = []
-                val.each_byte { |byte| a << byte }
-                a
-              end
-            end
-
+    bytes = val.to_a
     num_bytes = bytes.length
-    buf.put_int(num_bytes + 4)
-    buf.put(BYTE_TYPE)
-    buf.put_int(num_bytes)
-    buf.put_array(bytes)
+    subtype = val.respond_to?(:subtype) ? val.subtype : Binary::SUBTYPE_BYTES
+    if subtype == Binary::SUBTYPE_BYTES
+      buf.put_int(num_bytes + 4)
+      buf.put(subtype)
+      buf.put_int(num_bytes)
+      buf.put_array(bytes)
+    else
+      buf.put_int(num_bytes)
+      buf.put(subtype)
+      buf.put_array(bytes)
+    end
   end
 
   def serialize_undefined_element(buf, key)
@@ -420,7 +412,7 @@ class BSON
       NUMBER_INT
     when Numeric
       NUMBER
-    when XGen::Mongo::Driver::Binary, ByteBuffer # must be before String
+    when ByteBuffer
       BINARY
     when String
       # magic awful stuff - the DB requires that a where clause is sent as CODE
@@ -429,9 +421,9 @@ class BSON
       ARRAY
     when Regexp
       REGEX
-    when XGen::Mongo::Driver::ObjectID
+    when ObjectID
       OID
-    when XGen::Mongo::Driver::DBRef
+    when DBRef
       REF
     when true, false
       BOOLEAN
@@ -441,7 +433,7 @@ class BSON
       OBJECT
     when Symbol
       SYMBOL
-    when XGen::Mongo::Driver::Undefined
+    when Undefined
       UNDEFINED
     else
       raise "Unknown type of object: #{o.class.name}"

data/lib/mongo/util/xml_to_ruby.rb

@@ -45,7 +45,10 @@ class XMLToRuby
     when 'string', 'code'
       e.text.to_s
     when 'binary'
-      Base64.decode64(e.text.to_s).to_mongo_binary
+      bin = Binary.new
+      decoded = Base64.decode64(e.text.to_s)
+      decoded.each_byte { |b| bin.put(b) }
+      bin
     when 'symbol'
       e.text.to_s.intern
     when 'boolean'

data/mongo-ruby-driver.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
   s.name = 'mongo'
-  s.version = '0.4.4'
+  s.version = '0.5.0'
   s.platform = Gem::Platform::RUBY
   s.summary = 'Simple pure-Ruby driver for the 10gen Mongo DB'
   s.description = 'A pure-Ruby driver for the 10gen Mongo DB. For more information about Mongo, see http://www.mongodb.org.'
 
   s.require_paths = ['lib']
   
-  s.files = ['bin/mongo_console', 'bin/validate',
+  s.files = ['bin/mongo_console', 'bin/run_test_script',
              'examples/benchmarks.rb',
              'examples/blog.rb',
              'examples/index_test.rb',
@@ -17,6 +17,9 @@ Gem::Specification.new do |s|
              'lib/mongo/collection.rb',
              'lib/mongo/cursor.rb',
              'lib/mongo/db.rb',
+             'lib/mongo/gridfs/chunk.rb',
+             'lib/mongo/gridfs/grid_store.rb',
+             'lib/mongo/gridfs.rb',
              'lib/mongo/message/get_more_message.rb',
              'lib/mongo/message/insert_message.rb',
              'lib/mongo/message/kill_cursors_message.rb',
@@ -40,13 +43,22 @@ Gem::Specification.new do |s|
              'lib/mongo/util/ordered_hash.rb',
              'lib/mongo/util/xml_to_ruby.rb',
              'README.rdoc', 'Rakefile', 'mongo-ruby-driver.gemspec']
-  s.test_files = ['tests/test_admin.rb',
+  s.test_files = ['tests/mongo-qa/_common.rb',
+                  'tests/mongo-qa/capped',
+                  'tests/mongo-qa/circuar',
+                  'tests/mongo-qa/count1',
+                  'tests/mongo-qa/find',
+                  'tests/mongo-qa/remove',
+                  'tests/mongo-qa/test1',
+                  'tests/test_admin.rb',
                   'tests/test_bson.rb',
                   'tests/test_byte_buffer.rb',
+                  'tests/test_chunk.rb',
                   'tests/test_cursor.rb',
                   'tests/test_db.rb',
                   'tests/test_db_api.rb',
                   'tests/test_db_connection.rb',
+                  'tests/test_grid_store.rb',
                   'tests/test_message.rb',
                   'tests/test_mongo.rb',
                   'tests/test_objectid.rb',

data/tests/mongo-qa/_common.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH[0,0] = File.join(File.dirname(__FILE__), '../..', 'lib')
+require 'mongo'
+
+DEFAULT_HOST = '127.0.0.1'
+DEFAULT_PORT = 27017
+DEFAULT_DB = 'driver_test_framework'
+
+include XGen::Mongo::Driver

data/tests/mongo-qa/capped

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '_common.rb')
+db = Mongo.new(DEFAULT_HOST, DEFAULT_PORT).db(DEFAULT_DB)
+
+db.create_collection('capped1', :capped => true, :size => 500)
+coll = db.collection('capped1')
+coll.insert('x' => 1)
+coll.insert('x' => 2)
+
+db.create_collection('capped2', :capped => true, :size => 1000, :max => 11)
+coll = db.collection('capped2')
+str = ''
+100.times {
+  coll.insert('dashes' => str)
+  str << '-'
+}

data/tests/mongo-qa/count1

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '_common.rb')
+db = Mongo.new(DEFAULT_HOST, DEFAULT_PORT).db(DEFAULT_DB)
+
+puts db.collection('test1').count
+puts db.collection('test2').count
+puts db.collection('test3').count('i' => 'a')
+puts db.collection('test3').count('i' => 3)
+puts db.collection('test3').count({'i' => {'$gte' => 67}})

data/tests/mongo-qa/find

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '_common.rb')
+db = Mongo.new(DEFAULT_HOST, DEFAULT_PORT).db(DEFAULT_DB)
+
+db.collection('test').insert('a' => 2)

data/tests/mongo-qa/remove

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '_common.rb')
+db = Mongo.new(DEFAULT_HOST, DEFAULT_PORT).db(DEFAULT_DB)
+
+db.collection('remove1').clear
+db.collection('remove2').remove('a' => 3)

data/tests/mongo-qa/test1

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '_common.rb')
+db = Mongo.new(DEFAULT_HOST, DEFAULT_PORT).db(DEFAULT_DB)
+
+coll = db.collection('part1')
+100.times { |i| coll.insert('x' => i) }

data/tests/test_bson.rb

@@ -113,25 +113,40 @@ class BSONTest < Test::Unit::TestCase
   end
 
   def test_binary
-    bin = 'binstring'.to_mongo_binary
-    assert_kind_of Binary, bin
+    bin = Binary.new
+    'binstring'.each_byte { |b| bin.put(b) }
 
     doc = {'bin' => bin}
     @b.serialize(doc)
     doc2 = @b.deserialize
-    assert_equal 'binstring', doc2['bin']
+    bin2 = doc2['bin']
+    assert_kind_of Binary, bin2
+    assert_equal 'binstring', bin2.to_s
+  end
+
+  def test_binary_type
+    bin = Binary.new([1, 2, 3, 4, 5], Binary::SUBTYPE_USER_DEFINED)
+
+    doc = {'bin' => bin}
+    @b.serialize(doc)
+    doc2 = @b.deserialize
+    bin2 = doc2['bin']
+    assert_kind_of Binary, bin2
+    assert_equal [1, 2, 3, 4, 5], bin2.to_a
+    assert_equal Binary::SUBTYPE_USER_DEFINED, bin2.subtype
   end
 
   def test_binary_byte_buffer
     bb = ByteBuffer.new
-    10.times { |i| bb.put(i) }
+    5.times { |i| bb.put(i + 1) }
+
     doc = {'bin' => bb}
     @b.serialize(doc)
     doc2 = @b.deserialize
-
-    doc2_bytes = []
-    doc2['bin'].each_byte { |b| doc2_bytes << b }
-    assert_equal bb.to_a, doc2_bytes
+    bin2 = doc2['bin']
+    assert_kind_of Binary, bin2
+    assert_equal [1, 2, 3, 4, 5], bin2.to_a
+    assert_equal Binary::SUBTYPE_BYTES, bin2.subtype
   end
 
   def test_undefined

data/tests/test_chunk.rb

@@ -0,0 +1,87 @@
+$LOAD_PATH[0,0] = File.join(File.dirname(__FILE__), '..', 'lib')
+require 'test/unit'
+require 'mongo'
+require 'mongo/gridfs'
+
+class ChunkTest < Test::Unit::TestCase
+
+  include XGen::Mongo::Driver
+  include XGen::Mongo::GridFS
+
+  def setup
+    @host = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
+    @port = ENV['MONGO_RUBY_DRIVER_PORT'] || Mongo::DEFAULT_PORT
+    @db = Mongo.new(@host, @port).db('ruby-mongo-utils-test')
+
+    @files = @db.collection('gridfs.files')
+    @chunks = @db.collection('gridfs.chunks')
+    @chunks.clear
+    @files.clear
+
+    @f = GridStore.new(@db, 'foobar', 'w')
+    @c = @f.instance_variable_get('@curr_chunk')
+  end
+
+  def teardown
+    if @db && @db.connected?
+      @chunks.clear
+      @files.clear
+      @db.close
+    end
+  end
+
+  def test_pos
+    assert_equal 0, @c.pos
+    assert @c.eof?              # since data is empty
+
+    b = ByteBuffer.new
+    3.times { |i| b.put(i) }
+    c = Chunk.new(@f, 'data' => b)
+    assert !c.eof?
+  end
+
+  def test_getc
+    b = ByteBuffer.new
+    3.times { |i| b.put(i) }
+    c = Chunk.new(@f, 'data' => b)
+
+    assert !c.eof?
+    assert_equal 0, c.getc
+    assert !c.eof?
+    assert_equal 1, c.getc
+    assert !c.eof?
+    assert_equal 2, c.getc
+    assert c.eof?
+  end
+
+  def test_putc
+    3.times { |i| @c.putc(i) }
+    @c.pos = 0
+
+    assert !@c.eof?
+    assert_equal 0, @c.getc
+    assert !@c.eof?
+    assert_equal 1, @c.getc
+    assert !@c.eof?
+    assert_equal 2, @c.getc
+    assert @c.eof?
+  end
+
+  def test_truncate
+    10.times { |i| @c.putc(i) }
+    assert_equal 10, @c.size
+    @c.pos = 3
+    @c.truncate
+    assert_equal 3, @c.size
+
+    @c.pos = 0
+    assert !@c.eof?
+    assert_equal 0, @c.getc
+    assert !@c.eof?
+    assert_equal 1, @c.getc
+    assert !@c.eof?
+    assert_equal 2, @c.getc
+    assert @c.eof?
+  end
+
+end

data/tests/test_grid_store.rb

@@ -0,0 +1,235 @@
+$LOAD_PATH[0,0] = File.join(File.dirname(__FILE__), '..', 'lib')
+require 'test/unit'
+require 'mongo'
+require 'mongo/gridfs'
+
+class GridStoreTest < Test::Unit::TestCase
+
+  include XGen::Mongo::Driver
+  include XGen::Mongo::GridFS
+
+  def setup
+    @host = ENV['MONGO_RUBY_DRIVER_HOST'] || 'localhost'
+    @port = ENV['MONGO_RUBY_DRIVER_PORT'] || Mongo::DEFAULT_PORT
+    @db = Mongo.new(@host, @port).db('ruby-mongo-utils-test')
+
+    @files = @db.collection('gridfs.files')
+    @chunks = @db.collection('gridfs.chunks')
+    @chunks.clear
+    @files.clear
+
+    GridStore.open(@db, 'foobar', 'w') { |f| f.write("hello, world!") }
+  end
+
+  def teardown
+    if @db && @db.connected?
+      @chunks.clear
+      @files.clear
+      @db.close
+    end
+  end
+
+  def test_exist
+    assert GridStore.exist?(@db, 'foobar')
+    assert !GridStore.exist?(@db, 'does_not_exist')
+    assert !GridStore.exist?(@db, 'foobar', 'another_root')
+  end
+
+  def test_small_write
+    rows = @files.find({'filename' => 'foobar'}).to_a
+    assert_not_nil rows
+    assert_equal 1, rows.length
+    row = rows[0]
+    assert_not_nil row
+
+    file_id = row['_id']
+    assert_kind_of ObjectID, file_id
+    rows = @chunks.find({'files_id' => file_id}).to_a
+    assert_not_nil rows
+    assert_equal 1, rows.length
+  end
+
+  def test_small_file
+    rows = @files.find({'filename' => 'foobar'}).to_a
+    assert_not_nil rows
+    assert_equal 1, rows.length
+    row = rows[0]
+    assert_not_nil row
+    assert_equal "hello, world!", GridStore.read(@db, 'foobar')
+  end
+
+  def test_overwrite
+    GridStore.open(@db, 'foobar', 'w') { |f| f.write("overwrite") }
+    assert_equal "overwrite", GridStore.read(@db, 'foobar')
+  end
+
+  def test_read_length
+    assert_equal "hello", GridStore.read(@db, 'foobar', 5)
+  end
+
+  # Also tests seek
+  def test_read_with_offset
+    assert_equal "world", GridStore.read(@db, 'foobar', 5, 7)
+    assert_equal "world!", GridStore.read(@db, 'foobar', nil, 7)
+  end
+
+  def test_multi_chunk
+    @chunks.clear
+    @files.clear
+
+    size = 512
+    GridStore.open(@db, 'biggie', 'w') { |f|
+      f.chunk_size = size
+      f.write('x' * size)
+      f.write('y' * size)
+      f.write('z' * size)
+    }
+
+    assert_equal 3, @chunks.count
+    assert_equal ('x' * size) + ('y' * size) + ('z' * size), GridStore.read(@db, 'biggie')
+  end
+
+  def test_puts_and_readlines
+    GridStore.open(@db, 'multiline', 'w') { |f|
+      f.puts "line one"
+      f.puts "line two\n"
+      f.puts "line three"
+    }
+
+    lines = GridStore.readlines(@db, 'multiline')
+    assert_equal ["line one\n", "line two\n", "line three\n"], lines
+  end
+
+  def test_unlink
+    assert_equal 1, @files.count
+    assert_equal 1, @chunks.count
+    GridStore.unlink(@db, 'foobar')
+    assert_equal 0, @files.count
+    assert_equal 0, @chunks.count
+  end
+
+  def test_append
+    GridStore.open(@db, 'foobar', 'w+') { |f| f.write(" how are you?") }
+    assert_equal 1, @chunks.count
+    assert_equal "hello, world! how are you?", GridStore.read(@db, 'foobar')
+  end
+
+  def test_rewind_and_truncate_on_write
+    GridStore.open(@db, 'foobar', 'w') { |f|
+      f.write("some text is inserted here")
+      f.rewind
+      f.write("abc")
+    }
+    assert_equal "abc", GridStore.read(@db, 'foobar')
+  end
+
+  def test_tell
+    GridStore.open(@db, 'foobar', 'r') { |f|
+      f.read(5)
+      assert_equal 5, f.tell
+    }
+  end
+
+  def test_empty_block_ok
+    GridStore.open(@db, 'empty', 'w')
+  end
+
+  def test_save_empty_file
+    @chunks.clear
+    @files.clear
+    GridStore.open(@db, 'empty', 'w') {} # re-write with zero bytes
+    assert_equal 1, @files.count
+    assert_equal 0, @chunks.count
+  end
+
+  def test_empty_file_eof
+    GridStore.open(@db, 'empty', 'w')
+    GridStore.open(@db, 'empty', 'r') { |f|
+      assert f.eof?
+    }
+  end
+
+  def test_cannot_change_chunk_size_on_read
+    begin
+      GridStore.open(@db, 'foobar', 'r') { |f| f.chunk_size = 42 }
+      fail "should have seen error"
+    rescue => ex
+      assert_match /error: can only change chunk size/, ex.to_s
+    end
+  end
+
+  def test_cannot_change_chunk_size_after_data_written
+    begin
+      GridStore.open(@db, 'foobar', 'w') { |f|
+        f.write("some text")
+        f.chunk_size = 42
+      }
+      fail "should have seen error"
+    rescue => ex
+      assert_match /error: can only change chunk size/, ex.to_s
+    end
+  end
+
+  def test_change_chunk_size
+    GridStore.open(@db, 'new-file', 'w') { |f|
+      f.chunk_size = 42
+      f.write("foo")
+    }
+    GridStore.open(@db, 'new-file', 'r') { |f|
+      assert f.chunk_size == 42
+    }
+  end
+
+  def test_chunk_size_in_option
+    GridStore.open(@db, 'new-file', 'w', :chunk_size => 42) { |f| f.write("foo") }
+    GridStore.open(@db, 'new-file', 'r') { |f|
+      assert f.chunk_size == 42
+    }
+  end
+
+  def test_upload_date
+    now = Time.now
+    orig_file_upload_date = nil
+    GridStore.open(@db, 'foobar', 'r') { |f| orig_file_upload_date = f.upload_date }
+    assert_not_nil orig_file_upload_date
+    assert (orig_file_upload_date - now) < 5 # even a really slow system < 5 secs
+
+    sleep(2)
+    GridStore.open(@db, 'foobar', 'w') { |f| f.write "new data" }
+    file_upload_date = nil
+    GridStore.open(@db, 'foobar', 'r') { |f| file_upload_date = f.upload_date }
+    assert_equal orig_file_upload_date, file_upload_date
+  end
+
+  def test_content_type
+    ct = nil
+    GridStore.open(@db, 'foobar', 'r') { |f| ct = f.content_type }
+    assert_equal GridStore::DEFAULT_CONTENT_TYPE, ct
+
+    GridStore.open(@db, 'foobar', 'w+') { |f| f.content_type = 'text/html' }
+    ct2 = nil
+    GridStore.open(@db, 'foobar', 'r') { |f| ct2 = f.content_type }
+    assert_equal 'text/html', ct2
+  end
+
+  def test_content_type_option
+    GridStore.open(@db, 'new-file', 'w', :content_type => 'image/jpg') { |f| f.write('foo') }
+    ct = nil
+    GridStore.open(@db, 'new-file', 'r') { |f| ct = f.content_type }
+    assert_equal 'image/jpg', ct
+  end
+
+  def test_unknown_mode
+    GridStore.open(@db, 'foobar', 'x')
+    fail 'should have seen "illegal mode" error raised'
+  rescue => ex
+    assert_equal "error: illegal mode x", ex.to_s
+  end
+
+  def test_metadata
+    GridStore.open(@db, 'foobar', 'r') { |f| assert_nil f.metadata }
+    GridStore.open(@db, 'foobar', 'w+') { |f| f.metadata = {'a' => 1} }
+    GridStore.open(@db, 'foobar', 'r') { |f| assert_equal({'a' => 1}, f.metadata) }
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mongodb-mongo
 version: !ruby/object:Gem::Version 
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors: 
 - Jim Menard
@@ -23,7 +23,7 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - bin/mongo_console
-- bin/validate
+- bin/run_test_script
 - examples/benchmarks.rb
 - examples/blog.rb
 - examples/index_test.rb
@@ -33,6 +33,9 @@ files:
 - lib/mongo/collection.rb
 - lib/mongo/cursor.rb
 - lib/mongo/db.rb
+- lib/mongo/gridfs/chunk.rb
+- lib/mongo/gridfs/grid_store.rb
+- lib/mongo/gridfs.rb
 - lib/mongo/message/get_more_message.rb
 - lib/mongo/message/insert_message.rb
 - lib/mongo/message/kill_cursors_message.rb
@@ -87,13 +90,22 @@ signing_key:
 specification_version: 2
 summary: Simple pure-Ruby driver for the 10gen Mongo DB
 test_files: 
+- tests/mongo-qa/_common.rb
+- tests/mongo-qa/capped
+- tests/mongo-qa/circuar
+- tests/mongo-qa/count1
+- tests/mongo-qa/find
+- tests/mongo-qa/remove
+- tests/mongo-qa/test1
 - tests/test_admin.rb
 - tests/test_bson.rb
 - tests/test_byte_buffer.rb
+- tests/test_chunk.rb
 - tests/test_cursor.rb
 - tests/test_db.rb
 - tests/test_db_api.rb
 - tests/test_db_connection.rb
+- tests/test_grid_store.rb
 - tests/test_message.rb
 - tests/test_mongo.rb
 - tests/test_objectid.rb

data/bin/validate

@@ -1,51 +0,0 @@
-#!/usr/bin/env ruby
-#
-# usage: validate somefile.xson somefile.bson
-#
-# Reads somefile.xson file (XML that describes a Mongo-type document),
-# converts it into a Ruby OrderedHash, runs that through the BSON
-# serialization code, and writes the BSON bytes to somefile.bson.
-#
-# In addition, this script takes the generated BSON, reads it in then writes
-# it back out to a temp BSON file. If they are different, we report that error
-# to STDOUT.
-#
-# This script is used by the mongo-qa project
-# (http://github.com/mongodb/mongo-qa).
-
-$LOAD_PATH[0,0] = File.join(File.dirname(__FILE__), '..', 'lib')
-require 'mongo'
-require 'mongo/util/xml_to_ruby'
-
-if ARGV.length < 2
-  $stderr.puts "usage: validate somefile.xson somefile.bson"
-  exit 1
-end
-
-# Translate the .xson XML into a Ruby object, turn that object into BSON, and
-# write the BSON to the file as requested.
-obj = File.open(ARGV[0], 'rb') { |f| XMLToRuby.new.xml_to_ruby(f) }
-bson = BSON.new.serialize(obj).to_a
-File.open(ARGV[1], 'wb') { |f| bson.each { |b| f.putc(b) } }
-
-# Now the additional testing. Read the generated BSON back in, deserialize it,
-# and re-serialize the results. Compare that BSON with the BSON from the file
-# we output.
-bson = File.open(ARGV[1], 'rb') { |f| f.read }
-bson = if RUBY_VERSION >= '1.9'
-         bson.bytes.to_a
-       else
-         bson.split(//).collect { |c| c[0] }
-       end
-
-# Turn the Ruby object into BSON bytes and compare with the BSON bytes from
-# the file.
-bson_from_ruby = BSON.new.serialize(obj).to_a
-
-if bson.length != bson_from_ruby.length
-  $stderr.puts "error: round-trip BSON lengths differ when testing #{ARGV[0]}"
-  exit 1
-elsif bson != bson_from_ruby
-  $stderr.puts "error: round-trip BSON contents differ when testing #{ARGV[0]}"
-  exit 1
-end