jmongo 1.0.3 → 1.1.0

data/test/grid_io_test.rb

@@ -0,0 +1,210 @@
+__END__
+require './test/test_helper'
+
+class GridIOTest < Test::Unit::TestCase
+
+  context "GridIO" do
+    setup do
+      @db = standard_connection.db(MONGO_TEST_DB)
+      @files  = @db.collection('fs.files')
+      @chunks = @db.collection('fs.chunks')
+      @chunks.create_index([['files_id', Mongo::ASCENDING], ['n', Mongo::ASCENDING]])
+    end
+
+    teardown do
+      @files.remove
+      @chunks.remove
+    end
+
+    context "Options" do
+      setup do
+        @filename = 'test'
+        @mode     = 'w'
+      end
+
+      should "set default 256k chunk size" do
+        file = GridIO.new(@files, @chunks, @filename, @mode)
+        assert_equal 256 * 1024, file.chunk_size
+      end
+
+      should "set chunk size" do
+        file = GridIO.new(@files, @chunks, @filename, @mode, :chunk_size => 1000)
+        assert_equal 1000, file.chunk_size
+      end
+    end
+
+    context "StringIO methods" do
+      setup do
+        @filename = 'test'
+        @mode     = 'w'
+        @data     = "012345678\n" * 100000
+        @file = GridIO.new(@files, @chunks, @filename, @mode)
+        @file.write(@data)
+        @file.close
+      end
+
+      should "read data character by character using" do
+        bytes = 0
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        while char = file.getc
+          bytes += 1
+        end
+        assert_equal bytes, 1_000_000
+      end
+
+      should "read length is a length is given" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.gets(1000)
+        assert_equal string.length, 1000
+        bytes = 0
+        bytes += string.length
+        while string = file.gets(1000)
+          bytes += string.length
+        end
+        assert_equal bytes, 1_000_000
+      end
+
+      should "read to the end of the line by default and assign to $_" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.gets
+        assert_equal 10, string.length
+      end
+
+      should "read to the end of the file one line at a time" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        bytes = 0
+        while string = file.gets
+          bytes += string.length
+        end
+        assert_equal 1_000_000, bytes
+      end
+
+      should "read to the end of the file one multi-character separator at a time" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        bytes = 0
+        while string = file.gets("45")
+          bytes += string.length
+        end
+        assert_equal 1_000_000, bytes
+      end
+
+      should "read to a given separator" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.gets("5")
+        assert_equal 6, string.length
+      end
+
+      should "read a multi-character separator" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.gets("45")
+        assert_equal 6, string.length
+        string = file.gets("45")
+        assert_equal "678\n012345", string
+        string = file.gets("\n01")
+        assert_equal "678\n01", string
+      end
+
+      should "read a mult-character separator with a length" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.gets("45", 3)
+        assert_equal 3, string.length
+      end
+
+      should "tell position, eof, and rewind" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        string = file.read(1000)
+        assert_equal 1000, file.pos
+        assert !file.eof?
+        file.read
+        assert file.eof?
+        file.rewind
+        assert_equal 0, file.pos
+        assert_equal 1_000_000, file.read.length
+      end
+    end
+
+    context "Seeking" do
+      setup do
+        @filename = 'test'
+        @mode     = 'w'
+        @data     = "1" * 1024 * 1024
+        @file = GridIO.new(@files, @chunks, @filename, @mode)
+        @file.write(@data)
+        @file.close
+      end
+
+      should "read all data using read_length and then be able to seek" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        assert_equal @data, file.read(1024 * 1024)
+        file.seek(0)
+        assert_equal @data, file.read
+      end
+
+      should "read all data using read_all and then be able to seek" do
+        file = GridIO.new(@files, @chunks, nil, "r", :query => {:_id => @file.files_id})
+        assert_equal @data, file.read
+        file.seek(0)
+        assert_equal @data, file.read
+        file.seek(1024 * 512)
+        assert_equal 524288, file.file_position
+        assert_equal @data.length / 2, file.read.length
+        assert_equal 1048576, file.file_position
+        assert_nil file.read
+        file.seek(1024 * 512)
+        assert_equal 524288, file.file_position
+      end
+
+    end
+
+    context "Grid MD5 check" do
+      should "run in safe mode" do
+        file = GridIO.new(@files, @chunks, 'smallfile', 'w', :safe => true)
+        file.write("DATA" * 100)
+        assert file.close
+        assert_equal file.server_md5, file.client_md5
+      end
+
+      should "validate with a large file" do
+        io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
+        file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
+        file.write(io)
+        assert file.close
+        assert_equal file.server_md5, file.client_md5
+      end
+
+      should "raise an exception when check fails" do
+        io = File.open(File.join(File.dirname(__FILE__), 'data', 'sample_file.pdf'), 'r')
+        @db.stubs(:command).returns({'md5' => '12345'})
+        file = GridIO.new(@files, @chunks, 'bigfile', 'w', :safe => true)
+        file.write(io)
+        assert_raise GridMD5Failure do
+          assert file.close
+        end
+        assert_not_equal file.server_md5, file.client_md5
+      end
+    end
+
+    context "Content types" do
+      if defined?(MIME)
+        should "determine common content types from the extension" do
+          file = GridIO.new(@files, @chunks, 'sample.pdf', 'w')
+          assert_equal 'application/pdf', file.content_type
+
+          file = GridIO.new(@files, @chunks, 'sample.txt', 'w')
+          assert_equal 'text/plain', file.content_type
+        end
+      end
+
+      should "default to binary/octet-stream when type is unknown" do
+        file = GridIO.new(@files, @chunks, 'sample.l33t', 'w')
+        assert_equal 'binary/octet-stream', file.content_type
+      end
+
+      should "use any provided content type by default" do
+        file = GridIO.new(@files, @chunks, 'sample.l33t', 'w', :content_type => 'image/jpg')
+        assert_equal 'image/jpg', file.content_type
+      end
+    end
+  end
+
+end

data/test/grid_test.rb

@@ -0,0 +1,259 @@
+__END__
+require './test/test_helper'
+include Mongo
+
+def read_and_write_stream(filename, read_length, opts={})
+  io   = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
+  id   = @grid.put(io, opts.merge!(:filename => filename + read_length.to_s))
+  file = @grid.get(id)
+  io.rewind
+  data = io.read
+  if data.respond_to?(:force_encoding)
+    data.force_encoding("binary")
+  end
+  read_data = ""
+  while(chunk = file.read(read_length))
+    read_data << chunk
+    break if chunk.empty?
+  end
+  assert_equal data.length, read_data.length
+end
+
+class GridTest < Test::Unit::TestCase
+  context "Tests:" do
+    setup do
+      @db = standard_connection.db(MONGO_TEST_DB)
+      @files  = @db.collection('test-fs.files')
+      @chunks = @db.collection('test-fs.chunks')
+    end
+
+    teardown do
+      @files.remove
+      @chunks.remove
+    end
+
+    context "A one-chunk grid-stored file" do
+      setup do
+        @data = "GRIDDATA" * 5
+        @grid = Grid.new(@db, 'test-fs')
+        @id   = @grid.put(@data, :filename => 'sample',
+                          :metadata => {'app' => 'photos'})
+      end
+
+      should "retrieve the file" do
+        data = @grid.get(@id).data
+        assert_equal @data, data
+      end
+
+    end
+
+    context "A basic grid-stored file" do
+      setup do
+        @data = "GRIDDATA" * 50000
+        @grid = Grid.new(@db, 'test-fs')
+        @id   = @grid.put(@data, :filename => 'sample',
+                          :metadata => {'app' => 'photos'})
+      end
+
+      should "check existence" do
+        file = @grid.exist?(:filename => 'sample')
+        assert_equal 'sample', file['filename']
+      end
+
+      should "not be able to overwrite an exising file" do
+        assert_raise GridError do
+          @grid.put(@data, :filename => 'sample', :_id => @id, :safe => true)
+        end
+      end
+
+      should "return nil if it doesn't exist" do
+        assert_nil @grid.exist?(:metadata => 'foo')
+      end
+
+      should "retrieve the stored data" do
+        data = @grid.get(@id).data
+        assert_equal @data.length, data.length
+      end
+
+      should "have a unique index on chunks" do
+        assert @chunks.index_information['files_id_1_n_1']['unique']
+      end
+
+      should "store the filename" do
+        file = @grid.get(@id)
+        assert_equal 'sample', file.filename
+      end
+
+      should "store any relevant metadata" do
+        file = @grid.get(@id)
+        assert_equal 'photos', file.metadata['app']
+      end
+
+      should "delete the file and any chunks" do
+        @grid.delete(@id)
+        assert_raise GridFileNotFound do
+          @grid.get(@id)
+        end
+        assert_equal nil, @db['test-fs']['chunks'].find_one({:files_id => @id})
+      end
+    end
+
+    context "Filename not required" do
+      setup do
+        @data = "GRIDDATA" * 50000
+        @grid = Grid.new(@db, 'test-fs')
+        @metadata = {'app' => 'photos'}
+      end
+
+      should "store the file with the old filename api" do
+        id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
+        file = @grid.get(id)
+        assert_equal 'sample', file.filename
+        assert_equal @metadata, file.metadata
+      end
+
+      should "store without a filename" do
+        id = @grid.put(@data, :metadata => @metadata)
+        file = @grid.get(id)
+        assert_nil file.filename
+        file_doc = @files.find_one({'_id' => id})
+        assert !file_doc.has_key?('filename')
+        assert_equal @metadata, file.metadata
+      end
+
+      should "store with filename and metadata with the new api" do
+        id = @grid.put(@data, :filename => 'sample', :metadata => @metadata)
+        file = @grid.get(id)
+        assert_equal 'sample', file.filename
+        assert_equal @metadata, file.metadata
+      end
+    end
+
+    context "Writing arbitrary data fields" do
+      setup do
+        @data = "GRIDDATA" * 50000
+        @grid = Grid.new(@db, 'test-fs')
+      end
+
+      should "write random keys to the files collection" do
+        id = @grid.put(@data, :phrases => ["blimey", "ahoy!"])
+        file = @grid.get(id)
+
+        assert_equal ["blimey", "ahoy!"], file['phrases']
+      end
+
+      should "ignore special keys" do
+        id = @grid.put(@data, :file_length => 100, :phrase => "blimey")
+        file = @grid.get(id)
+
+        assert_equal "blimey", file['phrase']
+        assert_equal 400_000, file.file_length
+      end
+    end
+
+    context "Storing data with a length of zero" do
+      setup do
+        @grid = Grid.new(@db, 'test-fs')
+        @id   = @grid.put('', :filename => 'sample',
+                          :metadata => {'app' => 'photos'})
+      end
+
+      should "return the zero length" do
+        data = @grid.get(@id)
+        assert_equal 0, data.read.length
+      end
+    end
+
+    context "Grid streaming: " do
+      setup do
+        @grid = Grid.new(@db, 'test-fs')
+        filename = 'sample_data'
+        @io   = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
+        id    = @grid.put(@io, :filename => filename)
+        @file = @grid.get(id)
+        @io.rewind
+        @data = @io.read
+        if @data.respond_to?(:force_encoding)
+          @data.force_encoding("binary")
+        end
+      end
+
+      should "be equal in length" do
+        @io.rewind
+        assert_equal @io.read.length, @file.read.length
+      end
+
+      should "read the file" do
+        read_data = ""
+        @file.each do |chunk|
+          read_data << chunk
+        end
+        assert_equal @data.length, read_data.length
+      end
+
+      should "read the file if no block is given" do
+        read_data = @file.each
+        assert_equal @data.length, read_data.length
+      end
+    end
+
+    context "Grid streaming an empty file: " do
+      setup do
+        @grid = Grid.new(@db, 'test-fs')
+        filename = 'empty_data'
+        @io   = File.open(File.join(File.dirname(__FILE__), 'data', filename), 'r')
+        id    = @grid.put(@io, :filename => filename)
+        @file = @grid.get(id)
+        @io.rewind
+        @data = @io.read
+        if @data.respond_to?(:force_encoding)
+          @data.force_encoding("binary")
+        end
+      end
+
+      should "be equal in length" do
+        @io.rewind
+        assert_equal @io.read.length, @file.read.length
+      end
+
+      should "read the file" do
+        read_data = ""
+        @file.each do |chunk|
+          read_data << chunk
+        end
+        assert_equal @data.length, read_data.length
+      end
+
+      should "read the file if no block is given" do
+        read_data = @file.each
+        assert_equal @data.length, read_data.length
+      end
+    end
+
+    context "Streaming: " do || {}
+      setup do
+        @grid = Grid.new(@db, 'test-fs')
+      end
+
+      should "put and get a small io object with a small chunk size" do
+        read_and_write_stream('small_data.txt', 1, :chunk_size => 2)
+      end
+
+      should "put and get an empty io object" do
+        read_and_write_stream('empty_data', 1)
+      end
+
+      should "put and get a small io object" do
+        read_and_write_stream('small_data.txt', 1)
+      end
+
+      should "put and get a large io object if reading less than the chunk size" do
+        read_and_write_stream('sample_data', 256 * 1024)
+      end
+
+      should "put and get a large io object if reading more than the chunk size" do
+        read_and_write_stream('sample_data', 300 * 1024)
+      end
+    end
+  end
+end