aliyun-sdk 0.1.3 → 0.1.4

data/spec/aliyun/oss/client/resumable_download_spec.rb

@@ -75,8 +75,9 @@ module Aliyun
         expect(File.exist?("#{@file}.cpt")).to be false
         expect(Dir.glob("#{@file}.part.*").empty?).to be true
 
-        expect(File.read(@file)).to eq((1..10).map{ |i| mock_object(i) }.join)
-        expect(prg).to match_array((1..10).map {|i| i.to_f / 10 })
+        expect(File.read(@file).lines)
+          .to match_array((1..10).map{ |i| mock_object(i) })
+        expect(prg.size).to eq(10)
       end
 
       it "should resume when download part fails" do
@@ -103,7 +104,8 @@ module Aliyun
         success = false
         4.times do
           begin
-            @bucket.resumable_download(@object_key, @file, :part_size => 10)
+            @bucket.resumable_download(
+              @object_key, @file, :part_size => 10, :threads => 1)
             success = true
           rescue
             # pass
@@ -158,7 +160,8 @@ module Aliyun
         success = false
         4.times do
           begin
-            @bucket.resumable_download(@object_key, @file, :part_size => 10)
+            @bucket.resumable_download(
+              @object_key, @file, :part_size => 10, :threads => 1)
             success = true
           rescue
             # pass
@@ -197,7 +200,7 @@ module Aliyun
           begin
             @bucket.resumable_download(
               @object_key, @file, :part_size => 10,
-              :cpt_file => cpt_file, :disable_cpt => true)
+              :cpt_file => cpt_file, :disable_cpt => true, :threads => 1)
             success = true
           rescue
             # pass

data/spec/aliyun/oss/client/resumable_upload_spec.rb

@@ -96,7 +96,7 @@ module Aliyun
           :post, /#{object_url}\?uploadId.*/).times(1)
 
         expect(File.exist?("#{@file}.cpt")).to be false
-        expect(prg).to match_array((1..10).map {|i| i.to_f / 10 })
+        expect(prg.size).to eq(10)
       end
 
       it "should restart when begin txn fails" do
@@ -149,7 +149,8 @@ module Aliyun
         success = false
         4.times do
           begin
-            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            @bucket.resumable_upload(
+              @object_key, @file, part_size: 10, threads: 1)
             success = true
           rescue
             # pass
@@ -206,7 +207,8 @@ module Aliyun
         success = false
         4.times do
           begin
-            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            @bucket.resumable_upload(
+              @object_key, @file, part_size: 10, threads: 1)
             success = true
           rescue
             # pass
@@ -251,7 +253,8 @@ module Aliyun
         success = false
         3.times do
           begin
-            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            @bucket.resumable_upload(
+              @object_key, @file, part_size: 10, threads: 1)
             success = true
           rescue
             # pass

data/spec/aliyun/oss/multipart_spec.rb

@@ -38,7 +38,6 @@ module Aliyun
           xml.ListMultipartUploadsResult {
             {
               :prefix => 'Prefix',
-              :delimiter => 'Delimiter',
               :limit => 'MaxUploads',
               :id_marker => 'UploadIdMarker',
               :next_id_marker => 'NextUploadIdMarker',
@@ -414,7 +413,6 @@ module Aliyun
           query = {
             'uploads' => '',
             'prefix' => 'foo-',
-            'delimiter' => '-',
             'upload-id-marker' => 'id-marker',
             'key-marker' => 'key-marker',
             'max-uploads' => 10,
@@ -426,7 +424,6 @@ module Aliyun
           @protocol.list_multipart_uploads(
             @bucket,
             :prefix => 'foo-',
-            :delimiter => '-',
             :id_marker => 'id-marker',
             :key_marker => 'key-marker',
             :limit => 10,
@@ -442,7 +439,6 @@ module Aliyun
           query = {
             'uploads' => '',
             'prefix' => 'foo-',
-            'delimiter' => '-',
             'upload-id-marker' => 'id-marker',
             'key-marker' => 'key-marker',
             'max-uploads' => 100,
@@ -459,7 +455,6 @@ module Aliyun
 
           return_more = {
             :prefix => 'foo-',
-            :delimiter => '-',
             :id_marker => 'id-marker',
             :key_marker => 'key-marker',
             :next_id_marker => 'next-id-marker',
@@ -474,7 +469,6 @@ module Aliyun
           txns, more = @protocol.list_multipart_uploads(
                   @bucket,
                   :prefix => 'foo-',
-                  :delimiter => '-',
                   :id_marker => 'id-marker',
                   :key_marker => 'key-marker',
                   :limit => 100,
@@ -492,7 +486,6 @@ module Aliyun
           query = {
             'uploads' => '',
             'prefix' => 'foo-',
-            'delimiter' => '-',
             'upload-id-marker' => 'id-marker',
             'key-marker' => 'key-marker',
             'max-uploads' => 100,
@@ -516,7 +509,6 @@ module Aliyun
 
           return_more = {
             :prefix => 'foo-',
-            :delimiter => '中国のruby',
             :id_marker => 'id-marker',
             :key_marker => '杭州のruby',
             :next_id_marker => 'next-id-marker',
@@ -528,7 +520,6 @@ module Aliyun
 
           es_more = {
             :prefix => 'foo-',
-            :delimiter => CGI.escape('中国のruby'),
             :id_marker => 'id-marker',
             :key_marker => CGI.escape('杭州のruby'),
             :next_id_marker => 'next-id-marker',
@@ -545,7 +536,6 @@ module Aliyun
           txns, more = @protocol.list_multipart_uploads(
                   @bucket,
                   :prefix => 'foo-',
-                  :delimiter => '-',
                   :id_marker => 'id-marker',
                   :key_marker => 'key-marker',
                   :limit => 100,

data/spec/aliyun/oss/object_spec.rb

@@ -54,16 +54,15 @@ module Aliyun
       end
 
       def mock_delete(objects, opts = {})
-        Nokogiri::XML::Builder.new do |xml|
-          xml.Delete {
-            xml.Quiet opts[:quiet]? true : false
-            objects.each do |o|
-              xml.Object {
-                xml.Key o
-              }
-            end
-          }
-        end.to_xml
+        # It may have invisible chars in object key which will corrupt
+        # libxml. So we're constructing xml body manually here.
+        body = '<?xml version="1.0"?>'
+        body << '<Delete>'
+        body << '<Quiet>' << (opts[:quiet]? true : false).to_s << '</Quiet>'
+        objects.each { |k|
+          body << '<Object><Key>' << k << '</Key></Object>'
+        }
+        body << '</Delete>'
       end
 
       def mock_delete_result(deleted, opts = {})

data/tests/test_content_type.rb

@@ -0,0 +1,100 @@
+require 'minitest/autorun'
+require 'yaml'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+
+class TestContentType < Minitest::Test
+  def setup
+    Aliyun::OSS::Logging.set_log_level(Logger::DEBUG)
+    conf_file = '~/.oss.yml'
+    conf = YAML.load(File.read(File.expand_path(conf_file)))
+    client = Aliyun::OSS::Client.new(
+      :endpoint => conf['endpoint'],
+      :cname => conf['cname'],
+      :access_key_id => conf['id'],
+      :access_key_secret => conf['key'])
+    @bucket = client.get_bucket(conf['bucket'])
+
+    @types = {
+      "html" => "text/html",
+      "js" => "application/javascript",
+      "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "xltx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+      "potx" => "application/vnd.openxmlformats-officedocument.presentationml.template",
+      "ppsx" => "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+      "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      "sldx" => "application/vnd.openxmlformats-officedocument.presentationml.slide",
+      "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      "dotx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+      "xlam" => "application/vnd.ms-excel.addin.macroEnabled.12",
+      "xlsb" => "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+      "apk" => "application/vnd.android.package-archive",
+      "" => "application/octet-stream"
+    }
+
+    @prefix = "tests/content_type/"
+  end
+
+  def get_key(p, k)
+    "#{@prefix}#{p}obj" + (k.empty? ? "" : ".#{k}")
+  end
+
+  def test_type_from_key
+    @types.each do |k, v|
+      key = get_key('from_key', k)
+      @bucket.put_object(key)
+      assert_equal v, @bucket.get_object(key).content_type
+
+      copy_key = get_key('copy.from_key', k)
+      @bucket.copy_object(key, copy_key)
+      assert_equal v, @bucket.get_object(copy_key).content_type
+
+      append_key = get_key('append.from_key', k)
+      @bucket.append_object(append_key, 0)
+      assert_equal v, @bucket.get_object(append_key).content_type
+    end
+  end
+
+  def test_type_from_file
+    @types.each do |k, v|
+      upload_file = "/tmp/upload_file"
+      upload_file += ".#{k}" unless k.empty?
+      `touch #{upload_file}`
+
+      key = get_key('from_file', k)
+      @bucket.put_object(key, :file => upload_file)
+      assert_equal v, @bucket.get_object(key).content_type
+
+      append_key = get_key('append.from_file', k)
+      @bucket.append_object(append_key, 0, :file => upload_file)
+      assert_equal v, @bucket.get_object(append_key).content_type
+
+      multipart_key = get_key('multipart.from_file', k)
+      @bucket.resumable_upload(multipart_key, upload_file)
+      assert_equal v, @bucket.get_object(multipart_key).content_type
+    end
+  end
+
+  def test_type_from_user
+    @types.each do |k, v|
+      upload_file = "/tmp/upload_file.html"
+      `touch #{upload_file}`
+
+      key = get_key('from_user', k)
+      @bucket.put_object(key, :file => upload_file, :content_type => v)
+      assert_equal v, @bucket.get_object(key).content_type
+
+      copy_key = get_key('copy.from_user', k)
+      @bucket.copy_object(key, copy_key, :content_type => v)
+      assert_equal v, @bucket.get_object(copy_key).content_type
+
+      append_key = get_key('append.from_user', k)
+      @bucket.append_object(append_key, 0, :file => upload_file, :content_type => v)
+      assert_equal v, @bucket.get_object(append_key).content_type
+
+      multipart_key = get_key('multipart.from_file', k)
+      @bucket.resumable_upload(multipart_key, upload_file, :content_type => v)
+      assert_equal v, @bucket.get_object(multipart_key).content_type
+    end
+  end
+end

data/tests/test_large_file.rb

@@ -0,0 +1,66 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'benchmark'
+require 'yaml'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+
+class TestObjectKey < Minitest::Test
+  def setup
+    conf_file = '~/.oss.yml'
+    conf = YAML.load(File.read(File.expand_path(conf_file)))
+    client = Aliyun::OSS::Client.new(
+      :endpoint => conf['endpoint'],
+      :cname => conf['cname'],
+      :access_key_id => conf['id'],
+      :access_key_secret => conf['key'])
+    @bucket = client.get_bucket(conf['bucket'])
+    @prefix = 'tests/large_file/'
+  end
+
+  def get_key(k)
+    @prefix + k
+  end
+
+  def test_large_file_1gb
+    key = get_key("large_file_1gb")
+    Benchmark.bm(32) do |bm|
+      bm.report("Upload with put_object: ") do
+        @bucket.put_object(key, :file => './large_file_1gb')
+      end
+
+      bm.report("Upload with resumable_upload: ") do
+        @bucket.resumable_upload(key, './large_file_1gb')
+      end
+
+      bm.report("Download with get_object: ") do
+        @bucket.get_object(key, :file => './large_file_1gb')
+      end
+
+      bm.report("Download with resumable_download: ") do
+        @bucket.resumable_download(key, './large_file_1gb')
+      end
+    end
+  end
+
+  def test_large_file_8gb
+    key = get_key("large_file_8gb")
+    Benchmark.bm(32) do |bm|
+      bm.report("Upload with put_object: ") do
+        @bucket.put_object(key, :file => './large_file_8gb')
+      end
+
+      bm.report("Upload with resumable_upload: ") do
+        @bucket.resumable_upload(key, './large_file_8gb')
+      end
+
+      bm.report("Download with get_object: ") do
+        @bucket.get_object(key, :file => './large_file_8gb')
+      end
+
+      bm.report("Download with resumable_download: ") do
+        @bucket.resumable_download(key, './large_file_8gb')
+      end
+    end
+  end
+end

data/tests/test_multipart.rb

@@ -0,0 +1,105 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'yaml'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+
+class TestMultipart < Minitest::Unit::TestCase
+  def setup
+    conf_file = '~/.oss.yml'
+    conf = YAML.load(File.read(File.expand_path(conf_file)))
+    opts = {
+      endpoint: conf['endpoint'],
+      cname: conf['cname'],
+      access_key_id: conf['id'],
+      access_key_secret: conf['key'],
+    }
+    client = Aliyun::OSS::Client.new(opts)
+    @bucket_name = conf['bucket']
+    @bucket = client.get_bucket(@bucket_name)
+    @protocol = Aliyun::OSS::Protocol.new(Aliyun::OSS::Config.new(opts))
+    @prefix = 'tests/multipart/'
+  end
+
+  def get_key(k)
+    @prefix + k
+  end
+
+  def clear_uploads
+    all = @bucket.list_uploads.to_a
+    all.each { |t| @bucket.abort_upload(t.id, t.object) }
+  end
+
+  def test_key_marker
+    clear_uploads
+
+    # initiate 5 uploads
+    ids = []
+    5.times { |i|
+      id = @protocol.initiate_multipart_upload(@bucket_name, get_key("obj-#{i}"))
+      ids << id
+    }
+
+    all = @bucket.list_uploads(limit: 1).to_a
+    assert_equal ids, all.map(&:id)
+
+    after_1 = @bucket.list_uploads(key_marker: get_key("obj-0")).to_a
+    assert_equal ids[1, 5], after_1.map(&:id)
+
+    after_5 = @bucket.list_uploads(key_marker: get_key("obj-4")).to_a
+    assert after_5.empty?, after_5.to_s
+  end
+
+  def test_id_marker
+    clear_uploads
+
+    # initiate 5 uploads
+    ids = []
+    5.times { |i|
+      id = @protocol.initiate_multipart_upload(@bucket_name, get_key("object"))
+      ids << id
+    }
+    ids.sort!
+
+    all = @bucket.list_uploads.to_a
+    assert_equal ids, all.map(&:id)
+
+    # id_marker is ignored
+    after_1 = @bucket.list_uploads(id_marker: ids[0]).to_a
+    assert_equal ids, after_1.map(&:id)
+
+    # id_marker is ignored
+    after_5 = @bucket.list_uploads(id_marker: ids[4]).to_a
+    assert_equal ids, after_5.map(&:id)
+  end
+
+  def test_id_key_marker
+    clear_uploads
+
+    # initiate 5 uploads
+    foo_ids = []
+    5.times { |i|
+      id = @protocol.initiate_multipart_upload(@bucket_name, get_key("foo"))
+      foo_ids << id
+    }
+    foo_ids.sort!
+
+    bar_ids = []
+    5.times { |i|
+      id = @protocol.initiate_multipart_upload(@bucket_name, get_key("bar"))
+      bar_ids << id
+    }
+    bar_ids.sort!
+
+    after_1 = @bucket.list_uploads(
+      id_marker: bar_ids[0], key_marker: get_key("bar"), limit: 1).to_a
+    assert_equal bar_ids[1, 5] + foo_ids, after_1.map(&:id)
+
+    after_5 = @bucket.list_uploads(
+      id_marker: bar_ids[4], key_marker: get_key("bar")).to_a
+    assert_equal foo_ids, after_5.map(&:id)
+  end
+
+  def test_prefix
+  end
+end