aliyun-sdk 0.1.1

data/spec/aliyun/oss/client/resumable_download_spec.rb

@@ -0,0 +1,217 @@
+# -*- encoding: utf-8 -*-
+
+require 'spec_helper'
+require 'yaml'
+require 'nokogiri'
+
+module Aliyun
+  module OSS
+
+    describe "Resumable download" do
+
+      before :all do
+        @endpoint = 'oss-cn-hangzhou.aliyuncs.com'
+        @bucket_name = 'rubysdk-bucket'
+        @object_key = 'resumable_file'
+        @bucket = Client.new(
+          :endpoint => @endpoint,
+          :access_key_id => 'xxx',
+          :access_key_secret => 'yyy').get_bucket(@bucket_name)
+
+        @file = './download_file'
+      end
+
+      before :each do
+        File.delete("#{@file}.cpt") if File.exist?("#{@file}.cpt")
+      end
+
+      def object_url
+        "#{@bucket_name}.#{@endpoint}/#{@object_key}"
+      end
+
+      def mock_object(i)
+        i.to_s.rjust(9, '0') + "\n"
+      end
+
+      def mock_range(i)
+        "bytes=#{(i-1)*10}-#{i*10 - 1}"
+      end
+
+      def mock_error(code, message)
+        Nokogiri::XML::Builder.new do |xml|
+          xml.Error {
+            xml.Code code
+            xml.Message message
+            xml.RequestId '0000'
+          }
+        end.to_xml
+      end
+
+      it "should download file when all goes well" do
+        return_headers = {
+          'x-oss-object-type' => 'Normal',
+          'ETag' => 'xxxyyyzzz',
+          'Content-Length' => 100,
+          'Last-Modified' => Time.now.rfc822
+        }
+
+        # get object meta
+        stub_request(:head, object_url).to_return(:headers => return_headers)
+
+        # get object by range
+        stub_request(:get, object_url)
+          .to_return((1..10).map{ |i| {:body => mock_object(i)} })
+
+        prg = []
+        @bucket.resumable_download(
+          @object_key, @file, :part_size => 10) { |p| prg << p }
+
+        ranges = []
+        expect(WebMock).to have_requested(:get, object_url).with{ |req|
+          ranges << req.headers['Range']
+        }.times(10)
+
+        expect(ranges).to match_array((1..10).map{ |i| mock_range(i) })
+        expect(File.exist?("#{@file}.cpt")).to be false
+        expect(Dir.glob("#{@file}.part.*").empty?).to be true
+
+        expect(File.read(@file)).to eq((1..10).map{ |i| mock_object(i) }.join)
+        expect(prg).to match_array((1..10).map {|i| i.to_f / 10 })
+      end
+
+      it "should resume when download part fails" do
+        return_headers = {
+          'x-oss-object-type' => 'Normal',
+          'ETag' => 'xxxyyyzzz',
+          'Content-Length' => 100,
+          'Last-Modified' => Time.now.rfc822
+        }
+
+        # get object meta
+        stub_request(:head, object_url).to_return(:headers => return_headers)
+
+        code = 'Timeout'
+        message = 'Request timeout.'
+        # upload part
+        stub_request(:get, object_url)
+          .to_return((1..3).map{ |i| {:body => mock_object(i)} }).then
+          .to_return(:status => 500, :body => mock_error(code, message)).times(2).then
+          .to_return((4..9).map{ |i| {:body => mock_object(i)} }).then
+          .to_return(:status => 500, :body => mock_error(code, message)).then
+          .to_return((10..10).map{ |i| {:body => mock_object(i)} })
+
+        success = false
+        4.times do
+          begin
+            @bucket.resumable_download(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+        ranges = []
+        expect(WebMock).to have_requested(:get, object_url).with{ |req|
+          ranges << req.headers['Range']
+        }.times(13)
+
+        expect(ranges.uniq).to match_array((1..10).map{ |i| mock_range(i) })
+        expect(File.read(@file)).to eq((1..10).map{ |i| mock_object(i) }.join)
+      end
+
+      it "should resume when checkpoint fails" do
+        # Monkey patch to inject failures
+        class ::Aliyun::OSS::Multipart::Download
+          alias :old_checkpoint :checkpoint
+
+          def checkpoint_fails
+            @@fail_injections ||= [false, false, true, true, false, true, false]
+            @@fail_injections.shift
+          end
+
+          def checkpoint
+            t = checkpoint_fails
+            if t == true
+              raise ClientError.new("fail injection")
+            end
+
+            old_checkpoint
+          end
+        end
+
+        return_headers = {
+          'x-oss-object-type' => 'Normal',
+          'ETag' => 'xxxyyyzzz',
+          'Content-Length' => 100,
+          'Last-Modified' => Time.now.rfc822
+        }
+
+        # get object meta
+        stub_request(:head, object_url).to_return(:headers => return_headers)
+
+        # get object by range
+        returns = [1, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        stub_request(:get, object_url)
+          .to_return(returns.map{ |i| {:body => mock_object(i)} })
+
+        success = false
+        4.times do
+          begin
+            @bucket.resumable_download(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+        expect(WebMock).to have_requested(:get, object_url).times(13)
+        expect(File.read(@file)).to eq((1..10).map{ |i| mock_object(i) }.join)
+      end
+
+      it "should not resume when specify disable_cpt" do
+        return_headers = {
+          'x-oss-object-type' => 'Normal',
+          'ETag' => 'xxxyyyzzz',
+          'Content-Length' => 100,
+          'Last-Modified' => Time.now.rfc822
+        }
+
+        # get object meta
+        stub_request(:head, object_url).to_return(:headers => return_headers)
+
+        code = 'Timeout'
+        message = 'Request timeout.'
+        # upload part
+        stub_request(:get, object_url)
+          .to_return((1..3).map{ |i| {:body => mock_object(i)} }).then
+          .to_return(:status => 500, :body => mock_error(code, message)).times(2).then
+          .to_return((1..9).map{ |i| {:body => mock_object(i)} }).then
+          .to_return(:status => 500, :body => mock_error(code, message)).then
+          .to_return((1..10).map{ |i| {:body => mock_object(i)} })
+
+        cpt_file = "#{File.expand_path(@file)}.cpt"
+        success = false
+        4.times do
+          begin
+            @bucket.resumable_download(
+              @object_key, @file, :part_size => 10,
+              :cpt_file => cpt_file, :disable_cpt => true)
+            success = true
+          rescue
+            # pass
+          end
+
+          expect(File.exists?(cpt_file)).to be false
+        end
+
+        expect(success).to be true
+        expect(WebMock).to have_requested(:get, object_url).times(25)
+        expect(File.read(@file)).to eq((1..10).map{ |i| mock_object(i) }.join)
+        expect(Dir.glob("#{@file}.part.*").empty?).to be true
+      end
+
+    end # Resumable upload
+  end # OSS
+end # Aliyun

data/spec/aliyun/oss/client/resumable_upload_spec.rb

@@ -0,0 +1,318 @@
+# -*- encoding: utf-8 -*-
+
+require 'spec_helper'
+require 'yaml'
+require 'nokogiri'
+
+module Aliyun
+  module OSS
+
+    describe "Resumable upload" do
+
+      before :all do
+        @endpoint = 'oss-cn-hangzhou.aliyuncs.com'
+        @bucket_name = 'rubysdk-bucket'
+        @object_key = 'resumable_file'
+        @bucket = Client.new(
+          :endpoint => @endpoint,
+          :access_key_id => 'xxx',
+          :access_key_secret => 'yyy').get_bucket(@bucket_name)
+
+        @file = './file_to_upload'
+        # write 100B data
+        File.open(@file, 'w') do |f|
+          (1..10).each do |i|
+            f.puts i.to_s.rjust(9, '0')
+          end
+        end
+      end
+
+      before :each do
+        File.delete("#{@file}.cpt") if File.exist?("#{@file}.cpt")
+      end
+
+      def object_url
+        "#{@bucket_name}.#{@endpoint}/#{@object_key}"
+      end
+
+      def parse_query_from_uri(uri)
+        query = {}
+        str = uri.to_s[uri.to_s.index('?')+1..-1]
+        str.split("&").each do |q|
+          v = q.split('=')
+          query[v.at(0)] = v.at(1)
+        end
+
+        query
+      end
+
+      def mock_txn_id(txn_id)
+        Nokogiri::XML::Builder.new do |xml|
+          xml.InitiateMultipartUploadResult {
+            xml.Bucket @bucket
+            xml.Key @object
+            xml.UploadId txn_id
+          }
+        end.to_xml
+      end
+
+      def mock_error(code, message)
+        Nokogiri::XML::Builder.new do |xml|
+          xml.Error {
+            xml.Code code
+            xml.Message message
+            xml.RequestId '0000'
+          }
+        end.to_xml
+      end
+
+      it "should upload file when all goes well" do
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:body => mock_txn_id('upload_id'))
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+
+        prg = []
+        @bucket.resumable_upload(
+          @object_key, @file, :part_size => 10) { |p| prg << p }
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(1)
+
+        part_numbers = Set.new([])
+        upload_ids = Set.new([])
+
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).with{ |req|
+          query = parse_query_from_uri(req.uri)
+          part_numbers << query['partNumber']
+          upload_ids << query['uploadId']
+        }.times(10)
+
+        expect(part_numbers.to_a).to match_array((1..10).map{ |x| x.to_s })
+        expect(upload_ids.to_a).to match_array(['upload_id'])
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).times(1)
+
+        expect(File.exist?("#{@file}.cpt")).to be false
+        expect(prg).to match_array((1..10).map {|i| i.to_f / 10 })
+      end
+
+      it "should restart when begin txn fails" do
+        code = 'Timeout'
+        message = 'Request timeout.'
+
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:status => 500, :body => mock_error(code, message)).then
+          .to_return(:body => mock_txn_id('upload_id'))
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+
+        success = false
+        2.times do
+          begin
+            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(2)
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).times(10)
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).times(1)
+      end
+
+      it "should resume when upload part fails" do
+        # begin multipart
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:body => mock_txn_id('upload_id'))
+
+        # commit multipart
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+
+        code = 'Timeout'
+        message = 'Request timeout.'
+        # upload part
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+          .to_return(:status => 200).times(3).then
+          .to_return(:status => 500, :body => mock_error(code, message)).times(2).then
+          .to_return(:status => 200).times(6).then
+          .to_return(:status => 500, :body => mock_error(code, message)).then
+          .to_return(:status => 200)
+
+        success = false
+        4.times do
+          begin
+            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(1)
+
+        part_numbers = Set.new([])
+        upload_ids = Set.new([])
+
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).with{ |req|
+          query = parse_query_from_uri(req.uri)
+          part_numbers << query['partNumber']
+          upload_ids << query['uploadId']
+        }.times(13)
+
+        expect(part_numbers.to_a).to match_array((1..10).map{ |x| x.to_s })
+        expect(upload_ids.to_a).to match_array(['upload_id'])
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).times(1)
+      end
+
+      it "should resume when checkpoint fails" do
+        # Monkey patch to inject failures
+        class ::Aliyun::OSS::Multipart::Upload
+          alias :old_checkpoint :checkpoint
+
+          def checkpoint_fails
+            @@fail_injections ||= [false, false, true, true, false, true, false]
+            @@fail_injections.shift
+          end
+
+          def checkpoint
+            t = checkpoint_fails
+            if t == true
+              raise ClientError.new("fail injection")
+            end
+
+            old_checkpoint
+          end
+        end
+
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:body => mock_txn_id('upload_id'))
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+
+        success = false
+        4.times do
+          begin
+            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(1)
+
+        part_numbers = Set.new([])
+
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).with{ |req|
+          query = parse_query_from_uri(req.uri)
+          part_numbers << query['partNumber']
+          query['uploadId'] == 'upload_id'
+        }.times(13)
+
+        expect(part_numbers.to_a).to match_array((1..10).map{ |x| x.to_s })
+
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).times(1)
+      end
+
+      it "should resume when commit txn fails" do
+        # begin multipart
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:body => mock_txn_id('upload_id'))
+
+        # upload part
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+
+        code = 'Timeout'
+        message = 'Request timeout.'
+        # commit multipart
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+          .to_return(:status => 500, :body => mock_error(code, message)).times(2).then
+          .to_return(:status => 200)
+
+        success = false
+        3.times do
+          begin
+            @bucket.resumable_upload(@object_key, @file, :part_size => 10)
+            success = true
+          rescue
+            # pass
+          end
+        end
+
+        expect(success).to be true
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(1)
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).times(10)
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).with{ |req|
+          query = parse_query_from_uri(req.uri)
+          query['uploadId'] == 'upload_id'
+        }.times(3)
+      end
+
+      it "should not write checkpoint when specify disable_cpt" do
+        # begin multipart
+        stub_request(:post, /#{object_url}\?uploads.*/)
+          .to_return(:body => mock_txn_id('upload_id'))
+
+        # upload part
+        stub_request(:put, /#{object_url}\?partNumber.*/)
+
+        code = 'Timeout'
+        message = 'Request timeout.'
+        # commit multipart
+        stub_request(:post, /#{object_url}\?uploadId.*/)
+          .to_return(:status => 500, :body => mock_error(code, message)).times(2).then
+          .to_return(:status => 200)
+
+        cpt_file = "#{File.expand_path(@file)}.cpt"
+        success = false
+        3.times do
+          begin
+            @bucket.resumable_upload(
+              @object_key, @file, :part_size => 10,
+              :cpt_file => cpt_file, :disable_cpt => true)
+            success = true
+          rescue
+            # pass
+          end
+
+          expect(File.exists?(cpt_file)).to be false
+        end
+
+        expect(success).to be true
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploads.*/).times(3)
+        expect(WebMock).to have_requested(
+          :put, /#{object_url}\?partNumber.*/).times(30)
+        expect(WebMock).to have_requested(
+          :post, /#{object_url}\?uploadId.*/).with{ |req|
+          query = parse_query_from_uri(req.uri)
+          query['uploadId'] == 'upload_id'
+        }.times(3)
+      end
+
+    end # Resumable upload
+
+  end # OSS
+end # Aliyun