aliyun-sdk 0.4.1 → 0.5.0

data/lib/aliyun/oss/http.rb

@@ -43,10 +43,14 @@ module Aliyun
       # A stream is any class that responds to :read(bytes, outbuf)
       #
       class StreamWriter
-        def initialize
+        attr_reader :data_crc
+
+        def initialize(crc_enable = false, init_crc = 0)
           @buffer = ""
           @producer = Fiber.new { yield self if block_given? }
           @producer.resume
+          @data_crc = init_crc.to_i
+          @crc_enable = crc_enable
         end
 
         def read(bytes = nil, outbuf = nil)
@@ -84,6 +88,8 @@ module Aliyun
           # "". ios.read(positive-integer) returns nil.
           return nil if ret.empty? && !bytes.nil? && bytes > 0
 
+          @data_crc = Aliyun::OSS::Util.crc(ret, @data_crc) if @crc_enable
+
           ret
         end
 
@@ -115,8 +121,8 @@ module Aliyun
       #     net_http_do_request(http, req, payload ? payload.to_s : nil,
       #                     &@block_response)
       class StreamPayload
-        def initialize(&block)
-          @stream = StreamWriter.new(&block)
+        def initialize(crc_enable = false, init_crc = 0, &block)
+          @stream = StreamWriter.new(crc_enable, init_crc, &block)
         end
 
         def read(bytes = nil)
@@ -129,7 +135,6 @@ module Aliyun
         def closed?
           false
         end
-
       end
 
       include Common::Logging

data/lib/aliyun/oss/multipart.rb

@@ -30,7 +30,7 @@ module Aliyun
           md5= Util.get_content_md5(states.to_json)
 
           @mutex.synchronize {
-            File.open(file, 'w') {
+            File.open(file, 'wb') {
               |f| f.write(states.merge(md5: md5).to_json)
             }
           }

data/lib/aliyun/oss/protocol.rb

@@ -7,6 +7,7 @@ require 'time'
 module Aliyun
   module OSS
 
+
     ##
     # Protocol implement the OSS Open API which is low-level. User
     # should refer to {OSS::Client} for normal use.
@@ -534,9 +535,10 @@ module Aliyun
           headers[CALLBACK_HEADER] = opts[:callback].serialize
         end
 
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable, opts[:init_crc], &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name},
-          {:headers => headers, :body => HTTP::StreamPayload.new(&block)})
+          {:headers => headers, :body => payload})
 
         if r.code == 203
           e = CallbackError.new(r)
@@ -544,6 +546,11 @@ module Aliyun
           raise e
         end
 
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
+          data_crc = payload.read.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
+        end
+
         logger.debug('Done put object')
       end
 
@@ -586,9 +593,18 @@ module Aliyun
 
         headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers)
 
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable && !opts[:init_crc].nil?, opts[:init_crc], &block)
+
         r = @http.post(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
-          {:headers => headers, :body => HTTP::StreamPayload.new(&block)})
+          {:headers => headers, :body => payload})
+
+        if @config.upload_crc_enable && 
+          !r.headers[:x_oss_hash_crc64ecma].nil? && 
+          !opts[:init_crc].nil?
+          data_crc = payload.read.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'append')
+        end
 
         logger.debug('Done append object')
 
@@ -759,11 +775,22 @@ module Aliyun
             rewrites[:expires].httpdate if rewrites.key?(:expires)
         end
 
+        data_crc = opts[:init_crc].nil? ? 0 : opts[:init_crc]
         r = @http.get(
           {:bucket => bucket_name, :object => object_name,
            :sub_res => sub_res},
           {:headers => headers}
-        ) { |chunk| yield chunk if block_given? }
+        ) do |chunk| 
+          if block_given?
+            # crc enable and no range and oss server support crc
+            data_crc = Aliyun::OSS::Util.crc(chunk, data_crc) if @config.download_crc_enable && range.nil?
+            yield chunk
+          end
+        end
+
+        if @config.download_crc_enable && range.nil? && !r.headers[:x_oss_hash_crc64ecma].nil?
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'get')
+        end
 
         h = r.headers
         metas = {}
@@ -1082,9 +1109,16 @@ module Aliyun
                      "#{object_name}, txn id: #{txn_id}, part No: #{part_no}")
 
         sub_res = {'partNumber' => part_no, 'uploadId' => txn_id}
+
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable, &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
-          {:body => HTTP::StreamPayload.new(&block)})
+          {:body => payload})
+
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
+          data_crc = payload.read.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
+        end
 
         logger.debug("Done upload part")
 
@@ -1378,6 +1412,18 @@ module Aliyun
         Util.sign(@config.access_key_secret, string_to_sign)
       end
 
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @config.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @config.upload_crc_enable
+      end
+
       private
 
       # Parse body content to xml document
@@ -1493,7 +1539,6 @@ module Aliyun
           result
         end
       end
-
     end # Protocol
   end # OSS
 end # Aliyun

data/lib/aliyun/oss/util.rb

@@ -7,6 +7,7 @@ require 'digest/md5'
 
 module Aliyun
   module OSS
+
     ##
     # Util functions to help generate formatted Date, signatures,
     # etc.
@@ -75,6 +76,23 @@ module Aliyun
           end
         end
 
+        # Get a crc value of the data
+        def crc(data, init_crc = 0)
+          CrcX::crc64(init_crc, data, data.size)
+        end
+
+        # Calculate a value of the crc1 combine with crc2. 
+        def crc_combine(crc1, crc2, len2)
+          CrcX::crc64_combine(crc1, crc2, len2)
+        end
+
+        def crc_check(crc_a, crc_b, operation)
+          if crc_a.nil? || crc_b.nil? || crc_a.to_i != crc_b.to_i
+            logger.error("The crc of #{operation} between client and oss is not inconsistent. crc_a=#{crc_a} crc_b=#{crc_b}")
+            fail CrcInconsistentError.new("The crc of #{operation} between client and oss is not inconsistent.")
+          end
+        end
+
       end # self
     end # Util
   end # OSS
@@ -86,4 +104,4 @@ class String
     return true if self =~ /^true$/i
     false
   end
-end
+end

data/lib/aliyun/version.rb

@@ -2,6 +2,6 @@
 
 module Aliyun
 
-    VERSION = "0.4.1"
+    VERSION = "0.5.0"
 
 end # Aliyun

data/spec/aliyun/oss/bucket_spec.rb

@@ -591,6 +591,60 @@ module Aliyun
 
       end # acl, logging, cors, etc
 
+      context "crc" do
+        it "should download crc enable equal config setting" do
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => 'true'))
+          expect(protocol.download_crc_enable).to eq(true)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => true))
+          expect(protocol.download_crc_enable).to eq(true)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => 'false'))
+          expect(protocol.download_crc_enable).to eq(false)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => false))
+          expect(protocol.download_crc_enable).to eq(false)
+        end
+
+        it "should upload crc enable equal config setting" do
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => 'true'))
+          expect(protocol.upload_crc_enable).to eq(true)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => true))
+          expect(protocol.upload_crc_enable).to eq(true)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => 'false'))
+          expect(protocol.upload_crc_enable).to eq(false)
+
+          protocol = Protocol.new(
+                      Config.new(:endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => false))
+          expect(protocol.upload_crc_enable).to eq(false)
+        end
+      end # crc
+
     end # Bucket
 
   end # OSS

data/spec/aliyun/oss/client/bucket_spec.rb

@@ -549,6 +549,72 @@ module Aliyun
         end
       end # multipart operations
 
+      context "crc" do
+        it "should download crc enable equal config setting" do
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => 'true').get_bucket(@bucket_name)
+          expect(bucket.download_crc_enable).to eq(true)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => true).get_bucket(@bucket_name)
+          expect(bucket.download_crc_enable).to eq(true)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => 'false').get_bucket(@bucket_name)
+          expect(bucket.download_crc_enable).to eq(false)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :download_crc_enable => false).get_bucket(@bucket_name)
+          expect(bucket.download_crc_enable).to eq(false)
+
+          # check default value
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy').get_bucket(@bucket_name)
+          expect(bucket.download_crc_enable).to eq(false)
+        end
+
+        it "should upload crc enable equal config setting" do
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => 'true').get_bucket(@bucket_name)
+          expect(bucket.upload_crc_enable).to eq(true)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => true).get_bucket(@bucket_name)
+          expect(bucket.upload_crc_enable).to eq(true)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => 'false').get_bucket(@bucket_name)
+          expect(bucket.upload_crc_enable).to eq(false)
+
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy',
+                      :upload_crc_enable => false).get_bucket(@bucket_name)
+          expect(bucket.upload_crc_enable).to eq(false)
+
+          # check default value
+          bucket = Client.new(
+                      :endpoint => @endpoint,
+                      :access_key_id => 'xxx', :access_key_secret => 'yyy').get_bucket(@bucket_name)
+          expect(bucket.upload_crc_enable).to eq(true)
+        end
+      end # crc
+
     end # Bucket
   end # OSS
 end # Aliyun

data/spec/aliyun/oss/http_spec.rb

@@ -76,6 +76,32 @@ module Aliyun
           r = s.read(1000)
           expect(r.size).to eq(64)
         end
+
+        it "should read with a correct crc" do
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          s = HTTP::StreamWriter.new(true) do |sr|
+            sr << content
+          end
+
+          r = s.read(content.size + 1)
+          expect(r.size).to eq(content.size)
+          expect(s.data_crc).to eq(content_crc)
+        end
+
+        it "should read with a correct crc when setting init_crc" do
+          content = "hello world"
+          init_crc = 100
+          content_crc = Aliyun::OSS::Util.crc(content, init_crc)
+
+          s = HTTP::StreamWriter.new(true, init_crc) do |sr|
+            sr << content
+          end
+
+          r = s.read(content.size + 1)
+          expect(r.size).to eq(content.size)
+          expect(s.data_crc).to eq(content_crc)
+        end
       end # StreamWriter
 
     end # HTTP

data/spec/aliyun/oss/multipart_spec.rb

@@ -23,6 +23,15 @@ module Aliyun
         "#{@bucket}.#{@endpoint}/#{@object}"
       end
 
+      def crc_protocol
+        Protocol.new(
+          Config.new(:endpoint => @endpoint,
+                     :access_key_id => 'xxx', 
+                     :access_key_secret => 'yyy', 
+                     :upload_crc_enable => true,
+                     :download_crc_enable => true))
+      end
+
       def mock_txn_id(txn_id)
         Nokogiri::XML::Builder.new do |xml|
           xml.InitiateMultipartUploadResult {
@@ -207,6 +216,42 @@ module Aliyun
             .with(:body => nil, :query => query)
         end
 
+        it "should raise crc exception on error" do
+          txn_id = 'xxxyyyzzz'
+          part_no = 1
+          query = {'partNumber' => part_no, 'uploadId' => txn_id}
+
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+
+          stub_request(:put, request_path).with(:query => query).to_return(
+            :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc + 1})
+
+          expect {
+            crc_protocol.upload_part(@bucket, @object, txn_id, part_no) do |body|
+              body << content
+            end
+          }.to raise_error(CrcInconsistentError, "The crc of put between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception on error" do
+          txn_id = 'xxxyyyzzz'
+          part_no = 1
+          query = {'partNumber' => part_no, 'uploadId' => txn_id}
+
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+
+          stub_request(:put, request_path).with(:query => query).to_return(
+            :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc})
+
+          expect {
+            crc_protocol.upload_part(@bucket, @object, txn_id, part_no) do |body|
+              body << content
+            end
+          }.not_to raise_error
+        end
+
       end # upload part
 
       context "Upload part by copy object" do

data/spec/aliyun/oss/object_spec.rb

@@ -17,6 +17,15 @@ module Aliyun
         @bucket = 'rubysdk-bucket'
       end
 
+      def crc_protocol
+        Protocol.new(
+          Config.new(:endpoint => @endpoint,
+                     :access_key_id => 'xxx', 
+                     :access_key_secret => 'yyy', 
+                     :upload_crc_enable => true,
+                     :download_crc_enable => true))
+      end
+
       def get_request_path(object = nil)
         p = "#{@bucket}.#{@endpoint}/"
         p += CGI.escape(object) if object
@@ -190,6 +199,35 @@ module Aliyun
                     'x-oss-meta-people' => 'mary'})
         end
 
+        it "should raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:put, url).to_return(
+            :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          expect(crc_protocol.upload_crc_enable).to eq(true)
+          expect {
+            crc_protocol.put_object(@bucket, object_name) do |c|
+              c << content
+            end
+          }.to raise_error(CrcInconsistentError, "The crc of put between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:put, url).to_return(
+            :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc})
+          expect(crc_protocol.upload_crc_enable).to eq(true)
+          expect {
+            crc_protocol.put_object(@bucket, object_name) do |c|
+              c << content
+            end
+          }.not_to raise_error
+        end
       end # put object
 
       context "Append object" do
@@ -290,6 +328,72 @@ module Aliyun
                                  'x-oss-meta-year' => '2015',
                                  'x-oss-meta-people' => 'mary'})
         end
+
+        it "should raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+
+          query = {'append' => '', 'position' => 11}
+          return_headers = {'x-oss-next-append-position' => '101', :x_oss_hash_crc64ecma => content_crc.to_i + 1}
+          stub_request(:post, url).with(:query => query)
+            .to_return(:headers => return_headers)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
+          expect {
+            crc_protocol.append_object(@bucket, object_name, 11, :init_crc => 0) do |c|
+              c << content
+            end
+          }.to raise_error(CrcInconsistentError, "The crc of append between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception with init_crc" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+
+          query = {'append' => '', 'position' => 11}
+          return_headers = {'x-oss-next-append-position' => '101', :x_oss_hash_crc64ecma => content_crc}
+          stub_request(:post, url).with(:query => query)
+            .to_return(:headers => return_headers)
+
+          expect(crc_protocol.upload_crc_enable).to eq(true)
+          next_pos = 0
+          expect {
+            next_pos = crc_protocol.append_object(@bucket, object_name, 11, :init_crc => 0) do |c|
+              c << content
+            end
+          }.not_to raise_error
+
+          expect(WebMock).to have_requested(:post, url)
+            .with(:body => content, :query => query)
+          expect(next_pos).to eq(101)
+        end
+
+        it "should not raise crc exception without init_crc" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+
+          query = {'append' => '', 'position' => 11}
+          return_headers = {'x-oss-next-append-position' => '101', :x_oss_hash_crc64ecma => content_crc + 1}
+          stub_request(:post, url).with(:query => query)
+            .to_return(:headers => return_headers)
+            
+          expect(crc_protocol.upload_crc_enable).to eq(true)
+          next_pos = 0
+          expect {
+            next_pos = crc_protocol.append_object(@bucket, object_name, 11) do |c|
+              c << content
+            end
+          }.not_to raise_error
+
+          expect(WebMock).to have_requested(:post, url)
+            .with(:body => content, :query => query)
+          expect(next_pos).to eq(101)
+        end
       end # append object
 
       context "Copy object" do
@@ -557,6 +661,55 @@ module Aliyun
           expect(WebMock).to have_requested(:get, url)
             .with(:body => nil, :query => query)
         end
+
+
+        it "should raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.to raise_error(CrcInconsistentError, "The crc of get between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.not_to raise_error
+          expect(response_content).to eq(content)
+        end
+
+        it "should not raise crc exception on error when setting range" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world 0123456789"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name, {range: [0, 10]}) {|c| response_content << c}
+          }.not_to raise_error
+          expect(WebMock).to have_requested(:get, url)
+            .with(:body => nil, :query => {},
+                  :headers => {
+                    'Range' => 'bytes=0-9'
+                  })
+        end
       end # Get object
 
       context "Get object meta" do