aliyun-sdk 0.4.1 → 0.7.2

data/lib/aliyun/oss/client.rb

@@ -28,11 +28,15 @@ module Aliyun
       #  KEY SECRET，如果不填则会尝试匿名访问
       # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
       #  定的域名
+      # @option opts [Boolean] :upload_crc_enable [可选]指定上传处理
+      #  是否开启CRC校验，默认为开启(true)
+      # @option opts [Boolean] :download_crc_enable [可选]指定下载处理
+      #  是否开启CRC校验，默认为不开启(false)
       # @option opts [String] :sts_token [可选] 指定STS的
       #  SecurityToken，如果指定，则使用STS授权访问
-      # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时
+      # @option opts [Integer] :open_timeout [可选] 指定建立连接的超时
       #  时间，默认为10秒
-      # @option opts [Fixnum] :read_timeout [可选] 指定等待响应的超时
+      # @option opts [Integer] :read_timeout [可选] 指定等待响应的超时
       #  时间，默认为120秒
       # @example 标准endpoint
       #   oss-cn-hangzhou.aliyuncs.com
@@ -66,6 +70,7 @@ module Aliyun
       # @param opts [Hash] 创建Bucket的属性（可选）
       # @option opts [:location] [String] 指定bucket所在的区域，默认为oss-cn-hangzhou
       def create_bucket(name, opts = {})
+        Util.ensure_bucket_name_valid(name)
         @protocol.create_bucket(name, opts)
       end
 
@@ -73,6 +78,7 @@ module Aliyun
       # @param name [String] Bucket名字
       # @note 如果要删除的Bucket不为空（包含有object），则删除会失败
       def delete_bucket(name)
+        Util.ensure_bucket_name_valid(name)
         @protocol.delete_bucket(name)
       end
 
@@ -80,6 +86,7 @@ module Aliyun
       # @param name [String] Bucket名字
       # @return [Boolean] 如果Bucket存在则返回true，否则返回false
       def bucket_exists?(name)
+        Util.ensure_bucket_name_valid(name)
         exist = false
 
         begin
@@ -98,6 +105,7 @@ module Aliyun
       # @param name [String] Bucket名字
       # @return [Bucket] Bucket对象
       def get_bucket(name)
+        Util.ensure_bucket_name_valid(name)
         Bucket.new({:name => name}, @protocol)
       end
 

data/lib/aliyun/oss/config.rb

@@ -11,7 +11,8 @@ module Aliyun
 
       attrs :endpoint, :cname, :sts_token,
             :access_key_id, :access_key_secret,
-            :open_timeout, :read_timeout
+            :open_timeout, :read_timeout,
+            :download_crc_enable, :upload_crc_enable
 
       def initialize(opts = {})
         super(opts)
@@ -19,6 +20,8 @@ module Aliyun
         @access_key_id = @access_key_id.strip if @access_key_id
         @access_key_secret = @access_key_secret.strip if @access_key_secret
         normalize_endpoint if endpoint
+        @upload_crc_enable = (@upload_crc_enable == 'false' || @upload_crc_enable == false) ? false : true
+        @download_crc_enable = (@download_crc_enable == 'true' || @download_crc_enable == true) ? true : false
       end
 
       private

data/lib/aliyun/oss/download.rb

@@ -114,7 +114,7 @@ module Aliyun
 
           parts = sync_get_all_parts
           # concat all part files into the target file
-          File.open(@file, 'w') do |w|
+          File.open(@file, 'wb') do |w|
             parts.sort{ |x, y| x[:number] <=> y[:number] }.each do |p|
               File.open(get_part_file(p)) do |r|
                   w.write(r.read(READ_SIZE)) until r.eof?
@@ -177,7 +177,7 @@ module Aliyun
           logger.debug("Begin download part: #{p}")
 
           part_file = get_part_file(p)
-          File.open(part_file, 'w') do |w|
+          File.open(part_file, 'wb') do |w|
             @protocol.get_object(
               bucket, object,
               @options.merge(range: p[:range])) { |chunk| w.write(chunk) }

data/lib/aliyun/oss/exception.rb

@@ -64,6 +64,12 @@ module Aliyun
     class ClientError < Common::Exception
     end # ClientError
 
+    ##
+    # CrcInconsistentError will be raised after a upload operation,
+    # when the local crc is inconsistent with the response crc from server. 
+    #
+    class CrcInconsistentError < Common::Exception; end
+
     ##
     # FileInconsistentError happens in a resumable upload transaction,
     # when the file to upload has changed during the uploading

data/lib/aliyun/oss/http.rb

@@ -43,10 +43,14 @@ module Aliyun
       # A stream is any class that responds to :read(bytes, outbuf)
       #
       class StreamWriter
-        def initialize
+        attr_reader :data_crc
+
+        def initialize(crc_enable = false, init_crc = 0)
           @buffer = ""
           @producer = Fiber.new { yield self if block_given? }
           @producer.resume
+          @data_crc = init_crc.to_i
+          @crc_enable = crc_enable
         end
 
         def read(bytes = nil, outbuf = nil)
@@ -84,6 +88,8 @@ module Aliyun
           # "". ios.read(positive-integer) returns nil.
           return nil if ret.empty? && !bytes.nil? && bytes > 0
 
+          @data_crc = Aliyun::OSS::Util.crc(ret, @data_crc) if @crc_enable
+
           ret
         end
 
@@ -99,37 +105,12 @@ module Aliyun
           false
         end
 
-        def inspect
-          "@buffer: " + @buffer[0, 32].inspect + "...#{@buffer.size} bytes"
-        end
-      end
-
-      # RestClient requires the payload to respones to :read(bytes)
-      # and return a stream.
-      # We are not doing the real read here, just return a
-      # readable stream for RestClient playload.rb treats it as:
-      #     def read(bytes=nil)
-      #       @stream.read(bytes)
-      #     end
-      #     alias :to_s :read
-      #     net_http_do_request(http, req, payload ? payload.to_s : nil,
-      #                     &@block_response)
-      class StreamPayload
-        def initialize(&block)
-          @stream = StreamWriter.new(&block)
-        end
-
-        def read(bytes = nil)
-          @stream
-        end
-
         def close
         end
 
-        def closed?
-          false
+        def inspect
+          "@buffer: " + @buffer[0, 32].inspect + "...#{@buffer.size} bytes"
         end
-
       end
 
       include Common::Logging
@@ -140,12 +121,13 @@ module Aliyun
 
       def get_request_url(bucket, object)
         url = @config.endpoint.dup
+        url.query = nil
+        url.fragment = nil 
         isIP = !!(url.host =~ Resolv::IPv4::Regex)
         url.host = "#{bucket}." + url.host if bucket && !@config.cname && !isIP
         url.path = '/'
         url.path << "#{bucket}/" if bucket && isIP
-        url.path << "#{CGI.escape(object)}" if object
-
+        url.path << CGI.escape(object) if object
         url.to_s
       end
 
@@ -276,44 +258,46 @@ module Aliyun
         headers[:params] = (sub_res || {}).merge(http_options[:query] || {})
 
         block_response = ->(r) { handle_response(r, &block) } if block
-        r = RestClient::Request.execute(
+        request = RestClient::Request.new(
           :method => verb,
           :url => get_request_url(bucket, object),
           :headers => headers,
           :payload => http_options[:body],
           :block_response => block_response,
           :open_timeout => @config.open_timeout || OPEN_TIMEOUT,
-          :timeout => @config.read_timeout || READ_TIMEOUT
-        ) do |response, request, result, &blk|
-
-          if response.code >= 300
-            e = ServerError.new(response)
+          :read_timeout => @config.read_timeout || READ_TIMEOUT
+        )
+        response = request.execute do |resp, &blk|
+          if resp.code >= 300
+            e = ServerError.new(resp)
             logger.error(e.to_s)
             raise e
           else
-            response.return!(request, result, &blk)
+            resp.return!(&blk)
           end
         end
 
         # If streaming read_body is used, we need to create the
         # RestClient::Response ourselves
-        unless r.is_a?(RestClient::Response)
-          if r.code.to_i >= 300
-            r = RestClient::Response.create(
-              RestClient::Request.decode(r['content-encoding'], r.body),
-              r, nil, nil)
-            e = ServerError.new(r)
+        unless response.is_a?(RestClient::Response)
+          if response.code.to_i >= 300
+            body = response.body
+            if RestClient::version < '2.1.0'
+              body = RestClient::Request.decode(response['content-encoding'], response.body)
+            end
+            response = RestClient::Response.create(body, response, request)
+            e = ServerError.new(response)
             logger.error(e.to_s)
             raise e
           end
-          r = RestClient::Response.create(nil, r, nil, nil)
-          r.return!
+          response = RestClient::Response.create(nil, response, request)
+          response.return!
         end
 
-        logger.debug("Received HTTP response, code: #{r.code}, headers: " \
-                      "#{r.headers}, body: #{r.body}")
+        logger.debug("Received HTTP response, code: #{response.code}, headers: " \
+                      "#{response.headers}, body: #{response.body}")
 
-        r
+        response
       end
 
       def get_user_agent

data/lib/aliyun/oss/multipart.rb

@@ -30,7 +30,7 @@ module Aliyun
           md5= Util.get_content_md5(states.to_json)
 
           @mutex.synchronize {
-            File.open(file, 'w') {
+            File.open(file, 'wb') {
               |f| f.write(states.merge(md5: md5).to_json)
             }
           }

data/lib/aliyun/oss/protocol.rb

@@ -7,6 +7,7 @@ require 'time'
 module Aliyun
   module OSS
 
+
     ##
     # Protocol implement the OSS Open API which is low-level. User
     # should refer to {OSS::Client} for normal use.
@@ -345,10 +346,10 @@ module Aliyun
                     xml.Date Time.utc(
                                r.expiry.year, r.expiry.month, r.expiry.day)
                               .iso8601.sub('Z', '.000Z')
-                  elsif r.expiry.is_a?(Fixnum)
+                  elsif r.expiry.is_a?(Integer)
                     xml.Days r.expiry
                   else
-                    fail ClientError, "Expiry must be a Date or Fixnum."
+                    fail ClientError, "Expiry must be a Date or Integer."
                   end
                 }
               }
@@ -534,9 +535,10 @@ module Aliyun
           headers[CALLBACK_HEADER] = opts[:callback].serialize
         end
 
+        payload = HTTP::StreamWriter.new(@config.upload_crc_enable, opts[:init_crc], &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name},
-          {:headers => headers, :body => HTTP::StreamPayload.new(&block)})
+          {:headers => headers, :body => payload})
 
         if r.code == 203
           e = CallbackError.new(r)
@@ -544,6 +546,11 @@ module Aliyun
           raise e
         end
 
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
+          data_crc = payload.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
+        end
+
         logger.debug('Done put object')
       end
 
@@ -586,9 +593,19 @@ module Aliyun
 
         headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers)
 
+        payload = HTTP::StreamWriter.new(
+          @config.upload_crc_enable && !opts[:init_crc].nil?, opts[:init_crc], &block)
+
         r = @http.post(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
-          {:headers => headers, :body => HTTP::StreamPayload.new(&block)})
+          {:headers => headers, :body => payload})
+
+        if @config.upload_crc_enable &&
+          !r.headers[:x_oss_hash_crc64ecma].nil? &&
+          !opts[:init_crc].nil?
+          data_crc = payload.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'append')
+        end
 
         logger.debug('Done append object')
 
@@ -759,11 +776,22 @@ module Aliyun
             rewrites[:expires].httpdate if rewrites.key?(:expires)
         end
 
+        data_crc = opts[:init_crc].nil? ? 0 : opts[:init_crc]
         r = @http.get(
           {:bucket => bucket_name, :object => object_name,
            :sub_res => sub_res},
           {:headers => headers}
-        ) { |chunk| yield chunk if block_given? }
+        ) do |chunk|
+          if block_given?
+            # crc enable and no range and oss server support crc
+            data_crc = Aliyun::OSS::Util.crc(chunk, data_crc) if @config.download_crc_enable && range.nil?
+            yield chunk
+          end
+        end
+
+        if @config.download_crc_enable && range.nil? && !r.headers[:x_oss_hash_crc64ecma].nil?
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'get')
+        end
 
         h = r.headers
         metas = {}
@@ -1082,9 +1110,16 @@ module Aliyun
                      "#{object_name}, txn id: #{txn_id}, part No: #{part_no}")
 
         sub_res = {'partNumber' => part_no, 'uploadId' => txn_id}
+
+        payload = HTTP::StreamWriter.new(@config.upload_crc_enable, &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
-          {:body => HTTP::StreamPayload.new(&block)})
+          {:body => payload})
+
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
+          data_crc = payload.data_crc
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
+        end
 
         logger.debug("Done upload part")
 
@@ -1359,12 +1394,26 @@ module Aliyun
         @http.get_request_url(bucket, object)
       end
 
+      # Get bucket/object resource path
+      # @param [String] bucket the bucket name
+      # @param [String] object the bucket name
+      # @return [String] resource path for the bucket/object
+      def get_resource_path(bucket, object = nil)
+        @http.get_resource_path(bucket, object)
+      end
+
       # Get user's access key id
       # @return [String] the access key id
       def get_access_key_id
         @config.access_key_id
       end
 
+      # Get user's access key secret
+      # @return [String] the access key secret
+      def get_access_key_secret
+        @config.access_key_secret
+      end  
+
       # Get user's STS token
       # @return [String] the STS token
       def get_sts_token
@@ -1378,6 +1427,18 @@ module Aliyun
         Util.sign(@config.access_key_secret, string_to_sign)
       end
 
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @config.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @config.upload_crc_enable
+      end
+
       private
 
       # Parse body content to xml document
@@ -1470,7 +1531,7 @@ module Aliyun
       def get_bytes_range(range)
         if range &&
            (!range.is_a?(Array) || range.size != 2 ||
-            !range.at(0).is_a?(Fixnum) || !range.at(1).is_a?(Fixnum))
+            !range.at(0).is_a?(Integer) || !range.at(1).is_a?(Integer))
           fail ClientError, "Range must be an array containing 2 Integers."
         end
 
@@ -1493,7 +1554,6 @@ module Aliyun
           result
         end
       end
-
     end # Protocol
   end # OSS
 end # Aliyun

data/lib/aliyun/oss/struct.rb

@@ -107,10 +107,10 @@ module Aliyun
     # * id [String] the unique id of a rule
     # * enabled [Boolean] whether to enable this rule
     # * prefix [String] the prefix objects to apply this rule
-    # * expiry [Date] or [Fixnum] the expire time of objects
+    # * expiry [Date] or [Integer] the expire time of objects
     #   * if expiry is a Date, it specifies the absolute date to
     #     expire objects
-    #   * if expiry is a Fixnum, it specifies the relative date to
+    #   * if expiry is a Integer, it specifies the relative date to
     #     expire objects: how many days after the object's last
     #     modification time to expire the object
     # @example Specify expiry as Date

data/lib/aliyun/oss/util.rb

@@ -7,6 +7,7 @@ require 'digest/md5'
 
 module Aliyun
   module OSS
+
     ##
     # Util functions to help generate formatted Date, signatures,
     # etc.
@@ -75,6 +76,29 @@ module Aliyun
           end
         end
 
+        # Get a crc value of the data
+        def crc(data, init_crc = 0)
+          CrcX::crc64(init_crc, data, data.size)
+        end
+
+        # Calculate a value of the crc1 combine with crc2. 
+        def crc_combine(crc1, crc2, len2)
+          CrcX::crc64_combine(crc1, crc2, len2)
+        end
+
+        def crc_check(crc_a, crc_b, operation)
+          if crc_a.nil? || crc_b.nil? || crc_a.to_i != crc_b.to_i
+            logger.error("The crc of #{operation} between client and oss is not inconsistent. crc_a=#{crc_a} crc_b=#{crc_b}")
+            fail CrcInconsistentError.new("The crc of #{operation} between client and oss is not inconsistent.")
+          end
+        end
+
+        def ensure_bucket_name_valid(name)
+          unless (name =~ %r|^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$|)
+            fail ClientError, "The bucket name is invalid."
+          end
+        end  
+
       end # self
     end # Util
   end # OSS
@@ -86,4 +110,4 @@ class String
     return true if self =~ /^true$/i
     false
   end
-end
+end