aliyun-oss-ruby-sdk 0.4.1

data/lib/aliyun_sdk/oss/client.rb

@@ -0,0 +1,106 @@
+# -*- encoding: utf-8 -*-
+
+module AliyunSDK
+  module OSS
+
+    ##
+    # OSS服务的客户端，用于获取bucket列表，创建/删除bucket。Object相关
+    # 的操作请使用{OSS::Bucket}。
+    # @example 创建Client
+    #   endpoint = 'oss-cn-hangzhou.aliyuncs.com'
+    #   client = Client.new(
+    #     :endpoint => endpoint,
+    #     :access_key_id => 'access_key_id',
+    #     :access_key_secret => 'access_key_secret')
+    #   buckets = client.list_buckets
+    #   client.create_bucket('my-bucket')
+    #   client.delete_bucket('my-bucket')
+    #   bucket = client.get_bucket('my-bucket')
+    class Client
+
+      # 构造OSS client，用于操作buckets。
+      # @param opts [Hash] 构造Client时的参数选项
+      # @option opts [String] :endpoint [必填]OSS服务的地址，可以是以
+      #  oss-cn-hangzhou.aliyuncs.com的标准域名，也可以是用户绑定的域名
+      # @option opts [String] :access_key_id [可选]用户的ACCESS KEY ID，
+      #  如果不填则会尝试匿名访问
+      # @option opts [String] :access_key_secret [可选]用户的ACCESS
+      #  KEY SECRET，如果不填则会尝试匿名访问
+      # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
+      #  定的域名
+      # @option opts [String] :sts_token [可选] 指定STS的
+      #  SecurityToken，如果指定，则使用STS授权访问
+      # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时
+      #  时间，默认为10秒
+      # @option opts [Fixnum] :read_timeout [可选] 指定等待响应的超时
+      #  时间，默认为120秒
+      # @example 标准endpoint
+      #   oss-cn-hangzhou.aliyuncs.com
+      #   oss-cn-beijing.aliyuncs.com
+      # @example 用户绑定的域名
+      #   my-domain.com
+      #   foo.bar.com
+      def initialize(opts)
+        fail ClientError, "Endpoint must be provided" unless opts[:endpoint]
+
+        @config = Config.new(opts)
+        @protocol = Protocol.new(@config)
+      end
+
+      # 列出当前所有的bucket
+      # @param opts [Hash] 查询选项
+      # @option opts [String] :prefix 如果设置，则只返回以它为前缀的bucket
+      # @option opts [String] :marker 如果设置，则只返回名字在它之后
+      #  （字典序，不包含marker）的bucket
+      # @return [Enumerator<Bucket>] Bucket的迭代器
+      def list_buckets(opts = {})
+        if @config.cname
+          fail ClientError, "Cannot list buckets for a CNAME endpoint."
+        end
+
+        Iterator::Buckets.new(@protocol, opts).to_enum
+      end
+
+      # 创建一个bucket
+      # @param name [String] Bucket名字
+      # @param opts [Hash] 创建Bucket的属性（可选）
+      # @option opts [:location] [String] 指定bucket所在的区域，默认为oss-cn-hangzhou
+      def create_bucket(name, opts = {})
+        @protocol.create_bucket(name, opts)
+      end
+
+      # 删除一个bucket
+      # @param name [String] Bucket名字
+      # @note 如果要删除的Bucket不为空（包含有object），则删除会失败
+      def delete_bucket(name)
+        @protocol.delete_bucket(name)
+      end
+
+      # 判断一个bucket是否存在
+      # @param name [String] Bucket名字
+      # @return [Boolean] 如果Bucket存在则返回true，否则返回false
+      def bucket_exists?(name)
+        exist = false
+
+        begin
+          @protocol.get_bucket_acl(name)
+          exist = true
+        rescue ServerError => e
+          raise unless e.http_code == 404
+        end
+
+        exist
+      end
+
+      alias :bucket_exist? :bucket_exists?
+
+      # 获取一个Bucket对象，用于操作bucket中的objects。
+      # @param name [String] Bucket名字
+      # @return [Bucket] Bucket对象
+      def get_bucket(name)
+        Bucket.new({:name => name}, @protocol)
+      end
+
+    end # Client
+  end # OSS
+end # Aliyun

data/lib/aliyun_sdk/oss/config.rb

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+
+module AliyunSDK
+  module OSS
+
+    ##
+    # A place to store various configurations: credentials, api
+    # timeout, retry mechanism, etc
+    #
+    class Config < Common::Struct::Base
+
+      attrs :endpoint, :cname, :sts_token,
+            :access_key_id, :access_key_secret,
+            :open_timeout, :read_timeout
+
+      def initialize(opts = {})
+        super(opts)
+
+        @access_key_id = @access_key_id.strip if @access_key_id
+        @access_key_secret = @access_key_secret.strip if @access_key_secret
+        normalize_endpoint if endpoint
+      end
+
+      private
+
+      def normalize_endpoint
+        uri = URI.parse(endpoint)
+        uri = URI.parse("http://#{endpoint}") unless uri.scheme
+
+        if uri.scheme != 'http' and uri.scheme != 'https'
+          fail ClientError, "Only HTTP and HTTPS endpoint are accepted."
+        end
+
+        @endpoint = uri
+      end
+
+    end # Config
+  end # OSS
+end # Aliyun

data/lib/aliyun_sdk/oss/download.rb

@@ -0,0 +1,255 @@
+# -*- encoding: utf-8 -*-
+
+module AliyunSDK
+  module OSS
+    module Multipart
+      ##
+      # A multipart download transaction
+      #
+      class Download < Transaction
+
+        include Common::Logging
+
+        PART_SIZE = 10 * 1024 * 1024
+        READ_SIZE = 16 * 1024
+        NUM_THREAD = 10
+
+        def initialize(protocol, opts)
+          args = opts.dup
+          @protocol = protocol
+          @progress = args.delete(:progress)
+          @file = args.delete(:file)
+          @cpt_file = args.delete(:cpt_file)
+          super(args)
+
+          @object_meta = {}
+          @num_threads = options[:threads] || NUM_THREAD
+          @all_mutex = Mutex.new
+          @parts = []
+          @todo_mutex = Mutex.new
+          @todo_parts = []
+        end
+
+        # Run the download transaction, which includes 3 stages:
+        # * 1a. initiate(new downlaod) and divide parts
+        # * 1b. rebuild states(resumed download)
+        # * 2.  download each unfinished part
+        # * 3.  combine the downloaded parts into the final file
+        def run
+          logger.info("Begin download, file: #{@file}, "\
+                      "checkpoint file: #{@cpt_file}, "\
+                      "threads: #{@num_threads}")
+
+          # Rebuild transaction states from checkpoint file
+          # Or initiate new transaction states
+          rebuild
+
+          # Divide the target object into parts to download by ranges
+          divide_parts if @parts.empty?
+
+          # Download each part(object range)
+          @todo_parts = @parts.reject { |p| p[:done] }
+
+          (1..@num_threads).map {
+            Thread.new {
+              loop {
+                p = sync_get_todo_part
+                break unless p
+                download_part(p)
+              }
+            }
+          }.map(&:join)
+
+          # Combine the parts into the final file
+          commit
+
+          logger.info("Done download, file: #{@file}")
+        end
+
+        # Checkpoint structures:
+        # @example
+        #   states = {
+        #     :id => 'download_id',
+        #     :file => 'file',
+        #     :object_meta => {
+        #       :etag => 'xxx',
+        #       :size => 1024
+        #     },
+        #     :parts => [
+        #       {:number => 1, :range => [0, 100], :md5 => 'xxx', :done => false},
+        #       {:number => 2, :range => [100, 200], :md5 => 'yyy', :done => true}
+        #     ],
+        #     :md5 => 'states_md5'
+        #   }
+        def checkpoint
+          logger.debug("Begin make checkpoint, disable_cpt: "\
+                       "#{options[:disable_cpt] == true}")
+
+          ensure_object_not_changed
+
+          parts = sync_get_all_parts
+          states = {
+            :id => id,
+            :file => @file,
+            :object_meta => @object_meta,
+            :parts => parts
+          }
+
+          # report progress
+          if @progress
+            done = parts.count { |p| p[:done] }
+            @progress.call(done.to_f / parts.size) if done > 0
+          end
+
+          write_checkpoint(states, @cpt_file) unless options[:disable_cpt]
+
+          logger.debug("Done make checkpoint, states: #{states}")
+        end
+
+        private
+        # Combine the downloaded parts into the final file
+        # @todo avoid copy all part files
+        def commit
+          logger.info("Begin commit transaction, id: #{id}")
+
+          parts = sync_get_all_parts
+          # concat all part files into the target file
+          File.open(@file, 'w') do |w|
+            parts.sort{ |x, y| x[:number] <=> y[:number] }.each do |p|
+              File.open(get_part_file(p)) do |r|
+                  w.write(r.read(READ_SIZE)) until r.eof?
+              end
+            end
+          end
+
+          File.delete(@cpt_file) unless options[:disable_cpt]
+          parts.each{ |p| File.delete(get_part_file(p)) }
+
+          logger.info("Done commit transaction, id: #{id}")
+        end
+
+        # Rebuild the states of the transaction from checkpoint file
+        def rebuild
+          logger.info("Begin rebuild transaction, checkpoint: #{@cpt_file}")
+
+          if options[:disable_cpt] || !File.exists?(@cpt_file)
+            initiate
+          else
+            states = load_checkpoint(@cpt_file)
+
+            states[:parts].select{ |p| p[:done] }.each do |p|
+              part_file = get_part_file(p)
+
+              unless File.exist?(part_file)
+                fail PartMissingError, "The part file is missing: #{part_file}."
+              end
+
+              if p[:md5] != get_file_md5(part_file)
+                fail PartInconsistentError,
+                     "The part file is changed: #{part_file}."
+              end
+            end
+
+            @id = states[:id]
+            @object_meta = states[:object_meta]
+            @parts = states[:parts]
+          end
+
+          logger.info("Done rebuild transaction, states: #{states}")
+        end
+
+        def initiate
+          logger.info("Begin initiate transaction")
+
+          @id = generate_download_id
+          obj = @protocol.get_object_meta(bucket, object)
+          @object_meta = {
+            :etag => obj.etag,
+            :size => obj.size
+          }
+          checkpoint
+
+          logger.info("Done initiate transaction, id: #{id}")
+        end
+
+        # Download a part
+        def download_part(p)
+          logger.debug("Begin download part: #{p}")
+
+          part_file = get_part_file(p)
+          File.open(part_file, 'w') do |w|
+            @protocol.get_object(
+              bucket, object,
+              @options.merge(range: p[:range])) { |chunk| w.write(chunk) }
+          end
+
+          sync_update_part(p.merge(done: true, md5: get_file_md5(part_file)))
+
+          checkpoint
+
+          logger.debug("Done download part: #{p}")
+        end
+
+        # Devide the object to download into parts to download
+        def divide_parts
+          logger.info("Begin divide parts, object: #{@object}")
+
+          max_parts = 100
+          object_size = @object_meta[:size]
+          part_size =
+            [@options[:part_size] || PART_SIZE, object_size / max_parts].max
+          num_parts = (object_size - 1) / part_size + 1
+          @parts = (1..num_parts).map do |i|
+            {
+              :number => i,
+              :range => [(i - 1) * part_size, [i * part_size, object_size].min],
+              :done => false
+            }
+          end
+
+          checkpoint
+
+          logger.info("Done divide parts, parts: #{@parts}")
+        end
+
+        def sync_get_todo_part
+          @todo_mutex.synchronize {
+            @todo_parts.shift
+          }
+        end
+
+        def sync_update_part(p)
+          @all_mutex.synchronize {
+            @parts[p[:number] - 1] = p
+          }
+        end
+
+        def sync_get_all_parts
+          @all_mutex.synchronize {
+            @parts.dup
+          }
+        end
+
+        # Ensure file not changed during uploading
+        def ensure_object_not_changed
+          obj = @protocol.get_object_meta(bucket, object)
+          unless obj.etag == @object_meta[:etag]
+            fail ObjectInconsistentError,
+                 "The object to download is changed: #{object}."
+          end
+        end
+
+        # Generate a download id
+        def generate_download_id
+          "download_#{bucket}_#{object}_#{Time.now.to_i}"
+        end
+
+        # Get name for part file
+        def get_part_file(p)
+          "#{@file}.part.#{p[:number]}"
+        end
+      end # Download
+
+    end # Multipart
+  end # OSS
+end # Aliyun

data/lib/aliyun_sdk/oss/exception.rb

@@ -0,0 +1,108 @@
+# -*- encoding: utf-8 -*-
+
+require 'nokogiri'
+
+module AliyunSDK
+  module OSS
+
+    ##
+    # ServerError represents exceptions from the OSS
+    # service. i.e. Client receives a HTTP response whose status is
+    # NOT OK. #message provides the error message and #to_s gives
+    # detailed information probably including the OSS request id.
+    #
+    class ServerError < Common::Exception
+
+      attr_reader :http_code, :error_code, :message, :request_id
+
+      def initialize(response)
+        @http_code = response.code
+        @attrs = {'RequestId' => get_request_id(response)}
+
+        doc = Nokogiri::XML(response.body) do |config|
+          config.options |= Nokogiri::XML::ParseOptions::NOBLANKS
+        end rescue nil
+
+        if doc and doc.root
+          doc.root.children.each do |n|
+            @attrs[n.name] = n.text
+          end
+        end
+
+        @error_code = @attrs['Code']
+        @message = @attrs['Message']
+        @request_id = @attrs['RequestId']
+      end
+
+      def message
+        msg = @attrs['Message'] || "UnknownError[#{http_code}]."
+        "#{msg} RequestId: #{request_id}"
+      end
+
+      def to_s
+        @attrs.merge({'HTTPCode' => @http_code}).map do |k, v|
+          [k, v].join(": ")
+        end.join(", ")
+      end
+
+      private
+
+      def get_request_id(response)
+        r = response.headers[:x_oss_request_id] if response.headers
+        r.to_s
+      end
+
+    end # ServerError
+
+    class CallbackError < ServerError
+    end # CallbackError
+
+    ##
+    # ClientError represents client exceptions caused mostly by
+    # invalid parameters.
+    #
+    class ClientError < Common::Exception
+    end # ClientError
+
+    ##
+    # FileInconsistentError happens in a resumable upload transaction,
+    # when the file to upload has changed during the uploading
+    # process. Which means the transaction cannot go on. Or user may
+    # have inconsistent data uploaded to OSS.
+    #
+    class FileInconsistentError < ClientError; end
+
+    ##
+    # ObjectInconsistentError happens in a resumable download transaction,
+    # when the object to download has changed during the downloading
+    # process. Which means the transaction cannot go on. Or user may
+    # have inconsistent data downloaded to OSS.
+    #
+    class ObjectInconsistentError < ClientError; end
+
+    ##
+    # PartMissingError happens in a resumable download transaction,
+    # when a downloaded part cannot be found as the client tries to
+    # resume download. The process cannot go on until the part is
+    # restored.
+    #
+    class PartMissingError < ClientError; end
+
+    ##
+    # PartMissingError happens in a resumable download transaction,
+    # when a downloaded part has changed(MD5 mismatch) as the client
+    # tries to resume download. The process cannot go on until the
+    # part is restored.
+    #
+    class PartInconsistentError < ClientError; end
+
+    ##
+    # CheckpointBrokenError happens in a resumable upload/download
+    # transaction, when the client finds the checkpoint file has
+    # changed(MD5 mismatch) as it tries to resume upload/download. The
+    # process cannot go on until the checkpoint file is restored.
+    #
+    class CheckpointBrokenError < ClientError; end
+
+  end # OSS
+end # Aliyun