aliyun-oss-ruby-sdk 0.4.1

data/lib/aliyun_sdk/oss/struct.rb

@@ -0,0 +1,208 @@
+# -*- encoding: utf-8 -*-
+
+require 'base64'
+require 'json'
+require 'uri'
+
+module AliyunSDK
+  module OSS
+
+    ##
+    # Access Control List, it controls how the bucket/object can be
+    # accessed.
+    # * public-read-write: allow access(read&write) anonymously
+    # * public-read: allow read anonymously
+    # * private: access must be signatured
+    #
+    module ACL
+      PUBLIC_READ_WRITE = "public-read-write"
+      PUBLIC_READ = "public-read"
+      PRIVATE = "private"
+    end # ACL
+
+    ##
+    # A OSS object may carry some metas(String key-value pairs) with
+    # it. MetaDirective specifies what to do with the metas in the
+    # copy process.
+    # * COPY: metas are copied from the source object to the dest
+    #   object
+    # * REPLACE: source object's metas are NOT copied, use user
+    #   provided metas for the dest object
+    #
+    module MetaDirective
+      COPY = "COPY"
+      REPLACE = "REPLACE"
+    end # MetaDirective
+
+    ##
+    # The object key may contains unicode charactors which cannot be
+    # encoded in the request/response body(XML). KeyEncoding specifies
+    # the encoding type for the object key.
+    # * url: the object key is url-encoded
+    # @note url-encoding is the only supported KeyEncoding type
+    #
+    module KeyEncoding
+      URL = "url"
+
+      @@all = [URL]
+
+      def self.include?(enc)
+        all.include?(enc)
+      end
+
+      def self.all
+        @@all
+      end
+    end # KeyEncoding
+
+    ##
+    # Bucket Logging setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/logging.html OSS Bucket logging}
+    # Attributes:
+    # * enable [Boolean] whether to enable bucket logging
+    # * target_bucket [String] the target bucket to store access logs
+    # * target_prefix [String] the target object prefix to store access logs
+    # @example Enable bucket logging
+    #  bucket.logging = BucketLogging.new(
+    #    :enable => true, :target_bucket => 'log_bucket', :target_prefix => 'my-log')
+    # @example Disable bucket logging
+    #  bucket.logging = BucketLogging.new(:enable => false)
+    class BucketLogging < Common::Struct::Base
+      attrs :enable, :target_bucket, :target_prefix
+
+      def enabled?
+        enable == true
+      end
+    end
+
+    ##
+    # Bucket website setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/host-static-website.html OSS Website hosting}
+    # Attributes:
+    # * enable [Boolean] whether to enable website hosting for the bucket
+    # * index [String] the index object as the index page for the website
+    # * error [String] the error object as the error page for the website
+    class BucketWebsite < Common::Struct::Base
+      attrs :enable, :index, :error
+
+      def enabled?
+        enable == true
+      end
+    end
+
+    ##
+    # Bucket referer setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS Website hosting}
+    # Attributes:
+    # * allow_empty [Boolean] whether to allow requests with empty "Referer"
+    # * whitelist [Array<String>] the allowed origins for requests
+    class BucketReferer < Common::Struct::Base
+      attrs :allow_empty, :whitelist
+
+      def allow_empty?
+        allow_empty == true
+      end
+    end
+
+    ##
+    # LifeCycle rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/lifecycle.html OSS Bucket LifeCycle}
+    # Attributes:
+    # * id [String] the unique id of a rule
+    # * enabled [Boolean] whether to enable this rule
+    # * prefix [String] the prefix objects to apply this rule
+    # * expiry [Date] or [Fixnum] the expire time of objects
+    #   * if expiry is a Date, it specifies the absolute date to
+    #     expire objects
+    #   * if expiry is a Fixnum, it specifies the relative date to
+    #     expire objects: how many days after the object's last
+    #     modification time to expire the object
+    # @example Specify expiry as Date
+    #   LifeCycleRule.new(
+    #     :id => 'rule1',
+    #     :enabled => true,
+    #     :prefix => 'foo/',
+    #     :expiry => Date.new(2016, 1, 1))
+    # @example Specify expiry as days
+    #   LifeCycleRule.new(
+    #     :id => 'rule1',
+    #     :enabled => true,
+    #     :prefix => 'foo/',
+    #     :expiry => 15)
+    # @note the expiry date is treated as UTC time
+    class LifeCycleRule < Common::Struct::Base
+
+      attrs :id, :enable, :prefix, :expiry
+
+      def enabled?
+        enable == true
+      end
+    end # LifeCycleRule
+
+    ##
+    # CORS rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS CORS}
+    # Attributes:
+    # * allowed_origins [Array<String>] the allowed origins
+    # * allowed_methods [Array<String>] the allowed methods
+    # * allowed_headers [Array<String>] the allowed headers
+    # * expose_headers [Array<String>] the expose headers
+    # * max_age_seconds [Integer] the max age seconds
+    class CORSRule < Common::Struct::Base
+
+      attrs :allowed_origins, :allowed_methods, :allowed_headers,
+            :expose_headers, :max_age_seconds
+
+    end # CORSRule
+
+    ##
+    # Callback represents a HTTP call made by OSS to user's
+    # application server after an event happens, such as an object is
+    # successfully uploaded to OSS. See: {https://help.aliyun.com/document_detail/oss/api-reference/object/Callback.html}
+    # Attributes:
+    # * url [String] the URL *WITHOUT* the query string
+    # * query [Hash] the query to generate query string
+    # * body [String] the body of the request
+    # * content_type [String] the Content-Type of the request
+    # * host [String] the Host in HTTP header for this request
+    class Callback < Common::Struct::Base
+
+      attrs :url, :query, :body, :content_type, :host
+
+      include Common::Logging
+
+      def serialize
+        query_string = (query || {}).map { |k, v|
+          [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=') }.join('&')
+
+        cb = {
+          'callbackUrl' => "#{normalize_url(url)}?#{query_string}",
+          'callbackBody' => body,
+          'callbackBodyType' => content_type || default_content_type
+        }
+        cb['callbackHost'] = host if host
+
+        logger.debug("Callback json: #{cb}")
+
+        Base64.strict_encode64(cb.to_json)
+      end
+
+      private
+      def normalize_url(url)
+        uri = URI.parse(url)
+        uri = URI.parse("http://#{url}") unless uri.scheme
+
+        if uri.scheme != 'http' and uri.scheme != 'https'
+          fail ClientError, "Only HTTP and HTTPS endpoint are accepted."
+        end
+
+        unless uri.query.nil?
+          fail ClientError, "Query parameters should not appear in URL."
+        end
+
+        uri.to_s
+      end
+
+      def default_content_type
+        "application/x-www-form-urlencoded"
+      end
+
+    end # Callback
+
+  end # OSS
+end # Aliyun

data/lib/aliyun_sdk/oss/upload.rb

@@ -0,0 +1,238 @@
+# -*- encoding: utf-8 -*-
+
+module AliyunSDK
+  module OSS
+    module Multipart
+      ##
+      # A multipart upload transaction
+      #
+      class Upload < Transaction
+
+        include Common::Logging
+
+        PART_SIZE = 10 * 1024 * 1024
+        READ_SIZE = 16 * 1024
+        NUM_THREAD = 10
+
+        def initialize(protocol, opts)
+          args = opts.dup
+          @protocol = protocol
+          @progress = args.delete(:progress)
+          @file = args.delete(:file)
+          @cpt_file = args.delete(:cpt_file)
+          super(args)
+
+          @file_meta = {}
+          @num_threads = options[:threads] || NUM_THREAD
+          @all_mutex = Mutex.new
+          @parts = []
+          @todo_mutex = Mutex.new
+          @todo_parts = []
+        end
+
+        # Run the upload transaction, which includes 3 stages:
+        # * 1a. initiate(new upload) and divide parts
+        # * 1b. rebuild states(resumed upload)
+        # * 2.  upload each unfinished part
+        # * 3.  commit the multipart upload transaction
+        def run
+          logger.info("Begin upload, file: #{@file}, "\
+                      "checkpoint file: #{@cpt_file}, "\
+                      "threads: #{@num_threads}")
+
+          # Rebuild transaction states from checkpoint file
+          # Or initiate new transaction states
+          rebuild
+
+          # Divide the file to upload into parts to upload separately
+          divide_parts if @parts.empty?
+
+          # Upload each part
+          @todo_parts = @parts.reject { |p| p[:done] }
+
+          (1..@num_threads).map {
+            Thread.new {
+              loop {
+                p = sync_get_todo_part
+                break unless p
+                upload_part(p)
+              }
+            }
+          }.map(&:join)
+
+          # Commit the multipart upload transaction
+          commit
+
+          logger.info("Done upload, file: #{@file}")
+        end
+
+        # Checkpoint structures:
+        # @example
+        #   states = {
+        #     :id => 'upload_id',
+        #     :file => 'file',
+        #     :file_meta => {
+        #       :mtime => Time.now,
+        #       :md5 => 1024
+        #     },
+        #     :parts => [
+        #       {:number => 1, :range => [0, 100], :done => false},
+        #       {:number => 2, :range => [100, 200], :done => true}
+        #     ],
+        #     :md5 => 'states_md5'
+        #   }
+        def checkpoint
+          logger.debug("Begin make checkpoint, disable_cpt: "\
+                       "#{options[:disable_cpt] == true}")
+
+          ensure_file_not_changed
+
+          parts = sync_get_all_parts
+          states = {
+            :id => id,
+            :file => @file,
+            :file_meta => @file_meta,
+            :parts => parts
+          }
+
+          # report progress
+          if @progress
+            done = parts.count { |p| p[:done] }
+            @progress.call(done.to_f / parts.size) if done > 0
+          end
+
+          write_checkpoint(states, @cpt_file) unless options[:disable_cpt]
+
+          logger.debug("Done make checkpoint, states: #{states}")
+        end
+
+        private
+        # Commit the transaction when all parts are succefully uploaded
+        # @todo handle undefined behaviors: commit succeeds in server
+        #  but return error in client
+        def commit
+          logger.info("Begin commit transaction, id: #{id}")
+
+          parts = sync_get_all_parts.map{ |p|
+            Part.new(:number  => p[:number], :etag => p[:etag])
+          }
+          @protocol.complete_multipart_upload(
+            bucket, object, id, parts, @options[:callback])
+
+          File.delete(@cpt_file) unless options[:disable_cpt]
+
+          logger.info("Done commit transaction, id: #{id}")
+        end
+
+        # Rebuild the states of the transaction from checkpoint file
+        def rebuild
+          logger.info("Begin rebuild transaction, checkpoint: #{@cpt_file}")
+
+          if options[:disable_cpt] || !File.exists?(@cpt_file)
+            initiate
+          else
+            states = load_checkpoint(@cpt_file)
+
+            if states[:file_md5] != @file_meta[:md5]
+              fail FileInconsistentError.new("The file to upload is changed.")
+            end
+
+            @id = states[:id]
+            @file_meta = states[:file_meta]
+            @parts = states[:parts]
+          end
+
+          logger.info("Done rebuild transaction, states: #{states}")
+        end
+
+        def initiate
+          logger.info("Begin initiate transaction")
+
+          @id = @protocol.initiate_multipart_upload(bucket, object, options)
+          @file_meta = {
+            :mtime => File.mtime(@file),
+            :md5 => get_file_md5(@file)
+          }
+          checkpoint
+
+          logger.info("Done initiate transaction, id: #{id}")
+        end
+
+        # Upload a part
+        def upload_part(p)
+          logger.debug("Begin upload part: #{p}")
+
+          result = nil
+          File.open(@file) do |f|
+            range = p[:range]
+            pos = range.first
+            f.seek(pos)
+
+            result = @protocol.upload_part(bucket, object, id, p[:number]) do |sw|
+              while pos < range.at(1)
+                bytes = [READ_SIZE, range.at(1) - pos].min
+                sw << f.read(bytes)
+                pos += bytes
+              end
+            end
+          end
+
+          sync_update_part(p.merge(done: true, etag: result.etag))
+
+          checkpoint
+
+          logger.debug("Done upload part: #{p}")
+        end
+
+        # Devide the file into parts to upload
+        def divide_parts
+          logger.info("Begin divide parts, file: #{@file}")
+
+          max_parts = 10000
+          file_size = File.size(@file)
+          part_size = [@options[:part_size] || PART_SIZE, file_size / max_parts].max
+          num_parts = (file_size - 1) / part_size + 1
+          @parts = (1..num_parts).map do |i|
+            {
+              :number => i,
+              :range => [(i-1) * part_size, [i * part_size, file_size].min],
+              :done => false
+            }
+          end
+
+          checkpoint
+
+          logger.info("Done divide parts, parts: #{@parts}")
+        end
+
+        def sync_get_todo_part
+          @todo_mutex.synchronize {
+            @todo_parts.shift
+          }
+        end
+
+        def sync_update_part(p)
+          @all_mutex.synchronize {
+            @parts[p[:number] - 1] = p
+          }
+        end
+
+        def sync_get_all_parts
+          @all_mutex.synchronize {
+            @parts.dup
+          }
+        end
+
+        # Ensure file not changed during uploading
+        def ensure_file_not_changed
+          return if File.mtime(@file) == @file_meta[:mtime]
+
+          if @file_meta[:md5] != get_file_md5(@file)
+            fail FileInconsistentError, "The file to upload is changed."
+          end
+        end
+      end # Upload
+
+    end # Multipart
+  end # OSS
+end # Aliyun

data/lib/aliyun_sdk/oss/util.rb

@@ -0,0 +1,89 @@
+# -*- encoding: utf-8 -*-
+
+require 'time'
+require 'base64'
+require 'openssl'
+require 'digest/md5'
+
+module AliyunSDK
+  module OSS
+    ##
+    # Util functions to help generate formatted Date, signatures,
+    # etc.
+    #
+    module Util
+
+      # Prefix for OSS specific HTTP headers
+      HEADER_PREFIX = "x-oss-"
+
+      class << self
+
+        include Common::Logging
+
+        # Calculate request signatures
+        def get_signature(key, verb, headers, resources)
+          logger.debug("Sign, headers: #{headers}, resources: #{resources}")
+
+          content_md5 = headers['content-md5'] || ""
+          content_type = headers['content-type'] || ""
+          date = headers['date']
+
+          cano_headers = headers.select { |k, v| k.start_with?(HEADER_PREFIX) }
+                         .map { |k, v| [k.downcase.strip, v.strip] }
+                         .sort.map { |k, v| [k, v].join(":") + "\n" }.join
+
+          cano_res = resources[:path] || "/"
+          sub_res = (resources[:sub_res] || {})
+                    .sort.map { |k, v| v ? [k, v].join("=") : k }.join("&")
+          cano_res += "?#{sub_res}" unless sub_res.empty?
+
+          string_to_sign =
+            "#{verb}\n#{content_md5}\n#{content_type}\n#{date}\n" +
+            "#{cano_headers}#{cano_res}"
+
+          Util.sign(key, string_to_sign)
+        end
+
+        # Sign a string using HMAC and BASE64
+        # @param [String] key the secret key
+        # @param [String] string_to_sign the string to sign
+        # @return [String] the signature
+        def sign(key, string_to_sign)
+          logger.debug("String to sign: #{string_to_sign}")
+
+          Base64.strict_encode64(
+            OpenSSL::HMAC.digest('sha1', key, string_to_sign))
+        end
+
+        # Calculate content md5
+        def get_content_md5(content)
+          Base64.strict_encode64(OpenSSL::Digest::MD5.digest(content))
+        end
+
+        # Symbolize keys in Hash, recursively
+        def symbolize(v)
+          if v.is_a?(Hash)
+            result = {}
+            v.each_key { |k| result[k.to_sym] = symbolize(v[k]) }
+            result
+          elsif v.is_a?(Array)
+            result = []
+            v.each { |x| result << symbolize(x) }
+            result
+          else
+            v
+          end
+        end
+
+      end # self
+    end # Util
+  end # OSS
+end # Aliyun
+
+# Monkey patch to support #to_bool
+class String
+  def to_bool
+    return true if self =~ /^true$/i
+    false
+  end
+end