aliyun-sdk 0.1.1

data/lib/aliyun/oss/struct.rb

@@ -0,0 +1,199 @@
+# -*- encoding: utf-8 -*-
+
+module Aliyun
+  module OSS
+
+    ##
+    # Access Control List, it controls how the bucket/object can be
+    # accessed.
+    # * public-read-write: allow access(read&write) anonymously
+    # * public-read: allow read anonymously
+    # * private: access must be signatured
+    #
+    module ACL
+      PUBLIC_READ_WRITE = "public-read-write"
+      PUBLIC_READ = "public-read"
+      PRIVATE = "private"
+    end # ACL
+
+    ##
+    # A OSS object may carry some metas(String key-value pairs) with
+    # it. MetaDirective specifies what to do with the metas in the
+    # copy process.
+    # * COPY: metas are copied from the source object to the dest
+    #   object
+    # * REPLACE: source object's metas are NOT copied, use user
+    #   provided metas for the dest object
+    #
+    module MetaDirective
+      COPY = "COPY"
+      REPLACE = "REPLACE"
+    end # MetaDirective
+
+    ##
+    # The object key may contains unicode charactors which cannot be
+    # encoded in the request/response body(XML). KeyEncoding specifies
+    # the encoding type for the object key.
+    # * url: the object key is url-encoded
+    # @note url-encoding is the only supported KeyEncoding type
+    #
+    module KeyEncoding
+      URL = "url"
+
+      @@all = [URL]
+
+      def self.include?(enc)
+        all.include?(enc)
+      end
+
+      def self.all
+        @@all
+      end
+    end # KeyEncoding
+
+    ##
+    # Common structs used. It provides a 'attrs' helper method for
+    # subclass to define its attributes. 'attrs' is based on
+    # attr_reader and provide additional functionalities for classes
+    # that inherits Struct::Base :
+    # * the constuctor is provided to accept options and set the
+    #   corresponding attibute automatically
+    # * the #to_s method is rewrite to concatenate the defined
+    #   attributes keys and values
+    # @example
+    #   class X < Struct::Base
+    #     attrs :foo, :bar
+    #   end
+    #
+    #   x.new(:foo => 'hello', :bar => 'world')
+    #   x.foo # == "hello"
+    #   x.bar # == "world"
+    #   x.to_s # == "foo: hello, bar: world"
+    module Struct
+      class Base
+        module AttrHelper
+          def attrs(*s)
+            define_method(:attrs) {s}
+            attr_reader(*s)
+          end
+        end
+
+        extend AttrHelper
+
+        def initialize(opts = {})
+          extra_keys = opts.keys - attrs
+          unless extra_keys.empty?
+            fail ClientError, "Unexpected extra keys: #{extra_keys.join(', ')}"
+          end
+
+          attrs.each do |attr|
+            instance_variable_set("@#{attr}", opts[attr])
+          end
+        end
+
+        def to_s
+          attrs.map do |attr|
+            v = instance_variable_get("@#{attr}")
+            "#{attr.to_s}: #{v}"
+          end.join(", ")
+        end
+      end # Base
+    end # Struct
+
+    ##
+    # Bucket Logging setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/logging.html OSS Bucket logging}
+    # Attributes:
+    # * enable [Boolean] whether to enable bucket logging
+    # * target_bucket [String] the target bucket to store access logs
+    # * target_prefix [String] the target object prefix to store access logs
+    # @example Enable bucket logging
+    #  bucket.logging = BucketLogging.new(
+    #    :enable => true, :target_bucket => 'log_bucket', :target_prefix => 'my-log')
+    # @example Disable bucket logging
+    #  bucket.logging = BucketLogging.new(:enable => false)
+    class BucketLogging < Struct::Base
+      attrs :enable, :target_bucket, :target_prefix
+
+      def enabled?
+        enable == true
+      end
+    end
+
+    ##
+    # Bucket website setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/host-static-website.html OSS Website hosting}
+    # Attributes:
+    # * enable [Boolean] whether to enable website hosting for the bucket
+    # * index [String] the index object as the index page for the website
+    # * error [String] the error object as the error page for the website
+    class BucketWebsite < Struct::Base
+      attrs :enable, :index, :error
+
+      def enabled?
+        enable == true
+      end
+    end
+
+    ##
+    # Bucket referer setting. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS Website hosting}
+    # Attributes:
+    # * allow_empty [Boolean] whether to allow requests with empty "Referer"
+    # * whitelist [Array<String>] the allowed origins for requests
+    class BucketReferer < Struct::Base
+      attrs :allow_empty, :whitelist
+
+      def allow_empty?
+        allow_empty == true
+      end
+    end
+
+    ##
+    # LifeCycle rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/lifecycle.html OSS Bucket LifeCycle}
+    # Attributes:
+    # * id [String] the unique id of a rule
+    # * enabled [Boolean] whether to enable this rule
+    # * prefix [String] the prefix objects to apply this rule
+    # * expiry [Date] or [Fixnum] the expire time of objects
+    #   * if expiry is a Date, it specifies the absolute date to
+    #     expire objects
+    #   * if expiry is a Fixnum, it specifies the relative date to
+    #     expire objects: how many days after the object's last
+    #     modification time to expire the object
+    # @example Specify expiry as Date
+    #   LifeCycleRule.new(
+    #     :id => 'rule1',
+    #     :enabled => true,
+    #     :prefix => 'foo/',
+    #     :expiry => Date.new(2016, 1, 1))
+    # @example Specify expiry as days
+    #   LifeCycleRule.new(
+    #     :id => 'rule1',
+    #     :enabled => true,
+    #     :prefix => 'foo/',
+    #     :expiry => 15)
+    # @note the expiry date is treated as UTC time
+    class LifeCycleRule < Struct::Base
+
+      attrs :id, :enable, :prefix, :expiry
+
+      def enabled?
+        enable == true
+      end
+    end # LifeCycleRule
+
+    ##
+    # CORS rule for bucket. See: {http://help.aliyun.com/document_detail/oss/product-documentation/function/referer-white-list.html OSS CORS}
+    # Attributes:
+    # * allowed_origins [Array<String>] the allowed origins
+    # * allowed_methods [Array<String>] the allowed methods
+    # * allowed_headers [Array<String>] the allowed headers
+    # * expose_headers [Array<String>] the expose headers
+    # * max_age_seconds [Integer] the max age seconds
+    class CORSRule < Struct::Base
+
+      attrs :allowed_origins, :allowed_methods, :allowed_headers,
+            :expose_headers, :max_age_seconds
+
+    end # CORSRule
+
+  end # OSS
+end # Aliyun

data/lib/aliyun/oss/upload.rb

@@ -0,0 +1,195 @@
+# -*- encoding: utf-8 -*-
+
+module Aliyun
+  module OSS
+    module Multipart
+      ##
+      # A multipart upload transaction
+      #
+      class Upload < Transaction
+        PART_SIZE = 4 * 1024 * 1024
+        READ_SIZE = 16 * 1024
+
+        def initialize(protocol, opts)
+          args = opts.dup
+          @protocol = protocol
+          @progress = args.delete(:progress)
+          @file = args.delete(:file)
+          @checkpoint_file = args.delete(:cpt_file)
+          @file_meta = {}
+          @parts = []
+          super(args)
+        end
+
+        # Run the upload transaction, which includes 3 stages:
+        # * 1a. initiate(new upload) and divide parts
+        # * 1b. rebuild states(resumed upload)
+        # * 2.  upload each unfinished part
+        # * 3.  commit the multipart upload transaction
+        def run
+          logger.info("Begin upload, file: #{@file}, checkpoint file: " \
+                      "#{@checkpoint_file}")
+
+          # Rebuild transaction states from checkpoint file
+          # Or initiate new transaction states
+          rebuild
+
+          # Divide the file to upload into parts to upload separately
+          divide_parts if @parts.empty?
+
+          # Upload each part
+          @parts.reject { |p| p[:done] }.each { |p| upload_part(p) }
+
+          # Commit the multipart upload transaction
+          commit
+
+          logger.info("Done upload, file: #{@file}")
+        end
+
+        # Checkpoint structures:
+        # @example
+        #   states = {
+        #     :id => 'upload_id',
+        #     :file => 'file',
+        #     :file_meta => {
+        #       :mtime => Time.now,
+        #       :md5 => 1024
+        #     },
+        #     :parts => [
+        #       {:number => 1, :range => [0, 100], :done => false},
+        #       {:number => 2, :range => [100, 200], :done => true}
+        #     ],
+        #     :md5 => 'states_md5'
+        #   }
+        def checkpoint
+          logger.debug("Begin make checkpoint, disable_cpt: #{options[:disable_cpt]}")
+
+          ensure_file_not_changed
+
+          states = {
+            :id => id,
+            :file => @file,
+            :file_meta => @file_meta,
+            :parts => @parts
+          }
+
+          # report progress
+          if @progress
+            done = @parts.count { |p| p[:done] }
+            @progress.call(done.to_f / @parts.size) if done > 0
+          end
+
+          write_checkpoint(states, @checkpoint_file) unless options[:disable_cpt]
+
+          logger.debug("Done make checkpoint, states: #{states}")
+        end
+
+        private
+        # Commit the transaction when all parts are succefully uploaded
+        # @todo handle undefined behaviors: commit succeeds in server
+        #  but return error in client
+        def commit
+          logger.info("Begin commit transaction, id: #{id}")
+
+          parts = @parts.map{ |p| Part.new(:number  => p[:number], :etag => p[:etag])}
+          @protocol.complete_multipart_upload(bucket, object, id, parts)
+
+          File.delete(@checkpoint_file) unless options[:disable_cpt]
+
+          logger.info("Done commit transaction, id: #{id}")
+        end
+
+        # Rebuild the states of the transaction from checkpoint file
+        def rebuild
+          logger.info("Begin rebuild transaction, checkpoint: #{@checkpoint_file}")
+
+          if File.exists?(@checkpoint_file) and not options[:disable_cpt]
+            states = load_checkpoint(@checkpoint_file)
+
+            if states[:file_md5] != @file_meta[:md5]
+              fail FileInconsistentError.new("The file to upload is changed.")
+            end
+
+            @id = states[:id]
+            @file_meta = states[:file_meta]
+            @parts = states[:parts]
+          else
+            initiate
+          end
+
+          logger.info("Done rebuild transaction, states: #{states}")
+        end
+
+        def initiate
+          logger.info("Begin initiate transaction")
+
+          @id = @protocol.initiate_multipart_upload(bucket, object, options)
+          @file_meta = {
+            :mtime => File.mtime(@file),
+            :md5 => get_file_md5(@file)
+          }
+          checkpoint
+
+          logger.info("Done initiate transaction, id: #{id}")
+        end
+
+        # Upload a part
+        def upload_part(p)
+          logger.debug("Begin upload part: #{p}")
+
+          result = nil
+          File.open(@file) do |f|
+            range = p[:range]
+            pos = range.first
+            f.seek(pos)
+
+            result = @protocol.upload_part(bucket, object, id, p[:number]) do |sw|
+              while pos < range.at(1)
+                bytes = [READ_SIZE, range.at(1) - pos].min
+                sw << f.read(bytes)
+                pos += bytes
+              end
+            end
+          end
+          p[:done] = true
+          p[:etag] = result.etag
+
+          checkpoint
+
+          logger.debug("Done upload part: #{p}")
+        end
+
+        # Devide the file into parts to upload
+        def divide_parts
+          logger.info("Begin divide parts, file: #{@file}")
+
+          max_parts = 10000
+          file_size = File.size(@file)
+          part_size = [@options[:part_size] || PART_SIZE, file_size / max_parts].max
+          num_parts = (file_size - 1) / part_size + 1
+          @parts = (1..num_parts).map do |i|
+            {
+              :number => i,
+              :range => [(i-1) * part_size, [i * part_size, file_size].min],
+              :done => false
+            }
+          end
+
+          checkpoint
+
+          logger.info("Done divide parts, parts: #{@parts}")
+        end
+
+        # Ensure file not changed during uploading
+        def ensure_file_not_changed
+          return if File.mtime(@file) == @file_meta[:mtime]
+
+          if @file_meta[:md5] != get_file_md5(@file)
+            fail FileInconsistentError, "The file to upload is changed."
+          end
+        end
+      end # Upload
+
+    end # Multipart
+  end # OSS
+end # Aliyun

data/lib/aliyun/oss/util.rb

@@ -0,0 +1,88 @@
+# -*- encoding: utf-8 -*-
+
+require 'time'
+require 'base64'
+require 'openssl'
+
+module Aliyun
+  module OSS
+    ##
+    # Util functions to help generate formatted Date, signatures,
+    # etc.
+    #
+    module Util
+
+      # Prefix for OSS specific HTTP headers
+      HEADER_PREFIX = "x-oss-"
+
+      class << self
+
+        include Logging
+
+        # Calculate request signatures
+        def get_signature(key, verb, headers, resources)
+          logger.debug("Sign, headers: #{headers}, resources: #{resources}")
+
+          content_md5 = headers['Content-MD5'] || ""
+          content_type = headers['Content-Type'] || ""
+          date = headers['Date']
+
+          cano_headers = headers.select { |k, v| k.start_with?(HEADER_PREFIX) }
+                         .map { |k, v| [k.downcase.strip, v.strip] }
+                         .sort.map { |k, v| [k, v].join(":") + "\n" }.join
+
+          cano_res = resources[:path] || "/"
+          sub_res = (resources[:sub_res] || {})
+                    .sort.map { |k, v| v ? [k, v].join("=") : k }.join("&")
+          cano_res += "?#{sub_res}" unless sub_res.empty?
+
+          string_to_sign =
+            "#{verb}\n#{content_md5}\n#{content_type}\n#{date}\n" +
+            "#{cano_headers}#{cano_res}"
+
+          logger.debug("String to sign: #{string_to_sign}")
+
+          Util.sign(key, string_to_sign)
+        end
+
+        # Sign a string using HMAC and BASE64
+        # @param [String] key the secret key
+        # @param [String] string_to_sign the string to sign
+        # @return [String] the signature
+        def sign(key, string_to_sign)
+          Base64.strict_encode64(
+            OpenSSL::HMAC.digest('sha1', key, string_to_sign))
+        end
+
+        # Calculate content md5
+        def get_content_md5(content)
+          Base64.strict_encode64(OpenSSL::Digest::MD5.digest(content))
+        end
+
+      end # self
+    end # Util
+  end # OSS
+end # Aliyun
+
+# Monkey patch to support #to_bool
+class String
+  def to_bool
+    return true if self =~ /^true$/i
+    false
+  end
+end
+
+# Monkey patch to support #symbolize_keys!
+class Array
+  def symbolize_keys!
+    self.each { |v| v.symbolize_keys! if v.is_a?(Hash) or v.is_a?(Array) }
+  end
+end
+
+# Monkey patch to support #symbolize_keys!
+class Hash
+  def symbolize_keys!
+    self.keys.each { |k| self[k.to_sym] = self.delete(k) }
+    self.values.each { |v| v.symbolize_keys! if v.is_a?(Hash) or v.is_a?(Array) }
+  end
+end