aliyun-sdk 0.1.3 → 0.1.4

data/lib/aliyun/oss/http.rb

@@ -198,8 +198,12 @@ module Aliyun
         headers['Date'] = Time.now.httpdate
         headers['Content-Type'] ||= DEFAULT_CONTENT_TYPE
 
-        if body = http_options[:body] and body.respond_to?(:read)
-          headers['Transfer-Encoding'] = 'chunked'
+        if body = http_options[:body]
+          if body.respond_to?(:read)
+            headers['Transfer-Encoding'] = 'chunked'
+          else
+            headers['Content-MD5'] = Util.get_content_md5(body)
+          end
         end
 
         res = {

data/lib/aliyun/oss/iterator.rb

@@ -69,6 +69,24 @@ module Aliyun
         end
       end # Objects
 
+      ##
+      # Uploads iterator
+      #
+      class Uploads < Base
+        def initialize(protocol, bucket_name, opts = {})
+          super(protocol, opts)
+          @bucket = bucket_name
+        end
+
+        def fetch(more)
+          @results, cont = @protocol.list_multipart_uploads(@bucket, more)
+          @results = cont[:common_prefixes] + @results if cont[:common_prefixes]
+          @more[:id_marker] = cont[:next_id_marker]
+          @more[:key_marker] = cont[:next_key_marker]
+          @more[:truncated] = cont[:truncated] || false
+        end
+      end # Objects
+
     end # Iterator
   end # OSS
 end # Aliyun

data/lib/aliyun/oss/logging.rb

@@ -12,6 +12,8 @@ module Aliyun
     module Logging
 
       DEFAULT_LOG_FILE = "./oss_sdk.log"
+      MAX_NUM_LOG = 100
+      ROTATE_SIZE = 10 * 1024 * 1024
 
       # level = Logger::DEBUG | Logger::INFO | Logger::ERROR | Logger::FATAL
       def self.set_log_level(level)
@@ -20,7 +22,7 @@ module Aliyun
 
       # 设置日志输出的文件
       def self.set_log_file(file)
-        @@log_file = file
+        @log_file = file
       end
 
       # 获取logger
@@ -32,7 +34,8 @@ module Aliyun
 
       def self.logger
         unless @logger
-          @logger = Logger.new(@@log_file ||= DEFAULT_LOG_FILE)
+          @logger = Logger.new(
+            @log_file ||= DEFAULT_LOG_FILE, MAX_NUM_LOG, ROTATE_SIZE)
           @logger.level = Logger::INFO
         end
         @logger

data/lib/aliyun/oss/multipart.rb

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 require 'json'
+require 'digest/md5'
 
 module Aliyun
   module OSS
@@ -22,8 +23,8 @@ module Aliyun
         private
         # Persist transaction states to file
         def write_checkpoint(states, file)
-          states[:md5] = Util.get_content_md5(states.to_json)
-          File.open(file, 'w'){ |f| f.write(states.to_json) }
+          md5= Util.get_content_md5(states.to_json)
+          File.open(file, 'w') { |f| f.write(states.merge(md5: md5).to_json) }
         end
 
         # Load transaction states from file

data/lib/aliyun/oss/object.rb

@@ -8,7 +8,7 @@ module Aliyun
     #
     class Object < Struct::Base
 
-      attrs :key, :type, :size, :etag, :metas, :last_modified
+      attrs :key, :type, :size, :etag, :metas, :last_modified, :content_type
 
     end # Object
   end # OSS

data/lib/aliyun/oss/protocol.rb

@@ -745,7 +745,8 @@ module Aliyun
           :size => wrap(h[:content_length], &:to_i),
           :etag => h[:etag],
           :metas => metas,
-          :last_modified => wrap(h[:last_modified]) { |x| Time.parse(x) })
+          :last_modified => wrap(h[:last_modified]) { |x| Time.parse(x) },
+          :content_type => h[:content_type])
 
         logger.debug("Done get object")
 
@@ -786,7 +787,8 @@ module Aliyun
           :size => wrap(h[:content_length], &:to_i),
           :etag => h[:etag],
           :metas => metas,
-          :last_modified => wrap(h[:last_modified]) { |x| Time.parse(x) })
+          :last_modified => wrap(h[:last_modified]) { |x| Time.parse(x) },
+          :content_type => h[:content_type])
 
         logger.debug("Done get object meta")
 
@@ -871,7 +873,7 @@ module Aliyun
       # @param opts [Hash] options
       # @option opts [Boolean] :quiet indicates whether the server
       #  should return the delete result of the objects
-      # @option opts [String] :encoding-type the encoding type for
+      # @option opts [String] :encoding the encoding type for
       #  object key in the response body, only
       #  {OSS::KeyEncoding::URL} is supported now
       # @return [Array<String>] object names that have been
@@ -881,16 +883,16 @@ module Aliyun
                      "objects: #{object_names}, options: #{opts}")
 
         sub_res = {'delete' => nil}
-        body = Nokogiri::XML::Builder.new do |xml|
-          xml.Delete {
-            xml.Quiet opts[:quiet]? true : false
-            object_names.each do |o|
-              xml.Object {
-                xml.Key o
-              }
-            end
-          }
-        end.to_xml
+
+        # It may have invisible chars in object key which will corrupt
+        # libxml. So we're constructing xml body manually here.
+        body = '<?xml version="1.0"?>'
+        body << '<Delete>'
+        body << '<Quiet>' << (opts[:quiet]? true : false).to_s << '</Quiet>'
+        object_names.each { |k|
+          body << '<Object><Key>' << k << '</Key></Object>'
+        }
+        body << '</Delete>'
 
         query = {}
         query['encoding-type'] = opts[:encoding] if opts[:encoding]
@@ -1144,8 +1146,8 @@ module Aliyun
       # transactions. That is: thoses started and not aborted.
       # @param bucket_name [String] the bucket name
       # @param opts [Hash] options:
-      # @option opts [String] :id_marker return only thoese transactions with
-      #  txn id after :id_marker
+      # @option opts [String] :id_marker return only thoese
+      #  transactions with txn id after :id_marker
       # @option opts [String] :key_marker the object key marker for
       #  a multipart upload transaction.
       #  1. if +:id_marker+ is not set, return only those
@@ -1156,8 +1158,6 @@ module Aliyun
       # @option opts [String] :prefix the prefix of the object key
       #  for a multipart upload transaction. if set only return
       #  those transactions with the object key prefixed with it
-      # @option opts [String] :delimiter the delimiter for the
-      #  object key for a multipart upload transaction.
       # @option opts [String] :encoding the encoding of object key
       #  in the response body. Only {OSS::KeyEncoding::URL} is
       #  supported now.
@@ -1165,7 +1165,6 @@ module Aliyun
       #  the returned transactions and a hash including next tokens,
       #  which includes:
       #  * :prefix [String] the prefix used
-      #  * :delimiter [String] the delimiter used
       #  * :limit [Integer] the limit used
       #  * :id_marker [String] the upload id marker used
       #  * :next_id_marker [String] upload id marker to continue list
@@ -1183,7 +1182,6 @@ module Aliyun
         sub_res = {'uploads' => nil}
         params = {
           'prefix' => opts[:prefix],
-          'delimiter' => opts[:delimiter],
           'upload-id-marker' => opts[:id_marker],
           'key-marker' => opts[:key_marker],
           'max-uploads' => opts[:limit],
@@ -1210,7 +1208,6 @@ module Aliyun
 
         more = {
           :prefix => 'Prefix',
-          :delimiter => 'Delimiter',
           :limit => 'MaxUploads',
           :id_marker => 'UploadIdMarker',
           :next_id_marker => 'NextUploadIdMarker',
@@ -1227,7 +1224,6 @@ module Aliyun
           more, {
             :limit => ->(x) { x.to_i },
             :truncated => ->(x) { x.to_bool },
-            :delimiter => ->(x) { decode_key(x, encoding) },
             :key_marker => ->(x) { decode_key(x, encoding) },
             :next_key_marker => ->(x) { decode_key(x, encoding) }
           }

data/lib/aliyun/oss/upload.rb

@@ -9,16 +9,22 @@ module Aliyun
       class Upload < Transaction
         PART_SIZE = 4 * 1024 * 1024
         READ_SIZE = 16 * 1024
+        NUM_THREAD = 10
 
         def initialize(protocol, opts)
           args = opts.dup
           @protocol = protocol
           @progress = args.delete(:progress)
           @file = args.delete(:file)
-          @checkpoint_file = args.delete(:cpt_file)
+          @cpt_file = args.delete(:cpt_file)
+          super(args)
+
           @file_meta = {}
+          @num_threads = options[:threads] || NUM_THREAD
+          @all_mutex = Mutex.new
           @parts = []
-          super(args)
+          @todo_mutex = Mutex.new
+          @todo_parts = []
         end
 
         # Run the upload transaction, which includes 3 stages:
@@ -27,8 +33,9 @@ module Aliyun
         # * 2.  upload each unfinished part
         # * 3.  commit the multipart upload transaction
         def run
-          logger.info("Begin upload, file: #{@file}, checkpoint file: " \
-                      "#{@checkpoint_file}")
+          logger.info("Begin upload, file: #{@file}, "\
+                      "checkpoint file: #{@cpt_file}, "\
+                      "threads: #{@num_threads}")
 
           # Rebuild transaction states from checkpoint file
           # Or initiate new transaction states
@@ -38,7 +45,17 @@ module Aliyun
           divide_parts if @parts.empty?
 
           # Upload each part
-          @parts.reject { |p| p[:done] }.each { |p| upload_part(p) }
+          @todo_parts = @parts.reject { |p| p[:done] }
+
+          (1..@num_threads).map {
+            Thread.new {
+              loop {
+                p = sync_get_todo_part
+                break unless p
+                upload_part(p)
+              }
+            }
+          }.map(&:join)
 
           # Commit the multipart upload transaction
           commit
@@ -62,24 +79,26 @@ module Aliyun
         #     :md5 => 'states_md5'
         #   }
         def checkpoint
-          logger.debug("Begin make checkpoint, disable_cpt: #{options[:disable_cpt]}")
+          logger.debug("Begin make checkpoint, disable_cpt: "\
+                       "#{options[:disable_cpt] == true}")
 
           ensure_file_not_changed
 
+          parts = sync_get_all_parts
           states = {
             :id => id,
             :file => @file,
             :file_meta => @file_meta,
-            :parts => @parts
+            :parts => parts
           }
 
           # report progress
           if @progress
-            done = @parts.count { |p| p[:done] }
-            @progress.call(done.to_f / @parts.size) if done > 0
+            done = parts.count { |p| p[:done] }
+            @progress.call(done.to_f / parts.size) if done > 0
           end
 
-          write_checkpoint(states, @checkpoint_file) unless options[:disable_cpt]
+          write_checkpoint(states, @cpt_file) unless options[:disable_cpt]
 
           logger.debug("Done make checkpoint, states: #{states}")
         end
@@ -91,20 +110,24 @@ module Aliyun
         def commit
           logger.info("Begin commit transaction, id: #{id}")
 
-          parts = @parts.map{ |p| Part.new(:number  => p[:number], :etag => p[:etag])}
+          parts = sync_get_all_parts.map{ |p|
+            Part.new(:number  => p[:number], :etag => p[:etag])
+          }
           @protocol.complete_multipart_upload(bucket, object, id, parts)
 
-          File.delete(@checkpoint_file) unless options[:disable_cpt]
+          File.delete(@cpt_file) unless options[:disable_cpt]
 
           logger.info("Done commit transaction, id: #{id}")
         end
 
         # Rebuild the states of the transaction from checkpoint file
         def rebuild
-          logger.info("Begin rebuild transaction, checkpoint: #{@checkpoint_file}")
+          logger.info("Begin rebuild transaction, checkpoint: #{@cpt_file}")
 
-          if File.exists?(@checkpoint_file) and not options[:disable_cpt]
-            states = load_checkpoint(@checkpoint_file)
+          if options[:disable_cpt] || !File.exists?(@cpt_file)
+            initiate
+          else
+            states = load_checkpoint(@cpt_file)
 
             if states[:file_md5] != @file_meta[:md5]
               fail FileInconsistentError.new("The file to upload is changed.")
@@ -113,8 +136,6 @@ module Aliyun
             @id = states[:id]
             @file_meta = states[:file_meta]
             @parts = states[:parts]
-          else
-            initiate
           end
 
           logger.info("Done rebuild transaction, states: #{states}")
@@ -151,8 +172,8 @@ module Aliyun
               end
             end
           end
-          p[:done] = true
-          p[:etag] = result.etag
+
+          sync_update_part(p.merge(done: true, etag: result.etag))
 
           checkpoint
 
@@ -180,6 +201,24 @@ module Aliyun
           logger.info("Done divide parts, parts: #{@parts}")
         end
 
+        def sync_get_todo_part
+          @todo_mutex.synchronize {
+            @todo_parts.shift
+          }
+        end
+
+        def sync_update_part(p)
+          @all_mutex.synchronize {
+            @parts[p[:number] - 1] = p
+          }
+        end
+
+        def sync_get_all_parts
+          @all_mutex.synchronize {
+            @parts.dup
+          }
+        end
+
         # Ensure file not changed during uploading
         def ensure_file_not_changed
           return if File.mtime(@file) == @file_meta[:mtime]

data/lib/aliyun/oss/util.rb

@@ -3,6 +3,7 @@
 require 'time'
 require 'base64'
 require 'openssl'
+require 'digest/md5'
 
 module Aliyun
   module OSS

data/lib/aliyun/oss/version.rb

@@ -3,7 +3,7 @@
 module Aliyun
   module OSS
 
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
 
   end # OSS
 end # Aliyun

data/spec/aliyun/oss/client/bucket_spec.rb

@@ -62,6 +62,33 @@ module Aliyun
         end.to_xml
       end
 
+      def mock_uploads(txns, more = {})
+        Nokogiri::XML::Builder.new do |xml|
+          xml.ListMultipartUploadsResult {
+            {
+              :prefix => 'Prefix',
+              :delimiter => 'Delimiter',
+              :limit => 'MaxUploads',
+              :key_marker => 'KeyMarker',
+              :id_marker => 'UploadIdMarker',
+              :next_key_marker => 'NextKeyMarker',
+              :next_id_marker => 'NextUploadIdMarker',
+              :truncated => 'IsTruncated',
+              :encoding => 'EncodingType'
+            }.map do |k, v|
+              xml.send(v, more[k]) if more[k] != nil
+            end
+
+            txns.each do |t|
+              xml.Upload {
+                xml.Key t.object
+                xml.UploadId t.id
+              }
+            end
+          }
+        end.to_xml
+      end
+
       def mock_acl(acl)
         Nokogiri::XML::Builder.new do |xml|
           xml.AccessControlPolicy {
@@ -133,7 +160,8 @@ module Aliyun
         it "should list objects" do
           query_1 = {
             :prefix => 'list-',
-            :delimiter => '-'
+            :delimiter => '-',
+            'encoding-type' => 'url'
           }
           return_obj_1 = (1..5).map{ |i| Object.new(
             :key => "obj-#{i}",
@@ -148,7 +176,8 @@ module Aliyun
           query_2 = {
             :prefix => 'list-',
             :delimiter => '-',
-            :marker => 'foo'
+            :marker => 'foo',
+            'encoding-type' => 'url'
           }
           return_obj_2 = (6..8).map{ |i| Object.new(
             :key => "obj-#{i}",
@@ -337,6 +366,63 @@ module Aliyun
         end
       end # object operations
 
+      context "multipart operations" do
+        it "should list uploads" do
+          query_1 = {
+            :prefix => 'list-',
+            'encoding-type' => 'url',
+            'uploads' => ''
+          }
+          return_up_1 = (1..5).map{ |i| Multipart::Transaction.new(
+            :id => "txn-#{i}",
+            :object => "my-object",
+            :bucket => @bucket_name
+          )}
+          return_more_1 = {
+            :next_id_marker => "txn-5",
+            :truncated => true
+          }
+
+          query_2 = {
+            :prefix => 'list-',
+            'upload-id-marker' => 'txn-5',
+            'encoding-type' => 'url',
+            'uploads' => ''
+          }
+          return_up_2 = (6..8).map{ |i| Multipart::Transaction.new(
+            :id => "txn-#{i}",
+            :object => "my-object",
+            :bucket => @bucket_name
+          )}
+          return_more_2 = {
+            :next_id_marker => 'txn-8',
+            :truncated => false,
+          }
+
+          stub_request(:get, bucket_url)
+            .with(:query => query_1)
+            .to_return(:body => mock_uploads(return_up_1, return_more_1))
+
+          stub_request(:get, bucket_url)
+            .with(:query => query_2)
+            .to_return(:body => mock_uploads(return_up_2, return_more_2))
+
+          txns = @bucket.list_uploads(prefix: 'list-').to_a
+
+          expect(WebMock).to have_requested(:get, bucket_url)
+                         .with(:query => query_1).times(1)
+          expect(WebMock).to have_requested(:get, bucket_url)
+                         .with(:query => query_2).times(1)
+
+          all_txns = (1..8).map{ |i| Multipart::Transaction.new(
+            :id => "txn-#{i}",
+            :object => "my-object",
+            :bucket => @bucket_name
+          )}
+          expect(txns.map(&:to_s)).to match_array(all_txns.map(&:to_s))
+        end
+      end # multipart operations
+
     end # Bucket
   end # OSS
 end # Aliyun