fog-aliyun 0.3.18 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cff686947955adaead48c9bb70ec405a2ca807240ba53d7e86fe1da27b63ab4f
-  data.tar.gz: bec5610976802d1cefccbe9f8788943747cfe31059cf0d6b04d0ae589caace46
+  metadata.gz: 5f527fdb37f55f3f14e68b37fa7d038ba1b351e798d25bc761e18af8e2543c82
+  data.tar.gz: c7b4cdb85fe538f7934eafe57e4108d8fd411084fb3bb3cdd7c1cd0a28714923
 SHA512:
-  metadata.gz: 7bb3975226a2af2d130be85b64f057d61cab8564bd6ef9fc349080efa0008d79dff5149a3124142436bcf30ef893314e27d195b8706c9bc6c634d88c5452a33b
-  data.tar.gz: e21803702da802d2a3051c63428af92b0fb66c9f8a95f62c2c9c76a31ea80fa6ae0ee74081c9ee72d5c0570dae0f60c2764f8094d9692c39f8d93e1ade320ea0
+  metadata.gz: ef5fcc535228173a33fc33294c132ca1e6153bee3e9ac15c9626c06313eaba4e7e0e6a9f88457198e872042ee4a14697d040d772ec6e43203e891c89ee591fb4
+  data.tar.gz: 19054a55dd975915129f3e7cc0893e81aae38a844bc4d04363de0daf2d0cad484ea1cb62496f0813cb466d7689c65d1a3773a75c597ccbb7f5fcc30073e40d7b

data/CHANGELOG.md

@@ -1,4 +1,10 @@
-## 0.3.19 (Unreleased)
+## 0.3.20 (Unreleased)
+## 0.3.19 (August 17, 2020)
+
+IMPROVEMENTS:
+
+- Upgrade oss ruby sdk to support setting log level [GH-152](https://github.com/fog/fog-aliyun/pull/152)
+
 ## 0.3.18 (August 03, 2020)
 
 IMPROVEMENTS:

data/fog-aliyun.gemspec

@@ -28,9 +28,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'memory_profiler'
-  spec.add_development_dependency 'aliyun-sdk', '~> 0.7.3'
+  spec.add_development_dependency 'aliyun-sdk', '~> 0.8.0'
 
-  spec.add_dependency 'aliyun-sdk', '~> 0.7.3'
+  spec.add_dependency 'aliyun-sdk', '~> 0.8.0'
   spec.add_dependency 'fog-core'
   spec.add_dependency 'fog-json'
   spec.add_dependency 'ipaddress', '~> 0.8'

data/lib/fog/aliyun/models/storage/directories.rb

@@ -10,70 +10,47 @@ module Fog
         model Fog::Aliyun::Storage::Directory
 
         def all
-          containers = service.get_containers
-          return nil if containers.nil?
+          buckets = service.get_service[0]
+          return nil if buckets.size < 1
           data = []
           i = 0
-          containers.each do |entry|
-            key = entry['Prefix'][0]
-            key[-1] = ''
-            data[i] = { key: key }
+          buckets.each do |b|
+            data[i] = { key: b.name }
             i += 1
           end
-
           load(data)
         end
 
-        # get method used to get a specified directory.
-        # If the directory is not exist, this method will create a new with 'key'
-        # In order to support multi-buckets scenario which making bucket as a solo directory, it have been expanded.
-        # If key is a directory(including /), return an existed or a new one;
-        # If key does not contain /, if bucket, return '', else return an existed or a new one directory;
+
         def get(key, options = {})
-          if key.is_a? Array
-            key = key[0]
+          data = service.get_bucket(key, options)
+
+          directory = new(:key => key, :is_persisted => true)
+
+          options = data[1]
+          options[:max_keys] = options[:limit]
+          directory.files.merge_attributes(options)
+
+          objects = []
+          i = 0
+          data[0].each do |o|
+            objects[i] = {
+                'Key' => o.key,
+                'Type' => o.type,
+                'Size' => o.size,
+                'ETag' => o.etag,
+                'LastModified' => o.last_modified
+            }
+            i += 1
           end
-          if !key.nil? && key != '' && key != '.'
-            key = key.chomp('/')
-            if key.include? '/'
-              dir = key + '/'
-              ret = service.head_object(dir, options)
-              new(key: key) if ret.data[:status] == 200
-            else
-              remap_attributes(options, {
-                  :delimiter  => 'delimiter',
-                  :marker     => 'marker',
-                  :max_keys   => 'max-keys',
-                  :prefix     => 'prefix'
-              })
-              data = service.get_bucket(key, options)
-              directory = new(:key => data['Name'], :is_persisted => true)
-              options = {}
-              for k, v in data
-                if ['CommonPrefixes', 'Delimiter', 'IsTruncated', 'Marker', 'MaxKeys', 'Prefix'].include?(k)
-                  # Sometimes, the v will be a Array, like "Name"=>["blobstore-droplet1"], "Prefix"=>[{}], "Marker"=>[{}], "MaxKeys"=>["100"], "Delimiter"=>[{}], "IsTruncated"=>["false"]
-                  # and there needs to parse them
-                  if !v.nil? && (v.is_a? Array) && (v.size > 0)
-                    if v[0].is_a? Hash
-                      v = nil
-                    else
-                      v = v[0]
-                    end
-                  end
-                  options[k] = v
-                end
-              end
-              directory.files.merge_attributes(options)
-              if data.key?('Contents') && !data['Contents'].nil?
-                directory.files.load(data['Contents'])
-              end
-              directory
-            end
+          directory.files.load(objects)
+          directory
+        rescue AliyunOssSdk::ServerError => error
+          if error.error_code == "NoSuchBucket"
+            nil
           else
-            new(key: '')
+            raise(error)
           end
-        rescue Fog::Aliyun::Storage::NotFound
-          nil
         end
       end
     end

data/lib/fog/aliyun/models/storage/directory.rb

@@ -7,24 +7,62 @@ module Fog
   module Aliyun
     class Storage
       class Directory < Fog::Model
+        VALID_ACLS = ['private', 'public-read', 'public-read-write']
+
+        attr_reader :acl
         identity :key, :aliases => ['Key', 'Name', 'name']
 
+        attribute :creation_date, :aliases => 'CreationDate', :type => 'time'
+
+        def acl=(new_acl)
+          unless VALID_ACLS.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{VALID_ACLS.join(', ')}]")
+          else
+            @acl = new_acl
+          end
+        end
+
         def destroy
           requires :key
-          prefix = key + '/'
-          ret = service.list_objects(prefix: prefix)['Contents']
-
-          if ret.nil?
+          service.delete_bucket(key)
+          true
+        rescue AliyunOssSdk::ServerError => error
+          if error.error_code == "NoSuchBucket"
             false
-          elsif ret.size == 1
-            service.delete_container(key)
-            true
           else
-            raise Fog::Aliyun::Storage::Error, ' Forbidden: Direction not empty!'
+            raise(error)
+          end
+        end
+
+        def destroy!(options = {})
+          requires :key
+          options = {
+              timeout: Fog.timeout,
+              interval: Fog.interval,
+          }.merge(options)
+
+          begin
+            clear!
+            Fog.wait_for(options[:timeout], options[:interval]) { objects_keys.size == 0 }
+            service.delete_bucket(key)
+            true
+          rescue AliyunOssSdk::ServerError
             false
           end
         end
 
+        def location
+          region = @aliyun_region_id
+          region ||= Storage::DEFAULT_REGION
+          @location = (bucket_location || 'oss-' + region)
+        end
+
+        # NOTE: you can't change the region once the bucket is created
+        def location=(new_location)
+          new_location = 'oss-' + new_location unless new_location.start_with?('oss-')
+          @location = new_location
+        end
+
         def files
           @files ||= begin
             Fog::Aliyun::Storage::Files.new(
@@ -34,6 +72,12 @@ module Fog
           end
         end
 
+        # TODO
+        def public=(new_public)
+          nil
+        end
+
+        # TODO
         def public_url
           nil
         end
@@ -41,18 +85,53 @@ module Fog
         def save
           requires :key
 
-          # Checking whether the key is a bucket and meet the multi-bucket scenario.
-          # If the key is a existing bucket, return it directly.
-          key = key.chomp('/')
-          if !key.nil? && key != '' && key != '.' && !(key.include? '/')
-            data = service.get_bucket(key)
-            if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-              service.put_container(key)
-            end
-          end
+          options = {}
+
+          options['x-oss-acl'] = acl if acl
+
+          # https://help.aliyun.com/document_detail/31959.html
+          # if !persisted?
+          #   # There is a sdk bug that location can not be set
+          #   options[:location] = location
+          # end
+
+          service.put_bucket(key, options)
+          attributes[:is_persisted] = true
 
           true
         end
+
+        def persisted?
+          # is_persisted is true in case of directories.get or after #save
+          # creation_date is set in case of directories.all
+          attributes[:is_persisted] || !!attributes[:creation_date]
+        end
+
+        private
+
+        def bucket_location
+          requires :key
+          return nil unless persisted?
+          service.get_bucket_location(key)
+        end
+
+        def objects_keys
+          requires :key
+          bucket_query = service.get_bucket(key)
+
+          object_keys = []
+          i = 0
+          bucket_query[0].each do |o|
+            object_keys[i] = o.key
+            i += 1
+          end
+          object_keys
+        end
+
+        def clear!
+          requires :key
+          service.delete_multiple_objects(key, objects_keys) if objects_keys.size > 0
+        end
       end
     end
   end

data/lib/fog/aliyun/models/storage/file.rb

@@ -7,77 +7,123 @@ module Fog
     class Storage
       class File < Fog::Model
         identity :key, aliases: ['Key', 'Name', 'name']
+
+        attr_writer :body
+        attribute :cache_control, aliases: 'Cache-Control'
+        attribute :content_encoding, aliases: 'Content-Encoding'
         attribute :date, aliases: 'Date'
-        attribute :content_length, aliases: 'Content-Length', type: :integer
+        attribute :content_length, aliases: ['Content-Length', 'Size'], type: :integer
+        attribute :content_md5, aliases: 'Content-MD5'
         attribute :content_type, aliases: 'Content-Type'
         attribute :connection, aliases: 'Connection'
         attribute :content_disposition, aliases: 'Content-Disposition'
-        attribute :etag, aliases: 'Etag'
+        attribute :etag, aliases: ['Etag', 'ETag']
+        attribute :expires, aliases: 'Expires'
+        attribute :metadata
+        attribute :owner, aliases: 'Owner'
         attribute :last_modified, aliases: 'Last-Modified', type: :time
         attribute :accept_ranges, aliases: 'Accept-Ranges'
         attribute :server, aliases: 'Server'
-        attribute :object_type, aliases: 'x-oss-object-type'
+        attribute :object_type, aliases: ['x-oss-object-type', 'x_oss_object_type']
+
+        # @note Chunk size to use for multipart uploads.
+        #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
+        attr_reader :multipart_chunk_size
+        def multipart_chunk_size=(mp_chunk_size)
+          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          @multipart_chunk_size = mp_chunk_size
+        end
+
+        def acl
+          requires :directory, :key
+          service.get_object_acl(directory.key, key)
+        end
+
+        def acl=(new_acl)
+          valid_acls = ['private', 'public-read', 'public-read-write', 'default']
+          unless valid_acls.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{valid_acls.join(', ')}]")
+          end
+          @acl = new_acl
+        end
 
         def body
-          attributes[:body] ||=
-            if last_modified
-              collection.get(identity).body
-            else
-              ''
-            end
+          return attributes[:body] if attributes[:body]
+          return '' unless last_modified
+
+          file = collection.get(identity)
+          if file
+            attributes[:body] = file.body
+          else
+            attributes[:body] = ''
+          end
         end
 
         def body=(new_body)
           attributes[:body] = new_body
         end
 
-        attr_reader :directory
+        def directory
+          @directory
+        end
 
+        # Copy object from one bucket to other bucket.
+        #
+        #     required attributes: directory, key
+        #
+        # @param target_directory_key [String]
+        # @param target_file_key [String]
+        # @param options [Hash] options for copy_object method
+        # @return [String] Fog::Aliyun::Files#head status of directory contents
+        #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          source_bucket, directory_key = collection.check_directory_key(directory.key)
-          source_object = if directory_key == ''
-                            key
-                          else
-                            directory_key + '/' + key
-                          end
-          target_bucket, target_directory_key = collection.check_directory_key(target_directory_key)
-          target_object = if target_directory_key == ''
-                            target_file_key
-                          else
-                            target_directory_key + '/' + target_file_key
-                          end
-          service.copy_object(source_bucket, source_object, target_bucket, target_object, options)
-          target_directory = service.directories.new(key: target_directory_key)
-          target_directory.files.get(target_file_key)
-        end
-
-        def destroy
+          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          target_directory = service.directories.new(:key => target_directory_key)
+          target_directory.files.head(target_file_key)
+        end
+
+        def destroy(options = {})
           requires :directory, :key
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          service.delete_object(object, bucket: bucket_name)
+          # TODO support versionId
+          # attributes[:body] = nil if options['versionId'] == version
+          service.delete_object(directory.key, key, options)
           true
         end
 
+        remove_method :metadata
         def metadata
-          attributes[:metadata] ||= headers_to_metadata
+          attributes.reject {|key, value| !(key.to_s =~ /^x-oss-/)}
+        end
+
+        remove_method :metadata=
+        def metadata=(new_metadata)
+          merge_attributes(new_metadata)
         end
 
+        remove_method :owner=
         def owner=(new_owner)
           if new_owner
             attributes[:owner] = {
-              display_name: new_owner['DisplayName'],
-              id: new_owner['ID']
+                :display_name => new_owner['DisplayName'] || new_owner[:display_name],
+                :id           => new_owner['ID'] || new_owner[:id]
             }
           end
         end
 
+        # Set Access-Control-List permissions.
+        #
+        #     valid new_publics: public_read, private
+        #
+        # @param [String] new_public
+        # @return [String] new_public
+        #
         def public=(new_public)
+          if new_public
+            @acl = 'public-read'
+          else
+            @acl = 'private'
+          end
           new_public
         end
 
@@ -86,50 +132,32 @@ module Fog
         #     required attributes: directory, key
         #
         # @param expires [String] number of seconds (since 1970-01-01 00:00) before url expires
-        # @param options [Hash]
+        # @param options[Hash] No need to use
         # @return [String] url
         #
         def url(expires, options = {})
-
-          expires = expires.nil? ? 0 : expires.to_i
-
-          requires :directory, :key
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
-        end
-
-        def public_url
           requires :key
-          collection.get_url(key)
+          service.get_object_http_url_public(directory.key, key, expires)
         end
 
         def save(options = {})
           requires :body, :directory, :key
-          options['Content-Type'] = content_type if content_type
+          options['x-oss-object-acl'] ||= @acl if @acl
+          options['Cache-Control'] = cache_control if cache_control
           options['Content-Disposition'] = content_disposition if content_disposition
-          options.merge!(metadata_to_headers)
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          if body.is_a?(::File)
-            service.put_object(object, body, options.merge(bucket: bucket_name))
-            data = service.head_object(object, bucket: bucket_name)
-          elsif body.is_a?(String)
-            data = service.put_object_with_body(object, body, options.merge(bucket: bucket_name)).data
+          options['Content-Encoding'] = content_encoding if content_encoding
+          options['Content-MD5'] = content_md5 if content_md5
+          options['Content-Type'] = content_type if content_type
+          options['Expires'] = expires if expires
+          options.merge!(metadata)
+
+          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+            multipart_save(options)
           else
-            raise Fog::Aliyun::Storage::Error, " Forbidden: Invalid body type: #{body.class}!"
+            service.put_object(directory.key, key, body, options)
           end
-          update_attributes_from(data)
-          refresh_metadata
-
+          self.etag = self.etag.gsub('"','') if self.etag
           self.content_length = Fog::Storage.get_body_size(body)
           self.content_type ||= Fog::Storage.get_content_type(body)
           true
@@ -137,70 +165,43 @@ module Fog
 
         private
 
-        attr_writer :directory
-
-        def refresh_metadata
-          metadata.reject! { |_k, v| v.nil? }
-        end
-
-        def headers_to_metadata
-          key_map = key_mapping
-          Hash[metadata_attributes.map { |k, v| [key_map[k], v] }]
-        end
-
-        def key_mapping
-          key_map = metadata_attributes
-          key_map.each_pair { |k, _v| key_map[k] = header_to_key(k) }
+        def directory=(new_directory)
+          @directory = new_directory
         end
 
-        def header_to_key(opt)
-          opt.gsub(metadata_prefix, '').split('-').map { |k| k[0, 1].downcase + k[1..-1] }.join('_').to_sym
-        end
-
-        def metadata_to_headers
-          header_map = header_mapping
-          Hash[metadata.map { |k, v| [header_map[k], v] }]
-        end
-
-        def header_mapping
-          header_map = metadata.dup
-          header_map.each_pair { |k, _v| header_map[k] = key_to_header(k) }
-        end
-
-        def key_to_header(key)
-          metadata_prefix + key.to_s.split(/[-_]/).map(&:capitalize).join('-')
-        end
+        def multipart_save(options)
+          # Initiate the upload
+          upload_id = service.initiate_multipart_upload(directory.key, key, options)
 
-        def metadata_attributes
-          if last_modified
-            bucket_name, directory_key = collection.check_directory_key(directory.key)
-            object = if directory_key == ''
-                       key
-                     else
-                       directory_key + '/' + key
-                     end
+          # Store ETags of upload parts
+          part_tags = []
 
-            data = service.head_object(object, bucket: bucket_name).data
-            if data[:status] == 200
-              headers = data[:headers]
-              headers.select! { |k, _v| metadata_attribute?(k) }
-            end
-          else
-            {}
+          # Upload each part
+          # TODO: optionally upload chunks in parallel using threads
+          # (may cause network performance problems with many small chunks)
+          # TODO: Support large chunk sizes without reading the chunk into memory
+          if body.respond_to?(:rewind)
+            body.rewind  rescue nil
+          end
+          while (chunk = body.read(multipart_chunk_size)) do
+            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk)
+            part_tags << part_upload
           end
-        end
 
-        def metadata_attribute?(key)
-          key.to_s =~ /^#{metadata_prefix}/
-        end
+          if part_tags.empty? #it is an error to have a multipart upload with no parts
+            part_upload = service.upload_part(directory.key, key, upload_id, 1, '')
+            part_tags << part_upload
+          end
 
-        def metadata_prefix
-          'X-Object-Meta-'
+        rescue
+          # Abort the upload & reraise
+          service.abort_multipart_upload(directory.key, key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
-        def update_attributes_from(data)
-          merge_attributes(data[:headers].reject { |key, _value| ['Content-Length', 'Content-Type'].include?(key) })
-        end
       end
     end
   end