fog-aliyun 0.3.3 → 0.4.0

data/lib/fog/aliyun/models/storage/directory.rb

@@ -4,46 +4,134 @@ require 'fog/core/model'
 require 'fog/aliyun/models/storage/files'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directory < Fog::Model
-        identity :key
+        VALID_ACLS = ['private', 'public-read', 'public-read-write']
+
+        attr_reader :acl
+        identity :key, :aliases => ['Key', 'Name', 'name']
+
+        attribute :creation_date, :aliases => 'CreationDate', :type => 'time'
+
+        def acl=(new_acl)
+          unless VALID_ACLS.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{VALID_ACLS.join(', ')}]")
+          else
+            @acl = new_acl
+          end
+        end
 
         def destroy
           requires :key
-          prefix = key + '/'
-          ret = service.list_objects(prefix: prefix)['Contents']
-
-          if ret.nil?
-            puts ' Not found: Direction not exist!'
+          service.delete_bucket(key)
+          true
+        rescue AliyunOssSdk::ServerError => error
+          if error.error_code == "NoSuchBucket"
             false
-          elsif ret.size == 1
-            service.delete_container(key)
-            true
           else
-            raise Fog::Storage::Aliyun::Error, ' Forbidden: Direction not empty!'
+            raise(error)
+          end
+        end
+
+        def destroy!(options = {})
+          requires :key
+          options = {
+              timeout: Fog.timeout,
+              interval: Fog.interval,
+          }.merge(options)
+
+          begin
+            clear!
+            Fog.wait_for(options[:timeout], options[:interval]) { objects_keys.size == 0 }
+            service.delete_bucket(key)
+            true
+          rescue AliyunOssSdk::ServerError
             false
           end
         end
 
+        def location
+          region = @aliyun_region_id
+          region ||= Storage::DEFAULT_REGION
+          @location = (bucket_location || 'oss-' + region)
+        end
+
+        # NOTE: you can't change the region once the bucket is created
+        def location=(new_location)
+          new_location = 'oss-' + new_location unless new_location.start_with?('oss-')
+          @location = new_location
+        end
+
         def files
           @files ||= begin
-            Fog::Storage::Aliyun::Files.new(
+            Fog::Aliyun::Storage::Files.new(
               directory: self,
               service: service
             )
           end
         end
 
+        # TODO
+        def public=(new_public)
+          nil
+        end
+
+        # TODO
         def public_url
           nil
         end
 
         def save
           requires :key
-          service.put_container(key)
+
+          options = {}
+
+          options['x-oss-acl'] = acl if acl
+
+          # https://help.aliyun.com/document_detail/31959.html
+          # if !persisted?
+          #   # There is a sdk bug that location can not be set
+          #   options[:location] = location
+          # end
+
+          service.put_bucket(key, options)
+          attributes[:is_persisted] = true
+
           true
         end
+
+        def persisted?
+          # is_persisted is true in case of directories.get or after #save
+          # creation_date is set in case of directories.all
+          attributes[:is_persisted] || !!attributes[:creation_date]
+        end
+
+        private
+
+        def bucket_location
+          requires :key
+          return nil unless persisted?
+          service.get_bucket_location(key)
+        end
+
+        def objects_keys
+          requires :key
+          bucket_query = service.get_bucket(key)
+
+          object_keys = []
+          i = 0
+          bucket_query[0].each do |o|
+            object_keys[i] = o.key
+            i += 1
+          end
+          object_keys
+        end
+
+        def clear!
+          requires :key
+          service.delete_multiple_objects(key, objects_keys) if objects_keys.size > 0
+        end
       end
     end
   end

data/lib/fog/aliyun/models/storage/file.rb

@@ -3,78 +3,127 @@
 require 'fog/core/model'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class File < Fog::Model
-        identity :key, aliases: 'name'
+        identity :key, aliases: ['Key', 'Name', 'name']
+
+        attr_writer :body
+        attribute :cache_control, aliases: 'Cache-Control'
+        attribute :content_encoding, aliases: 'Content-Encoding'
         attribute :date, aliases: 'Date'
-        attribute :content_length, aliases: 'Content-Length', type: :integer
+        attribute :content_length, aliases: ['Content-Length', 'Size'], type: :integer
+        attribute :content_md5, aliases: 'Content-MD5'
         attribute :content_type, aliases: 'Content-Type'
         attribute :connection, aliases: 'Connection'
         attribute :content_disposition, aliases: 'Content-Disposition'
-        attribute :etag, aliases: 'Etag'
+        attribute :etag, aliases: ['Etag', 'ETag']
+        attribute :expires, aliases: 'Expires'
+        attribute :metadata
+        attribute :owner, aliases: 'Owner'
         attribute :last_modified, aliases: 'Last-Modified', type: :time
         attribute :accept_ranges, aliases: 'Accept-Ranges'
         attribute :server, aliases: 'Server'
-        attribute :object_type, aliases: 'x-oss-object-type'
+        attribute :object_type, aliases: ['x-oss-object-type', 'x_oss_object_type']
+
+        # @note Chunk size to use for multipart uploads.
+        #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
+        attr_reader :multipart_chunk_size
+        def multipart_chunk_size=(mp_chunk_size)
+          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          @multipart_chunk_size = mp_chunk_size
+        end
+
+        def acl
+          requires :directory, :key
+          service.get_object_acl(directory.key, key)
+        end
+
+        def acl=(new_acl)
+          valid_acls = ['private', 'public-read', 'public-read-write', 'default']
+          unless valid_acls.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{valid_acls.join(', ')}]")
+          end
+          @acl = new_acl
+        end
 
         def body
-          attributes[:body] ||=
-            if last_modified
-              collection.get(identity).body
-            else
-              ''
-            end
+          return attributes[:body] if attributes[:body]
+          return '' unless last_modified
+
+          file = collection.get(identity)
+          if file
+            attributes[:body] = file.body
+          else
+            attributes[:body] = ''
+          end
         end
 
         def body=(new_body)
           attributes[:body] = new_body
         end
 
-        attr_reader :directory
+        def directory
+          @directory
+        end
 
+        # Copy object from one bucket to other bucket.
+        #
+        #     required attributes: directory, key
+        #
+        # @param target_directory_key [String]
+        # @param target_file_key [String]
+        # @param options [Hash] options for copy_object method
+        # @return [String] Fog::Aliyun::Files#head status of directory contents
+        #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          source_object = if directory.key == ''
-                            key
-                          else
-                            directory.key + '/' + key
-                          end
-          target_object = if target_directory_key == ''
-                            target_file_key
-                          else
-                            target_directory_key + '/' + target_file_key
-                          end
-          service.copy_object(nil, source_object, nil, target_object, options)
-          target_directory = service.directories.new(key: target_directory_key)
-          target_directory.files.get(target_file_key)
-        end
-
-        def destroy
+          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          target_directory = service.directories.new(:key => target_directory_key)
+          target_directory.files.head(target_file_key)
+        end
+
+        def destroy(options = {})
           requires :directory, :key
-          object = if directory.key == ''
-                     key
-                   else
-                     directory.key + '/' + key
-                   end
-          service.delete_object(object)
+          # TODO support versionId
+          # attributes[:body] = nil if options['versionId'] == version
+          service.delete_object(directory.key, key, options)
           true
         end
 
+        remove_method :metadata
         def metadata
-          attributes[:metadata] ||= headers_to_metadata
+          attributes.reject {|key, value| !(key.to_s =~ /^x-oss-/)}
+        end
+
+        remove_method :metadata=
+        def metadata=(new_metadata)
+          merge_attributes(new_metadata)
         end
 
+        remove_method :owner=
         def owner=(new_owner)
           if new_owner
             attributes[:owner] = {
-              display_name: new_owner['DisplayName'],
-              id: new_owner['ID']
+                :display_name => new_owner['DisplayName'] || new_owner[:display_name],
+                :id           => new_owner['ID'] || new_owner[:id]
             }
           end
         end
 
+        # Set Access-Control-List permissions.
+        #
+        #     valid new_publics: public_read, private
+        #
+        # @param [String] new_public
+        # @return [String] new_public
+        #
         def public=(new_public)
+          if new_public
+            @acl = 'public-read'
+          else
+            @acl = 'private'
+          end
           new_public
         end
 
@@ -83,48 +132,32 @@ module Fog
         #     required attributes: directory, key
         #
         # @param expires [String] number of seconds (since 1970-01-01 00:00) before url expires
-        # @param options [Hash]
+        # @param options[Hash] No need to use
         # @return [String] url
         #
         def url(expires, options = {})
-
-          expires = expires.nil? ? 0 : expires.to_i
-
-          requires :directory, :key
-          object = if directory.key == ''
-                     key
-                   else
-                     directory.key + '/' + key
-                   end
-          service.get_object_http_url_public(object, expires, options)
-        end
-
-        def public_url
           requires :key
-          collection.get_url(key)
+          service.get_object_http_url_public(directory.key, key, expires)
         end
 
         def save(options = {})
           requires :body, :directory, :key
-          options['Content-Type'] = content_type if content_type
+          options['x-oss-object-acl'] ||= @acl if @acl
+          options['Cache-Control'] = cache_control if cache_control
           options['Content-Disposition'] = content_disposition if content_disposition
-          options.merge!(metadata_to_headers)
-
-          object = if directory.key == ''
-                     key
-                   else
-                     directory.key + '/' + key
-                   end
-          if body.is_a?(::File)
-            data = service.put_object(object, body, options).data
-          elsif body.is_a?(String)
-            data = service.put_object_with_body(object, body, options).data
+          options['Content-Encoding'] = content_encoding if content_encoding
+          options['Content-MD5'] = content_md5 if content_md5
+          options['Content-Type'] = content_type if content_type
+          options['Expires'] = expires if expires
+          options.merge!(metadata)
+
+          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+            multipart_save(options)
           else
-            raise Fog::Storage::Aliyun::Error, " Forbidden: Invalid body type: #{body.class}!"
+            service.put_object(directory.key, key, body, options)
           end
-          update_attributes_from(data)
-          refresh_metadata
-
+          self.etag = self.etag.gsub('"','') if self.etag
           self.content_length = Fog::Storage.get_body_size(body)
           self.content_type ||= Fog::Storage.get_content_type(body)
           true
@@ -132,66 +165,43 @@ module Fog
 
         private
 
-        attr_writer :directory
-
-        def refresh_metadata
-          metadata.reject! { |_k, v| v.nil? }
-        end
-
-        def headers_to_metadata
-          key_map = key_mapping
-          Hash[metadata_attributes.map { |k, v| [key_map[k], v] }]
-        end
-
-        def key_mapping
-          key_map = metadata_attributes
-          key_map.each_pair { |k, _v| key_map[k] = header_to_key(k) }
+        def directory=(new_directory)
+          @directory = new_directory
         end
 
-        def header_to_key(opt)
-          opt.gsub(metadata_prefix, '').split('-').map { |k| k[0, 1].downcase + k[1..-1] }.join('_').to_sym
-        end
-
-        def metadata_to_headers
-          header_map = header_mapping
-          Hash[metadata.map { |k, v| [header_map[k], v] }]
-        end
-
-        def header_mapping
-          header_map = metadata.dup
-          header_map.each_pair { |k, _v| header_map[k] = key_to_header(k) }
-        end
-
-        def key_to_header(key)
-          metadata_prefix + key.to_s.split(/[-_]/).map(&:capitalize).join('-')
-        end
+        def multipart_save(options)
+          # Initiate the upload
+          upload_id = service.initiate_multipart_upload(directory.key, key, options)
 
-        def metadata_attributes
-          if last_modified
-            object = if directory.key == ''
-                       key
-                     else
-                       directory.key + '/' + key
-                     end
+          # Store ETags of upload parts
+          part_tags = []
 
-            headers = service.head_object(object).data[:headers]
-            headers.select! { |k, _v| metadata_attribute?(k) }
-          else
-            {}
+          # Upload each part
+          # TODO: optionally upload chunks in parallel using threads
+          # (may cause network performance problems with many small chunks)
+          # TODO: Support large chunk sizes without reading the chunk into memory
+          if body.respond_to?(:rewind)
+            body.rewind  rescue nil
+          end
+          while (chunk = body.read(multipart_chunk_size)) do
+            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk)
+            part_tags << part_upload
           end
-        end
 
-        def metadata_attribute?(key)
-          key.to_s =~ /^#{metadata_prefix}/
-        end
+          if part_tags.empty? #it is an error to have a multipart upload with no parts
+            part_upload = service.upload_part(directory.key, key, upload_id, 1, '')
+            part_tags << part_upload
+          end
 
-        def metadata_prefix
-          'X-Object-Meta-'
+        rescue
+          # Abort the upload & reraise
+          service.abort_multipart_upload(directory.key, key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
-        def update_attributes_from(data)
-          merge_attributes(data[:headers].reject { |key, _value| ['Content-Length', 'Content-Type'].include?(key) })
-        end
       end
     end
   end