fog-aliyun 0.3.7 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71f324e607c567e9377a324698f35edd82667312c28fce1d3d8dc2e7e771af19
-  data.tar.gz: 608ea19f6289e5fc3e6ab8642f1326b7b88cb81639853b91f98be251923633e5
+  metadata.gz: c47b8a9b6a56d553d1b0161be29b2d2444671f41f083e8af34707bd1eadc1718
+  data.tar.gz: 17e83f9d11b7ae7b1802941ddb13427346bbd971b563d7ac04436f7d036d7ef7
 SHA512:
-  metadata.gz: 80f63cc617a5e1d1758955342ae252c94e89dbb429575ad8da406100d3dd56a446dd34cbfa896be2c8908e7fd118e7cfb6e87f648def81a52b812c1f72832be8
-  data.tar.gz: 1da3cd3a1a3d8bc19d2290fdce54f2e9efe1e26849a93a8a5a2b7310f3303c46e1550d0e445cd316b1a7a8fdd5c39f2a07f79a19351e3857b02bd6cf117df37d
+  metadata.gz: 52060f2e1a3881ccb72ccc6e5fbeddd988bd2820416c419a7409e714c6d4f08fb7b37f5f98ff1e8fc66b47c0f2aa5656349da06e939edfb0c78c4adba05bf1ef
+  data.tar.gz: 30bdcf81e3e78e2983545d391ef2062ebc49975bee90c9d93226f30a631b3eaa7627916b179d15c684b38211bbd2e9e78e84210a2471b8185c5c3d21fbfe3001

data/.gitignore

@@ -23,3 +23,5 @@ tmp
 *.a
 mkmf.log
 gemfiles/*.lock
+morethan100m
+*.log

data/README.md

@@ -30,9 +30,9 @@ Since it's a bad practice to have your credentials in source code, you should lo
 
 ```
 default:
-  :aliyun_accesskey_id:     <YOUR_ACCESS_KEY_ID>,
-  :aliyun_accesskey_secret: <YOUR_SECRET_ACCESS_KEY>,
-  :aliyun_region_id:        <YOUR_TARGET_REGION>
+  aliyun_accesskey_id:     "<YOUR_ACCESS_KEY_ID>"
+  aliyun_accesskey_secret: "<YOUR_SECRET_ACCESS_KEY>"
+  aliyun_region_id:        "<YOUR_TARGET_REGION>"
 ```
 
 ### Connecting to OSS
@@ -307,6 +307,64 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Testing
+
+
+
+To run test suite use the following command:
+
+```
+rake spec
+```
+
+### Code coverage
+
+To run test suite with code coverage:
+
+```
+export COVERAGE=true
+rake spec
+```
+
+The result will be generated in `coverage` folder.
+
+### Integration tests
+
+To run integration tests please prepare a set of AliCloud credentials to be used by integration tests.
+
+Define the credentials and bucket in `~/.fog` file in using following format:
+
+```
+default:
+  aliyun_accesskey_id:     "...access key..." # You can create a set of credentials  
+  aliyun_accesskey_secret: "...secret..." # using Alicloud console portal
+  aliyun_region_id:        "...region name..." # Example: cn-shanghai
+  aliyun_oss_bucket:       "...name of the bucket..." # Example: fog-integration-test-bucket
+```
+
+WARNING: Do NOT use any productive account credentials and buckets for the testing, it may be harmful to your data!
+
+The tests are using [https://github.com/aliyun/aliyun-cli#installation](Aliyun CLI) to setup integration bucket and content for tests, 
+please install it locally before running integration tests.
+
+Aliyun CLI will be configured automatically as part of test execution using the credentials provided for fog connection.
+
+Then run the test suite with `INTEGRATION` environment variable to activate integration tests:
+
+```
+export INTEGRATION=true
+rake spec
+```
+
+### Performance test
+
+Performance tests are providing memory consumption report for download/upload operations.
+
+```
+export PERFORMANCE=true
+rake spec
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/fog-aliyun.gemspec

@@ -7,10 +7,10 @@ require 'fog/aliyun/version'
 Gem::Specification.new do |spec|
   spec.name = 'fog-aliyun'
   spec.version = Fog::Aliyun::VERSION
-  spec.authors = ['Qinsi Deng, Jianxun Li, Jane Han']
-  spec.email = ['dengqinsi@sina.com']
+  spec.authors = ['Qinsi Deng, Jianxun Li, Jane Han, Guimin He']
+  spec.email = ['dengqinsi@sina.com', 'guimin.hgm@alibaba-inc.com']
 
-  spec.summary = 'Fog provider for Aliyun Web Services.'
+  spec.summary = 'Fog provider for Alibaba Cloud Web Services.'
   spec.description = 'As a FOG provider, fog-aliyun support aliyun OSS/ECS. It will support more aliyun services later.'
   spec.homepage = 'https://github.com/fog/fog-aliyun'
   spec.license = 'MIT'
@@ -26,9 +26,13 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'memory_profiler'
+  spec.add_development_dependency 'aliyun-sdk'
 
   spec.add_dependency 'fog-core'
   spec.add_dependency 'fog-json'
   spec.add_dependency 'ipaddress', '~> 0.8'
   spec.add_dependency 'xml-simple', '~> 1.1'
+  spec.add_dependency 'aliyun-sdk'
 end

data/lib/fog/aliyun.rb

@@ -2,19 +2,16 @@
 
 require 'fog/core'
 require 'fog/json'
-require 'fog/aliyun/version'
+require File.expand_path('../aliyun/version', __FILE__)
 
 module Fog
-  module Compute
-    autoload :Aliyun, 'fog/aliyun/compute'
-  end
-
-  module Storage
-    autoload :Aliyun, 'fog/aliyun/storage'
-  end
-
   module Aliyun
     extend Fog::Provider
+
+    # Services
+    autoload :Compute,          File.expand_path('../aliyun/compute', __FILE__)
+    autoload :Storage,          File.expand_path('../aliyun/storage', __FILE__)
+
     service(:compute, 'Compute')
     service(:storage, 'Storage')
   end

data/lib/fog/aliyun/models/storage/directories.rb

@@ -4,10 +4,10 @@ require 'fog/core/collection'
 require 'fog/aliyun/models/storage/directory'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directories < Fog::Collection
-        model Fog::Storage::Aliyun::Directory
+        model Fog::Aliyun::Storage::Directory
 
         def all
           containers = service.get_containers
@@ -30,6 +30,9 @@ module Fog
         # If key is a directory(including /), return an existed or a new one;
         # If key does not contain /, if bucket, return '', else return an existed or a new one directory;
         def get(key, options = {})
+          if key.is_a? Array
+            key = key[0]
+          end
           if !key.nil? && key != '' && key != '.'
             key = key.chomp('/')
             if key.include? '/'
@@ -37,21 +40,39 @@ module Fog
               ret = service.head_object(dir, options)
               new(key: key) if ret.data[:status] == 200
             else
-              data = service.get_bucket(key)
-              puts "[DEBUG] Getting the bucket named with #{key}..."
-              puts data
-              if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-                dir = key + '/'
-                ret = service.head_object(dir, options)
-                new(key: key) if ret.data[:status] == 200
-              else
-                new(key: '')
+              remap_attributes(options, {
+                  :delimiter  => 'delimiter',
+                  :marker     => 'marker',
+                  :max_keys   => 'max-keys',
+                  :prefix     => 'prefix'
+              })
+              data = service.get_bucket(key, options)
+              directory = new(:key => data['Name'], :is_persisted => true)
+              options = {}
+              for k, v in data
+                if ['CommonPrefixes', 'Delimiter', 'IsTruncated', 'Marker', 'MaxKeys', 'Prefix'].include?(k)
+                  # Sometimes, the v will be a Array, like "Name"=>["blobstore-droplet1"], "Prefix"=>[{}], "Marker"=>[{}], "MaxKeys"=>["100"], "Delimiter"=>[{}], "IsTruncated"=>["false"]
+                  # and there needs to parse them
+                  if !v.nil? && (v.is_a? Array) && (v.size > 0)
+                    if v[0].is_a? Hash
+                      v = nil
+                    else
+                      v = v[0]
+                    end
+                  end
+                  options[k] = v
+                end
+              end
+              directory.files.merge_attributes(options)
+              if data.key?('Contents') && !data['Contents'].nil?
+                directory.files.load(data['Contents'])
               end
+              directory
             end
           else
             new(key: '')
           end
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end
       end

data/lib/fog/aliyun/models/storage/directory.rb

@@ -4,10 +4,10 @@ require 'fog/core/model'
 require 'fog/aliyun/models/storage/files'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directory < Fog::Model
-        identity :key
+        identity :key, :aliases => ['Key', 'Name', 'name']
 
         def destroy
           requires :key
@@ -15,20 +15,19 @@ module Fog
           ret = service.list_objects(prefix: prefix)['Contents']
 
           if ret.nil?
-            puts ' Not found: Direction not exist!'
             false
           elsif ret.size == 1
             service.delete_container(key)
             true
           else
-            raise Fog::Storage::Aliyun::Error, ' Forbidden: Direction not empty!'
+            raise Fog::Aliyun::Storage::Error, ' Forbidden: Direction not empty!'
             false
           end
         end
 
         def files
           @files ||= begin
-            Fog::Storage::Aliyun::Files.new(
+            Fog::Aliyun::Storage::Files.new(
               directory: self,
               service: service
             )
@@ -41,7 +40,17 @@ module Fog
 
         def save
           requires :key
-          service.put_container(key)
+
+          # Checking whether the key is a bucket and meet the multi-bucket scenario.
+          # If the key is a existing bucket, return it directly.
+          key = key.chomp('/')
+          if !key.nil? && key != '' && key != '.' && !(key.include? '/')
+            data = service.get_bucket(key)
+            if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
+              service.put_container(key)
+            end
+          end
+
           true
         end
       end

data/lib/fog/aliyun/models/storage/file.rb

@@ -3,10 +3,10 @@
 require 'fog/core/model'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class File < Fog::Model
-        identity :key, aliases: 'name'
+        identity :key, aliases: ['Key', 'Name', 'name']
         attribute :date, aliases: 'Date'
         attribute :content_length, aliases: 'Content-Length', type: :integer
         attribute :content_type, aliases: 'Content-Type'
@@ -35,29 +35,32 @@ module Fog
 
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          source_object = if directory.key == ''
+          source_bucket, directory_key = collection.check_directory_key(directory.key)
+          source_object = if directory_key == ''
                             key
                           else
-                            directory.key + '/' + key
+                            directory_key + '/' + key
                           end
+          target_bucket, target_directory_key = collection.check_directory_key(target_directory_key)
           target_object = if target_directory_key == ''
                             target_file_key
                           else
                             target_directory_key + '/' + target_file_key
                           end
-          service.copy_object(nil, source_object, nil, target_object, options)
+          service.copy_object(source_bucket, source_object, target_bucket, target_object, options)
           target_directory = service.directories.new(key: target_directory_key)
           target_directory.files.get(target_file_key)
         end
 
         def destroy
           requires :directory, :key
-          object = if directory.key == ''
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
-          service.delete_object(object)
+          service.delete_object(object, bucket: bucket_name)
           true
         end
 
@@ -91,12 +94,13 @@ module Fog
           expires = expires.nil? ? 0 : expires.to_i
 
           requires :directory, :key
-          object = if directory.key == ''
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
-          service.get_object_http_url_public(object, expires, options)
+          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def public_url
@@ -109,18 +113,18 @@ module Fog
           options['Content-Type'] = content_type if content_type
           options['Content-Disposition'] = content_disposition if content_disposition
           options.merge!(metadata_to_headers)
-
-          object = if directory.key == ''
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
           if body.is_a?(::File)
-            data = service.put_object(object, body, options).data
+            data = service.put_object(object, body, options.merge(bucket: bucket_name)).data
           elsif body.is_a?(String)
-            data = service.put_object_with_body(object, body, options).data
+            data = service.put_object_with_body(object, body, options.merge(bucket: bucket_name)).data
           else
-            raise Fog::Storage::Aliyun::Error, " Forbidden: Invalid body type: #{body.class}!"
+            raise Fog::Aliyun::Storage::Error, " Forbidden: Invalid body type: #{body.class}!"
           end
           update_attributes_from(data)
           refresh_metadata
@@ -168,13 +172,14 @@ module Fog
 
         def metadata_attributes
           if last_modified
-            object = if directory.key == ''
+            bucket_name, directory_key = collection.check_directory_key(directory.key)
+            object = if directory_key == ''
                        key
                      else
-                       directory.key + '/' + key
+                       directory_key + '/' + key
                      end
 
-            data = service.head_object(object).data
+            data = service.head_object(object, bucket: bucket_name).data
             if data[:status] == 200
               headers = data[:headers]
               headers.select! { |k, _v| metadata_attribute?(k) }

data/lib/fog/aliyun/models/storage/files.rb

@@ -2,23 +2,67 @@
 
 require 'fog/core/collection'
 require 'fog/aliyun/models/storage/file'
+require 'aliyun/oss'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Files < Fog::Collection
         attribute :directory
         attribute :limit
-        attribute :marker
+        attribute :prefix,          :aliases => 'Prefix'
         attribute :path
-        attribute :prefix
-
-        model Fog::Storage::Aliyun::File
+        attribute :common_prefixes, :aliases => 'CommonPrefixes'
+        attribute :delimiter,       :aliases => 'Delimiter'
+        attribute :is_truncated,    :aliases => 'IsTruncated'
+        attribute :marker,          :aliases => 'Marker'
+        attribute :max_keys,        :aliases => ['MaxKeys', 'max-keys']
+
+        model Fog::Aliyun::Storage::File
+
+        # check_directory_key have two functions:
+        # 1. trim the directory_key suffix '/'
+        # 2. checking whether the directory_key is a bucket.
+        #    If so, it will return directly to avoid to create a new redundant folder named with directory_key.
+        #    This point will be applied to multi-bucket and make bucket as a directory scenario.
+        def check_directory_key(directory_key)
+          bucket_name = nil
+          if directory_key.is_a? Array
+            directory_key = directory_key[0]
+          end
+          if directory_key != ''
+            # trim the suffix '/'
+            directory_key = directory_key.chomp('/')
+            # The bucket name can not contain '/', so if directory_key, return directory.
+            if directory_key.include? '/'
+              directory_key
+            else
+              data = service.get_bucket(directory_key)
+              if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
+                directory_key
+              else
+                bucket_name = directory_key
+                directory_key = ''
+              end
+            end
+          end
+          return bucket_name, directory_key
+        end
 
-        def all(_options = {})
+        def all(options = {})
           requires :directory
-          prefix = directory.key + '/' if directory.key != '' && directory.key != '.' && !directory.key.nil?
-          files = service.list_objects(prefix: prefix)['Contents']
+          bucket_name, directory_key = check_directory_key(directory.key)
+          remap_attributes(options, {
+              :delimiter  => 'delimiter',
+              :marker     => 'marker',
+              :max_keys   => 'max-keys',
+              :prefix     => 'prefix'
+          })
+          prefix_value = options['prefix']
+          prefix_value = directory_key + '/' + prefix if directory_key != '' && directory_key != '.' && !directory_key.nil?
+          options['prefix'] = prefix_value
+          options['bucket'] = bucket_name
+          files = service.list_objects(options)['Contents']
           return if files.nil?
           data = []
           i = 0
@@ -59,112 +103,88 @@ module Fog
 
         def get(key)
           requires :directory
-          object = if directory.key == ''
+          bucket_name, directory_key = check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
           begin
-            data = service.get_object(object)
-          rescue StandardError => error
-            case error.response.body
-            when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
+            data = service.get_object(object, nil, bucket: bucket_name)
+            lastModified = data['headers'][:last_modified]
+            last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
+
+            date = data['headers'][:date]
+            date = (Time.parse(date).localtime if !date.nil? && date != '')
+            file_data = {
+                body: data[:body],
+                content_length: data['headers'][:content_length].to_i,
+                key: key,
+                last_modified: last_modified,
+                content_type: data['headers'][:content_type],
+                etag: data['headers'][:etag],
+                date: date,
+                connection: data['headers'][:connection],
+                accept_ranges: data['headers'][:accept_ranges],
+                server: data['headers'][:server],
+                object_type: data['headers'][:x_oss_object_type]
+            }
+
+            new(file_data)
+          rescue AliyunOssSdk::ServerError => error
+            case error.error_code
+            when %r{NoSuchKey},%r{SymlinkTargetNotExist}
               nil
             else
               raise(error)
             end
           end
-
-          contentLen = data[:headers]['Content-Length'].to_i
-
-          if block_given?
-            pagesNum = (contentLen + Excon::CHUNK_SIZE - 1) / Excon::CHUNK_SIZE
-
-            for i in 1..pagesNum
-              _start = (i - 1) * Excon::CHUNK_SIZE
-              _end = i * Excon::CHUNK_SIZE - 1
-              range = "#{_start}-#{_end}"
-              begin
-                data = service.get_object(object, range)
-                chunk = data[:body]
-              rescue StandardError => error
-                case error.response.body
-                  when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
-                    chunk = ''
-                  else
-                    raise(error)
-                end
-              end
-              yield(chunk)
-              body = nil
-            end
-          else
-            body = data[:body]
-          end
-
-          lastModified = data[:headers]['Last-Modified']
-          last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-
-          date = data[:headers]['Date']
-          date = (Time.parse(date).localtime if !date.nil? && date != '')
-          file_data = {
-            body: body,
-            content_length: contentLen,
-            key: key,
-            last_modified: last_modified,
-            content_type: data[:headers]['Content-Type'],
-            etag: data[:headers]['ETag'],
-            date: date,
-            connection: data[:headers]['Connection'],
-            accept_ranges: data[:headers]['Accept-Ranges'],
-            server: data[:headers]['Server'],
-            object_type: data[:headers]['x-oss-object-type'],
-            content_disposition: data[:headers]['Content-Disposition']
-          }
-
-          new(file_data)
         end
 
         def get_url(key)
           requires :directory
-          object = if directory.key == ''
+          bucket_name, directory_key = check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
-          service.get_object_http_url_public(object, 3600)
+          service.get_object_http_url_public(object, 3600, bucket: bucket_name)
         end
 
         def get_http_url(key, expires, options = {})
           requires :directory
-          object = if directory.key == ''
+          bucket_name, directory_key = check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
           expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_http_url_public(object, expires, options)
+          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def get_https_url(key, expires, options = {})
           requires :directory
-          object = if directory.key == ''
+          bucket_name, directory_key = check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
           expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_https_url_public(object, expires, options)
+          service.get_object_https_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def head(key, _options = {})
           requires :directory
-          object = if directory.key == ''
+          bucket_name, directory_key = check_directory_key(directory.key)
+          object = if directory_key == ''
                      key
                    else
-                     directory.key + '/' + key
+                     directory_key + '/' + key
                    end
-          data = service.head_object(object).data
+          data = service.head_object(object, bucket: bucket_name).data
           return nil if data[:status] == 404
           lastModified = data[:headers]['Last-Modified']
           last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
@@ -185,12 +205,19 @@ module Fog
             object_type: data[:headers]['x-oss-object-type']
           }
           new(file_data)
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end
 
         def new(attributes = {})
           requires :directory
+          # Sometimes, the v will be a Array, like "Prefix"=>[{}], "Marker"=>[xxxx], "MaxKeys"=>["100"], "IsTruncated"=>["false"]
+          # and there needs to parse them
+          for k, v in attributes
+            if !v.nil? && (v.is_a? Array) && (v.size > 0)
+              attributes[k] = v[0]
+            end
+          end
           super({ directory: directory }.merge!(attributes))
         end
       end