fog-aliyun 0.3.8 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 111b9682a34af25b9856551baebc1acaff7a7b41c613d37f65ef47a9d5fd2e96
-  data.tar.gz: 242585bce56cba8e443a991e44f5b1b7306b61ded43c0b57e8af7a4e48bcd408
+  metadata.gz: e3be4a1832619663a88cd9ac70eada6d06c24b9e8b7f11da0242516d2e379acd
+  data.tar.gz: 5d9338f0e592f82b9f2565ee8fb5c46077d2b1156e72a808a7c476a17e7da8fe
 SHA512:
-  metadata.gz: fc4e02bae46ba2520af281d5321fe517b4030439239d771eddc89a235052627831ab13768e9784100654d0e6ba883bb6b15de0ddd5234cac446ac38d007f9424
-  data.tar.gz: df23687c6c0103993eb2bc5c276ce9f5c45ec8ff7c9478e80de1b7faff5d6cf32764c520316854fae4d3404425778a7912703f274216fe2a1671842142822380
+  metadata.gz: 7aebf30618f7e098f37c4a0a0c9a10bba29d33062bc9789547d92caf0dd220a75c7e303b22a64f1c27eec7385be4c4e18b3542aff1430ba5020dea7e763866c0
+  data.tar.gz: dd9e04bac82d1acfd60ef696d43b02f181d810aad04a9ad8bff8e6c9038c9a464fd91ad3dbab2c65b491bb1b929ca6cb4f35b9bd06f4485e65a67fe60baaad03

data/.gitignore

@@ -23,3 +23,5 @@ tmp
 *.a
 mkmf.log
 gemfiles/*.lock
+morethan100m
+*.log

data/Gemfile

@@ -4,3 +4,4 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fog-aliyun.gemspec
 gemspec
+gem 'aliyun-sdk', git: 'https://github.com/aliyun/aliyun-oss-ruby-sdk.git', branch: 'master'

data/README.md

@@ -30,9 +30,9 @@ Since it's a bad practice to have your credentials in source code, you should lo
 
 ```
 default:
-  :aliyun_accesskey_id:     <YOUR_ACCESS_KEY_ID>,
-  :aliyun_accesskey_secret: <YOUR_SECRET_ACCESS_KEY>,
-  :aliyun_region_id:        <YOUR_TARGET_REGION>
+  aliyun_accesskey_id:     "<YOUR_ACCESS_KEY_ID>"
+  aliyun_accesskey_secret: "<YOUR_SECRET_ACCESS_KEY>"
+  aliyun_region_id:        "<YOUR_TARGET_REGION>"
 ```
 
 ### Connecting to OSS
@@ -307,6 +307,64 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Testing
+
+
+
+To run test suite use the following command:
+
+```
+rake spec
+```
+
+### Code coverage
+
+To run test suite with code coverage:
+
+```
+export COVERAGE=true
+rake spec
+```
+
+The result will be generated in `coverage` folder.
+
+### Integration tests
+
+To run integration tests please prepare a set of AliCloud credentials to be used by integration tests.
+
+Define the credentials and bucket in `~/.fog` file in using following format:
+
+```
+default:
+  aliyun_accesskey_id:     "...access key..." # You can create a set of credentials  
+  aliyun_accesskey_secret: "...secret..." # using Alicloud console portal
+  aliyun_region_id:        "...region name..." # Example: cn-shanghai
+  aliyun_oss_bucket:       "...name of the bucket..." # Example: fog-integration-test-bucket
+```
+
+WARNING: Do NOT use any productive account credentials and buckets for the testing, it may be harmful to your data!
+
+The tests are using [https://github.com/aliyun/aliyun-cli#installation](Aliyun CLI) to setup integration bucket and content for tests, 
+please install it locally before running integration tests.
+
+Aliyun CLI will be configured automatically as part of test execution using the credentials provided for fog connection.
+
+Then run the test suite with `INTEGRATION` environment variable to activate integration tests:
+
+```
+export INTEGRATION=true
+rake spec
+```
+
+### Performance test
+
+Performance tests are providing memory consumption report for download/upload operations.
+
+```
+export PERFORMANCE=true
+rake spec
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/fog-aliyun.gemspec

@@ -26,6 +26,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'memory_profiler'
 
   spec.add_dependency 'fog-core'
   spec.add_dependency 'fog-json'

data/lib/fog/aliyun.rb

@@ -2,19 +2,16 @@
 
 require 'fog/core'
 require 'fog/json'
-require 'fog/aliyun/version'
+require File.expand_path('../aliyun/version', __FILE__)
 
 module Fog
-  module Compute
-    autoload :Aliyun, 'fog/aliyun/compute'
-  end
-
-  module Storage
-    autoload :Aliyun, 'fog/aliyun/storage'
-  end
-
   module Aliyun
     extend Fog::Provider
+
+    # Services
+    autoload :Compute,          File.expand_path('../aliyun/compute', __FILE__)
+    autoload :Storage,          File.expand_path('../aliyun/storage', __FILE__)
+
     service(:compute, 'Compute')
     service(:storage, 'Storage')
   end

data/lib/fog/aliyun/models/storage/directories.rb

@@ -4,10 +4,10 @@ require 'fog/core/collection'
 require 'fog/aliyun/models/storage/directory'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directories < Fog::Collection
-        model Fog::Storage::Aliyun::Directory
+        model Fog::Aliyun::Storage::Directory
 
         def all
           containers = service.get_containers
@@ -30,6 +30,9 @@ module Fog
         # If key is a directory(including /), return an existed or a new one;
         # If key does not contain /, if bucket, return '', else return an existed or a new one directory;
         def get(key, options = {})
+          if key.is_a? Array
+            key = key[0]
+          end
           if !key.nil? && key != '' && key != '.'
             key = key.chomp('/')
             if key.include? '/'
@@ -37,19 +40,39 @@ module Fog
               ret = service.head_object(dir, options)
               new(key: key) if ret.data[:status] == 200
             else
-              data = service.get_bucket(key)
-              if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-                dir = key + '/'
-                ret = service.head_object(dir, options)
-                new(key: key) if ret.data[:status] == 200
-              else
-                new(key: '')
+              remap_attributes(options, {
+                  :delimiter  => 'delimiter',
+                  :marker     => 'marker',
+                  :max_keys   => 'max-keys',
+                  :prefix     => 'prefix'
+              })
+              data = service.get_bucket(key, options)
+              directory = new(:key => data['Name'], :is_persisted => true)
+              options = {}
+              for k, v in data
+                if ['CommonPrefixes', 'Delimiter', 'IsTruncated', 'Marker', 'MaxKeys', 'Prefix'].include?(k)
+                  # Sometimes, the v will be a Array, like "Name"=>["blobstore-droplet1"], "Prefix"=>[{}], "Marker"=>[{}], "MaxKeys"=>["100"], "Delimiter"=>[{}], "IsTruncated"=>["false"]
+                  # and there needs to parse them
+                  if !v.nil? && (v.is_a? Array) && (v.size > 0)
+                    if v[0].is_a? Hash
+                      v = nil
+                    else
+                      v = v[0]
+                    end
+                  end
+                  options[k] = v
+                end
+              end
+              directory.files.merge_attributes(options)
+              if data.key?('Contents') && !data['Contents'].nil?
+                directory.files.load(data['Contents'])
               end
+              directory
             end
           else
             new(key: '')
           end
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end
       end

data/lib/fog/aliyun/models/storage/directory.rb

@@ -4,10 +4,10 @@ require 'fog/core/model'
 require 'fog/aliyun/models/storage/files'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directory < Fog::Model
-        identity :key
+        identity :key, :aliases => ['Key', 'Name', 'name']
 
         def destroy
           requires :key
@@ -15,20 +15,19 @@ module Fog
           ret = service.list_objects(prefix: prefix)['Contents']
 
           if ret.nil?
-            puts ' Not found: Direction not exist!'
             false
           elsif ret.size == 1
             service.delete_container(key)
             true
           else
-            raise Fog::Storage::Aliyun::Error, ' Forbidden: Direction not empty!'
+            raise Fog::Aliyun::Storage::Error, ' Forbidden: Direction not empty!'
             false
           end
         end
 
         def files
           @files ||= begin
-            Fog::Storage::Aliyun::Files.new(
+            Fog::Aliyun::Storage::Files.new(
               directory: self,
               service: service
             )
@@ -48,7 +47,6 @@ module Fog
           if !key.nil? && key != '' && key != '.' && !(key.include? '/')
             data = service.get_bucket(key)
             if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-              puts "[INFO] The key #{key} is not a bucket and create one folder named with it."
               service.put_container(key)
             end
           end

data/lib/fog/aliyun/models/storage/file.rb

@@ -3,10 +3,10 @@
 require 'fog/core/model'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class File < Fog::Model
-        identity :key, aliases: 'name'
+        identity :key, aliases: ['Key', 'Name', 'name']
         attribute :date, aliases: 'Date'
         attribute :content_length, aliases: 'Content-Length', type: :integer
         attribute :content_type, aliases: 'Content-Type'
@@ -35,32 +35,32 @@ module Fog
 
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          directory_key = collection.check_directory_key(directory.key)
+          source_bucket, directory_key = collection.check_directory_key(directory.key)
           source_object = if directory_key == ''
                             key
                           else
                             directory_key + '/' + key
                           end
-          target_directory_key = collection.check_directory_key(target_directory_key)
+          target_bucket, target_directory_key = collection.check_directory_key(target_directory_key)
           target_object = if target_directory_key == ''
                             target_file_key
                           else
                             target_directory_key + '/' + target_file_key
                           end
-          service.copy_object(nil, source_object, nil, target_object, options)
+          service.copy_object(source_bucket, source_object, target_bucket, target_object, options)
           target_directory = service.directories.new(key: target_directory_key)
           target_directory.files.get(target_file_key)
         end
 
         def destroy
           requires :directory, :key
-          directory_key = collection.check_directory_key(directory.key)
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
-          service.delete_object(object)
+          service.delete_object(object, bucket: bucket_name)
           true
         end
 
@@ -94,13 +94,13 @@ module Fog
           expires = expires.nil? ? 0 : expires.to_i
 
           requires :directory, :key
-          directory_key = collection.check_directory_key(directory.key)
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
-          service.get_object_http_url_public(object, expires, options)
+          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def public_url
@@ -113,18 +113,18 @@ module Fog
           options['Content-Type'] = content_type if content_type
           options['Content-Disposition'] = content_disposition if content_disposition
           options.merge!(metadata_to_headers)
-          directory_key = collection.check_directory_key(directory.key)
+          bucket_name, directory_key = collection.check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
           if body.is_a?(::File)
-            data = service.put_object(object, body, options).data
+            data = service.put_object(object, body, options.merge(bucket: bucket_name)).data
           elsif body.is_a?(String)
-            data = service.put_object_with_body(object, body, options).data
+            data = service.put_object_with_body(object, body, options.merge(bucket: bucket_name)).data
           else
-            raise Fog::Storage::Aliyun::Error, " Forbidden: Invalid body type: #{body.class}!"
+            raise Fog::Aliyun::Storage::Error, " Forbidden: Invalid body type: #{body.class}!"
           end
           update_attributes_from(data)
           refresh_metadata
@@ -172,14 +172,14 @@ module Fog
 
         def metadata_attributes
           if last_modified
-            directory_key = collection.check_directory_key(directory.key)
+            bucket_name, directory_key = collection.check_directory_key(directory.key)
             object = if directory_key == ''
                        key
                      else
                        directory_key + '/' + key
                      end
 
-            data = service.head_object(object).data
+            data = service.head_object(object, bucket: bucket_name).data
             if data[:status] == 200
               headers = data[:headers]
               headers.select! { |k, _v| metadata_attribute?(k) }

data/lib/fog/aliyun/models/storage/files.rb

@@ -2,18 +2,23 @@
 
 require 'fog/core/collection'
 require 'fog/aliyun/models/storage/file'
+require 'aliyun/oss'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Files < Fog::Collection
         attribute :directory
         attribute :limit
-        attribute :marker
+        attribute :prefix,          :aliases => 'Prefix'
         attribute :path
-        attribute :prefix
+        attribute :common_prefixes, :aliases => 'CommonPrefixes'
+        attribute :delimiter,       :aliases => 'Delimiter'
+        attribute :is_truncated,    :aliases => 'IsTruncated'
+        attribute :marker,          :aliases => 'Marker'
+        attribute :max_keys,        :aliases => ['MaxKeys', 'max-keys']
 
-        model Fog::Storage::Aliyun::File
+        model Fog::Aliyun::Storage::File
 
         # check_directory_key have two functions:
         # 1. trim the directory_key suffix '/'
@@ -21,6 +26,10 @@ module Fog
         #    If so, it will return directly to avoid to create a new redundant folder named with directory_key.
         #    This point will be applied to multi-bucket and make bucket as a directory scenario.
         def check_directory_key(directory_key)
+          bucket_name = nil
+          if directory_key.is_a? Array
+            directory_key = directory_key[0]
+          end
           if directory_key != ''
             # trim the suffix '/'
             directory_key = directory_key.chomp('/')
@@ -29,25 +38,31 @@ module Fog
               directory_key
             else
               data = service.get_bucket(directory_key)
-              puts "[DEBUG] Getting the bucket named with directory name #{directory_key}..."
               if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-                puts "[INFO] The directory name #{directory_key} is not a bucket and will create one folder named with it."
                 directory_key
               else
-                puts "[INFO] The directory name #{directory_key} is a bucket and store objects directly."
-                ''
+                bucket_name = directory_key
+                directory_key = ''
               end
             end
-          else
-            ''
           end
+          return bucket_name, directory_key
         end
 
-        def all(_options = {})
+        def all(options = {})
           requires :directory
-          directory_key = check_directory_key(directory.key)
-          prefix = directory_key + '/' if directory_key != '' && directory_key != '.' && !directory_key.nil?
-          files = service.list_objects(prefix: prefix)['Contents']
+          bucket_name, directory_key = check_directory_key(directory.key)
+          remap_attributes(options, {
+              :delimiter  => 'delimiter',
+              :marker     => 'marker',
+              :max_keys   => 'max-keys',
+              :prefix     => 'prefix'
+          })
+          prefix_value = options['prefix']
+          prefix_value = directory_key + '/' + prefix if directory_key != '' && directory_key != '.' && !directory_key.nil?
+          options['prefix'] = prefix_value
+          options['bucket'] = bucket_name
+          files = service.list_objects(options)['Contents']
           return if files.nil?
           data = []
           i = 0
@@ -88,117 +103,88 @@ module Fog
 
         def get(key)
           requires :directory
-          directory_key = check_directory_key(directory.key)
+          bucket_name, directory_key = check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
           begin
-            data = service.get_object(object)
-          rescue StandardError => error
-            case error.response.body
-            when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
+            data = service.get_object(object, nil, bucket: bucket_name)
+            lastModified = data['headers'][:last_modified]
+            last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
+
+            date = data['headers'][:date]
+            date = (Time.parse(date).localtime if !date.nil? && date != '')
+            file_data = {
+                body: data[:body],
+                content_length: data['headers'][:content_length].to_i,
+                key: key,
+                last_modified: last_modified,
+                content_type: data['headers'][:content_type],
+                etag: data['headers'][:etag],
+                date: date,
+                connection: data['headers'][:connection],
+                accept_ranges: data['headers'][:accept_ranges],
+                server: data['headers'][:server],
+                object_type: data['headers'][:x_oss_object_type]
+            }
+
+            new(file_data)
+          rescue AliyunOssSdk::ServerError => error
+            case error.error_code
+            when %r{NoSuchKey},%r{SymlinkTargetNotExist}
               nil
             else
               raise(error)
             end
           end
-
-          contentLen = data[:headers]['Content-Length'].to_i
-
-          if block_given?
-            pagesNum = (contentLen + Excon::CHUNK_SIZE - 1) / Excon::CHUNK_SIZE
-
-            for i in 1..pagesNum
-              _start = (i - 1) * Excon::CHUNK_SIZE
-              _end = i * Excon::CHUNK_SIZE - 1
-              range = "#{_start}-#{_end}"
-              begin
-                data = service.get_object(object, range)
-                chunk = data[:body]
-              rescue StandardError => error
-                case error.response.body
-                  when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
-                    chunk = ''
-                  else
-                    raise(error)
-                end
-              end
-              yield(chunk)
-              body = nil
-            end
-          else
-            body = data[:body]
-          end
-
-          lastModified = data[:headers]['Last-Modified']
-          last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-
-          date = data[:headers]['Date']
-          date = (Time.parse(date).localtime if !date.nil? && date != '')
-          file_data = {
-            body: body,
-            content_length: contentLen,
-            key: key,
-            last_modified: last_modified,
-            content_type: data[:headers]['Content-Type'],
-            etag: data[:headers]['ETag'],
-            date: date,
-            connection: data[:headers]['Connection'],
-            accept_ranges: data[:headers]['Accept-Ranges'],
-            server: data[:headers]['Server'],
-            object_type: data[:headers]['x-oss-object-type'],
-            content_disposition: data[:headers]['Content-Disposition']
-          }
-
-          new(file_data)
         end
 
         def get_url(key)
           requires :directory
-          directory_key = check_directory_key(directory.key)
+          bucket_name, directory_key = check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
-          service.get_object_http_url_public(object, 3600)
+          service.get_object_http_url_public(object, 3600, bucket: bucket_name)
         end
 
         def get_http_url(key, expires, options = {})
           requires :directory
-          directory_key = check_directory_key(directory.key)
+          bucket_name, directory_key = check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
           expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_http_url_public(object, expires, options)
+          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def get_https_url(key, expires, options = {})
           requires :directory
-          directory_key = check_directory_key(directory.key)
+          bucket_name, directory_key = check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
           expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_https_url_public(object, expires, options)
+          service.get_object_https_url_public(object, expires, options.merge(bucket: bucket_name))
         end
 
         def head(key, _options = {})
           requires :directory
-          directory_key = check_directory_key(directory.key)
+          bucket_name, directory_key = check_directory_key(directory.key)
           object = if directory_key == ''
                      key
                    else
                      directory_key + '/' + key
                    end
-          data = service.head_object(object).data
+          data = service.head_object(object, bucket: bucket_name).data
           return nil if data[:status] == 404
           lastModified = data[:headers]['Last-Modified']
           last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
@@ -219,12 +205,19 @@ module Fog
             object_type: data[:headers]['x-oss-object-type']
           }
           new(file_data)
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end
 
         def new(attributes = {})
           requires :directory
+          # Sometimes, the v will be a Array, like "Prefix"=>[{}], "Marker"=>[xxxx], "MaxKeys"=>["100"], "IsTruncated"=>["false"]
+          # and there needs to parse them
+          for k, v in attributes
+            if !v.nil? && (v.is_a? Array) && (v.size > 0)
+              attributes[k] = v[0]
+            end
+          end
           super({ directory: directory }.merge!(attributes))
         end
       end