fog-aliyun 0.3.10 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5da51b9b9b9dba901f7d457ca4dc24ac63a2dc271691920b648f47041102d913
-  data.tar.gz: f76b4c63734441c9fc4d1d7089f70f9d37d9d91b433e23d8400bd84e009eb14c
+  metadata.gz: 91aa2154dd185e6481eff2429ffc9c04e8651764dd8d6cf196c2b689a38c461b
+  data.tar.gz: a7aff361e70ead42b81f2d07aff5cbb92c5dcf6624f6dbfd0e0e1e80cc888a1e
 SHA512:
-  metadata.gz: a87085ec484fcdf4a1140cd3f0ae390fe5cdc444791024e3263facb2977318ee5a6a1c2e76ff0d3b47deda665abe61405f3956e421dc36be43ba071e0ca5e5f4
-  data.tar.gz: a3382c51257b146bf3e3d7803042cc36b241965330d72a345816dcedb8d92799aed0e84c6b8fbf57a53659fa9dab9fb2f9be9e6e7337fc9c5288e64a43c68acc
+  metadata.gz: bdc525d0d99894eb5e0dd63a6e0f4eff5fc9ebddce5c2eecbed0559010bfe25db134076d1f8a6bacc5f72d3efe4772efd02925e27481e8f3a413a6c5eb13ed01
+  data.tar.gz: f4726017f3fd50a33d4015adf8c1f5870645f2401acb79ab43b88fd845cef67fc838fb7a0044eed53c7007aa4ee2705819ecbe04e3f7eca3b12c78d90d6fbed8

data/.gitignore

@@ -23,3 +23,5 @@ tmp
 *.a
 mkmf.log
 gemfiles/*.lock
+morethan100m
+*.log

data/CHANGELOG.md

@@ -0,0 +1,55 @@
+## 0.3.17 (Unreleased)
+## 0.3.16 (June 18, 2020)
+
+IMPROVEMENTS:
+- [Enhance tests][Files & Directory scenarios] Test get nested directories and files in nested directory [GH-98](https://github.com/fog/fog-aliyun/pull/98)
+- remove get_bucket_location and use ruby sdk to improve performance when uploading object [GH-97](https://github.com/fog/fog-aliyun/pull/97)
+- using bucket_exist to checking bucket [GH-95](https://github.com/fog/fog-aliyun/pull/95)
+- add change log [GH-94](https://github.com/fog/fog-aliyun/pull/94)
+
+BUG FIXES:
+- fix delete all of files bug when specifying a prefix [GH-102](https://github.com/fog/fog-aliyun/pull/102)
+
+## 0.3.15 (June 05, 2020)
+
+BUG FIXES:
+- change dependence ruby sdk to gems [GH-92](https://github.com/fog/fog-aliyun/pull/92)
+
+## 0.3.13 (June 02, 2020)
+
+IMPROVEMENTS:
+- using ruby sdk to delete object [GH-90](https://github.com/fog/fog-aliyun/pull/90)
+
+## 0.3.12 (May 28, 2020 )
+
+BUG FIXES:
+- add missing dependence [GH-88](https://github.com/fog/fog-aliyun/pull/88)
+
+## 0.3.11 (May 25, 2020)
+
+IMPROVEMENTS:
+- using oss ruby sdk to improve downloading object performance [GH-86](https://github.com/fog/fog-aliyun/pull/86)
+- Add performance tests [GH-85](https://github.com/fog/fog-aliyun/pull/85)
+- [Enhance tests][Entity operations]Add tests for each type of entity that validates the CURD operations [GH-84](https://github.com/fog/fog-aliyun/pull/84)
+- [Enhance tests][Auth & Connectivity scenarios] Test region is selected according to provider configuration [GH-83](https://github.com/fog/fog-aliyun/pull/83)
+- [Enhance tests][Files & Directory scenarios] test file listing using parameters such as prefix, marker, delimeter and maxKeys [GH-82](https://github.com/fog/fog-aliyun/pull/82)
+- [Enhance tests][Files & Directory scenarios]test directory listing using parameters such as prefix, marker, delimeter and maxKeys [GH-81](https://github.com/fog/fog-aliyun/pull/81)
+- [Enhance tests][Files & Directory scenarios]Test that it is possible to upload (write) large file (multi part upload) [GH-79](https://github.com/fog/fog-aliyun/pull/79)
+- upgrade deprecated code [GH-78](https://github.com/fog/fog-aliyun/pull/78)
+- improve fog/integration_spec [GH-77](https://github.com/fog/fog-aliyun/pull/77)
+- [Enhance tests][Files & Directory scenarios]Test that it is possible to upload (write) a file [GH-76](https://github.com/fog/fog-aliyun/pull/76)
+- upgrade deprecated code [GH-74](https://github.com/fog/fog-aliyun/pull/74)
+- support https scheme [GH-71](https://github.com/fog/fog-aliyun/pull/71)
+- [Enhance tests][Files & Directory scenarios]Test that it is possible to destroy a file/directory [GH-69](https://github.com/fog/fog-aliyun/pull/69)
+- improve fog/integration_spec [GH-68](https://github.com/fog/fog-aliyun/pull/68)
+- Implement basic integration tests [GH-66](https://github.com/fog/fog-aliyun/pull/66)
+
+## 0.3.10 (May 07, 2020)
+
+IMPROVEMENTS:
+- Set max limitation to 1000 when get objects [GH-64](https://github.com/fog/fog-aliyun/pull/64)
+
+## 0.3.9 (May 07, 2020)
+
+BUG FIXES:
+- diectories.get supports options to filter the specified objects [GH-62](https://github.com/fog/fog-aliyun/pull/62)

data/README.md

@@ -30,9 +30,9 @@ Since it's a bad practice to have your credentials in source code, you should lo
 
 ```
 default:
-  :aliyun_accesskey_id:     <YOUR_ACCESS_KEY_ID>,
-  :aliyun_accesskey_secret: <YOUR_SECRET_ACCESS_KEY>,
-  :aliyun_region_id:        <YOUR_TARGET_REGION>
+  aliyun_accesskey_id:     "<YOUR_ACCESS_KEY_ID>"
+  aliyun_accesskey_secret: "<YOUR_SECRET_ACCESS_KEY>"
+  aliyun_region_id:        "<YOUR_TARGET_REGION>"
 ```
 
 ### Connecting to OSS
@@ -307,6 +307,64 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Testing
+
+
+
+To run test suite use the following command:
+
+```
+rake spec
+```
+
+### Code coverage
+
+To run test suite with code coverage:
+
+```
+export COVERAGE=true
+rake spec
+```
+
+The result will be generated in `coverage` folder.
+
+### Integration tests
+
+To run integration tests please prepare a set of AliCloud credentials to be used by integration tests.
+
+Define the credentials and bucket in `~/.fog` file in using following format:
+
+```
+default:
+  aliyun_accesskey_id:     "...access key..." # You can create a set of credentials  
+  aliyun_accesskey_secret: "...secret..." # using Alicloud console portal
+  aliyun_region_id:        "...region name..." # Example: cn-shanghai
+  aliyun_oss_bucket:       "...name of the bucket..." # Example: fog-integration-test-bucket
+```
+
+WARNING: Do NOT use any productive account credentials and buckets for the testing, it may be harmful to your data!
+
+The tests are using [https://github.com/aliyun/aliyun-cli#installation](Aliyun CLI) to setup integration bucket and content for tests, 
+please install it locally before running integration tests.
+
+Aliyun CLI will be configured automatically as part of test execution using the credentials provided for fog connection.
+
+Then run the test suite with `INTEGRATION` environment variable to activate integration tests:
+
+```
+export INTEGRATION=true
+rake spec
+```
+
+### Performance test
+
+Performance tests are providing memory consumption report for download/upload operations.
+
+```
+export PERFORMANCE=true
+rake spec
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/fog-aliyun.gemspec

@@ -26,7 +26,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'memory_profiler'
+  spec.add_development_dependency 'aliyun-sdk', '~> 0.7.2'
 
+  spec.add_dependency 'aliyun-sdk', '~> 0.7.2'
   spec.add_dependency 'fog-core'
   spec.add_dependency 'fog-json'
   spec.add_dependency 'ipaddress', '~> 0.8'

data/lib/fog/aliyun.rb

@@ -2,19 +2,16 @@
 
 require 'fog/core'
 require 'fog/json'
-require 'fog/aliyun/version'
+require File.expand_path('../aliyun/version', __FILE__)
 
 module Fog
-  module Compute
-    autoload :Aliyun, 'fog/aliyun/compute'
-  end
-
-  module Storage
-    autoload :Aliyun, 'fog/aliyun/storage'
-  end
-
   module Aliyun
     extend Fog::Provider
+
+    # Services
+    autoload :Compute,          File.expand_path('../aliyun/compute', __FILE__)
+    autoload :Storage,          File.expand_path('../aliyun/storage', __FILE__)
+
     service(:compute, 'Compute')
     service(:storage, 'Storage')
   end

data/lib/fog/aliyun/models/storage/directories.rb

@@ -4,10 +4,10 @@ require 'fog/core/collection'
 require 'fog/aliyun/models/storage/directory'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directories < Fog::Collection
-        model Fog::Storage::Aliyun::Directory
+        model Fog::Aliyun::Storage::Directory
 
         def all
           containers = service.get_containers
@@ -72,7 +72,7 @@ module Fog
           else
             new(key: '')
           end
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end
       end

data/lib/fog/aliyun/models/storage/directory.rb

@@ -4,8 +4,8 @@ require 'fog/core/model'
 require 'fog/aliyun/models/storage/files'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Directory < Fog::Model
         identity :key, :aliases => ['Key', 'Name', 'name']
 
@@ -20,14 +20,14 @@ module Fog
             service.delete_container(key)
             true
           else
-            raise Fog::Storage::Aliyun::Error, ' Forbidden: Direction not empty!'
+            raise Fog::Aliyun::Storage::Error, ' Forbidden: Direction not empty!'
             false
           end
         end
 
         def files
           @files ||= begin
-            Fog::Storage::Aliyun::Files.new(
+            Fog::Aliyun::Storage::Files.new(
               directory: self,
               service: service
             )

data/lib/fog/aliyun/models/storage/file.rb

@@ -3,8 +3,8 @@
 require 'fog/core/model'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class File < Fog::Model
         identity :key, aliases: ['Key', 'Name', 'name']
         attribute :date, aliases: 'Date'
@@ -120,11 +120,12 @@ module Fog
                      directory_key + '/' + key
                    end
           if body.is_a?(::File)
-            data = service.put_object(object, body, options.merge(bucket: bucket_name)).data
+            service.put_object(object, body, options.merge(bucket: bucket_name))
+            data = service.head_object(object, bucket: bucket_name)
           elsif body.is_a?(String)
             data = service.put_object_with_body(object, body, options.merge(bucket: bucket_name)).data
           else
-            raise Fog::Storage::Aliyun::Error, " Forbidden: Invalid body type: #{body.class}!"
+            raise Fog::Aliyun::Storage::Error, " Forbidden: Invalid body type: #{body.class}!"
           end
           update_attributes_from(data)
           refresh_metadata

data/lib/fog/aliyun/models/storage/files.rb

@@ -2,10 +2,11 @@
 
 require 'fog/core/collection'
 require 'fog/aliyun/models/storage/file'
+require 'aliyun/oss'
 
 module Fog
-  module Storage
-    class Aliyun
+  module Aliyun
+    class Storage
       class Files < Fog::Collection
         attribute :directory
         attribute :limit
@@ -17,7 +18,7 @@ module Fog
         attribute :marker,          :aliases => 'Marker'
         attribute :max_keys,        :aliases => ['MaxKeys', 'max-keys']
 
-        model Fog::Storage::Aliyun::File
+        model Fog::Aliyun::Storage::File
 
         # check_directory_key have two functions:
         # 1. trim the directory_key suffix '/'
@@ -35,44 +36,36 @@ module Fog
             # The bucket name can not contain '/', so if directory_key, return directory.
             if directory_key.include? '/'
               directory_key
+            elsif service.bucket_exists?(directory_key)
+              bucket_name = directory_key
+              directory_key = ''
             else
-              data = service.get_bucket(directory_key)
-              if data.class == Hash && data.key?('Code') && !data['Code'].nil? && !data['Code'].empty?
-                directory_key
-              else
-                bucket_name = directory_key
-                directory_key = ''
-              end
+              directory_key
             end
           end
           return bucket_name, directory_key
         end
 
-        def all(_options = {})
+        def all(options = {})
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          prefix_value = prefix
-          prefix_value = directory_key + '/' + prefix if directory_key != '' && directory_key != '.' && !directory_key.nil?
-          files = service.list_objects(prefix: prefix_value, bucket: bucket_name)['Contents']
-          return if files.nil?
-          data = []
-          i = 0
-          files.each do |file|
-            next unless file['Key'][0][-1] != '/'
-            content_length = file['Size'][0].to_i
-            key = file['Key'][0]
-            lastModified = file['LastModified'][0]
-            last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-            type = file['Type'][0]
-            data[i] = { content_length: content_length,
-                        key: key,
-                        last_modified: last_modified,
-                        etag: file['ETag'][0],
-                        object_type: type }
-            i += 1
+          options = {
+              'delimiter'   => delimiter,
+              'marker'      => marker,
+              'max-keys'    => max_keys.to_i,
+              'prefix'      => prefix
+          }.merge!(options)
+          options = options.reject {|key,value| value.nil? || value.to_s.empty?}
+          merge_attributes(options)
+          parent = directory.collection.get(
+              directory.key,
+              options
+          )
+          if parent
+            merge_attributes(parent.files.attributes)
+            load(parent.files.map {|file| file.attributes})
+          else
+            nil
           end
-
-          load(data)
         end
 
         alias each_file_this_page each
@@ -102,63 +95,34 @@ module Fog
                    end
           begin
             data = service.get_object(object, nil, bucket: bucket_name)
-          rescue StandardError => error
-            case error.response.body
-            when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
+            lastModified = data['headers'][:last_modified]
+            last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
+
+            date = data['headers'][:date]
+            date = (Time.parse(date).localtime if !date.nil? && date != '')
+            file_data = {
+                body: data[:body],
+                content_length: data['headers'][:content_length].to_i,
+                key: key,
+                last_modified: last_modified,
+                content_type: data['headers'][:content_type],
+                etag: data['headers'][:etag],
+                date: date,
+                connection: data['headers'][:connection],
+                accept_ranges: data['headers'][:accept_ranges],
+                server: data['headers'][:server],
+                object_type: data['headers'][:x_oss_object_type]
+            }
+
+            new(file_data)
+          rescue AliyunOssSdk::ServerError => error
+            case error.error_code
+            when %r{NoSuchKey},%r{SymlinkTargetNotExist}
               nil
             else
               raise(error)
             end
           end
-
-          contentLen = data[:headers]['Content-Length'].to_i
-
-          if block_given?
-            pagesNum = (contentLen + Excon::CHUNK_SIZE - 1) / Excon::CHUNK_SIZE
-
-            for i in 1..pagesNum
-              _start = (i - 1) * Excon::CHUNK_SIZE
-              _end = i * Excon::CHUNK_SIZE - 1
-              range = "#{_start}-#{_end}"
-              begin
-                data = service.get_object(object, range, bucket: bucket_name)
-                chunk = data[:body]
-              rescue StandardError => error
-                case error.response.body
-                  when %r{<Code>NoSuchKey</Code>},%r{<Code>SymlinkTargetNotExist</Code>}
-                    chunk = ''
-                  else
-                    raise(error)
-                end
-              end
-              yield(chunk)
-              body = nil
-            end
-          else
-            body = data[:body]
-          end
-
-          lastModified = data[:headers]['Last-Modified']
-          last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-
-          date = data[:headers]['Date']
-          date = (Time.parse(date).localtime if !date.nil? && date != '')
-          file_data = {
-            body: body,
-            content_length: contentLen,
-            key: key,
-            last_modified: last_modified,
-            content_type: data[:headers]['Content-Type'],
-            etag: data[:headers]['ETag'],
-            date: date,
-            connection: data[:headers]['Connection'],
-            accept_ranges: data[:headers]['Accept-Ranges'],
-            server: data[:headers]['Server'],
-            object_type: data[:headers]['x-oss-object-type'],
-            content_disposition: data[:headers]['Content-Disposition']
-          }
-
-          new(file_data)
         end
 
         def get_url(key)
@@ -225,7 +189,7 @@ module Fog
             object_type: data[:headers]['x-oss-object-type']
           }
           new(file_data)
-        rescue Fog::Storage::Aliyun::NotFound
+        rescue Fog::Aliyun::Storage::NotFound
           nil
         end