fog-aws 3.6.5 → 3.9.0

data/lib/fog/aws/storage.rb

@@ -112,6 +112,7 @@ module Fog
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region
@@ -227,7 +228,7 @@ module Fog
           when %r{\Acn-.*}
             "s3.#{region}.amazonaws.com.cn"
           else
-            "s3-#{region}.amazonaws.com"
+            "s3.#{region}.amazonaws.com"
           end
         end
 

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.6.5"
+    VERSION = "3.9.0"
   end
 end

data/tests/credentials_tests.rb

@@ -60,6 +60,47 @@ Shindo.tests('AWS | credentials', ['aws']) do
 
     ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
 
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = File.dirname(__FILE__) + '/lorem.txt'
+    ENV['AWS_ROLE_ARN'] = "dummyrole"
+    ENV['AWS_ROLE_SESSION_NAME'] = "dummyrolesessionname"
+    document = 
+      '<AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">'\
+        '<AssumeRoleWithWebIdentityResult>'\
+          '<Credentials>'\
+            '<SessionToken>dummytoken</SessionToken>'\
+            '<SecretAccessKey>dummysecret</SecretAccessKey>'\
+            "<Expiration>#{expires_at.xmlschema}</Expiration>"\
+            '<AccessKeyId>dummykey</AccessKeyId>'\
+          '</Credentials>'\
+        '</AssumeRoleWithWebIdentityResult>'\
+      '</AssumeRoleWithWebIdentityResponse>'
+    
+    Excon.stub({method: :get, path: "/", idempotent: true}, { status: 200, body: document})
+
+    tests('#fetch_credentials token based') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    ENV['AWS_ROLE_SESSION_NAME'] = nil
+
+    tests('#fetch_credentials token based without session name') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
+
     compute = Fog::AWS::Compute.new(use_iam_profile: true)
 
     tests('#refresh_credentials_if_expired') do
@@ -100,6 +141,7 @@ Shindo.tests('AWS | credentials', ['aws']) do
     end
   ensure
     ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
     Excon.stubs.clear
     Excon.defaults[:mock] = old_mock_value
     Fog.mock! if fog_was_mocked

data/tests/helpers/succeeds_helper.rb

@@ -1,8 +1,8 @@
 module Shindo
   class Tests
-    def succeeds
+    def succeeds(&block)
       test('succeeds') do
-        !instance_eval(&Proc.new).nil?
+        !instance_eval(&Proc.new(&block)).nil?
       end
     end
   end

data/tests/models/storage/files_tests.rb

@@ -50,6 +50,38 @@ Shindo.tests("Storage[:aws] | files", ["aws"]) do
       end
     end
 
+    tests('#normalize_headers') do
+      files = @directory.files
+      response = Excon::Response.new
+      current_time = Time.new(2021, 02, 21)
+
+      response.headers['last-modified'] = current_time.to_s
+      response.headers['etag'] = '12345'
+      response.headers['ETAG'] = '12345'
+      response.headers['Cache-Control'] = 'no-cache'
+      response.headers['Content-disposition'] = 'attachment'
+      response.headers['content-length'] = 100
+      response.headers['content-Encoding'] = 'gzip'
+      response.headers['content-md5'] = 'ABCDEAB'
+      response.headers['content-Md5'] = 'ABCDEAB'
+      response.headers['ConTent-Type'] = 'application/json'
+
+      expected = {
+        'Last-Modified' => current_time,
+        'ETag' => '12345',
+        'Cache-Control' => 'no-cache',
+        'Content-Disposition' => 'attachment',
+        'Content-Length' => 100,
+        'Content-Encoding' => 'gzip',
+        'Content-MD5' => 'ABCDEAB',
+        'Content-Type' => 'application/json'
+      }
+
+      tests('header keys are normalized').returns(expected) do
+        files.normalize_headers(response)
+        response.headers
+      end
+    end
   end
 
   @directory.versions.each(&:destroy)

data/tests/requests/storage/multipart_copy_tests.rb

@@ -0,0 +1,93 @@
+require 'securerandom'
+
+Shindo.tests('Fog::Storage[:aws] | copy requests', ["aws"]) do
+
+  @directory = Fog::Storage[:aws].directories.create(:key => uniq_id('fogmultipartcopytests'))
+  @large_data = SecureRandom.hex * 600000
+  @large_blob = Fog::Storage[:aws].put_object(@directory.identity, 'large_object', @large_data)
+
+  tests('copies an empty object') do
+    Fog::Storage[:aws].put_object(@directory.identity, 'empty_object', '')
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('empty_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', 'empty_copied_object'").succeeds do
+      file.copy(@directory.identity, 'empty_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('empty_copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a small object') do
+    Fog::Storage[:aws].put_object(@directory.identity, 'fog_object', lorem_file)
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('fog_object')
+
+    tests("#copy_object('#{@directory.identity}', 'copied_object'").succeeds do
+      file.copy(@directory.identity, 'copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file needing a single part') do
+    data = '*' * Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+    Fog::Storage[:aws].put_object(@directory.identity, '1_part_object', data)
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('1_part_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', '1_part_copied_object'").succeeds do
+      file.copy(@directory.identity, '1_part_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('1_part_copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file with many parts') do
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', 'large_copied_object'").succeeds do
+      file.copy(@directory.identity, 'large_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_copied_object')
+
+    test("concurrency defaults to 1") { file.concurrency == 1 }
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file with many parts with 10 threads') do
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+    file.concurrency = 10
+
+    test("concurrency is set to 10") { file.concurrency == 10 }
+
+    tests("#copy_object('#{@directory.identity}', 'copied_object_with_10_threads'").succeeds do
+      file.copy(@directory.identity, 'copied_object_with_10_threads')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('copied_object_with_10_threads')
+
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies an object with unknown headers') do
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+    file.concurrency = 10
+
+    tests("#copy_object('#{@directory.identity}', 'copied_object'").succeeds do
+      file.copy(@directory.identity, 'copied_object', { unknown: 1 } )
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+end

data/tests/requests/storage/object_tests.rb

@@ -18,6 +18,13 @@ Shindo.tests('AWS::Storage | object requests', ['aws']) do
       Fog::Storage[:aws].put_object(@directory.identity, 'fog_object', lorem_file)
     end
 
+    tests("#put_object('#{@directory.identity}', 'fog_object', lorem_file at EOF)").returns(lorem_file.read) do
+      file = lorem_file
+      file.read
+      Fog::Storage[:aws].put_object(@directory.identity, 'fog_object', file)
+      Fog::Storage[:aws].get_object(@directory.identity, 'fog_object').body
+    end
+
     tests("#copy_object('#{@directory.identity}', 'fog_object', '#{@directory.identity}', 'fog_other_object')").succeeds do
       Fog::Storage[:aws].copy_object(@directory.identity, 'fog_object', @directory.identity, 'fog_other_object')
     end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fog-aws
 version: !ruby/object:Gem::Version
-  version: 3.6.5
+  version: 3.9.0
 platform: ruby
 authors:
 - Josh Lane
 - Wesley Beary
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-22 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -754,6 +754,7 @@ files:
 - lib/fog/aws/parsers/storage/initiate_multipart_upload.rb
 - lib/fog/aws/parsers/storage/list_multipart_uploads.rb
 - lib/fog/aws/parsers/storage/list_parts.rb
+- lib/fog/aws/parsers/storage/upload_part_copy_object.rb
 - lib/fog/aws/parsers/sts/assume_role.rb
 - lib/fog/aws/parsers/sts/assume_role_with_saml.rb
 - lib/fog/aws/parsers/sts/assume_role_with_web_identity.rb
@@ -1442,6 +1443,7 @@ files:
 - lib/fog/aws/requests/storage/shared_mock_methods.rb
 - lib/fog/aws/requests/storage/sync_clock.rb
 - lib/fog/aws/requests/storage/upload_part.rb
+- lib/fog/aws/requests/storage/upload_part_copy.rb
 - lib/fog/aws/requests/sts/assume_role.rb
 - lib/fog/aws/requests/sts/assume_role_with_saml.rb
 - lib/fog/aws/requests/sts/assume_role_with_web_identity.rb
@@ -1709,6 +1711,7 @@ files:
 - tests/requests/storage/bucket_tests.rb
 - tests/requests/storage/cors_utils_tests.rb
 - tests/requests/storage/delete_multiple_objects_tests.rb
+- tests/requests/storage/multipart_copy_tests.rb
 - tests/requests/storage/multipart_upload_tests.rb
 - tests/requests/storage/object_tests.rb
 - tests/requests/storage/versioning_tests.rb
@@ -1726,7 +1729,7 @@ homepage: https://github.com/fog/fog-aws
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -1741,8 +1744,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Module for the 'fog' gem to support Amazon Web Services.
 test_files: []