fog-aws 3.6.7 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b18145d8a9c814854524a02640ca995f074ae8a63c232ab50a6fc84f7387237c
-  data.tar.gz: 36346830f2b660a561b372b9fd8b7c48d58f95bb53971d480199477909d4fe9b
+  metadata.gz: 9ac0628bad4edf849a2b1395bd829d18ae77bff352841e68fbc7ecdb475ef82b
+  data.tar.gz: 5a545b7e8ad44c321273fac7b87b11b79be3a9af1c2d038b2790ea95c529df30
 SHA512:
-  metadata.gz: 29957fd751f2ee82a52cf0bbbff5740a5e80e6e3c6a9cd9aa5da486b4cd1c2fbe1a19486a2ecddfd92731183e4650ca839886ce2ba7d6a5ff991db6d814fbd3e
-  data.tar.gz: f869e131ec87666ad2e0741e858d2f107613fc7e1151683e32dc650e844420ec0692beaae74eeb018e3297d346c14cbe59a76af7c1d754c62fc9052426e80bfb
+  metadata.gz: 06156a44eec55618672a3d5beed5d35c0e7195ba07a3a515e944db462edbd6b4e56f6ff693bd5dd0013090c2083f395cf9f310013a164d3a3163c7ea82f96b9e
+  data.tar.gz: ebf606599061ed26cf2f1995e91690aebb7b409b15fe1d6ee465ef3e36a87b9c714e376e9bf6fee9c276a45a994972d94a24ef49d27f92e8b64ee6a47a9ae082

data/.travis.yml

@@ -19,6 +19,14 @@ matrix:
     gemfile: Gemfile
   - rvm: 2.5.0
     gemfile: gemfiles/Gemfile-edge
+  - rvm: 2.6.6
+    gemfile: Gemfile
+  - rvm: 2.6.6
+    gemfile: gemfiles/Gemfile-edge
+  - rvm: 2.7.2
+    gemfile: Gemfile
+  - rvm: 2.7.2
+    gemfile: gemfiles/Gemfile-edge
   - rvm: jruby-head
     gemfile: Gemfile
   allow_failures:
@@ -32,6 +40,10 @@ matrix:
     gemfile: gemfiles/Gemfile-edge
   - rvm: 2.5.0
     gemfile: gemfiles/Gemfile-edge
+  - rvm: 2.6.6
+    gemfile: gemfiles/Gemfile-edge
+  - rvm: 2.7.2
+    gemfile: gemfiles/Gemfile-edge
 notifications:
   email: false
   irc:

data/CHANGELOG.md

@@ -1,8 +1,23 @@
 # Change Log
 
-## [v3.6.7](https://github.com/fog/fog-aws/tree/HEAD)(2020-08-26)
+## [v3.7.0](https://github.com/fog/fog-aws/tree/v3.7.0) (2020-12-01)
+[Full Changelog](https://github.com/fog/fog-aws/compare/v3.7.0...HEAD)
 
-[Full Changelog](https://github.com/fog/fog-aws/compare/v3.6.6...HEAD)
+**Closed issues:**
+
+- File\#copy does not support files above 5 GB [\#577](https://github.com/fog/fog-aws/issues/577)
+- fog-aws: AWS extended length resource ID issues \(8-\>18\) [\#517](https://github.com/fog/fog-aws/issues/517)
+
+**Merged pull requests:**
+
+- Add all m6gd, r6g, r6gd, c6g, and c6gd instance classes [\#582](https://github.com/fog/fog-aws/pull/582) ([calebwoofenden](https://github.com/calebwoofenden))
+- Test Ruby v2.6.6 and v2.7.2 in CI [\#581](https://github.com/fog/fog-aws/pull/581) ([stanhu](https://github.com/stanhu))
+- Add multi-threaded support for File\#copy [\#579](https://github.com/fog/fog-aws/pull/579) ([stanhu](https://github.com/stanhu))
+- Add support for multipart Fog::AWS::Storage::File\#copy [\#578](https://github.com/fog/fog-aws/pull/578) ([stanhu](https://github.com/stanhu))
+- Add AssumeRoleWithWebIdentity to fetch\_credentials [\#576](https://github.com/fog/fog-aws/pull/576) ([jpac-run](https://github.com/jpac-run))
+
+## [v3.6.7](https://github.com/fog/fog-aws/tree/v3.6.7) (2020-08-26)
+[Full Changelog](https://github.com/fog/fog-aws/compare/v3.6.6...v3.6.7)
 
 **Merged pull requests:**
 
@@ -293,6 +308,7 @@
 
 - Update changelog for 1.4.0 [\#383](https://github.com/fog/fog-aws/pull/383) ([greysteil](https://github.com/greysteil))
 - Allow specifying kms key id to use [\#382](https://github.com/fog/fog-aws/pull/382) ([fcheung](https://github.com/fcheung))
+- added support to retrieve and create vpc with ipv6 cidr block [\#381](https://github.com/fog/fog-aws/pull/381) ([chanakyacool](https://github.com/chanakyacool))
 - Add MaxResults filter to describe reserved instances offerings [\#376](https://github.com/fog/fog-aws/pull/376) ([KevinLoiseau](https://github.com/KevinLoiseau))
 - Fix Fog::Compute::AWS::Images\#all [\#375](https://github.com/fog/fog-aws/pull/375) ([eddiej](https://github.com/eddiej))
 - Fix AWS credential mocking [\#374](https://github.com/fog/fog-aws/pull/374) ([v-yarotsky](https://github.com/v-yarotsky))
@@ -310,7 +326,6 @@
 
 **Merged pull requests:**
 
-- added support to retrieve and create vpc with ipv6 cidr block [\#381](https://github.com/fog/fog-aws/pull/381) ([chanakyacool](https://github.com/chanakyacool))
 - add NextContinuationToken support to GetBucket operation [\#370](https://github.com/fog/fog-aws/pull/370) ([khoan](https://github.com/khoan))
 - Add a top-level require that matches the gem name [\#367](https://github.com/fog/fog-aws/pull/367) ([lanej](https://github.com/lanej))
 - Fixed credential refresh when instance metadata host is inaccessible [\#366](https://github.com/fog/fog-aws/pull/366) ([ankane](https://github.com/ankane))

data/Gemfile

@@ -11,4 +11,4 @@ end
 group :test do
   gem "simplecov"
   gem "codeclimate-test-reporter", "~> 1.0.0"
-end
+end

data/README.md

@@ -105,6 +105,26 @@ directory.files
 directory.files.new(key: 'user/1/Gemfile').url(Time.now + 60)
 ```
 
+#### Copying a file
+
+```ruby
+directory = s3.directories.new(key: 'gaudi-portal-dev')
+file = directory.files.get('user/1/Gemfile')
+file.copy("target-bucket", "user/2/Gemfile.copy")
+```
+
+To speed transfers of large files, the `concurrency` option can be used
+to spawn multiple threads. Note that the file must be at least 5 MB for
+multipart uploads to work. For example:
+
+```ruby
+directory = s3.directories.new(key: 'gaudi-portal-dev')
+file = directory.files.get('user/1/Gemfile')
+file.multipart_chunk_size = 10 * 1024 * 1024
+file.concurrency = 10
+file.copy("target-bucket", "user/2/Gemfile.copy")
+```
+
 ## Documentation
 
 See the [online documentation](http://www.rubydoc.info/github/fog/fog-aws) for a complete API reference.

data/lib/fog/aws/credential_fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Fog
   module AWS
     module CredentialFetcher
@@ -9,6 +11,8 @@ module Fog
 
       CONTAINER_CREDENTIALS_HOST = "http://169.254.170.2"
 
+      STS_GLOBAL_ENDPOINT = "https://sts.amazonaws.com"
+
       module ServiceMethods
         def fetch_credentials(options)
           if options[:use_iam_profile] && Fog.mocking?
@@ -23,6 +27,30 @@ module Fog
                 connection = options[:connection] || Excon.new(CONTAINER_CREDENTIALS_HOST)
                 credential_path = options[:credential_path] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
                 role_data = connection.get(:path => credential_path, :idempotent => true, :expects => 200).body
+                session = Fog::JSON.decode(role_data)
+
+                if region.nil?
+                  connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
+                  token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
+                end
+              elsif ENV["AWS_WEB_IDENTITY_TOKEN_FILE"]
+                params = {
+                  :Action => "AssumeRoleWithWebIdentity",
+                  :RoleArn => options[:role_arn] || ENV.fetch("AWS_ROLE_ARN"),
+                  :RoleSessionName => options[:role_session_name] || ENV.fetch("AWS_ROLE_SESSION_NAME"),
+                  :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")),
+                  :Version => "2011-06-15",
+                }
+                connection = options[:connection] || Excon.new(STS_GLOBAL_ENDPOINT, :query => params)
+                document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body)
+
+                session = {
+                  "AccessKeyId" => document.css("AccessKeyId").children.text,
+                  "SecretAccessKey" => document.css("SecretAccessKey").children.text,
+                  "Token" => document.css("SessionToken").children.text,
+                  "Expiration" => document.css("Expiration").children.text,
+                }
 
                 if region.nil?
                   connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
@@ -34,16 +62,17 @@ module Fog
                 token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
                 role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200, :headers => token_header).body
                 role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200, :headers => token_header).body
+                session = Fog::JSON.decode(role_data)
+                
                 region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
               end
-
-              session = Fog::JSON.decode(role_data)
+              
               credentials = {}
               credentials[:aws_access_key_id] = session['AccessKeyId']
               credentials[:aws_secret_access_key] = session['SecretAccessKey']
               credentials[:aws_session_token] = session['Token']
               credentials[:aws_credentials_expire_at] = Time.xmlschema session['Expiration']
-
+              
               # set region by default to the one the instance is in.
               credentials[:region] = region
               #these indicate the metadata service is unavailable or has no profile setup

data/lib/fog/aws/models/compute/flavors.rb

@@ -366,6 +366,106 @@ module Fog
           :ebs_optimized_available => true,
           :instance_store_volumes  => 0
         },
+        {
+          :id                      => 'm6g.metal',
+          :name                    => 'M6G Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 0,
+          :ram                     => 274878,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'm6gd.medium',
+          :name                    => 'M6GD Medium',
+          :bits                    => 64,
+          :cores                   => 1,
+          :disk                    => 59,
+          :ram                     => 4295,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.large',
+          :name                    => 'M6GD Large',
+          :bits                    => 64,
+          :cores                   => 2,
+          :disk                    => 118,
+          :ram                     => 8590,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.xlarge',
+          :name                    => 'M6GD Extra Large',
+          :bits                    => 64,
+          :cores                   => 4,
+          :disk                    => 237,
+          :ram                     => 17180,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.2xlarge',
+          :name                    => 'M6GD Double Extra Large',
+          :bits                    => 64,
+          :cores                   => 8,
+          :disk                    => 474,
+          :ram                     => 34360,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.4xlarge',
+          :name                    => 'M6GD Quadruple Extra Large',
+          :bits                    => 64,
+          :cores                   => 16,
+          :disk                    => 950,
+          :ram                     => 68719,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.8xlarge',
+          :name                    => 'M6GD Octuple Extra Large',
+          :bits                    => 64,
+          :cores                   => 32,
+          :disk                    => 1900,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'm6gd.12xlarge',
+          :name                    => 'M6GD Twelve Extra Large',
+          :bits                    => 64,
+          :cores                   => 48,
+          :disk                    => 2850,
+          :ram                     => 206158,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'm6gd.16xlarge',
+          :name                    => 'M6GD Sixteen Extra Large',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 274878,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'm6gd.metal',
+          :name                    => 'M6GD Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 274878,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
         {
           :id                      => 'm1.small',
           :name                    => 'M1 Small Instance',
@@ -776,6 +876,186 @@ module Fog
           :ebs_optimized_available => true,
           :instance_store_volumes  => 0
         },
+        {
+          :id                      => 'c6g.medium',
+          :name                    => 'C6G Medium',
+          :bits                    => 64,
+          :cores                   => 1,
+          :disk                    => 0,
+          :ram                     => 2147,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.large',
+          :name                    => 'C6G Large',
+          :bits                    => 64,
+          :cores                   => 2,
+          :disk                    => 0,
+          :ram                     => 4295,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.xlarge',
+          :name                    => 'C6G Extra Large',
+          :bits                    => 64,
+          :cores                   => 4,
+          :disk                    => 0,
+          :ram                     => 8590,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.2xlarge',
+          :name                    => 'C6G Double Extra Large',
+          :bits                    => 64,
+          :cores                   => 8,
+          :disk                    => 0,
+          :ram                     => 17180,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.4xlarge',
+          :name                    => 'C6G Quadruple Extra Large',
+          :bits                    => 64,
+          :cores                   => 16,
+          :disk                    => 0,
+          :ram                     => 34360,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.8xlarge',
+          :name                    => 'C6G Octuple Extra Large',
+          :bits                    => 64,
+          :cores                   => 32,
+          :disk                    => 0,
+          :ram                     => 68719,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.12xlarge',
+          :name                    => 'C6G Twelve Extra Large',
+          :bits                    => 64,
+          :cores                   => 48,
+          :disk                    => 0,
+          :ram                     => 103079,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.16xlarge',
+          :name                    => 'C6G Sixteen Extra Large',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 0,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6g.metal',
+          :name                    => 'C6G Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 0,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'c6gd.medium',
+          :name                    => 'C6GD Medium',
+          :bits                    => 64,
+          :cores                   => 1,
+          :disk                    => 59,
+          :ram                     => 2147,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.large',
+          :name                    => 'C6GD Large',
+          :bits                    => 64,
+          :cores                   => 2,
+          :disk                    => 118,
+          :ram                     => 4295,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.xlarge',
+          :name                    => 'C6GD Extra Large',
+          :bits                    => 64,
+          :cores                   => 4,
+          :disk                    => 237,
+          :ram                     => 8590,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.2xlarge',
+          :name                    => 'C6GD Double Extra Large',
+          :bits                    => 64,
+          :cores                   => 8,
+          :disk                    => 474,
+          :ram                     => 17180,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.4xlarge',
+          :name                    => 'C6GD Quadruple Extra Large',
+          :bits                    => 64,
+          :cores                   => 16,
+          :disk                    => 950,
+          :ram                     => 34360,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.8xlarge',
+          :name                    => 'C6GD Octuple Extra Large',
+          :bits                    => 64,
+          :cores                   => 32,
+          :disk                    => 1900,
+          :ram                     => 68719,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'c6gd.12xlarge',
+          :name                    => 'C6GD Twelve Extra Large',
+          :bits                    => 64,
+          :cores                   => 48,
+          :disk                    => 2850,
+          :ram                     => 103079,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'c6gd.16xlarge',
+          :name                    => 'C6GD Sixteen Extra Large',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'c6gd.metal',
+          :name                    => 'C6GD Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
         {
           :id                      => 'g2.2xlarge',
           :name                    => 'GPU Double Extra Large',
@@ -1696,6 +1976,186 @@ module Fog
           :ebs_optimized_available => true,
           :instance_store_volumes  => 4
         },
+        {
+          :id                      => 'r6g.medium',
+          :name                    => 'R6G Medium',
+          :bits                    => 64,
+          :cores                   => 1,
+          :disk                    => 0,
+          :ram                     => 8590,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.large',
+          :name                    => 'R6G Large',
+          :bits                    => 64,
+          :cores                   => 2,
+          :disk                    => 0,
+          :ram                     => 17180,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.xlarge',
+          :name                    => 'R6G Extra Large',
+          :bits                    => 64,
+          :cores                   => 4,
+          :disk                    => 0,
+          :ram                     => 34360,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.2xlarge',
+          :name                    => 'R6G Double Extra Large',
+          :bits                    => 64,
+          :cores                   => 8,
+          :disk                    => 0,
+          :ram                     => 68719,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.4xlarge',
+          :name                    => 'R6G Quadruple Extra Large',
+          :bits                    => 64,
+          :cores                   => 16,
+          :disk                    => 0,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.8xlarge',
+          :name                    => 'R6G Octuple Extra Large',
+          :bits                    => 64,
+          :cores                   => 32,
+          :disk                    => 0,
+          :ram                     => 274878,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.12xlarge',
+          :name                    => 'R6G Twelve Extra Large',
+          :bits                    => 64,
+          :cores                   => 48,
+          :disk                    => 0,
+          :ram                     => 412317,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.16xlarge',
+          :name                    => 'R6G Sixteen Extra Large',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 0,
+          :ram                     => 549756,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6g.metal',
+          :name                    => 'R6G Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 0,
+          :ram                     => 549756,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 0
+        },
+        {
+          :id                      => 'r6gd.medium',
+          :name                    => 'R6GD Medium',
+          :bits                    => 64,
+          :cores                   => 1,
+          :disk                    => 59,
+          :ram                     => 8590,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.large',
+          :name                    => 'R6GD Large',
+          :bits                    => 64,
+          :cores                   => 2,
+          :disk                    => 118,
+          :ram                     => 17180,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.xlarge',
+          :name                    => 'R6GD Extra Large',
+          :bits                    => 64,
+          :cores                   => 4,
+          :disk                    => 237,
+          :ram                     => 34360,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.2xlarge',
+          :name                    => 'R6GD Double Extra Large',
+          :bits                    => 64,
+          :cores                   => 8,
+          :disk                    => 474,
+          :ram                     => 68719,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.4xlarge',
+          :name                    => 'R6GD Quadruple Extra Large',
+          :bits                    => 64,
+          :cores                   => 16,
+          :disk                    => 950,
+          :ram                     => 137439,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.8xlarge',
+          :name                    => 'R6GD Octuple Extra Large',
+          :bits                    => 64,
+          :cores                   => 32,
+          :disk                    => 1900,
+          :ram                     => 274878,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 1
+        },
+        {
+          :id                      => 'r6gd.12xlarge',
+          :name                    => 'R6GD Twelve Extra Large',
+          :bits                    => 64,
+          :cores                   => 48,
+          :disk                    => 2850,
+          :ram                     => 412317,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'r6gd.16xlarge',
+          :name                    => 'R6GD Sixteen Extra Large',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 549756,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
+        {
+          :id                      => 'r6gd.metal',
+          :name                    => 'R6GD Metal',
+          :bits                    => 64,
+          :cores                   => 64,
+          :disk                    => 3800,
+          :ram                     => 549756,
+          :ebs_optimized_available => true,
+          :instance_store_volumes  => 2
+        },
         {
           :id                      => "x1.16xlarge",
           :name                    => "X1 Sixteen Extra Large",

data/lib/fog/aws/models/storage/file.rb

@@ -4,6 +4,10 @@ module Fog
   module AWS
     class Storage
       class File < Fog::Model
+        MIN_MULTIPART_CHUNK_SIZE = 5242880
+        MAX_SINGLE_PUT_SIZE = 5368709120
+        MULTIPART_COPY_THRESHOLD = 15728640
+
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
 
         identity  :key,             :aliases => 'Key'
@@ -27,14 +31,54 @@ module Fog
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
         attribute :tags,                :aliases => 'x-amz-tagging'
 
+        UploadPartData = Struct.new(:part_number, :upload_options, :etag)
+
+        class PartList
+          def initialize(parts = [])
+            @parts = parts
+            @mutex = Mutex.new
+          end
+
+          def push(part)
+            @mutex.synchronize { @parts.push(part) }
+          end
+
+          def shift
+            @mutex.synchronize { @parts.shift }
+          end
+
+          def clear!
+            @mutex.synchronize { @parts.clear }
+          end
+
+          def size
+            @mutex.synchronize { @parts.size }
+          end
+
+          def to_a
+            @mutex.synchronize { @parts.dup }
+          end
+        end
+
         # @note Chunk size to use for multipart uploads.
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
           @multipart_chunk_size = mp_chunk_size
         end
 
+        # @note Number of threads used to copy files.
+        def concurrency=(concurrency)
+          raise ArgumentError.new('minimum concurrency is 1') if concurrency.to_i < 1
+
+          @concurrency = concurrency.to_i
+        end
+
+        def concurrency
+          @concurrency || 1
+        end
+
         def acl
           requires :directory, :key
           service.get_object_acl(directory.key, key).body['AccessControlList']
@@ -99,7 +143,17 @@ module Fog
         #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+
+          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+
+          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+            multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          else
+            service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          end
+
           target_directory = service.directories.new(:key => target_directory_key)
           target_directory.files.head(target_file_key)
         end
@@ -214,7 +268,7 @@ module Fog
           options.merge!(encryption_headers)
 
           # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
 
           if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
@@ -294,6 +348,30 @@ module Fog
           service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
+        def multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          # Initiate the upload
+          res = service.initiate_multipart_upload(target_directory_key, target_file_key, options)
+          upload_id = res.body["UploadId"]
+
+          # Store ETags of upload parts
+          part_tags = []
+          pending = PartList.new(create_part_list(upload_part_options))
+          thread_count = self.concurrency
+          completed = PartList.new
+          errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+
+          raise error.first if errors.any?
+
+          part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
+        rescue => e
+          # Abort the upload & reraise
+          service.abort_multipart_upload(target_directory_key, target_file_key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(target_directory_key, target_file_key, upload_id, part_tags)
+        end
+
         def encryption_headers
           if encryption && encryption_key
             encryption_customer_key_headers
@@ -318,6 +396,49 @@ module Fog
             'x-amz-server-side-encryption-customer-key-md5' => Base64.encode64(OpenSSL::Digest::MD5.digest(encryption_key.to_s)).chomp!
           }
         end
+
+        def create_part_list(upload_part_options)
+          current_pos = 0
+          count = 0
+          pending = []
+
+          while current_pos < self.content_length do
+            start_pos = current_pos
+            end_pos = [current_pos + self.multipart_chunk_size, self.content_length - 1].min
+            range = "bytes=#{start_pos}-#{end_pos}"
+            part_options = upload_part_options.dup
+            part_options['x-amz-copy-source-range'] = range
+            pending << UploadPartData.new(count + 1, part_options, nil)
+            count += 1
+            current_pos = end_pos + 1
+          end
+
+          pending
+        end
+
+        def upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+          threads = []
+
+          thread_count.times do
+            thread = Thread.new do
+              begin
+                while part = pending.shift
+                  part_upload = service.upload_part_copy(target_directory_key, target_file_key, upload_id, part.part_number, part.upload_options)
+                  part.etag = part_upload.body['ETag']
+                  completed.push(part)
+                end
+              rescue => error
+                pending.clear!
+                error
+              end
+            end
+
+            thread.abort_on_exception = true
+            threads << thread
+          end
+
+          threads.map(&:value).compact
+        end
       end
     end
   end

data/lib/fog/aws/parsers/storage/upload_part_copy_object.rb

@@ -0,0 +1,18 @@
+module Fog
+  module Parsers
+    module AWS
+      module Storage
+        class UploadPartCopyObject < Fog::Parsers::Base
+          def end_element(name)
+            case name
+            when 'ETag'
+              @response[name] = value.gsub('"', '')
+            when 'LastModified'
+              @response[name] = Time.parse(value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/compute/request_spot_instances.rb

@@ -113,7 +113,7 @@ module Fog
               raise Fog::AWS::Compute::Error.new(message)
             end
 
-            if !image_id.match(/^ami-[a-f0-9]{8}$/)
+            if !image_id.match(/^ami-[a-f0-9]{8,17}$/)
               message = "The image id '[#{image_id}]' does not exist"
               raise Fog::AWS::Compute::NotFound.new(message)
             end

data/lib/fog/aws/requests/storage/upload_part_copy.rb

@@ -0,0 +1,92 @@
+module Fog
+  module AWS
+    class Storage
+      class Real
+        require 'fog/aws/parsers/storage/upload_part_copy_object'
+
+        # Upload a part for a multipart copy
+        #
+        # @param target_bucket_name [String] Name of bucket to create copy in
+        # @param target_object_name [String] Name for new copy of object
+        # @param upload_id [String] Id of upload to add part to
+        # @param part_number [String] Index of part in upload
+        # @param options [Hash]:
+        # @option options [String] x-amz-metadata-directive Specifies whether to copy metadata from source or replace with data in request.  Must be in ['COPY', 'REPLACE']
+        # @option options [String] x-amz-copy_source-if-match Copies object if its etag matches this value
+        # @option options [Time] x-amz-copy_source-if-modified_since Copies object it it has been modified since this time
+        # @option options [String] x-amz-copy_source-if-none-match Copies object if its etag does not match this value
+        # @option options [Time] x-amz-copy_source-if-unmodified-since Copies object it it has not been modified since this time
+        # @option options [Time] x-amz-copy-source-range Specifes the range of bytes to copy from the source object
+        #
+        # @return [Excon::Response]
+        #   * body [Hash]:
+        #     * ETag [String] - etag of new object
+        #     * LastModified [Time] - date object was last modified
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+        #
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          headers = options
+          request({
+            :expects    => 200,
+            :idempotent => true,
+            :headers    => headers,
+            :bucket_name => target_bucket_name,
+            :object_name => target_object_name,
+            :method     => 'PUT',
+            :query      => {'uploadId' => upload_id, 'partNumber' => part_number},
+            :parser   => Fog::Parsers::AWS::Storage::UploadPartCopyObject.new,
+          })
+        end
+      end # Real
+
+      class Mock # :nodoc:all
+        require 'fog/aws/requests/storage/shared_mock_methods'
+        include Fog::AWS::Storage::SharedMockMethods
+
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          copy_source = options['x-amz-copy-source']
+          copy_range = options['x-amz-copy-source-range']
+
+          raise 'No x-amz-copy-source header provided' unless copy_source
+          raise 'No x-amz-copy-source-range header provided' unless copy_range
+
+          source_bucket_name, source_object_name = copy_source.split('/', 2)
+          verify_mock_bucket_exists(source_bucket_name)
+
+          source_bucket = self.data[:buckets][source_bucket_name]
+          source_object = source_bucket && source_bucket[:objects][source_object_name] && source_bucket[:objects][source_object_name].first
+          upload_info = get_upload_info(target_bucket_name, upload_id)
+
+          response = Excon::Response.new
+
+          if source_object
+            start_pos, end_pos = byte_range(copy_range, source_object[:body].length)
+            upload_info[:parts][part_number] = source_object[:body][start_pos..end_pos]
+
+            response.status = 200
+            response.body = {
+              # just use the part number as the ETag, for simplicity
+              'ETag'          => part_number.to_i,
+              'LastModified'  => Time.parse(source_object['Last-Modified'])
+            }
+            response
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+        end
+
+        def byte_range(range, size)
+          matches = range.match(/bytes=(\d*)-(\d*)/)
+
+          return nil unless matches
+
+          end_pos = [matches[2].to_i, size].min
+
+          [matches[1].to_i, end_pos]
+        end
+      end # Mock
+    end # Storage
+  end # AWS
+end # Fog

data/lib/fog/aws/storage.rb

@@ -112,6 +112,7 @@ module Fog
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.6.7"
+    VERSION = "3.7.0"
   end
 end

data/tests/credentials_tests.rb

@@ -60,6 +60,35 @@ Shindo.tests('AWS | credentials', ['aws']) do
 
     ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
 
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = File.dirname(__FILE__) + '/lorem.txt'
+    ENV['AWS_ROLE_ARN'] = "dummyrole"
+    ENV['AWS_ROLE_SESSION_NAME'] = "dummyrolesessionname"
+    document = 
+      '<AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">'\
+        '<AssumeRoleWithWebIdentityResult>'\
+          '<Credentials>'\
+            '<SessionToken>dummytoken</SessionToken>'\
+            '<SecretAccessKey>dummysecret</SecretAccessKey>'\
+            "<Expiration>#{expires_at.xmlschema}</Expiration>"\
+            '<AccessKeyId>dummykey</AccessKeyId>'\
+          '</Credentials>'\
+        '</AssumeRoleWithWebIdentityResult>'\
+      '</AssumeRoleWithWebIdentityResponse>'
+    
+    Excon.stub({method: :get, path: "/", idempotent: true}, { status: 200, body: document})
+
+    tests('#fetch_credentials token based') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
+
     compute = Fog::AWS::Compute.new(use_iam_profile: true)
 
     tests('#refresh_credentials_if_expired') do
@@ -100,6 +129,7 @@ Shindo.tests('AWS | credentials', ['aws']) do
     end
   ensure
     ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
     Excon.stubs.clear
     Excon.defaults[:mock] = old_mock_value
     Fog.mock! if fog_was_mocked

data/tests/requests/storage/multipart_copy_tests.rb

@@ -0,0 +1,80 @@
+require 'securerandom'
+
+Shindo.tests('Fog::Storage[:aws] | copy requests', ["aws"]) do
+
+  @directory = Fog::Storage[:aws].directories.create(:key => uniq_id('fogmultipartcopytests'))
+  @large_data = SecureRandom.hex * 19 * 1024 * 1024
+  @large_blob = Fog::Storage[:aws].put_object(@directory.identity, 'large_object', @large_data)
+
+  tests('copies an empty object') do
+    Fog::Storage[:aws].put_object(@directory.identity, 'empty_object', '')
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('empty_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', 'empty_copied_object'").succeeds do
+      file.copy(@directory.identity, 'empty_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('empty_copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a small object') do
+    Fog::Storage[:aws].put_object(@directory.identity, 'fog_object', lorem_file)
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('fog_object')
+
+    tests("#copy_object('#{@directory.identity}', 'copied_object'").succeeds do
+      file.copy(@directory.identity, 'copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file needing a single part') do
+    data = '*' * Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+    Fog::Storage[:aws].put_object(@directory.identity, '1_part_object', data)
+
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('1_part_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', '1_part_copied_object'").succeeds do
+      file.copy(@directory.identity, '1_part_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('1_part_copied_object')
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file with many parts') do
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+
+    tests("#copy_object('#{@directory.identity}', 'large_copied_object'").succeeds do
+      file.copy(@directory.identity, 'large_copied_object')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_copied_object')
+
+    test("concurrency defaults to 1") { file.concurrency == 1 }
+    test("copied is the same") { copied.body == file.body }
+  end
+
+  tests('copies a file with many parts with 10 threads') do
+    file = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('large_object')
+    file.multipart_chunk_size = Fog::AWS::Storage::File::MIN_MULTIPART_CHUNK_SIZE
+    file.concurrency = 10
+
+    test("concurrency is set to 10") { file.concurrency == 10 }
+
+    tests("#copy_object('#{@directory.identity}', 'copied_object_with_10_threads'").succeeds do
+      file.copy(@directory.identity, 'copied_object_with_10_threads')
+    end
+
+    copied = Fog::Storage[:aws].directories.new(key: @directory.identity).files.get('copied_object_with_10_threads')
+
+    test("copied is the same") { copied.body == file.body }
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fog-aws
 version: !ruby/object:Gem::Version
-  version: 3.6.7
+  version: 3.7.0
 platform: ruby
 authors:
 - Josh Lane
 - Wesley Beary
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-26 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -754,6 +754,7 @@ files:
 - lib/fog/aws/parsers/storage/initiate_multipart_upload.rb
 - lib/fog/aws/parsers/storage/list_multipart_uploads.rb
 - lib/fog/aws/parsers/storage/list_parts.rb
+- lib/fog/aws/parsers/storage/upload_part_copy_object.rb
 - lib/fog/aws/parsers/sts/assume_role.rb
 - lib/fog/aws/parsers/sts/assume_role_with_saml.rb
 - lib/fog/aws/parsers/sts/assume_role_with_web_identity.rb
@@ -1442,6 +1443,7 @@ files:
 - lib/fog/aws/requests/storage/shared_mock_methods.rb
 - lib/fog/aws/requests/storage/sync_clock.rb
 - lib/fog/aws/requests/storage/upload_part.rb
+- lib/fog/aws/requests/storage/upload_part_copy.rb
 - lib/fog/aws/requests/sts/assume_role.rb
 - lib/fog/aws/requests/sts/assume_role_with_saml.rb
 - lib/fog/aws/requests/sts/assume_role_with_web_identity.rb
@@ -1709,6 +1711,7 @@ files:
 - tests/requests/storage/bucket_tests.rb
 - tests/requests/storage/cors_utils_tests.rb
 - tests/requests/storage/delete_multiple_objects_tests.rb
+- tests/requests/storage/multipart_copy_tests.rb
 - tests/requests/storage/multipart_upload_tests.rb
 - tests/requests/storage/object_tests.rb
 - tests/requests/storage/versioning_tests.rb
@@ -1726,7 +1729,7 @@ homepage: https://github.com/fog/fog-aws
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -1741,8 +1744,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Module for the 'fog' gem to support Amazon Web Services.
 test_files: []