fog-aws 3.10.0 → 3.12.0

data/lib/fog/aws/storage.rb

@@ -14,6 +14,9 @@ module Fog
         'https' => 443
       }
 
+      MIN_MULTIPART_CHUNK_SIZE = 5242880
+      MAX_SINGLE_PUT_SIZE = 5368709120
+
       VALID_QUERY_KEYS = %w[
         acl
         cors
@@ -43,7 +46,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -117,6 +120,17 @@ module Fog
       module Utils
         attr_accessor :region
 
+        # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
+        # Other S3-compatible storages (like, Ceph) do not have such limit.
+        # Ceph shows much better performance when file is copied as a whole, in a single request.
+        # fog-aws user can use these settings to configure chunk sizes.
+        # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
+        #
+        # @return [Integer]
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
+        attr_reader :max_put_chunk_size
+        attr_reader :max_copy_chunk_size
+
         def cdn
           @cdn ||= Fog::AWS::CDN.new(
             :aws_access_key_id => @aws_access_key_id,
@@ -171,6 +185,12 @@ module Fog
           params_to_url(params)
         end
 
+        # @param value [int]
+        # @param description [str]
+        def validate_chunk_size(value, description)
+          raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
+        end
+
         private
 
         def validate_signature_version!
@@ -179,6 +199,16 @@ module Fog
           end
         end
 
+        def init_max_put_chunk_size!(options = {})
+          @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
+        end
+
+        def init_max_copy_chunk_size!(options = {})
+          @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
+        end
+
         def v4_signed_params_for_url(params, expires)
           now = Fog::Time.now
 
@@ -452,6 +482,10 @@ module Fog
 
 
           @path_style = options[:path_style] || false
+
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -515,6 +549,9 @@ module Fog
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.10.0"
+    VERSION = "3.12.0"
   end
 end

data/tests/credentials_tests.rb

@@ -83,6 +83,7 @@ Shindo.tests('AWS | credentials', ['aws']) do
         aws_secret_access_key: 'dummysecret',
         aws_session_token: 'dummytoken',
         region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
         aws_credentials_expire_at: expires_at
       ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
@@ -95,10 +96,50 @@ Shindo.tests('AWS | credentials', ['aws']) do
         aws_secret_access_key: 'dummysecret',
         aws_session_token: 'dummytoken',
         region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
+    end
+
+    ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional"
+
+    tests('#fetch_credentials with no region specified') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    tests('#fetch_credentials with regional STS endpoint') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.us-west-1.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
+    end
+
+    ENV["AWS_DEFAULT_REGION"] = "us-west-1"
+
+    tests('#fetch_credentials with regional STS endpoint with region in env') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.us-west-1.amazonaws.com",
         aws_credentials_expire_at: expires_at
       ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
 
+    ENV["AWS_STS_REGIONAL_ENDPOINTS"] = nil
+    ENV["AWS_DEFAULT_REGION"] = nil
     ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
 
     compute = Fog::AWS::Compute.new(use_iam_profile: true)

data/tests/requests/compute/security_group_tests.rb

@@ -19,6 +19,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         'groups'      => [{ 'groupName' => Fog::Nullable::String, 'userId' => String, 'groupId' => String }],
         'ipProtocol'  => String,
         'ipRanges'    => [Fog::Nullable::Hash],
+        'ipv6Ranges'  => [Fog::Nullable::Hash],
         'toPort'      => Fog::Nullable::Integer,
       }],
       'ipPermissionsEgress' => [],
@@ -54,16 +55,19 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "toPort"=>65535},
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"udp",
         "toPort"=>65535},
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>-1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"icmp",
         "toPort"=>-1}
     ]
@@ -88,6 +92,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>1,
         "toPort"=>65535},
@@ -95,6 +100,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"udp",
         "fromPort"=>1,
         "toPort"=>65535},
@@ -102,6 +108,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"icmp",
         "fromPort"=>-1,
         "toPort"=>-1}
@@ -133,6 +140,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[],
         "ipRanges"=>[{"cidrIp"=>"10.0.0.0/8"}],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>22,
         "toPort"=>22}
@@ -164,7 +172,8 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
       'IpPermissions' => [
         {
           'IpProtocol' => 'tcp', 'FromPort' => '80', 'ToPort' => '80',
-          'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }]
+          'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }],
+          'Ipv6Ranges' => []
         }
       ]
     }
@@ -177,6 +186,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[],
         "ipRanges"=>[{"cidrIp"=>"192.168.0.0/24"}],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>80,
         "toPort"=>80}
@@ -204,6 +214,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[{"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>8000,
         "toPort"=>8000}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fog-aws
 version: !ruby/object:Gem::Version
-  version: 3.10.0
+  version: 3.12.0
 platform: ruby
 authors:
 - Josh Lane
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-22 00:00:00.000000000 Z
+date: 2021-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,14 +59,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 2.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 2.3.0
 - !ruby/object:Gem::Dependency
   name: shindo
   requirement: !ruby/object:Gem::Requirement
@@ -147,21 +147,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/dependabot.yml"
-- ".github/workflows/ruby.yml"
-- ".github/workflows/stale.yml"
-- ".gitignore"
 - CHANGELOG.md
 - CONTRIBUTING.md
 - CONTRIBUTORS.md
-- Gemfile
 - LICENSE.md
 - README.md
-- Rakefile
-- bin/console
-- bin/setup
 - fog-aws.gemspec
-- gemfiles/Gemfile-edge
 - lib/fog-aws.rb
 - lib/fog/aws.rb
 - lib/fog/aws/auto_scaling.rb
@@ -185,9 +176,7 @@ files:
 - lib/fog/aws/federation.rb
 - lib/fog/aws/glacier.rb
 - lib/fog/aws/iam.rb
-- lib/fog/aws/iam/default_policies.json
 - lib/fog/aws/iam/default_policies.rb
-- lib/fog/aws/iam/default_policy_versions.json
 - lib/fog/aws/iam/paged_collection.rb
 - lib/fog/aws/kinesis.rb
 - lib/fog/aws/kms.rb
@@ -1744,7 +1733,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: Module for the 'fog' gem to support Amazon Web Services.

data/.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "bundler"
-    directory: "/"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"

data/.github/workflows/ruby.yml

@@ -1,36 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
-# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
-
-name: Ruby
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  test:
-
-    env:
-      BUNDLER_GEMFILE: gemfiles/Gemfile-edge
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        ruby-version: ['2.3', '2.4', '2.5', '2.6', '2.7', '3.0', 'head']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Install dependencies
-      run: bundle install
-    - name: Run tests
-      run: bundle exec rake

data/.github/workflows/stale.yml

@@ -1,23 +0,0 @@
-name: Mark stale issues and pull requests
-
-on:
-  schedule:
-  - cron: "30 1 * * *"
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 60
-        days-before-close: 7
-        exempt-issue-labels: 'pinned,security'
-        exempt-pr-labels: 'pinned,security'
-        stale-issue-message: 'This issue has been marked inactive and will be closed if no further activity occurs.'
-        stale-pr-message: 'This pr has been marked inactive and will be closed if no further activity occurs.'
-        stale-issue-label: 'no-issue-activity'
-        stale-pr-label: 'no-pr-activity'

data/.gitignore

@@ -1,17 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log
-tests/.fog
-gemfiles/Gemfile-edge.lock
-.idea

data/Gemfile

@@ -1,14 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fog-aws.gemspec
-gemspec
-
-group :test, :default do
-  gem 'pry-nav'
-  gem 'mime-types', '~> 3.1'
-end
-
-group :test do
-  gem "simplecov"
-  gem "codeclimate-test-reporter", "~> 1.0.0"
-end

data/Rakefile

@@ -1,14 +0,0 @@
-require "bundler/gem_tasks"
-require "github_changelog_generator/task"
-
-task :default => :test
-
-mock = ENV['FOG_MOCK'] || 'true'
-task :test do
-  sh("export FOG_MOCK=#{mock} && bundle exec shindont")
-end
-
-GitHubChangelogGenerator::RakeTask.new :changelog do |config|
-  config.user = 'fog'
-  config.project = 'fog-aws'
-end

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "fog-aws"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/gemfiles/Gemfile-edge

@@ -1,14 +0,0 @@
-source "https://rubygems.org"
-
-# Shared components
-gem "fog-core", :github => "fog/fog-core"
-gem "fog-json", :github => "fog/fog-json"
-
-group :test, :default do
-  gem 'pry-nav'
-  gem 'mime-types', '~> 3.1'
-end
-
-gem "codeclimate-test-reporter", group: :test, require: nil
-
-gemspec :path => "../"