fog-aws 3.10.0 → 3.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,9 @@ module Fog
14
14
  'https' => 443
15
15
  }
16
16
 
17
+ MIN_MULTIPART_CHUNK_SIZE = 5242880
18
+ MAX_SINGLE_PUT_SIZE = 5368709120
19
+
17
20
  VALID_QUERY_KEYS = %w[
18
21
  acl
19
22
  cors
@@ -43,7 +46,7 @@ module Fog
43
46
  ]
44
47
 
45
48
  requires :aws_access_key_id, :aws_secret_access_key
46
- recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
49
+ recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size
47
50
 
48
51
  secrets :aws_secret_access_key, :hmac
49
52
 
@@ -117,6 +120,17 @@ module Fog
117
120
  module Utils
118
121
  attr_accessor :region
119
122
 
123
+ # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
124
+ # Other S3-compatible storages (like, Ceph) do not have such limit.
125
+ # Ceph shows much better performance when file is copied as a whole, in a single request.
126
+ # fog-aws user can use these settings to configure chunk sizes.
127
+ # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
128
+ #
129
+ # @return [Integer]
130
+ # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
131
+ attr_reader :max_put_chunk_size
132
+ attr_reader :max_copy_chunk_size
133
+
120
134
  def cdn
121
135
  @cdn ||= Fog::AWS::CDN.new(
122
136
  :aws_access_key_id => @aws_access_key_id,
@@ -171,6 +185,12 @@ module Fog
171
185
  params_to_url(params)
172
186
  end
173
187
 
188
+ # @param value [int]
189
+ # @param description [str]
190
+ def validate_chunk_size(value, description)
191
+ raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
192
+ end
193
+
174
194
  private
175
195
 
176
196
  def validate_signature_version!
@@ -179,6 +199,16 @@ module Fog
179
199
  end
180
200
  end
181
201
 
202
+ def init_max_put_chunk_size!(options = {})
203
+ @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
204
+ validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
205
+ end
206
+
207
+ def init_max_copy_chunk_size!(options = {})
208
+ @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
209
+ validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
210
+ end
211
+
182
212
  def v4_signed_params_for_url(params, expires)
183
213
  now = Fog::Time.now
184
214
 
@@ -452,6 +482,10 @@ module Fog
452
482
 
453
483
 
454
484
  @path_style = options[:path_style] || false
485
+
486
+ init_max_put_chunk_size!(options)
487
+ init_max_copy_chunk_size!(options)
488
+
455
489
  @signature_version = options.fetch(:aws_signature_version, 4)
456
490
  validate_signature_version!
457
491
  setup_credentials(options)
@@ -515,6 +549,9 @@ module Fog
515
549
  validate_signature_version!
516
550
  @path_style = options[:path_style] || false
517
551
 
552
+ init_max_put_chunk_size!(options)
553
+ init_max_copy_chunk_size!(options)
554
+
518
555
  @region = options[:region] || DEFAULT_REGION
519
556
 
520
557
  if @endpoint = options[:endpoint]
@@ -1,5 +1,5 @@
1
1
  module Fog
2
2
  module AWS
3
- VERSION = "3.10.0"
3
+ VERSION = "3.12.0"
4
4
  end
5
5
  end
@@ -83,6 +83,7 @@ Shindo.tests('AWS | credentials', ['aws']) do
83
83
  aws_secret_access_key: 'dummysecret',
84
84
  aws_session_token: 'dummytoken',
85
85
  region: 'us-west-1',
86
+ sts_endpoint: "https://sts.amazonaws.com",
86
87
  aws_credentials_expire_at: expires_at
87
88
  ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
88
89
  end
@@ -95,10 +96,50 @@ Shindo.tests('AWS | credentials', ['aws']) do
95
96
  aws_secret_access_key: 'dummysecret',
96
97
  aws_session_token: 'dummytoken',
97
98
  region: 'us-west-1',
99
+ sts_endpoint: "https://sts.amazonaws.com",
100
+ aws_credentials_expire_at: expires_at
101
+ ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
102
+ end
103
+
104
+ ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional"
105
+
106
+ tests('#fetch_credentials with no region specified') do
107
+ returns(
108
+ aws_access_key_id: 'dummykey',
109
+ aws_secret_access_key: 'dummysecret',
110
+ aws_session_token: 'dummytoken',
111
+ region: 'us-west-1',
112
+ sts_endpoint: "https://sts.amazonaws.com",
113
+ aws_credentials_expire_at: expires_at
114
+ ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
115
+ end
116
+
117
+ tests('#fetch_credentials with regional STS endpoint') do
118
+ returns(
119
+ aws_access_key_id: 'dummykey',
120
+ aws_secret_access_key: 'dummysecret',
121
+ aws_session_token: 'dummytoken',
122
+ region: 'us-west-1',
123
+ sts_endpoint: "https://sts.us-west-1.amazonaws.com",
124
+ aws_credentials_expire_at: expires_at
125
+ ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
126
+ end
127
+
128
+ ENV["AWS_DEFAULT_REGION"] = "us-west-1"
129
+
130
+ tests('#fetch_credentials with regional STS endpoint with region in env') do
131
+ returns(
132
+ aws_access_key_id: 'dummykey',
133
+ aws_secret_access_key: 'dummysecret',
134
+ aws_session_token: 'dummytoken',
135
+ region: 'us-west-1',
136
+ sts_endpoint: "https://sts.us-west-1.amazonaws.com",
98
137
  aws_credentials_expire_at: expires_at
99
138
  ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
100
139
  end
101
140
 
141
+ ENV["AWS_STS_REGIONAL_ENDPOINTS"] = nil
142
+ ENV["AWS_DEFAULT_REGION"] = nil
102
143
  ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
103
144
 
104
145
  compute = Fog::AWS::Compute.new(use_iam_profile: true)
@@ -19,6 +19,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
19
19
  'groups' => [{ 'groupName' => Fog::Nullable::String, 'userId' => String, 'groupId' => String }],
20
20
  'ipProtocol' => String,
21
21
  'ipRanges' => [Fog::Nullable::Hash],
22
+ 'ipv6Ranges' => [Fog::Nullable::Hash],
22
23
  'toPort' => Fog::Nullable::Integer,
23
24
  }],
24
25
  'ipPermissionsEgress' => [],
@@ -54,16 +55,19 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
54
55
  {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
55
56
  "fromPort"=>1,
56
57
  "ipRanges"=>[],
58
+ "ipv6Ranges"=>[],
57
59
  "ipProtocol"=>"tcp",
58
60
  "toPort"=>65535},
59
61
  {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
60
62
  "fromPort"=>1,
61
63
  "ipRanges"=>[],
64
+ "ipv6Ranges"=>[],
62
65
  "ipProtocol"=>"udp",
63
66
  "toPort"=>65535},
64
67
  {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
65
68
  "fromPort"=>-1,
66
69
  "ipRanges"=>[],
70
+ "ipv6Ranges"=>[],
67
71
  "ipProtocol"=>"icmp",
68
72
  "toPort"=>-1}
69
73
  ]
@@ -88,6 +92,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
88
92
  [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
89
93
  {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
90
94
  "ipRanges"=>[],
95
+ "ipv6Ranges"=>[],
91
96
  "ipProtocol"=>"tcp",
92
97
  "fromPort"=>1,
93
98
  "toPort"=>65535},
@@ -95,6 +100,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
95
100
  [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
96
101
  {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
97
102
  "ipRanges"=>[],
103
+ "ipv6Ranges"=>[],
98
104
  "ipProtocol"=>"udp",
99
105
  "fromPort"=>1,
100
106
  "toPort"=>65535},
@@ -102,6 +108,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
102
108
  [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
103
109
  {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
104
110
  "ipRanges"=>[],
111
+ "ipv6Ranges"=>[],
105
112
  "ipProtocol"=>"icmp",
106
113
  "fromPort"=>-1,
107
114
  "toPort"=>-1}
@@ -133,6 +140,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
133
140
  expected_permissions += [
134
141
  {"groups"=>[],
135
142
  "ipRanges"=>[{"cidrIp"=>"10.0.0.0/8"}],
143
+ "ipv6Ranges"=>[],
136
144
  "ipProtocol"=>"tcp",
137
145
  "fromPort"=>22,
138
146
  "toPort"=>22}
@@ -164,7 +172,8 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
164
172
  'IpPermissions' => [
165
173
  {
166
174
  'IpProtocol' => 'tcp', 'FromPort' => '80', 'ToPort' => '80',
167
- 'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }]
175
+ 'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }],
176
+ 'Ipv6Ranges' => []
168
177
  }
169
178
  ]
170
179
  }
@@ -177,6 +186,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
177
186
  expected_permissions += [
178
187
  {"groups"=>[],
179
188
  "ipRanges"=>[{"cidrIp"=>"192.168.0.0/24"}],
189
+ "ipv6Ranges"=>[],
180
190
  "ipProtocol"=>"tcp",
181
191
  "fromPort"=>80,
182
192
  "toPort"=>80}
@@ -204,6 +214,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
204
214
  expected_permissions += [
205
215
  {"groups"=>[{"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
206
216
  "ipRanges"=>[],
217
+ "ipv6Ranges"=>[],
207
218
  "ipProtocol"=>"tcp",
208
219
  "fromPort"=>8000,
209
220
  "toPort"=>8000}
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fog-aws
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.10.0
4
+ version: 3.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Josh Lane
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2021-03-22 00:00:00.000000000 Z
12
+ date: 2021-08-23 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -59,14 +59,14 @@ dependencies:
59
59
  requirements:
60
60
  - - "~>"
61
61
  - !ruby/object:Gem::Version
62
- version: 1.3.0
62
+ version: 2.3.0
63
63
  type: :development
64
64
  prerelease: false
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
67
  - - "~>"
68
68
  - !ruby/object:Gem::Version
69
- version: 1.3.0
69
+ version: 2.3.0
70
70
  - !ruby/object:Gem::Dependency
71
71
  name: shindo
72
72
  requirement: !ruby/object:Gem::Requirement
@@ -147,21 +147,12 @@ executables: []
147
147
  extensions: []
148
148
  extra_rdoc_files: []
149
149
  files:
150
- - ".github/dependabot.yml"
151
- - ".github/workflows/ruby.yml"
152
- - ".github/workflows/stale.yml"
153
- - ".gitignore"
154
150
  - CHANGELOG.md
155
151
  - CONTRIBUTING.md
156
152
  - CONTRIBUTORS.md
157
- - Gemfile
158
153
  - LICENSE.md
159
154
  - README.md
160
- - Rakefile
161
- - bin/console
162
- - bin/setup
163
155
  - fog-aws.gemspec
164
- - gemfiles/Gemfile-edge
165
156
  - lib/fog-aws.rb
166
157
  - lib/fog/aws.rb
167
158
  - lib/fog/aws/auto_scaling.rb
@@ -185,9 +176,7 @@ files:
185
176
  - lib/fog/aws/federation.rb
186
177
  - lib/fog/aws/glacier.rb
187
178
  - lib/fog/aws/iam.rb
188
- - lib/fog/aws/iam/default_policies.json
189
179
  - lib/fog/aws/iam/default_policies.rb
190
- - lib/fog/aws/iam/default_policy_versions.json
191
180
  - lib/fog/aws/iam/paged_collection.rb
192
181
  - lib/fog/aws/kinesis.rb
193
182
  - lib/fog/aws/kms.rb
@@ -1744,7 +1733,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
1744
1733
  - !ruby/object:Gem::Version
1745
1734
  version: '0'
1746
1735
  requirements: []
1747
- rubygems_version: 3.1.2
1736
+ rubygems_version: 3.2.15
1748
1737
  signing_key:
1749
1738
  specification_version: 4
1750
1739
  summary: Module for the 'fog' gem to support Amazon Web Services.
@@ -1,10 +0,0 @@
1
- version: 2
2
- updates:
3
- - package-ecosystem: "bundler"
4
- directory: "/"
5
- schedule:
6
- interval: "daily"
7
- - package-ecosystem: "github-actions"
8
- directory: "/"
9
- schedule:
10
- interval: "daily"
@@ -1,36 +0,0 @@
1
- # This workflow uses actions that are not certified by GitHub.
2
- # They are provided by a third-party and are governed by
3
- # separate terms of service, privacy policy, and support
4
- # documentation.
5
- # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
6
- # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
7
-
8
- name: Ruby
9
-
10
- on:
11
- push:
12
- branches: [ master ]
13
- pull_request:
14
- branches: [ master ]
15
-
16
- jobs:
17
- test:
18
-
19
- env:
20
- BUNDLER_GEMFILE: gemfiles/Gemfile-edge
21
- runs-on: ubuntu-latest
22
- strategy:
23
- matrix:
24
- ruby-version: ['2.3', '2.4', '2.5', '2.6', '2.7', '3.0', 'head']
25
-
26
- steps:
27
- - uses: actions/checkout@v2
28
- - name: Set up Ruby
29
- uses: ruby/setup-ruby@v1
30
- with:
31
- ruby-version: ${{ matrix.ruby-version }}
32
- bundler-cache: true # runs 'bundle install' and caches installed gems automatically
33
- - name: Install dependencies
34
- run: bundle install
35
- - name: Run tests
36
- run: bundle exec rake
@@ -1,23 +0,0 @@
1
- name: Mark stale issues and pull requests
2
-
3
- on:
4
- schedule:
5
- - cron: "30 1 * * *"
6
-
7
- jobs:
8
- stale:
9
-
10
- runs-on: ubuntu-latest
11
-
12
- steps:
13
- - uses: actions/stale@v3
14
- with:
15
- repo-token: ${{ secrets.GITHUB_TOKEN }}
16
- days-before-stale: 60
17
- days-before-close: 7
18
- exempt-issue-labels: 'pinned,security'
19
- exempt-pr-labels: 'pinned,security'
20
- stale-issue-message: 'This issue has been marked inactive and will be closed if no further activity occurs.'
21
- stale-pr-message: 'This pr has been marked inactive and will be closed if no further activity occurs.'
22
- stale-issue-label: 'no-issue-activity'
23
- stale-pr-label: 'no-pr-activity'
data/.gitignore DELETED
@@ -1,17 +0,0 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
10
- *.bundle
11
- *.so
12
- *.o
13
- *.a
14
- mkmf.log
15
- tests/.fog
16
- gemfiles/Gemfile-edge.lock
17
- .idea
data/Gemfile DELETED
@@ -1,14 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- # Specify your gem's dependencies in fog-aws.gemspec
4
- gemspec
5
-
6
- group :test, :default do
7
- gem 'pry-nav'
8
- gem 'mime-types', '~> 3.1'
9
- end
10
-
11
- group :test do
12
- gem "simplecov"
13
- gem "codeclimate-test-reporter", "~> 1.0.0"
14
- end
data/Rakefile DELETED
@@ -1,14 +0,0 @@
1
- require "bundler/gem_tasks"
2
- require "github_changelog_generator/task"
3
-
4
- task :default => :test
5
-
6
- mock = ENV['FOG_MOCK'] || 'true'
7
- task :test do
8
- sh("export FOG_MOCK=#{mock} && bundle exec shindont")
9
- end
10
-
11
- GitHubChangelogGenerator::RakeTask.new :changelog do |config|
12
- config.user = 'fog'
13
- config.project = 'fog-aws'
14
- end
data/bin/console DELETED
@@ -1,14 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "bundler/setup"
4
- require "fog-aws"
5
-
6
- # You can add fixtures and/or initialization code here to make experimenting
7
- # with your gem easier. You can also use a different console, if you like.
8
-
9
- # (If you use this, don't forget to add pry to your Gemfile!)
10
- # require "pry"
11
- # Pry.start
12
-
13
- require "irb"
14
- IRB.start(__FILE__)
data/bin/setup DELETED
@@ -1,8 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
- IFS=$'\n\t'
4
- set -vx
5
-
6
- bundle install
7
-
8
- # Do any other automated setup that you need to do here
@@ -1,14 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- # Shared components
4
- gem "fog-core", :github => "fog/fog-core"
5
- gem "fog-json", :github => "fog/fog-json"
6
-
7
- group :test, :default do
8
- gem 'pry-nav'
9
- gem 'mime-types', '~> 3.1'
10
- end
11
-
12
- gem "codeclimate-test-reporter", group: :test, require: nil
13
-
14
- gemspec :path => "../"