middleman-s3_sync 4.5.0 → 4.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.s3_sync.sample +22 -0
- data/Changelog.md +59 -0
- data/README.md +217 -36
- data/lib/middleman/s3_sync/cloudfront.rb +211 -0
- data/lib/middleman/s3_sync/version.rb +1 -1
- data/lib/middleman/s3_sync.rb +34 -3
- data/lib/middleman-s3_sync/commands.rb +39 -0
- data/lib/middleman-s3_sync/extension.rb +7 -0
- data/middleman-s3_sync.gemspec +1 -0
- data/spec/cloudfront_spec.rb +511 -0
- data/spec/s3_sync_integration_spec.rb +132 -0
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad34e26fb6ea816a0426ef33e543c4eabc3c5323151f1c40512a95183ae6ed7e
|
4
|
+
data.tar.gz: a5b875195ed49d7b311eb87a6672a67aba69773e9d85f50fc1022c185238cf41
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c8f1e1678fd3e45350cdc4efefdd3b2520ad90abd075478dbfcfabb85d8c6294aef1c78acffc7451a9f19cb69b890fd90f6049f35a3000960c87906c7188059f
|
7
|
+
data.tar.gz: 8b062b097077d16019a1aef109338ab3603941dc4a57d06ea0da23374fc5a947ac0f93b0bbedb08eac1f87e3fd1df38fd7db5f3c116d6a04f4c8bf7f79ee1c39
|
data/.s3_sync.sample
CHANGED
@@ -1,3 +1,25 @@
|
|
1
1
|
---
|
2
2
|
aws_access_key_id: <AWS Access Key>
|
3
3
|
aws_secret_access_key: <AWS Secret Access Key>
|
4
|
+
bucket: <S3 Bucket Name>
|
5
|
+
region: us-east-1
|
6
|
+
delete: true
|
7
|
+
after_build: false
|
8
|
+
prefer_gzip: true
|
9
|
+
path_style: true
|
10
|
+
reduced_redundancy_storage: false
|
11
|
+
acl: public-read
|
12
|
+
encryption: false
|
13
|
+
prefix: ''
|
14
|
+
version_bucket: false
|
15
|
+
index_document: index.html
|
16
|
+
error_document: 404.html
|
17
|
+
|
18
|
+
# CloudFront Invalidation Settings
|
19
|
+
cloudfront_distribution_id: <CloudFront Distribution ID> # e.g., E1234567890123
|
20
|
+
cloudfront_invalidate: false # Set to true to enable
|
21
|
+
cloudfront_invalidate_all: false # Set to true to invalidate all paths (/*)
|
22
|
+
cloudfront_invalidation_batch_size: 1000 # Max paths per invalidation request
|
23
|
+
cloudfront_invalidation_max_retries: 5 # Max retries for rate-limited requests
|
24
|
+
cloudfront_invalidation_batch_delay: 2 # Delay in seconds between invalidation batches
|
25
|
+
cloudfront_wait: false # Set to true to wait for invalidation to complete
|
data/Changelog.md
CHANGED
@@ -2,6 +2,65 @@
|
|
2
2
|
|
3
3
|
The gem that tries really hard not to push files to S3.
|
4
4
|
|
5
|
+
## v4.6.1
|
6
|
+
|
7
|
+
* Add CloudFront rate limit handling with exponential backoff retry logic
|
8
|
+
* Add configurable retry settings: `cloudfront_invalidation_max_retries` and `cloudfront_invalidation_batch_delay`
|
9
|
+
* Improve CloudFront error handling for "Rate exceeded" and "Throttling" errors
|
10
|
+
* Add command line options for retry configuration
|
11
|
+
* Update documentation with retry configuration examples
|
12
|
+
* Increase default batch delay from 1 to 2 seconds for better rate limit prevention
|
13
|
+
|
14
|
+
## v4.6.0
|
15
|
+
|
16
|
+
* Add comprehensive CloudFront invalidation support with smart path tracking
|
17
|
+
* Add CloudFront configuration options and command line switches
|
18
|
+
* Add batch processing for CloudFront invalidations to respect API limits
|
19
|
+
* Add path normalization and deduplication for efficient invalidations
|
20
|
+
* Add dry-run support for CloudFront invalidations
|
21
|
+
* Add wait functionality for CI/CD pipeline integration
|
22
|
+
* Update README with CloudFront documentation and best practices
|
23
|
+
|
24
|
+
## v4.5.0
|
25
|
+
|
26
|
+
* Migrate from Fog gem to native AWS SDK S3 client
|
27
|
+
* Remove numerous transitive dependencies by dropping Fog
|
28
|
+
* Fix path handling inconsistencies with leading slashes
|
29
|
+
* Improve resource handling for AWS SDK v3 compatibility
|
30
|
+
* Add Ruby 3.2 compatibility fixes
|
31
|
+
* Fix build behavior to not auto-build unless explicitly requested
|
32
|
+
* Optimize resource processing for better performance
|
33
|
+
* Update nokogiri dependency for security
|
34
|
+
* Enhance test coverage and mocking
|
35
|
+
|
36
|
+
## v4.4.0
|
37
|
+
|
38
|
+
* Add support for newer Ruby versions (3.0+)
|
39
|
+
* Update dependencies for security and compatibility
|
40
|
+
* Fix deprecation warnings with newer Ruby versions
|
41
|
+
* Improve error handling and logging
|
42
|
+
|
43
|
+
## v4.3.0
|
44
|
+
|
45
|
+
* Enhanced S3 client configuration options
|
46
|
+
* Improved AWS credential handling
|
47
|
+
* Better support for custom S3 endpoints
|
48
|
+
* Performance optimizations for large sites
|
49
|
+
|
50
|
+
## v4.2.0
|
51
|
+
|
52
|
+
* Add support for S3 transfer acceleration
|
53
|
+
* Improve concurrent upload handling
|
54
|
+
* Enhanced progress reporting
|
55
|
+
* Better error messages and debugging
|
56
|
+
|
57
|
+
## v4.1.0
|
58
|
+
|
59
|
+
* Add support for custom content types
|
60
|
+
* Improve gzip handling and encoding detection
|
61
|
+
* Enhanced caching policy management
|
62
|
+
* Better support for redirects and metadata
|
63
|
+
|
5
64
|
## v4.0.1
|
6
65
|
|
7
66
|
* Fix order of manipulator chain so that S3 Sync is always the last action
|
data/README.md
CHANGED
@@ -54,6 +54,9 @@ activate :s3_sync do |s3_sync|
|
|
54
54
|
s3_sync.version_bucket = false
|
55
55
|
s3_sync.index_document = 'index.html'
|
56
56
|
s3_sync.error_document = '404.html'
|
57
|
+
s3_sync.cloudfront_distribution_id = 'E1234567890123' # CloudFront distribution ID
|
58
|
+
s3_sync.cloudfront_invalidate = false # Enable CloudFront invalidation
|
59
|
+
s3_sync.cloudfront_invalidate_all = false # Invalidate all paths (/*) or only changed files
|
57
60
|
end
|
58
61
|
```
|
59
62
|
|
@@ -76,44 +79,45 @@ The following defaults apply to the configuration items:
|
|
76
79
|
| encryption | ```false``` |
|
77
80
|
| acl | ```'public-read'``` |
|
78
81
|
| version_bucket | ```false``` |
|
82
|
+
| cloudfront_distribution_id | - |
|
83
|
+
| cloudfront_invalidate | ```false``` |
|
84
|
+
| cloudfront_invalidate_all | ```false``` |
|
85
|
+
| cloudfront_wait | ```false``` |
|
79
86
|
|
80
87
|
## Setting AWS Credentials
|
81
88
|
|
82
|
-
There are several ways to provide
|
83
|
-
S3 bucket. However, you can still use the following methods::We
|
89
|
+
There are several secure ways to provide AWS credentials for s3_sync. Using temporary, least-privilege credentials is strongly recommended.
|
84
90
|
|
85
|
-
####
|
91
|
+
#### Best Practices for AWS Credentials (Recommended)
|
86
92
|
|
87
|
-
|
88
|
-
that is passed to the activate method.
|
93
|
+
##### 1. AWS IAM Roles (Most Secure)
|
89
94
|
|
90
|
-
|
91
|
-
|
95
|
+
###### For CI/CD and Cloud Environments
|
96
|
+
- **EC2 Instance Profiles**: If running on EC2, use IAM roles attached to your instance. Credentials are automatically rotated and managed by AWS.
|
97
|
+
- **ECS Task Roles**: For container workloads, use task roles to provide permissions to specific containers.
|
98
|
+
- **CI/CD Service Roles**: Most CI/CD services (GitHub Actions, CircleCI, etc.) offer native AWS integrations that support assuming IAM roles.
|
92
99
|
|
93
|
-
|
100
|
+
###### For Local Development
|
101
|
+
- **AWS IAM Identity Center (SSO)** and configured profiles in your AWS config file
|
102
|
+
- **AWS CLI credential process** to integrate with external identity providers
|
103
|
+
- **Role assumption** with short-lived credentials through `aws sts assume-role`
|
94
104
|
|
95
|
-
|
96
|
-
The credentials are passed in the YAML format. The keys match the
|
97
|
-
options keys.
|
98
|
-
|
99
|
-
The .s3_sync file takes precedence to the configuration passed in the
|
100
|
-
activate method.
|
101
|
-
|
102
|
-
A sample `.s3_sync` file is included at the root of this repo.
|
103
|
-
|
104
|
-
> Make sure to add .s3_sync to your ignore list if you choose this approach. Not doing so may expose
|
105
|
-
> credentials to the world.
|
105
|
+
To use these methods, you don't need to specify credentials in your Middleman configuration. The AWS SDK will automatically detect and use them.
|
106
106
|
|
107
|
-
|
107
|
+
##### 2. Environment Variables with Temporary Credentials
|
108
108
|
|
109
|
-
|
110
|
-
`--aws_access_key_id` (`-k`) and `--aws_secret_access_key` (`-s`). They should override
|
111
|
-
any other settings if specified.
|
109
|
+
Using environment variables with short-lived credentials from role assumption:
|
112
110
|
|
113
|
-
|
111
|
+
```bash
|
112
|
+
# Obtain temporary credentials via assume-role or similar
|
113
|
+
# Then set these environment variables
|
114
|
+
export AWS_ACCESS_KEY_ID="temporary-access-key"
|
115
|
+
export AWS_SECRET_ACCESS_KEY="temporary-secret-key"
|
116
|
+
export AWS_SESSION_TOKEN="temporary-session-token"
|
117
|
+
export AWS_BUCKET="your-bucket-name"
|
118
|
+
```
|
114
119
|
|
115
|
-
|
116
|
-
map to the following values:
|
120
|
+
These environment variables are used when credentials are not otherwise specified:
|
117
121
|
|
118
122
|
| Setting | Environment Variable |
|
119
123
|
| --------------------- | ---------------------------------- |
|
@@ -122,37 +126,214 @@ map to the following values:
|
|
122
126
|
| aws_session_token | ```ENV['AWS_SESSION_TOKEN']``` |
|
123
127
|
| bucket | ```ENV['AWS_BUCKET']``` |
|
124
128
|
|
125
|
-
|
126
|
-
method or passed through the ```.s3_sync``` configuration file.
|
129
|
+
#### Alternative Methods (Not Recommended for Production)
|
127
130
|
|
128
|
-
|
131
|
+
The following methods are less secure and should be avoided in production environments:
|
129
132
|
|
130
|
-
|
133
|
+
##### Through `.s3_sync` File
|
131
134
|
|
132
|
-
|
135
|
+
You can create a `.s3_sync` at the root of your middleman project.
|
136
|
+
The credentials are passed in the YAML format. The keys match the options keys.
|
137
|
+
|
138
|
+
A sample `.s3_sync` file is included at the root of this repo.
|
139
|
+
|
140
|
+
> **SECURITY WARNING**: If using this approach, ensure you add `.s3_sync` to your `.gitignore` to prevent
|
141
|
+
> accidentally committing credentials to your repository. Consider using this only for local development
|
142
|
+
> and only with temporary credentials.
|
143
|
+
|
144
|
+
##### Through `config.rb`
|
145
|
+
|
146
|
+
You can set the AWS credentials in the activation block, but this is strongly discouraged:
|
147
|
+
|
148
|
+
> **SECURITY WARNING**: This method could lead to credentials being committed to version control,
|
149
|
+
> potentially exposing sensitive information. Never use long-lived credentials with this method.
|
150
|
+
|
151
|
+
##### Through Command Line
|
152
|
+
|
153
|
+
Credentials can be passed via command line options, but this may expose them in shell history:
|
154
|
+
|
155
|
+
> **SECURITY WARNING**: Command line parameters may be visible in process listings or shell history.
|
156
|
+
> Consider using environment variables or IAM roles instead.
|
157
|
+
|
158
|
+
## CloudFront Invalidation
|
159
|
+
|
160
|
+
The gem can automatically invalidate CloudFront cache after a successful sync. This ensures that your CloudFront distribution serves the latest content immediately after deployment.
|
161
|
+
|
162
|
+
### Configuration
|
163
|
+
|
164
|
+
```ruby
|
165
|
+
activate :s3_sync do |s3_sync|
|
166
|
+
# ... other configuration ...
|
167
|
+
s3_sync.cloudfront_distribution_id = 'E1234567890123' # Your CloudFront distribution ID
|
168
|
+
s3_sync.cloudfront_invalidate = true # Enable invalidation
|
169
|
+
s3_sync.cloudfront_invalidate_all = false # Invalidate only changed files
|
170
|
+
end
|
171
|
+
```
|
172
|
+
|
173
|
+
### Configuration Options
|
174
|
+
|
175
|
+
| Setting | Default | Description |
|
176
|
+
| ------------------------------------- | ----------- | ----------- |
|
177
|
+
| cloudfront_distribution_id | - | CloudFront distribution ID to invalidate |
|
178
|
+
| cloudfront_invalidate | ```false``` | Enable CloudFront invalidation after sync |
|
179
|
+
| cloudfront_invalidate_all | ```false``` | Invalidate all paths (/*) instead of only changed files |
|
180
|
+
| cloudfront_invalidation_batch_size | ```1000``` | Maximum paths per invalidation request |
|
181
|
+
| cloudfront_invalidation_max_retries | ```5``` | Maximum retries for rate-limited requests |
|
182
|
+
| cloudfront_invalidation_batch_delay | ```2``` | Delay in seconds between invalidation batches |
|
183
|
+
| cloudfront_wait | ```false``` | Wait for CloudFront invalidation to complete |
|
184
|
+
|
185
|
+
### Command Line Options
|
186
|
+
|
187
|
+
You can also control CloudFront invalidation via command line:
|
188
|
+
|
189
|
+
```bash
|
190
|
+
# Enable CloudFront invalidation for this sync
|
191
|
+
middleman s3_sync --cloudfront-invalidate --cloudfront-distribution-id E1234567890123
|
192
|
+
|
193
|
+
# Invalidate all paths instead of just changed files
|
194
|
+
middleman s3_sync --cloudfront-invalidate-all --cloudfront-distribution-id E1234567890123
|
195
|
+
|
196
|
+
# Wait for invalidation to complete before exiting
|
197
|
+
middleman s3_sync --cloudfront-invalidate --cloudfront-wait --cloudfront-distribution-id E1234567890123
|
198
|
+
|
199
|
+
# Custom batch size for large numbers of files
|
200
|
+
middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-batch-size 500 --cloudfront-distribution-id E1234567890123
|
201
|
+
|
202
|
+
# Adjust retry behavior for rate limiting
|
203
|
+
middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-max-retries 3 --cloudfront-invalidation-batch-delay 5 --cloudfront-distribution-id E1234567890123
|
204
|
+
|
205
|
+
# Short aliases
|
206
|
+
middleman s3_sync -c -d E1234567890123 # Basic invalidation
|
207
|
+
middleman s3_sync -a -d E1234567890123 # Invalidate all paths
|
208
|
+
middleman s3_sync -c -w -d E1234567890123 # Invalidate and wait
|
209
|
+
middleman s3_sync -c -a -w -d E1234567890123 # Invalidate all and wait
|
210
|
+
```
|
211
|
+
|
212
|
+
#### Available CloudFront Command Line Options
|
213
|
+
|
214
|
+
| Option | Short | Description |
|
215
|
+
| ------ | ----- | ----------- |
|
216
|
+
| `--cloudfront-distribution-id` | `-d` | CloudFront distribution ID |
|
217
|
+
| `--cloudfront-invalidate` | `-c` | Enable CloudFront invalidation |
|
218
|
+
| `--cloudfront-invalidate-all` | `-a` | Invalidate all paths (/*) |
|
219
|
+
| `--cloudfront-invalidation-batch-size` | - | Max paths per request (default: 1000) |
|
220
|
+
| `--cloudfront-invalidation-max-retries` | - | Max retries for rate limits (default: 5) |
|
221
|
+
| `--cloudfront-invalidation-batch-delay` | - | Delay between batches in seconds (default: 2) |
|
222
|
+
| `--cloudfront-wait` | `-w` | Wait for invalidation to complete |
|
223
|
+
|
224
|
+
### How It Works
|
225
|
+
|
226
|
+
1. **Smart Invalidation**: By default, only files that were created, updated, or deleted during the sync are invalidated
|
227
|
+
2. **Path Optimization**: Duplicate paths are removed and redundant paths (covered by wildcards) are eliminated
|
228
|
+
3. **Batch Processing**: Large numbers of paths are split into multiple invalidation requests to respect CloudFront limits
|
229
|
+
4. **Rate Limit Handling**: Automatic retry with exponential backoff when CloudFront rate limits are hit
|
230
|
+
5. **Error Handling**: Invalidation failures are logged but don't stop the sync process
|
231
|
+
6. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
|
133
232
|
|
134
|
-
|
135
|
-
contained in a bucket named "mysite.com":
|
233
|
+
### IAM Permissions
|
136
234
|
|
235
|
+
Your AWS credentials need CloudFront permissions in addition to S3:
|
236
|
+
|
237
|
+
```json
|
238
|
+
{
|
239
|
+
"Version": "2012-10-17",
|
240
|
+
"Statement": [
|
241
|
+
{
|
242
|
+
"Effect": "Allow",
|
243
|
+
"Action": [
|
244
|
+
"cloudfront:CreateInvalidation",
|
245
|
+
"cloudfront:GetInvalidation",
|
246
|
+
"cloudfront:ListInvalidations"
|
247
|
+
],
|
248
|
+
"Resource": "arn:aws:cloudfront::*:distribution/E1234567890123"
|
249
|
+
}
|
250
|
+
]
|
251
|
+
}
|
137
252
|
```
|
253
|
+
|
254
|
+
### Cost Considerations
|
255
|
+
|
256
|
+
- CloudFront allows 1,000 free invalidation paths per month
|
257
|
+
- Additional invalidations cost $0.005 per path
|
258
|
+
- Use `cloudfront_invalidate_all: true` for major updates to minimize costs (counts as 1 path)
|
259
|
+
- Consider the trade-off between immediate cache invalidation and cost
|
260
|
+
|
261
|
+
#### IAM Policy
|
262
|
+
|
263
|
+
Here's a sample IAM policy with least-privilege permissions that will allow syncing to a bucket named "mysite.com":
|
264
|
+
|
265
|
+
```json
|
138
266
|
{
|
139
267
|
"Version": "2012-10-17",
|
140
268
|
"Statement": [
|
141
269
|
{
|
142
270
|
"Effect": "Allow",
|
143
|
-
"Action":
|
271
|
+
"Action": [
|
272
|
+
"s3:ListBucket",
|
273
|
+
"s3:GetBucketLocation",
|
274
|
+
"s3:GetBucketVersioning"
|
275
|
+
],
|
144
276
|
"Resource": "arn:aws:s3:::mysite.com"
|
145
277
|
},
|
146
278
|
{
|
147
279
|
"Effect": "Allow",
|
148
|
-
"Action":
|
280
|
+
"Action": [
|
281
|
+
"s3:PutObject",
|
282
|
+
"s3:PutObjectAcl",
|
283
|
+
"s3:GetObject",
|
284
|
+
"s3:GetObjectAcl",
|
285
|
+
"s3:DeleteObject",
|
286
|
+
"s3:HeadObject"
|
287
|
+
],
|
149
288
|
"Resource": "arn:aws:s3:::mysite.com/*"
|
150
289
|
}
|
151
290
|
]
|
152
291
|
}
|
153
292
|
```
|
154
293
|
|
155
|
-
|
294
|
+
If you're using additional features, you may need these permissions as well:
|
295
|
+
|
296
|
+
```json
|
297
|
+
{
|
298
|
+
"Version": "2012-10-17",
|
299
|
+
"Statement": [
|
300
|
+
{
|
301
|
+
"Effect": "Allow",
|
302
|
+
"Action": [
|
303
|
+
"s3:PutBucketWebsite",
|
304
|
+
"s3:PutBucketVersioning"
|
305
|
+
],
|
306
|
+
"Resource": "arn:aws:s3:::mysite.com",
|
307
|
+
"Condition": {
|
308
|
+
"Bool": {
|
309
|
+
"aws:SecureTransport": "true"
|
310
|
+
}
|
311
|
+
}
|
312
|
+
}
|
313
|
+
]
|
314
|
+
}
|
315
|
+
```
|
316
|
+
|
317
|
+
This policy grants only the specific permissions needed:
|
318
|
+
|
319
|
+
- **For the bucket itself**:
|
320
|
+
- `s3:ListBucket`: To list objects in the bucket
|
321
|
+
- `s3:GetBucketLocation`: To determine the bucket's region
|
322
|
+
- `s3:GetBucketVersioning`: To check versioning status
|
323
|
+
- `s3:PutBucketVersioning`: If using the `version_bucket` option
|
324
|
+
- `s3:PutBucketWebsite`: If using website configuration (index/error documents)
|
325
|
+
|
326
|
+
- **For objects in the bucket**:
|
327
|
+
- `s3:PutObject`: To create/update objects
|
328
|
+
- `s3:PutObjectAcl`: To set ACLs on objects
|
329
|
+
- `s3:GetObject`: To retrieve objects for comparison
|
330
|
+
- `s3:GetObjectAcl`: To read existing ACLs
|
331
|
+
- `s3:DeleteObject`: To delete stray objects (when `delete: true`)
|
332
|
+
- `s3:HeadObject`: To retrieve object metadata via HEAD requests
|
333
|
+
|
334
|
+
The source code shows that middleman-s3_sync uses HEAD requests (`object.head`) to compare resources, checks and sets bucket versioning when configured, and can set website configuration for index and error documents.
|
335
|
+
|
336
|
+
Note: You can further restrict these permissions by adding conditions or limiting them to specific prefixes if you're only publishing to a subdirectory of the bucket.
|
156
337
|
|
157
338
|
## Command Line Usage
|
158
339
|
|
@@ -0,0 +1,211 @@
|
|
1
|
+
require 'aws-sdk-cloudfront'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
module Middleman
|
5
|
+
module S3Sync
|
6
|
+
module CloudFront
|
7
|
+
class << self
|
8
|
+
include Status
|
9
|
+
|
10
|
+
def invalidate(invalidation_paths, options)
|
11
|
+
return unless should_invalidate?(options)
|
12
|
+
return if invalidation_paths.empty? && !options.cloudfront_invalidate_all
|
13
|
+
|
14
|
+
paths = prepare_invalidation_paths(invalidation_paths, options)
|
15
|
+
return if paths.empty?
|
16
|
+
|
17
|
+
say_status "Invalidating CloudFront distribution #{options.cloudfront_distribution_id}"
|
18
|
+
|
19
|
+
if options.dry_run
|
20
|
+
say_status "#{ANSI.yellow{'DRY RUN:'}} Would invalidate #{paths.length} paths in CloudFront"
|
21
|
+
paths.each { |path| say_status " #{path}" } if options.verbose
|
22
|
+
return
|
23
|
+
end
|
24
|
+
|
25
|
+
# Split paths into batches to respect CloudFront limits
|
26
|
+
batch_size = [options.cloudfront_invalidation_batch_size, 3000].min
|
27
|
+
path_batches = paths.each_slice(batch_size).to_a
|
28
|
+
|
29
|
+
invalidation_ids = []
|
30
|
+
|
31
|
+
path_batches.each_with_index do |batch, index|
|
32
|
+
say_status "Creating invalidation batch #{index + 1}/#{path_batches.length} (#{batch.length} paths)"
|
33
|
+
|
34
|
+
invalidation_id = create_invalidation_with_retry(batch, options)
|
35
|
+
invalidation_ids << invalidation_id if invalidation_id
|
36
|
+
|
37
|
+
# Add a delay between batches to avoid rate limiting
|
38
|
+
delay = options.cloudfront_invalidation_batch_delay || 2
|
39
|
+
sleep(delay) if path_batches.length > 1 && index < path_batches.length - 1
|
40
|
+
end
|
41
|
+
|
42
|
+
if invalidation_ids.any?
|
43
|
+
say_status "CloudFront invalidation(s) created: #{invalidation_ids.join(', ')}"
|
44
|
+
|
45
|
+
if options.cloudfront_wait
|
46
|
+
say_status "Waiting for CloudFront invalidation(s) to complete..."
|
47
|
+
wait_for_invalidations(invalidation_ids, options)
|
48
|
+
say_status "CloudFront invalidation(s) completed successfully"
|
49
|
+
else
|
50
|
+
say_status "Invalidations may take 10-15 minutes to complete"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
invalidation_ids
|
55
|
+
rescue Aws::CloudFront::Errors::ServiceError => e
|
56
|
+
say_status "#{ANSI.red{'CloudFront invalidation failed:'}} #{e.message}"
|
57
|
+
raise e unless options.verbose # Re-raise unless we're being verbose
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def should_invalidate?(options)
|
63
|
+
return false unless options.cloudfront_invalidate
|
64
|
+
|
65
|
+
unless options.cloudfront_distribution_id
|
66
|
+
say_status "#{ANSI.yellow{'CloudFront invalidation skipped:'}} no distribution ID provided"
|
67
|
+
return false
|
68
|
+
end
|
69
|
+
|
70
|
+
true
|
71
|
+
end
|
72
|
+
|
73
|
+
def prepare_invalidation_paths(invalidation_paths, options)
|
74
|
+
if options.cloudfront_invalidate_all
|
75
|
+
return ['/*']
|
76
|
+
end
|
77
|
+
|
78
|
+
# Normalize paths for CloudFront
|
79
|
+
paths = invalidation_paths.map do |path|
|
80
|
+
# Ensure path starts with /
|
81
|
+
normalized_path = path.start_with?('/') ? path : "/#{path}"
|
82
|
+
|
83
|
+
# Remove any double slashes
|
84
|
+
normalized_path.gsub(/\/+/, '/')
|
85
|
+
end.uniq.sort
|
86
|
+
|
87
|
+
# Remove any paths that would be covered by a wildcard
|
88
|
+
if paths.include?('/*')
|
89
|
+
paths = ['/*']
|
90
|
+
else
|
91
|
+
# Remove redundant paths (e.g., if we have /path/* and /path/file.html)
|
92
|
+
paths = remove_redundant_paths(paths)
|
93
|
+
end
|
94
|
+
|
95
|
+
say_status "Prepared #{paths.length} paths for CloudFront invalidation" if options.verbose
|
96
|
+
|
97
|
+
paths
|
98
|
+
end
|
99
|
+
|
100
|
+
def remove_redundant_paths(paths)
|
101
|
+
# Sort paths to ensure wildcards come before specific files
|
102
|
+
sorted_paths = paths.sort
|
103
|
+
result = []
|
104
|
+
|
105
|
+
sorted_paths.each do |path|
|
106
|
+
# Check if this path is already covered by a wildcard we've added
|
107
|
+
is_redundant = result.any? do |existing_path|
|
108
|
+
if existing_path.end_with?('/*')
|
109
|
+
# Check if current path is under this wildcard
|
110
|
+
wildcard_prefix = existing_path[0..-3] # Remove /*
|
111
|
+
path.start_with?(wildcard_prefix + '/')
|
112
|
+
else
|
113
|
+
false
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
result << path unless is_redundant
|
118
|
+
end
|
119
|
+
|
120
|
+
result
|
121
|
+
end
|
122
|
+
|
123
|
+
def create_invalidation_with_retry(paths, options)
|
124
|
+
max_retries = options.cloudfront_invalidation_max_retries || 5
|
125
|
+
retries = 0
|
126
|
+
base_delay = 1
|
127
|
+
|
128
|
+
begin
|
129
|
+
create_invalidation(paths, options)
|
130
|
+
rescue Aws::CloudFront::Errors::ServiceError => e
|
131
|
+
if (e.message.include?('Rate exceeded') || e.message.include?('Throttling')) && retries < max_retries
|
132
|
+
retries += 1
|
133
|
+
delay = base_delay * (2 ** (retries - 1)) + rand(1..3) # Exponential backoff with jitter
|
134
|
+
say_status "#{ANSI.yellow{"Rate limit hit, retrying in #{delay} seconds..."}} (attempt #{retries}/#{max_retries})"
|
135
|
+
sleep(delay)
|
136
|
+
retry
|
137
|
+
else
|
138
|
+
say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
|
139
|
+
say_status "Paths: #{paths.join(', ')}" if options.verbose
|
140
|
+
raise e unless options.verbose
|
141
|
+
nil
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def create_invalidation(paths, options)
|
147
|
+
caller_reference = "middleman-s3_sync-#{Time.now.to_i}-#{SecureRandom.hex(4)}"
|
148
|
+
|
149
|
+
response = cloudfront_client(options).create_invalidation({
|
150
|
+
distribution_id: options.cloudfront_distribution_id,
|
151
|
+
invalidation_batch: {
|
152
|
+
paths: {
|
153
|
+
quantity: paths.length,
|
154
|
+
items: paths
|
155
|
+
},
|
156
|
+
caller_reference: caller_reference
|
157
|
+
}
|
158
|
+
})
|
159
|
+
|
160
|
+
response.invalidation.id
|
161
|
+
end
|
162
|
+
|
163
|
+
def cloudfront_client(options)
|
164
|
+
client_options = {
|
165
|
+
region: 'us-east-1' # CloudFront is always in us-east-1
|
166
|
+
}
|
167
|
+
|
168
|
+
# Use the same credentials as S3 if available
|
169
|
+
if options.aws_access_key_id && options.aws_secret_access_key
|
170
|
+
client_options.merge!({
|
171
|
+
access_key_id: options.aws_access_key_id,
|
172
|
+
secret_access_key: options.aws_secret_access_key
|
173
|
+
})
|
174
|
+
|
175
|
+
# If using an assumed role
|
176
|
+
client_options.merge!({
|
177
|
+
session_token: options.aws_session_token
|
178
|
+
}) if options.aws_session_token
|
179
|
+
end
|
180
|
+
|
181
|
+
Aws::CloudFront::Client.new(client_options)
|
182
|
+
end
|
183
|
+
|
184
|
+
def wait_for_invalidations(invalidation_ids, options)
|
185
|
+
invalidation_ids.each do |invalidation_id|
|
186
|
+
say_status "Waiting for invalidation #{invalidation_id}..."
|
187
|
+
|
188
|
+
client = cloudfront_client(options)
|
189
|
+
client.wait_until(:invalidation_completed,
|
190
|
+
distribution_id: options.cloudfront_distribution_id,
|
191
|
+
id: invalidation_id
|
192
|
+
) do |waiter|
|
193
|
+
waiter.max_attempts = 30 # Wait up to 30 minutes (30 * 60s checks)
|
194
|
+
waiter.delay = 60 # Check every 60 seconds
|
195
|
+
|
196
|
+
waiter.before_attempt do |attempt|
|
197
|
+
say_status "Checking invalidation status (attempt #{attempt}/30)..." if options.verbose
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
rescue Aws::Waiters::Errors::WaiterFailed => e
|
202
|
+
say_status "#{ANSI.yellow{'Warning:'}} CloudFront invalidation wait timed out: #{e.message}"
|
203
|
+
say_status "Invalidation is still in progress but sync will continue"
|
204
|
+
rescue Aws::CloudFront::Errors::ServiceError => e
|
205
|
+
say_status "#{ANSI.yellow{'Warning:'}} CloudFront invalidation wait failed: #{e.message}"
|
206
|
+
say_status "Invalidation may still be in progress"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|