middleman-s3_sync 4.6.5 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +29 -0
- data/.github/workflows/release.yml +53 -0
- data/README.md +88 -0
- data/lib/middleman/s3_sync/options.rb +12 -1
- data/lib/middleman/s3_sync/resource.rb +26 -7
- data/lib/middleman/s3_sync/version.rb +1 -1
- data/lib/middleman/s3_sync.rb +116 -0
- data/lib/middleman-s3_sync/commands.rb +3 -1
- data/lib/middleman-s3_sync/extension.rb +3 -0
- data/middleman-s3_sync.gemspec +22 -18
- data/spec/aws_sdk_parameters_spec.rb +54 -0
- data/spec/resource_spec.rb +206 -0
- data/spec/s3_sync_integration_spec.rb +293 -4
- metadata +78 -64
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6b94ebf16b78ae47530a08332b802f50aa9c4510a47b07b60cab9a8b004576e0
|
|
4
|
+
data.tar.gz: 6d097be5659e364f9c789164da5838bed6781fbb0db3a4724433d2b78d43cbad
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0f29594758ad18658482ee676c9fa2d505dbe133ccd164e9a76816a7803982736f644f62193b61a45aa5bc4204439dad2c0911852dad6b596897caed93fbfd4a
|
|
7
|
+
data.tar.gz: 5f309d8e5aa6b43e0b6cb76cf7be241b4d198cff4b87d561f33c140c2886cd9337be6f7187b70f38235bed8e469deeb9ce02fb7433c82c8ca08f208cb9a28a02
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [master]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
ruby-version: ['3.1', '3.2', '3.3', '3.4']
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
20
|
+
uses: ruby/setup-ruby@v1
|
|
21
|
+
with:
|
|
22
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
23
|
+
bundler-cache: true
|
|
24
|
+
|
|
25
|
+
- name: Run tests
|
|
26
|
+
run: bundle exec rspec
|
|
27
|
+
|
|
28
|
+
- name: Build gem
|
|
29
|
+
run: gem build middleman-s3_sync.gemspec
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
release:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
contents: write # For creating GitHub releases
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Set up Ruby
|
|
18
|
+
uses: ruby/setup-ruby@v1
|
|
19
|
+
with:
|
|
20
|
+
ruby-version: '3.4'
|
|
21
|
+
bundler-cache: true
|
|
22
|
+
|
|
23
|
+
- name: Run tests
|
|
24
|
+
run: bundle exec rspec
|
|
25
|
+
|
|
26
|
+
- name: Build gem
|
|
27
|
+
run: gem build middleman-s3_sync.gemspec
|
|
28
|
+
|
|
29
|
+
- name: Get version
|
|
30
|
+
id: version
|
|
31
|
+
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
|
32
|
+
|
|
33
|
+
- name: Generate checksums
|
|
34
|
+
run: |
|
|
35
|
+
sha256sum middleman-s3_sync-*.gem > checksums.txt
|
|
36
|
+
cat checksums.txt
|
|
37
|
+
|
|
38
|
+
- name: Publish to RubyGems
|
|
39
|
+
run: |
|
|
40
|
+
mkdir -p ~/.gem
|
|
41
|
+
echo -e "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials
|
|
42
|
+
chmod 0600 ~/.gem/credentials
|
|
43
|
+
gem push middleman-s3_sync-*.gem
|
|
44
|
+
env:
|
|
45
|
+
RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
|
|
46
|
+
|
|
47
|
+
- name: Create GitHub Release
|
|
48
|
+
uses: softprops/action-gh-release@v2
|
|
49
|
+
with:
|
|
50
|
+
files: |
|
|
51
|
+
middleman-s3_sync-*.gem
|
|
52
|
+
checksums.txt
|
|
53
|
+
generate_release_notes: true
|
data/README.md
CHANGED
|
@@ -19,6 +19,33 @@ that are no longer needed.
|
|
|
19
19
|
* Use middleman-s3_sync version 4.x for Middleman 4.x
|
|
20
20
|
* Use middleman-s3_sync version 3.x for Middleman 3.x
|
|
21
21
|
|
|
22
|
+
## What's New in 4.7.0
|
|
23
|
+
|
|
24
|
+
**New Features**
|
|
25
|
+
- `after_s3_sync` callback for post-sync hooks (notifications, custom actions)
|
|
26
|
+
- `scan_build_dir` option to sync files outside the Middleman sitemap
|
|
27
|
+
- `routing_rules` option for S3 website redirect configuration
|
|
28
|
+
- Improved content type detection with mime-types gem fallback
|
|
29
|
+
|
|
30
|
+
**Performance & Efficiency**
|
|
31
|
+
- Batch deletes using S3 `delete_objects` (up to 1,000 keys per request)
|
|
32
|
+
- Streaming uploads to reduce memory usage on large files
|
|
33
|
+
- Single-pass MD5 computation avoids redundant file reads
|
|
34
|
+
- Single-pass resource categorization (create/update/delete)
|
|
35
|
+
- Faster redundant-path pruning for CloudFront invalidations
|
|
36
|
+
|
|
37
|
+
**Reliability**
|
|
38
|
+
- Thread-safe CloudFront invalidation path tracking (mutex-protected Set)
|
|
39
|
+
- Cached CloudFront client to reduce re-instantiation overhead
|
|
40
|
+
- Proper sitemap population before sync (`ensure_resource_list_updated!`)
|
|
41
|
+
- Fixed redirect detection to return boolean values
|
|
42
|
+
|
|
43
|
+
**Developer Experience**
|
|
44
|
+
- Extension now properly delegates option writers (`verbose=`, `dry_run=`, etc.)
|
|
45
|
+
- GitHub Actions CI and release workflows
|
|
46
|
+
- Tightened gemspec with bounded dependency versions
|
|
47
|
+
- Ruby >= 3.0 requirement
|
|
48
|
+
|
|
22
49
|
## Installation
|
|
23
50
|
|
|
24
51
|
Add this line to your application's Gemfile:
|
|
@@ -258,6 +285,57 @@ Your AWS credentials need CloudFront permissions in addition to S3:
|
|
|
258
285
|
- Use `cloudfront_invalidate_all: true` for major updates to minimize costs (counts as 1 path)
|
|
259
286
|
- Consider the trade-off between immediate cache invalidation and cost
|
|
260
287
|
|
|
288
|
+
## Callbacks
|
|
289
|
+
|
|
290
|
+
### after_s3_sync
|
|
291
|
+
|
|
292
|
+
You can configure a callback that runs after the sync completes. This is useful for triggering notifications, updating external services, or running post-deployment tasks.
|
|
293
|
+
|
|
294
|
+
```ruby
|
|
295
|
+
activate :s3_sync do |s3_sync|
|
|
296
|
+
# ... other configuration ...
|
|
297
|
+
|
|
298
|
+
# Using a lambda/proc
|
|
299
|
+
s3_sync.after_s3_sync = ->(results) {
|
|
300
|
+
puts "Created: #{results[:created]} files"
|
|
301
|
+
puts "Updated: #{results[:updated]} files"
|
|
302
|
+
puts "Deleted: #{results[:deleted]} files"
|
|
303
|
+
puts "Invalidation paths: #{results[:invalidation_paths].join(', ')}"
|
|
304
|
+
}
|
|
305
|
+
end
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
The callback receives a hash with sync results:
|
|
309
|
+
|
|
310
|
+
| Key | Type | Description |
|
|
311
|
+
| --------------------- | ------- | ----------- |
|
|
312
|
+
| `:created` | Integer | Number of files created |
|
|
313
|
+
| `:updated` | Integer | Number of files updated |
|
|
314
|
+
| `:deleted` | Integer | Number of files deleted |
|
|
315
|
+
| `:invalidation_paths` | Array | CloudFront paths that were invalidated |
|
|
316
|
+
|
|
317
|
+
You can also use a symbol to call a method on the Middleman app:
|
|
318
|
+
|
|
319
|
+
```ruby
|
|
320
|
+
# In config.rb
|
|
321
|
+
def notify_slack(results)
|
|
322
|
+
# Send deployment notification to Slack
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
activate :s3_sync do |s3_sync|
|
|
326
|
+
# ... other configuration ...
|
|
327
|
+
s3_sync.after_s3_sync = :notify_slack
|
|
328
|
+
end
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
Callbacks that take no arguments are also supported:
|
|
332
|
+
|
|
333
|
+
```ruby
|
|
334
|
+
activate :s3_sync do |s3_sync|
|
|
335
|
+
s3_sync.after_s3_sync = -> { puts "Sync complete!" }
|
|
336
|
+
end
|
|
337
|
+
```
|
|
338
|
+
|
|
261
339
|
#### IAM Policy
|
|
262
340
|
|
|
263
341
|
Here's a sample IAM policy with least-privilege permissions that will allow syncing to a bucket named "mysite.com":
|
|
@@ -514,6 +592,16 @@ The full values and their semantics are [documented on AWS's
|
|
|
514
592
|
documentation
|
|
515
593
|
site](http://docs.aws.amazon.com/AmazonS3/latest/dev/ACLOverview.html#CannedACL).
|
|
516
594
|
|
|
595
|
+
##### Buckets with ACLs Disabled
|
|
596
|
+
|
|
597
|
+
If your bucket uses "Object Ownership: Bucket owner enforced" (ACLs disabled), set:
|
|
598
|
+
|
|
599
|
+
```ruby
|
|
600
|
+
s3_sync.acl = '' # or: s3_sync.acl = nil
|
|
601
|
+
```
|
|
602
|
+
|
|
603
|
+
The gem will also auto-detect buckets that reject ACL headers and transparently retry uploads without the `:acl` parameter.
|
|
604
|
+
|
|
517
605
|
#### Encryption
|
|
518
606
|
|
|
519
607
|
You can ask Amazon to encrypt your files at rest by setting the
|
|
@@ -26,13 +26,16 @@ module Middleman
|
|
|
26
26
|
:ignore_paths,
|
|
27
27
|
:index_document,
|
|
28
28
|
:error_document,
|
|
29
|
+
:routing_rules,
|
|
30
|
+
:scan_build_dir,
|
|
29
31
|
:cloudfront_distribution_id,
|
|
30
32
|
:cloudfront_invalidate,
|
|
31
33
|
:cloudfront_invalidate_all,
|
|
32
34
|
:cloudfront_invalidation_batch_size,
|
|
33
35
|
:cloudfront_invalidation_max_retries,
|
|
34
36
|
:cloudfront_invalidation_batch_delay,
|
|
35
|
-
:cloudfront_wait
|
|
37
|
+
:cloudfront_wait,
|
|
38
|
+
:after_s3_sync
|
|
36
39
|
]
|
|
37
40
|
attr_accessor *OPTIONS
|
|
38
41
|
|
|
@@ -113,6 +116,14 @@ module Middleman
|
|
|
113
116
|
@version_bucket.nil? ? false : @version_bucket
|
|
114
117
|
end
|
|
115
118
|
|
|
119
|
+
def routing_rules
|
|
120
|
+
@routing_rules || []
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def scan_build_dir
|
|
124
|
+
@scan_build_dir.nil? ? false : @scan_build_dir
|
|
125
|
+
end
|
|
126
|
+
|
|
116
127
|
end
|
|
117
128
|
end
|
|
118
129
|
end
|
|
@@ -6,9 +6,11 @@ module Middleman
|
|
|
6
6
|
|
|
7
7
|
include Status
|
|
8
8
|
|
|
9
|
-
def initialize(resource, partial_s3_resource)
|
|
9
|
+
def initialize(resource, partial_s3_resource, path: nil)
|
|
10
10
|
@resource = resource
|
|
11
|
-
@path = if
|
|
11
|
+
@path = if path
|
|
12
|
+
path.sub(/^\//, '')
|
|
13
|
+
elsif resource
|
|
12
14
|
resource.destination_path.sub(/^\//, '')
|
|
13
15
|
elsif partial_s3_resource&.key
|
|
14
16
|
partial_s3_resource.key.sub(/^\//, '')
|
|
@@ -224,7 +226,8 @@ module Middleman
|
|
|
224
226
|
end
|
|
225
227
|
|
|
226
228
|
def local?
|
|
227
|
-
|
|
229
|
+
# For orphan files (scan_build_dir), resource is nil but file exists
|
|
230
|
+
File.exist?(local_path)
|
|
228
231
|
end
|
|
229
232
|
|
|
230
233
|
def remote?
|
|
@@ -232,8 +235,8 @@ module Middleman
|
|
|
232
235
|
end
|
|
233
236
|
|
|
234
237
|
def redirect?
|
|
235
|
-
(resource && resource.respond_to?(:redirect?) && resource.redirect?) ||
|
|
236
|
-
(full_s3_resource && full_s3_resource.respond_to?(:website_redirect_location) && full_s3_resource.website_redirect_location)
|
|
238
|
+
!!(resource && resource.respond_to?(:redirect?) && resource.redirect?) ||
|
|
239
|
+
!!(full_s3_resource && full_s3_resource.respond_to?(:website_redirect_location) && full_s3_resource.website_redirect_location)
|
|
237
240
|
end
|
|
238
241
|
|
|
239
242
|
def metadata_match?
|
|
@@ -323,8 +326,24 @@ module Middleman
|
|
|
323
326
|
end
|
|
324
327
|
|
|
325
328
|
def content_type
|
|
326
|
-
@content_type ||=
|
|
327
|
-
|
|
329
|
+
@content_type ||= begin
|
|
330
|
+
# Priority: content_types option > mm_resource > mime-types > default
|
|
331
|
+
ct = options.content_types[local_path] if options.content_types
|
|
332
|
+
ct ||= options.content_types[path] if options.content_types
|
|
333
|
+
ct ||= Middleman::S3Sync.content_types[local_path]
|
|
334
|
+
ct ||= Middleman::S3Sync.content_types[path]
|
|
335
|
+
ct ||= resource.content_type if resource&.respond_to?(:content_type)
|
|
336
|
+
ct ||= detect_content_type_from_extension
|
|
337
|
+
ct || 'application/octet-stream'
|
|
338
|
+
end
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def detect_content_type_from_extension
|
|
342
|
+
return nil unless defined?(MIME::Types)
|
|
343
|
+
extension = File.extname(original_path).delete_prefix('.')
|
|
344
|
+
return nil if extension.empty?
|
|
345
|
+
types = MIME::Types.type_for(extension)
|
|
346
|
+
types.first&.content_type
|
|
328
347
|
end
|
|
329
348
|
|
|
330
349
|
def caching_policy
|
data/lib/middleman/s3_sync.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'aws-sdk-s3'
|
|
2
2
|
require 'digest/md5'
|
|
3
|
+
require 'mime/types'
|
|
3
4
|
require 'middleman/s3_sync/version'
|
|
4
5
|
require 'middleman/s3_sync/options'
|
|
5
6
|
require 'middleman/s3_sync/caching_policy'
|
|
@@ -37,6 +38,12 @@ module Middleman
|
|
|
37
38
|
@app ||= ::Middleman::Application.new
|
|
38
39
|
@invalidation_paths = Set.new
|
|
39
40
|
|
|
41
|
+
# Ensure sitemap is fully populated before syncing
|
|
42
|
+
# This catches resources from extensions activated in :build mode
|
|
43
|
+
if @app.respond_to?(:sitemap) && @app.sitemap.respond_to?(:ensure_resource_list_updated!)
|
|
44
|
+
@app.sitemap.ensure_resource_list_updated!
|
|
45
|
+
end
|
|
46
|
+
|
|
40
47
|
say_status "Let's see if there's work to be done..."
|
|
41
48
|
unless work_to_be_done?
|
|
42
49
|
say_status "All S3 files are up to date."
|
|
@@ -63,6 +70,9 @@ module Middleman
|
|
|
63
70
|
if s3_sync_options.cloudfront_invalidate
|
|
64
71
|
CloudFront.invalidate(@invalidation_paths.to_a, s3_sync_options)
|
|
65
72
|
end
|
|
73
|
+
|
|
74
|
+
# Run after_s3_sync callback if provided
|
|
75
|
+
run_after_s3_sync_callback
|
|
66
76
|
end
|
|
67
77
|
|
|
68
78
|
def bucket
|
|
@@ -100,6 +110,44 @@ module Middleman
|
|
|
100
110
|
@content_types || {}
|
|
101
111
|
end
|
|
102
112
|
|
|
113
|
+
# Run the after_s3_sync callback if configured
|
|
114
|
+
def run_after_s3_sync_callback
|
|
115
|
+
callback = s3_sync_options.after_s3_sync
|
|
116
|
+
return unless callback
|
|
117
|
+
|
|
118
|
+
say_status 'callback', 'Running after_s3_sync callback...'
|
|
119
|
+
|
|
120
|
+
# Build sync results hash
|
|
121
|
+
results = {
|
|
122
|
+
created: files_to_create.size,
|
|
123
|
+
updated: files_to_update.size,
|
|
124
|
+
deleted: files_to_delete.size,
|
|
125
|
+
invalidation_paths: @invalidation_paths.to_a
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
begin
|
|
129
|
+
if callback.respond_to?(:call)
|
|
130
|
+
# Lambda/Proc callback - check arity to decide whether to pass results
|
|
131
|
+
if callback.arity == 0 || callback.arity == -1 && callback.parameters.empty?
|
|
132
|
+
callback.call
|
|
133
|
+
else
|
|
134
|
+
callback.call(results)
|
|
135
|
+
end
|
|
136
|
+
elsif callback.is_a?(Symbol) && @app.respond_to?(callback)
|
|
137
|
+
# Symbol method name - call on app
|
|
138
|
+
if @app.method(callback).arity == 0
|
|
139
|
+
@app.send(callback)
|
|
140
|
+
else
|
|
141
|
+
@app.send(callback, results)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
say_status 'callback', 'after_s3_sync completed successfully'
|
|
146
|
+
rescue => e
|
|
147
|
+
say_status 'error', "after_s3_sync callback failed: #{e.message}"
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
103
151
|
protected
|
|
104
152
|
def update_bucket_versioning
|
|
105
153
|
s3_client.put_bucket_versioning({
|
|
@@ -115,10 +163,41 @@ module Middleman
|
|
|
115
163
|
opts[:index_document] = { suffix: s3_sync_options.index_document } if s3_sync_options.index_document
|
|
116
164
|
opts[:error_document] = { key: s3_sync_options.error_document } if s3_sync_options.error_document
|
|
117
165
|
|
|
166
|
+
# Add routing rules if specified
|
|
167
|
+
if s3_sync_options.routing_rules && !s3_sync_options.routing_rules.empty?
|
|
168
|
+
opts[:routing_rules] = s3_sync_options.routing_rules.map do |rule|
|
|
169
|
+
routing_rule = {}
|
|
170
|
+
|
|
171
|
+
# Handle condition (optional)
|
|
172
|
+
if rule[:condition] || rule['condition']
|
|
173
|
+
condition = rule[:condition] || rule['condition']
|
|
174
|
+
routing_rule[:condition] = {}
|
|
175
|
+
routing_rule[:condition][:key_prefix_equals] = condition[:key_prefix_equals] || condition['key_prefix_equals'] if condition[:key_prefix_equals] || condition['key_prefix_equals']
|
|
176
|
+
routing_rule[:condition][:http_error_code_returned_equals] = condition[:http_error_code_returned_equals] || condition['http_error_code_returned_equals'] if condition[:http_error_code_returned_equals] || condition['http_error_code_returned_equals']
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Handle redirect (required)
|
|
180
|
+
redirect = rule[:redirect] || rule['redirect']
|
|
181
|
+
routing_rule[:redirect] = {}
|
|
182
|
+
routing_rule[:redirect][:host_name] = redirect[:host_name] || redirect['host_name'] if redirect[:host_name] || redirect['host_name']
|
|
183
|
+
routing_rule[:redirect][:http_redirect_code] = redirect[:http_redirect_code] || redirect['http_redirect_code'] if redirect[:http_redirect_code] || redirect['http_redirect_code']
|
|
184
|
+
routing_rule[:redirect][:protocol] = redirect[:protocol] || redirect['protocol'] if redirect[:protocol] || redirect['protocol']
|
|
185
|
+
routing_rule[:redirect][:replace_key_prefix_with] = redirect[:replace_key_prefix_with] || redirect['replace_key_prefix_with'] if redirect[:replace_key_prefix_with] || redirect['replace_key_prefix_with']
|
|
186
|
+
routing_rule[:redirect][:replace_key_with] = redirect[:replace_key_with] || redirect['replace_key_with'] if redirect[:replace_key_with] || redirect['replace_key_with']
|
|
187
|
+
|
|
188
|
+
routing_rule
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
118
192
|
if opts[:error_document] && !opts[:index_document]
|
|
119
193
|
raise 'S3 requires `index_document` if `error_document` is specified'
|
|
120
194
|
end
|
|
121
195
|
|
|
196
|
+
# S3 requires index_document if routing_rules are specified
|
|
197
|
+
if opts[:routing_rules] && !opts[:index_document]
|
|
198
|
+
raise 'S3 requires `index_document` if `routing_rules` are specified'
|
|
199
|
+
end
|
|
200
|
+
|
|
122
201
|
unless opts.empty?
|
|
123
202
|
say_status "Putting bucket website: #{opts.to_json}"
|
|
124
203
|
s3_client.put_bucket_website({
|
|
@@ -234,6 +313,11 @@ module Middleman
|
|
|
234
313
|
def work_to_be_done?
|
|
235
314
|
Parallel.each(mm_resources, in_threads: THREADS_COUNT, progress: "Processing sitemap") { |mm_resource| add_local_resource(mm_resource) }
|
|
236
315
|
|
|
316
|
+
# Scan build directory for orphan files (not in sitemap)
|
|
317
|
+
if s3_sync_options.scan_build_dir
|
|
318
|
+
discover_orphan_files
|
|
319
|
+
end
|
|
320
|
+
|
|
237
321
|
Parallel.each(remote_only_paths, in_threads: THREADS_COUNT, progress: "Processing remote files") do |remote_path|
|
|
238
322
|
s3_sync_resources[remote_path] ||= S3Sync::Resource.new(nil, remote_resource_for_path(remote_path)).tap(&:status)
|
|
239
323
|
end
|
|
@@ -241,6 +325,38 @@ module Middleman
|
|
|
241
325
|
!(files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?)
|
|
242
326
|
end
|
|
243
327
|
|
|
328
|
+
# Discover files in build directory that are not in the sitemap
|
|
329
|
+
# This handles files generated by after_build callbacks, image optimizers, etc.
|
|
330
|
+
def discover_orphan_files
|
|
331
|
+
return unless build_dir && File.directory?(build_dir)
|
|
332
|
+
|
|
333
|
+
orphan_files = []
|
|
334
|
+
|
|
335
|
+
Dir.glob(File.join(build_dir, '**', '*')).each do |file_path|
|
|
336
|
+
next if File.directory?(file_path)
|
|
337
|
+
|
|
338
|
+
# Get path relative to build_dir
|
|
339
|
+
relative_path = file_path.sub(/^#{Regexp.escape(build_dir)}\/?/, '')
|
|
340
|
+
|
|
341
|
+
# Skip if already in sitemap resources
|
|
342
|
+
next if s3_sync_resources.key?(relative_path)
|
|
343
|
+
|
|
344
|
+
orphan_files << relative_path
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
return if orphan_files.empty?
|
|
348
|
+
|
|
349
|
+
say_status "Found #{orphan_files.size} files outside sitemap"
|
|
350
|
+
|
|
351
|
+
Parallel.each(orphan_files, in_threads: THREADS_COUNT, progress: "Processing orphan files") do |relative_path|
|
|
352
|
+
# Create a Resource with nil mm_resource but explicit path
|
|
353
|
+
# It will use mime-types for content type detection
|
|
354
|
+
remote = remote_resource_for_path(relative_path)
|
|
355
|
+
resource = S3Sync::Resource.new(nil, remote, path: relative_path)
|
|
356
|
+
s3_sync_resources[relative_path] = resource.tap(&:status)
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
|
|
244
360
|
# Single-pass categorization of resources by status
|
|
245
361
|
# Avoids multiple iterations over s3_sync_resources
|
|
246
362
|
def categorized_resources
|
|
@@ -102,7 +102,9 @@ module Middleman
|
|
|
102
102
|
verbose = options[:verbose] ? 0 : 1
|
|
103
103
|
instrument = options[:instrument]
|
|
104
104
|
|
|
105
|
-
|
|
105
|
+
# Always use :build mode to ensure build-time extensions are active
|
|
106
|
+
# (e.g., asset_hash in `configure :build` blocks)
|
|
107
|
+
mode = :build
|
|
106
108
|
|
|
107
109
|
::Middleman::S3Sync.app = ::Middleman::Application.new do
|
|
108
110
|
config[:mode] = mode
|
|
@@ -26,8 +26,10 @@ module Middleman
|
|
|
26
26
|
option :dry_run, false, 'Whether to perform a dry-run'
|
|
27
27
|
option :index_document, nil, 'S3 custom index document path'
|
|
28
28
|
option :error_document, nil, 'S3 custom error document path'
|
|
29
|
+
option :routing_rules, [], 'S3 website routing rules (array of rule hashes)'
|
|
29
30
|
option :content_types, {}, 'Custom content types'
|
|
30
31
|
option :ignore_paths, [], 'Paths that should be ignored during sync, strings or regex are allowed'
|
|
32
|
+
option :scan_build_dir, false, 'Scan build directory for files not in sitemap (e.g., after_build files)'
|
|
31
33
|
option :cloudfront_distribution_id, nil, 'CloudFront distribution ID for invalidation'
|
|
32
34
|
option :cloudfront_invalidate, false, 'Whether to invalidate CloudFront cache after sync'
|
|
33
35
|
option :cloudfront_invalidate_all, false, 'Whether to invalidate all paths (/*) or only changed files'
|
|
@@ -35,6 +37,7 @@ module Middleman
|
|
|
35
37
|
option :cloudfront_invalidation_max_retries, 5, 'Maximum number of retries for rate-limited invalidation requests'
|
|
36
38
|
option :cloudfront_invalidation_batch_delay, 2, 'Delay in seconds between invalidation batches'
|
|
37
39
|
option :cloudfront_wait, false, 'Whether to wait for CloudFront invalidation to complete'
|
|
40
|
+
option :after_s3_sync, nil, 'Proc/lambda to call after sync completes (receives changed_paths array)'
|
|
38
41
|
|
|
39
42
|
expose_to_config :s3_sync_options, :default_caching_policy, :caching_policy
|
|
40
43
|
|
data/middleman-s3_sync.gemspec
CHANGED
|
@@ -13,28 +13,32 @@ Gem::Specification.new do |gem|
|
|
|
13
13
|
gem.homepage = "http://github.com/fredjean/middleman-s3_sync"
|
|
14
14
|
gem.license = 'MIT'
|
|
15
15
|
|
|
16
|
+
gem.required_ruby_version = '>= 3.0'
|
|
17
|
+
|
|
16
18
|
gem.files = `git ls-files`.split($/)
|
|
17
19
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
|
18
20
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
19
21
|
gem.require_paths = ["lib"]
|
|
20
22
|
|
|
21
|
-
|
|
22
|
-
gem.add_runtime_dependency 'middleman-
|
|
23
|
-
gem.add_runtime_dependency '
|
|
24
|
-
gem.add_runtime_dependency 'aws-sdk-
|
|
25
|
-
gem.add_runtime_dependency '
|
|
26
|
-
gem.add_runtime_dependency '
|
|
27
|
-
gem.add_runtime_dependency '
|
|
28
|
-
gem.add_runtime_dependency '
|
|
29
|
-
gem.add_runtime_dependency '
|
|
23
|
+
# Runtime dependencies
|
|
24
|
+
gem.add_runtime_dependency 'middleman-core', '~> 4.4'
|
|
25
|
+
gem.add_runtime_dependency 'middleman-cli', '~> 4.4'
|
|
26
|
+
gem.add_runtime_dependency 'aws-sdk-s3', '~> 1.187', '>= 1.187.0'
|
|
27
|
+
gem.add_runtime_dependency 'aws-sdk-cloudfront', '~> 1.0'
|
|
28
|
+
gem.add_runtime_dependency 'parallel', '~> 1.20'
|
|
29
|
+
gem.add_runtime_dependency 'ruby-progressbar', '~> 1.11'
|
|
30
|
+
gem.add_runtime_dependency 'ansi', '~> 1.5'
|
|
31
|
+
gem.add_runtime_dependency 'mime-types', '~> 3.4'
|
|
32
|
+
gem.add_runtime_dependency 'nokogiri', '~> 1.18', '>= 1.18.4'
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
gem.add_development_dependency '
|
|
33
|
-
gem.add_development_dependency 'pry
|
|
34
|
-
gem.add_development_dependency '
|
|
35
|
-
gem.add_development_dependency 'rspec
|
|
36
|
-
gem.add_development_dependency 'rspec-
|
|
37
|
-
gem.add_development_dependency 'rspec-
|
|
38
|
-
gem.add_development_dependency '
|
|
39
|
-
gem.add_development_dependency '
|
|
34
|
+
# Development dependencies
|
|
35
|
+
gem.add_development_dependency 'rake', '~> 13.0'
|
|
36
|
+
gem.add_development_dependency 'pry', '~> 0.14'
|
|
37
|
+
gem.add_development_dependency 'pry-byebug', '~> 3.10'
|
|
38
|
+
gem.add_development_dependency 'rspec', '~> 3.12'
|
|
39
|
+
gem.add_development_dependency 'rspec-support', '~> 3.12'
|
|
40
|
+
gem.add_development_dependency 'rspec-its', '~> 2.0'
|
|
41
|
+
gem.add_development_dependency 'rspec-mocks', '~> 3.12'
|
|
42
|
+
gem.add_development_dependency 'timerizer', '~> 0.3'
|
|
43
|
+
gem.add_development_dependency 'webrick', '~> 1.8'
|
|
40
44
|
end
|
|
@@ -87,6 +87,60 @@ describe 'AWS SDK Parameter Validation' do
|
|
|
87
87
|
}.to raise_error('S3 requires `index_document` if `error_document` is specified')
|
|
88
88
|
end
|
|
89
89
|
end
|
|
90
|
+
|
|
91
|
+
context 'when routing_rules are set' do
|
|
92
|
+
before do
|
|
93
|
+
options.routing_rules = [
|
|
94
|
+
{
|
|
95
|
+
condition: { key_prefix_equals: 'docs/' },
|
|
96
|
+
redirect: { replace_key_prefix_with: 'documents/' }
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
condition: { http_error_code_returned_equals: '404' },
|
|
100
|
+
redirect: { host_name: 'example.com', replace_key_with: 'error.html' }
|
|
101
|
+
}
|
|
102
|
+
]
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it 'includes routing_rules in website configuration' do
|
|
106
|
+
expect(s3_client).to receive(:put_bucket_website) do |params|
|
|
107
|
+
expect(params[:website_configuration]).to have_key(:routing_rules)
|
|
108
|
+
rules = params[:website_configuration][:routing_rules]
|
|
109
|
+
expect(rules).to be_an(Array)
|
|
110
|
+
expect(rules.length).to eq(2)
|
|
111
|
+
|
|
112
|
+
# First rule
|
|
113
|
+
expect(rules[0][:condition][:key_prefix_equals]).to eq('docs/')
|
|
114
|
+
expect(rules[0][:redirect][:replace_key_prefix_with]).to eq('documents/')
|
|
115
|
+
|
|
116
|
+
# Second rule
|
|
117
|
+
expect(rules[1][:condition][:http_error_code_returned_equals]).to eq('404')
|
|
118
|
+
expect(rules[1][:redirect][:host_name]).to eq('example.com')
|
|
119
|
+
expect(rules[1][:redirect][:replace_key_with]).to eq('error.html')
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
Middleman::S3Sync.send(:update_bucket_website)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
context 'when routing_rules are set without index_document' do
|
|
127
|
+
before do
|
|
128
|
+
options.index_document = nil
|
|
129
|
+
options.error_document = nil
|
|
130
|
+
options.routing_rules = [
|
|
131
|
+
{
|
|
132
|
+
condition: { key_prefix_equals: 'old/' },
|
|
133
|
+
redirect: { replace_key_prefix_with: 'new/' }
|
|
134
|
+
}
|
|
135
|
+
]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
it 'raises an error because S3 requires index_document if routing_rules are specified' do
|
|
139
|
+
expect {
|
|
140
|
+
Middleman::S3Sync.send(:update_bucket_website)
|
|
141
|
+
}.to raise_error('S3 requires `index_document` if `routing_rules` are specified')
|
|
142
|
+
end
|
|
143
|
+
end
|
|
90
144
|
end
|
|
91
145
|
|
|
92
146
|
describe 'put_bucket_versioning parameters' do
|