middleman-s3_sync 4.6.4 → 4.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6b576cfe91e75975edadd71f0cfd319e84772edecd882aabe2e3c8c06c90082
-  data.tar.gz: 86bc102e074f9b47980e4cbf93060babd7f7b8a5117cbf35b60ea78ba7476ff0
+  metadata.gz: 6b94ebf16b78ae47530a08332b802f50aa9c4510a47b07b60cab9a8b004576e0
+  data.tar.gz: 6d097be5659e364f9c789164da5838bed6781fbb0db3a4724433d2b78d43cbad
 SHA512:
-  metadata.gz: 20b465799e297cff4e8085ba9c9e25637a25a3a77242d862f1d2b93291dd6112c32835fc91eb8fd92d0e4cd11df6676c034330d804e4556b5fc0ba4c2312215e
-  data.tar.gz: 36a4b96fa0c6831047a90ef4140c598bfab3eb5279fffa09adeee5e23673cb30a811806019ea63de16d1c52383b9a644a2ed6cf0bf400ccd1e9eaade616ae00a
+  metadata.gz: 0f29594758ad18658482ee676c9fa2d505dbe133ccd164e9a76816a7803982736f644f62193b61a45aa5bc4204439dad2c0911852dad6b596897caed93fbfd4a
+  data.tar.gz: 5f309d8e5aa6b43e0b6cb76cf7be241b4d198cff4b87d561f33c140c2886cd9337be6f7187b70f38235bed8e469deeb9ce02fb7433c82c8ca08f208cb9a28a02

data/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['3.1', '3.2', '3.3', '3.4']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+
+      - name: Run tests
+        run: bundle exec rspec
+
+      - name: Build gem
+        run: gem build middleman-s3_sync.gemspec

data/.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # For creating GitHub releases
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+
+      - name: Run tests
+        run: bundle exec rspec
+
+      - name: Build gem
+        run: gem build middleman-s3_sync.gemspec
+
+      - name: Get version
+        id: version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+
+      - name: Generate checksums
+        run: |
+          sha256sum middleman-s3_sync-*.gem > checksums.txt
+          cat checksums.txt
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p ~/.gem
+          echo -e "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials
+          chmod 0600 ~/.gem/credentials
+          gem push middleman-s3_sync-*.gem
+        env:
+          RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            middleman-s3_sync-*.gem
+            checksums.txt
+          generate_release_notes: true

data/Changelog.md

@@ -2,6 +2,18 @@
 
 The gem that tries really hard not to push files to S3.
 
+## v4.6.5
+- Performance and stability improvements
+  - Thread-safe invalidation path tracking (use Set + mutex) when running in parallel
+  - Cache CloudFront client (with reset hook for tests)
+  - Single-pass resource categorization (reduce multiple iterations over resources)
+  - Batch S3 deletes via delete_objects (up to 1000 keys/request)
+  - Stream file uploads to reduce memory; compute MD5s in a single read when possible
+  - Optimize CloudFront path deduplication to O(n × path_depth)
+  - CLI/extension: support option writers (e.g., verbose=, dry_run=) to fix NoMethodError
+- Tests: add coverage for CloudFront, batch delete, and streaming uploads
+- No breaking changes; default behavior preserved
+
 ## v4.6.4
 * Remove map gem dependency and replace with native Ruby implementation
 * Add IndifferentHash class to provide string/symbol indifferent access without external dependencies

data/README.md

@@ -19,6 +19,33 @@ that are no longer needed.
 * Use middleman-s3_sync version 4.x for Middleman 4.x
 * Use middleman-s3_sync version 3.x for Middleman 3.x
 
+## What's New in 4.7.0
+
+**New Features**
+- `after_s3_sync` callback for post-sync hooks (notifications, custom actions)
+- `scan_build_dir` option to sync files outside the Middleman sitemap
+- `routing_rules` option for S3 website redirect configuration
+- Improved content type detection with mime-types gem fallback
+
+**Performance & Efficiency**
+- Batch deletes using S3 `delete_objects` (up to 1,000 keys per request)
+- Streaming uploads to reduce memory usage on large files
+- Single-pass MD5 computation avoids redundant file reads
+- Single-pass resource categorization (create/update/delete)
+- Faster redundant-path pruning for CloudFront invalidations
+
+**Reliability**
+- Thread-safe CloudFront invalidation path tracking (mutex-protected Set)
+- Cached CloudFront client to reduce re-instantiation overhead
+- Proper sitemap population before sync (`ensure_resource_list_updated!`)
+- Fixed redirect detection to return boolean values
+
+**Developer Experience**
+- Extension now properly delegates option writers (`verbose=`, `dry_run=`, etc.)
+- GitHub Actions CI and release workflows
+- Tightened gemspec with bounded dependency versions
+- Ruby >= 3.0 requirement
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -258,6 +285,57 @@ Your AWS credentials need CloudFront permissions in addition to S3:
 - Use `cloudfront_invalidate_all: true` for major updates to minimize costs (counts as 1 path)
 - Consider the trade-off between immediate cache invalidation and cost
 
+## Callbacks
+
+### after_s3_sync
+
+You can configure a callback that runs after the sync completes. This is useful for triggering notifications, updating external services, or running post-deployment tasks.
+
+```ruby
+activate :s3_sync do |s3_sync|
+  # ... other configuration ...
+  
+  # Using a lambda/proc
+  s3_sync.after_s3_sync = ->(results) {
+    puts "Created: #{results[:created]} files"
+    puts "Updated: #{results[:updated]} files"
+    puts "Deleted: #{results[:deleted]} files"
+    puts "Invalidation paths: #{results[:invalidation_paths].join(', ')}"
+  }
+end
+```
+
+The callback receives a hash with sync results:
+
+| Key                   | Type    | Description |
+| --------------------- | ------- | ----------- |
+| `:created`            | Integer | Number of files created |
+| `:updated`            | Integer | Number of files updated |
+| `:deleted`            | Integer | Number of files deleted |
+| `:invalidation_paths` | Array   | CloudFront paths that were invalidated |
+
+You can also use a symbol to call a method on the Middleman app:
+
+```ruby
+# In config.rb
+def notify_slack(results)
+  # Send deployment notification to Slack
+end
+
+activate :s3_sync do |s3_sync|
+  # ... other configuration ...
+  s3_sync.after_s3_sync = :notify_slack
+end
+```
+
+Callbacks that take no arguments are also supported:
+
+```ruby
+activate :s3_sync do |s3_sync|
+  s3_sync.after_s3_sync = -> { puts "Sync complete!" }
+end
+```
+
 #### IAM Policy
 
 Here's a sample IAM policy with least-privilege permissions that will allow syncing to a bucket named "mysite.com":
@@ -514,6 +592,16 @@ The full values and their semantics are [documented on AWS's
 documentation
 site](http://docs.aws.amazon.com/AmazonS3/latest/dev/ACLOverview.html#CannedACL).
 
+##### Buckets with ACLs Disabled
+
+If your bucket uses "Object Ownership: Bucket owner enforced" (ACLs disabled), set:
+
+```ruby
+s3_sync.acl = ''   # or: s3_sync.acl = nil
+```
+
+The gem will also auto-detect buckets that reject ACL headers and transparently retry uploads without the `:acl` parameter.
+
 #### Encryption
 
 You can ask Amazon to encrypt your files at rest by setting the

data/WARP.md

@@ -100,6 +100,10 @@ The gem determines what to do with each file by comparing:
 - **`config.rb`**: Middleman configuration with `activate :s3_sync` block
 - **Environment Variables**: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_BUCKET, etc.
 
+## Development Rules
+
+**All changes must be accompanied with unit tests.** This is non-negotiable for maintaining code quality and preventing regressions.
+
 ## Common Development Patterns
 
 When adding new functionality:
@@ -109,4 +113,4 @@ When adding new functionality:
 4. Add comprehensive specs following existing patterns
 5. Update README.md with new configuration options
 
-The codebase emphasizes security (credential handling), efficiency (parallel operations), and reliability (comprehensive error handling and dry-run support).
+The codebase emphasizes security (credential handling), efficiency (parallel operations), and reliability (comprehensive error handling and dry-run support).

data/lib/middleman/s3_sync/cloudfront.rb

@@ -1,5 +1,6 @@
 require 'aws-sdk-cloudfront'
 require 'securerandom'
+require 'set'
 
 module Middleman
   module S3Sync
@@ -101,25 +102,42 @@ module Middleman
           # Sort paths to ensure wildcards come before specific files
           sorted_paths = paths.sort
           result = []
+          # Use a Set for O(1) lookup of wildcard prefixes
+          wildcard_prefixes = Set.new
           
           sorted_paths.each do |path|
-            # Check if this path is already covered by a wildcard we've added
-            is_redundant = result.any? do |existing_path|
-              if existing_path.end_with?('/*')
-                # Check if current path is under this wildcard
-                wildcard_prefix = existing_path[0..-3] # Remove /*
-                path.start_with?(wildcard_prefix + '/')
-              else
-                false
+            # Check if this path is covered by any existing wildcard prefix
+            # by checking all parent directories of this path
+            is_redundant = path_covered_by_wildcard?(path, wildcard_prefixes)
+            
+            unless is_redundant
+              result << path
+              # If this is a wildcard path, add its prefix for future lookups
+              if path.end_with?('/*')
+                wildcard_prefixes.add(path[0..-3]) # Remove /*
               end
             end
-            
-            result << path unless is_redundant
           end
           
           result
         end
 
+        # Check if a path is covered by any wildcard prefix in O(path_depth) time
+        def path_covered_by_wildcard?(path, wildcard_prefixes)
+          return false if wildcard_prefixes.empty?
+          
+          # Check each parent directory of the path
+          segments = path.split('/')
+          current_path = ''
+          
+          segments[0..-2].each do |segment|  # Exclude the last segment
+            current_path = current_path.empty? ? segment : "#{current_path}/#{segment}"
+            return true if wildcard_prefixes.include?(current_path)
+          end
+          
+          false
+        end
+
         def create_invalidation_with_retry(paths, options)
           max_retries = options.cloudfront_invalidation_max_retries || 5
           retries = 0
@@ -161,24 +179,30 @@ module Middleman
         end
 
         def cloudfront_client(options)
-          client_options = {
-            region: 'us-east-1' # CloudFront is always in us-east-1
-          }
-
-          # Use the same credentials as S3 if available
-          if options.aws_access_key_id && options.aws_secret_access_key
-            client_options.merge!({
-              access_key_id: options.aws_access_key_id,
-              secret_access_key: options.aws_secret_access_key
-            })
-
-            # If using an assumed role
-            client_options.merge!({
-              session_token: options.aws_session_token
-            }) if options.aws_session_token
+          @cloudfront_client ||= begin
+            client_options = {
+              region: 'us-east-1' # CloudFront is always in us-east-1
+            }
+
+            # Use the same credentials as S3 if available
+            if options.aws_access_key_id && options.aws_secret_access_key
+              client_options.merge!({
+                access_key_id: options.aws_access_key_id,
+                secret_access_key: options.aws_secret_access_key
+              })
+
+              # If using an assumed role
+              client_options.merge!({
+                session_token: options.aws_session_token
+              }) if options.aws_session_token
+            end
+
+            Aws::CloudFront::Client.new(client_options)
           end
+        end
 
-          Aws::CloudFront::Client.new(client_options)
+        def reset_cloudfront_client!
+          @cloudfront_client = nil
         end
 
         def wait_for_invalidations(invalidation_ids, options)

data/lib/middleman/s3_sync/options.rb

@@ -26,13 +26,16 @@ module Middleman
         :ignore_paths,
         :index_document,
         :error_document,
+        :routing_rules,
+        :scan_build_dir,
         :cloudfront_distribution_id,
         :cloudfront_invalidate,
         :cloudfront_invalidate_all,
         :cloudfront_invalidation_batch_size,
         :cloudfront_invalidation_max_retries,
         :cloudfront_invalidation_batch_delay,
-        :cloudfront_wait
+        :cloudfront_wait,
+        :after_s3_sync
       ]
       attr_accessor *OPTIONS
 
@@ -113,6 +116,14 @@ module Middleman
         @version_bucket.nil? ? false : @version_bucket
       end
 
+      def routing_rules
+        @routing_rules || []
+      end
+
+      def scan_build_dir
+        @scan_build_dir.nil? ? false : @scan_build_dir
+      end
+
     end
   end
 end

data/lib/middleman/s3_sync/resource.rb

@@ -6,9 +6,11 @@ module Middleman
 
       include Status
 
-      def initialize(resource, partial_s3_resource)
+      def initialize(resource, partial_s3_resource, path: nil)
         @resource = resource
-        @path = if resource
+        @path = if path
+                  path.sub(/^\//, '')
+                elsif resource
                   resource.destination_path.sub(/^\//, '')
                 elsif partial_s3_resource&.key
                   partial_s3_resource.key.sub(/^\//, '')
@@ -116,20 +118,25 @@ module Middleman
 
       def upload!
         object = bucket.object(remote_path.sub(/^\//, ''))
-        upload_options = build_upload_options
-
-        begin
-          object.put(upload_options)
-        rescue Aws::S3::Errors::AccessControlListNotSupported => e
-          # Bucket has ACLs disabled - retry without ACL
-          if upload_options.key?(:acl)
-            say_status "#{ANSI.yellow{"Note"}} Bucket does not support ACLs, retrying without ACL parameter"
-            # Automatically disable ACLs for this bucket going forward
-            options.acl = ''
-            upload_options.delete(:acl)
-            retry
-          else
-            raise e
+        
+        # Use streaming upload for memory efficiency with large files
+        File.open(local_path, 'rb') do |file|
+          upload_options = build_upload_options_for_stream(file)
+          
+          begin
+            object.put(upload_options)
+          rescue Aws::S3::Errors::AccessControlListNotSupported => e
+            # Bucket has ACLs disabled - retry without ACL
+            if upload_options.key?(:acl)
+              say_status "#{ANSI.yellow{"Note"}} Bucket does not support ACLs, retrying without ACL parameter"
+              # Automatically disable ACLs for this bucket going forward
+              options.acl = ''
+              upload_options.delete(:acl)
+              file.rewind  # Reset file position for retry
+              retry
+            else
+              raise e
+            end
           end
         end
       end
@@ -219,7 +226,8 @@ module Middleman
       end
 
       def local?
-        File.exist?(local_path) && resource
+        # For orphan files (scan_build_dir), resource is nil but file exists
+        File.exist?(local_path)
       end
 
       def remote?
@@ -227,8 +235,8 @@ module Middleman
       end
 
       def redirect?
-        (resource && resource.respond_to?(:redirect?) && resource.redirect?) || 
-          (full_s3_resource && full_s3_resource.respond_to?(:website_redirect_location) && full_s3_resource.website_redirect_location)
+        !!(resource && resource.respond_to?(:redirect?) && resource.redirect?) || 
+          !!(full_s3_resource && full_s3_resource.respond_to?(:website_redirect_location) && full_s3_resource.website_redirect_location)
       end
 
       def metadata_match?
@@ -278,12 +286,24 @@ module Middleman
       end
 
       def local_object_md5
-        @local_object_md5 ||= Digest::MD5.hexdigest(File.read(local_path))
+        @local_object_md5 ||= begin
+          # When not gzipped, compute both MD5s in single read to avoid redundant I/O
+          if !gzipped && local_path == original_path
+            compute_md5s_single_read
+            @local_object_md5
+          else
+            Digest::MD5.hexdigest(File.read(local_path))
+          end
+        end
       end
 
       def local_content_md5
         @local_content_md5 ||= begin
-          if File.exist?(original_path)
+          # When not gzipped, compute both MD5s in single read to avoid redundant I/O
+          if !gzipped && local_path == original_path
+            compute_md5s_single_read
+            @local_content_md5
+          elsif File.exist?(original_path)
             Digest::MD5.hexdigest(File.read(original_path))
           else
             nil
@@ -291,13 +311,39 @@ module Middleman
         end
       end
 
+      # Compute both MD5s from a single file read when they're the same file
+      def compute_md5s_single_read
+        return if @md5s_computed
+        content = File.read(local_path)
+        md5 = Digest::MD5.hexdigest(content)
+        @local_object_md5 = md5
+        @local_content_md5 = md5
+        @md5s_computed = true
+      end
+
       def original_path
         gzipped ? local_path.gsub(/\.gz$/, '') : local_path
       end
 
       def content_type
-        @content_type ||= Middleman::S3Sync.content_types[local_path]
-        @content_type ||= !resource.nil? && resource.respond_to?(:content_type) ? resource.content_type : nil
+        @content_type ||= begin
+          # Priority: content_types option > mm_resource > mime-types > default
+          ct = options.content_types[local_path] if options.content_types
+          ct ||= options.content_types[path] if options.content_types
+          ct ||= Middleman::S3Sync.content_types[local_path]
+          ct ||= Middleman::S3Sync.content_types[path]
+          ct ||= resource.content_type if resource&.respond_to?(:content_type)
+          ct ||= detect_content_type_from_extension
+          ct || 'application/octet-stream'
+        end
+      end
+
+      def detect_content_type_from_extension
+        return nil unless defined?(MIME::Types)
+        extension = File.extname(original_path).delete_prefix('.')
+        return nil if extension.empty?
+        types = MIME::Types.type_for(extension)
+        types.first&.content_type
       end
 
       def caching_policy
@@ -314,9 +360,10 @@ module Middleman
 
       protected
       
-      def build_upload_options
+      # Build upload options with a file stream as the body
+      def build_upload_options_for_stream(file_stream)
         upload_options = {
-          body: local_content,
+          body: file_stream,
           content_type: content_type
         }
         # Only add ACL if enabled (not for buckets with ACLs disabled)

data/lib/middleman/s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "4.6.4"
+    VERSION = "4.7.0"
   end
 end