middleman-s3_sync 4.6.0 → 4.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d5baff2d2863949eb480b7ac94d105d3fdcc2c7df67fab66c649f575dd686a4
-  data.tar.gz: 70c12f42e60eabb0bf063d47a3e42f2766ea883bf47c17557cf323b15a9e546a
+  metadata.gz: 61870e8a5164a26db62b7463da7be2c74272800ab76cd6eab3fe83ec68e58081
+  data.tar.gz: d901b5c020cc4fe8ddc695ceeb2fde34a3555f4645aca7ec04cea7d513a0e37c
 SHA512:
-  metadata.gz: 663ab12a585095543adb3843976e1d24bbf5d3e6bb52a5285b3ce7de5cc7796cb916b72dc330d110225e191325add7e6eed81797abccae9c4e66d0b1d84c8608
-  data.tar.gz: b9b720083616f701318747b661aa04d9257c9539136ae0a2d153cc314eced337baa7ca158cb6e10de1d86ef1bd29cb90cd49b7ee1160c69bdf7b9bab09d4c27f
+  metadata.gz: 40ff859c4ae0b722c2132b3192bd14ab454988b612afb18f442645d7fbfb894b48e7d5ba502a1aae36439ec751c8f2f03e8496df255c736c5c2718770f69a7be
+  data.tar.gz: 0a1fe315c71b79ae80041efaf2beb206cdc4caa1db49a7635ede51c8e53c8bbb1cd95a2b0a28cf2e9eb2dcc4c98101e5a6847c3792fc567d6f2b2d26c4bd92f4

data/.envrc

@@ -0,0 +1,5 @@
+# Adds bin to the path.
+if ! test -d bin || test Gemfile.lock -nt bin; then
+  bundle binstubs --all
+fi
+PATH_add bin

data/.gitignore

@@ -16,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .aider*
+/bin

data/.s3_sync.sample

@@ -20,4 +20,6 @@ cloudfront_distribution_id: <CloudFront Distribution ID>  # e.g., E1234567890123
 cloudfront_invalidate: false                              # Set to true to enable
 cloudfront_invalidate_all: false                          # Set to true to invalidate all paths (/*) 
 cloudfront_invalidation_batch_size: 1000                  # Max paths per invalidation request
+cloudfront_invalidation_max_retries: 5                    # Max retries for rate-limited requests
+cloudfront_invalidation_batch_delay: 2                    # Delay in seconds between invalidation batches
 cloudfront_wait: false                                    # Set to true to wait for invalidation to complete

data/Changelog.md

@@ -2,6 +2,74 @@
 
 The gem that tries really hard not to push files to S3.
 
+## v4.6.2
+
+* Fix AWS SDK parameter format issues from Fog migration
+* Fix website configuration parameters to use symbol keys instead of strings
+* Fix S3 object metadata parameter format to use correct key suffixes
+* Remove obsolete Fog-style constants (CONTENT_MD5_KEY, REDIRECT_KEY)
+* Add comprehensive test suite for AWS SDK parameter validation (18 new tests)
+* Improve compatibility with AWS SDK v3 to prevent API errors
+
+## v4.6.1
+
+* Add CloudFront rate limit handling with exponential backoff retry logic
+* Add configurable retry settings: `cloudfront_invalidation_max_retries` and `cloudfront_invalidation_batch_delay`
+* Improve CloudFront error handling for "Rate exceeded" and "Throttling" errors
+* Add command line options for retry configuration
+* Update documentation with retry configuration examples
+* Increase default batch delay from 1 to 2 seconds for better rate limit prevention
+
+## v4.6.0
+
+* Add comprehensive CloudFront invalidation support with smart path tracking
+* Add CloudFront configuration options and command line switches
+* Add batch processing for CloudFront invalidations to respect API limits
+* Add path normalization and deduplication for efficient invalidations
+* Add dry-run support for CloudFront invalidations
+* Add wait functionality for CI/CD pipeline integration
+* Update README with CloudFront documentation and best practices
+
+## v4.5.0
+
+* Migrate from Fog gem to native AWS SDK S3 client
+* Remove numerous transitive dependencies by dropping Fog
+* Fix path handling inconsistencies with leading slashes
+* Improve resource handling for AWS SDK v3 compatibility
+* Add Ruby 3.2 compatibility fixes
+* Fix build behavior to not auto-build unless explicitly requested
+* Optimize resource processing for better performance
+* Update nokogiri dependency for security
+* Enhance test coverage and mocking
+
+## v4.4.0
+
+* Add support for newer Ruby versions (3.0+)
+* Update dependencies for security and compatibility
+* Fix deprecation warnings with newer Ruby versions
+* Improve error handling and logging
+
+## v4.3.0
+
+* Enhanced S3 client configuration options
+* Improved AWS credential handling
+* Better support for custom S3 endpoints
+* Performance optimizations for large sites
+
+## v4.2.0
+
+* Add support for S3 transfer acceleration
+* Improve concurrent upload handling
+* Enhanced progress reporting
+* Better error messages and debugging
+
+## v4.1.0
+
+* Add support for custom content types
+* Improve gzip handling and encoding detection
+* Enhanced caching policy management
+* Better support for redirects and metadata
+
 ## v4.0.1
 
 * Fix order of manipulator chain so that S3 Sync is always the last action

data/README.md

@@ -172,14 +172,15 @@ end
 
 ### Configuration Options
 
-| Setting                           | Default     | Description |
-| --------------------------------- | ----------- | ----------- |
-| cloudfront_distribution_id        | -           | CloudFront distribution ID to invalidate |
-| cloudfront_invalidate             | ```false``` | Enable CloudFront invalidation after sync |
-| cloudfront_invalidate_all         | ```false``` | Invalidate all paths (/*) instead of only changed files |
-| cloudfront_invalidation_batch_size| ```1000```  | Maximum paths per invalidation request |
-| cloudfront_wait                   | ```false``` | Wait for CloudFront invalidation to complete |
-</edits>
+| Setting                               | Default     | Description |
+| ------------------------------------- | ----------- | ----------- |
+| cloudfront_distribution_id            | -           | CloudFront distribution ID to invalidate |
+| cloudfront_invalidate                 | ```false``` | Enable CloudFront invalidation after sync |
+| cloudfront_invalidate_all             | ```false``` | Invalidate all paths (/*) instead of only changed files |
+| cloudfront_invalidation_batch_size    | ```1000```  | Maximum paths per invalidation request |
+| cloudfront_invalidation_max_retries   | ```5```     | Maximum retries for rate-limited requests |
+| cloudfront_invalidation_batch_delay   | ```2```     | Delay in seconds between invalidation batches |
+| cloudfront_wait                       | ```false``` | Wait for CloudFront invalidation to complete |
 
 ### Command Line Options
 
@@ -198,6 +199,9 @@ middleman s3_sync --cloudfront-invalidate --cloudfront-wait --cloudfront-distrib
 # Custom batch size for large numbers of files
 middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-batch-size 500 --cloudfront-distribution-id E1234567890123
 
+# Adjust retry behavior for rate limiting
+middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-max-retries 3 --cloudfront-invalidation-batch-delay 5 --cloudfront-distribution-id E1234567890123
+
 # Short aliases
 middleman s3_sync -c -d E1234567890123              # Basic invalidation
 middleman s3_sync -a -d E1234567890123              # Invalidate all paths
@@ -213,6 +217,8 @@ middleman s3_sync -c -a -w -d E1234567890123        # Invalidate all and wait
 | `--cloudfront-invalidate` | `-c` | Enable CloudFront invalidation |
 | `--cloudfront-invalidate-all` | `-a` | Invalidate all paths (/*) |
 | `--cloudfront-invalidation-batch-size` | - | Max paths per request (default: 1000) |
+| `--cloudfront-invalidation-max-retries` | - | Max retries for rate limits (default: 5) |
+| `--cloudfront-invalidation-batch-delay` | - | Delay between batches in seconds (default: 2) |
 | `--cloudfront-wait` | `-w` | Wait for invalidation to complete |
 
 ### How It Works
@@ -220,8 +226,9 @@ middleman s3_sync -c -a -w -d E1234567890123        # Invalidate all and wait
 1. **Smart Invalidation**: By default, only files that were created, updated, or deleted during the sync are invalidated
 2. **Path Optimization**: Duplicate paths are removed and redundant paths (covered by wildcards) are eliminated
 3. **Batch Processing**: Large numbers of paths are split into multiple invalidation requests to respect CloudFront limits
-4. **Error Handling**: Invalidation failures are logged but don't stop the sync process
-5. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
+4. **Rate Limit Handling**: Automatic retry with exponential backoff when CloudFront rate limits are hit
+5. **Error Handling**: Invalidation failures are logged but don't stop the sync process
+6. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
 
 ### IAM Permissions
 

data/WARP.md

@@ -0,0 +1,112 @@
+# WARP.md
+
+This file provides guidance to WARP (warp.dev) when working with code in this repository.
+
+## Project Overview
+
+middleman-s3_sync is a Ruby gem that provides intelligent S3 synchronization for Middleman static sites. Unlike other sync tools, it only transfers files that have been added, updated, or deleted, making deployments more efficient. The gem also supports CloudFront cache invalidation and advanced caching policies.
+
+## Key Commands
+
+### Development
+```bash
+# Install dependencies
+bundle install
+
+# Run tests
+bundle exec rspec
+
+# Run specific test file
+bundle exec rspec spec/resource_spec.rb
+
+# Run tests with verbose output
+bundle exec rspec -fd
+
+# Build the gem
+bundle exec rake build
+
+# Install the gem locally for testing
+bundle exec rake install
+```
+
+### Testing & Quality
+```bash
+# Run all specs
+bundle exec rake spec  # or just `rake` (default task)
+
+# Run specific test patterns
+bundle exec rspec spec/*_spec.rb --pattern "*policy*"
+
+# Check gem build without installing
+gem build middleman-s3_sync.gemspec
+```
+
+## Architecture
+
+### Core Components
+
+**Main Sync Engine** (`lib/middleman/s3_sync.rb`)
+- Central orchestration of sync operations
+- Thread-safe S3 operations using mutexes
+- Parallel processing (8 threads by default) for file operations
+- Tracks CloudFront invalidation paths during sync
+
+**Extension Integration** (`lib/middleman-s3_sync/extension.rb`)
+- Middleman extension that hooks into the build process
+- Configurable options with environment variable fallbacks
+- Resource list manipulation to prepare files for sync
+
+**CLI Commands** (`lib/middleman-s3_sync/commands.rb`)
+- Thor-based command-line interface
+- Extensive option parsing for CloudFront, AWS credentials, and sync behavior
+- Support for dry-run mode and build-then-sync workflows
+
+**Resource Management** (`lib/middleman/s3_sync/resource.rb`)
+- Individual file resource handling and status determination
+- MD5-based change detection and caching policy application
+
+**CloudFront Integration** (`lib/middleman/s3_sync/cloudfront.rb`)
+- Intelligent cache invalidation with batch processing
+- Rate limit handling and retry logic
+- Path optimization and wildcard support
+
+### Key Design Patterns
+
+- **Thread Safety**: Uses mutexes for bucket and bucket_files operations
+- **Parallel Processing**: Leverages `parallel` gem for concurrent S3 operations
+- **Status-Based Operations**: Resources maintain state (create, update, delete, ignore)
+- **Configuration Cascade**: CLI options override config.rb options override .s3_sync file options override environment variables
+
+### File Status Logic
+
+The gem determines what to do with each file by comparing:
+- Local file MD5 hashes vs S3 ETags
+- Presence in local build vs S3 bucket
+- Caching policies and content encoding preferences
+
+## Testing Strategy
+
+**RSpec Structure**:
+- `caching_policy_spec.rb`: HTTP caching header generation
+- `cloudfront_spec.rb`: CloudFront invalidation logic (comprehensive, 18k lines)
+- `resource_spec.rb`: Individual file resource operations
+- `s3_sync_integration_spec.rb`: End-to-end sync workflows
+
+**Mock Strategy**: Uses AWS SDK stub responses and custom S3 object mocks for isolated testing without real AWS calls.
+
+## Configuration Files
+
+- **`.s3_sync`**: YAML configuration file for credentials and options (should be gitignored)
+- **`config.rb`**: Middleman configuration with `activate :s3_sync` block
+- **Environment Variables**: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_BUCKET, etc.
+
+## Common Development Patterns
+
+When adding new functionality:
+1. Add option to `extension.rb` with appropriate defaults
+2. Add CLI flag to `commands.rb` if user-facing
+3. Implement core logic in main `s3_sync.rb` module
+4. Add comprehensive specs following existing patterns
+5. Update README.md with new configuration options
+
+The codebase emphasizes security (credential handling), efficiency (parallel operations), and reliability (comprehensive error handling and dry-run support).

data/lib/middleman/s3_sync/cloudfront.rb

@@ -31,11 +31,12 @@ module Middleman
           path_batches.each_with_index do |batch, index|
             say_status "Creating invalidation batch #{index + 1}/#{path_batches.length} (#{batch.length} paths)"
             
-            invalidation_id = create_invalidation(batch, options)
+            invalidation_id = create_invalidation_with_retry(batch, options)
             invalidation_ids << invalidation_id if invalidation_id
             
-            # Add a small delay between batches to avoid rate limiting
-            sleep(1) if path_batches.length > 1 && index < path_batches.length - 1
+            # Add a delay between batches to avoid rate limiting
+            delay = options.cloudfront_invalidation_batch_delay || 2
+            sleep(delay) if path_batches.length > 1 && index < path_batches.length - 1
           end
 
           if invalidation_ids.any?
@@ -119,6 +120,29 @@ module Middleman
           result
         end
 
+        def create_invalidation_with_retry(paths, options)
+          max_retries = options.cloudfront_invalidation_max_retries || 5
+          retries = 0
+          base_delay = 1
+          
+          begin
+            create_invalidation(paths, options)
+          rescue Aws::CloudFront::Errors::ServiceError => e
+            if (e.message.include?('Rate exceeded') || e.message.include?('Throttling')) && retries < max_retries
+              retries += 1
+              delay = base_delay * (2 ** (retries - 1)) + rand(1..3) # Exponential backoff with jitter
+              say_status "#{ANSI.yellow{"Rate limit hit, retrying in #{delay} seconds..."}} (attempt #{retries}/#{max_retries})"
+              sleep(delay)
+              retry
+            else
+              say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
+              say_status "Paths: #{paths.join(', ')}" if options.verbose
+              raise e unless options.verbose
+              nil
+            end
+          end
+        end
+
         def create_invalidation(paths, options)
           caller_reference = "middleman-s3_sync-#{Time.now.to_i}-#{SecureRandom.hex(4)}"
           
@@ -134,11 +158,6 @@ module Middleman
           })
 
           response.invalidation.id
-        rescue Aws::CloudFront::Errors::ServiceError => e
-          say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
-          say_status "Paths: #{paths.join(', ')}" if options.verbose
-          raise e unless options.verbose
-          nil
         end
 
         def cloudfront_client(options)

data/lib/middleman/s3_sync/resource.rb

@@ -3,8 +3,6 @@ module Middleman
     class Resource
       attr_accessor :path, :resource, :partial_s3_resource, :full_s3_resource, :content_type, :gzipped, :options
 
-      CONTENT_MD5_KEY = 'x-amz-meta-content-md5'
-      REDIRECT_KEY = 'x-amz-website-redirect-location'
 
       include Status
 
@@ -59,7 +57,7 @@ module Middleman
           :key => key,
           :acl => options.acl,
           :content_type => content_type,
-          CONTENT_MD5_KEY => local_content_md5
+          'content-md5' => local_content_md5
         }
 
         if caching_policy
@@ -80,7 +78,7 @@ module Middleman
         end
 
         if redirect?
-          attributes[REDIRECT_KEY] = redirect_url
+          attributes['website-redirect-location'] = redirect_url
         end
 
         attributes
@@ -125,7 +123,7 @@ module Middleman
 
         # Add metadata if present
         if local_content_md5
-          upload_options[:metadata] = { CONTENT_MD5_KEY => local_content_md5 }
+          upload_options[:metadata] = { 'content-md5' => local_content_md5 }
         end
 
         # Add redirect if present
@@ -292,7 +290,7 @@ module Middleman
 
       def remote_content_md5
         if full_s3_resource && full_s3_resource.metadata
-          full_s3_resource.metadata[CONTENT_MD5_KEY]
+          full_s3_resource.metadata['content-md5']
         end
       end
 

data/lib/middleman/s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "4.6.0"
+    VERSION = "4.6.2"
   end
 end

data/lib/middleman/s3_sync.rb

@@ -107,10 +107,10 @@ module Middleman
 
       def update_bucket_website
         opts = {}
-        opts[:IndexDocument] = s3_sync_options.index_document if s3_sync_options.index_document
-        opts[:ErrorDocument] = s3_sync_options.error_document if s3_sync_options.error_document
+        opts[:index_document] = { suffix: s3_sync_options.index_document } if s3_sync_options.index_document
+        opts[:error_document] = { key: s3_sync_options.error_document } if s3_sync_options.error_document
 
-        if opts[:ErrorDocument] && !opts[:IndexDocument]
+        if opts[:error_document] && !opts[:index_document]
           raise 'S3 requires `index_document` if `error_document` is specified'
         end
 

data/lib/middleman-s3_sync/commands.rb

@@ -84,6 +84,14 @@ module Middleman
                    type: :numeric,
                    desc: 'Maximum number of paths to invalidate in a single request (default: 1000).'
 
+      class_option :cloudfront_invalidation_max_retries,
+                   type: :numeric,
+                   desc: 'Maximum number of retries for rate-limited invalidation requests (default: 5).'
+
+      class_option :cloudfront_invalidation_batch_delay,
+                   type: :numeric,
+                   desc: 'Delay in seconds between invalidation batches (default: 2).'
+
       class_option :cloudfront_wait,
                    aliases: '-w',
                    type: :boolean,
@@ -127,6 +135,8 @@ module Middleman
         s3_sync_options.cloudfront_invalidate = options[:cloudfront_invalidate] if options[:cloudfront_invalidate]
         s3_sync_options.cloudfront_invalidate_all = options[:cloudfront_invalidate_all] if options[:cloudfront_invalidate_all]
         s3_sync_options.cloudfront_invalidation_batch_size = options[:cloudfront_invalidation_batch_size] if options[:cloudfront_invalidation_batch_size]
+        s3_sync_options.cloudfront_invalidation_max_retries = options[:cloudfront_invalidation_max_retries] if options[:cloudfront_invalidation_max_retries]
+        s3_sync_options.cloudfront_invalidation_batch_delay = options[:cloudfront_invalidation_batch_delay] if options[:cloudfront_invalidation_batch_delay]
         s3_sync_options.cloudfront_wait = options[:cloudfront_wait] if options[:cloudfront_wait]
 
         ::Middleman::S3Sync.sync()

data/lib/middleman-s3_sync/extension.rb

@@ -33,6 +33,8 @@ module Middleman
     option :cloudfront_invalidate, false, 'Whether to invalidate CloudFront cache after sync'
     option :cloudfront_invalidate_all, false, 'Whether to invalidate all paths (/*) or only changed files'
     option :cloudfront_invalidation_batch_size, 1000, 'Maximum number of paths to invalidate in a single request'
+    option :cloudfront_invalidation_max_retries, 5, 'Maximum number of retries for rate-limited invalidation requests'
+    option :cloudfront_invalidation_batch_delay, 2, 'Delay in seconds between invalidation batches'
     option :cloudfront_wait, false, 'Whether to wait for CloudFront invalidation to complete'
 
     expose_to_config :s3_sync_options, :default_caching_policy, :caching_policy

data/spec/aws_sdk_parameters_spec.rb

@@ -0,0 +1,415 @@
+require 'spec_helper'
+
+describe 'AWS SDK Parameter Validation' do
+  let(:options) { Middleman::S3Sync::Options.new }
+  let(:s3_client) { instance_double(Aws::S3::Client) }
+  let(:s3_resource) { instance_double(Aws::S3::Resource) }
+  let(:bucket) { instance_double(Aws::S3::Bucket) }
+  let(:s3_object) { instance_double(Aws::S3::Object) }
+
+  before do
+    Middleman::S3Sync.s3_sync_options = options
+    options.build_dir = "build"
+    options.bucket = "test-bucket"
+    options.acl = "public-read"
+    options.index_document = "index.html"
+    options.error_document = "404.html"
+    options.version_bucket = true
+
+    allow(Aws::S3::Client).to receive(:new).and_return(s3_client)
+    allow(Aws::S3::Resource).to receive(:new).and_return(s3_resource)
+    allow(s3_resource).to receive(:bucket).and_return(bucket)
+    allow(bucket).to receive(:exists?).and_return(true)
+    allow(bucket).to receive(:object).and_return(s3_object)
+    allow(s3_object).to receive(:put).and_return(true)
+
+    # Allow Middleman::S3Sync to use our mocked client/bucket
+    allow(Middleman::S3Sync).to receive(:s3_client).and_return(s3_client)
+    allow(Middleman::S3Sync).to receive(:bucket).and_return(bucket)
+    allow(Middleman::S3Sync).to receive(:say_status)
+  end
+
+  describe 'put_bucket_website parameters' do
+    it 'uses symbol keys for index_document and error_document' do
+      expect(s3_client).to receive(:put_bucket_website) do |params|
+        expect(params[:bucket]).to eq("test-bucket")
+        expect(params[:website_configuration]).to have_key(:index_document)
+        expect(params[:website_configuration]).to have_key(:error_document)
+        
+        # Verify the nested structure uses symbols, not strings
+        expect(params[:website_configuration][:index_document]).to have_key(:suffix)
+        expect(params[:website_configuration][:error_document]).to have_key(:key)
+        
+        # Verify the values are correct
+        expect(params[:website_configuration][:index_document][:suffix]).to eq("index.html")
+        expect(params[:website_configuration][:error_document][:key]).to eq("404.html")
+      end
+
+      Middleman::S3Sync.send(:update_bucket_website)
+    end
+
+    context 'when only index_document is set' do
+      before do
+        options.error_document = nil
+      end
+
+      it 'only includes index_document in website configuration' do
+        expect(s3_client).to receive(:put_bucket_website) do |params|
+          expect(params[:website_configuration]).to have_key(:index_document)
+          expect(params[:website_configuration]).not_to have_key(:error_document)
+        end
+
+        Middleman::S3Sync.send(:update_bucket_website)
+      end
+    end
+
+    context 'when neither document is set' do
+      before do
+        options.index_document = nil
+        options.error_document = nil
+      end
+
+      it 'does not call put_bucket_website' do
+        expect(s3_client).not_to receive(:put_bucket_website)
+
+        Middleman::S3Sync.send(:update_bucket_website)
+      end
+    end
+
+    context 'when only error_document is set' do
+      before do
+        options.index_document = nil
+      end
+
+      it 'raises an error because S3 requires index_document if error_document is specified' do
+        expect {
+          Middleman::S3Sync.send(:update_bucket_website)
+        }.to raise_error('S3 requires `index_document` if `error_document` is specified')
+      end
+    end
+  end
+
+  describe 'put_bucket_versioning parameters' do
+    it 'uses correct parameter structure' do
+      expect(s3_client).to receive(:put_bucket_versioning) do |params|
+        expect(params[:bucket]).to eq("test-bucket")
+        expect(params[:versioning_configuration]).to be_a(Hash)
+        expect(params[:versioning_configuration][:status]).to eq("Enabled")
+      end
+
+      Middleman::S3Sync.send(:update_bucket_versioning)
+    end
+
+    context 'when version_bucket is false' do
+      before do
+        options.version_bucket = false
+      end
+
+      it 'does not call put_bucket_versioning' do
+        expect(s3_client).not_to receive(:put_bucket_versioning)
+
+        Middleman::S3Sync.send(:update_bucket_versioning)
+      end
+    end
+  end
+
+  describe 'S3 object upload parameters' do
+    let(:mm_resource) do
+      double(
+        destination_path: 'test/file.html',
+        content_type: 'text/html'
+      )
+    end
+
+    let(:resource) { Middleman::S3Sync::Resource.new(mm_resource, nil) }
+
+    before do
+      allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+      allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
+      allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      allow(s3_object).to receive(:head).and_return(nil)
+      options.dry_run = false
+    end
+
+    it 'uses correct metadata key format' do
+      expect(s3_object).to receive(:put) do |upload_options|
+        # Verify basic parameters
+        expect(upload_options[:body]).to eq('test content')
+        expect(upload_options[:content_type]).to eq('text/html')
+        expect(upload_options[:acl]).to eq('public-read')
+        
+        # Verify metadata uses correct key format (suffix only, not full header name)
+        expect(upload_options[:metadata]).to be_a(Hash)
+        expect(upload_options[:metadata]).to have_key('content-md5')
+        expect(upload_options[:metadata]).not_to have_key('x-amz-meta-content-md5')
+        
+        # Verify metadata value is the MD5 hash
+        expected_md5 = Digest::MD5.hexdigest('test content')
+        expect(upload_options[:metadata]['content-md5']).to eq(expected_md5)
+      end
+
+      resource.upload!
+    end
+
+    context 'when gzip is enabled' do
+      before do
+        options.prefer_gzip = true
+        allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(true)
+        allow(File).to receive(:read).with('build/test/file.html.gz').and_return('gzipped content')
+        allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+        allow(File).to receive(:read).with('build/test/file.html').and_return('original content')
+        
+        # Mock the HEAD response to avoid calling it during redirect?
+        head_response = double(
+          metadata: {},
+          etag: '"abc123"',
+          content_encoding: nil,
+          cache_control: nil,
+          website_redirect_location: nil
+        )
+        allow(s3_object).to receive(:head).and_return(head_response)
+        resource.instance_variable_set(:@full_s3_resource, head_response)
+      end
+
+      it 'includes content_encoding parameter' do
+        expect(s3_object).to receive(:put) do |upload_options|
+          expect(upload_options[:content_encoding]).to eq('gzip')
+        end
+
+        resource.upload!
+      end
+    end
+
+    context 'when reduced redundancy storage is enabled' do
+      before do
+        options.reduced_redundancy_storage = true
+      end
+
+      it 'includes storage_class parameter' do
+        expect(s3_object).to receive(:put) do |upload_options|
+          expect(upload_options[:storage_class]).to eq('REDUCED_REDUNDANCY')
+        end
+
+        resource.upload!
+      end
+    end
+
+    context 'when encryption is enabled' do
+      before do
+        options.encryption = true
+      end
+
+      it 'includes server_side_encryption parameter' do
+        expect(s3_object).to receive(:put) do |upload_options|
+          expect(upload_options[:server_side_encryption]).to eq('AES256')
+        end
+
+        resource.upload!
+      end
+    end
+
+    context 'when resource has a redirect' do
+      let(:mm_resource) do
+        double(
+          destination_path: 'redirect/file.html',
+          content_type: 'text/html',
+          redirect?: true,
+          target_url: 'https://example.com/new-location'
+        )
+      end
+
+      before do
+        allow(File).to receive(:exist?).with('build/redirect/file.html').and_return(true)
+        allow(File).to receive(:exist?).with('build/redirect/file.html.gz').and_return(false)
+        allow(File).to receive(:read).with('build/redirect/file.html').and_return('redirect content')
+        allow(File).to receive(:directory?).with('build/redirect/file.html').and_return(false)
+        allow(s3_object).to receive(:head).and_return(nil)
+        allow(resource).to receive(:redirect?).and_return(true)
+        allow(resource).to receive(:redirect_url).and_return('https://example.com/new-location')
+      end
+
+      it 'includes website_redirect_location parameter' do
+        expect(s3_object).to receive(:put) do |upload_options|
+          expect(upload_options[:website_redirect_location]).to eq('https://example.com/new-location')
+        end
+
+        resource.upload!
+      end
+    end
+  end
+
+  describe 'S3 object metadata retrieval' do
+    let(:mm_resource) do
+      double(
+        destination_path: 'test/file.html',
+        content_type: 'text/html'
+      )
+    end
+
+    let(:resource) { Middleman::S3Sync::Resource.new(mm_resource, nil) }
+    let(:head_response) do
+      double(
+        metadata: { 'content-md5' => 'abc123def456' },
+        etag: '"def456abc123"',
+        content_encoding: nil,
+        cache_control: nil,
+        website_redirect_location: nil
+      )
+    end
+
+    before do
+      allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+      allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      allow(s3_object).to receive(:head).and_return(head_response)
+      resource.instance_variable_set(:@full_s3_resource, head_response)
+    end
+
+    it 'reads metadata using correct key format' do
+      expect(resource.remote_content_md5).to eq('abc123def456')
+    end
+
+    it 'does not try to read metadata with old header format' do
+      # Ensure it's not looking for the full header name
+      expect(head_response.metadata).not_to receive(:[]).with('x-amz-meta-content-md5')
+      
+      resource.remote_content_md5
+    end
+  end
+
+  describe 'to_h method for legacy compatibility' do
+    let(:mm_resource) do
+      double(
+        destination_path: 'test/file.html',
+        content_type: 'text/html'
+      )
+    end
+
+    let(:resource) { Middleman::S3Sync::Resource.new(mm_resource, nil) }
+
+    before do
+      allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+      allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
+      allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      allow(s3_object).to receive(:head).and_return(nil)
+    end
+
+    it 'returns attributes with correct key formats' do
+      attributes = resource.to_h
+
+      expect(attributes[:key]).to eq('test/file.html')
+      expect(attributes[:acl]).to eq('public-read')
+      expect(attributes[:content_type]).to eq('text/html')
+      expect(attributes['content-md5']).to be_a(String)
+      expect(attributes).not_to have_key('x-amz-meta-content-md5')
+    end
+
+    context 'when resource has a redirect' do
+      let(:mm_resource) do
+        double(
+          destination_path: 'redirect/file.html',
+          content_type: 'text/html',
+          redirect?: true,
+          target_url: 'https://example.com/new-location'
+        )
+      end
+
+      before do
+        allow(File).to receive(:exist?).with('build/redirect/file.html').and_return(true)
+        allow(File).to receive(:exist?).with('build/redirect/file.html.gz').and_return(false)
+        allow(File).to receive(:read).with('build/redirect/file.html').and_return('redirect content')
+        allow(File).to receive(:directory?).with('build/redirect/file.html').and_return(false)
+        allow(s3_object).to receive(:head).and_return(nil)
+        allow(resource).to receive(:redirect?).and_return(true)
+        allow(resource).to receive(:redirect_url).and_return('https://example.com/new-location')
+      end
+
+      it 'includes redirect with correct key format' do
+        attributes = resource.to_h
+
+        expect(attributes['website-redirect-location']).to eq('https://example.com/new-location')
+        expect(attributes).not_to have_key('x-amz-website-redirect-location')
+      end
+    end
+  end
+
+  describe 'Regression tests for Fog-style parameter issues' do
+    # These tests validate that we've fixed the old Fog-style parameter formatting
+    # and demonstrate what would fail if we reverted to the old style
+
+    it 'does not use string keys for website configuration (old Fog style)' do
+      # This would fail if we reverted to the old format:
+      # opts[:index_document] = { "suffix" => s3_sync_options.index_document }
+      
+      expect(s3_client).to receive(:put_bucket_website) do |params|
+        config = params[:website_configuration]
+        
+        # Ensure we're not using string keys (old Fog style)
+        expect(config[:index_document]).not_to have_key("suffix")
+        expect(config[:error_document]).not_to have_key("key")
+        
+        # Ensure we ARE using symbol keys (correct AWS SDK style)
+        expect(config[:index_document]).to have_key(:suffix)
+        expect(config[:error_document]).to have_key(:key)
+      end
+
+      Middleman::S3Sync.send(:update_bucket_website)
+    end
+
+    it 'does not use full header names in metadata (old style)' do
+      mm_resource = double(
+        destination_path: 'test/file.html',
+        content_type: 'text/html'
+      )
+      resource = Middleman::S3Sync::Resource.new(mm_resource, nil)
+      
+      allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+      allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
+      allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      allow(s3_object).to receive(:head).and_return(nil)
+      options.dry_run = false
+
+      expect(s3_object).to receive(:put) do |upload_options|
+        # Ensure we're not using the old full header format
+        expect(upload_options[:metadata]).not_to have_key('x-amz-meta-content-md5')
+        
+        # Ensure we ARE using the correct suffix-only format
+        expect(upload_options[:metadata]).to have_key('content-md5')
+      end
+
+      resource.upload!
+    end
+
+    it 'validates that old constants are no longer used' do
+      # This test ensures the old constants were removed/changed
+      # If they still existed, this would be a sign we didn't clean up properly
+      
+      mm_resource = double(
+        destination_path: 'test/file.html',
+        content_type: 'text/html'
+      )
+      resource = Middleman::S3Sync::Resource.new(mm_resource, nil)
+      
+      allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
+      allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
+      allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      allow(s3_object).to receive(:head).and_return(nil)
+      allow(resource).to receive(:redirect?).and_return(true)
+      allow(resource).to receive(:redirect_url).and_return('https://example.com/redirect')
+
+      attributes = resource.to_h
+      
+      # Validate that the old constant values are not used
+      expect(attributes).not_to have_key('x-amz-meta-content-md5')
+      expect(attributes).not_to have_key('x-amz-website-redirect-location')
+      
+      # Validate that the correct formats are used
+      expect(attributes).to have_key('content-md5')
+      expect(attributes).to have_key('website-redirect-location')
+    end
+  end
+end

data/spec/cloudfront_spec.rb

@@ -8,6 +8,8 @@ describe Middleman::S3Sync::CloudFront do
       cloudfront_distribution_id: 'E1234567890123',
       cloudfront_invalidate_all: false,
       cloudfront_invalidation_batch_size: 1000,
+      cloudfront_invalidation_max_retries: 5,
+      cloudfront_invalidation_batch_delay: 2,
       cloudfront_wait: false,
       aws_access_key_id: 'test_key',
       aws_secret_access_key: 'test_secret',
@@ -92,6 +94,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: true,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -201,6 +205,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 2,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 1,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -239,6 +245,112 @@ describe Middleman::S3Sync::CloudFront do
         }.to raise_error(Aws::CloudFront::Errors::ServiceError)
       end
 
+      context 'when rate limit is exceeded' do
+        let(:rate_error) { Aws::CloudFront::Errors::ServiceError.new(nil, 'Rate exceeded') }
+
+        it 'retries with exponential backoff' do
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          # Fail twice with rate limit, then succeed
+          call_count = 0
+          allow(client).to receive(:create_invalidation) do
+            call_count += 1
+            if call_count <= 2
+              raise rate_error
+            else
+              invalidation_response
+            end
+          end
+          
+          # Allow normal status messages but expect retry messages
+          allow(described_class).to receive(:say_status)
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/5/)
+          ).ordered
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 2\/5/)
+          ).ordered
+          
+          # Expect sleep calls for backoff
+          expect(described_class).to receive(:sleep).twice
+          
+          result = described_class.invalidate(['/path1'], options)
+          expect(result).to eq(['I1234567890123'])
+        end
+
+        it 'gives up after max retries and raises error' do
+          rate_limited_options = double(
+            cloudfront_invalidate: true,
+            cloudfront_distribution_id: 'E1234567890123',
+            cloudfront_invalidate_all: false,
+            cloudfront_invalidation_batch_size: 1000,
+            cloudfront_invalidation_max_retries: 2,
+            cloudfront_invalidation_batch_delay: 2,
+            cloudfront_wait: false,
+            aws_access_key_id: 'test_key',
+            aws_secret_access_key: 'test_secret',
+            aws_session_token: nil,
+            dry_run: false,
+            verbose: false
+          )
+          
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          # Fail max_retries + 1 times
+          expect(client).to receive(:create_invalidation).exactly(3).times.and_raise(rate_error)
+          
+          # Allow normal status messages
+          allow(described_class).to receive(:say_status)
+          
+          # Expect retry status messages
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/2/)
+          )
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 2\/2/)
+          )
+          expect(described_class).to receive(:say_status).with(
+            match(/Failed to create CloudFront invalidation.*Rate exceeded/)
+          )
+          
+          # Expect sleep calls for backoff
+          expect(described_class).to receive(:sleep).twice
+          
+          expect {
+            described_class.invalidate(['/path1'], rate_limited_options)
+          }.to raise_error(Aws::CloudFront::Errors::ServiceError)
+        end
+
+        it 'handles throttling errors the same as rate exceeded' do
+          throttling_error = Aws::CloudFront::Errors::ServiceError.new(nil, 'Throttling: Request was throttled')
+          
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          call_count = 0
+          allow(client).to receive(:create_invalidation) do
+            call_count += 1
+            if call_count == 1
+              raise throttling_error
+            else
+              invalidation_response
+            end
+          end
+          
+          # Allow normal status messages but expect retry message
+          allow(described_class).to receive(:say_status)
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/5/)
+          ).ordered
+          expect(described_class).to receive(:sleep).once
+          
+          result = described_class.invalidate(['/path1'], options)
+          expect(result).to eq(['I1234567890123'])
+        end
+      end
+
       context 'when verbose mode is enabled' do
         let(:options) do
           double(
@@ -246,6 +358,8 @@ describe Middleman::S3Sync::CloudFront do
             cloudfront_distribution_id: 'E1234567890123',
             cloudfront_invalidate_all: false,
             cloudfront_invalidation_batch_size: 1000,
+            cloudfront_invalidation_max_retries: 5,
+            cloudfront_invalidation_batch_delay: 2,
             cloudfront_wait: false,
             aws_access_key_id: 'test_key',
             aws_secret_access_key: 'test_secret',
@@ -285,6 +399,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: true,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -336,6 +452,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -367,6 +485,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: nil,
           aws_secret_access_key: nil,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: middleman-s3_sync
 version: !ruby/object:Gem::Version
-  version: 4.6.0
+  version: 4.6.2
 platform: ruby
 authors:
 - Frederic Jean
 - Will Koehler
 bindir: bin
 cert_chain: []
-date: 2025-05-26 00:00:00.000000000 Z
+date: 2025-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: middleman-core
@@ -311,6 +311,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".envrc"
 - ".gitignore"
 - ".rspec"
 - ".s3_sync.sample"
@@ -320,6 +321,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- WARP.md
 - lib/middleman-s3_sync.rb
 - lib/middleman-s3_sync/commands.rb
 - lib/middleman-s3_sync/extension.rb
@@ -333,6 +335,7 @@ files:
 - lib/middleman/s3_sync/version.rb
 - lib/middleman_extension.rb
 - middleman-s3_sync.gemspec
+- spec/aws_sdk_parameters_spec.rb
 - spec/caching_policy_spec.rb
 - spec/cloudfront_spec.rb
 - spec/resource_spec.rb
@@ -360,6 +363,7 @@ rubygems_version: 3.6.2
 specification_version: 4
 summary: Tries really, really hard not to push files to S3.
 test_files:
+- spec/aws_sdk_parameters_spec.rb
 - spec/caching_policy_spec.rb
 - spec/cloudfront_spec.rb
 - spec/resource_spec.rb