middleman-s3_sync 4.6.0 → 4.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d5baff2d2863949eb480b7ac94d105d3fdcc2c7df67fab66c649f575dd686a4
-  data.tar.gz: 70c12f42e60eabb0bf063d47a3e42f2766ea883bf47c17557cf323b15a9e546a
+  metadata.gz: ad34e26fb6ea816a0426ef33e543c4eabc3c5323151f1c40512a95183ae6ed7e
+  data.tar.gz: a5b875195ed49d7b311eb87a6672a67aba69773e9d85f50fc1022c185238cf41
 SHA512:
-  metadata.gz: 663ab12a585095543adb3843976e1d24bbf5d3e6bb52a5285b3ce7de5cc7796cb916b72dc330d110225e191325add7e6eed81797abccae9c4e66d0b1d84c8608
-  data.tar.gz: b9b720083616f701318747b661aa04d9257c9539136ae0a2d153cc314eced337baa7ca158cb6e10de1d86ef1bd29cb90cd49b7ee1160c69bdf7b9bab09d4c27f
+  metadata.gz: c8f1e1678fd3e45350cdc4efefdd3b2520ad90abd075478dbfcfabb85d8c6294aef1c78acffc7451a9f19cb69b890fd90f6049f35a3000960c87906c7188059f
+  data.tar.gz: 8b062b097077d16019a1aef109338ab3603941dc4a57d06ea0da23374fc5a947ac0f93b0bbedb08eac1f87e3fd1df38fd7db5f3c116d6a04f4c8bf7f79ee1c39

data/.s3_sync.sample

@@ -20,4 +20,6 @@ cloudfront_distribution_id: <CloudFront Distribution ID>  # e.g., E1234567890123
 cloudfront_invalidate: false                              # Set to true to enable
 cloudfront_invalidate_all: false                          # Set to true to invalidate all paths (/*) 
 cloudfront_invalidation_batch_size: 1000                  # Max paths per invalidation request
+cloudfront_invalidation_max_retries: 5                    # Max retries for rate-limited requests
+cloudfront_invalidation_batch_delay: 2                    # Delay in seconds between invalidation batches
 cloudfront_wait: false                                    # Set to true to wait for invalidation to complete

data/Changelog.md

@@ -2,6 +2,65 @@
 
 The gem that tries really hard not to push files to S3.
 
+## v4.6.1
+
+* Add CloudFront rate limit handling with exponential backoff retry logic
+* Add configurable retry settings: `cloudfront_invalidation_max_retries` and `cloudfront_invalidation_batch_delay`
+* Improve CloudFront error handling for "Rate exceeded" and "Throttling" errors
+* Add command line options for retry configuration
+* Update documentation with retry configuration examples
+* Increase default batch delay from 1 to 2 seconds for better rate limit prevention
+
+## v4.6.0
+
+* Add comprehensive CloudFront invalidation support with smart path tracking
+* Add CloudFront configuration options and command line switches
+* Add batch processing for CloudFront invalidations to respect API limits
+* Add path normalization and deduplication for efficient invalidations
+* Add dry-run support for CloudFront invalidations
+* Add wait functionality for CI/CD pipeline integration
+* Update README with CloudFront documentation and best practices
+
+## v4.5.0
+
+* Migrate from Fog gem to native AWS SDK S3 client
+* Remove numerous transitive dependencies by dropping Fog
+* Fix path handling inconsistencies with leading slashes
+* Improve resource handling for AWS SDK v3 compatibility
+* Add Ruby 3.2 compatibility fixes
+* Fix build behavior to not auto-build unless explicitly requested
+* Optimize resource processing for better performance
+* Update nokogiri dependency for security
+* Enhance test coverage and mocking
+
+## v4.4.0
+
+* Add support for newer Ruby versions (3.0+)
+* Update dependencies for security and compatibility
+* Fix deprecation warnings with newer Ruby versions
+* Improve error handling and logging
+
+## v4.3.0
+
+* Enhanced S3 client configuration options
+* Improved AWS credential handling
+* Better support for custom S3 endpoints
+* Performance optimizations for large sites
+
+## v4.2.0
+
+* Add support for S3 transfer acceleration
+* Improve concurrent upload handling
+* Enhanced progress reporting
+* Better error messages and debugging
+
+## v4.1.0
+
+* Add support for custom content types
+* Improve gzip handling and encoding detection
+* Enhanced caching policy management
+* Better support for redirects and metadata
+
 ## v4.0.1
 
 * Fix order of manipulator chain so that S3 Sync is always the last action

data/README.md

@@ -172,14 +172,15 @@ end
 
 ### Configuration Options
 
-| Setting                           | Default     | Description |
-| --------------------------------- | ----------- | ----------- |
-| cloudfront_distribution_id        | -           | CloudFront distribution ID to invalidate |
-| cloudfront_invalidate             | ```false``` | Enable CloudFront invalidation after sync |
-| cloudfront_invalidate_all         | ```false``` | Invalidate all paths (/*) instead of only changed files |
-| cloudfront_invalidation_batch_size| ```1000```  | Maximum paths per invalidation request |
-| cloudfront_wait                   | ```false``` | Wait for CloudFront invalidation to complete |
-</edits>
+| Setting                               | Default     | Description |
+| ------------------------------------- | ----------- | ----------- |
+| cloudfront_distribution_id            | -           | CloudFront distribution ID to invalidate |
+| cloudfront_invalidate                 | ```false``` | Enable CloudFront invalidation after sync |
+| cloudfront_invalidate_all             | ```false``` | Invalidate all paths (/*) instead of only changed files |
+| cloudfront_invalidation_batch_size    | ```1000```  | Maximum paths per invalidation request |
+| cloudfront_invalidation_max_retries   | ```5```     | Maximum retries for rate-limited requests |
+| cloudfront_invalidation_batch_delay   | ```2```     | Delay in seconds between invalidation batches |
+| cloudfront_wait                       | ```false``` | Wait for CloudFront invalidation to complete |
 
 ### Command Line Options
 
@@ -198,6 +199,9 @@ middleman s3_sync --cloudfront-invalidate --cloudfront-wait --cloudfront-distrib
 # Custom batch size for large numbers of files
 middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-batch-size 500 --cloudfront-distribution-id E1234567890123
 
+# Adjust retry behavior for rate limiting
+middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-max-retries 3 --cloudfront-invalidation-batch-delay 5 --cloudfront-distribution-id E1234567890123
+
 # Short aliases
 middleman s3_sync -c -d E1234567890123              # Basic invalidation
 middleman s3_sync -a -d E1234567890123              # Invalidate all paths
@@ -213,6 +217,8 @@ middleman s3_sync -c -a -w -d E1234567890123        # Invalidate all and wait
 | `--cloudfront-invalidate` | `-c` | Enable CloudFront invalidation |
 | `--cloudfront-invalidate-all` | `-a` | Invalidate all paths (/*) |
 | `--cloudfront-invalidation-batch-size` | - | Max paths per request (default: 1000) |
+| `--cloudfront-invalidation-max-retries` | - | Max retries for rate limits (default: 5) |
+| `--cloudfront-invalidation-batch-delay` | - | Delay between batches in seconds (default: 2) |
 | `--cloudfront-wait` | `-w` | Wait for invalidation to complete |
 
 ### How It Works
@@ -220,8 +226,9 @@ middleman s3_sync -c -a -w -d E1234567890123        # Invalidate all and wait
 1. **Smart Invalidation**: By default, only files that were created, updated, or deleted during the sync are invalidated
 2. **Path Optimization**: Duplicate paths are removed and redundant paths (covered by wildcards) are eliminated
 3. **Batch Processing**: Large numbers of paths are split into multiple invalidation requests to respect CloudFront limits
-4. **Error Handling**: Invalidation failures are logged but don't stop the sync process
-5. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
+4. **Rate Limit Handling**: Automatic retry with exponential backoff when CloudFront rate limits are hit
+5. **Error Handling**: Invalidation failures are logged but don't stop the sync process
+6. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
 
 ### IAM Permissions
 

data/lib/middleman/s3_sync/cloudfront.rb

@@ -31,11 +31,12 @@ module Middleman
           path_batches.each_with_index do |batch, index|
             say_status "Creating invalidation batch #{index + 1}/#{path_batches.length} (#{batch.length} paths)"
             
-            invalidation_id = create_invalidation(batch, options)
+            invalidation_id = create_invalidation_with_retry(batch, options)
             invalidation_ids << invalidation_id if invalidation_id
             
-            # Add a small delay between batches to avoid rate limiting
-            sleep(1) if path_batches.length > 1 && index < path_batches.length - 1
+            # Add a delay between batches to avoid rate limiting
+            delay = options.cloudfront_invalidation_batch_delay || 2
+            sleep(delay) if path_batches.length > 1 && index < path_batches.length - 1
           end
 
           if invalidation_ids.any?
@@ -119,6 +120,29 @@ module Middleman
           result
         end
 
+        def create_invalidation_with_retry(paths, options)
+          max_retries = options.cloudfront_invalidation_max_retries || 5
+          retries = 0
+          base_delay = 1
+          
+          begin
+            create_invalidation(paths, options)
+          rescue Aws::CloudFront::Errors::ServiceError => e
+            if (e.message.include?('Rate exceeded') || e.message.include?('Throttling')) && retries < max_retries
+              retries += 1
+              delay = base_delay * (2 ** (retries - 1)) + rand(1..3) # Exponential backoff with jitter
+              say_status "#{ANSI.yellow{"Rate limit hit, retrying in #{delay} seconds..."}} (attempt #{retries}/#{max_retries})"
+              sleep(delay)
+              retry
+            else
+              say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
+              say_status "Paths: #{paths.join(', ')}" if options.verbose
+              raise e unless options.verbose
+              nil
+            end
+          end
+        end
+
         def create_invalidation(paths, options)
           caller_reference = "middleman-s3_sync-#{Time.now.to_i}-#{SecureRandom.hex(4)}"
           
@@ -134,11 +158,6 @@ module Middleman
           })
 
           response.invalidation.id
-        rescue Aws::CloudFront::Errors::ServiceError => e
-          say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
-          say_status "Paths: #{paths.join(', ')}" if options.verbose
-          raise e unless options.verbose
-          nil
         end
 
         def cloudfront_client(options)

data/lib/middleman/s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "4.6.0"
+    VERSION = "4.6.1"
   end
 end

data/lib/middleman-s3_sync/commands.rb

@@ -84,6 +84,14 @@ module Middleman
                    type: :numeric,
                    desc: 'Maximum number of paths to invalidate in a single request (default: 1000).'
 
+      class_option :cloudfront_invalidation_max_retries,
+                   type: :numeric,
+                   desc: 'Maximum number of retries for rate-limited invalidation requests (default: 5).'
+
+      class_option :cloudfront_invalidation_batch_delay,
+                   type: :numeric,
+                   desc: 'Delay in seconds between invalidation batches (default: 2).'
+
       class_option :cloudfront_wait,
                    aliases: '-w',
                    type: :boolean,
@@ -127,6 +135,8 @@ module Middleman
         s3_sync_options.cloudfront_invalidate = options[:cloudfront_invalidate] if options[:cloudfront_invalidate]
         s3_sync_options.cloudfront_invalidate_all = options[:cloudfront_invalidate_all] if options[:cloudfront_invalidate_all]
         s3_sync_options.cloudfront_invalidation_batch_size = options[:cloudfront_invalidation_batch_size] if options[:cloudfront_invalidation_batch_size]
+        s3_sync_options.cloudfront_invalidation_max_retries = options[:cloudfront_invalidation_max_retries] if options[:cloudfront_invalidation_max_retries]
+        s3_sync_options.cloudfront_invalidation_batch_delay = options[:cloudfront_invalidation_batch_delay] if options[:cloudfront_invalidation_batch_delay]
         s3_sync_options.cloudfront_wait = options[:cloudfront_wait] if options[:cloudfront_wait]
 
         ::Middleman::S3Sync.sync()

data/lib/middleman-s3_sync/extension.rb

@@ -33,6 +33,8 @@ module Middleman
     option :cloudfront_invalidate, false, 'Whether to invalidate CloudFront cache after sync'
     option :cloudfront_invalidate_all, false, 'Whether to invalidate all paths (/*) or only changed files'
     option :cloudfront_invalidation_batch_size, 1000, 'Maximum number of paths to invalidate in a single request'
+    option :cloudfront_invalidation_max_retries, 5, 'Maximum number of retries for rate-limited invalidation requests'
+    option :cloudfront_invalidation_batch_delay, 2, 'Delay in seconds between invalidation batches'
     option :cloudfront_wait, false, 'Whether to wait for CloudFront invalidation to complete'
 
     expose_to_config :s3_sync_options, :default_caching_policy, :caching_policy

data/spec/cloudfront_spec.rb

@@ -8,6 +8,8 @@ describe Middleman::S3Sync::CloudFront do
       cloudfront_distribution_id: 'E1234567890123',
       cloudfront_invalidate_all: false,
       cloudfront_invalidation_batch_size: 1000,
+      cloudfront_invalidation_max_retries: 5,
+      cloudfront_invalidation_batch_delay: 2,
       cloudfront_wait: false,
       aws_access_key_id: 'test_key',
       aws_secret_access_key: 'test_secret',
@@ -92,6 +94,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: true,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -201,6 +205,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 2,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 1,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -239,6 +245,112 @@ describe Middleman::S3Sync::CloudFront do
         }.to raise_error(Aws::CloudFront::Errors::ServiceError)
       end
 
+      context 'when rate limit is exceeded' do
+        let(:rate_error) { Aws::CloudFront::Errors::ServiceError.new(nil, 'Rate exceeded') }
+
+        it 'retries with exponential backoff' do
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          # Fail twice with rate limit, then succeed
+          call_count = 0
+          allow(client).to receive(:create_invalidation) do
+            call_count += 1
+            if call_count <= 2
+              raise rate_error
+            else
+              invalidation_response
+            end
+          end
+          
+          # Allow normal status messages but expect retry messages
+          allow(described_class).to receive(:say_status)
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/5/)
+          ).ordered
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 2\/5/)
+          ).ordered
+          
+          # Expect sleep calls for backoff
+          expect(described_class).to receive(:sleep).twice
+          
+          result = described_class.invalidate(['/path1'], options)
+          expect(result).to eq(['I1234567890123'])
+        end
+
+        it 'gives up after max retries and raises error' do
+          rate_limited_options = double(
+            cloudfront_invalidate: true,
+            cloudfront_distribution_id: 'E1234567890123',
+            cloudfront_invalidate_all: false,
+            cloudfront_invalidation_batch_size: 1000,
+            cloudfront_invalidation_max_retries: 2,
+            cloudfront_invalidation_batch_delay: 2,
+            cloudfront_wait: false,
+            aws_access_key_id: 'test_key',
+            aws_secret_access_key: 'test_secret',
+            aws_session_token: nil,
+            dry_run: false,
+            verbose: false
+          )
+          
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          # Fail max_retries + 1 times
+          expect(client).to receive(:create_invalidation).exactly(3).times.and_raise(rate_error)
+          
+          # Allow normal status messages
+          allow(described_class).to receive(:say_status)
+          
+          # Expect retry status messages
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/2/)
+          )
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 2\/2/)
+          )
+          expect(described_class).to receive(:say_status).with(
+            match(/Failed to create CloudFront invalidation.*Rate exceeded/)
+          )
+          
+          # Expect sleep calls for backoff
+          expect(described_class).to receive(:sleep).twice
+          
+          expect {
+            described_class.invalidate(['/path1'], rate_limited_options)
+          }.to raise_error(Aws::CloudFront::Errors::ServiceError)
+        end
+
+        it 'handles throttling errors the same as rate exceeded' do
+          throttling_error = Aws::CloudFront::Errors::ServiceError.new(nil, 'Throttling: Request was throttled')
+          
+          client = double('cloudfront_client')
+          allow(Aws::CloudFront::Client).to receive(:new).and_return(client)
+          
+          call_count = 0
+          allow(client).to receive(:create_invalidation) do
+            call_count += 1
+            if call_count == 1
+              raise throttling_error
+            else
+              invalidation_response
+            end
+          end
+          
+          # Allow normal status messages but expect retry message
+          allow(described_class).to receive(:say_status)
+          expect(described_class).to receive(:say_status).with(
+            match(/Rate limit hit, retrying in \d+ seconds.*attempt 1\/5/)
+          ).ordered
+          expect(described_class).to receive(:sleep).once
+          
+          result = described_class.invalidate(['/path1'], options)
+          expect(result).to eq(['I1234567890123'])
+        end
+      end
+
       context 'when verbose mode is enabled' do
         let(:options) do
           double(
@@ -246,6 +358,8 @@ describe Middleman::S3Sync::CloudFront do
             cloudfront_distribution_id: 'E1234567890123',
             cloudfront_invalidate_all: false,
             cloudfront_invalidation_batch_size: 1000,
+            cloudfront_invalidation_max_retries: 5,
+            cloudfront_invalidation_batch_delay: 2,
             cloudfront_wait: false,
             aws_access_key_id: 'test_key',
             aws_secret_access_key: 'test_secret',
@@ -285,6 +399,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: true,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -336,6 +452,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: 'test_key',
           aws_secret_access_key: 'test_secret',
@@ -367,6 +485,8 @@ describe Middleman::S3Sync::CloudFront do
           cloudfront_distribution_id: 'E1234567890123',
           cloudfront_invalidate_all: false,
           cloudfront_invalidation_batch_size: 1000,
+          cloudfront_invalidation_max_retries: 5,
+          cloudfront_invalidation_batch_delay: 2,
           cloudfront_wait: false,
           aws_access_key_id: nil,
           aws_secret_access_key: nil,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: middleman-s3_sync
 version: !ruby/object:Gem::Version
-  version: 4.6.0
+  version: 4.6.1
 platform: ruby
 authors:
 - Frederic Jean