middleman-s3_sync 4.0.3 → 4.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e3827f761cc837c4b40aab6fd29a7737fc7039b1
-  data.tar.gz: 3c42ac240f148cb08d7daa3e72dacb29c11c59c0
+SHA256:
+  metadata.gz: 7d5baff2d2863949eb480b7ac94d105d3fdcc2c7df67fab66c649f575dd686a4
+  data.tar.gz: 70c12f42e60eabb0bf063d47a3e42f2766ea883bf47c17557cf323b15a9e546a
 SHA512:
-  metadata.gz: edbf4fb9046f29261590403220112597f01221350ac223b815856d7a89f833969fbf2a9eb9cb430a9241ffc37603e8baa049baa6ddc64ac1f5b62c6205773453
-  data.tar.gz: 370d1524220e776febbce93868341fc2d381163768dbfb7b698c64e802625f495aefa0ff833735c270362d9c16f0d812523c517ac0ea05c71c7551e3c8b93efe
+  metadata.gz: 663ab12a585095543adb3843976e1d24bbf5d3e6bb52a5285b3ce7de5cc7796cb916b72dc330d110225e191325add7e6eed81797abccae9c4e66d0b1d84c8608
+  data.tar.gz: b9b720083616f701318747b661aa04d9257c9539136ae0a2d153cc314eced337baa7ca158cb6e10de1d86ef1bd29cb90cd49b7ee1160c69bdf7b9bab09d4c27f

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.aider*

data/.s3_sync.sample

@@ -1,3 +1,23 @@
 ---
 aws_access_key_id: <AWS Access Key>
 aws_secret_access_key: <AWS Secret Access Key>
+bucket: <S3 Bucket Name>
+region: us-east-1
+delete: true
+after_build: false
+prefer_gzip: true
+path_style: true
+reduced_redundancy_storage: false
+acl: public-read
+encryption: false
+prefix: ''
+version_bucket: false
+index_document: index.html
+error_document: 404.html
+
+# CloudFront Invalidation Settings
+cloudfront_distribution_id: <CloudFront Distribution ID>  # e.g., E1234567890123
+cloudfront_invalidate: false                              # Set to true to enable
+cloudfront_invalidate_all: false                          # Set to true to invalidate all paths (/*) 
+cloudfront_invalidation_batch_size: 1000                  # Max paths per invalidation request
+cloudfront_wait: false                                    # Set to true to wait for invalidation to complete

data/README.md

@@ -1,6 +1,6 @@
 # Middleman::S3Sync
 
-[![Join the chat at https://gitter.im/fredjean/middleman-s3_sync](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/fredjean/middleman-s3_sync?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Climate](https://codeclimate.com/github/fredjean/middleman-s3_sync.png)](https://codeclimate.com/github/fredjean/middleman-s3_sync) [![Build Status](https://travis-ci.org/fredjean/middleman-s3_sync.png?branch=master)](https://travis-ci.org/fredjean/middleman-s3_sync)
+[![Join the chat at https://gitter.im/fredjean/middleman-s3_sync](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/fredjean/middleman-s3_sync?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Climate](https://codeclimate.com/github/fredjean/middleman-s3_sync.svg)](https://codeclimate.com/github/fredjean/middleman-s3_sync) [![Build Status](https://travis-ci.org/fredjean/middleman-s3_sync.svg?branch=master)](https://travis-ci.org/fredjean/middleman-s3_sync)
 
 This gem determines which files need to be added, updated and optionally deleted
 and only transfer these files up. This reduces the impact of an update
@@ -54,6 +54,9 @@ activate :s3_sync do |s3_sync|
   s3_sync.version_bucket             = false
   s3_sync.index_document             = 'index.html'
   s3_sync.error_document             = '404.html'
+  s3_sync.cloudfront_distribution_id = 'E1234567890123' # CloudFront distribution ID
+  s3_sync.cloudfront_invalidate      = false # Enable CloudFront invalidation
+  s3_sync.cloudfront_invalidate_all  = false # Invalidate all paths (/*) or only changed files
 end
 ```
 
@@ -76,81 +79,254 @@ The following defaults apply to the configuration items:
 | encryption                 | ```false```                        |
 | acl                        | ```'public-read'```                |
 | version_bucket             | ```false```                        |
+| cloudfront_distribution_id | -                                  |
+| cloudfront_invalidate      | ```false```                        |
+| cloudfront_invalidate_all  | ```false```                        |
+| cloudfront_wait            | ```false```                        |
 
 ## Setting AWS Credentials
 
-There are several ways to provide the AWS credentials for s3_sync:
+There are several secure ways to provide AWS credentials for s3_sync. Using temporary, least-privilege credentials is strongly recommended.
 
-#### Through `config.rb`
+#### Best Practices for AWS Credentials (Recommended)
 
-You can set the aws_access_key_id and aws_secret_access_key in the block
-that is passed to the activate method.
+##### 1. AWS IAM Roles (Most Secure)
 
-> I strongly discourage using this method. This will lead you to add and commit these changes
-> to your SCM and potentially expose sensitive information to the world.
+###### For CI/CD and Cloud Environments
+- **EC2 Instance Profiles**: If running on EC2, use IAM roles attached to your instance. Credentials are automatically rotated and managed by AWS.
+- **ECS Task Roles**: For container workloads, use task roles to provide permissions to specific containers.
+- **CI/CD Service Roles**: Most CI/CD services (GitHub Actions, CircleCI, etc.) offer native AWS integrations that support assuming IAM roles.
 
-#### Through `.s3_sync` File
+###### For Local Development
+- **AWS IAM Identity Center (SSO)** and configured profiles in your AWS config file
+- **AWS CLI credential process** to integrate with external identity providers
+- **Role assumption** with short-lived credentials through `aws sts assume-role`
 
-You can create a `.s3_sync` at the root of your middleman project.
-The credentials are passed in the YAML format. The keys match the
-options keys.
-
-The .s3_sync file takes precedence to the configuration passed in the
-activate method.
-
-A sample `.s3_sync` file is included at the root of this repo.
-
-> Make sure to add .s3_sync to your ignore list if you choose this approach. Not doing so may expose
-> credentials to the world.
+To use these methods, you don't need to specify credentials in your Middleman configuration. The AWS SDK will automatically detect and use them.
 
-#### Through the Command Line
+##### 2. Environment Variables with Temporary Credentials
 
-The aws credentials can also be passed via a command line options
-`--aws_access_key_id` (`-k`) and `--aws_secret_access_key` (`-s`). They should override
-any other settings if specified.
+Using environment variables with short-lived credentials from role assumption:
 
-#### Through Environment
+```bash
+# Obtain temporary credentials via assume-role or similar
+# Then set these environment variables
+export AWS_ACCESS_KEY_ID="temporary-access-key"
+export AWS_SECRET_ACCESS_KEY="temporary-secret-key"
+export AWS_SESSION_TOKEN="temporary-session-token"
+export AWS_BUCKET="your-bucket-name"
+```
 
-You can also pass the credentials through environment variables. They
-map to the following values:
+These environment variables are used when credentials are not otherwise specified:
 
 | Setting               | Environment Variable               |
 | --------------------- | ---------------------------------- |
 | aws_access_key_id     | ```ENV['AWS_ACCESS_KEY_ID']```     |
 | aws_secret_access_key | ```ENV['AWS_SECRET_ACCESS_KEY']``` |
+| aws_session_token     | ```ENV['AWS_SESSION_TOKEN']```     |
 | bucket                | ```ENV['AWS_BUCKET']```            |
 
-The environment is used when the credentials are not set in the activate
-method or passed through the ```.s3_sync``` configuration file.
+#### Alternative Methods (Not Recommended for Production)
 
-#### Through IAM role
+The following methods are less secure and should be avoided in production environments:
 
-Alternatively, if you are running builds on EC2 instance which has approrpiate IAM role, then you don't need to think about specifying credentials at all – they will be pulled from AWS metadata service.
+##### Through `.s3_sync` File
 
-#### IAM Policy
+You can create a `.s3_sync` at the root of your middleman project.
+The credentials are passed in the YAML format. The keys match the options keys.
+
+A sample `.s3_sync` file is included at the root of this repo.
+
+> **SECURITY WARNING**: If using this approach, ensure you add `.s3_sync` to your `.gitignore` to prevent
+> accidentally committing credentials to your repository. Consider using this only for local development
+> and only with temporary credentials.
+
+##### Through `config.rb`
+
+You can set the AWS credentials in the activation block, but this is strongly discouraged:
+
+> **SECURITY WARNING**: This method could lead to credentials being committed to version control,
+> potentially exposing sensitive information. Never use long-lived credentials with this method.
+
+##### Through Command Line
+
+Credentials can be passed via command line options, but this may expose them in shell history:
+
+> **SECURITY WARNING**: Command line parameters may be visible in process listings or shell history.
+> Consider using environment variables or IAM roles instead.
+
+## CloudFront Invalidation
+
+The gem can automatically invalidate CloudFront cache after a successful sync. This ensures that your CloudFront distribution serves the latest content immediately after deployment.
+
+### Configuration
+
+```ruby
+activate :s3_sync do |s3_sync|
+  # ... other configuration ...
+  s3_sync.cloudfront_distribution_id = 'E1234567890123'  # Your CloudFront distribution ID
+  s3_sync.cloudfront_invalidate      = true             # Enable invalidation
+  s3_sync.cloudfront_invalidate_all  = false            # Invalidate only changed files
+end
+```
+
+### Configuration Options
+
+| Setting                           | Default     | Description |
+| --------------------------------- | ----------- | ----------- |
+| cloudfront_distribution_id        | -           | CloudFront distribution ID to invalidate |
+| cloudfront_invalidate             | ```false``` | Enable CloudFront invalidation after sync |
+| cloudfront_invalidate_all         | ```false``` | Invalidate all paths (/*) instead of only changed files |
+| cloudfront_invalidation_batch_size| ```1000```  | Maximum paths per invalidation request |
+| cloudfront_wait                   | ```false``` | Wait for CloudFront invalidation to complete |
+</edits>
+
+### Command Line Options
+
+You can also control CloudFront invalidation via command line:
+
+```bash
+# Enable CloudFront invalidation for this sync
+middleman s3_sync --cloudfront-invalidate --cloudfront-distribution-id E1234567890123
+
+# Invalidate all paths instead of just changed files
+middleman s3_sync --cloudfront-invalidate-all --cloudfront-distribution-id E1234567890123
+
+# Wait for invalidation to complete before exiting
+middleman s3_sync --cloudfront-invalidate --cloudfront-wait --cloudfront-distribution-id E1234567890123
+
+# Custom batch size for large numbers of files
+middleman s3_sync --cloudfront-invalidate --cloudfront-invalidation-batch-size 500 --cloudfront-distribution-id E1234567890123
+
+# Short aliases
+middleman s3_sync -c -d E1234567890123              # Basic invalidation
+middleman s3_sync -a -d E1234567890123              # Invalidate all paths
+middleman s3_sync -c -w -d E1234567890123           # Invalidate and wait
+middleman s3_sync -c -a -w -d E1234567890123        # Invalidate all and wait
+```
 
-Here's a sample IAM policy that will allow a user to update the site
-contained in a bucket named "mysite.com":
+#### Available CloudFront Command Line Options
 
+| Option | Short | Description |
+| ------ | ----- | ----------- |
+| `--cloudfront-distribution-id` | `-d` | CloudFront distribution ID |
+| `--cloudfront-invalidate` | `-c` | Enable CloudFront invalidation |
+| `--cloudfront-invalidate-all` | `-a` | Invalidate all paths (/*) |
+| `--cloudfront-invalidation-batch-size` | - | Max paths per request (default: 1000) |
+| `--cloudfront-wait` | `-w` | Wait for invalidation to complete |
+
+### How It Works
+
+1. **Smart Invalidation**: By default, only files that were created, updated, or deleted during the sync are invalidated
+2. **Path Optimization**: Duplicate paths are removed and redundant paths (covered by wildcards) are eliminated
+3. **Batch Processing**: Large numbers of paths are split into multiple invalidation requests to respect CloudFront limits
+4. **Error Handling**: Invalidation failures are logged but don't stop the sync process
+5. **Dry Run Support**: Use `--dry-run` to see what would be invalidated without making actual API calls
+
+### IAM Permissions
+
+Your AWS credentials need CloudFront permissions in addition to S3:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "cloudfront:CreateInvalidation",
+        "cloudfront:GetInvalidation",
+        "cloudfront:ListInvalidations"
+      ],
+      "Resource": "arn:aws:cloudfront::*:distribution/E1234567890123"
+    }
+  ]
+}
 ```
+
+### Cost Considerations
+
+- CloudFront allows 1,000 free invalidation paths per month
+- Additional invalidations cost $0.005 per path
+- Use `cloudfront_invalidate_all: true` for major updates to minimize costs (counts as 1 path)
+- Consider the trade-off between immediate cache invalidation and cost
+
+#### IAM Policy
+
+Here's a sample IAM policy with least-privilege permissions that will allow syncing to a bucket named "mysite.com":
+
+```json
 {
   "Version": "2012-10-17",
   "Statement": [
     {
       "Effect": "Allow",
-      "Action": "s3:*",
+      "Action": [
+        "s3:ListBucket",
+        "s3:GetBucketLocation",
+        "s3:GetBucketVersioning"
+      ],
       "Resource": "arn:aws:s3:::mysite.com"
     },
     {
       "Effect": "Allow",
-      "Action": "s3:*",
+      "Action": [
+        "s3:PutObject",
+        "s3:PutObjectAcl",
+        "s3:GetObject",
+        "s3:GetObjectAcl",
+        "s3:DeleteObject",
+        "s3:HeadObject"
+      ],
       "Resource": "arn:aws:s3:::mysite.com/*"
     }
   ]
 }
 ```
 
-This will give full access to both the bucket and it's contents.
+If you're using additional features, you may need these permissions as well:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutBucketWebsite",
+        "s3:PutBucketVersioning"
+      ],
+      "Resource": "arn:aws:s3:::mysite.com",
+      "Condition": {
+        "Bool": {
+          "aws:SecureTransport": "true"
+        }
+      }
+    }
+  ]
+}
+```
+
+This policy grants only the specific permissions needed:
+
+- **For the bucket itself**:
+  - `s3:ListBucket`: To list objects in the bucket
+  - `s3:GetBucketLocation`: To determine the bucket's region
+  - `s3:GetBucketVersioning`: To check versioning status
+  - `s3:PutBucketVersioning`: If using the `version_bucket` option
+  - `s3:PutBucketWebsite`: If using website configuration (index/error documents)
+
+- **For objects in the bucket**:
+  - `s3:PutObject`: To create/update objects
+  - `s3:PutObjectAcl`: To set ACLs on objects
+  - `s3:GetObject`: To retrieve objects for comparison
+  - `s3:GetObjectAcl`: To read existing ACLs
+  - `s3:DeleteObject`: To delete stray objects (when `delete: true`)
+  - `s3:HeadObject`: To retrieve object metadata via HEAD requests
+
+The source code shows that middleman-s3_sync uses HEAD requests (`object.head`) to compare resources, checks and sets bucket versioning when configured, and can set website configuration for index and error documents.
+
+Note: You can further restrict these permissions by adding conditions or limiting them to specific prefixes if you're only publishing to a subdirectory of the bucket.
 
 ## Command Line Usage
 
@@ -182,17 +358,17 @@ You can specify which environment to run Middleman under using the
 
     $ middleman s3_sync --environment=production
 
-You can set up separate sync environments in config.rb like this: 
+You can set up separate sync environments in config.rb like this:
 
 ```ruby
 	configure :staging do
 		activate :s3_sync do |s3_sync|
 			s3_sync.bucket = '<bucket'
-			... 
+			...
     	end
     end
 ```
-    
+
 See the Usage section above for all the s3_sync. options to include. Currently, the .s3_sync file does not allow separate environments.
 
 #### Dry Run
@@ -291,7 +467,7 @@ The following keys can be set:
 You can pass the `expires` key to the `caching_policy` and
 `default_caching_policy` methods if you insist on setting the expires
 header on a results. You will need to pass it a Time object indicating
-when the resourse is set to expire.
+when the resource is set to expire.
 
 > Note that the `Cache-Control` header will take precedence over the
 > `Expires` header if both are present.

data/lib/middleman/redirect.rb

@@ -0,0 +1,21 @@
+module Middleman
+  module Sitemap
+    class Resource
+      def redirect?
+        false
+      end
+    end
+
+    module Extensions
+      class RedirectResource < Resource
+        def target_url
+          @target_url ||= ::Middleman::Util.url_for(@store.app, @request_path, relative: false, find_resource: true)
+        end
+
+        def redirect?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/middleman/s3_sync/cloudfront.rb

@@ -0,0 +1,192 @@
+require 'aws-sdk-cloudfront'
+require 'securerandom'
+
+module Middleman
+  module S3Sync
+    module CloudFront
+      class << self
+        include Status
+
+        def invalidate(invalidation_paths, options)
+          return unless should_invalidate?(options)
+          return if invalidation_paths.empty? && !options.cloudfront_invalidate_all
+
+          paths = prepare_invalidation_paths(invalidation_paths, options)
+          return if paths.empty?
+
+          say_status "Invalidating CloudFront distribution #{options.cloudfront_distribution_id}"
+          
+          if options.dry_run
+            say_status "#{ANSI.yellow{'DRY RUN:'}} Would invalidate #{paths.length} paths in CloudFront"
+            paths.each { |path| say_status "  #{path}" } if options.verbose
+            return
+          end
+
+          # Split paths into batches to respect CloudFront limits
+          batch_size = [options.cloudfront_invalidation_batch_size, 3000].min
+          path_batches = paths.each_slice(batch_size).to_a
+
+          invalidation_ids = []
+          
+          path_batches.each_with_index do |batch, index|
+            say_status "Creating invalidation batch #{index + 1}/#{path_batches.length} (#{batch.length} paths)"
+            
+            invalidation_id = create_invalidation(batch, options)
+            invalidation_ids << invalidation_id if invalidation_id
+            
+            # Add a small delay between batches to avoid rate limiting
+            sleep(1) if path_batches.length > 1 && index < path_batches.length - 1
+          end
+
+          if invalidation_ids.any?
+            say_status "CloudFront invalidation(s) created: #{invalidation_ids.join(', ')}"
+            
+            if options.cloudfront_wait
+              say_status "Waiting for CloudFront invalidation(s) to complete..."
+              wait_for_invalidations(invalidation_ids, options)
+              say_status "CloudFront invalidation(s) completed successfully"
+            else
+              say_status "Invalidations may take 10-15 minutes to complete"
+            end
+          end
+
+          invalidation_ids
+        rescue Aws::CloudFront::Errors::ServiceError => e
+          say_status "#{ANSI.red{'CloudFront invalidation failed:'}} #{e.message}"
+          raise e unless options.verbose # Re-raise unless we're being verbose
+        end
+
+        private
+
+        def should_invalidate?(options)
+          return false unless options.cloudfront_invalidate
+          
+          unless options.cloudfront_distribution_id
+            say_status "#{ANSI.yellow{'CloudFront invalidation skipped:'}} no distribution ID provided"
+            return false
+          end
+
+          true
+        end
+
+        def prepare_invalidation_paths(invalidation_paths, options)
+          if options.cloudfront_invalidate_all
+            return ['/*']
+          end
+
+          # Normalize paths for CloudFront
+          paths = invalidation_paths.map do |path|
+            # Ensure path starts with /
+            normalized_path = path.start_with?('/') ? path : "/#{path}"
+            
+            # Remove any double slashes
+            normalized_path.gsub(/\/+/, '/')
+          end.uniq.sort
+
+          # Remove any paths that would be covered by a wildcard
+          if paths.include?('/*')
+            paths = ['/*']
+          else
+            # Remove redundant paths (e.g., if we have /path/* and /path/file.html)
+            paths = remove_redundant_paths(paths)
+          end
+
+          say_status "Prepared #{paths.length} paths for CloudFront invalidation" if options.verbose
+
+          paths
+        end
+
+        def remove_redundant_paths(paths)
+          # Sort paths to ensure wildcards come before specific files
+          sorted_paths = paths.sort
+          result = []
+          
+          sorted_paths.each do |path|
+            # Check if this path is already covered by a wildcard we've added
+            is_redundant = result.any? do |existing_path|
+              if existing_path.end_with?('/*')
+                # Check if current path is under this wildcard
+                wildcard_prefix = existing_path[0..-3] # Remove /*
+                path.start_with?(wildcard_prefix + '/')
+              else
+                false
+              end
+            end
+            
+            result << path unless is_redundant
+          end
+          
+          result
+        end
+
+        def create_invalidation(paths, options)
+          caller_reference = "middleman-s3_sync-#{Time.now.to_i}-#{SecureRandom.hex(4)}"
+          
+          response = cloudfront_client(options).create_invalidation({
+            distribution_id: options.cloudfront_distribution_id,
+            invalidation_batch: {
+              paths: {
+                quantity: paths.length,
+                items: paths
+              },
+              caller_reference: caller_reference
+            }
+          })
+
+          response.invalidation.id
+        rescue Aws::CloudFront::Errors::ServiceError => e
+          say_status "#{ANSI.red{'Failed to create CloudFront invalidation:'}} #{e.message}"
+          say_status "Paths: #{paths.join(', ')}" if options.verbose
+          raise e unless options.verbose
+          nil
+        end
+
+        def cloudfront_client(options)
+          client_options = {
+            region: 'us-east-1' # CloudFront is always in us-east-1
+          }
+
+          # Use the same credentials as S3 if available
+          if options.aws_access_key_id && options.aws_secret_access_key
+            client_options.merge!({
+              access_key_id: options.aws_access_key_id,
+              secret_access_key: options.aws_secret_access_key
+            })
+
+            # If using an assumed role
+            client_options.merge!({
+              session_token: options.aws_session_token
+            }) if options.aws_session_token
+          end
+
+          Aws::CloudFront::Client.new(client_options)
+        end
+
+        def wait_for_invalidations(invalidation_ids, options)
+          invalidation_ids.each do |invalidation_id|
+            say_status "Waiting for invalidation #{invalidation_id}..."
+            
+            client = cloudfront_client(options)
+            client.wait_until(:invalidation_completed, 
+              distribution_id: options.cloudfront_distribution_id,
+              id: invalidation_id
+            ) do |waiter|
+              waiter.max_attempts = 30  # Wait up to 30 minutes (30 * 60s checks)
+              waiter.delay = 60         # Check every 60 seconds
+              
+              waiter.before_attempt do |attempt|
+                say_status "Checking invalidation status (attempt #{attempt}/30)..." if options.verbose
+              end
+            end
+          end
+        rescue Aws::Waiters::Errors::WaiterFailed => e
+          say_status "#{ANSI.yellow{'Warning:'}} CloudFront invalidation wait timed out: #{e.message}"
+          say_status "Invalidation is still in progress but sync will continue"
+        rescue Aws::CloudFront::Errors::ServiceError => e
+          say_status "#{ANSI.yellow{'Warning:'}} CloudFront invalidation wait failed: #{e.message}"
+          say_status "Invalidation may still be in progress"
+        end
+      end
+    end
+  end
+end

data/lib/middleman/s3_sync/options.rb

@@ -6,6 +6,7 @@ module Middleman
         :http_prefix,
         :acl,
         :bucket,
+        :endpoint,
         :region,
         :aws_access_key_id,
         :aws_secret_access_key,
@@ -22,6 +23,7 @@ module Middleman
         :dry_run,
         :verbose,
         :content_types,
+        :ignore_paths,
         :index_document,
         :error_document
       ]
@@ -67,6 +69,10 @@ module Middleman
         (@path_style.nil? ? true : @path_style)
       end
 
+      def ignore_paths
+        @ignore_paths.nil? ? [] : @ignore_paths
+      end
+
       def prefix=(prefix)
         http_prefix = @http_prefix ? @http_prefix.sub(%r{^/}, "") : ""
         if http_prefix.split("/").first == prefix