middleman-s3_sync 4.6.4 → 4.7.0

data/lib/middleman/s3_sync.rb

@@ -1,5 +1,6 @@
 require 'aws-sdk-s3'
 require 'digest/md5'
+require 'mime/types'
 require 'middleman/s3_sync/version'
 require 'middleman/s3_sync/options'
 require 'middleman/s3_sync/caching_policy'
@@ -11,6 +12,7 @@ require 'middleman/redirect'
 require 'parallel'
 require 'ruby-progressbar'
 require 'thread'
+require 'set'
 
 module Middleman
   module S3Sync
@@ -20,6 +22,7 @@ module Middleman
 
       @@bucket_lock = Mutex.new
       @@bucket_files_lock = Mutex.new
+      @@invalidation_paths_lock = Mutex.new
 
       attr_accessor :s3_sync_options
       attr_accessor :mm_resources
@@ -28,11 +31,18 @@ module Middleman
       THREADS_COUNT = 8
       
       # Track paths that were changed during sync for CloudFront invalidation
-      attr_accessor :invalidation_paths
+      # Using a Set for O(1) lookups
+      attr_reader :invalidation_paths
 
       def sync()
         @app ||= ::Middleman::Application.new
-        @invalidation_paths = []
+        @invalidation_paths = Set.new
+
+        # Ensure sitemap is fully populated before syncing
+        # This catches resources from extensions activated in :build mode
+        if @app.respond_to?(:sitemap) && @app.sitemap.respond_to?(:ensure_resource_list_updated!)
+          @app.sitemap.ensure_resource_list_updated!
+        end
 
         say_status "Let's see if there's work to be done..."
         unless work_to_be_done?
@@ -58,8 +68,11 @@ module Middleman
         
         # Invalidate CloudFront cache if requested
         if s3_sync_options.cloudfront_invalidate
-          CloudFront.invalidate(@invalidation_paths, s3_sync_options)
+          CloudFront.invalidate(@invalidation_paths.to_a, s3_sync_options)
         end
+
+        # Run after_s3_sync callback if provided
+        run_after_s3_sync_callback
       end
 
       def bucket
@@ -77,10 +90,12 @@ module Middleman
       end
       
       def add_invalidation_path(path)
-        @invalidation_paths ||= []
-        # Normalize path for CloudFront (ensure it starts with /)
-        normalized_path = path.start_with?('/') ? path : "/#{path}"
-        @invalidation_paths << normalized_path unless @invalidation_paths.include?(normalized_path)
+        @@invalidation_paths_lock.synchronize do
+          @invalidation_paths ||= Set.new
+          # Normalize path for CloudFront (ensure it starts with /)
+          normalized_path = path.start_with?('/') ? path : "/#{path}"
+          @invalidation_paths.add(normalized_path)
+        end
       end
 
       def remote_only_paths
@@ -95,6 +110,44 @@ module Middleman
         @content_types || {}
       end
 
+      # Run the after_s3_sync callback if configured
+      def run_after_s3_sync_callback
+        callback = s3_sync_options.after_s3_sync
+        return unless callback
+
+        say_status 'callback', 'Running after_s3_sync callback...'
+        
+        # Build sync results hash
+        results = {
+          created: files_to_create.size,
+          updated: files_to_update.size,
+          deleted: files_to_delete.size,
+          invalidation_paths: @invalidation_paths.to_a
+        }
+
+        begin
+          if callback.respond_to?(:call)
+            # Lambda/Proc callback - check arity to decide whether to pass results
+            if callback.arity == 0 || callback.arity == -1 && callback.parameters.empty?
+              callback.call
+            else
+              callback.call(results)
+            end
+          elsif callback.is_a?(Symbol) && @app.respond_to?(callback)
+            # Symbol method name - call on app
+            if @app.method(callback).arity == 0
+              @app.send(callback)
+            else
+              @app.send(callback, results)
+            end
+          end
+          
+          say_status 'callback', 'after_s3_sync completed successfully'
+        rescue => e
+          say_status 'error', "after_s3_sync callback failed: #{e.message}"
+        end
+      end
+
       protected
       def update_bucket_versioning
         s3_client.put_bucket_versioning({
@@ -110,10 +163,41 @@ module Middleman
         opts[:index_document] = { suffix: s3_sync_options.index_document } if s3_sync_options.index_document
         opts[:error_document] = { key: s3_sync_options.error_document } if s3_sync_options.error_document
 
+        # Add routing rules if specified
+        if s3_sync_options.routing_rules && !s3_sync_options.routing_rules.empty?
+          opts[:routing_rules] = s3_sync_options.routing_rules.map do |rule|
+            routing_rule = {}
+            
+            # Handle condition (optional)
+            if rule[:condition] || rule['condition']
+              condition = rule[:condition] || rule['condition']
+              routing_rule[:condition] = {}
+              routing_rule[:condition][:key_prefix_equals] = condition[:key_prefix_equals] || condition['key_prefix_equals'] if condition[:key_prefix_equals] || condition['key_prefix_equals']
+              routing_rule[:condition][:http_error_code_returned_equals] = condition[:http_error_code_returned_equals] || condition['http_error_code_returned_equals'] if condition[:http_error_code_returned_equals] || condition['http_error_code_returned_equals']
+            end
+            
+            # Handle redirect (required)
+            redirect = rule[:redirect] || rule['redirect']
+            routing_rule[:redirect] = {}
+            routing_rule[:redirect][:host_name] = redirect[:host_name] || redirect['host_name'] if redirect[:host_name] || redirect['host_name']
+            routing_rule[:redirect][:http_redirect_code] = redirect[:http_redirect_code] || redirect['http_redirect_code'] if redirect[:http_redirect_code] || redirect['http_redirect_code']
+            routing_rule[:redirect][:protocol] = redirect[:protocol] || redirect['protocol'] if redirect[:protocol] || redirect['protocol']
+            routing_rule[:redirect][:replace_key_prefix_with] = redirect[:replace_key_prefix_with] || redirect['replace_key_prefix_with'] if redirect[:replace_key_prefix_with] || redirect['replace_key_prefix_with']
+            routing_rule[:redirect][:replace_key_with] = redirect[:replace_key_with] || redirect['replace_key_with'] if redirect[:replace_key_with] || redirect['replace_key_with']
+            
+            routing_rule
+          end
+        end
+
         if opts[:error_document] && !opts[:index_document]
           raise 'S3 requires `index_document` if `error_document` is specified'
         end
 
+        # S3 requires index_document if routing_rules are specified
+        if opts[:routing_rules] && !opts[:index_document]
+          raise 'S3 requires `index_document` if `routing_rules` are specified'
+        end
+
         unless opts.empty?
           say_status "Putting bucket website: #{opts.to_json}"
           s3_client.put_bucket_website({
@@ -204,10 +288,22 @@ module Middleman
       end
 
       def delete_resources
-        Parallel.map(files_to_delete, in_threads: THREADS_COUNT) do |resource|
-          resource.destroy!
+        resources = files_to_delete
+        return if resources.empty?
+
+        # Print status messages for all resources being deleted
+        resources.each do |resource|
+          say_status "#{ANSI.red{"Deleting"}} #{resource.remote_path}"
           add_invalidation_path(resource.path)
         end
+
+        # Batch delete using S3's delete_objects API (up to 1000 objects per request)
+        unless s3_sync_options.dry_run
+          resources.each_slice(1000) do |batch|
+            objects_to_delete = batch.map { |r| { key: r.remote_path.sub(/^\//, '') } }
+            bucket.delete_objects(delete: { objects: objects_to_delete })
+          end
+        end
       end
 
       def ignore_resources
@@ -217,6 +313,11 @@ module Middleman
       def work_to_be_done?
         Parallel.each(mm_resources, in_threads: THREADS_COUNT, progress: "Processing sitemap") { |mm_resource| add_local_resource(mm_resource) }
 
+        # Scan build directory for orphan files (not in sitemap)
+        if s3_sync_options.scan_build_dir
+          discover_orphan_files
+        end
+
         Parallel.each(remote_only_paths, in_threads: THREADS_COUNT, progress: "Processing remote files") do |remote_path|
           s3_sync_resources[remote_path] ||= S3Sync::Resource.new(nil, remote_resource_for_path(remote_path)).tap(&:status)
         end
@@ -224,24 +325,73 @@ module Middleman
         !(files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?)
       end
 
+      # Discover files in build directory that are not in the sitemap
+      # This handles files generated by after_build callbacks, image optimizers, etc.
+      def discover_orphan_files
+        return unless build_dir && File.directory?(build_dir)
+
+        orphan_files = []
+        
+        Dir.glob(File.join(build_dir, '**', '*')).each do |file_path|
+          next if File.directory?(file_path)
+          
+          # Get path relative to build_dir
+          relative_path = file_path.sub(/^#{Regexp.escape(build_dir)}\/?/, '')
+          
+          # Skip if already in sitemap resources
+          next if s3_sync_resources.key?(relative_path)
+          
+          orphan_files << relative_path
+        end
+
+        return if orphan_files.empty?
+
+        say_status "Found #{orphan_files.size} files outside sitemap"
+        
+        Parallel.each(orphan_files, in_threads: THREADS_COUNT, progress: "Processing orphan files") do |relative_path|
+          # Create a Resource with nil mm_resource but explicit path
+          # It will use mime-types for content type detection
+          remote = remote_resource_for_path(relative_path)
+          resource = S3Sync::Resource.new(nil, remote, path: relative_path)
+          s3_sync_resources[relative_path] = resource.tap(&:status)
+        end
+      end
+
+      # Single-pass categorization of resources by status
+      # Avoids multiple iterations over s3_sync_resources
+      def categorized_resources
+        @categorized_resources ||= begin
+          result = { create: [], update: [], delete: [], ignore: [] }
+          s3_sync_resources.values.each do |resource|
+            case
+            when resource.to_create? then result[:create] << resource
+            when resource.to_update? then result[:update] << resource
+            when resource.to_delete? then result[:delete] << resource
+            when resource.to_ignore? then result[:ignore] << resource
+            end
+          end
+          result
+        end
+      end
+
       def files_to_delete
         if s3_sync_options.delete
-          s3_sync_resources.values.select { |r| r.to_delete? }
+          categorized_resources[:delete]
         else
           []
         end
       end
 
       def files_to_create
-        s3_sync_resources.values.select { |r| r.to_create? }
+        categorized_resources[:create]
       end
 
       def files_to_update
-        s3_sync_resources.values.select { |r| r.to_update? }
+        categorized_resources[:update]
       end
 
       def files_to_ignore
-        s3_sync_resources.values.select { |r| r.to_ignore? }
+        categorized_resources[:ignore]
       end
 
       def build_dir

data/lib/middleman-s3_sync/commands.rb

@@ -102,7 +102,9 @@ module Middleman
         verbose = options[:verbose] ? 0 : 1
         instrument = options[:instrument]
 
-        mode = options[:build] ? :build : :config
+        # Always use :build mode to ensure build-time extensions are active
+        # (e.g., asset_hash in `configure :build` blocks)
+        mode = :build
 
         ::Middleman::S3Sync.app = ::Middleman::Application.new do
           config[:mode] = mode

data/lib/middleman-s3_sync/extension.rb

@@ -26,8 +26,10 @@ module Middleman
     option :dry_run, false, 'Whether to perform a dry-run'
     option :index_document, nil, 'S3 custom index document path'
     option :error_document, nil, 'S3 custom error document path'
+    option :routing_rules, [], 'S3 website routing rules (array of rule hashes)'
     option :content_types, {}, 'Custom content types'
     option :ignore_paths, [], 'Paths that should be ignored during sync, strings or regex are allowed'
+    option :scan_build_dir, false, 'Scan build directory for files not in sitemap (e.g., after_build files)'
     option :cloudfront_distribution_id, nil, 'CloudFront distribution ID for invalidation'
     option :cloudfront_invalidate, false, 'Whether to invalidate CloudFront cache after sync'
     option :cloudfront_invalidate_all, false, 'Whether to invalidate all paths (/*) or only changed files'
@@ -35,6 +37,7 @@ module Middleman
     option :cloudfront_invalidation_max_retries, 5, 'Maximum number of retries for rate-limited invalidation requests'
     option :cloudfront_invalidation_batch_delay, 2, 'Delay in seconds between invalidation batches'
     option :cloudfront_wait, false, 'Whether to wait for CloudFront invalidation to complete'
+    option :after_s3_sync, nil, 'Proc/lambda to call after sync completes (receives changed_paths array)'
 
     expose_to_config :s3_sync_options, :default_caching_policy, :caching_policy
 
@@ -87,16 +90,30 @@ module Middleman
       true
     end
 
-    # Delegate option readers to the options object
+    # Delegate option readers and writers to the options object
     def method_missing(method, *args, &block)
-      if options.respond_to?(method)
-        options.send(method, *args, &block)
-      else
-        super
+      method_str = method.to_s
+      
+      # Handle setter methods (e.g., verbose=)
+      if method_str.end_with?('=')
+        option_name = method_str.chomp('=').to_sym
+        if options.respond_to?(option_name)
+          options[option_name] = args.first
+          return args.first
+        end
+      elsif options.respond_to?(method)
+        return options.send(method, *args, &block)
       end
+      
+      super
     end
 
     def respond_to_missing?(method, include_private = false)
+      method_str = method.to_s
+      if method_str.end_with?('=')
+        option_name = method_str.chomp('=').to_sym
+        return options.respond_to?(option_name)
+      end
       options.respond_to?(method) || super
     end
 

data/middleman-s3_sync.gemspec

@@ -13,28 +13,32 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://github.com/fredjean/middleman-s3_sync"
   gem.license       = 'MIT'
 
+  gem.required_ruby_version = '>= 3.0'
+
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency 'middleman-core'
-  gem.add_runtime_dependency 'middleman-cli'
-  gem.add_runtime_dependency 'aws-sdk-s3', '>= 1.187.0'
-  gem.add_runtime_dependency 'aws-sdk-cloudfront'
-  gem.add_runtime_dependency 'parallel'
-  gem.add_runtime_dependency 'ruby-progressbar'
-  gem.add_runtime_dependency 'ansi', '~> 1.5.0'
-  gem.add_runtime_dependency 'mime-types', '~> 3.1'
-  gem.add_runtime_dependency 'nokogiri', '>= 1.18.4'
+  # Runtime dependencies
+  gem.add_runtime_dependency 'middleman-core', '~> 4.4'
+  gem.add_runtime_dependency 'middleman-cli', '~> 4.4'
+  gem.add_runtime_dependency 'aws-sdk-s3', '~> 1.187', '>= 1.187.0'
+  gem.add_runtime_dependency 'aws-sdk-cloudfront', '~> 1.0'
+  gem.add_runtime_dependency 'parallel', '~> 1.20'
+  gem.add_runtime_dependency 'ruby-progressbar', '~> 1.11'
+  gem.add_runtime_dependency 'ansi', '~> 1.5'
+  gem.add_runtime_dependency 'mime-types', '~> 3.4'
+  gem.add_runtime_dependency 'nokogiri', '~> 1.18', '>= 1.18.4'
 
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'pry-byebug'
-  gem.add_development_dependency 'rspec'
-  gem.add_development_dependency 'rspec-support'
-  gem.add_development_dependency 'rspec-its'
-  gem.add_development_dependency 'rspec-mocks'
-  gem.add_development_dependency 'timerizer'
-  gem.add_development_dependency 'webrick'
+  # Development dependencies
+  gem.add_development_dependency 'rake', '~> 13.0'
+  gem.add_development_dependency 'pry', '~> 0.14'
+  gem.add_development_dependency 'pry-byebug', '~> 3.10'
+  gem.add_development_dependency 'rspec', '~> 3.12'
+  gem.add_development_dependency 'rspec-support', '~> 3.12'
+  gem.add_development_dependency 'rspec-its', '~> 2.0'
+  gem.add_development_dependency 'rspec-mocks', '~> 3.12'
+  gem.add_development_dependency 'timerizer', '~> 0.3'
+  gem.add_development_dependency 'webrick', '~> 1.8'
 end

data/spec/aws_sdk_parameters_spec.rb

@@ -87,6 +87,60 @@ describe 'AWS SDK Parameter Validation' do
         }.to raise_error('S3 requires `index_document` if `error_document` is specified')
       end
     end
+
+    context 'when routing_rules are set' do
+      before do
+        options.routing_rules = [
+          {
+            condition: { key_prefix_equals: 'docs/' },
+            redirect: { replace_key_prefix_with: 'documents/' }
+          },
+          {
+            condition: { http_error_code_returned_equals: '404' },
+            redirect: { host_name: 'example.com', replace_key_with: 'error.html' }
+          }
+        ]
+      end
+
+      it 'includes routing_rules in website configuration' do
+        expect(s3_client).to receive(:put_bucket_website) do |params|
+          expect(params[:website_configuration]).to have_key(:routing_rules)
+          rules = params[:website_configuration][:routing_rules]
+          expect(rules).to be_an(Array)
+          expect(rules.length).to eq(2)
+          
+          # First rule
+          expect(rules[0][:condition][:key_prefix_equals]).to eq('docs/')
+          expect(rules[0][:redirect][:replace_key_prefix_with]).to eq('documents/')
+          
+          # Second rule
+          expect(rules[1][:condition][:http_error_code_returned_equals]).to eq('404')
+          expect(rules[1][:redirect][:host_name]).to eq('example.com')
+          expect(rules[1][:redirect][:replace_key_with]).to eq('error.html')
+        end
+
+        Middleman::S3Sync.send(:update_bucket_website)
+      end
+    end
+
+    context 'when routing_rules are set without index_document' do
+      before do
+        options.index_document = nil
+        options.error_document = nil
+        options.routing_rules = [
+          {
+            condition: { key_prefix_equals: 'old/' },
+            redirect: { replace_key_prefix_with: 'new/' }
+          }
+        ]
+      end
+
+      it 'raises an error because S3 requires index_document if routing_rules are specified' do
+        expect {
+          Middleman::S3Sync.send(:update_bucket_website)
+        }.to raise_error('S3 requires `index_document` if `routing_rules` are specified')
+      end
+    end
   end
 
   describe 'put_bucket_versioning parameters' do
@@ -128,14 +182,17 @@ describe 'AWS SDK Parameter Validation' do
       allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
       allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
       allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
+      # Stub File.open to return a StringIO for streaming upload tests
+      allow(File).to receive(:open).with('build/test/file.html', 'rb').and_yield(StringIO.new('test content'))
       allow(s3_object).to receive(:head).and_return(nil)
       options.dry_run = false
     end
 
     it 'uses correct metadata key format' do
       expect(s3_object).to receive(:put) do |upload_options|
-        # Verify basic parameters
-        expect(upload_options[:body]).to eq('test content')
+        # Verify body is a readable IO object (for streaming)
+        expect(upload_options[:body]).to respond_to(:read)
+        expect(upload_options[:body].read).to eq('test content')
         expect(upload_options[:content_type]).to eq('text/html')
         expect(upload_options[:acl]).to eq('public-read')
         
@@ -160,7 +217,7 @@ describe 'AWS SDK Parameter Validation' do
       it 'does not include acl parameter in upload' do
         expect(s3_object).to receive(:put) do |upload_options|
           expect(upload_options).not_to have_key(:acl)
-          expect(upload_options[:body]).to eq('test content')
+          expect(upload_options[:body]).to respond_to(:read)
           expect(upload_options[:content_type]).to eq('text/html')
         end
 
@@ -176,7 +233,7 @@ describe 'AWS SDK Parameter Validation' do
       it 'does not include acl parameter in upload' do
         expect(s3_object).to receive(:put) do |upload_options|
           expect(upload_options).not_to have_key(:acl)
-          expect(upload_options[:body]).to eq('test content')
+          expect(upload_options[:body]).to respond_to(:read)
           expect(upload_options[:content_type]).to eq('text/html')
         end
 
@@ -192,6 +249,10 @@ describe 'AWS SDK Parameter Validation' do
 
       it 'automatically retries without ACL when AccessControlListNotSupported error occurs' do
         call_count = 0
+        # Use a reusable StringIO that can be rewound
+        file_io = StringIO.new('test content')
+        allow(File).to receive(:open).with('build/test/file.html', 'rb').and_yield(file_io)
+        
         expect(s3_object).to receive(:put).twice do |upload_options|
           call_count += 1
           if call_count == 1
@@ -201,7 +262,7 @@ describe 'AWS SDK Parameter Validation' do
           else
             # Second call should not include ACL
             expect(upload_options).not_to have_key(:acl)
-            expect(upload_options[:body]).to eq('test content')
+            expect(upload_options[:body]).to respond_to(:read)
             expect(upload_options[:content_type]).to eq('text/html')
           end
         end
@@ -219,7 +280,7 @@ describe 'AWS SDK Parameter Validation' do
             expect(upload_options[:acl]).to eq('public-read')
             raise Aws::S3::Errors::AccessControlListNotSupported.new(nil, 'The bucket does not allow ACLs')
           else
-            # Second call should succeed without ACL
+            # Subsequent calls should succeed without ACL
             expect(upload_options).not_to have_key(:acl)
             true
           end
@@ -241,6 +302,7 @@ describe 'AWS SDK Parameter Validation' do
         allow(File).to receive(:read).with('build/test/file.html.gz').and_return('gzipped content')
         allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
         allow(File).to receive(:read).with('build/test/file.html').and_return('original content')
+        allow(File).to receive(:open).with('build/test/file.html.gz', 'rb').and_yield(StringIO.new('gzipped content'))
         
         # Mock the HEAD response to avoid calling it during redirect?
         head_response = double(
@@ -305,6 +367,7 @@ describe 'AWS SDK Parameter Validation' do
         allow(File).to receive(:exist?).with('build/redirect/file.html').and_return(true)
         allow(File).to receive(:exist?).with('build/redirect/file.html.gz').and_return(false)
         allow(File).to receive(:read).with('build/redirect/file.html').and_return('redirect content')
+        allow(File).to receive(:open).with('build/redirect/file.html', 'rb').and_yield(StringIO.new('redirect content'))
         allow(File).to receive(:directory?).with('build/redirect/file.html').and_return(false)
         allow(s3_object).to receive(:head).and_return(nil)
         allow(resource).to receive(:redirect?).and_return(true)
@@ -478,6 +541,7 @@ describe 'AWS SDK Parameter Validation' do
       allow(File).to receive(:exist?).with('build/test/file.html').and_return(true)
       allow(File).to receive(:exist?).with('build/test/file.html.gz').and_return(false)
       allow(File).to receive(:read).with('build/test/file.html').and_return('test content')
+      allow(File).to receive(:open).with('build/test/file.html', 'rb').and_yield(StringIO.new('test content'))
       allow(File).to receive(:directory?).with('build/test/file.html').and_return(false)
       allow(s3_object).to receive(:head).and_return(nil)
       options.dry_run = false

data/spec/cloudfront_spec.rb

@@ -27,6 +27,8 @@ describe Middleman::S3Sync::CloudFront do
 
   before do
     allow(described_class).to receive(:say_status)
+    # Reset cached CloudFront client between tests to prevent double leakage
+    described_class.send(:reset_cloudfront_client!)
   end
 
   describe '.invalidate' do