middleman-s3_sync 4.0.3 → 4.6.0

data/lib/middleman/s3_sync/resource.rb

@@ -4,12 +4,19 @@ module Middleman
       attr_accessor :path, :resource, :partial_s3_resource, :full_s3_resource, :content_type, :gzipped, :options
 
       CONTENT_MD5_KEY = 'x-amz-meta-content-md5'
+      REDIRECT_KEY = 'x-amz-website-redirect-location'
 
       include Status
 
       def initialize(resource, partial_s3_resource)
         @resource = resource
-        @path = resource ? resource.destination_path : partial_s3_resource.key
+        @path = if resource
+                  resource.destination_path.sub(/^\//, '')
+                elsif partial_s3_resource&.key
+                  partial_s3_resource.key.sub(/^\//, '')
+                else
+                  ''
+                end
         @partial_s3_resource = partial_s3_resource
       end
 
@@ -19,13 +26,33 @@ module Middleman
 
       # S3 resource as returned by a HEAD request
       def full_s3_resource
-        @full_s3_resource ||= bucket.files.head(remote_path)
+        @full_s3_resource ||= begin
+          bucket.object(remote_path.sub(/^\//, '')).head
+        rescue Aws::S3::Errors::NotFound
+          nil
+        end
       end
 
       def remote_path
-        s3_resource ? s3_resource.key : "#{options.prefix}#{path}"
+        if s3_resource
+          if s3_resource.respond_to?(:key)
+            s3_resource.key.sub(/^\//, '')
+          else
+            # For HeadObjectOutput objects which don't have key method
+            options.prefix ? normalize_path(options.prefix, path) : path.sub(/^\//, '')
+          end
+        else
+          options.prefix ? normalize_path(options.prefix, path) : path.sub(/^\//, '')
+        end.sub(/^\//, '')  # Ensure no leading slash
       end
       alias :key :remote_path
+      
+      def normalize_path(prefix, path)
+        # Remove any trailing slash from prefix and leading slash from path
+        prefix = prefix.chomp('/')
+        path = path.sub(/^\//, '')
+        "#{prefix}/#{path}"
+      end
 
       def to_h
         attributes = {
@@ -52,18 +79,19 @@ module Middleman
           attributes[:encryption] = 'AES256'
         end
 
+        if redirect?
+          attributes[REDIRECT_KEY] = redirect_url
+        end
+
         attributes
       end
       alias :attributes :to_h
 
       def update!
-        local_content { |body|
-          say_status "#{ANSI.blue{"Updating"}} #{remote_path}#{ gzipped ? ANSI.white {' (gzipped)'} : ''}"
-          s3_resource.merge_attributes(to_h)
-          s3_resource.body = body
-
-          s3_resource.save unless options.dry_run
-        }
+        say_status "#{ANSI.blue{"Updating"}} #{remote_path}#{ gzipped ? ANSI.white {' (gzipped)'} : ''}"
+        unless options.dry_run
+          upload!
+        end
       end
 
       def local_path
@@ -77,14 +105,52 @@ module Middleman
 
       def destroy!
         say_status "#{ANSI.red{"Deleting"}} #{remote_path}"
-        bucket.files.destroy remote_path unless options.dry_run
+        bucket.object(remote_path.sub(/^\//, '')).delete unless options.dry_run
       end
 
       def create!
         say_status "#{ANSI.green{"Creating"}} #{remote_path}#{ gzipped ? ANSI.white {' (gzipped)'} : ''}"
-        local_content { |body|
-          bucket.files.create(to_h.merge(body: body)) unless options.dry_run
+        unless options.dry_run
+          upload!
+        end
+      end
+
+      def upload!
+        object = bucket.object(remote_path.sub(/^\//, ''))
+        upload_options = {
+          body: local_content,
+          content_type: content_type,
+          acl: options.acl
         }
+
+        # Add metadata if present
+        if local_content_md5
+          upload_options[:metadata] = { CONTENT_MD5_KEY => local_content_md5 }
+        end
+
+        # Add redirect if present
+        upload_options[:website_redirect_location] = redirect_url if redirect?
+
+        # Add content encoding if present
+        upload_options[:content_encoding] = "gzip" if options.prefer_gzip && gzipped
+
+        # Add cache control and expires if present
+        if caching_policy
+          upload_options[:cache_control] = caching_policy.cache_control
+          upload_options[:expires] = caching_policy.expires
+        end
+
+        # Add storage class if needed
+        if options.reduced_redundancy_storage
+          upload_options[:storage_class] = 'REDUCED_REDUNDANCY'
+        end
+
+        # Add encryption if needed
+        if options.encryption
+          upload_options[:server_side_encryption] = 'AES256'
+        end
+
+        object.put(upload_options)
       end
 
       def ignore!
@@ -122,12 +188,18 @@ module Middleman
         status == :ignored || status == :alternate_encoding
       end
 
-      def local_content(&block)
-        File.open(local_path, &block)
+      def local_content
+        if block_given?
+          File.open(local_path) { |f| yield f.read }
+        else
+          File.read(local_path)
+        end
       end
 
       def status
-        @status ||= if directory?
+        @status ||= if shunned?
+                      :ignored
+                    elsif directory?
                       if remote?
                         :deleted
                       else
@@ -136,7 +208,7 @@ module Middleman
                     elsif local? && remote?
                       if options.force
                         :updated
-                      elsif not caching_policy_match?
+                      elsif not metadata_match?
                         :updated
                       elsif local_object_md5 == remote_object_md5
                         :identical
@@ -170,11 +242,36 @@ module Middleman
       end
 
       def remote?
-        !!s3_resource
+        !full_s3_resource.nil?
       end
 
       def redirect?
-        full_s3_resource && full_s3_resource.metadata.has_key?('x-amz-website-redirect-location')
+        (resource && resource.respond_to?(:redirect?) && resource.redirect?) || 
+          (full_s3_resource && full_s3_resource.respond_to?(:website_redirect_location) && full_s3_resource.website_redirect_location)
+      end
+
+      def metadata_match?
+        redirect_match? && caching_policy_match?
+      end
+
+      def redirect_match?
+        if redirect?
+          redirect_url == remote_redirect_url
+        else
+          true
+        end
+      end
+
+      def shunned?
+        !!path[Regexp.union(options.ignore_paths)]
+      end
+
+      def remote_redirect_url
+        full_s3_resource&.website_redirect_location
+      end
+
+      def redirect_url
+        resource.respond_to?(:target_url) ? resource.target_url : nil
       end
 
       def directory?
@@ -186,7 +283,7 @@ module Middleman
       end
 
       def remote_object_md5
-        s3_resource.etag
+        s3_resource.etag.gsub(/"/, '') if s3_resource.etag
       end
 
       def encoding_match?
@@ -194,7 +291,9 @@ module Middleman
       end
 
       def remote_content_md5
-        full_s3_resource.metadata[CONTENT_MD5_KEY]
+        if full_s3_resource && full_s3_resource.metadata
+          full_s3_resource.metadata[CONTENT_MD5_KEY]
+        end
       end
 
       def local_object_md5
@@ -202,7 +301,13 @@ module Middleman
       end
 
       def local_content_md5
-        @local_content_md5 ||= Digest::MD5.hexdigest(File.read(original_path))
+        @local_content_md5 ||= begin
+          if File.exist?(original_path)
+            Digest::MD5.hexdigest(File.read(original_path))
+          else
+            nil
+          end
+        end
       end
 
       def original_path
@@ -211,7 +316,7 @@ module Middleman
 
       def content_type
         @content_type ||= Middleman::S3Sync.content_types[local_path]
-        @content_type ||= !resource.nil? ? resource.content_type : nil
+        @content_type ||= !resource.nil? && resource.respond_to?(:content_type) ? resource.content_type : nil
       end
 
       def caching_policy
@@ -219,7 +324,7 @@ module Middleman
       end
 
       def caching_policy_match?
-        if (caching_policy)
+        if caching_policy && full_s3_resource && full_s3_resource.respond_to?(:cache_control)
           caching_policy.cache_control == full_s3_resource.cache_control
         else
           true

data/lib/middleman/s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "4.0.3"
+    VERSION = "4.6.0"
   end
 end

data/lib/middleman/s3_sync.rb

@@ -1,12 +1,13 @@
-require 'fog/aws'
-require 'fog/aws/storage'
+require 'aws-sdk-s3'
 require 'digest/md5'
 require 'middleman/s3_sync/version'
 require 'middleman/s3_sync/options'
 require 'middleman/s3_sync/caching_policy'
 require 'middleman/s3_sync/status'
 require 'middleman/s3_sync/resource'
+require 'middleman/s3_sync/cloudfront'
 require 'middleman-s3_sync/extension'
+require 'middleman/redirect'
 require 'parallel'
 require 'ruby-progressbar'
 require 'thread'
@@ -24,12 +25,23 @@ module Middleman
       attr_accessor :mm_resources
       attr_reader   :app
 
+      THREADS_COUNT = 8
+      
+      # Track paths that were changed during sync for CloudFront invalidation
+      attr_accessor :invalidation_paths
+
       def sync()
         @app ||= ::Middleman::Application.new
+        @invalidation_paths = []
 
         say_status "Let's see if there's work to be done..."
         unless work_to_be_done?
           say_status "All S3 files are up to date."
+          
+          # Still run CloudFront invalidation if requested for all paths
+          if s3_sync_options.cloudfront_invalidate && s3_sync_options.cloudfront_invalidate_all
+            CloudFront.invalidate([], s3_sync_options)
+          end
           return
         end
 
@@ -43,13 +55,18 @@ module Middleman
         create_resources
         update_resources
         delete_resources
+        
+        # Invalidate CloudFront cache if requested
+        if s3_sync_options.cloudfront_invalidate
+          CloudFront.invalidate(@invalidation_paths, s3_sync_options)
+        end
       end
 
       def bucket
         @@bucket_lock.synchronize do
           @bucket ||= begin
-                        bucket = connection.directories.get(s3_sync_options.bucket, :prefix => s3_sync_options.prefix)
-                        raise "Bucket #{s3_sync_options.bucket} doesn't exist!" unless bucket
+                        bucket = s3_resource.bucket(s3_sync_options.bucket)
+                        raise "Bucket #{s3_sync_options.bucket} doesn't exist!" unless bucket.exists?
                         bucket
                       end
         end
@@ -58,6 +75,13 @@ module Middleman
       def add_local_resource(mm_resource)
         s3_sync_resources[mm_resource.destination_path] = S3Sync::Resource.new(mm_resource, remote_resource_for_path(mm_resource.destination_path)).tap(&:status)
       end
+      
+      def add_invalidation_path(path)
+        @invalidation_paths ||= []
+        # Normalize path for CloudFront (ensure it starts with /)
+        normalized_path = path.start_with?('/') ? path : "/#{path}"
+        @invalidation_paths << normalized_path unless @invalidation_paths.include?(normalized_path)
+      end
 
       def remote_only_paths
         paths - s3_sync_resources.keys
@@ -73,7 +97,12 @@ module Middleman
 
       protected
       def update_bucket_versioning
-        connection.put_bucket_versioning(s3_sync_options.bucket, "Enabled") if s3_sync_options.version_bucket
+        s3_client.put_bucket_versioning({
+          bucket: s3_sync_options.bucket,
+          versioning_configuration: {
+            status: "Enabled"
+          }
+        }) if s3_sync_options.version_bucket
       end
 
       def update_bucket_website
@@ -87,30 +116,47 @@ module Middleman
 
         unless opts.empty?
           say_status "Putting bucket website: #{opts.to_json}"
-          connection.put_bucket_website(s3_sync_options.bucket, opts)
+          s3_client.put_bucket_website({
+            bucket: s3_sync_options.bucket,
+            website_configuration: opts
+          })
         end
       end
 
-      def connection
-        connection_options = {
-          :region => s3_sync_options.region,
-          :path_style => s3_sync_options.path_style
-        }
+      def s3_client
+        @s3_client ||= Aws::S3::Client.new(connection_options)
+      end
 
-        if s3_sync_options.aws_access_key_id && s3_sync_options.aws_secret_access_key
-          connection_options.merge!({
-            :aws_access_key_id => s3_sync_options.aws_access_key_id,
-            :aws_secret_access_key => s3_sync_options.aws_secret_access_key
-          })
-        else
-          connection_options.merge!({ :use_iam_profile => true })
-        end
+      def s3_resource
+        @s3_resource ||= Aws::S3::Resource.new(client: s3_client)
+      end
+
+      def connection_options
+        @connection_options ||= begin
+          connection_options = {
+            endpoint: s3_sync_options.endpoint,
+            region: s3_sync_options.region,
+            force_path_style: s3_sync_options.path_style
+          }
+
+          if s3_sync_options.aws_access_key_id && s3_sync_options.aws_secret_access_key
+            connection_options.merge!({
+              access_key_id: s3_sync_options.aws_access_key_id,
+              secret_access_key: s3_sync_options.aws_secret_access_key
+            })
 
-        @connection ||= Fog::Storage::AWS.new(connection_options)
+            # If using an assumed role
+            connection_options.merge!({
+              session_token: s3_sync_options.aws_session_token
+            }) if s3_sync_options.aws_session_token
+          end
+
+          connection_options
+        end
       end
 
       def remote_resource_for_path(path)
-        bucket_files.find { |f| f.key == "#{s3_sync_options.prefix}#{path}" }
+        bucket_files[path]
       end
 
       def s3_sync_resources
@@ -125,7 +171,7 @@ module Middleman
 
       def remote_paths
         @remote_paths ||= if s3_sync_options.delete
-                            bucket_files.map(&:key)
+                            bucket_files.keys
                           else
                             []
                           end
@@ -133,42 +179,45 @@ module Middleman
 
       def bucket_files
         @@bucket_files_lock.synchronize do
-          @bucket_files ||= [].tap { |files|
-            bucket.files.each { |f|
-              files << f
-            }
-          }
+          @bucket_files ||= begin
+            files = {}
+            bucket.objects.each do |object|
+              files[object.key] = object
+            end
+            files
+          end
         end
       end
 
       def create_resources
-        files_to_create.each do |r|
-          r.create!
+        Parallel.map(files_to_create, in_threads: THREADS_COUNT) do |resource|
+          resource.create!
+          add_invalidation_path(resource.path)
         end
       end
 
       def update_resources
-        files_to_update.each do |r|
-          r.update!
+        Parallel.map(files_to_update, in_threads: THREADS_COUNT) do |resource|
+          resource.update!
+          add_invalidation_path(resource.path)
         end
       end
 
       def delete_resources
-        files_to_delete.each do |r|
-          r.destroy!
+        Parallel.map(files_to_delete, in_threads: THREADS_COUNT) do |resource|
+          resource.destroy!
+          add_invalidation_path(resource.path)
         end
       end
 
       def ignore_resources
-        files_to_ignore.each do |r|
-          r.ignore!
-        end
+        Parallel.map(files_to_ignore, in_threads: THREADS_COUNT, &:ignore!)
       end
 
       def work_to_be_done?
-        Parallel.each(mm_resources, in_threads: 8, progress: "Processing sitemap") { |mm_resource| add_local_resource(mm_resource) }
+        Parallel.each(mm_resources, in_threads: THREADS_COUNT, progress: "Processing sitemap") { |mm_resource| add_local_resource(mm_resource) }
 
-        Parallel.each(remote_only_paths, in_threads: 8, progress: "Processing remote files") do |remote_path|
+        Parallel.each(remote_only_paths, in_threads: THREADS_COUNT, progress: "Processing remote files") do |remote_path|
           s3_sync_resources[remote_path] ||= S3Sync::Resource.new(nil, remote_resource_for_path(remote_path)).tap(&:status)
         end
 

data/lib/middleman-s3_sync/commands.rb

@@ -65,13 +65,39 @@ module Middleman
                    type: :string,
                    desc: 'Print instrument messages.'
 
+      class_option :cloudfront_distribution_id,
+                   aliases: '-d',
+                   type: :string,
+                   desc: 'CloudFront distribution ID for invalidation.'
+
+      class_option :cloudfront_invalidate,
+                   aliases: '-c',
+                   type: :boolean,
+                   desc: 'Invalidate CloudFront cache after sync.'
+
+      class_option :cloudfront_invalidate_all,
+                   aliases: '-a',
+                   type: :boolean,
+                   desc: 'Invalidate all paths (/*) instead of only changed files.'
+
+      class_option :cloudfront_invalidation_batch_size,
+                   type: :numeric,
+                   desc: 'Maximum number of paths to invalidate in a single request (default: 1000).'
+
+      class_option :cloudfront_wait,
+                   aliases: '-w',
+                   type: :boolean,
+                   desc: 'Wait for CloudFront invalidation to complete before exiting.'
+
       def s3_sync
         env = options[:environment].to_s.to_sym
         verbose = options[:verbose] ? 0 : 1
         instrument = options[:instrument]
 
+        mode = options[:build] ? :build : :config
+
         ::Middleman::S3Sync.app = ::Middleman::Application.new do
-          config[:mode] = :build
+          config[:mode] = mode
           config[:environment] = env
           ::Middleman::Logger.singleton(verbose, instrument)
         end
@@ -97,6 +123,11 @@ module Middleman
           s3_sync_options.prefix = s3_sync_options.prefix.end_with?('/') ? s3_sync_options.prefix : s3_sync_options.prefix + '/'
         end
         s3_sync_options.dry_run = options[:dry_run] if options[:dry_run]
+        s3_sync_options.cloudfront_distribution_id = options[:cloudfront_distribution_id] if options[:cloudfront_distribution_id]
+        s3_sync_options.cloudfront_invalidate = options[:cloudfront_invalidate] if options[:cloudfront_invalidate]
+        s3_sync_options.cloudfront_invalidate_all = options[:cloudfront_invalidate_all] if options[:cloudfront_invalidate_all]
+        s3_sync_options.cloudfront_invalidation_batch_size = options[:cloudfront_invalidation_batch_size] if options[:cloudfront_invalidation_batch_size]
+        s3_sync_options.cloudfront_wait = options[:cloudfront_wait] if options[:cloudfront_wait]
 
         ::Middleman::S3Sync.sync()
       end

data/lib/middleman-s3_sync/extension.rb

@@ -9,9 +9,11 @@ module Middleman
     option :http_prefix, nil, 'Path prefix of the resources'
     option :acl, 'public-read', 'ACL for the resources being pushed to S3'
     option :bucket, nil, 'The name of the bucket we are pushing to.'
+    option :endpoint, nil, 'The name of the endpoint to use - useful when using S3 compatible storage'
     option :region, 'us-east-1', 'The name of the AWS region hosting the S3 bucket'
     option :aws_access_key_id, ENV['AWS_ACCESS_KEY_ID'] , 'The AWS access key id'
     option :aws_secret_access_key, ENV['AWS_SECRET_ACCESS_KEY'], 'The AWS secret access key'
+    option :aws_session_token, ENV['AWS_SESSION_TOKEN'] || ENV['AWS_SECURITY_TOKEN'], 'The AWS session token (for assuming roles)'
     option :after_build, false, 'Whether to synchronize right after the build'
     option :build_dir, nil, 'Where the built site is stored'
     option :delete, true, 'Whether to delete resources that do not have a local equivalent'
@@ -26,6 +28,12 @@ module Middleman
     option :index_document, nil, 'S3 custom index document path'
     option :error_document, nil, 'S3 custom error document path'
     option :content_types, {}, 'Custom content types'
+    option :ignore_paths, [], 'Paths that should be ignored during sync, strings or regex are allowed'
+    option :cloudfront_distribution_id, nil, 'CloudFront distribution ID for invalidation'
+    option :cloudfront_invalidate, false, 'Whether to invalidate CloudFront cache after sync'
+    option :cloudfront_invalidate_all, false, 'Whether to invalidate all paths (/*) or only changed files'
+    option :cloudfront_invalidation_batch_size, 1000, 'Maximum number of paths to invalidate in a single request'
+    option :cloudfront_wait, false, 'Whether to wait for CloudFront invalidation to complete'
 
     expose_to_config :s3_sync_options, :default_caching_policy, :caching_policy
 
@@ -40,6 +48,7 @@ module Middleman
       read_config
       options.aws_access_key_id ||= ENV['AWS_ACCESS_KEY_ID']
       options.aws_secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY']
+      options.aws_session_token ||= ENV['AWS_SESSION_TOKEN'] || ENV['AWS_SECURITY_TOKEN']
       options.bucket ||= ENV['AWS_BUCKET']
       options.http_prefix = app.http_prefix if app.respond_to? :http_prefix
       options.build_dir ||= app.build_dir if app.respond_to? :build_dir
@@ -54,8 +63,15 @@ module Middleman
       ::Middleman::S3Sync.sync() if options.after_build
     end
 
-    def manipulate_resource_list(mm_resources)
-      ::Middleman::S3Sync.mm_resources = mm_resources
+    def manipulate_resource_list(resources)
+      ::Middleman::S3Sync.mm_resources = resources.each_with_object([]) do |resource, list|
+        next if resource.ignored?
+
+        list << resource
+        list << resource.target_resource if resource.respond_to?(:target_resource)
+      end
+
+      resources
     end
 
     def s3_sync_options
@@ -73,7 +89,7 @@ module Middleman
         config_file_path = File.join(root_path, ".s3_sync")
 
         # skip if config file does not exist
-        return unless File.exists?(config_file_path)
+        return unless File.exist?(config_file_path)
 
         io = File.open(config_file_path, "r")
       end

data/lib/middleman-s3_sync.rb

@@ -1,3 +1,4 @@
+require 'logger'
 require 'middleman-core'
 require 'middleman-s3_sync/commands'
 require 'middleman/s3_sync'

data/middleman-s3_sync.gemspec

@@ -18,22 +18,26 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency 'middleman-core', '>= 4.0.0'
+  gem.add_runtime_dependency 'middleman-core'
   gem.add_runtime_dependency 'middleman-cli'
   gem.add_runtime_dependency 'unf'
-  gem.add_runtime_dependency 'fog-aws', '>= 0.1.1'
+  gem.add_runtime_dependency 'aws-sdk-s3'
+  gem.add_runtime_dependency 'aws-sdk-cloudfront'
   gem.add_runtime_dependency 'map'
   gem.add_runtime_dependency 'parallel'
   gem.add_runtime_dependency 'ruby-progressbar'
   gem.add_runtime_dependency 'ansi', '~> 1.5.0'
+  gem.add_runtime_dependency 'mime-types', '~> 3.1'
+  gem.add_runtime_dependency 'base64'
+  gem.add_runtime_dependency 'nokogiri', '>= 1.18.4'
 
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'pry-byebug'
-  gem.add_development_dependency 'rspec', '>= 3.0.0'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rspec-support'
   gem.add_development_dependency 'rspec-its'
   gem.add_development_dependency 'rspec-mocks'
   gem.add_development_dependency 'timerizer'
-  gem.add_development_dependency 'travis'
-  gem.add_development_dependency 'travis-lint'
+  gem.add_development_dependency 'webrick'
 end