middleman-s3_sync 3.0.10 → 3.0.11

data/Changelog.md

@@ -0,0 +1,10 @@
+# Middleman::S3Sync Changelog
+
+## v3.0.11
+
+* Adds support for GZipped resources (fixes #3)
+* Quiets Fog's warning messages (fixes #10)
+* Rename the options method to s3_sync_options to remove a method name collision (fixes #9)
+* Colorize the output.
+
+

data/README.md

@@ -1,5 +1,8 @@
 # Middleman::S3Sync
 
+[![Code
+Climate](https://codeclimate.com/github/fredjean/middleman-s3_sync.png)](https://codeclimate.com/github/fredjean/middleman-s3_sync)
+
 This gem determines which files need to be added, updated and optionally deleted
 and only transfer these files up. This reduces the impact of an update
 on a web site hosted on S3.
@@ -38,6 +41,7 @@ activate :s3_sync do |s3_sync|
   s3_sync.aws_secret_access_key = 'AWS SECRET KEY'
   s3_sync.delete                = false # We delete stray files by default.
   s3_sync.after_build           = false # We chain after the build step by default. This may not be your desired behavior...
+  s3_sync.prefer_gzip           = true
 end
 ```
 
@@ -50,6 +54,13 @@ such case, you can pass the ```--force``` option:
 
     $ middleman s3_sync --force
 
+## Overriding the destination bucket
+
+You can now override the destination bucket using the --bucket switch.
+The command is:
+
+    $ middleman s3_sync --bucket=my.new.bucket
+
 ## HTTP Caching
 
 By default, ```middleman-s3_sync``` does not set caching headers. In
@@ -121,6 +132,14 @@ are (I'm looking at you Chrome!). Setting the `Cache-Control` or
 that stand between them and your content will behave the way you want
 them to. YMMV.
 
+### GZipped Content Encoding
+
+You can set the ```prefer_gzip``` option to look for a gzipped version
+of a resource. The gzipped version of the resource will be pushed to S3
+instead of the original and the ```Content-Encoding``` and ```Content-Type```
+headers will be set correctly. This will cause Amazon to serve the
+compressed version of the resource.
+
 ## A Debt of Gratitude
 
 I used Middleman Sync as a template for building a Middleman extension.

data/lib/middleman-s3_sync.rb

@@ -1,9 +1,14 @@
 require 'middleman-core'
 require 'fog'
-require 'parallel'
+require 'pmap'
+require 'ruby-progressbar'
 require 'digest/md5'
 require 'middleman-s3_sync/version'
 require 'middleman-s3_sync/commands'
+require 'middleman/s3_sync/status'
+require 'middleman/s3_sync/resource'
+
+Fog::Logger[:warning] = nil
 
 ::Middleman::Extensions.register(:s3_sync, '>= 3.0.0') do
   require 'middleman-s3_sync/extension'
@@ -13,140 +18,112 @@ end
 module Middleman
   module S3Sync
     class << self
+      include Status
+
       def sync
-        if files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?
-          puts "\nAll S3 files are up to date."
+        unless work_to_be_done?
+          say_status "\nAll S3 files are up to date."
           return
         end
 
-        puts "\nReady to apply updates to #{options.bucket}."
-
-        files_to_create.each do |f|
-          puts "Creating #{f}"
-          file_hash = {
-            :key => f,
-            :body => File.open(local_path(f)),
-            :public => true,
-            :acl => 'public-read',
-            :content_type => MIME::Types.of(f).first
-          }
-
-          # Add cache-control headers
-          if policy = options.caching_policy_for(file_hash[:content_type])
-            file_hash[:cache_control] = policy.cache_control if policy.cache_control
-            file_hash[:expires] = policy.expires if policy.expires
-          end
-
-          bucket.files.create(file_hash)
-        end
+        say_status "\nReady to apply updates to #{s3_sync_options.bucket}."
 
-        files_to_update.each do |f|
-          puts "Updating #{f}"
-          file = s3_files.get(f)
-          file.body = File.open(local_path(f))
-          file.public = true
-          file.content_type = MIME::Types.of(f).first
-          if policy = options.caching_policy_for(file.content_type)
-            file.cache_control = policy.cache_control if policy.cache_control
-            file.expires = policy.expires if policy.expires
-          end
-
-          file.save
-        end
+        create_resources
+        update_resources
+        delete_resources
+      end
 
-        files_to_delete.each do |f|
-          puts "Deleting #{f}"
-          if file = s3_files.get(f)
-            file.destroy
-          end
-        end
+      def bucket
+        @bucket ||= connection.directories.get(s3_sync_options.bucket)
       end
 
       protected
       def connection
         @connection ||= Fog::Storage.new({
           :provider => 'AWS',
-          :aws_access_key_id => options.aws_access_key_id,
-          :aws_secret_access_key => options.aws_secret_access_key,
-          :region => options.region
+          :aws_access_key_id => s3_sync_options.aws_access_key_id,
+          :aws_secret_access_key => s3_sync_options.aws_secret_access_key,
+          :region => s3_sync_options.region
         })
       end
 
-      def bucket
-        @bucket ||= connection.directories.get(options.bucket)
+      def resources
+        @resources ||= paths.pmap do |p|
+          progress_bar.increment
+          S3Sync::Resource.new(p)
+        end
       end
 
-      def s3_files
-        @s3_files ||= bucket.files
+      def progress_bar
+        @progress_bar ||= ProgressBar.create(total: paths.length)
       end
 
-      def remote_files
-        @remote_files ||= begin
-                            puts "Gathering remote files from #{options.bucket}"
-                            bucket.files.map { |f| f.key }
-                          end
+      def paths
+        @paths ||= begin
+                     say_status "Gathering the paths to evaluate."
+                     (remote_paths + local_paths).uniq.sort
+                   end
       end
-      def local_files
-        @local_files ||= begin
-                           puts "Gathering local files."
-                           (Dir[build_dir + "/**/*"] + Dir[build_dir + "/**/.*"])
-                             .reject { |f| File.directory?(f) }
-                             .map { |f| f.gsub(/^#{build_dir}\//, '') }
+
+      def local_paths
+        @local_paths ||= begin
+                           local_paths = (Dir[build_dir + "/**/*"] + Dir[build_dir + "/**/.*"])
+                                           .reject { |p| File.directory?(p) }
+
+                           if s3_sync_options.prefer_gzip
+                             local_paths.reject! { |p| p =~ /\.gz$/ && File.exist?(p.gsub(/\.gz$/, '')) }
+                           end
+
+                           local_paths.pmap { |p| p.gsub(/#{build_dir}\//, '') }
                          end
       end
 
-      def files_to_delete
-        @files_to_delete ||= begin
-                               if options.delete
-                                 puts "\nDetermine which files to delete from #{options.bucket}"
-                                 remote_files - local_files
-                               else
-                                 []
-                               end
-                             end
+      def remote_paths
+        @remote_paths ||= bucket.files.map{ |f| f.key }
       end
 
-      def files_to_create
-        @files_to_create ||= begin
-                               puts "Determine files to add to #{options.bucket}."
-                               local_files - remote_files
-                             end
+      def create_resources
+        files_to_create.each do |r|
+          r.create!
+        end
       end
 
-      def files_to_evaluate
-        @files_to_evaluate ||= begin
-                                 local_files - files_to_create
-                               end
+      def update_resources
+        files_to_update.each do |r|
+          r.update!
+        end
       end
 
-      def files_to_update
-        return files_to_evaluate if options.force
-
-        @files_to_update ||= begin
-                               puts "Determine which local files to update their S3 counterparts"
-                               files_to_update = []
-                               Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
-                                 print '.'
-                                 remote_file = s3_files.get(f)
-                                 local_mtime = File.mtime(local_path(f))
-                                 remote_mtime = remote_file.last_modified
-                                 if remote_mtime < local_mtime
-                                   local_md5 = Digest::MD5.hexdigest(File.read(local_path(f)))
-                                   remote_md5 = remote_file.etag
-                                   files_to_update << f if local_md5 != remote_md5
-                                 end
-                               end
-                               puts ""
-                               files_to_update
+      def delete_resources
+        files_to_delete.each do |r|
+          r.destroy!
+        end
+      end
+
+      def work_to_be_done?
+        !(files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?)
+      end
+
+      def files_to_delete
+        @files_to_delete ||= if s3_sync_options.delete
+                                 resources.select { |r| r.to_delete? }
+                             else
+                               []
                              end
       end
 
-      def local_path(f)
-        "#{build_dir}/#{f}"
+      def files_to_create
+        @files_to_create ||= resources.select { |r| r.to_create? }
+      end
+
+      def files_to_update
+        return resources.select { |r| r.local? } if s3_sync_options.force
+
+        @files_to_update ||= resources.select { |r| r.to_update? }
       end
 
       def build_dir
-        @build_dir ||= options.build_dir
+        @build_dir ||= s3_sync_options.build_dir
       end
     end
   end

data/lib/middleman-s3_sync/commands.rb

@@ -18,16 +18,21 @@ module Middleman
       option :bucket, type: :string,
                       desc: "Specify which bucket to use, overrides the configured bucket.",
                       aliases: :b
+      option :verbose, type: :boolean,
+                       desc: "Adds more verbosity...",
+                       aliases: :v
 
       def s3_sync
         shared_inst = ::Middleman::Application.server.inst
-        bucket = shared_inst.options.bucket
+        bucket = shared_inst.s3_sync_options.bucket
         if (!bucket)
           raise Thor::Error.new "You need to activate this extension."
         end
 
-        shared_inst.options.force = options[:force] if options[:force]
-        shared_inst.options.bucket = options[:bucket] if options[:bucket]
+        # Override options based on what was passed on the command line...
+        shared_inst.s3_sync_options.force = options[:force] if options[:force]
+        shared_inst.s3_sync_options.bucket = options[:bucket] if options[:bucket]
+        shared_inst.s3_sync_options.verbose = options[:verbose] if options[:verbose]
 
         ::Middleman::S3Sync.sync
       end

data/lib/middleman-s3_sync/extension.rb

@@ -13,7 +13,9 @@ module Middleman
       :delete,
       :existing_remote_file,
       :build_dir,
-      :force
+      :force,
+      :prefer_gzip,
+      :verbose
     )
 
       def add_caching_policy(content_type, options)
@@ -66,7 +68,7 @@ module Middleman
     end
 
     class << self
-      def options
+      def s3_sync_options
         @@options
       end
 
@@ -90,16 +92,16 @@ module Middleman
       alias :included :registered
 
       module Helpers
-        def options
-          ::Middleman::S3Sync.options
+        def s3_sync_options
+          ::Middleman::S3Sync.s3_sync_options
         end
 
         def default_caching_policy(policy = {})
-          options.add_caching_policy(:default, policy)
+          s3_sync_options.add_caching_policy(:default, policy)
         end
 
         def caching_policy(content_type, policy = {})
-          options.add_caching_policy(content_type, policy)
+          s3_sync_options.add_caching_policy(content_type, policy)
         end
       end
     end

data/lib/middleman-s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "3.0.10"
+    VERSION = "3.0.11"
   end
 end

data/lib/middleman/s3_sync/resource.rb

@@ -0,0 +1,168 @@
+module Middleman
+  module S3Sync
+    class Resource
+      attr_accessor :path, :s3_resource, :content_type, :gzipped
+
+      CONTENT_MD5_KEY = 'x-amz-meta-content-md5'
+
+      include Status
+
+      def initialize(path)
+        @path = path
+        @s3_resource = bucket.files.get(path) rescue nil
+      end
+
+      def remote_path
+        s3_resource ? s3_resource.key : path
+      end
+      alias :key :remote_path
+
+      def to_h
+        attributes = {
+          :key => key,
+          :body => body,
+          :public => true,
+          :acl => 'public-read',
+          :content_type => content_type,
+          CONTENT_MD5_KEY => content_md5
+        }
+
+        if caching_policy
+          attributes[:cache_control] = caching_policy.cache_control
+          attributes[:expires] = caching_policy.expires
+        end
+
+        if options.prefer_gzip && gzipped
+          attributes[:content_encoding] = "gzip"
+        end
+
+        attributes
+      end
+      alias :attributes :to_h
+
+      def update!
+        say_status "Updating".blue + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
+        if options.verbose
+          say_status "Original:    #{original_path.white}"
+          say_status "Local Path:  #{local_path.white}"
+          say_status "remote md5:  #{remote_md5.white}"
+          say_status "content md5: #{content_md5.white}"
+        end
+        s3_resource.body = body
+        s3_resource.public = true
+        s3_resource.acl = 'public-read'
+        s3_resource.content_type = content_type
+        s3_resource.metadata = { CONTENT_MD5_KEY => content_md5 }
+
+        if caching_policy
+          s3_resource.cache_control = caching_policy.cache_control
+          s3_resource.expires = caching_policy.expires
+        end
+
+        if options.prefer_gzip && gzipped
+          s3_resource.content_encoding = "gzip"
+        end
+
+        s3_resource.save
+      end
+
+      def local_path
+        local_path = build_dir + '/' + path
+        if options.prefer_gzip && File.exist?(local_path + ".gz")
+          @gzipped = true
+          local_path += ".gz"
+        end
+        local_path
+      end
+
+      def destroy!
+        say_status "Deleting".red + " #{path}".red
+        s3_resource.destroy
+      end
+
+      def create!
+        say_status "Creating".green + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
+        bucket.files.create(to_h)
+      end
+
+      def to_delete?
+        status == :deleted
+      end
+
+      def to_create?
+        status == :new
+      end
+
+      def identical?
+        status == :identical
+      end
+
+      def to_update?
+        status == :updated
+      end
+
+      def body
+        @body = File.open(local_path)
+      end
+
+      def status
+        @status ||= if local? && remote?
+                      if content_md5 != remote_md5
+                        :updated
+                      else
+                        :identical
+                      end
+                    elsif local?
+                      :new
+                    else
+                      :deleted
+                    end
+      end
+
+      def local?
+        File.exist?(local_path)
+      end
+
+      def remote?
+        s3_resource
+      end
+
+      def relative_path
+        @relative_path ||= local_path.gsub(/#{build_dir}/, '')
+      end
+
+      def remote_md5
+        s3_resource.metadata[CONTENT_MD5_KEY] || s3_resource.etag
+      end
+
+      def content_md5
+        @content_md5 ||= Digest::MD5.hexdigest(File.read(original_path))
+      end
+
+      def original_path
+        gzipped ? local_path.gsub(/\.gz$/, '') : local_path
+      end
+
+      def content_type
+        @content_type ||= MIME::Types.of(path).first
+      end
+
+      def caching_policy
+        @caching_policy ||= options.caching_policy_for(content_type)
+      end
+
+      protected
+      def bucket
+        Middleman::S3Sync.bucket
+      end
+
+      def build_dir
+        options.build_dir
+      end
+
+      def options
+        Middleman::S3Sync.s3_sync_options
+      end
+    end
+  end
+end

data/lib/middleman/s3_sync/status.rb

@@ -0,0 +1,11 @@
+require 'colorize'
+
+module Middleman
+  module S3Sync
+    module Status
+      def say_status(status)
+        puts :s3_sync.to_s.rjust(12).green + "  #{status}"
+      end
+    end
+  end
+end

data/middleman-s3_sync.gemspec

@@ -19,8 +19,10 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'middleman-core', '>= 3.0.0'
   gem.add_runtime_dependency 'fog'
-  gem.add_runtime_dependency 'parallel'
   gem.add_runtime_dependency 'map'
+  gem.add_runtime_dependency 'pmap'
+  gem.add_runtime_dependency 'ruby-progressbar'
+  gem.add_runtime_dependency 'colorize'
 
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'pry'

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: middleman-s3_sync
 version: !ruby/object:Gem::Version
+  version: 3.0.11
   prerelease: 
-  version: 3.0.10
 platform: ruby
 authors:
 - Frederic Jean
@@ -10,33 +10,42 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-17 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  version_requirements: !ruby/object:Gem::Requirement
+  name: middleman-core
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: middleman-core
+  type: :runtime
   prerelease: false
-  requirement: !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  version_requirements: !ruby/object:Gem::Requirement
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: fog
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: map
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -44,15 +53,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
-- !ruby/object:Gem::Dependency
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: parallel
-  prerelease: false
+- !ruby/object:Gem::Dependency
+  name: pmap
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -60,15 +69,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
-- !ruby/object:Gem::Dependency
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: map
-  prerelease: false
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -76,15 +85,31 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   name: rake
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -92,15 +117,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
-- !ruby/object:Gem::Dependency
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   name: pry
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -108,15 +133,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
-- !ruby/object:Gem::Dependency
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   name: pry-nav
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -124,6 +149,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Only syncs files that have been updated to S3.
 email:
 - fred@fredjean.net
@@ -132,6 +164,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- Changelog.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -140,6 +173,8 @@ files:
 - lib/middleman-s3_sync/commands.rb
 - lib/middleman-s3_sync/extension.rb
 - lib/middleman-s3_sync/version.rb
+- lib/middleman/s3_sync/resource.rb
+- lib/middleman/s3_sync/status.rb
 - lib/middleman_extension.rb
 - middleman-s3_sync.gemspec
 homepage: http://github.com/fredjean/middleman-s3_sync
@@ -153,19 +188,19 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
+      version: '0'
       segments:
       - 0
-      hash: 2846065098887891118
-      version: '0'
+      hash: 963813138401624681
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
+      version: '0'
       segments:
       - 0
-      hash: 2846065098887891118
-      version: '0'
+      hash: 963813138401624681
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23