middleman-s3_sync 3.0.21 → 3.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e8a5cb2c096e224e6c21b405cb142a157711dac
-  data.tar.gz: 5b906c2f9d25530a297a1a8d2da9b8e8ad147681
+  metadata.gz: bccd80d4f353aef53c1cead353fc6939f6b5eaa5
+  data.tar.gz: dc63b6bbcca6a99d0e01796681124872c37f472b
 SHA512:
-  metadata.gz: 757e5a276590f998ef30a67367e64be4ed8544380ceab2fe431e9f4ad6212522113924c792b1cb713c41550d5979fa37a4784ad45dc4d8a368f9210eca8a8a85
-  data.tar.gz: cc64348884fc3a0d94879f173f7ceee02282670f098a6798d6435565d2f676d0174ebaa46c5f8c4807a32ff60011dee28c02dc502d3ee4feddbbadac42aece97
+  metadata.gz: 39e3a98584cf44195741c2e839f278f7442ddfa015ee7a3a7c5ca3e021ca7b22cc88f2cd465647bf413535b6cb70e9653dfb4d031ed131e9703f7af4de518f20
+  data.tar.gz: 7a4b0c20fa87bc333bdd85856401e009ae9de9020f7be96080114ee6836e9e740082b4e299020f867eb2bf73d5714b7cb7244ac02be7f8f224f97b82ea91e284

data/Changelog.md

@@ -2,6 +2,12 @@
 
 The gem that tries really hard not to push files to S3.
 
+## v3.0.22
+
+* Fixes a bug where files were not closed, leading to an exhaustion of 
+  file handles with large web sites.
+* Internal fixes.
+
 ## v3.0.17
 
 * Limits the number of concurrent threads used while processing the

data/README.md

@@ -100,6 +100,8 @@ A sample ```.s3_sync``` file is included at the root of this repo.
 You can also pass the credentials through environment variables. They
 map to the following values:
 
+| Setting               | Environment Variable               |
+| --------------------- | ---------------------------------- |
 | aws_access_key_id     | ```ENV['AWS_ACCESS_KEY_ID```       |
 | aws_secret_access_key | ```ENV['AWS_SECRET_ACCESS_KEY']``` |
 

data/lib/middleman/s3_sync/resource.rb

@@ -1,15 +1,24 @@
 module Middleman
   module S3Sync
     class Resource
-      attr_accessor :path, :s3_resource, :content_type, :gzipped
+      attr_accessor :path, :partial_s3_resource, :content_type, :gzipped
 
       CONTENT_MD5_KEY = 'x-amz-meta-content-md5'
 
       include Status
+      
+      def s3_resource
+        @full_s3_resource || @partial_s3_resource
+      end
+
+      # S3 resource as returned by a HEAD request
+      def full_s3_resource
+        @full_s3_resource ||= bucket.files.head(path)
+      end
 
-      def initialize(path)
+      def initialize(path, partial_s3_resource)
         @path = path
-        @s3_resource = bucket.files.head(path)
+        @partial_s3_resource = partial_s3_resource
       end
 
       def remote_path
@@ -20,10 +29,9 @@ module Middleman
       def to_h
         attributes = {
           :key => key,
-          :body => body,
           :acl => options.acl,
           :content_type => content_type,
-          CONTENT_MD5_KEY => content_md5
+          CONTENT_MD5_KEY => local_content_md5
         }
 
         if caching_policy
@@ -48,36 +56,39 @@ module Middleman
       alias :attributes :to_h
 
       def update!
-        say_status "Updating".blue + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
-        if options.verbose
-          say_status "Original:    #{original_path.white}"
-          say_status "Local Path:  #{local_path.white}"
-          say_status "remote md5:  #{remote_md5.white}"
-          say_status "content md5: #{content_md5.white}"
-        end
-        s3_resource.body = body
-        s3_resource.acl = options.acl
-        s3_resource.content_type = content_type
-        s3_resource.metadata = { CONTENT_MD5_KEY => content_md5 }
-
-        if caching_policy
-          s3_resource.cache_control = caching_policy.cache_control
-          s3_resource.expires = caching_policy.expires
-        end
-
-        if options.prefer_gzip && gzipped
-          s3_resource.content_encoding = "gzip"
-        end
-
-        if options.reduced_redundancy_storage
-          s3_resource.storage_class = 'REDUCED_REDUNDANCY'
-        end
-
-        if options.encryption
-          s3_resource.encryption = 'AES256'
-        end
-
-        s3_resource.save
+        body { |body|
+          say_status "Updating".blue + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
+          if options.verbose
+            say_status "Original:    #{original_path.white}"
+            say_status "Local Path:  #{local_path.white}"
+            say_status "remote md5:  #{remote_object_md5.white} / #{remote_content_md5}"
+            say_status "content md5: #{local_object_md5.white} / #{local_content_md5}"
+          end
+          s3_resource.body = body
+
+          s3_resource.acl = options.acl
+          s3_resource.content_type = content_type
+          s3_resource.metadata = { CONTENT_MD5_KEY => local_content_md5 }
+
+          if caching_policy
+            s3_resource.cache_control = caching_policy.cache_control
+            s3_resource.expires = caching_policy.expires
+          end
+
+          if options.prefer_gzip && gzipped
+            s3_resource.content_encoding = "gzip"
+          end
+
+          if options.reduced_redundancy_storage
+            s3_resource.storage_class = 'REDUCED_REDUNDANCY'
+          end
+
+          if options.encryption
+            s3_resource.encryption = 'AES256'
+          end
+
+          s3_resource.save
+        }
       end
 
       def local_path
@@ -91,7 +102,7 @@ module Middleman
 
       def destroy!
         say_status "Deleting".red + " #{path}"
-        s3_resource.destroy
+        bucket.files.destroy remote_path
       end
 
       def create!
@@ -99,9 +110,11 @@ module Middleman
         if options.verbose
           say_status "Original:    #{original_path.white}"
           say_status "Local Path:  #{local_path.white}"
-          say_status "content md5: #{content_md5.white}"
+          say_status "content md5: #{local_content_md5.white}"
         end
-        bucket.files.create(to_h)
+        body { |body|
+          bucket.files.create(to_h.merge(:body => body))
+        }
       end
 
       def ignore!
@@ -109,6 +122,8 @@ module Middleman
                    :redirect
                  elsif directory?
                    :directory
+                 elsif alternate_encoding?
+                   'alternate encoding'
                  end
         say_status "Ignoring".yellow + " #{path} #{ reason ? "(#{reason})".white : "" }"
       end
@@ -120,6 +135,10 @@ module Middleman
       def to_create?
         status == :new
       end
+      
+      def alternate_encoding?
+        status == :alternate_encoding
+      end
 
       def identical?
         status == :identical
@@ -130,21 +149,36 @@ module Middleman
       end
 
       def to_ignore?
-        status == :ignored
+        status == :ignored || status == :alternate_encoding
       end
 
-      def body
-        @body = File.open(local_path)
+      def body(&block)
+        File.open(local_path, &block)
       end
 
       def status
         @status ||= if directory?
-                      :ignored
-                    elsif local? && remote?
-                      if content_md5 != remote_md5
-                        :updated
+                      if remote?
+                        :deleted
                       else
+                        :ignored
+                      end
+                    elsif local? && remote?
+                      if local_object_md5 == remote_object_md5
                         :identical
+                      else
+                        if !gzipped
+                          # we're not gzipped, object hashes being different indicates updated content
+                          :updated
+                        elsif (local_content_md5 != remote_content_md5)
+                          # we're gzipped, so we checked the content MD5, and it also changed
+                          :updated
+                        else
+                          # we're gzipped, the object hashes differ, but the content hashes are equal
+                          # this means the gzipped bits changed while the compressed bits did not
+                          # what's more, we spent a HEAD request to find out
+                          :alternate_encoding
+                        end
                       end
                     elsif local?
                       :new
@@ -154,7 +188,7 @@ module Middleman
                       :deleted
                     end
       end
-
+      
       def local?
         File.exist?(local_path)
       end
@@ -174,13 +208,21 @@ module Middleman
       def relative_path
         @relative_path ||= local_path.gsub(/#{build_dir}/, '')
       end
-
-      def remote_md5
-        s3_resource.metadata[CONTENT_MD5_KEY] || s3_resource.etag
+      
+      def remote_object_md5
+        s3_resource.etag
+      end
+      
+      def remote_content_md5
+        full_s3_resource.metadata[CONTENT_MD5_KEY]
       end
 
-      def content_md5
-        @content_md5 ||= Digest::MD5.hexdigest(File.read(original_path))
+      def local_object_md5
+        @local_object_md5 ||= Digest::MD5.hexdigest(File.read(local_path))
+      end
+      
+      def local_content_md5
+        @local_content_md5 ||= Digest::MD5.hexdigest(File.read(original_path))
       end
 
       def original_path

data/lib/middleman/s3_sync/version.rb

@@ -1,5 +1,5 @@
 module Middleman
   module S3Sync
-    VERSION = "3.0.21"
+    VERSION = "3.0.22"
   end
 end

data/lib/middleman/s3_sync.rb

@@ -46,7 +46,7 @@ module Middleman
       def resources
         @resources ||= paths.pmap(32) do |p|
           progress_bar.increment
-          S3Sync::Resource.new(p)
+          S3Sync::Resource.new(p, bucket_files.find { |f| f.key == p }).tap(&:status)
         end
       end
 
@@ -75,7 +75,15 @@ module Middleman
       end
 
       def remote_paths
-        @remote_paths ||= bucket.files.map{ |f| f.key }
+        @remote_paths ||= bucket_files.map(&:key)
+      end
+      
+      def bucket_files
+        @bucket_files ||= [].tap { |files|
+          bucket.files.each { |f|
+            files << f
+          }
+        }
       end
 
       def create_resources

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: middleman-s3_sync
 version: !ruby/object:Gem::Version
-  version: 3.0.21
+  version: 3.0.22
 platform: ruby
 authors:
 - Frederic Jean
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-11 00:00:00.000000000 Z
+date: 2013-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: middleman-core
@@ -256,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Tries really, really hard not to push files to S3.