filey-diff 1.4.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d859b05f01d263d190fe2458c2b2ffc4b58395ce
-  data.tar.gz: 3fbda83260167397c8cd1bb7b71ff63b63103394
+  metadata.gz: cc23f9bf625a91c7d80733abc57047d431ff600a
+  data.tar.gz: 2f6f4ec21efb418cf0e5b9e704d34fe5d26908e5
 SHA512:
-  metadata.gz: 78cccad67134289ccf0be21024ba3f37e90abd7bac9278e830bdd14c16974f633a21672923fb7834fe695504585095abd61ad5d01258c321c642ebba7a2ea000
-  data.tar.gz: 7210db02b36df164b4684f2492f45f01fbdb62e37ea0e3ce55172b004dc43adcf6f3c21259c17af0faa2f80ab4e4faa6d60261c9066d2e3a11022354bf304007
+  metadata.gz: 7e37c4f64dda53464693dc2f5b670a1059a2b08867634f36556406083ba0518f556613365da7a9dbd09fa211d273b865ae84f171cba916cca5323da46a1781b7
+  data.tar.gz: c5109d899be4af7c5fa12566cce6a96c2232b2d3abcf9b2fe8aa6cc5e2f10409736ab2662166faaa3f933a886e67e4b201cd278878436927fae3596fefd6ba6c

data/changelog.md

@@ -2,6 +2,14 @@
 
 This project is [Semantically Versioned](http://semver.org).
 
+## 2.0.0
+
+* Improve performance significantly (see https://github.com/laurilehmijoki/s3_website/issues/44)
+
+#### Non-backwards compatible changes:
+* Calculate MD5 of gzipped S3 object using its gzipped content
+  (previously MD5 was always calculated using raw, non-gzipped file content).
+
 ## 1.4.3
 
 * Fix issue <https://github.com/laurilehmijoki/filey-diff/pull/8>

data/filey-diff.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'filey-diff'
-  s.version           = '1.4.3'
+  s.version           = '2.0.0'
 
   s.summary     = "Compare two data sources that contain file-like objects"
   s.description =

data/lib/filey-diff/comparison.rb

@@ -2,33 +2,29 @@ module Filey
   class Comparison
     def self.list_outdated(data_source_a, data_source_b)
       select_in_outer_array(data_source_b, data_source_a) { |b_item, a_item|
-        b_item.full_path == a_item.full_path and
-          b_item.last_modified < a_item.last_modified
+        !a_item.nil? && b_item.last_modified < a_item.last_modified
       }
     end
 
     def self.list_changed(data_source_a, data_source_b)
       select_in_outer_array(data_source_b, data_source_a) { |b_item, a_item|
-        b_item.full_path == a_item.full_path and
-          b_item.md5 != a_item.md5
+        !a_item.nil? && b_item.md5 != a_item.md5
       }
     end
 
     def self.list_missing(data_source_a, data_source_b)
-      intersection = select_in_outer_array(data_source_a, data_source_b) do
-        |a_item, b_item|
-        b_item.full_path == a_item.full_path
-      end
+      intersection = select_in_outer_array(data_source_a, data_source_b) { |a_item, b_item|
+        !b_item.nil?
+      }
       data_source_a.get_fileys - intersection
     end
 
     private
 
     def self.select_in_outer_array(outer, inner)
+      inner_hash = inner.get_fileys.reduce({}) { |h, e| h[e.full_path] = e; h }
       outer.get_fileys.select { |outer_item|
-        inner.get_fileys.select { |inner_item|
-          yield outer_item, inner_item
-        }.length > 0
+        yield outer_item, inner_hash[outer_item.full_path]
       }
     end
   end

data/lib/filey-diff/data-sources/aws_sdk_s3.rb

@@ -54,13 +54,7 @@ module Filey
           name = s3_object.key
         end
 
-        if (s3_object.head[:content_encoding] == "gzip")
-          last_modified, md5 = last_modified_and_md5_from_gzipped(
-            s3_object, path
-          )
-        else
-          last_modified, md5 = last_modified_and_md5(s3_object)
-        end
+        last_modified, md5 = last_modified_and_md5(s3_object)
 
         normalised_path = "./#{path}"
         filey = Filey.new(
@@ -78,35 +72,6 @@ module Filey
         md5 = s3_object.etag.gsub(/"/, '').split('-',2).first
         [last_modified, md5]
       end
-
-      def last_modified_and_md5_from_gzipped(s3_object, path)
-        s3_object_contents = s3_object.read
-        if is_already_decoded s3_object_contents
-          md5 = Digest::MD5.hexdigest(s3_object_contents)
-          [s3_object.last_modified, md5]
-        else
-          tempfile = Tempfile.new(File.basename(path))
-          tempfile.binmode
-          tempfile.write s3_object_contents
-          tempfile.close
-
-          gz = Zlib::GzipReader.open(tempfile.path)
-          last_modified = gz.mtime
-          md5 = Digest::MD5.hexdigest(gz.read)
-          gz.close
-          [last_modified, md5]
-        end
-      end
-
-      # Check if the two first bytes are the magic numbers of the gzip format.
-      # We double-check here because Ruby 2.0.0 decodes gzip'ed HTTP responses
-      # automatically. As a result, we get decoded gzip data from the
-      # s3_object#read method when we are using Ruby 2.0.0, and encoded data
-      # when we are using previous versions of Ruby.
-      def is_already_decoded(gzipped_on_server)
-        is_gzipped = gzipped_on_server.bytes.to_a[0] == 0x1f && gzipped_on_server.bytes.to_a[1] == 0x8b
-        is_gzipped == false
-      end
     end
   end
 end

data/spec/data_sources_spec.rb

@@ -160,35 +160,15 @@ describe Filey::DataSources::AwsSdkS3 do
       )]))
     }
 
-    it 'provides the original md5/mtime of a gzipped file' do
+    it 'provides the md5/mtime of a gzipped file' do
       filey = data_source_with_one_gzipped_object.get_fileys[0]
-      filey.md5.should eq(Digest::MD5.hexdigest(objects.first[:content]))
+      gzipped_content = IO.binread(gzip_tempfile_and_path[0])
+      filey.md5.should eq(Digest::MD5.hexdigest(gzipped_content))
       # GzipWriter seems to cut off fractions of a second,
       # to_i adjusts the original file to match
       filey.last_modified.to_i.should eq(objects.first[:mtime].to_i)
     end
 
-    context 'working with Ruby 2.0.0 automatic decoding of gzipped HTTP responses' do
-      let(:data_source_with_decoded_object_and_gzip_header) {
-        data_source = Filey::DataSources::AwsSdkS3.new(S3Bucket.new([S3Object.new(
-          objects.first[:path],
-          objects.first[:mtime],
-          objects.first[:content],
-          { :content_encoding => 'gzip' }
-        )]))
-      }
-
-      it 'detects the case where the gzipped data has already been decoded' do
-        filey = data_source_with_decoded_object_and_gzip_header.get_fileys.first
-        filey.last_modified.should eq(objects.first[:mtime])
-      end
-
-      it 'returns the md5 of the gzip-decoded content' do
-        filey = data_source_with_decoded_object_and_gzip_header.get_fileys.first
-        filey.md5.should eq(Digest::MD5.hexdigest(objects.first[:content]))
-      end
-    end
-
     def gzip_into_tmp_file(content, mtime)
       tempfile = Tempfile.new("temp")
 

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: filey-diff
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 2.0.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-01 00:00:00.000000000 Z
+date: 2014-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.11'
 description: "\n    Find missing or outdated files.\n    For example, compare your
@@ -45,8 +45,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - README.md
 - Rakefile
@@ -72,17 +72,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Compare two data sources that contain file-like objects