zip_tricks 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bc3917bd654f3fca15ae6bce1769eb0fa45dd12
-  data.tar.gz: 12f495895ed59e23cc89bf7b86b91bd9422c1608
+  metadata.gz: d3b168fde79da25b24e3cb4b69caac818f1337f3
+  data.tar.gz: e1d7b9995636811c9c662a70e2caffb3a085709b
 SHA512:
-  metadata.gz: 66fe048eddc9c00ed459e02e30ba8607a76cf297de88bf595694d14eec42e7b1075752b42fae8e86dcfe06ca2814db985452ed0244f803207491b100baed1041
-  data.tar.gz: 717bd4d51c597fefdd77bfe0f7df3c41f2ea8ae0122f0b5330ab6cf63b6183370e5dd8bbc19b868a6f52ae53a84147bfdb8bee54bbc44b63df563074192f7ec8
+  metadata.gz: 65f27dacbcc5fd09e229f8543497fe3b3ff3da861696b355e01688f8b78dc7d02f4163de11e4a909c83f273a42005f63be7c1c7b7196f8657f0dd4d26cc4cc4f
+  data.tar.gz: 66b7db54cdc4702d0f867b8f263ea1bca2016405dd19db43bc4a7368ddb0d4b912b67344b7d0efc3156b0fbfe5868aa78b3cc6b8c22c87cddfd57282c8842dec

data/README.md

@@ -42,16 +42,20 @@ The block will only be called when actually sending the response to the client
 
 Use the `SizeEstimator` to compute the correct size of the resulting archive.
 
+    # Prepare the response body. The block will only be called when the response starts to be written.
     zip_body = ZipTricks::RackBody.new do | zip |
       zip.add_stored_entry(filename: "myfile1.bin", size: 9090821, crc32: 12485)
       zip << read_file('myfile1.bin')
       zip.add_stored_entry(filename: "myfile2.bin", size: 458678, crc32: 89568)
       zip << read_file('myfile2.bin')
     end
+    
+    # Precompute the Content-Length ahead of time
     bytesize = ZipTricks::SizeEstimator.estimate do |z|
      z.add_stored_entry(filename: 'myfile1.bin', size: 9090821)
      z.add_stored_entry(filename: 'myfile2.bin', size: 458678)
     end
+    
     [200, {'Content-Length' => bytesize.to_s}, zip_body]
 
 ## Other usage examples
@@ -79,55 +83,10 @@ to that socket using some accelerated writing technique, and only use the Stream
       zip.simulate_write(my_temp_file.size)
     end
 
-## RackBody
-
-Can be used to output a streamed ZIP archive directly through a Rack response body.
-The block given to the constructor will be called when the response body will be read by the webserver,
-and will receive a {ZipTricks::Streamer} as it's block argument. You can then add entries to the Streamer as usual.
-The archive will be automatically closed at the end of the block.
-
-    # Precompute the Content-Length ahead of time
-    content_length = ZipTricks::SizeEstimator.estimate do | estimator |
-      estimator.add_stored_entry('large.tif', size=1289894)
-    end
-    
-    # Prepare the response body. The block will only be called when the response starts to be written.
-    body = ZipTricks::RackBody.new do | streamer |
-      streamer.add_stored_entry('large.tif', size=1289894, crc32=198210)
-      streamer << large_file.read(1024*1024) until large_file.eof?
-      ...
-    end
-    
-    [200, {'Content-Type' => 'binary/octet-stream', 'Content-Length' => content_length.to_s}, body]
-  
-## BlockWrite
-
-Can be used as the destination IO, but will call the given block instead on every call to `:<<`.
-This can be used to attach the output of the zip compressor to the Rack response body, or another
-destination. For Rack/Rails just use RackBody since it sets this up for you.
-
-    io = ZipTricks::BlockWrite.new{|data| socket << data }
-    ZipTricks::Streamer.open(io) do | zip |
-      zip.add_stored_entry("first-file.bin", raw_file.size, raw_file_crc32)
-      ....
-    end
-
-## SizeEstimator
-
-Is used to predict the size of the ZIP archive after output. This can be used to generate, say, a `Content-Length` header,
-or to predict the size of the resulting archive on the storage device. The size is estimated using a very fast "fake archiving"
-procedure, so it computes the sizes of all the headers and the central directory very accurately.
-
-    expected_zip_archive_size = SizeEstimator.estimate do | estimator |
-      estimator.add_stored_entry("file.doc", size=898291)
-      estimator.add_compressed_entry("family.JPG", size=89281911, compressed_size=89218)
-    end
-
-
-## StreamCRC32
+## Computing the CRC32 value of a large file
 
-Computes the CRC32 value in a streaming fashion. Is slightly more convenient for the purpose than using the raw Zlib
-library functions.
+`BlockCRC32` computes the CRC32 checksum of an IO in a streaming fashion. It is slightly more convenient for the purpose
+than using the raw Zlib library functions.
 
     crc = ZipTricks::StreamCRC32.new
     crc << large_file.read(1024 * 12) until large_file.eof?

data/lib/zip_tricks.rb

@@ -1,5 +1,5 @@
 module ZipTricks
-  VERSION = '3.0.0'
+  VERSION = '3.1.0'
   
   # Require all the sub-components except myself
   Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }

data/lib/zip_tricks/file_reader.rb

@@ -366,16 +366,47 @@ class ZipTricks::FileReader
     # and a soft read (we might not be able to read as many bytes as we want)
     file_io.seek(implied_position_of_eocd_record, IO::SEEK_SET)
     str_containing_eocd_record = file_io.read(MAX_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE)
-
-    # TODO: what to do if multiple occurrences of the signature are found, somehow?
-    eocd_sig = [0x06054b50].pack(C_V)
-    eocd_idx_in_buf = str_containing_eocd_record.index(eocd_sig)
-
+    eocd_idx_in_buf = locate_eocd_signature(str_containing_eocd_record)
+    
     raise "Could not find the EOCD signature in the buffer - maybe a malformed ZIP file" unless eocd_idx_in_buf
 
     implied_position_of_eocd_record + eocd_idx_in_buf
   end
 
+  # This is tricky. Essentially, we have to scan the maximum possible number of bytes (that the EOCD can
+  # theoretically occupy including the comment), and we have to find a combination of:
+  #   [EOCD signature, <some ZIP medatata>, comment byte size, the comment of that size, eof].
+  # The only way I could find to do this was with a sliding window, but there probably is a better way.
+  def locate_eocd_signature(in_str)
+    # We have to scan from the _very_ tail. We read the very minimum size
+    # the EOCD record can have (up to and including the comment size), using
+    # a sliding window. Once our end offset matches the comment size we found our
+    # EOCD marker.
+    eocd_signature_int = 0x06054b50
+    unpack_pattern = 'VvvvvVVv'
+    minimum_record_size = 22
+    end_location = minimum_record_size * -1
+    loop do
+      # If the window is nil, we have rolled off the start of the string, nothing to do here.
+      # We use negative values because if we used positive slice indices
+      # we would have to detect the rollover ourselves
+      break unless window = in_str[end_location, minimum_record_size]
+      
+      window_location = in_str.bytesize + end_location
+      unpacked = window.unpack(unpack_pattern)
+      
+      # If we found the signarue, pick up the comment size, and check if the size of the window
+      # plus that comment size is where we are in the string. If we are - bingo.
+      if unpacked[0] == 0x06054b50 && comment_size = unpacked[-1] 
+        assumed_eocd_location = in_str.bytesize - comment_size - minimum_record_size
+        # if the comment size is where we should be at - we found our EOCD
+        return assumed_eocd_location if assumed_eocd_location == window_location
+      end
+      
+      end_location -= 1 # Shift the window back, by one byte, and try again.
+    end
+  end
+  
   # Find the Zip64 EOCD locator segment offset. Do this by seeking backwards from the
   # EOCD record in the archive by fixed offsets
   def get_zip64_eocd_locator_offset(file_io, eocd_offset)

data/lib/zip_tricks/zip_writer.rb

@@ -221,8 +221,9 @@ class ZipTricks::ZipWriter
   # @param start_of_central_directory_location[Fixnum] byte offset of the start of central directory form the beginning of ZIP file
   # @param central_directory_size[Fixnum] the size of the central directory (only file headers) in bytes
   # @param num_files_in_archive[Fixnum] How many files the archive contains
+  # @param comment[String] the comment for the archive (defaults to ZIP_TRICKS_COMMENT)
   # @return [void]
-  def write_end_of_central_directory(io:, start_of_central_directory_location:, central_directory_size:, num_files_in_archive:)
+  def write_end_of_central_directory(io:, start_of_central_directory_location:, central_directory_size:, num_files_in_archive:, comment: ZIP_TRICKS_COMMENT)
     zip64_eocdr_offset = start_of_central_directory_location + central_directory_size
     
     zip64_required = central_directory_size > FOUR_BYTE_MAX_UINT ||
@@ -296,8 +297,8 @@ class ZipTricks::ZipWriter
                                                             # directory with respect to
                                                             # the starting disk number        4 bytes
     end
-    io << [ZIP_TRICKS_COMMENT.bytesize].pack(C_v)           # .ZIP file comment length        2 bytes
-    io << ZIP_TRICKS_COMMENT                                # .ZIP file comment       (variable size)
+    io << [comment.bytesize].pack(C_v)                      # .ZIP file comment length        2 bytes
+    io << comment                                           # .ZIP file comment       (variable size)
   end
   
   private

data/spec/zip_tricks/file_reader_spec.rb

@@ -44,4 +44,26 @@ describe ZipTricks::FileReader do
     expect(readback.bytesize).to eq(tolstoy.bytesize)
     expect(readback[0..10]).to eq(tolstoy[0..10])
   end
+  
+  it 'is able to latch to the EOCD location even if the signature for the EOCD record appears all over the ZIP' do
+    # A VERY evil ZIP file which has this signature all over
+    eocd_sig = [0x06054b50].pack('V')
+    evil_str = "#{eocd_sig} and #{eocd_sig}"
+    
+    z = StringIO.new
+    w = ZipTricks::ZipWriter.new
+    w.write_local_file_header(io: z, filename: evil_str, compressed_size: evil_str.bytesize,
+       uncompressed_size: evil_str.bytesize, crc32: 0x06054b50, gp_flags: 0, mtime: Time.now, storage_mode: 0)
+    z << evil_str
+    where = z.tell
+    w.write_central_directory_file_header(io: z, local_file_header_location: 0, gp_flags: 0, storage_mode: 0,
+      filename: evil_str, compressed_size: evil_str.bytesize,
+      uncompressed_size: evil_str.bytesize, mtime: Time.now, crc32: 0x06054b50)
+    w.write_end_of_central_directory(io: z, start_of_central_directory_location: where,
+      central_directory_size: z.tell - where, num_files_in_archive: 1, comment: evil_str)
+    
+    z.rewind
+    entries = described_class.read_zip_structure(z)
+    expect(entries.length).to eq(1)
+  end
 end

data/spec/zip_tricks/zip_writer_spec.rb

@@ -330,6 +330,17 @@ describe ZipTricks::ZipWriter do
       expect(br.read_n(comment_length)).to match(/ZipTricks/)
     end
     
+    it 'writes out the custom comment' do
+      buf = ''
+      comment = 'Ohai mate'
+      subject.write_end_of_central_directory(io: buf, start_of_central_directory_location: 9091211,
+        central_directory_size: 9091, num_files_in_archive: 4, comment: comment)
+      
+      size_and_comment = buf[((comment.bytesize + 2) * -1)..-1]
+      comment_size = size_and_comment.unpack('v')[0]
+      expect(comment_size).to eq(comment.bytesize)
+    end
+    
     it 'writes out the Zip64 EOCD as well if the central directory is located beyound 4GB in the archive' do
       buf = StringIO.new
     

data/zip_tricks.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: zip_tricks 3.0.0 ruby lib
+# stub: zip_tricks 3.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "zip_tricks"
-  s.version = "3.0.0"
+  s.version = "3.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-08-03"
+  s.date = "2016-08-16"
   s.description = "Makes rubyzip stream, for real"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-03 00:00:00.000000000 Z
+date: 2016-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip