zip_tricks 4.0.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b478a7ffbae2dbb20270a1e4c4b63f495feef87
-  data.tar.gz: d9a8ca3a1596ee653fcf2e2b9153a9e59bd912fc
+  metadata.gz: f6d3a4df27461f1235e526d7d40eb089f4df5c64
+  data.tar.gz: 179df69ea7d164f9164aabcc0c19d951e8d5748f
 SHA512:
-  metadata.gz: 689d25f85a79987750fe3bbdf845cfe76c987713709a3d7fc8d2bc944a570958168c2b6c38a6f657b254d499052c079aad84c85b6421b9c228740dd3c9f79903
-  data.tar.gz: 434ffe473e5b0339b40ead5437e070cc1bd6dcce910db58e516a207806cd0eab16f86cc04d342201e2f31f121438308351239d923f4b4e5311850755c77ce339
+  metadata.gz: 68131f0f180074731223f145f06d42f65d7ee327bfcc73c7445a9ae2be198ba8ed2372cffb9deba8729c7b3af46eb4cacdeb3ae408fd37a202c848933ce93ffa
+  data.tar.gz: 9e38d0ce079b7e367b75db3b6ebd032fef11802f12926cf586f061bdb7f059c1087505b2556cfd76062e288601923d51e4c7a1ca5ca5bb928fcae385ee92807b

data/README.md

@@ -74,7 +74,7 @@ to that socket using some accelerated writing technique, and only use the Stream
     ZipTricks::Streamer.open(io) do | zip |
       # raw_file is written "as is" (STORED mode).
       # Write the local file header first..
-      zip.add_stored_entry("first-file.bin", raw_file.size, raw_file_crc32)
+      zip.add_stored_entry(filename: "first-file.bin", size: raw_file.size, crc32: raw_file_crc32)
       
       # then send the actual file contents bypassing the Streamer interface
       io.sendfile(my_temp_file)

data/Rakefile

@@ -18,8 +18,8 @@ Jeweler::Tasks.new do |gem|
   gem.homepage = "http://github.com/wetransfer/zip_tricks"
   gem.license = "MIT"
   gem.version = ZipTricks::VERSION
-  gem.summary = %Q{Makes rubyzip stream, for real}
-  gem.description = %Q{Makes rubyzip stream, for real}
+  gem.summary = 'Stream out ZIP files from Ruby'
+  gem.description = 'Stream out ZIP files from Ruby'
   gem.email = "me@julik.nl"
   gem.authors = ["Julik Tarkhanov"]
   gem.files.exclude "testing/**/*"

data/lib/zip_tricks.rb

@@ -1,5 +1,5 @@
 module ZipTricks
-  VERSION = '4.0.0'
+  VERSION = '4.1.0'
   
   # Require all the sub-components except myself
   Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }

data/lib/zip_tricks/file_reader.rb

@@ -41,11 +41,20 @@ require 'stringio'
 #
 # ## Mode of operation
 #
-# Basically, `FileReader` _ignores_ the data in local file headers (as it is often unreliable).
+# By default, `FileReader` _ignores_ the data in local file headers (as it is often unreliable).
 # It reads the ZIP file "from the tail", finds the end-of-central-directory signatures, then
 # reads the central directory entries, reconstitutes the entries with their filenames, attributes
 # and so on, and sets these entries up with the absolute _offsets_ into the source file/IO object.
 # These offsets can then be used to extract the actual compressed data of the files and to expand it.
+#
+# ## Recovering damaged or incomplete ZIP files
+#
+# If the ZIP file you are trying to read does not contain the central directory records `read_zip_structure`
+# will not work, since it starts the read process from the EOCD marker at the end of the central directory
+# and then crawls "back" in the IO to figure out the rest. You can explicitly apply a fallback for reading the
+# archive "straight ahead" instead using `read_zip_straight_ahead` - the method will instead scan your IO from
+# the very start, skipping over the actual entry data. This is less efficient than central directory parsing since
+# it involves a much larger number of reads (1 read from the IO per entry in the ZIP).
 class ZipTricks::FileReader
   require_relative 'file_reader/stored_reader'
   require_relative 'file_reader/inflating_reader'
@@ -58,6 +67,11 @@ class ZipTricks::FileReader
       "The compressed data offset is not available (local header has not been read)" 
     end
   end
+  MissingEOCD = Class.new(StandardError) do
+    def message
+      "Could not find the EOCD signature in the buffer - maybe a malformed ZIP file"
+    end
+  end
   
   private_constant :StoredReader, :InflatingReader
   
@@ -139,6 +153,12 @@ class ZipTricks::FileReader
       !@compressed_data_offset.nil?
     end
     
+    # Tells whether the entry uses a data descriptor (this is defined
+    # by bit 3 in the GP flags).
+    def uses_data_descriptor?
+      (gp_flags & 0x0008) == 0x0008
+    end
+    
     # Sets the offset at which the compressed data for this file starts in the ZIP.
     # By default, the value will be set by the Reader for you. If you use delayed
     # reading, you need to set it by using the `get_compressed_data_offset` on the Reader:
@@ -165,7 +185,7 @@ class ZipTricks::FileReader
   #   (since the reads have not been performed yet). As a rule, this option can be left in it's
   #   default setting (`true`) unless you want to _only_ read the central directory, or you need
   #   to limit the number of HTTP requests.
-  # @return [Array<Entry>] an array of entries within the ZIP being parsed
+  # @return [Array<ZipEntry>] an array of entries within the ZIP being parsed
   def read_zip_structure(io:, read_local_headers: true)
     zip_file_size = io.size
     eocd_offset = get_eocd_offset(io, zip_file_size)
@@ -194,52 +214,130 @@ class ZipTricks::FileReader
     entries
   end
 
-  # Get the offset in the IO at which the actual compressed data of the file starts within the ZIP.
+  # Sometimes you might encounter truncated ZIP files, which do not contain any central directory
+  # whatsoever - or where the central directory is truncated. In that case, employing the technique
+  # of reading the ZIP "from the end" is impossible, and the only recourse is reading each local file header
+  # in sucession. If the entries in such a ZIP use data descriptors, you would need to scan after the entry until
+  # you encounter the data descriptor signature - and that might be unreliable at best. Therefore, this reading
+  # technique does not support data descriptors. It can however recover the entries you still can read if these
+  # entries contain all the necessary information about the contained file.
+  #
+  # @param io[#tell, #read, #seek] the IO-ish object to read the local file headers from
+  # @return [Array<ZipEntry>] an array of entries that could be recovered before hitting EOF
+  def read_zip_straight_ahead(io:)
+    entries = []
+    loop do
+      cur_offset = io.tell
+      entry = read_local_file_header(io: io)
+      if entry.uses_data_descriptor?
+        raise UnsupportedFeature, "The local file header at #{cur_offset} uses a data descriptor and the start of next entry cannot be found"
+      end
+      entries << entry
+      next_local_header_offset = entry.compressed_data_offset + entry.compressed_size
+      log { 'Recovered a local file file header at offset %d, seeking to the next at %d' % [cur_offset, next_local_header_offset] }
+      seek(io, next_local_header_offset) # Seek to the next entry, and raise if seek is impossible
+    end
+    entries
+  rescue ReadError
+    log { 'Got a read/seek error after reaching %d, no more entries can be recovered' % cur_offset }
+    entries
+  end
+  
+  # Parse the local header entry and get the offset in the IO at which the actual compressed data of the
+  # file starts within the ZIP.
   # The method will eager-read the entire local header for the file (the maximum size the local header may use),
   # starting at the given offset, and will then compute its size. That size plus the local header offset
   # given will be the compressed data offset of the entry (read starting at this offset to get the data).
   #
-  # @param io[#seek, #read] an IO-ish object the ZIP file can be read from
-  # @param local_header_offset[Fixnum] absolute offset (0-based) where the local file header is supposed to begin
-  # @return [Fixnum] absolute offset (0-based) of where the compressed data begins for this file within the ZIP
-  def get_compressed_data_offset(io:, local_file_header_offset:)
-    seek(io, local_file_header_offset)
-    
+  # @param io[#read] an IO-ish object the ZIP file can be read from
+  # @return [Array<ZipEntry, Fixnum>] the parsed local header entry and the compressed data offset
+  def read_local_file_header(io:)
+    local_file_header_offset = io.tell
+
     # Reading in bulk is cheaper - grab the maximum length of the local header,
-    # including any headroom
+    # including any headroom for extra fields etc.
     local_file_header_str_plus_headroom = io.read(MAX_LOCAL_HEADER_SIZE)
+    raise ReadError if local_file_header_str_plus_headroom.nil? # reached EOF
+    
     io_starting_at_local_header = StringIO.new(local_file_header_str_plus_headroom)
 
     assert_signature(io_starting_at_local_header, 0x04034b50)
-
-    # The rest is unreliable, and we have that information from the central directory already.
-    # So just skip over it to get at the offset where the compressed data begins
-    skip_ahead_2(io_starting_at_local_header) # Version needed to extract
-    skip_ahead_2(io_starting_at_local_header) # gp flags
-    skip_ahead_2(io_starting_at_local_header) # storage mode
-    skip_ahead_2(io_starting_at_local_header) # dos time
-    skip_ahead_2(io_starting_at_local_header) # dos date
-    skip_ahead_4(io_starting_at_local_header) # CRC32
-
-    skip_ahead_4(io_starting_at_local_header) # Comp size
-    skip_ahead_4(io_starting_at_local_header) # Uncomp size
+    e = ZipEntry.new
+    e.version_needed_to_extract = read_2b(io_starting_at_local_header) # Version needed to extract
+    e.gp_flags = read_2b(io_starting_at_local_header) # gp flags
+    e.storage_mode = read_2b(io_starting_at_local_header) # storage mode
+    e.dos_time = read_2b(io_starting_at_local_header) # dos time
+    e.dos_date = read_2b(io_starting_at_local_header) # dos date
+    e.crc32 = read_4b(io_starting_at_local_header) # CRC32
+    e.compressed_size = read_4b(io_starting_at_local_header) # Comp size
+    e.uncompressed_size = read_4b(io_starting_at_local_header) # Uncomp size
 
     filename_size = read_2b(io_starting_at_local_header)
     extra_size = read_2b(io_starting_at_local_header)
+    e.filename = read_n(io_starting_at_local_header, filename_size)
+    extra_fields_str = read_n(io_starting_at_local_header, extra_size)
+    
+    # Parse out the extra fields
+    extra_table = parse_out_extra_fields(extra_fields_str)
+    
+    # ...of which we really only need the Zip64 extra
+    if zip64_extra_contents = extra_table[1]
+      # If the Zip64 extra is present, we let it override all
+      # the values fetched from the conventional header
+      zip64_extra = StringIO.new(zip64_extra_contents)
+      log { 'Will read Zip64 extra data from local header field for %s, %d bytes' % [e.filename, zip64_extra.size] }
+      # Now here be dragons. The APPNOTE specifies that
+      #
+      # > The order of the fields in the ZIP64 extended 
+      # > information record is fixed, but the fields will
+      # > only appear if the corresponding Local or Central
+      # > directory record field is set to 0xFFFF or 0xFFFFFFFF.
+      #
+      # It means that before we read this stuff we need to check if the previously-read
+      # values are at overflow, and only _then_ proceed to read them. Bah.
+      e.uncompressed_size = read_8b(zip64_extra) if e.uncompressed_size == 0xFFFFFFFF
+      e.compressed_size = read_8b(zip64_extra) if e.compressed_size == 0xFFFFFFFF
+    end
 
-    skip_ahead_n(io_starting_at_local_header, filename_size)
-    skip_ahead_n(io_starting_at_local_header, extra_size)
+    offset = local_file_header_offset + io_starting_at_local_header.tell
+    e.compressed_data_offset = offset
 
-    local_file_header_offset + io_starting_at_local_header.tell
+    e
   end
-  
-  # Parse an IO handle to a ZIP archive into an array of Entry objects.
+
+  # Get the offset in the IO at which the actual compressed data of the file starts within the ZIP.
+  # The method will eager-read the entire local header for the file (the maximum size the local header may use),
+  # starting at the given offset, and will then compute its size. That size plus the local header offset
+  # given will be the compressed data offset of the entry (read starting at this offset to get the data).
+  #
+  # @param io[#seek, #read] an IO-ish object the ZIP file can be read from
+  # @param local_header_offset[Fixnum] absolute offset (0-based) where the local file header is supposed to begin
+  # @return [Fixnum] absolute offset (0-based) of where the compressed data begins for this file within the ZIP
+  def get_compressed_data_offset(io:, local_file_header_offset:)
+    seek(io, local_file_header_offset)
+    entry_recovered_from_local_file_header = read_local_file_header(io: io)
+    entry_recovered_from_local_file_header.compressed_data_offset
+  end
+
+  # Parse an IO handle to a ZIP archive into an array of Entry objects, reading from the end
+  # of the IO object.
   #
+  # @see {#read_zip_structure}
   # @param options[Hash] any options the instance method of the same name accepts
-  # @return [Array<Entry>] an array of entries within the ZIP being parsed
+  # @return [Array<ZipEntry>] an array of entries within the ZIP being parsed
   def self.read_zip_structure(**options)
     new.read_zip_structure(**options)
   end
+
+  # Parse an IO handle to a ZIP archive into an array of Entry objects, reading from the start of
+  # the file and parsing local file headers one-by-one
+  #
+  # @see {#read_zip_straight_ahead}
+  # @param options[Hash] any options the instance method of the same name accepts
+  # @return [Array<ZipEntry>] an array of entries within the ZIP being parsed
+  def self.read_zip_straight_ahead(**options)
+    new.read_zip_straight_ahead(**options)
+  end
   
   private
 
@@ -334,14 +432,7 @@ class ZipTricks::FileReader
       e.comment = read_n(io, comment_len)
 
       # Parse out the extra fields
-      extra_table = {}
-      extras_buf = StringIO.new(extras)
-      until extras_buf.eof? do
-        extra_id = read_2b(extras_buf)
-        extra_size = read_2b(extras_buf)
-        extra_contents = read_n(extras_buf, extra_size)
-        extra_table[extra_id] = extra_contents
-      end
+      extra_table = parse_out_extra_fields(extras)
 
       # ...of which we really only need the Zip64 extra
       if zip64_extra_contents = extra_table[1]
@@ -378,7 +469,7 @@ class ZipTricks::FileReader
     str_containing_eocd_record = file_io.read(MAX_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE)
     eocd_idx_in_buf = locate_eocd_signature(str_containing_eocd_record)
     
-    raise "Could not find the EOCD signature in the buffer - maybe a malformed ZIP file" unless eocd_idx_in_buf
+    raise MissingEOCD unless eocd_idx_in_buf
     
     eocd_offset = implied_position_of_eocd_record + eocd_idx_in_buf
     log { 'Found EOCD signature at offset %d' % eocd_offset }
@@ -546,4 +637,16 @@ class ZipTricks::FileReader
     # The most minimal implementation for the method is just this:
     # $stderr.puts(yield)
   end
+  
+  def parse_out_extra_fields(extra_fields_str)
+    extra_table = {}
+    extras_buf = StringIO.new(extra_fields_str)
+    until extras_buf.eof? do
+      extra_id = read_2b(extras_buf)
+      extra_size = read_2b(extras_buf)
+      extra_contents = read_n(extras_buf, extra_size)
+      extra_table[extra_id] = extra_contents
+    end
+    extra_table
+  end
 end

data/spec/zip_tricks/file_reader_spec.rb

@@ -1,6 +1,78 @@
 require 'spec_helper'
 describe ZipTricks::FileReader do
   
+  describe 'with a file without EOCD' do
+    it 'raises the MissingEOCD exception and refuses to read' do
+      f = StringIO.new
+      10.times { f << ('A' * 1024 ) }
+      f.rewind
+      
+      expect {
+        described_class.read_zip_structure(io: f)
+      }.to raise_error(described_class::MissingEOCD)
+    end
+  end
+  
+  describe 'read_zip_straight_ahead' do
+    it 'returns all the entries it can recover' do
+      zipfile = StringIO.new
+      war_and_peace = File.read(__dir__ + '/war-and-peace.txt')
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        zip.add_stored_entry filename: 'text1.txt', crc32: Zlib.crc32(war_and_peace), size: war_and_peace.bytesize
+        zip << war_and_peace
+        zip.add_stored_entry filename: 'text2.txt', crc32: Zlib.crc32(war_and_peace), size: war_and_peace.bytesize
+        zip << war_and_peace
+        zip.add_stored_entry filename: 'text3.txt', crc32: Zlib.crc32(war_and_peace), size: war_and_peace.bytesize
+        zip << war_and_peace
+      end
+      zipfile.rewind
+
+      recovered_entries = described_class.read_zip_straight_ahead(io: zipfile)
+      expect(recovered_entries.length).to eq(3)
+      recovered_entries.each do |entry|
+        expect(entry.storage_mode).to eq(0)
+        expect(entry.compressed_size).to eq(496006)
+        expect(entry.uncompressed_size).to eq(496006)
+      end
+      
+      first, second, third = recovered_entries
+      expect(first.compressed_data_offset).to eq(39)
+      expect(second.compressed_data_offset).to eq(496084)
+      expect(third.compressed_data_offset).to eq(992129)
+      
+      recovered_entries.each do |entry|
+        zipfile.seek(entry.compressed_data_offset)
+        expect(zipfile.read(5)).to eq(war_and_peace[0...5])
+      end
+    end
+    
+    it 'recovers an entry that uses Zip64 extra fields' do
+      zipfile = StringIO.new
+      w = ZipTricks::ZipWriter.new
+      w.write_local_file_header(io: zipfile, filename: 'big.bin', compressed_size: 0xFFFFFFFFFF, uncompressed_size: 0xFFFFFFFFF,
+        crc32: 0, gp_flags: 0, mtime: Time.now, storage_mode: 0)
+      zipfile.rewind
+      recovered_entries = described_class.read_zip_straight_ahead(io: zipfile)
+      expect(recovered_entries.length).to eq(1)
+      entry = recovered_entries.shift
+      expect(entry.compressed_size).to eq(0xFFFFFFFFFF)
+    end
+    
+    it 'raises when an entry uses a data descriptor' do
+      zipfile = StringIO.new
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        zip.write_deflated_file('war-and-peace.txt') do |sink|
+          sink << File.read(__dir__ + '/war-and-peace.txt')
+        end
+      end
+      zipfile.rewind
+      
+      expect {
+        described_class.read_zip_straight_ahead(io: zipfile)
+      }.to raise_error(described_class::UnsupportedFeature)
+    end
+  end
+  
   describe 'with an end-to-end ZIP file to read' do
     it 'reads and uncompresses the file written deflated with data descriptors' do
       zipfile = StringIO.new

data/zip_tricks.gemspec

@@ -2,17 +2,17 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: zip_tricks 4.0.0 ruby lib
+# stub: zip_tricks 4.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "zip_tricks"
-  s.version = "4.0.0"
+  s.version = "4.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-08-19"
-  s.description = "Makes rubyzip stream, for real"
+  s.date = "2016-09-14"
+  s.description = "Stream out ZIP files from Ruby"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
     "LICENSE.txt",
@@ -70,8 +70,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/wetransfer/zip_tricks"
   s.licenses = ["MIT"]
-  s.rubygems_version = "2.2.2"
-  s.summary = "Makes rubyzip stream, for real"
+  s.rubygems_version = "2.5.1"
+  s.summary = "Stream out ZIP files from Ruby"
 
   if s.respond_to? :specification_version then
     s.specification_version = 4

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.1.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-19 00:00:00.000000000 Z
+date: 2016-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -162,7 +162,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.1
-description: Makes rubyzip stream, for real
+description: Stream out ZIP files from Ruby
 email: me@julik.nl
 executables: []
 extensions: []
@@ -238,8 +238,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Makes rubyzip stream, for real
+summary: Stream out ZIP files from Ruby
 test_files: []