zip_tricks 3.1.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3434ce881bfe7fdf3494ed494f8e251082d94510
-  data.tar.gz: 30062193918dacf4a018d7bde923303f0a14c47f
+  metadata.gz: 6b478a7ffbae2dbb20270a1e4c4b63f495feef87
+  data.tar.gz: d9a8ca3a1596ee653fcf2e2b9153a9e59bd912fc
 SHA512:
-  metadata.gz: 88be7f1bebd8e0faa8906203b594e022b173390039732a3eaafa53ec903770b0a619f7a0895a8203b75c6c09f5908ca400d86611f443e34fdead53554edd8e14
-  data.tar.gz: 5c4362394b3666ce528e16c657b019a8185875555af2202a8c976df523c309017ac060ec61082617b3573722cb1d2e4eb002502ef9c8b770227859756e0afe5f
+  metadata.gz: 689d25f85a79987750fe3bbdf845cfe76c987713709a3d7fc8d2bc944a570958168c2b6c38a6f657b254d499052c079aad84c85b6421b9c228740dd3c9f79903
+  data.tar.gz: 434ffe473e5b0339b40ead5437e070cc1bd6dcce910db58e516a207806cd0eab16f86cc04d342201e2f31f121438308351239d923f4b4e5311850755c77ce339

data/lib/zip_tricks.rb

@@ -1,5 +1,5 @@
 module ZipTricks
-  VERSION = '3.1.1'
+  VERSION = '4.0.0'
   
   # Require all the sub-components except myself
   Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }

data/lib/zip_tricks/file_reader.rb

@@ -47,71 +47,18 @@ require 'stringio'
 # and so on, and sets these entries up with the absolute _offsets_ into the source file/IO object.
 # These offsets can then be used to extract the actual compressed data of the files and to expand it.
 class ZipTricks::FileReader
+  require_relative 'file_reader/stored_reader'
+  require_relative 'file_reader/inflating_reader'
+
   ReadError = Class.new(StandardError)
   UnsupportedFeature = Class.new(StandardError)
   InvalidStructure = Class.new(ReadError)
-  
-  class InflatingReader
-    def initialize(from_io, compressed_data_size)
-      @io = from_io
-      @compressed_data_size = compressed_data_size
-      @already_read = 0
-      @zlib_inflater = ::Zlib::Inflate.new(-Zlib::MAX_WBITS)
-    end
-
-    def extract(n_bytes=nil)
-      n_bytes ||= (@compressed_data_size - @already_read)
-
-      return if eof?
-
-      available = @compressed_data_size - @already_read
-
-      return if available.zero?
-
-      n_bytes = available if n_bytes > available
-
-      return '' if n_bytes.zero?
-
-      compressed_chunk = @io.read(n_bytes)
-      @already_read += compressed_chunk.bytesize
-      @zlib_inflater.inflate(compressed_chunk)
-    end
-
-    def eof?
-      @zlib_inflater.finished?
-    end
-  end
-
-  class StoredReader
-    def initialize(from_io, compressed_data_size)
-      @io = from_io
-      @compressed_data_size = compressed_data_size
-      @already_read = 0
-    end
-
-    def extract(n_bytes=nil)
-      n_bytes ||= (@compressed_data_size - @already_read)
-
-      return if eof?
-
-      available = @compressed_data_size - @already_read
-
-      return if available.zero?
-
-      n_bytes = available if n_bytes > available
-
-      return '' if n_bytes.zero?
-
-      compressed_chunk = @io.read(n_bytes)
-      @already_read += compressed_chunk.bytesize
-      compressed_chunk
-    end
-
-    def eof?
-      @already_read >= @compressed_data_size
+  LocalHeaderPending = Class.new(StandardError) do
+    def message
+      "The compressed data offset is not available (local header has not been read)" 
     end
   end
-
+  
   private_constant :StoredReader, :InflatingReader
   
   # Represents a file within the ZIP archive being read
@@ -162,13 +109,9 @@ class ZipTricks::FileReader
     # @return [String] the file comment
     attr_accessor :comment
 
-    # @return [Fixnum] at what offset you should start reading
-    #       for the compressed data in your original IO object
-    attr_accessor :compressed_data_offset
-
     # Returns a reader for the actual compressed data of the entry.
     #
-    #   reader = entry.reader(source_file)
+    #   reader = entry.extractor_from(source_file)
     #   outfile << reader.extract(512 * 1024) until reader.eof?
     #
     # @return [#extract(n_bytes), #eof?] the reader for the data
@@ -180,17 +123,50 @@ class ZipTricks::FileReader
       when 0
         StoredReader.new(from_io, compressed_size)
       else
-        raise "Unsupported storage mode for reading (#{storage_mode})"
+        raise UnsupportedFeature, "Unsupported storage mode for reading - %d" % storage_mode
       end
     end
+    
+    # @return [Fixnum] at what offset you should start reading
+    #       for the compressed data in your original IO object
+    def compressed_data_offset
+      @compressed_data_offset or raise LocalHeaderPending
+    end
+    
+    # Tells whether the compressed data offset is already known for this entry
+    # @return [Boolean]
+    def known_offset?
+      !@compressed_data_offset.nil?
+    end
+    
+    # Sets the offset at which the compressed data for this file starts in the ZIP.
+    # By default, the value will be set by the Reader for you. If you use delayed
+    # reading, you need to set it by using the `get_compressed_data_offset` on the Reader:
+    #
+    #     entry.compressed_data_offset = reader.get_compressed_data_offset(io: file,
+    #            local_file_header_offset: entry.local_header_offset)
+    def compressed_data_offset=(offset)
+      @compressed_data_offset = offset.to_i
+    end
   end
 
   # Parse an IO handle to a ZIP archive into an array of Entry objects.
   #
   # @param io[#tell, #seek, #read, #size] an IO-ish object
-  # @param read_local_headers[Boolean] whether to proceed to read the local headers in addition to the central directory
+  # @param read_local_headers[Boolean] whether the local headers must be read upfront. When reading
+  #   a locally available ZIP file this option will not have much use since the small reads from
+  #   the file handle are not going to be that important. However, if you are using remote reads
+  #   to decipher a ZIP file located on an HTTP server, the operation _must_ perform an HTTP
+  #   request for _each entry in the ZIP file_ to determine where the actual file data starts.
+  #   This, for a ZIP archive of 1000 files, will incur 1000 extra HTTP requests - which you might
+  #   not want to perform upfront, or - at least - not want to perform _at once_. When the option is
+  #   set to `false`, you will be getting instances of `LazyEntry` instead of `Entry`. Those objects
+  #   will raise an exception when you attempt to access their compressed data offset in the ZIP
+  #   (since the reads have not been performed yet). As a rule, this option can be left in it's
+  #   default setting (`true`) unless you want to _only_ read the central directory, or you need
+  #   to limit the number of HTTP requests.
   # @return [Array<Entry>] an array of entries within the ZIP being parsed
-  def read_zip_structure(io, read_local_headers: true)
+  def read_zip_structure(io:, read_local_headers: true)
     zip_file_size = io.size
     eocd_offset = get_eocd_offset(io, zip_file_size)
 
@@ -213,24 +189,68 @@ class ZipTricks::FileReader
       read_cdir_entry(central_directory_io)
     end
     
-    entries.each_with_index do |entry, i|
-      if read_local_headers
-        log { 'Reading the local header for entry %d at offset %d' % [i, entry.local_file_header_offset] }
-        entry.compressed_data_offset = find_compressed_data_start_offset(io, entry.local_file_header_offset)
-      end
-    end
+    read_local_headers(entries, io) if read_local_headers
+    
+    entries
   end
 
+  # Get the offset in the IO at which the actual compressed data of the file starts within the ZIP.
+  # The method will eager-read the entire local header for the file (the maximum size the local header may use),
+  # starting at the given offset, and will then compute its size. That size plus the local header offset
+  # given will be the compressed data offset of the entry (read starting at this offset to get the data).
+  #
+  # @param io[#seek, #read] an IO-ish object the ZIP file can be read from
+  # @param local_header_offset[Fixnum] absolute offset (0-based) where the local file header is supposed to begin
+  # @return [Fixnum] absolute offset (0-based) of where the compressed data begins for this file within the ZIP
+  def get_compressed_data_offset(io:, local_file_header_offset:)
+    seek(io, local_file_header_offset)
+    
+    # Reading in bulk is cheaper - grab the maximum length of the local header,
+    # including any headroom
+    local_file_header_str_plus_headroom = io.read(MAX_LOCAL_HEADER_SIZE)
+    io_starting_at_local_header = StringIO.new(local_file_header_str_plus_headroom)
+
+    assert_signature(io_starting_at_local_header, 0x04034b50)
+
+    # The rest is unreliable, and we have that information from the central directory already.
+    # So just skip over it to get at the offset where the compressed data begins
+    skip_ahead_2(io_starting_at_local_header) # Version needed to extract
+    skip_ahead_2(io_starting_at_local_header) # gp flags
+    skip_ahead_2(io_starting_at_local_header) # storage mode
+    skip_ahead_2(io_starting_at_local_header) # dos time
+    skip_ahead_2(io_starting_at_local_header) # dos date
+    skip_ahead_4(io_starting_at_local_header) # CRC32
+
+    skip_ahead_4(io_starting_at_local_header) # Comp size
+    skip_ahead_4(io_starting_at_local_header) # Uncomp size
+
+    filename_size = read_2b(io_starting_at_local_header)
+    extra_size = read_2b(io_starting_at_local_header)
+
+    skip_ahead_n(io_starting_at_local_header, filename_size)
+    skip_ahead_n(io_starting_at_local_header, extra_size)
+
+    local_file_header_offset + io_starting_at_local_header.tell
+  end
+  
   # Parse an IO handle to a ZIP archive into an array of Entry objects.
   #
-  # @param io[#tell, #seek, #read, #size] an IO-ish object
+  # @param options[Hash] any options the instance method of the same name accepts
   # @return [Array<Entry>] an array of entries within the ZIP being parsed
-  def self.read_zip_structure(io)
-    new.read_zip_structure(io)
+  def self.read_zip_structure(**options)
+    new.read_zip_structure(**options)
   end
   
   private
 
+  def read_local_headers(entries, io)
+    entries.each_with_index do |entry, i|
+      log { 'Reading the local header for entry %d at offset %d' % [i, entry.local_file_header_offset] }
+      off = get_compressed_data_offset(io: io, local_file_header_offset: entry.local_file_header_offset)
+      entry.compressed_data_offset = off
+    end
+  end
+  
   def skip_ahead_2(io)
     skip_ahead_n(io, 2)
   end
@@ -287,40 +307,7 @@ class ZipTricks::FileReader
     read_n(io, 8).unpack(C_Qe).shift
   end
 
-  def find_compressed_data_start_offset(file_io, local_header_offset)
-    seek(file_io, local_header_offset)
-    
-    # Reading in bulk is cheaper - grab the maximum length of the local header, including
-    # any headroom
-    local_file_header_str_plus_headroom = file_io.read(MAX_LOCAL_HEADER_SIZE)
-    io = StringIO.new(local_file_header_str_plus_headroom)
-
-    assert_signature(io, 0x04034b50)
-
-    # The rest is unreliable, and we have that information from the central directory already.
-    # So just skip over it to get at the offset where the compressed data begins
-    skip_ahead_2(io) # Version needed to extract
-    skip_ahead_2(io) # gp flags
-    skip_ahead_2(io) # storage mode
-    skip_ahead_2(io) # dos time
-    skip_ahead_2(io) # dos date
-    skip_ahead_4(io) # CRC32
-
-    skip_ahead_4(io) # Comp size
-    skip_ahead_4(io) # Uncomp size
-
-    filename_size = read_2b(io)
-    extra_size = read_2b(io)
-
-    skip_ahead_n(io, filename_size)
-    skip_ahead_n(io, extra_size)
-
-    local_header_offset + io.tell
-  end
-
-
   def read_cdir_entry(io)
-    expected_at = io.tell
     assert_signature(io, 0x02014b50)
     ZipEntry.new.tap do |e|
       e.made_by = read_2b(io)
@@ -357,7 +344,9 @@ class ZipTricks::FileReader
       end
 
       # ...of which we really only need the Zip64 extra
-      if zip64_extra_contents = extra_table[1] # Zip64 extra
+      if zip64_extra_contents = extra_table[1]
+        # If the Zip64 extra is present, we let it override all
+        # the values fetched from the conventional header
         zip64_extra = StringIO.new(zip64_extra_contents)
         log { 'Will read Zip64 extra data for %s, %d bytes' % [e.filename, zip64_extra.size] }
         # Now here be dragons. The APPNOTE specifies that
@@ -533,10 +522,11 @@ class ZipTricks::FileReader
   def num_files_and_central_directory_offset(file_io, eocd_offset)
     seek(file_io, eocd_offset)
 
-    io = StringIO.new(read_n(file_io, SIZE_OF_USABLE_EOCD_RECORD))
-    
-    assert_signature(io, 0x06054b50)
+    # The size of the EOCD record is known upfront, so use a strict read
+    eocd_record_str = read_n(file_io, SIZE_OF_USABLE_EOCD_RECORD) 
+    io = StringIO.new(eocd_record_str)
 
+    assert_signature(io, 0x06054b50)
     skip_ahead_2(io) # number_of_this_disk
     skip_ahead_2(io) # number of the disk with the EOCD record
     skip_ahead_2(io) # number of entries in the central directory of this disk

data/lib/zip_tricks/file_reader/inflating_reader.rb

@@ -0,0 +1,30 @@
+class ZipTricks::FileReader::InflatingReader
+  def initialize(from_io, compressed_data_size)
+    @io = from_io
+    @compressed_data_size = compressed_data_size
+    @already_read = 0
+    @zlib_inflater = ::Zlib::Inflate.new(-Zlib::MAX_WBITS)
+  end
+
+  def extract(n_bytes=nil)
+    n_bytes ||= (@compressed_data_size - @already_read)
+
+    return if eof?
+
+    available = @compressed_data_size - @already_read
+
+    return if available.zero?
+
+    n_bytes = available if n_bytes > available
+
+    return '' if n_bytes.zero?
+
+    compressed_chunk = @io.read(n_bytes)
+    @already_read += compressed_chunk.bytesize
+    @zlib_inflater.inflate(compressed_chunk)
+  end
+
+  def eof?
+    @zlib_inflater.finished?
+  end
+end

data/lib/zip_tricks/file_reader/stored_reader.rb

@@ -0,0 +1,29 @@
+class ZipTricks::FileReader::StoredReader
+  def initialize(from_io, compressed_data_size)
+    @io = from_io
+    @compressed_data_size = compressed_data_size
+    @already_read = 0
+  end
+
+  def extract(n_bytes=nil)
+    n_bytes ||= (@compressed_data_size - @already_read)
+
+    return if eof?
+
+    available = @compressed_data_size - @already_read
+
+    return if available.zero?
+
+    n_bytes = available if n_bytes > available
+
+    return '' if n_bytes.zero?
+
+    compressed_chunk = @io.read(n_bytes)
+    @already_read += compressed_chunk.bytesize
+    compressed_chunk
+  end
+
+  def eof?
+    @already_read >= @compressed_data_size
+  end
+end

data/lib/zip_tricks/remote_io.rb

@@ -29,8 +29,7 @@ class ZipTricks::RemoteIO
   end
   
   # Emulates IO#read, but requires the number of bytes to read
-  # The method will raise if the number of bytes read from remote does
-  # not match the number requested. The read will be limited to the
+  # The read will be limited to the
   # size of the remote resource relative to the current offset in the IO,
   # so if you are at offset 0 in the IO of size 10, doing a `read(20)`
   # will only return you 10 bytes of result, and not raise any exceptions. 

data/lib/zip_tricks/remote_uncap.rb

@@ -7,55 +7,23 @@
 # before you use this module.
 class ZipTricks::RemoteUncap
 
-  # Represents a file embedded within a remote ZIP archive
-  class RemoteZipEntry
-
-    # @return [String] filename of the file in the remote ZIP
-    attr_accessor :name
-
-    # @return [Fixnum] size in bytes of the file when uncompressed
-    attr_accessor :size_uncompressed
-
-    # @return [Fixnum] size in bytes of the file when compressed (the segment in the ZIP)
-    attr_accessor :size_compressed
-
-    # @return [Fixnum] compression method (0 for stored, 8 for deflate)
-    attr_accessor :compression_method
-
-    # @return [Fixnum] where the file data starts within the ZIP
-    attr_accessor :starts_at_offset
-
-    # @return [Fixnum] where the file data ends within the zip.
-    #     Will be equal to starts_at_offset if the file is empty
-    attr_accessor :ends_at_offset
-
-    # Yields the object during initialization
-    def initialize
-      yield self
-    end
-  end
-
   # @param uri[String] the HTTP(S) URL to read the ZIP footer from 
-  # @return [Array<RemoteZipEntry>] metadata about the files within the remote archive
-  def self.files_within_zip_at(uri)
+  # @param reader_class[Class] which class to use for reading
+  # @param options_for_zip_reader[Hash] any additional options to give to {ZipTricks::FileReader} when reading
+  # @return [Array<ZipTricks::FileReader::ZipEntry>] metadata about the files within the remote archive
+  def self.files_within_zip_at(uri, reader_class: ZipTricks::FileReader, **options_for_zip_reader)
     fetcher = new(uri)
     fake_io = ZipTricks::RemoteIO.new(fetcher)
-    entries = ZipTricks.const_get(:FileReader).read_zip_structure(fake_io)
-    entries.map do | remote_entry |
-      RemoteZipEntry.new do | entry |
-        entry.name               = remote_entry.filename
-        entry.starts_at_offset   = remote_entry.compressed_data_offset
-        entry.size_uncompressed  = remote_entry.uncompressed_size
-        entry.size_compressed    = remote_entry.compressed_size
-        entry.compression_method = remote_entry.storage_mode
-      end
-    end
+    reader = reader_class.new
+    reader.read_zip_structure(io: fake_io, **options_for_zip_reader)
   end
 
   def initialize(uri)
     @uri = URI(uri)
   end
 
+  # Only used internally when reading the remote ZIP.
+  #
   # @param range[Range] the HTTP range of data to fetch from remote
   # @return [String] the response body of the ranged request
   def request_range(range)
@@ -65,6 +33,8 @@ class ZipTricks::RemoteUncap
     http.request(request).body
   end
 
+  # Only used internally when reading the remote ZIP.
+  #
   # @return [Fixnum] the byte size of the ranged request
   def request_object_size
     http = Net::HTTP.start(@uri.hostname, @uri.port)

data/lib/zip_tricks/streamer/writable.rb

@@ -16,5 +16,5 @@ class ZipTricks::Streamer::Writable
   #
   # @param d[String] the binary string to write (part of the uncompressed file)
   # @return [Fixnum] the number of bytes written
-  def write(d); @writer << d; end
+  def write(d); @writer << d; d.bytesize; end
 end

data/spec/spec_helper.rb

@@ -7,6 +7,17 @@ require 'digest'
 require 'fileutils'
 require 'shellwords'
 require 'zip'
+require 'delegate'
+
+class ReadMonitor < SimpleDelegator
+  def read(*)
+    super.tap { @num_reads ||= 0; @num_reads += 1 }
+  end
+  
+  def num_reads
+    @num_reads || 0
+  end
+end
 
 module Keepalive
   # Travis-CI kills the build if it does not receive output on standard out or standard error
@@ -45,7 +56,7 @@ module ZipInspection
     $zip_inspection_buf ||= StringIO.new
     $zip_inspection_buf.puts "\n"
     $zip_inspection_buf.puts "Inspecting ZIP output of #{inspect}." # The only way to get at the RSpec example without using the block argument
-    $zip_inspection_buf.puts "Be aware that the zipinfo version on OSX is too old to deal with Zip6."
+    $zip_inspection_buf.puts "Be aware that the zipinfo version on OSX is too old to deal with Zip64."
     escaped_cmd = Shellwords.join([zipinfo_path, '-tlhvz', path_to_zip])
     $zip_inspection_buf.puts `#{escaped_cmd}`
   end

data/spec/zip_tricks/file_reader_spec.rb

@@ -1,48 +1,142 @@
 require 'spec_helper'
-
 describe ZipTricks::FileReader do
-  it 'reads and uncompresses the file written deflated with data descriptors' do
-    zipfile = StringIO.new
-    tolstoy = File.read(__dir__ + '/war-and-peace.txt')
-    tolstoy.force_encoding(Encoding::BINARY)
-
-    ZipTricks::Streamer.open(zipfile) do |zip|
-      zip.write_deflated_file('war-and-peace.txt') do |sink|
-        sink << tolstoy
+  
+  describe 'with an end-to-end ZIP file to read' do
+    it 'reads and uncompresses the file written deflated with data descriptors' do
+      zipfile = StringIO.new
+      tolstoy = File.read(__dir__ + '/war-and-peace.txt')
+      tolstoy.force_encoding(Encoding::BINARY)
+
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        zip.write_deflated_file('war-and-peace.txt') do |sink|
+          sink << tolstoy
+        end
       end
+
+      entries = described_class.read_zip_structure(io: zipfile)
+      expect(entries.length).to eq(1)
+
+      entry = entries.first
+
+      readback = ''
+      reader = entry.extractor_from(zipfile)
+      readback << reader.extract(10) until reader.eof?
+
+      expect(readback.bytesize).to eq(tolstoy.bytesize)
+      expect(readback[0..10]).to eq(tolstoy[0..10])
+      expect(readback[-10..-1]).to eq(tolstoy[-10..-1])
     end
 
-    entries = described_class.read_zip_structure(zipfile)
-    expect(entries.length).to eq(1)
+    it 'performs local file header reads by default' do
+      zipfile = StringIO.new
+      tolstoy = File.read(__dir__ + '/war-and-peace.txt')
+      tolstoy.force_encoding(Encoding::BINARY)
 
-    entry = entries.first
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        40.times do |i|
+          zip.write_deflated_file('war-and-peace-%d.txt' % i) { |sink| sink << tolstoy }
+        end
+      end
+      zipfile.rewind
+    
+      read_monitor = ReadMonitor.new(zipfile)
+      entries = described_class.read_zip_structure(io: read_monitor, read_local_headers: true)
+      expect(read_monitor.num_reads).to eq(44)
+    end
 
-    readback = ''
-    reader = entry.extractor_from(zipfile)
-    readback << reader.extract(10) until reader.eof?
+    it 'performs local file header reads when `read_local_headers` is set to true' do
+      zipfile = StringIO.new
+      tolstoy = File.read(__dir__ + '/war-and-peace.txt')
+      tolstoy.force_encoding(Encoding::BINARY)
 
-    expect(readback.bytesize).to eq(tolstoy.bytesize)
-    expect(readback[0..10]).to eq(tolstoy[0..10])
-    expect(readback[-10..-1]).to eq(tolstoy[-10..-1])
-  end
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        40.times do |i|
+          zip.write_deflated_file('war-and-peace-%d.txt' % i) { |sink| sink << tolstoy }
+        end
+      end
+      zipfile.rewind
+    
+      read_monitor = ReadMonitor.new(zipfile)
+      entries = described_class.read_zip_structure(io: read_monitor, read_local_headers: true)
+      expect(read_monitor.num_reads).to eq(44)
 
-  it 'reads the file written stored with data descriptors' do
-    zipfile = StringIO.new
-    tolstoy = File.read(__dir__ + '/war-and-peace.txt')
-    ZipTricks::Streamer.open(zipfile) do |zip|
-      zip.write_stored_file('war-and-peace.txt') do |sink|
-        sink << tolstoy
+      expect(entries.length).to eq(40)
+      entry = entries.first
+      expect(entry).to be_known_offset
+    end
+
+    it 'performs a limited number of reads when `read_local_headers` is set to false' do
+      zipfile = StringIO.new
+      tolstoy = File.read(__dir__ + '/war-and-peace.txt')
+      tolstoy.force_encoding(Encoding::BINARY)
+
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        40.times do |i|
+          zip.write_deflated_file('war-and-peace-%d.txt' % i) { |sink| sink << tolstoy }
+        end
       end
+      zipfile.rewind
+      read_monitor = ReadMonitor.new(zipfile)
+
+      entries = described_class.read_zip_structure(io: read_monitor, read_local_headers: false)
+
+      expect(read_monitor.num_reads).to eq(4)
+      expect(entries.length).to eq(40)
+      entry = entries.first
+      expect(entry).not_to be_known_offset
+      expect {
+        entry.compressed_data_offset
+      }.to raise_error(/read/)
     end
 
-    entries = described_class.read_zip_structure(zipfile)
-    expect(entries.length).to eq(1)
+    it 'reads the file written stored with data descriptors' do
+      zipfile = StringIO.new
+      tolstoy = File.read(__dir__ + '/war-and-peace.txt')
+      ZipTricks::Streamer.open(zipfile) do |zip|
+        zip.write_stored_file('war-and-peace.txt') do |sink|
+          sink << tolstoy
+        end
+      end
 
-    entry = entries.first
+      entries = described_class.read_zip_structure(io: zipfile)
+      expect(entries.length).to eq(1)
 
-    readback = entry.extractor_from(zipfile).extract
-    expect(readback.bytesize).to eq(tolstoy.bytesize)
-    expect(readback[0..10]).to eq(tolstoy[0..10])
+      entry = entries.first
+
+      readback = entry.extractor_from(zipfile).extract
+      expect(readback.bytesize).to eq(tolstoy.bytesize)
+      expect(readback[0..10]).to eq(tolstoy[0..10])
+    end
+  end
+  
+  describe '#get_compressed_data_offset' do
+    it 'reads the offset for an entry having Zip64 extra fields' do
+      w = ZipTricks::ZipWriter.new
+      out = StringIO.new
+      out << Random.new.bytes(7656177)
+      w.write_local_file_header(io: out, filename: 'some file',
+        compressed_size: 0xFFFFFFFF + 5, uncompressed_size: 0xFFFFFFFFF, crc32: 123, gp_flags: 4,
+        mtime: Time.now, storage_mode: 8)
+      
+      out.rewind
+      
+      compressed_data_offset = subject.get_compressed_data_offset(io: out, local_file_header_offset: 7656177)
+      expect(compressed_data_offset).to eq(7656236)
+    end
+    
+    it 'reads the offset for an entry having a long name' do
+      w = ZipTricks::ZipWriter.new
+      out = StringIO.new
+      out << Random.new.bytes(7)
+      w.write_local_file_header(io: out, filename: 'This is a file with a ridiculously long name.doc',
+        compressed_size: 10, uncompressed_size: 15, crc32: 123, gp_flags: 4,
+        mtime: Time.now, storage_mode: 8)
+      
+      out.rewind
+      
+      compressed_data_offset = subject.get_compressed_data_offset(io: out, local_file_header_offset: 7)
+      expect(compressed_data_offset).to eq(85)
+    end
   end
   
   it 'is able to latch to the EOCD location even if the signature for the EOCD record appears all over the ZIP' do
@@ -63,7 +157,7 @@ describe ZipTricks::FileReader do
       central_directory_size: z.tell - where, num_files_in_archive: 1, comment: evil_str)
     
     z.rewind
-    entries = described_class.read_zip_structure(z)
+    entries = described_class.read_zip_structure(io: z)
     expect(entries.length).to eq(1)
   end
   

data/spec/zip_tricks/remote_uncap_spec.rb

@@ -16,19 +16,10 @@ describe ZipTricks::RemoteUncap, webmock: true do
     payload2 << Random.new.bytes(1024 * 1024 * 3)
     payload2.flush; payload2.rewind
 
-    payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
-    payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
-
     File.open('temp.zip', 'wb') do |f|
       ZipTricks::Streamer.open(f) do | zip |
-        zip.add_stored_entry(filename: 'first-file.bin', size: payload1.size, crc32: payload1_crc)
-        while blob = payload1.read(1024 * 5)
-          zip << blob
-        end
-        zip.add_stored_entry(filename: 'second-file.bin', size: payload2.size, crc32: payload2_crc)
-        while blob = payload2.read(1024 * 5)
-          zip << blob
-        end
+        zip.write_stored_file('first-file.bin') { |w| IO.copy_stream(payload1, w) }
+        zip.write_stored_file('second-file.bin') { |w| IO.copy_stream(payload2, w) }
       end
     end
     payload1.rewind; payload2.rewind
@@ -53,17 +44,17 @@ describe ZipTricks::RemoteUncap, webmock: true do
 
     first, second = *files
 
-    expect(first.name).to eq('first-file.bin')
-    expect(first.size_uncompressed).to eq(payload1.size)
+    expect(first.filename).to eq('first-file.bin')
+    expect(first.uncompressed_size).to eq(payload1.size)
     File.open('temp.zip', 'rb') do |readback|
-      readback.seek(first.starts_at_offset, IO::SEEK_SET)
+      readback.seek(first.compressed_data_offset, IO::SEEK_SET)
       expect(readback.read(12)).to eq(payload1.read(12))
     end
 
-    expect(second.name).to eq('second-file.bin')
-    expect(second.size_uncompressed).to eq(payload2.size)
+    expect(second.filename).to eq('second-file.bin')
+    expect(second.uncompressed_size).to eq(payload2.size)
     File.open('temp.zip', 'rb') do |readback|
-      readback.seek(second.starts_at_offset, IO::SEEK_SET)
+      readback.seek(second.compressed_data_offset, IO::SEEK_SET)
       expect(readback.read(12)).to eq(payload2.read(12))
     end
   end
@@ -79,50 +70,31 @@ describe ZipTricks::RemoteUncap, webmock: true do
     payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
     payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
 
-    File.open('temp.zip', 'wb') do |f|
-      ZipTricks::Streamer.open(f) do | zip |
-        zip.add_stored_entry(filename: 'first-file.bin', size: payload1.size, crc32: payload1_crc)
-        zip << '' # It is empty, so a read() would return nil
-        zip.add_stored_entry(filename: 'second-file.bin', size: payload2.size, crc32: payload2_crc)
-        while blob = payload2.read(1024 * 5)
-          zip << blob
-        end
-      end
+    readable_zip = Tempfile.new 'somezip'
+    ZipTricks::Streamer.open(readable_zip) do | zip |
+      zip.add_stored_entry(filename: 'first-file-zero-size.bin', size: payload1.size, crc32: payload1_crc)
+      zip.write_stored_file('second-file.bin') {|w| IO.copy_stream(payload2, w) }
     end
-    payload1.rewind; payload2.rewind
-
-    expect(File).to be_exist('temp.zip')
+    readable_zip.flush; readable_zip.rewind
 
     allow_any_instance_of(described_class).to receive(:request_object_size) {
-      File.size('temp.zip')
+      readable_zip.size
     }
     allow_any_instance_of(described_class).to receive(:request_range) {|_instance, range|
-      File.open('temp.zip', 'rb') do |f|
-        f.seek(range.begin)
-        f.read(range.end - range.begin + 1)
-      end
+      readable_zip.seek(range.begin, IO::SEEK_SET)
+      readable_zip.read(range.end - range.begin + 1)
     }
 
     payload1.rewind; payload2.rewind
 
-    files = described_class.files_within_zip_at('http://fake.example.com')
-    expect(files).to be_kind_of(Array)
-    expect(files.length).to eq(2)
-
-    first, second = *files
-
-    expect(first.name).to eq('first-file.bin')
-    expect(first.size_uncompressed).to eq(payload1.size)
-    File.open('temp.zip', 'rb') do |readback|
-      readback.seek(first.starts_at_offset, IO::SEEK_SET)
-      expect(readback.read(0)).to eq(payload1.read(0))
-    end
+    first, second  = described_class.files_within_zip_at('http://fake.example.com')
 
-    expect(second.name).to eq('second-file.bin')
-    expect(second.size_uncompressed).to eq(payload2.size)
-    File.open('temp.zip', 'rb') do |readback|
-      readback.seek(second.starts_at_offset, IO::SEEK_SET)
-      expect(readback.read(12)).to eq(payload2.read(12))
-    end
+    expect(first.filename).to eq('first-file-zero-size.bin')
+    expect(first.compressed_size).to be_zero
+    
+    expect(second.filename).to eq('second-file.bin')
+    expect(second.uncompressed_size).to eq(payload2.size)
+    readable_zip.seek(second.compressed_data_offset, IO::SEEK_SET)
+    expect(readable_zip.read(12)).to eq(payload2.read(12))
   end
 end

data/spec/zip_tricks/streamer/writable_spec.rb

@@ -0,0 +1,27 @@
+require_relative '../../spec_helper'
+
+describe ZipTricks::Streamer::Writable do
+  describe '#<<' do
+    it 'writes the given data to the destination and returns self' do
+      buf = StringIO.new
+      subject = described_class.new(buf)
+
+      result = subject << 'hello!'
+
+      expect(buf.string).to eq('hello!')
+      expect(result).to eq(subject)
+    end
+  end
+
+  describe '#write' do
+    it 'writes the given data to the destination and returns the number of bytes written' do
+      buf = StringIO.new
+      subject = described_class.new(buf)
+
+      result = subject.write('hello!')
+
+      expect(buf.string).to eq('hello!')
+      expect(result).to eq(6)
+    end
+  end
+end

data/zip_tricks.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: zip_tricks 3.1.1 ruby lib
+# stub: zip_tricks 4.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "zip_tricks"
-  s.version = "3.1.1"
+  s.version = "4.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-08-17"
+  s.date = "2016-08-19"
   s.description = "Makes rubyzip stream, for real"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -36,6 +36,8 @@ Gem::Specification.new do |s|
     "lib/zip_tricks/block_deflate.rb",
     "lib/zip_tricks/block_write.rb",
     "lib/zip_tricks/file_reader.rb",
+    "lib/zip_tricks/file_reader/inflating_reader.rb",
+    "lib/zip_tricks/file_reader/stored_reader.rb",
     "lib/zip_tricks/null_writer.rb",
     "lib/zip_tricks/rack_body.rb",
     "lib/zip_tricks/remote_io.rb",
@@ -59,6 +61,7 @@ Gem::Specification.new do |s|
     "spec/zip_tricks/remote_uncap_spec.rb",
     "spec/zip_tricks/size_estimator_spec.rb",
     "spec/zip_tricks/stream_crc32_spec.rb",
+    "spec/zip_tricks/streamer/writable_spec.rb",
     "spec/zip_tricks/streamer_spec.rb",
     "spec/zip_tricks/war-and-peace.txt",
     "spec/zip_tricks/write_and_tell_spec.rb",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-17 00:00:00.000000000 Z
+date: 2016-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -187,6 +187,8 @@ files:
 - lib/zip_tricks/block_deflate.rb
 - lib/zip_tricks/block_write.rb
 - lib/zip_tricks/file_reader.rb
+- lib/zip_tricks/file_reader/inflating_reader.rb
+- lib/zip_tricks/file_reader/stored_reader.rb
 - lib/zip_tricks/null_writer.rb
 - lib/zip_tricks/rack_body.rb
 - lib/zip_tricks/remote_io.rb
@@ -210,6 +212,7 @@ files:
 - spec/zip_tricks/remote_uncap_spec.rb
 - spec/zip_tricks/size_estimator_spec.rb
 - spec/zip_tricks/stream_crc32_spec.rb
+- spec/zip_tricks/streamer/writable_spec.rb
 - spec/zip_tricks/streamer_spec.rb
 - spec/zip_tricks/war-and-peace.txt
 - spec/zip_tricks/write_and_tell_spec.rb