zip_tricks 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0cda66fb0e3453ab3c239126c67686d124fcce5
-  data.tar.gz: 28327929e311c13ad556b85cef2ddf73ee1fc653
+  metadata.gz: 33d767b5968d6743916c79ae85953e802a3f3e4a
+  data.tar.gz: 30bcfd521a5db58cfea176d6b36c82c09609c485
 SHA512:
-  metadata.gz: 5f5e227ce148f6d7468371f8d8653b8fa15331f0495e674f0e8972fa32d532865757acfb1c0051b147fe44fcbc81e0798821bd3c7cf1dd75e4e3e85852715e90
-  data.tar.gz: 5f455aed3939b423b1459e510a00cfc749701102e1dc714677fc74a2e2de6b9b18f7aebf23a09ffc68b5bd1c51bb627e22c305f1d1ffc0b9c933a4954f111828
+  metadata.gz: 41b5f45f0ae560ca28386e939da7cbb4648fe34fe0c84acbff1a761786b797d1a17a30a33cf72201596783a0dfa307d8d7910089872d76aecaf8b10986a9ccd9
+  data.tar.gz: 0539bde9484ba1cb6bf580cfbc252271a1498bdbca9703701d22333a67f6d495360558590e6eaf2e945742a46e63bbe0d519f32664bdc329908a1b80f6ad8b76

data/.travis.yml

@@ -1,5 +1,9 @@
 rvm:
 - 2.1
 - 2.2
+- jruby-9.0
 sudo: false
 cache: bundler
+matrix:
+  allow_failures:
+    - rvm: jruby-9.0

data/Gemfile

@@ -1,13 +1,14 @@
 source "http://rubygems.org"
 
-gem 'rubyzip', '~> 1.1.7'
+gem 'rubyzip', '~> 1.1', '>= 1.1.7'
 gem 'very_tiny_state_machine', '~> 2'
 
 group :development do
+  gem 'range_utils'
+  gem 'rack', '~> 1.6' # For Jeweler
   gem 'rake', '~> 10.4'
   gem "rspec", "~> 3.2.0", '< 3.3'
-  gem "rdoc", "~> 3.12"
+  gem "yard", "~> 0.8"
   gem "bundler", "~> 1.0"
   gem "jeweler", "~> 2.0.1"
-  gem 'range_utils'
 end

data/README.md

@@ -6,7 +6,12 @@ Makes Rubyzip sing, dance and play saxophone for streaming applications.
 Spiritual successor to [zipline](https://github.com/fringd/zipline)
 
 Requires Ruby 2.1+, rubyzip and a couple of other gems (all available to jRuby as well).
-The library is composed of a loose set of modules which are described below.
+The library is composed of a loose set of modules.
+
+## Usage by example
+
+Check out the `examples/` directory at the root of the project. This will give you a good idea
+of various use cases the library supports.
 
 ## BlockDeflate
 

data/Rakefile

@@ -40,12 +40,10 @@ end
 
 task :default => :spec
 
-require 'rdoc/task'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "zip_tricks #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+require 'yard'
+desc "Generate YARD documentation"
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ['lib/**/*.rb', 'ext/**/*.c' ]
+  t.options = ['--markup markdown']
+  t.stats_options = ['--list-undoc']
 end

data/examples/archive_size_estimate.rb

@@ -0,0 +1,13 @@
+require_relative '../lib/zip_tricks'
+
+# Predict how large a ZIP file is going to be without having access to the actual
+# file contents, but using just the filenames (influences the file size) and the size
+# of the files
+zip_archive_size_in_bytes = ZipTricks::StoredSizeEstimator.perform_fake_archiving do |zip|
+  # Pretend we are going to make a ZIP file which contains a few
+  # MP4 files (those do not compress all too well)
+  zip.add_stored_entry("MOV_1234.MP4", 898090)
+  zip.add_stored_entry("MOV_1235.MP4", 7855126)
+end
+
+zip_archive_size_in_bytes #=> 8753438

data/examples/config.ru

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/rack_application.rb'
+
+# Demonstrates a Rack app that can offer a ZIP download composed
+# at runtime (see rack_application.rb)
+run ZipDownload.new

data/examples/parallel_compression_with_block_deflate.rb

@@ -0,0 +1,75 @@
+require_relative '../lib/zip_tricks'
+require 'tempfile'
+
+# This shows how to perform compression in parallel (a-la pigz, but in a less
+# advanced fashion since the compression tables are not shared - to minimize shared state).
+#
+# When using this approach, compressing a large file can be performed as a map-reduce operation.
+# First you prepare all the data per part of your (potentially very large) file, and then you use
+# the reduce task to combine that data into one linear zip. In this example we will generate threads
+# and collect their return values in the order the threads were launched, which guarantees a consistent
+# reduce.
+#
+# So, let each thread generate a part of the file, and also
+# compute the CRC32 of it. The thread will compress it's own part
+# as well, in an independent deflate segment - the threads do not share anything. You could also
+# multiplex this over multiple processes or even machines.
+threads = (0..12).map do
+  Thread.new do
+    source_tempfile = Tempfile.new 't'
+    source_tempfile.binmode
+  
+    # Fill the part with random content
+    12.times { source_tempfile << Random.new.bytes(1 * 1024 * 1024) }
+    source_tempfile.rewind
+  
+    # Compute the CRC32 of the source file
+    part_crc = ZipTricks::StreamCRC32.from_io(source_tempfile)
+    source_tempfile.rewind
+  
+    # Create a compressed part
+    compressed_tempfile = Tempfile.new('tc')
+    compressed_tempfile.binmode
+    ZipTricks::BlockDeflate.deflate_in_blocks(source_tempfile, compressed_tempfile)
+    
+    source_tempfile.close!
+    # The data that the splicing process needs.
+    [compressed_tempfile, part_crc, source_tempfile.size]
+  end
+end
+
+# Threads return us a tuple with [compressed_tempfile, source_part_size, source_part_crc]
+compressed_tempfiles_and_crc_of_parts = threads.map(&:join).map(&:value)
+
+# Now we need to compute the CRC32 of the _entire_ file, and it has to be the CRC32
+# of the _source_ file (uncompressed), not of the compressed variant. Handily we know
+entire_file_crc = ZipTricks::StreamCRC32.new
+compressed_tempfiles_and_crc_of_parts.each do | _, source_part_crc, source_part_size|
+  entire_file_crc.append(source_part_crc, source_part_size)
+end
+
+# We need to append the the terminator bytes to the end of the last part.
+last_compressed_part = compressed_tempfiles_and_crc_of_parts[-1][0]
+ZipTricks::BlockDeflate.write_terminator(last_compressed_part)
+
+# and we need to know how big the deflated segment of the ZIP is going to be, in total.
+# To figure that out we just sum the sizes of the files
+compressed_part_files = compressed_tempfiles_and_crc_of_parts.map(&:first)
+size_of_deflated_segment = compressed_part_files.map(&:size).inject(&:+)
+size_of_uncompressed_file = compressed_tempfiles_and_crc_of_parts.map{|e| e[2]}.inject(&:+)
+
+# And now we can create a ZIP with our compressed file in it's entirety.
+# We use a File as a destination here, but you can also use a socket or a
+# non-rewindable IO. ZipTricks never needs to rewind your output, since it is
+# made for streaming.
+output = File.open('zip_created_in_parallel.zip', 'wb')
+
+ZipTricks::Streamer.open(output) do | zip |
+  zip.add_compressed_entry("parallel.bin", size_of_uncompressed_file, entire_file_crc.to_i, size_of_deflated_segment)
+  compressed_part_files.each do |part_file|
+    part_file.rewind
+    while blob = part_file.read(2048)
+      zip << blob
+    end
+  end
+end

data/examples/rack_application.rb

@@ -0,0 +1,59 @@
+require_relative '../lib/zip_tricks'
+
+# An example of how you can create a Rack endpoint for your ZIP downloads.
+# NEVER run this in production - it is a huge security risk.
+# What this app will do is pick PATH_INFO (your request URL path)
+# and grab a file located at this path on your filesystem. The file will then
+# be added to a ZIP archive created completely programmatically. No data will be cached
+# on disk and the contents of the ZIP file will _not_ be buffered in it's entirety
+# before sending. Unless you use a buffering Rack server of course (WEBrick or Thin).
+class ZipDownload
+  def call(env)
+    file_path = env['PATH_INFO'] # Should be the absolute path on the filesystem
+    
+    # Open the file for binary reading
+    f = File.open(file_path, 'rb')
+    filename = File.basename(file_path)
+    
+    # Compute the CRC32 upfront. We do not use local footers for post-computing the CRC32,
+    # so you _do_ have to precompute it beforehand. Ideally, you would do that before
+    # storing the files you will be sending out later on.
+    crc32 = ZipTricks::StreamCRC32.from_io(f)
+    f.rewind
+    
+    # Compute the size of the download, so that a
+    # real Content-Length header can be sent. Also, if your download
+    # stops at some point, the downloading browser will be able to tell
+    # the user that the download stalled or was aborted in-flight.
+    # Note that using the size estimator here does _not_ read or compress
+    # your original file, so it is very fast.
+    size = ZipTricks::StoredSizeEstimator.perform_fake_archiving do |ar|
+      ar.add_stored_entry(filename, f.size)
+    end
+    
+    # Create a suitable Rack response body, that will support each(),
+    # close() and all the other methods. We can then return it up the stack.
+    zip_response_body = ZipTricks::RackBody.new do |zip|
+      begin
+        # We are adding only one file to the ZIP here, but you could do that
+        # with an arbitrary number of files of course.
+        zip.add_stored_entry(filename, f.size, crc32)
+        # Write the contents of the file. It is stored, so the writes go directly
+        # to the Rack output, bypassing any RubyZip deflaters/compressors. In fact you
+        # are yielding the "blob" string here directly to the Rack server handler.
+        while blob = f.read(1024 * 128)
+          zip << blob
+        end
+      ensure
+        f.close # Make sure the opened file we read from gets closed
+      end
+    end
+    
+    # Add a Content-Disposition so that the download has a .zip extension
+    # (this will not work well with UTF-8 filenames on Windows, but hey!)
+    content_disposition = 'attachment; filename=%s.zip' % filename
+    
+    # and return the response, adding the Content-Length we have computed earlier
+    [200, {'Content-Length' => size.to_s, 'Content-Disposition' => content_disposition}, zip_response_body]
+  end
+end

data/lib/zip_tricks/manifest.rb

@@ -15,7 +15,7 @@ class ZipTricks::Manifest < Struct.new(:zip_streamer, :io, :part_list)
   #     zip_spans[0] #=> Manifest::ZipSpan(part_type: :entry_header, byte_range_in_zip: 0..44, ...)
   #     zip_spans[-1] #=> Manifest::ZipSpan(part_type: :central_directory, byte_range_in_zip: 776721..898921, ...)
   #
-  # @return [Array<Array<ZipSpan>, Fixnum>] an array of byte spans within the final ZIP, and the total size of the archive
+  # @return [Array<ZipSpan>, Fixnum] an array of byte spans within the final ZIP, and the total size of the archive
   # @yield [Manifest] the manifest object you can add entries to
   def self.build
     output_io = ZipTricks::WriteAndTell.new(ZipTricks::NullWriter)

data/lib/zip_tricks/remote_io.rb

@@ -0,0 +1,85 @@
+# An object that fakes just-enough of an IO to be dangerous
+# - or, more precisely, to be useful as a source for the RubyZip
+# central directory parser
+class ZipTricks::RemoteIO
+  
+  # @param fetcher[#request_object_size, #request_range] an object that can fetch
+  def initialize(fetcher = :NOT_SET)
+    @pos = 0
+    @fetcher = fetcher
+    @remote_size = false
+  end
+  
+  # Emulates IO#seek
+  def seek(offset, mode = IO::SEEK_SET)
+    case mode
+      when IO::SEEK_SET
+        @remote_size ||= request_object_size
+        @pos = clamp(0, offset, @remote_size)
+      when IO::SEEK_END
+        @remote_size ||= request_object_size
+        @pos = clamp(0, @remote_size + offset, @remote_size)
+      else
+        raise Errno::ENOTSUP, "Seek mode #{mode.inspect} not supported"
+    end
+    0 # always return 0!
+  end
+
+  # Emulates IO#read
+  def read(n_bytes = nil)
+    @remote_size ||= request_object_size
+    
+    # If the resource is empty there is nothing to read
+    return nil if @remote_size.zero?
+    
+    maximum_avaialable = @remote_size - @pos
+    n_bytes ||= maximum_avaialable # nil == read to the end of file
+    raise ArgumentError, "No negative reads(#{n_bytes})" if n_bytes < 0
+    
+    n_bytes = clamp(0, n_bytes, maximum_avaialable)
+    
+    read_n_bytes_from_remote(@pos, n_bytes).tap do |data|
+      if data.bytesize != n_bytes
+        raise "Remote read returned #{data.bytesize} bytes instead of #{n_bytes} as requested"
+      end
+      @pos = clamp(0, @pos + data.bytesize, @remote_size)
+    end
+  end
+
+  # Returns the current pointer position within the IO.
+  # Not used by RubyZip but used in tests of our own
+  #
+  # @return [Fixnum]
+  def pos
+    @pos
+  end
+  
+  protected
+
+  def request_range(range)
+    @fetcher.request_range(range)
+  end
+  
+  def request_object_size
+    @fetcher.request_object_size
+  end
+  
+  # Reads N bytes at offset from remote
+  def read_n_bytes_from_remote(start_at, n_bytes)
+    range = (start_at..(start_at + n_bytes - 1))
+    request_range(range)
+  end
+  
+  # Reads the Content-Length and caches it
+  def remote_size
+    @remote_size ||= request_object_size
+  end
+  
+  private
+  
+  def clamp(a,b,c)
+    return a if b < a
+    return c if b > c
+    b
+  end
+end

data/lib/zip_tricks/remote_uncap.rb

@@ -0,0 +1,73 @@
+# Alows reading the central directory of a remote ZIP file without
+# downloading the entire file. The central directory provides the
+# offsets at which the actual file contents is located. You can then
+# use the `Range:` HTTP headers to download those entries separately.
+class ZipTricks::RemoteUncap
+    
+  # Represents a file embedded within a remote ZIP archive
+  class RemoteZipEntry
+    
+    # @return [String] filename of the file in the remote ZIP
+    attr_accessor :name
+    
+    # @return [Fixnum] size in bytes of the file when uncompressed
+    attr_accessor :size_uncompressed
+
+    # @return [Fixnum] size in bytes of the file when compressed (the segment in the ZIP)
+    attr_accessor :size_compressed
+    
+    # @return [Fixnum] compression method (0 for stored, 8 for deflate)
+    attr_accessor :compression_method
+    
+    # @return [Fixnum] where the file data starts within the ZIP
+    attr_accessor :starts_at_offset
+    
+    # @return [Fixnum] where the file data ends within the zip.
+    #     Will be equal to starts_at_offset if the file is empty
+    attr_accessor :ends_at_offset
+    
+    # Yields the object during initialization
+    def initialize
+      yield self
+    end
+  end
+  
+  # @param uri[String] the HTTP(S) URL to read the ZIP footer from 
+  # @return [Array<RemoteZipEntry>] metadata about the files within the remote archive
+  def self.files_within_zip_at(uri)
+    fetcher = new(uri)
+    fake_io = ZipTricks::RemoteIO.new(fetcher)
+    dir = Zip::CentralDirectory.read_from_stream(fake_io)
+    
+    dir.entries.map do | rubyzip_entry |
+      RemoteZipEntry.new do | entry |
+        entry.name = rubyzip_entry.name
+        entry.size_uncompressed = rubyzip_entry.size
+        entry.size_compressed = rubyzip_entry.compressed_size
+        entry.compression_method = rubyzip_entry.compression_method
+        
+        entry.starts_at_offset = rubyzip_entry.local_header_offset + rubyzip_entry.calculate_local_header_size
+        entry.ends_at_offset = entry.starts_at_offset + rubyzip_entry.compressed_size
+      end
+    end
+  end
+  
+  def initialize(uri)
+    @uri = URI(uri)
+  end
+  
+  # @param range[Range] the HTTP range of data to fetch from remote
+  # @return [String] the response body of the ranged request
+  def request_range(range)
+    request = Net::HTTP::Get.new(@uri)
+    request.range = range
+    http = Net::HTTP.start(@uri.hostname, @uri.port)
+    http.request(request).body
+  end
+
+  # @return [Fixnum] the byte size of the ranged request
+  def request_object_size
+    http = Net::HTTP.start(@uri.hostname, @uri.port)
+    http.request_head(uri)['Content-Length'].to_i
+  end
+end

data/lib/zip_tricks.rb

@@ -2,7 +2,7 @@ require 'zip'
 require 'very_tiny_state_machine'
 
 module ZipTricks
-  VERSION = '2.5.0'
+  VERSION = '2.6.0'
   
   # Require all the sub-components except myself
   Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }

data/spec/zip_tricks/remote_io_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+
+describe ZipTricks::RemoteIO do
+  
+  context 'working with the fetcher object' do
+    it 'asks the fetcher object to obtain the object size and the actual data when reading' do
+      mock_fetcher = double(request_object_size: 120, request_range: 'abc')
+      subject = described_class.new(mock_fetcher)
+      expect(subject.read(3)).to eq('abc')
+    end
+  end
+  
+  context 'when it internally addresses a remote resource' do
+    it 'requests the size of the resource once via #request_object_size and does neet to read if resource is empty' do
+      subject = described_class.new
+      expect(subject).to receive(:request_object_size).and_return(0)
+      expect(subject.read).to be_nil
+    end
+    
+    it 'performs remote reads when repeatedly requesting the same chunk, via #request_range' do
+      subject = described_class.new
+      
+      expect(subject).to receive(:request_object_size).and_return(120)
+      allow(subject).to receive(:request_range) {|range|
+        expect(range).to eq(5..14)
+        Random.new.bytes(10)
+      }
+      20.times do
+        subject.seek(5, IO::SEEK_SET)
+        subject.read(10)
+      end
+    end
+  end
+  
+  describe '#seek' do
+    context 'with an unsupported mode' do
+      it 'raises an error' do
+        uncap = described_class.new
+        expect {
+          uncap.seek(123, :UNSUPPORTED)
+        }.to raise_error(Errno::ENOTSUP)
+      end
+    end
+
+    context 'with SEEK_SET mode' do
+      it 'returns the offset of 10 when asked to seek to 10' do
+        uncap = described_class.new
+        expect(uncap).to receive(:request_object_size).and_return(100)
+        mode = IO::SEEK_SET
+        expect(uncap.seek(10, mode)).to eq(0)
+      end
+    end
+
+    context 'with SEEK_END mode' do
+      it 'seens to 10 bytes to the end of the IO' do
+        uncap = described_class.new
+        expect(uncap).to receive(:request_object_size).and_return(100)
+        
+        mode = IO::SEEK_END
+        offset = -10
+        expect(uncap.seek(-10, IO::SEEK_END)).to eq(0)
+        expect(uncap.pos).to eq(90)
+      end
+    end
+  end
+
+  describe '#read' do
+    before :each do
+      @buf = Tempfile.new('simulated-http')
+      @buf.binmode
+      5.times { @buf << Random.new.bytes(1024 * 1024 * 3) }
+      @buf.rewind
+      
+      @subject = described_class.new
+      
+      allow(@subject).to receive(:request_object_size).and_return(@buf.size)
+      allow(@subject).to receive(:request_range) {|range|
+        @buf.read[range].tap { @buf.rewind }
+      }
+    end
+    
+    after :each do
+      @buf.close; @buf.unlink
+    end
+    
+    context 'without arguments' do
+      it 'reads the entire buffer and alters the position pointer' do
+        expect(@subject.pos).to eq(0)
+        read = @subject.read
+        expect(read.bytesize).to eq(@buf.size)
+        expect(@subject.pos).to eq(@buf.size)
+      end
+    end
+
+    context 'with length' do
+      it 'returns exact amount of bytes at the start of the buffer' do
+        bytes_read = @subject.read(10)
+        expect(@subject.pos).to eq(10)
+        @buf.seek(0)
+        expect(bytes_read).to eq(@buf.read(10))
+      end
+
+      it 'returns exact amount of bytes from the middle of the buffer' do
+        @subject.seek(456, IO::SEEK_SET)
+        
+        bytes_read = @subject.read(10)
+        expect(@subject.pos).to eq(456+10)
+        
+        @buf.seek(456)
+        expect(bytes_read).to eq(@buf.read(10))
+      end
+
+      it 'returns the last N bytes it can read' do
+        at_end = @buf.size - 4
+        @subject.seek(at_end, IO::SEEK_SET)
+        
+        expect(@subject.pos).to eq(15728636)
+        bytes_read = @subject.read(10)
+        expect(@subject.pos).to eq(@buf.size) # Should have moved the pos pointer to the end
+        
+        expect(bytes_read.bytesize).to eq(4)
+
+        expect(@subject.pos).to eq(@buf.size)
+      end
+    end
+  end
+end

data/spec/zip_tricks/remote_uncap_spec.rb

@@ -0,0 +1,128 @@
+require 'spec_helper'
+
+describe ZipTricks::RemoteUncap, webmock: true do
+  let(:uri) { URI.parse('http://example.com/file.zip') }
+
+  after :each do
+    File.unlink('temp.zip') rescue Errno::ENOENT
+  end
+  
+  it 'returns an array of remote entries that can be used to fetch the segments from within the ZIP' do
+    payload1 = Tempfile.new 'payload1'
+    payload1 << Random.new.bytes((1024 * 1024 * 5) + 10)
+    payload1.flush; payload1.rewind;
+    
+    payload2 = Tempfile.new 'payload2'
+    payload2 << Random.new.bytes(1024 * 1024 * 3)
+    payload2.flush; payload2.rewind
+    
+    payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
+    payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
+
+    File.open('temp.zip', 'wb') do |f|
+      ZipTricks::Streamer.open(f) do | zip |
+        zip.add_stored_entry('first-file.bin', payload1.size, payload1_crc)
+        while blob = payload1.read(1024 * 5)
+          zip << blob
+        end
+        zip.add_stored_entry('second-file.bin', payload2.size, payload2_crc)
+        while blob = payload2.read(1024 * 5)
+          zip << blob
+        end
+      end
+    end
+    payload1.rewind; payload2.rewind
+    
+    expect(File).to be_exist('temp.zip')
+    
+    allow_any_instance_of(described_class).to receive(:request_object_size) {
+      File.size('temp.zip')
+    }
+    allow_any_instance_of(described_class).to receive(:request_range) {|_instance, range|
+      File.open('temp.zip', 'rb') do |f|
+        f.seek(range.begin)
+        f.read(range.end - range.begin + 1)
+      end
+    }
+    
+    payload1.rewind; payload2.rewind
+    
+    files = described_class.files_within_zip_at('http://fake.example.com')
+    expect(files).to be_kind_of(Array)
+    expect(files.length).to eq(2)
+    
+    first, second = *files
+    
+    expect(first.name).to eq('first-file.bin')
+    expect(first.size_uncompressed).to eq(payload1.size)
+    File.open('temp.zip', 'rb') do |readback|
+      readback.seek(first.starts_at_offset, IO::SEEK_SET)
+      expect(readback.read(12)).to eq(payload1.read(12))
+    end
+    
+    expect(second.name).to eq('second-file.bin')
+    expect(second.size_uncompressed).to eq(payload2.size)
+    File.open('temp.zip', 'rb') do |readback|
+      readback.seek(second.starts_at_offset, IO::SEEK_SET)
+      expect(readback.read(12)).to eq(payload2.read(12))
+    end
+  end
+  
+  it 'can cope with an empty file within the zip' do
+    payload1 = Tempfile.new 'payload1'
+    payload1.flush; payload1.rewind;
+    
+    payload2 = Tempfile.new 'payload2'
+    payload2 << Random.new.bytes(1024)
+    payload2.flush; payload2.rewind
+    
+    payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
+    payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
+
+    File.open('temp.zip', 'wb') do |f|
+      ZipTricks::Streamer.open(f) do | zip |
+        zip.add_stored_entry('first-file.bin', payload1.size, payload1_crc)
+        zip << '' # It is empty, so a read() would return nil
+        zip.add_stored_entry('second-file.bin', payload2.size, payload2_crc)
+        while blob = payload2.read(1024 * 5)
+          zip << blob
+        end
+      end
+    end
+    payload1.rewind; payload2.rewind
+    
+    expect(File).to be_exist('temp.zip')
+    
+    allow_any_instance_of(described_class).to receive(:request_object_size) {
+      File.size('temp.zip')
+    }
+    allow_any_instance_of(described_class).to receive(:request_range) {|_instance, range|
+      File.open('temp.zip', 'rb') do |f|
+        f.seek(range.begin)
+        f.read(range.end - range.begin + 1)
+      end
+    }
+    
+    payload1.rewind; payload2.rewind
+    
+    files = described_class.files_within_zip_at('http://fake.example.com')
+    expect(files).to be_kind_of(Array)
+    expect(files.length).to eq(2)
+    
+    first, second = *files
+    
+    expect(first.name).to eq('first-file.bin')
+    expect(first.size_uncompressed).to eq(payload1.size)
+    File.open('temp.zip', 'rb') do |readback|
+      readback.seek(first.starts_at_offset, IO::SEEK_SET)
+      expect(readback.read(0)).to eq(payload1.read(0))
+    end
+    
+    expect(second.name).to eq('second-file.bin')
+    expect(second.size_uncompressed).to eq(payload2.size)
+    File.open('temp.zip', 'rb') do |readback|
+      readback.seek(second.starts_at_offset, IO::SEEK_SET)
+      expect(readback.read(12)).to eq(payload2.read(12))
+    end
+  end
+end

data/zip_tricks.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: zip_tricks 2.5.0 ruby lib
+# stub: zip_tricks 2.6.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "zip_tricks"
-  s.version = "2.5.0"
+  s.version = "2.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-05-17"
+  s.date = "2016-07-01"
   s.description = "Makes rubyzip stream, for real"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -27,12 +27,18 @@ Gem::Specification.new do |s|
     "LICENSE.txt",
     "README.md",
     "Rakefile",
+    "examples/archive_size_estimate.rb",
+    "examples/config.ru",
+    "examples/parallel_compression_with_block_deflate.rb",
+    "examples/rack_application.rb",
     "lib/zip_tricks.rb",
     "lib/zip_tricks/block_deflate.rb",
     "lib/zip_tricks/block_write.rb",
     "lib/zip_tricks/manifest.rb",
     "lib/zip_tricks/null_writer.rb",
     "lib/zip_tricks/rack_body.rb",
+    "lib/zip_tricks/remote_io.rb",
+    "lib/zip_tricks/remote_uncap.rb",
     "lib/zip_tricks/stored_size_estimator.rb",
     "lib/zip_tricks/stream_crc32.rb",
     "lib/zip_tricks/streamer.rb",
@@ -42,6 +48,8 @@ Gem::Specification.new do |s|
     "spec/zip_tricks/block_write_spec.rb",
     "spec/zip_tricks/manifest_spec.rb",
     "spec/zip_tricks/rack_body_spec.rb",
+    "spec/zip_tricks/remote_io_spec.rb",
+    "spec/zip_tricks/remote_uncap_spec.rb",
     "spec/zip_tricks/stored_size_estimator_spec.rb",
     "spec/zip_tricks/stream_crc32_spec.rb",
     "spec/zip_tricks/streamer_spec.rb",
@@ -58,33 +66,36 @@ Gem::Specification.new do |s|
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<rubyzip>, ["~> 1.1.7"])
+      s.add_runtime_dependency(%q<rubyzip>, [">= 1.1.7", "~> 1.1"])
       s.add_runtime_dependency(%q<very_tiny_state_machine>, ["~> 2"])
+      s.add_development_dependency(%q<range_utils>, [">= 0"])
+      s.add_development_dependency(%q<rack>, ["~> 1.6"])
       s.add_development_dependency(%q<rake>, ["~> 10.4"])
       s.add_development_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
-      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<yard>, ["~> 0.8"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
-      s.add_development_dependency(%q<range_utils>, [">= 0"])
     else
-      s.add_dependency(%q<rubyzip>, ["~> 1.1.7"])
+      s.add_dependency(%q<rubyzip>, [">= 1.1.7", "~> 1.1"])
       s.add_dependency(%q<very_tiny_state_machine>, ["~> 2"])
+      s.add_dependency(%q<range_utils>, [">= 0"])
+      s.add_dependency(%q<rack>, ["~> 1.6"])
       s.add_dependency(%q<rake>, ["~> 10.4"])
       s.add_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
-      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<yard>, ["~> 0.8"])
       s.add_dependency(%q<bundler>, ["~> 1.0"])
       s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
-      s.add_dependency(%q<range_utils>, [">= 0"])
     end
   else
-    s.add_dependency(%q<rubyzip>, ["~> 1.1.7"])
+    s.add_dependency(%q<rubyzip>, [">= 1.1.7", "~> 1.1"])
     s.add_dependency(%q<very_tiny_state_machine>, ["~> 2"])
+    s.add_dependency(%q<range_utils>, [">= 0"])
+    s.add_dependency(%q<rack>, ["~> 1.6"])
     s.add_dependency(%q<rake>, ["~> 10.4"])
     s.add_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
-    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<yard>, ["~> 0.8"])
     s.add_dependency(%q<bundler>, ["~> 1.0"])
     s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
-    s.add_dependency(%q<range_utils>, [">= 0"])
   end
 end
 

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.6.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-17 00:00:00.000000000 Z
+date: 2016-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.7
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.7
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: very_tiny_state_machine
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +44,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: range_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -73,19 +107,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.2.0
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -114,20 +148,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.1
-- !ruby/object:Gem::Dependency
-  name: range_utils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Makes rubyzip stream, for real
 email: me@julik.nl
 executables: []
@@ -144,12 +164,18 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/archive_size_estimate.rb
+- examples/config.ru
+- examples/parallel_compression_with_block_deflate.rb
+- examples/rack_application.rb
 - lib/zip_tricks.rb
 - lib/zip_tricks/block_deflate.rb
 - lib/zip_tricks/block_write.rb
 - lib/zip_tricks/manifest.rb
 - lib/zip_tricks/null_writer.rb
 - lib/zip_tricks/rack_body.rb
+- lib/zip_tricks/remote_io.rb
+- lib/zip_tricks/remote_uncap.rb
 - lib/zip_tricks/stored_size_estimator.rb
 - lib/zip_tricks/stream_crc32.rb
 - lib/zip_tricks/streamer.rb
@@ -159,6 +185,8 @@ files:
 - spec/zip_tricks/block_write_spec.rb
 - spec/zip_tricks/manifest_spec.rb
 - spec/zip_tricks/rack_body_spec.rb
+- spec/zip_tricks/remote_io_spec.rb
+- spec/zip_tricks/remote_uncap_spec.rb
 - spec/zip_tricks/stored_size_estimator_spec.rb
 - spec/zip_tricks/stream_crc32_spec.rb
 - spec/zip_tricks/streamer_spec.rb