zip_tricks 2.5.0

data/lib/zip_tricks/streamer.rb

@@ -0,0 +1,175 @@
+# Is used to write streamed ZIP archives into the provided IO-ish object.
+# The output IO is never going to be rewound or seeked, so the output
+# of this object can be coupled directly to, say, a Rack output.
+#
+# Allows for splicing raw files (for "stored" entries without compression)
+# and splicing of deflated files (for "deflated" storage mode).
+#
+# For stored entries, you need to know the CRC32 (as a uint) and the filesize upfront,
+# before the writing of the entry body starts.
+#
+# For compressed entries, you need to know the bytesize of the precompressed entry
+# as well.
+class ZipTricks::Streamer
+  EntryBodySizeMismatch = Class.new(StandardError)
+  InvalidOutput = Class.new(ArgumentError)
+  
+  # Language encoding flag (EFS) bit (general purpose bit 11)
+  EFS = 0b100000000000
+  
+  # Default general purpose flags for each entry.
+  DEFAULT_GP_FLAGS = 0b00000000000
+  
+  # Creates a new Streamer on top of the given IO-ish object and yields it. Once the given block
+  # returns, the Streamer will have it's `close` method called, which will write out the central
+  # directory of the archive to the output.
+  #
+  # @param stream [IO] the destination IO for the ZIP (should respond to `tell` and `<<`)
+  # @yield [Streamer] the streamer that can be written to
+  def self.open(stream)
+    archive = new(stream)
+    yield(archive)
+    archive.close
+  end
+  
+  # Creates a new Streamer on top of the given IO-ish object.
+  #
+  # @param stream [IO] the destination IO for the ZIP (should respond to `tell` and `<<`)
+  def initialize(stream)
+    raise InvalidOutput, "The stream should respond to #<<" unless stream.respond_to?(:<<)
+    stream = ZipTricks::WriteAndTell.new(stream) unless stream.respond_to?(:tell) && stream.respond_to?(:advance_position_by)
+    @output_stream = stream
+    
+    @state_monitor = VeryTinyStateMachine.new(:before_entry, callbacks_to=self)
+    @state_monitor.permit_state :in_entry_header, :in_entry_body, :in_central_directory, :closed
+    @state_monitor.permit_transition :before_entry => :in_entry_header
+    @state_monitor.permit_transition :in_entry_header => :in_entry_body
+    @state_monitor.permit_transition :in_entry_body => :in_entry_header
+    @state_monitor.permit_transition :in_entry_body => :in_central_directory
+    @state_monitor.permit_transition :in_central_directory => :closed
+    
+    @entry_set = ::Zip::EntrySet.new
+  end
+
+  # Writes a part of a zip entry body (actual binary data of the entry) into the output stream.
+  #
+  # @param binary_data [String] a String in binary encoding
+  # @return self
+  def <<(binary_data)
+    @state_monitor.transition_or_maintain! :in_entry_body
+    @output_stream << binary_data
+    @bytes_written_for_entry += binary_data.bytesize
+    self
+  end
+  
+  # Advances the internal IO pointer to keep the offsets of the ZIP file in check. Use this if you are going
+  # to use accelerated writes to the socket (like the `sendfile()` call) after writing the headers, or if you
+  # just need to figure out the size of the archive.
+  #
+  # @param num_bytes [Numeric] how many bytes are going to be written bypassing the Streamer
+  # @return [Numeric] position in the output stream / ZIP archive
+  def simulate_write(num_bytes)
+    @state_monitor.transition_or_maintain! :in_entry_body
+    @output_stream.advance_position_by(num_bytes)
+    @bytes_written_for_entry += num_bytes
+    @output_stream.tell
+  end
+  
+  # Writes out the local header for an entry (file in the ZIP) that is using the deflated storage model (is compressed).
+  # Once this method is called, the `<<` method has to be called to write the actual contents of the body.
+  #
+  # Note that the deflated body that is going to be written into the output has to be _precompressed_ (pre-deflated)
+  # before writing it into the Streamer, because otherwise it is impossible to know it's size upfront.
+  #
+  # @param entry_name [String] the name of the file in the entry
+  # @param uncompressed_size [Fixnum] the size of the entry when uncompressed, in bytes
+  # @param crc32 [Fixnum] the CRC32 checksum of the entry when uncompressed
+  # @param compressed_size [Fixnum] the size of the compressed entry that is going to be written into the archive
+  # @return [Fixnum] the offset the output IO is at after writing the entry header
+  def add_compressed_entry(entry_name, uncompressed_size, crc32, compressed_size)
+    @state_monitor.transition! :in_entry_header
+    
+    entry = ::Zip::Entry.new(@file_name, entry_name)
+    entry.compression_method = Zip::Entry::DEFLATED
+    entry.crc = crc32
+    entry.size = uncompressed_size
+    entry.compressed_size = compressed_size
+    set_gp_flags_for_filename(entry, entry_name)
+    
+    @entry_set << entry
+    entry.write_local_entry(@output_stream)
+    @expected_bytes_for_entry = compressed_size
+    @bytes_written_for_entry = 0
+    @output_stream.tell
+  end
+  
+  # Writes out the local header for an entry (file in the ZIP) that is using the stored storage model (is stored as-is).
+  # Once this method is called, the `<<` method has to be called one or more times to write the actual contents of the body.
+  #
+  # @param entry_name [String] the name of the file in the entry
+  # @param uncompressed_size [Fixnum] the size of the entry when uncompressed, in bytes
+  # @param crc32 [Fixnum] the CRC32 checksum of the entry when uncompressed
+  # @return [Fixnum] the offset the output IO is at after writing the entry header
+  def add_stored_entry(entry_name, uncompressed_size, crc32)
+    @state_monitor.transition! :in_entry_header
+    
+    entry = ::Zip::Entry.new(@file_name, entry_name)
+    entry.compression_method = Zip::Entry::STORED
+    entry.crc = crc32
+    entry.size = uncompressed_size
+    entry.compressed_size = uncompressed_size
+    set_gp_flags_for_filename(entry, entry_name)
+    @entry_set << entry
+    entry.write_local_entry(@output_stream)
+    @bytes_written_for_entry = 0
+    @expected_bytes_for_entry = uncompressed_size
+    @output_stream.tell
+  end
+  
+  
+  # Writes out the global footer and the directory entry header and the global directory of the ZIP
+  # archive using the information about the entries added using `add_stored_entry` and `add_compressed_entry`.
+  #
+  # Once this method is called, the `Streamer` should be discarded (the ZIP archive is complete).
+  #
+  # @return [Fixnum] the offset the output IO is at after writing the central directory
+  def write_central_directory!
+    @state_monitor.transition! :in_central_directory
+    cdir = Zip::CentralDirectory.new(@entry_set, comment = nil)
+    cdir.write_to_stream(@output_stream)
+    @output_stream.tell
+  end
+  
+  # Closes the archive. Writes the central directory if it has not yet been written.
+  # Switches the Streamer into a state where it can no longer be written to.
+  #
+  # Once this method is called, the `Streamer` should be discarded (the ZIP archive is complete).
+  #
+  # @return [Fixnum] the offset the output IO is at after closing the archive
+  def close
+    write_central_directory! unless @state_monitor.in_state?(:in_central_directory)
+    @state_monitor.transition! :closed
+    @output_stream.tell
+  end
+    
+  private
+  
+  # Set the general purpose flags for the entry. The only flag we care about is the EFS
+  # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
+  # bit so that the unarchiving application knows that the filename in the archive is UTF-8
+  # encoded, and not some DOS default. For ASCII entries it does not matter.
+  def set_gp_flags_for_filename(entry, filename)
+    filename.encode(Encoding::ASCII)
+    entry.gp_flags = DEFAULT_GP_FLAGS
+  rescue Encoding::UndefinedConversionError #=> UTF8 filename
+    entry.gp_flags = DEFAULT_GP_FLAGS | EFS
+  end
+  
+  # Checks whether the number of bytes written conforms to the declared entry size
+  def leaving_in_entry_body_state
+    if @bytes_written_for_entry != @expected_bytes_for_entry
+      msg = "Wrong number of bytes written for entry (expected %d, got %d)" % [@expected_bytes_for_entry, @bytes_written_for_entry]
+      raise EntryBodySizeMismatch, msg
+    end
+  end
+end

data/lib/zip_tricks/write_and_tell.rb

@@ -0,0 +1,33 @@
+# A tiny wrapper over any object that supports :<<.
+# Adds :tell and :advance_position_by.
+class ZipTricks::WriteAndTell
+  def initialize(io)
+    @io = io
+    @pos = 0
+  end
+  
+  def <<(bytes)
+    return self if bytes.nil?
+    binary_bytes = binary(bytes)
+    @io << binary_bytes
+    @pos += binary_bytes.bytesize
+    self
+  end
+  
+  def advance_position_by(num_bytes)
+    @pos += num_bytes
+  end
+  
+  def tell
+    @pos
+  end
+  
+  private
+  
+  def binary(str)
+    return str if str.encoding == Encoding::BINARY
+    str.force_encoding(Encoding::BINARY)
+  rescue RuntimeError # the string is frozen
+    str.dup.force_encoding(Encoding::BINARY)
+  end
+end

data/lib/zip_tricks.rb

@@ -0,0 +1,9 @@
+require 'zip'
+require 'very_tiny_state_machine'
+
+module ZipTricks
+  VERSION = '2.5.0'
+  
+  # Require all the sub-components except myself
+  Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'zip_tricks'
+require 'digest'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end

data/spec/zip_tricks/block_deflate_spec.rb

@@ -0,0 +1,111 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::BlockDeflate do
+  def tag_deflated(deflated_string, raw_string)
+    [120, 156].pack("C*") + deflated_string + [3,0].pack("C*") + [Zlib.adler32(raw_string)].pack("N")
+  end
+  
+  describe '.deflate_chunk' do
+    it 'compresses a blob that can be inflated later, when the header, footer and adler32 are added' do
+      blob = 'compressible' * 1024 * 4
+      compressed = described_class.deflate_chunk(blob)
+      expect(compressed.bytesize).to be < blob.bytesize
+      complete_deflated_segment = tag_deflated(compressed, blob)
+      expect(Zlib.inflate(complete_deflated_segment)).to eq(blob)
+    end
+    
+    it 'removes the header' do
+      blob = 'compressible' * 1024 * 4
+      compressed = described_class.deflate_chunk(blob)
+      expect(compressed[0..1]).not_to eq([120, 156].pack("C*"))
+    end
+    
+    it 'removes the adler32' do
+      blob = 'compressible' * 1024 * 4
+      compressed = described_class.deflate_chunk(blob)
+      adler = [Zlib.adler32(blob)].pack("N")
+      expect(compressed).not_to end_with(adler)
+    end
+    
+    it 'removes the end marker' do
+      blob = 'compressible' * 1024 * 4
+      compressed = described_class.deflate_chunk(blob)
+      expect(compressed[-7..-5]).not_to eq([3,0].pack("C*"))
+    end
+    
+    it 'honors the compression level' do
+      deflater = Zlib::Deflate.new
+      expect(Zlib::Deflate).to receive(:new).with(2) { deflater }
+      blob = 'compressible' * 1024 * 4
+      compressed = described_class.deflate_chunk(blob, level: 2)
+    end
+  end
+  
+  describe 'deflate_in_blocks_and_terminate' do
+    it 'uses deflate_in_blocks' do
+      data = 'compressible' * 1024 * 1024 * 10
+      input = StringIO.new(data)
+      output = StringIO.new
+      block_size = 1024 * 64
+      expect(described_class).to receive(:deflate_in_blocks).with(input, output, level: -1, block_size: block_size).and_call_original
+      described_class.deflate_in_blocks_and_terminate(input, output, block_size: block_size)
+    end
+    
+    it 'passes a custom compression level' do
+      data = 'compressible' * 1024 * 1024 * 10
+      input = StringIO.new(data)
+      output = StringIO.new
+      expect(described_class).to receive(:deflate_in_blocks).with(input, output, level: 9, block_size: 5242880).and_call_original
+      described_class.deflate_in_blocks_and_terminate(input, output, level: Zlib::BEST_COMPRESSION)
+    end
+    
+    it 'writes the end marker' do
+      data = 'compressible' * 1024 * 1024 * 10
+      input = StringIO.new(data)
+      output = StringIO.new
+      described_class.deflate_in_blocks_and_terminate(input, output)
+      expect(output.string[-2..-1]).to eq([3,0].pack("C*"))
+    end
+  end
+  
+  describe '.write_terminator' do
+    it 'writes the terminator and returns 2 for number of bytes written' do
+      buf = double('IO')
+      expect(buf).to receive(:<<).with([3,0].pack("C*"))
+      expect(described_class.write_terminator(buf)).to eq(2)
+    end
+  end
+  
+  describe '.deflate_in_blocks' do
+    it 'honors the block size' do
+      data = 'compressible' * 1024 * 1024 * 10
+      input = StringIO.new(data)
+      output = StringIO.new
+      block_size = 1024 * 64
+    
+      num_chunks = (data.bytesize / block_size.to_f).ceil
+      expect(described_class).to receive(:deflate_chunk).exactly(num_chunks).times.and_call_original
+      expect(input).to receive(:read).with(block_size).exactly(num_chunks + 1).times.and_call_original
+      expect(output).to receive(:<<).exactly(num_chunks).times.and_call_original
+      
+      described_class.deflate_in_blocks(input, output, block_size: block_size)
+    end
+    
+    it 'does not write the end marker' do
+      input_string = 'compressible' * 1024 * 1024 * 10
+      output_string = ''
+      
+      described_class.deflate_in_blocks(StringIO.new(input_string), StringIO.new(output_string))
+      expect(output_string).not_to be_empty
+      expect(output_string).not_to end_with([3,0].pack("C*"))
+    end
+    
+    it 'returns the number of bytes written' do
+      input_string = 'compressible' * 1024 * 1024 * 10
+      output_string = ''
+      
+      num_bytes = described_class.deflate_in_blocks(StringIO.new(input_string), StringIO.new(output_string))
+      expect(num_bytes).to eq(245016)
+    end
+  end
+end

data/spec/zip_tricks/block_write_spec.rb

@@ -0,0 +1,95 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::BlockWrite do
+  it 'calls the given block each time data is written' do
+    blobs = []
+    adapter = described_class.new{|s|
+      blobs << s
+    }
+    
+    adapter << 'hello'
+    adapter << 'world'
+    adapter << '!'
+    
+    expect(blobs).to eq(['hello', 'world', '!'])
+  end
+  
+  it 'supports chained shovel' do
+    blobs = []
+    adapter = described_class.new{|s|
+      blobs << s
+    }
+    
+    adapter << 'hello' << 'world' << '!'
+    
+    expect(blobs).to eq(['hello', 'world', '!'])
+  end
+  
+  it 'can write in all possible encodings, even if the strings are frozen' do
+    destination = ''.encode(Encoding::BINARY)
+    
+    accum_string = ''
+    adapter = described_class.new{|s| accum_string << s }
+    
+    adapter << 'hello'
+    adapter << 'привет'
+    adapter << 'привет'.freeze
+    adapter << '!'
+    adapter << SecureRandom.random_bytes(1024)
+    
+    expect(accum_string.bytesize).to eq(1054)
+  end
+  
+  it 'can be closed' do
+    expect(described_class.new{}.close).to be_nil
+  end
+  
+  it 'forces the written strings to binary encoding' do
+    blobs = []
+    adapter = described_class.new{|s|
+      blobs << s
+    }
+    adapter << 'hello'.encode(Encoding::UTF_8)
+    adapter << 'world'.encode(Encoding::BINARY)
+    adapter << '!'
+    expect(blobs).not_to be_empty
+    blobs.each {|s| expect(s.encoding).to eq(Encoding::BINARY) }
+  end
+  
+  it 'omits strings of zero length' do
+    blobs = []
+    adapter = described_class.new{|s|
+      blobs << s
+    }
+    adapter << 'hello'
+    adapter << ''
+    adapter << '!'
+    expect(blobs).to eq(['hello', '!'])
+  end
+  
+  it 'omits nils' do
+    blobs = []
+    adapter = described_class.new{|s|
+      blobs << s
+    }
+    adapter << 'hello'
+    adapter << nil
+    adapter << '!'
+    expect(blobs).to eq(['hello', '!'])
+  end
+  
+  it 'raises a TypeError on specific unsupported methods' do
+    adapter = described_class.new {|s| }
+    expect {
+      adapter.seek(123)
+    }.to raise_error(/non\-rewindable/)
+    
+    expect {
+      adapter.to_s
+    }.to raise_error(/non\-rewindable/)
+    
+    expect {
+      adapter.pos = 123
+    }.to raise_error(/non\-rewindable/)
+  end
+end

data/spec/zip_tricks/manifest_spec.rb

@@ -0,0 +1,60 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::Manifest do
+  it 'builds a map of the contained ranges, and has its cumulative size match the predicted archive size exactly' do
+    # Generate a couple of random files
+    raw_file_1 = SecureRandom.random_bytes(1024 * 20)
+    raw_file_2 = SecureRandom.random_bytes(1024 * 128)
+    raw_file_3 = SecureRandom.random_bytes(1258695)
+    
+    manifest, bytesize = described_class.build do | builder |
+      r = builder.add_stored_entry(name: "first-file.bin", size_uncompressed: raw_file_1.size)
+      expect(r).to eq(builder), "add_stored_entry should return self"
+      
+      builder.add_stored_entry(name: "second-file.bin", size_uncompressed: raw_file_2.size)
+      
+      r = builder.add_compressed_entry(name: "second-file-comp.bin", size_uncompressed: raw_file_2.size, 
+        size_compressed: raw_file_3.size, segment_info: 'http://example.com/second-file-deflated-segment.bin')
+      expect(r).to eq(builder), "add_compressed_entry should return self"
+    end
+    
+    require 'range_utils'
+    
+    expect(manifest).to be_kind_of(Array)
+    total_size_of_all_parts = manifest.inject(0) do | total_bytes, span |
+      total_bytes + RangeUtils.size_from_range(span.byte_range_in_zip)
+    end
+    expect(total_size_of_all_parts).to eq(1410595)
+    expect(bytesize).to eq(1410595)
+    
+    expect(manifest.length).to eq(7)
+    
+    first_header = manifest[0]
+    expect(first_header.part_type).to eq(:entry_header)
+    expect(first_header.byte_range_in_zip).to eq(0..43)
+    expect(first_header.filename).to eq("first-file.bin")
+    expect(first_header.additional_metadata).to be_nil
+    
+    first_body = manifest[1]
+    expect(first_body.part_type).to eq(:entry_body)
+    expect(first_body.byte_range_in_zip).to eq(44..20523)
+    expect(first_body.filename).to eq("first-file.bin")
+    expect(first_body.additional_metadata).to be_nil
+    
+    third_header = manifest[4]
+    expect(third_header.part_type).to eq(:entry_header)
+    expect(third_header.byte_range_in_zip).to eq(151641..151690)
+    expect(third_header.filename).to eq("second-file-comp.bin")
+    expect(third_header.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
+    
+    third_body = manifest[5]
+    expect(third_body.part_type).to eq(:entry_body)
+    expect(third_body.byte_range_in_zip).to eq(151691..1410385)
+    expect(third_body.filename).to eq("second-file-comp.bin")
+    expect(third_body.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
+    
+    cd = manifest[-1]
+    expect(cd.part_type).to eq(:central_directory)
+    expect(cd.byte_range_in_zip).to eq(1410386..1410594)
+  end
+end

data/spec/zip_tricks/rack_body_spec.rb

@@ -0,0 +1,34 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::RackBody do
+  it 'is usable as a Rack response body, supports each() and close()' do
+    output_buf = Tempfile.new('output')
+    
+    file_body = SecureRandom.random_bytes(1024 * 1024 + 8981)
+    
+    body = described_class.new do | zip |
+      zip.add_stored_entry("A file", file_body.bytesize, Zlib.crc32(file_body))
+      zip << file_body
+    end
+    
+    body.each do | some_data |
+      output_buf << some_data
+    end
+    body.close
+    
+    output_buf.rewind
+    expect(output_buf.size).to eq(1057667)
+    
+    per_filename = {}
+    Zip::File.open(output_buf.path) do |zip_file|
+      # Handle entries one by one
+      zip_file.each do |entry|
+        # The entry name gets returned with a binary encoding, we have to force it back.
+        per_filename[entry.name] = entry.get_input_stream.read
+      end
+    end
+    
+    expect(per_filename).to have_key('A file')
+    expect(per_filename['A file'].bytesize).to eq(file_body.bytesize)
+  end
+end

data/spec/zip_tricks/stored_size_estimator_spec.rb

@@ -0,0 +1,22 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::StoredSizeEstimator do
+  it 'accurately predicts the output zip size' do
+    # Generate a couple of random files
+    raw_file_1 = SecureRandom.random_bytes(1024 * 20)
+    raw_file_2 = SecureRandom.random_bytes(1024 * 128)
+    raw_file_3 = SecureRandom.random_bytes(1258695)
+    
+    predicted_size = described_class.perform_fake_archiving do | estimator |
+      r = estimator.add_stored_entry("first-file.bin", raw_file_1.size)
+      expect(r).to eq(estimator), "add_stored_entry should return self"
+      
+      estimator.add_stored_entry("second-file.bin", raw_file_2.size)
+      
+      r = estimator.add_compressed_entry("second-file.bin", raw_file_2.size, raw_file_3.size)
+      expect(r).to eq(estimator), "add_compressed_entry should return self"
+    end
+    
+    expect(predicted_size).to eq(1410524)
+  end
+end

data/spec/zip_tricks/stream_crc32_spec.rb

@@ -0,0 +1,38 @@
+require_relative '../spec_helper'
+
+describe ZipTricks::StreamCRC32 do
+  it 'computes the CRC32 of a large binary file' do
+    raw = StringIO.new(SecureRandom.random_bytes(45 * 1024 * 1024))
+    crc = Zlib.crc32(raw.string)
+    via_from_io = described_class.from_io(raw)
+    expect(via_from_io).to eq(crc)
+  end
+  
+  it 'allows in-place updates' do
+    raw = StringIO.new(SecureRandom.random_bytes(45 * 1024 * 1024))
+    crc = Zlib.crc32(raw.string)
+    
+    stream_crc = described_class.new
+    stream_crc << raw.read(1024 * 64) until raw.eof?
+    expect(stream_crc.to_i).to eq(crc)
+  end
+  
+  it 'supports chained shovel' do
+    str = 'abcdef'
+    crc = Zlib.crc32(str)
+    
+    stream_crc = described_class.new
+    stream_crc << 'a' << 'b' << 'c' << 'd' << 'e' << 'f'
+    
+    expect(stream_crc.to_i).to eq(crc)
+  end
+  
+  it 'allows in-place update with a known value' do
+    crc = Zlib.crc32
+    
+    stream_crc = described_class.new
+    stream_crc << "This is some data"
+    stream_crc.append(45678, 12910)
+    expect(stream_crc.to_i).to eq(1555667875)
+  end
+end