zip_kit 6.0.0

data/lib/zip_kit/stream_crc32.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# A simple stateful class for keeping track of a CRC32 value through multiple writes
+class ZipKit::StreamCRC32
+  include ZipKit::WriteShovel
+
+  STRINGS_HAVE_CAPACITY_SUPPORT = begin
+    String.new("", capacity: 1)
+    true
+  rescue ArgumentError
+    false
+  end
+  CRC_BUF_SIZE = 1024 * 512
+  private_constant :STRINGS_HAVE_CAPACITY_SUPPORT, :CRC_BUF_SIZE
+
+  # Compute a CRC32 value from an IO object. The object should respond to `read` and `eof?`
+  #
+  # @param io[IO] the IO to read the data from
+  # @return [Fixnum] the computed CRC32 value
+  def self.from_io(io)
+    # If we can specify the string capacity upfront we will not have to resize
+    # the string during operation. This saves time but is only available on
+    # recent Ruby 2.x versions.
+    blob = STRINGS_HAVE_CAPACITY_SUPPORT ? String.new("", capacity: CRC_BUF_SIZE) : +""
+    crc = new
+    crc << io.read(CRC_BUF_SIZE, blob) until io.eof?
+    crc.to_i
+  end
+
+  # Creates a new streaming CRC32 calculator
+  def initialize
+    @crc = Zlib.crc32
+  end
+
+  # Append data to the CRC32. Updates the contained CRC32 value in place.
+  #
+  # @param blob[String] the string to compute the CRC32 from
+  # @return [self]
+  def <<(blob)
+    @crc = Zlib.crc32(blob, @crc)
+    self
+  end
+
+  # Returns the CRC32 value computed so far
+  #
+  # @return [Fixnum] the updated CRC32 value for all the blobs so far
+  def to_i
+    @crc
+  end
+
+  # Appends a known CRC32 value to the current one, and combines the
+  # contained CRC32 value in-place.
+  #
+  # @param crc32[Fixnum] the CRC32 value to append
+  # @param blob_size[Fixnum] the size of the daata the `crc32` is computed from
+  # @return [Fixnum] the updated CRC32 value for all the blobs so far
+  def append(crc32, blob_size)
+    @crc = Zlib.crc32_combine(@crc, crc32, blob_size)
+  end
+end

data/lib/zip_kit/streamer/deflated_writer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Sends writes to the given `io` compressed using a `Zlib::Deflate`. Also
+# registers data passing through it in a CRC32 checksum calculator. Is made to be completely
+# interchangeable with the StoredWriter in terms of interface.
+class ZipKit::Streamer::DeflatedWriter
+  include ZipKit::WriteShovel
+
+  # The amount of bytes we will buffer before computing the intermediate
+  # CRC32 checksums. Benchmarks show that the optimum is 64KB (see
+  # `bench/buffered_crc32_bench.rb), if that is exceeded Zlib is going
+  # to perform internal CRC combine calls which will make the speed go down again.
+  CRC32_BUFFER_SIZE = 64 * 1024
+
+  def initialize(io)
+    @compressed_io = io
+    @deflater = ::Zlib::Deflate.new(Zlib::DEFAULT_COMPRESSION, -::Zlib::MAX_WBITS)
+    @crc = ZipKit::StreamCRC32.new
+    @crc_buf = ZipKit::WriteBuffer.new(@crc, CRC32_BUFFER_SIZE)
+  end
+
+  # Writes the given data into the deflater, and flushes the deflater
+  # after having written more than FLUSH_EVERY_N_BYTES bytes of data
+  #
+  # @param data[String] data to be written
+  # @return self
+  def <<(data)
+    @deflater.deflate(data) { |chunk| @compressed_io << chunk }
+    @crc_buf << data
+    self
+  end
+
+  # Returns the amount of data received for writing, the amount of
+  # compressed data written and the CRC32 checksum. The return value
+  # can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
+  #
+  # @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
+  def finish
+    @compressed_io << @deflater.finish until @deflater.finished?
+    @crc_buf.flush
+    {crc32: @crc.to_i, compressed_size: @deflater.total_out, uncompressed_size: @deflater.total_in}
+  ensure
+    @deflater.close
+  end
+end

data/lib/zip_kit/streamer/entry.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Is used internally by Streamer to keep track of entries in the archive during writing.
+# Normally you will not have to use this class directly
+class ZipKit::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_size,
+  :uncompressed_size, :storage_mode, :mtime,
+  :use_data_descriptor, :local_header_offset, :bytes_used_for_local_header, :bytes_used_for_data_descriptor, :unix_permissions)
+  def initialize(*)
+    super
+    filename.force_encoding(Encoding::UTF_8)
+    @requires_efs_flag = !(begin
+      filename.encode(Encoding::ASCII)
+    rescue
+      false
+    end)
+  end
+
+  def total_bytes_used
+    bytes_used_for_local_header + compressed_size + bytes_used_for_data_descriptor
+  end
+
+  # Set the general purpose flags for the entry. We care about is the EFS
+  # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
+  # bit so that the unarchiving application knows that the filename in the archive is UTF-8
+  # encoded, and not some DOS default. For ASCII entries it does not matter.
+  # Additionally, we care about bit 3 which toggles the use of the postfix data descriptor.
+  def gp_flags
+    flag = 0b00000000000
+    flag |= 0b100000000000 if @requires_efs_flag # bit 11
+    flag |= 0x0008 if use_data_descriptor # bit 3
+    flag
+  end
+
+  def filler?
+    false
+  end
+end

data/lib/zip_kit/streamer/filler.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# Is used internally by Streamer to keep track of entries in the archive during writing.
+# Normally you will not have to use this class directly
+class ZipKit::Streamer::Filler < Struct.new(:total_bytes_used)
+  def filler?
+    true
+  end
+end

data/lib/zip_kit/streamer/heuristic.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Will be used to pick whether to store a file in the `stored` or
+# `deflated` mode, by compressing the first N bytes of the file and
+# comparing the stored and deflated data sizes. If deflate produces
+# a sizable compression gain for this data, it will create a deflated
+# file inside the ZIP archive. If the file doesn't compress well, it
+# will use the "stored" mode for the entry. About 128KB of the
+# file will be buffered to pick the appropriate storage mode. The
+# Heuristic will call either `write_stored_file` or `write_deflated_file`
+# on the Streamer passed into it once it knows which compression
+# method should be applied
+class ZipKit::Streamer::Heuristic
+  include ZipKit::WriteShovel
+
+  BYTES_WRITTEN_THRESHOLD = 128 * 1024
+  MINIMUM_VIABLE_COMPRESSION = 0.75
+
+  def initialize(streamer, filename, **write_file_options)
+    @streamer = streamer
+    @filename = filename
+    @write_file_options = write_file_options
+
+    @buf = StringIO.new.binmode
+    @deflater = ::Zlib::Deflate.new(Zlib::DEFAULT_COMPRESSION, -::Zlib::MAX_WBITS)
+    @bytes_deflated = 0
+
+    @winner = nil
+  end
+
+  def <<(bytes)
+    if @winner
+      @winner << bytes
+    else
+      @buf << bytes
+      @deflater.deflate(bytes) { |chunk| @bytes_deflated += chunk.bytesize }
+      decide if @buf.size > BYTES_WRITTEN_THRESHOLD
+    end
+    self
+  end
+
+  def write(bytes)
+    self << bytes
+    bytes.bytesize
+  end
+
+  def close
+    decide unless @winner
+    @winner.close
+  end
+
+  private def decide
+    # Finish and then close the deflater - it has likely buffered some data
+    @bytes_deflated += @deflater.finish.bytesize until @deflater.finished?
+    # If the deflated version is smaller than the stored one
+    # - use deflate, otherwise stored
+    ratio = @bytes_deflated / @buf.size.to_f
+    @winner = if ratio <= MINIMUM_VIABLE_COMPRESSION
+      @streamer.write_deflated_file(@filename, **@write_file_options)
+    else
+      @streamer.write_stored_file(@filename, **@write_file_options)
+    end
+    # Copy the buffered uncompressed data into the newly initialized writable
+    @buf.rewind
+    IO.copy_stream(@buf, @winner)
+    @buf.truncate(0)
+  end
+end

data/lib/zip_kit/streamer/stored_writer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Sends writes to the given `io`, and also registers all the data passing
+# through it in a CRC32 checksum calculator. Is made to be completely
+# interchangeable with the DeflatedWriter in terms of interface.
+class ZipKit::Streamer::StoredWriter
+  include ZipKit::WriteShovel
+
+  # The amount of bytes we will buffer before computing the intermediate
+  # CRC32 checksums. Benchmarks show that the optimum is 64KB (see
+  # `bench/buffered_crc32_bench.rb), if that is exceeded Zlib is going
+  # to perform internal CRC combine calls which will make the speed go down again.
+  CRC32_BUFFER_SIZE = 64 * 1024
+
+  def initialize(io)
+    @io = ZipKit::WriteAndTell.new(io)
+    @crc_compute = ZipKit::StreamCRC32.new
+    @crc = ZipKit::WriteBuffer.new(@crc_compute, CRC32_BUFFER_SIZE)
+  end
+
+  # Writes the given data to the contained IO object.
+  #
+  # @param data[String] data to be written
+  # @return self
+  def <<(data)
+    @io << data
+    @crc << data
+    self
+  end
+
+  # Returns the amount of data written and the CRC32 checksum. The return value
+  # can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
+  #
+  # @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
+  def finish
+    @crc.flush
+    {crc32: @crc_compute.to_i, compressed_size: @io.tell, uncompressed_size: @io.tell}
+  end
+end

data/lib/zip_kit/streamer/writable.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# Gets yielded from the writing methods of the Streamer
+# and accepts the data being written into the ZIP for deflate
+# or stored modes. Can be used as a destination for `IO.copy_stream`
+#
+#    IO.copy_stream(File.open('source.bin', 'rb), writable)
+class ZipKit::Streamer::Writable
+  include ZipKit::WriteShovel
+
+  # Initializes a new Writable with the object it delegates the writes to.
+  # Normally you would not need to use this method directly
+  def initialize(streamer, writer)
+    @streamer = streamer
+    @writer = writer
+    @closed = false
+  end
+
+  # Writes the given data to the output stream
+  #
+  # @param d[String] the binary string to write (part of the uncompressed file)
+  # @return [self]
+  def <<(d)
+    raise "Trying to write to a closed Writable" if @closed
+    @writer << d
+    self
+  end
+
+  # Flushes the writer and recovers the CRC32/size values. It then calls
+  # `update_last_entry_and_write_data_descriptor` on the given Streamer.
+  def close
+    return if @closed
+    @streamer.update_last_entry_and_write_data_descriptor(**@writer.finish)
+    @closed = true
+  end
+end