zip_kit 6.3.4 → 6.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f513e5300c8daa205994e7aa5a957c2270ea7378fab0dd13f57effd8f431d1ea
-  data.tar.gz: 9873a993a94a45339e9a1f383862ed1cc942db2fd23f4d09f6da70705afcf123
+  metadata.gz: 83cdf922faca6692b384bc30ba8290f0f15a4567ebce61fccb792a5b5d7993fc
+  data.tar.gz: ba3a6fee022ba663d77306850246ac8b33ba5d1e10d81f636ace18cc882dc99f
 SHA512:
-  metadata.gz: 4e3ace1ea348f7232d04932981bb5a3d4fb2f1f5e32f01523d5420bff97cc119b7a54bc9bd0f2269f92785167fa2acfdf1e013551def137d69f02e559deae244
-  data.tar.gz: 41c9791db83d11845dfff560526bf4d32cfaf7a0be99933a20ef2677d2027064bd7dd8f1071ac6a7b1864362c6cefa439728b422dcdb9c7529394233ad1dd85b
+  metadata.gz: ad24aba7c7359f01a11e94ddd1e469fdac2a77c59ece3a113cacebaf900e1cec9a9eee462b385c05d9f1789f9f5c6b48da5098b060e2c5b2c99cc5e312c4d435
+  data.tar.gz: 96eb1200ee5bdf95e2c9f6b8f91d856607f7981f0eabf7c5cee75dbb3efa200c7642f0886ffdad1965da123281c9cfe8d545614616702cadf4dc207da83a293d

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## Unreleased
+
+## 6.3.5
+
+* Make `WriteBuffer#<<` much faster for lots of tiny writes, like the XML fragments written by libraries such as caxlsx. Strings are no longer copied with `String#b` before being appended: on Ruby 3.4+ `String#append_as_bytes` is used, on older Rubies the string is converted only if Ruby refuses to append it. Mixing binary strings and non-ASCII strings still works, and the writable still always receives binary strings.
+* `WriteBuffer` now flushes before a write would make the buffer exceed its size (instead of after), so that it never outputs chunks larger than the buffer size, and writes which are larger than the buffer size are still passed through without getting copied.
+* `WriteBuffer` no longer preallocates (and immediately discards) a zero-filled String of twice its buffer size, which makes writing ZIPs with lots of small files faster.
+* Require `stringio` in `ZipWriter`, which uses it. Previously writing a ZIP could fail with a `NameError` if nothing else in the process had loaded `stringio`.
+* Add `bench/write_buffer_bench.rb`
+
 ## 6.3.4
 
 * Fix a bug whereby `rollback!` would cause an exception without any entries having been written yet (rollback on first entry).

data/bench/write_buffer_bench.rb

@@ -0,0 +1,183 @@
+# Benchmarks ZipKit::WriteBuffer with lots of tiny writes (like the XML fragments
+# a library such as caxlsx produces) and with large writes, comparing it to the
+# previous implementations of WriteBuffer.
+#
+#   bundle exec ruby bench/write_buffer_bench.rb
+require "bundler"
+Bundler.setup
+
+require "benchmark"
+require "benchmark/ips"
+require_relative "../lib/zip_kit"
+
+# Initialization and flushing as in zip_kit 6.3.x. These are not subclasses of ZipKit::WriteBuffer
+# on purpose: with Ruby 3.4, instances of a subclass can have a different object shape, which
+# can make the instance variable caches of methods they share with ZipKit::WriteBuffer miss.
+module WriteBuffer63x
+  def initialize(writable, buffer_size)
+    @buf = ("\0".b * (buffer_size * 2)).clear
+    @buffer_size = buffer_size
+    @writable = writable
+  end
+
+  def flush
+    unless @buf.empty?
+      @writable << @buf
+      @buf.clear
+    end
+    self
+  end
+end
+
+# WriteBuffer#<< as of zip_kit 6.3.2
+class WriteBuffer632
+  include WriteBuffer63x
+
+  def <<(data)
+    if data.bytesize >= @buffer_size
+      flush unless @buf.empty?
+      @writable << data
+    else
+      @buf << data
+      flush if @buf.bytesize >= @buffer_size
+    end
+    self
+  end
+end
+
+# WriteBuffer#<< as of zip_kit 6.3.3/6.3.4, which copies every string with String#b
+class WriteBuffer634
+  include WriteBuffer63x
+
+  def <<(string)
+    if string.bytesize >= @buffer_size
+      flush
+      @writable << string.b
+    else
+      @buf << string.b
+      flush if @buf.bytesize >= @buffer_size
+    end
+    self
+  end
+end
+
+# The simplest possible buffer, which does not deal with encodings or large writes
+class NaiveBuffer
+  def initialize(io, buffer_size)
+    @io = io
+    @buffer_size = buffer_size
+    @buf = "".b
+  end
+
+  def <<(fragment)
+    @buf << fragment
+    flush if @buf.bytesize >= @buffer_size
+    self
+  end
+
+  def flush
+    return if @buf.empty?
+    @io << @buf
+    @buf.clear
+  end
+end
+
+BUFFER_SIZE = 64 * 1024
+IMPLEMENTATIONS = {
+  "WriteBuffer" => ZipKit::WriteBuffer,
+  "WriteBuffer (6.3.2)" => WriteBuffer632,
+  "WriteBuffer (6.3.4, String#b)" => WriteBuffer634,
+  "Naive String buffer" => NaiveBuffer
+}
+
+# Fragments like the ones produced by caxlsx when writing out a worksheet: mostly frozen
+# literals and short dynamic strings (UTF-8 or US-ASCII), mostly ASCII-only.
+fragments = []
+20_000.times do |row|
+  fragments << "<row r=\"" << (row + 1).to_s << "\">"
+  6.times do |col|
+    fragments << "<c r=\"" << "#{("A".ord + col).chr}#{row + 1}" << "\" s=\"" << "1" << "\""
+    fragments << ((col == 3) ? " t=\"inlineStr\"><is><t>Grüße, #{row}</t></is>" : "><v>#{row * col}</v>")
+    fragments << "</c>"
+  end
+  fragments << "</row>"
+end
+n_bytes = fragments.sum(&:bytesize)
+
+puts RUBY_DESCRIPTION
+puts "#{fragments.length} tiny writes (#{n_bytes} bytes, #{(n_bytes.to_f / fragments.length).round(1)} bytes per write on average)"
+
+Benchmark.ips do |x|
+  x.config(time: 5, warmup: 2)
+  IMPLEMENTATIONS.each do |name, buffer_class|
+    x.report("#{name}, tiny writes into CRC32") do
+      buf = buffer_class.new(ZipKit::StreamCRC32.new, BUFFER_SIZE)
+      fragments.each { |fragment| buf << fragment }
+      buf.flush
+    end
+  end
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  x.config(time: 5, warmup: 2)
+  IMPLEMENTATIONS.each do |name, buffer_class|
+    x.report("#{name}, tiny writes into write_deflated_file") do
+      ZipKit::Streamer.open(ZipKit::NullWriter) do |zip|
+        zip.write_deflated_file("sheet.xml") do |sink|
+          buf = buffer_class.new(sink, BUFFER_SIZE)
+          fragments.each { |fragment| buf << fragment }
+          buf.flush
+        end
+      end
+    end
+  end
+  x.report("No buffer, tiny writes into write_deflated_file") do
+    ZipKit::Streamer.open(ZipKit::NullWriter) do |zip|
+      zip.write_deflated_file("sheet.xml") do |sink|
+        fragments.each { |fragment| sink << fragment }
+      end
+    end
+  end
+  x.compare!
+end
+
+large_chunk = Random.new(42).bytes(1024 * 1024)
+Benchmark.ips do |x|
+  x.config(time: 5, warmup: 2)
+  IMPLEMENTATIONS.each do |name, buffer_class|
+    x.report("#{name}, 64 writes of 1MB into CRC32") do
+      buf = buffer_class.new(ZipKit::StreamCRC32.new, BUFFER_SIZE)
+      64.times { buf << large_chunk }
+      buf.flush
+    end
+  end
+  x.compare!
+end
+
+__END__
+
+Apple M1 Pro, macOS 15.7
+
+ruby 3.4.1 (2024-12-25 revision 48d4efcb85) +PRISM [arm64-darwin24], without YJIT
+920000 tiny writes (5189483 bytes, 5.6 bytes per write on average)
+
+Comparison:
+Naive String buffer, tiny writes into CRC32:       12.1 i/s
+WriteBuffer, tiny writes into CRC32:       12.0 i/s - 1.01x  slower
+WriteBuffer (6.3.2), tiny writes into CRC32:       10.3 i/s - 1.17x  slower
+WriteBuffer (6.3.4, String#b), tiny writes into CRC32:        8.0 i/s - 1.51x  slower
+
+Comparison:
+Naive String buffer, tiny writes into write_deflated_file:        7.6 i/s
+WriteBuffer, tiny writes into write_deflated_file:        7.4 i/s - 1.02x  slower
+WriteBuffer (6.3.2), tiny writes into write_deflated_file:        6.8 i/s - 1.12x  slower
+WriteBuffer (6.3.4, String#b), tiny writes into write_deflated_file:        5.7 i/s - 1.33x  slower
+No buffer, tiny writes into write_deflated_file:        1.8 i/s - 4.18x  slower
+
+Comparison:
+WriteBuffer (6.3.2), 64 writes of 1MB into CRC32:      464.4 i/s
+WriteBuffer (6.3.4, String#b), 64 writes of 1MB into CRC32:      463.0 i/s - same-ish: difference falls within error
+WriteBuffer, 64 writes of 1MB into CRC32:      462.9 i/s - same-ish: difference falls within error
+Naive String buffer, 64 writes of 1MB into CRC32:      121.5 i/s - 3.82x  slower
+

data/lib/zip_kit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZipKit
-  VERSION = "6.3.4"
+  VERSION = "6.3.5"
 end

data/lib/zip_kit/write_buffer.rb

@@ -12,10 +12,23 @@
 # lots of very small writes, and some degree of speedup (about 20%) can be achieved
 # with a buffer of a few KB.
 #
-# Note that there is no guarantee that the write buffer is going to flush at or above
-# the given `buffer_size`, because for writes which exceed the buffer size it will
-# first `flush` and then write through the oversized chunk, without buffering it. This
-# helps conserve memory. Also note that the buffer will *not* duplicate strings for you
+# The WriteBuffer is also useful in front of a `write_file` / `write_deflated_file` writable
+# if you are going to be appending lots of tiny strings (like XML fragments) to it. Every write
+# into a writable goes through Zlib separately, so coalescing those writes into bigger chunks
+# is much faster.
+#
+# All strings appended to the WriteBuffer are appended as bytes, and the buffer String
+# given to the writable is always in binary encoding (`Encoding::BINARY`). You can therefore mix
+# binary strings and strings in other encodings (for instance UTF-8 with non-ASCII characters)
+# without getting an `Encoding::CompatibilityError`. No intermediate copies of the strings
+# you append (like `String#b` would create) are made.
+#
+# Note that there is no guarantee that the write buffer is going to flush at exactly
+# the given `buffer_size`. The buffer gets flushed when the next write would make it exceed
+# `buffer_size`, so the chunks it outputs are usually a bit smaller than that (strings with
+# multibyte characters can make it go slightly over). For writes of `buffer_size` or larger
+# it will first `flush` and then write through the oversized chunk, without buffering it.
+# This helps conserve memory. Also note that the buffer will *not* duplicate strings for you
 # and *will* yield the same buffer String over and over, so if you are storing it in an
 # Array you might need to duplicate it.
 #
@@ -25,16 +38,19 @@
 # to `<<`. Therefore, if you need to retain the output of the WriteBuffer in, say, an Array,
 # you might need to `.dup` the `String` it gives you.
 class ZipKit::WriteBuffer
+  # String#append_as_bytes (Ruby 3.4+) appends the bytes of the string without any encoding
+  # negotiation, so the buffer always stays binary. Without it we use String#<<, see `append_bytes`.
+  APPEND_AS_BYTES = String.instance_methods.include?(:append_as_bytes)
+
   # Creates a new WriteBuffer bypassing into a given writable object
   #
   # @param writable[#<<] An object that responds to `#<<` with a String as argument
   # @param buffer_size[Integer] How many bytes to buffer
   def initialize(writable, buffer_size)
-    # Allocating the buffer using a zero-padded String as a variation
-    # on using capacity:, which JRuby apparently does not like very much. The
-    # desire here is that the buffer doesn't have to be resized during the lifetime
-    # of the object.
-    @buf = ("\0".b * (buffer_size * 2)).clear
+    # No capacity gets preallocated. String#clear releases the memory held by the String,
+    # so after the first flush the buffer would have to grow again anyway - and many
+    # WriteBuffers (like the ones used for the CRC32 of small ZIP entries) never fill up.
+    @buf = "".b
     @buffer_size = buffer_size
     @writable = writable
   end
@@ -45,11 +61,15 @@ class ZipKit::WriteBuffer
   # @param string[String] data to be written
   # @return self
   def <<(string)
-    if string.bytesize >= @buffer_size
-      flush # <- this is were we can output less than @buffer_size
+    if @buf.bytesize + string.bytesize < @buffer_size
+      APPEND_AS_BYTES ? @buf.append_as_bytes(string) : append_bytes(string)
+    elsif string.bytesize >= @buffer_size
+      flush
+      # String#b does not copy the bytes of a large String, the new String shares them
       @writable << string.b
     else
-      @buf << string.b
+      flush if @buf.bytesize + string.bytesize > @buffer_size
+      append_bytes(string)
       flush if @buf.bytesize >= @buffer_size
     end
     self
@@ -60,7 +80,8 @@ class ZipKit::WriteBuffer
   # @return self
   def flush
     unless @buf.empty?
-      @writable << @buf
+      # force_encoding does not copy the String, it only changes its encoding
+      @writable << @buf.force_encoding(Encoding::BINARY)
       @buf.clear
     end
     self
@@ -68,4 +89,19 @@ class ZipKit::WriteBuffer
 
   # `flush!` was renamed to `flush` but we preserve this method for backwards compatibility
   alias_method :flush!, :flush
+
+  private
+
+  # Appends the bytes of the string without copying it. Without String#append_as_bytes (Ruby < 3.4)
+  # String#<< is used, which may change the encoding of the buffer or raise if the encodings are
+  # incompatible - in that case we append the bytes of the string instead. The buffer is forced
+  # back into binary before it is handed to the writable, see `flush`.
+  def append_bytes(string)
+    return @buf.append_as_bytes(string) if APPEND_AS_BYTES
+
+    @buf << string
+  rescue Encoding::CompatibilityError
+    @buf.force_encoding(Encoding::BINARY)
+    @buf << string.b
+  end
 end

data/lib/zip_kit/zip_writer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "stringio"
+
 # A low-level ZIP file data writer. You can use it to write out various headers and central directory elements
 # separately. The class handles the actual encoding of the data according to the ZIP format APPNOTE document.
 #

data/rbi/zip_kit.rbi

@@ -1,6 +1,6 @@
 # typed: strong
 module ZipKit
-  VERSION = T.let("6.3.4", T.untyped)
+  VERSION = T.let("6.3.5", T.untyped)
 
   class Railtie < Rails::Railtie
   end
@@ -1737,10 +1737,23 @@ end, T.untyped)
   # lots of very small writes, and some degree of speedup (about 20%) can be achieved
   # with a buffer of a few KB.
   # 
-  # Note that there is no guarantee that the write buffer is going to flush at or above
-  # the given `buffer_size`, because for writes which exceed the buffer size it will
-  # first `flush` and then write through the oversized chunk, without buffering it. This
-  # helps conserve memory. Also note that the buffer will *not* duplicate strings for you
+  # The WriteBuffer is also useful in front of a `write_file` / `write_deflated_file` writable
+  # if you are going to be appending lots of tiny strings (like XML fragments) to it. Every write
+  # into a writable goes through Zlib separately, so coalescing those writes into bigger chunks
+  # is much faster.
+  # 
+  # All strings appended to the WriteBuffer are appended as bytes, and the buffer String
+  # given to the writable is always in binary encoding (`Encoding::BINARY`). You can therefore mix
+  # binary strings and strings in other encodings (for instance UTF-8 with non-ASCII characters)
+  # without getting an `Encoding::CompatibilityError`. No intermediate copies of the strings
+  # you append (like `String#b` would create) are made.
+  # 
+  # Note that there is no guarantee that the write buffer is going to flush at exactly
+  # the given `buffer_size`. The buffer gets flushed when the next write would make it exceed
+  # `buffer_size`, so the chunks it outputs are usually a bit smaller than that (strings with
+  # multibyte characters can make it go slightly over). For writes of `buffer_size` or larger
+  # it will first `flush` and then write through the oversized chunk, without buffering it.
+  # This helps conserve memory. Also note that the buffer will *not* duplicate strings for you
   # and *will* yield the same buffer String over and over, so if you are storing it in an
   # Array you might need to duplicate it.
   # 
@@ -1750,6 +1763,8 @@ end, T.untyped)
   # to `<<`. Therefore, if you need to retain the output of the WriteBuffer in, say, an Array,
   # you might need to `.dup` the `String` it gives you.
   class WriteBuffer
+    APPEND_AS_BYTES = T.let(String.instance_methods.include?(:append_as_bytes), T.untyped)
+
     # sord duck - #<< looks like a duck type, replacing with untyped
     # Creates a new WriteBuffer bypassing into a given writable object
     # 
@@ -1773,6 +1788,15 @@ end, T.untyped)
     # _@return_ — self
     sig { returns(T.untyped) }
     def flush; end
+
+    # sord omit - no YARD type given for "string", using untyped
+    # sord omit - no YARD return type given, using untyped
+    # Appends the bytes of the string without copying it. Without String#append_as_bytes (Ruby < 3.4)
+    # String#<< is used, which may change the encoding of the buffer or raise if the encodings are
+    # incompatible - in that case we append the bytes of the string instead. The buffer is forced
+    # back into binary before it is handed to the writable, see `flush`.
+    sig { params(string: T.untyped).returns(T.untyped) }
+    def append_bytes(string); end
   end
 
   # A lot of objects in ZipKit accept bytes that may be sent

data/rbi/zip_kit.rbs

@@ -1510,10 +1510,23 @@ module ZipKit
   # lots of very small writes, and some degree of speedup (about 20%) can be achieved
   # with a buffer of a few KB.
   # 
-  # Note that there is no guarantee that the write buffer is going to flush at or above
-  # the given `buffer_size`, because for writes which exceed the buffer size it will
-  # first `flush` and then write through the oversized chunk, without buffering it. This
-  # helps conserve memory. Also note that the buffer will *not* duplicate strings for you
+  # The WriteBuffer is also useful in front of a `write_file` / `write_deflated_file` writable
+  # if you are going to be appending lots of tiny strings (like XML fragments) to it. Every write
+  # into a writable goes through Zlib separately, so coalescing those writes into bigger chunks
+  # is much faster.
+  # 
+  # All strings appended to the WriteBuffer are appended as bytes, and the buffer String
+  # given to the writable is always in binary encoding (`Encoding::BINARY`). You can therefore mix
+  # binary strings and strings in other encodings (for instance UTF-8 with non-ASCII characters)
+  # without getting an `Encoding::CompatibilityError`. No intermediate copies of the strings
+  # you append (like `String#b` would create) are made.
+  # 
+  # Note that there is no guarantee that the write buffer is going to flush at exactly
+  # the given `buffer_size`. The buffer gets flushed when the next write would make it exceed
+  # `buffer_size`, so the chunks it outputs are usually a bit smaller than that (strings with
+  # multibyte characters can make it go slightly over). For writes of `buffer_size` or larger
+  # it will first `flush` and then write through the oversized chunk, without buffering it.
+  # This helps conserve memory. Also note that the buffer will *not* duplicate strings for you
   # and *will* yield the same buffer String over and over, so if you are storing it in an
   # Array you might need to duplicate it.
   # 
@@ -1523,6 +1536,8 @@ module ZipKit
   # to `<<`. Therefore, if you need to retain the output of the WriteBuffer in, say, an Array,
   # you might need to `.dup` the `String` it gives you.
   class WriteBuffer
+    APPEND_AS_BYTES: untyped
+
     # sord duck - #<< looks like a duck type, replacing with untyped
     # Creates a new WriteBuffer bypassing into a given writable object
     # 
@@ -1543,6 +1558,14 @@ module ZipKit
     # 
     # _@return_ — self
     def flush: () -> untyped
+
+    # sord omit - no YARD type given for "string", using untyped
+    # sord omit - no YARD return type given, using untyped
+    # Appends the bytes of the string without copying it. Without String#append_as_bytes (Ruby < 3.4)
+    # String#<< is used, which may change the encoding of the buffer or raise if the encodings are
+    # incompatible - in that case we append the bytes of the string instead. The buffer is forced
+    # back into binary before it is handed to the writable, see `flush`.
+    def append_bytes: (untyped string) -> untyped
   end
 
   # A lot of objects in ZipKit accept bytes that may be sent

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zip_kit
 version: !ruby/object:Gem::Version
-  version: 6.3.4
+  version: 6.3.5
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -11,7 +11,7 @@ authors:
 - Felix Bünemann
 bindir: exe
 cert_chain: []
-date: 2025-11-17 00:00:00.000000000 Z
+date: 2026-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -367,6 +367,7 @@ files:
 - README.md
 - RUBYZIP_DIFFERENCES.md
 - Rakefile
+- bench/write_buffer_bench.rb
 - examples/archive_size_estimate.rb
 - examples/config.ru
 - examples/deferred_write.rb