zip_kit 6.3.2 → 6.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ff5f4284066004d435d36f28020e6ef2279fe0abe36c520436db3bd39d7608a
-  data.tar.gz: ac9f9c4312c632410cee6ffb98c2ec471f9de16e5260793a30df8ac28639218d
+  metadata.gz: f513e5300c8daa205994e7aa5a957c2270ea7378fab0dd13f57effd8f431d1ea
+  data.tar.gz: 9873a993a94a45339e9a1f383862ed1cc942db2fd23f4d09f6da70705afcf123
 SHA512:
-  metadata.gz: 34389f0a2d38a532af341c7694fcd10c2bbfb2f819ce9d1168afd51fbfc815b8c4ef0b36929f286f3ad31ead98479b07b871ec35d1d8f5adefbf2bb2717f3f58
-  data.tar.gz: 821151643cc5adafd9fe3446412e9d9de60ff049f6bab3ca1e9a7c0dc24dbcef0688b8bc0ab1fa51d88feb85892b3707ccbf8483e41c532680c2e9ddb073e823
+  metadata.gz: 4e3ace1ea348f7232d04932981bb5a3d4fb2f1f5e32f01523d5420bff97cc119b7a54bc9bd0f2269f92785167fa2acfdf1e013551def137d69f02e559deae244
+  data.tar.gz: 41c9791db83d11845dfff560526bf4d32cfaf7a0be99933a20ef2677d2027064bd7dd8f1071ac6a7b1864362c6cefa439728b422dcdb9c7529394233ad1dd85b

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 6.3.4
+
+* Fix a bug whereby `rollback!` would cause an exception without any entries having been written yet (rollback on first entry).
+
+## 6.3.3
+
+* Make sure `Writable#<<` converts the strings it is given into binary if they are not already in binary. This fixes an issue where `Heuristic` would suddenly start forwarding strings as-is to downstream callees. There is a lot of spots where the string-to-write gets forwarded and converting in every single one will be quite wasteful, but it can be handy to do in a few key places.
+* Make sure `WritableBuffer#<<` converts the strings it is given into binary if they are not already in binary. This helps prevent an issue where the receiving object the buffer flushes to is in a different encoding than binary (and all of our use cases assume bytes anyway, except for filenames).
+* When rescuing a failed `write_file`, differentiate between `#close`
+  and `#release_resources_on_failure!`. Closing a Writable can still try
+  to do things to the Streamer output, it can try to write to the destination
+  IO which is no longer accepting writes and so on. What we do want is to
+  safely destroy the zlib deflaters.
+
 ## 6.3.2
 
 * Make sure `rollback!` correctly works with `write_file` and the original exception gets re-raised from `write_file` if

data/Rakefile

@@ -3,7 +3,6 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require "yard"
-require "rubocop/rake_task"
 require "standard/rake"
 
 task :format do
@@ -16,6 +15,7 @@ RSpec::Core::RakeTask.new(:spec)
 
 task :generate_typedefs do
   `bundle exec sord rbi/zip_kit.rbi`
+  `bundle exec sord rbi/zip_kit.rbs`
 end
 
 task default: [:spec, :standard, :generate_typedefs]

data/lib/zip_kit/streamer/deflated_writer.rb

@@ -5,6 +5,7 @@
 # interchangeable with the StoredWriter in terms of interface.
 class ZipKit::Streamer::DeflatedWriter
   include ZipKit::WriteShovel
+  include ZipKit::ZlibCleanup
 
   # The amount of bytes we will buffer before computing the intermediate
   # CRC32 checksums. Benchmarks show that the optimum is 64KB (see
@@ -42,4 +43,8 @@ class ZipKit::Streamer::DeflatedWriter
   ensure
     @deflater.close
   end
+
+  def release_resources_on_failure!
+    safely_dispose_of_incomplete_deflater(@deflater)
+  end
 end

data/lib/zip_kit/streamer/heuristic.rb

@@ -13,6 +13,8 @@ require "zlib"
 # on the Streamer passed into it once it knows which compression
 # method should be applied
 class ZipKit::Streamer::Heuristic < ZipKit::Streamer::Writable
+  include ZipKit::ZlibCleanup
+
   BYTES_WRITTEN_THRESHOLD = 128 * 1024
   MINIMUM_VIABLE_COMPRESSION = 0.75
 
@@ -21,7 +23,7 @@ class ZipKit::Streamer::Heuristic < ZipKit::Streamer::Writable
     @filename = filename
     @write_file_options = write_file_options
 
-    @buf = StringIO.new.binmode
+    @buf = +"".b # Just use a mutable String
     @deflater = ::Zlib::Deflate.new(Zlib::DEFAULT_COMPRESSION, -::Zlib::MAX_WBITS)
     @bytes_deflated = 0
 
@@ -35,7 +37,7 @@ class ZipKit::Streamer::Heuristic < ZipKit::Streamer::Writable
     else
       @buf << bytes
       @deflater.deflate(bytes) { |chunk| @bytes_deflated += chunk.bytesize }
-      decide if @buf.size > BYTES_WRITTEN_THRESHOLD
+      decide if @buf.bytesize > BYTES_WRITTEN_THRESHOLD
     end
     self
   end
@@ -48,13 +50,18 @@ class ZipKit::Streamer::Heuristic < ZipKit::Streamer::Writable
     @winner.close
   end
 
+  def release_resources_on_failure!
+    safely_dispose_of_incomplete_deflater(@deflater)
+    @winner&.release_resources_on_failure!
+  end
+
   private def decide
     # Finish and then close the deflater - it has likely buffered some data
     @bytes_deflated += @deflater.finish.bytesize until @deflater.finished?
 
     # If the deflated version is smaller than the stored one
     # - use deflate, otherwise stored
-    ratio = @bytes_deflated / @buf.size.to_f
+    ratio = @bytes_deflated / @buf.bytesize.to_f
     @winner = if ratio <= MINIMUM_VIABLE_COMPRESSION
       @streamer.write_deflated_file(@filename, **@write_file_options)
     else
@@ -62,9 +69,8 @@ class ZipKit::Streamer::Heuristic < ZipKit::Streamer::Writable
     end
 
     # Copy the buffered uncompressed data into the newly initialized writable
-    @buf.rewind
-    IO.copy_stream(@buf, @winner)
-    @buf.truncate(0)
+    @winner << @buf
+    @buf.clear
   ensure
     @deflater.close
   end

data/lib/zip_kit/streamer/stored_writer.rb

@@ -36,4 +36,8 @@ class ZipKit::Streamer::StoredWriter
     @crc.flush
     {crc32: @crc_compute.to_i, compressed_size: @io.tell, uncompressed_size: @io.tell}
   end
+
+  def release_resources_on_failure!
+    # Nothing to do
+  end
 end

data/lib/zip_kit/streamer/writable.rb

@@ -18,11 +18,11 @@ class ZipKit::Streamer::Writable
 
   # Writes the given data to the output stream
   #
-  # @param d[String] the binary string to write (part of the uncompressed file)
+  # @param string[String] the string to write (part of the uncompressed file)
   # @return [self]
-  def <<(d)
+  def <<(string)
     raise "Trying to write to a closed Writable" if @closed
-    @writer << d
+    @writer << string.b
     self
   end
 
@@ -33,4 +33,10 @@ class ZipKit::Streamer::Writable
     @streamer.update_last_entry_and_write_data_descriptor(**@writer.finish)
     @closed = true
   end
+
+  def release_resources_on_failure!
+    return if @closed
+    @closed = true
+    @writer.release_resources_on_failure!
+  end
 end

data/lib/zip_kit/streamer.rb

@@ -273,6 +273,11 @@ class ZipKit::Streamer
   #    output (using `IO.copy_stream` is a good approach).
   # @return [ZipKit::Streamer::Writable] without a block - the Writable sink which has to be closed manually
   def write_file(filename, modification_time: Time.now.utc, unix_permissions: nil, &blk)
+    # Reset rollback state when starting a new entry attempt, so that if this entry
+    # fails before writing a header, rollback! won't use stale values from a previous entry
+    @offset_before_last_local_file_header = nil
+    @remove_last_file_at_rollback = false
+
     writable = ZipKit::Streamer::Heuristic.new(self, filename, modification_time: modification_time, unix_permissions: unix_permissions)
     yield_or_return_writable(writable, &blk)
   end
@@ -510,8 +515,13 @@ class ZipKit::Streamer
     end
 
     # Create filler for the truncated or unusable local file entry that did get written into the output
-    filler_size_bytes = @out.tell - @offset_before_last_local_file_header
-    @files << Filler.new(filler_size_bytes)
+    # Only create a filler if a local file header was actually written (indicated by
+    # @offset_before_last_local_file_header being set). If it's nil, no header was written,
+    # so there's nothing to create a filler for.
+    if @offset_before_last_local_file_header
+      filler_size_bytes = @out.tell - @offset_before_last_local_file_header
+      @files << Filler.new(filler_size_bytes)
+    end
 
     @out.tell
   end
@@ -524,7 +534,7 @@ class ZipKit::Streamer
         yield(writable)
         writable.close
       rescue
-        writable.close
+        writable.release_resources_on_failure!
         rollback!
         raise
       end

data/lib/zip_kit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZipKit
-  VERSION = "6.3.2"
+  VERSION = "6.3.4"
 end

data/lib/zip_kit/write_buffer.rb

@@ -42,14 +42,14 @@ class ZipKit::WriteBuffer
   # Appends the given data to the write buffer, and flushes the buffer into the
   # writable if the buffer size exceeds the `buffer_size` given at initialization
   #
-  # @param data[String] data to be written
+  # @param string[String] data to be written
   # @return self
-  def <<(data)
-    if data.bytesize >= @buffer_size
-      flush unless @buf.empty? # <- this is were we can output less than @buffer_size
-      @writable << data
+  def <<(string)
+    if string.bytesize >= @buffer_size
+      flush # <- this is were we can output less than @buffer_size
+      @writable << string.b
     else
-      @buf << data
+      @buf << string.b
       flush if @buf.bytesize >= @buffer_size
     end
     self

data/lib/zip_kit/zlib_cleanup.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ZipKit::ZlibCleanup
+  # This method is used to flush and close the native zlib handles
+  # should an archiving routine encounter an error. This is necessary,
+  # since otherwise unclosed deflaters may hang around in memory
+  # indefinitely, creating leaks.
+  #
+  # @param [Zlib::Deflater?]deflater the deflater to safely finish and close
+  # @return void
+  def safely_dispose_of_incomplete_deflater(deflater)
+    return unless deflater
+
+    # It can be a bit tricky to close and dealloc the deflater correctly.
+    # We want to do the right things for it to be GCd, including the
+    # native zlib handle. Also, leaving zlib handles dangling around
+    # creates warnings with "...with N bytes remaining to read", which are an
+    # eyesore. But they are there for a reason - so that we don't forget to do
+    # exactly this.
+    if !deflater.closed? && !deflater.finished?
+      deflater.finish until deflater.finished?
+    end
+    deflater.close unless deflater.closed?
+  end
+end

data/lib/zip_kit.rb

@@ -24,6 +24,7 @@ module ZipKit
   autoload :WriteShovel, File.dirname(__FILE__) + "/zip_kit/write_shovel.rb"
   autoload :RackChunkedBody, File.dirname(__FILE__) + "/zip_kit/rack_chunked_body.rb"
   autoload :RackTempfileBody, File.dirname(__FILE__) + "/zip_kit/rack_tempfile_body.rb"
+  autoload :ZlibCleanup, File.dirname(__FILE__) + "/zip_kit/zlib_cleanup.rb"
 
   require_relative "zip_kit/railtie" if defined?(::Rails)
 end

data/rbi/zip_kit.rbi

@@ -1,6 +1,6 @@
 # typed: strong
 module ZipKit
-  VERSION = T.let("6.3.1", T.untyped)
+  VERSION = T.let("6.3.4", T.untyped)
 
   class Railtie < Rails::Railtie
   end
@@ -106,8 +106,12 @@ module ZipKit
   # Is used to write ZIP archives without having to read them back or to overwrite
   # data. It outputs into any object that supports `<<` or `write`, namely:
   # 
-  # An `Array`, `File`, `IO`, `Socket` and even `String` all can be output destinations
-  # for the `Streamer`.
+  # * `Array` - will contain binary strings
+  # * `File` - data will be written to it as it gets generated
+  # * `IO` (`Socket`, `StringIO`) - data gets written into it
+  # * `String` - in binary encoding and unfrozen - also makes a decent output target
+  # 
+  # or anything else that responds to `#<<` or `#write`.
   # 
   # You can also combine output through the `Streamer` with direct output to the destination,
   # all while preserving the correct offsets in the ZIP file structures. This allows usage
@@ -514,6 +518,10 @@ module ZipKit
     # is likely already on the wire. However, excluding the entry from the central directory of the ZIP
     # file will allow better-behaved ZIP unarchivers to extract the entries which did store correctly,
     # provided they read the ZIP from the central directory and not straight-ahead.
+    # Rolling back does not perform any writes.
+    # 
+    # `rollback!` gets called for you if an exception is raised inside the block of `write_file`,
+    # `write_deflated_file` and `write_stored_file`.
     # 
     # _@return_ — position in the output stream / ZIP archive
     # 
@@ -670,9 +678,9 @@ module ZipKit
 
       # Writes the given data to the output stream
       # 
-      # _@param_ `d` — the binary string to write (part of the uncompressed file)
-      sig { params(d: String).returns(T.self_type) }
-      def <<(d); end
+      # _@param_ `string` — the string to write (part of the uncompressed file)
+      sig { params(string: String).returns(T.self_type) }
+      def <<(string); end
 
       # sord omit - no YARD return type given, using untyped
       # Flushes the writer and recovers the CRC32/size values. It then calls
@@ -680,6 +688,10 @@ module ZipKit
       sig { returns(T.untyped) }
       def close; end
 
+      # sord omit - no YARD return type given, using untyped
+      sig { returns(T.untyped) }
+      def release_resources_on_failure!; end
+
       # Writes the given data to the output stream. Allows the object to be used as
       # a target for `IO.copy_stream(from, to)`
       # 
@@ -701,6 +713,7 @@ module ZipKit
     # on the Streamer passed into it once it knows which compression
     # method should be applied
     class Heuristic < ZipKit::Streamer::Writable
+      include ZipKit::ZlibCleanup
       BYTES_WRITTEN_THRESHOLD = T.let(128 * 1024, T.untyped)
       MINIMUM_VIABLE_COMPRESSION = T.let(0.75, T.untyped)
 
@@ -718,9 +731,25 @@ module ZipKit
       sig { returns(T.untyped) }
       def close; end
 
+      # sord omit - no YARD return type given, using untyped
+      sig { returns(T.untyped) }
+      def release_resources_on_failure!; end
+
       # sord omit - no YARD return type given, using untyped
       sig { returns(T.untyped) }
       def decide; end
+
+      # sord warn - "Zlib::Deflater?" does not appear to be a type
+      # This method is used to flush and close the native zlib handles
+      # should an archiving routine encounter an error. This is necessary,
+      # since otherwise unclosed deflaters may hang around in memory
+      # indefinitely, creating leaks.
+      # 
+      # _@param_ `deflater` — the deflater to safely finish and close
+      # 
+      # _@return_ — void
+      sig { params(deflater: SORD_ERROR_ZlibDeflater).returns(T.untyped) }
+      def safely_dispose_of_incomplete_deflater(deflater); end
     end
 
     # Sends writes to the given `io`, and also registers all the data passing
@@ -749,6 +778,10 @@ module ZipKit
       sig { returns(T::Hash[T.untyped, T.untyped]) }
       def finish; end
 
+      # sord omit - no YARD return type given, using untyped
+      sig { returns(T.untyped) }
+      def release_resources_on_failure!; end
+
       # Writes the given data to the output stream. Allows the object to be used as
       # a target for `IO.copy_stream(from, to)`
       # 
@@ -764,6 +797,7 @@ module ZipKit
     # interchangeable with the StoredWriter in terms of interface.
     class DeflatedWriter
       include ZipKit::WriteShovel
+      include ZipKit::ZlibCleanup
       CRC32_BUFFER_SIZE = T.let(64 * 1024, T.untyped)
 
       # sord omit - no YARD type given for "io", using untyped
@@ -787,6 +821,22 @@ module ZipKit
       sig { returns(T::Hash[T.untyped, T.untyped]) }
       def finish; end
 
+      # sord omit - no YARD return type given, using untyped
+      sig { returns(T.untyped) }
+      def release_resources_on_failure!; end
+
+      # sord warn - "Zlib::Deflater?" does not appear to be a type
+      # This method is used to flush and close the native zlib handles
+      # should an archiving routine encounter an error. This is necessary,
+      # since otherwise unclosed deflaters may hang around in memory
+      # indefinitely, creating leaks.
+      # 
+      # _@param_ `deflater` — the deflater to safely finish and close
+      # 
+      # _@return_ — void
+      sig { params(deflater: SORD_ERROR_ZlibDeflater).returns(T.untyped) }
+      def safely_dispose_of_incomplete_deflater(deflater); end
+
       # Writes the given data to the output stream. Allows the object to be used as
       # a target for `IO.copy_stream(from, to)`
       # 
@@ -1712,11 +1762,11 @@ end, T.untyped)
     # Appends the given data to the write buffer, and flushes the buffer into the
     # writable if the buffer size exceeds the `buffer_size` given at initialization
     # 
-    # _@param_ `data` — data to be written
+    # _@param_ `string` — data to be written
     # 
     # _@return_ — self
-    sig { params(data: String).returns(T.untyped) }
-    def <<(data); end
+    sig { params(string: String).returns(T.untyped) }
+    def <<(string); end
 
     # Explicitly flushes the buffer if it contains anything
     # 
@@ -1745,6 +1795,20 @@ end, T.untyped)
     def write(bytes); end
   end
 
+  module ZlibCleanup
+    # sord warn - "Zlib::Deflater?" does not appear to be a type
+    # This method is used to flush and close the native zlib handles
+    # should an archiving routine encounter an error. This is necessary,
+    # since otherwise unclosed deflaters may hang around in memory
+    # indefinitely, creating leaks.
+    # 
+    # _@param_ `deflater` — the deflater to safely finish and close
+    # 
+    # _@return_ — void
+    sig { params(deflater: SORD_ERROR_ZlibDeflater).returns(T.untyped) }
+    def safely_dispose_of_incomplete_deflater(deflater); end
+  end
+
   # Permits Deflate compression in independent blocks. The workflow is as follows:
   # 
   # * Run every block to compress through deflate_chunk, remove the header,