zip_tricks 5.2.0 → 5.6.0

data/lib/zip_tricks/output_enumerator.rb

@@ -1,43 +1,64 @@
 # frozen_string_literal: true
 
-# Can be used as a Rack response body directly. Will yield
-# a {ZipTricks::Streamer} for adding entries to the archive and writing
-# zip entry bodies.
+# The output enumerator makes it possible to "pull" from a ZipTricks streamer
+# object instead of having it "push" writes to you. It will "stash" the block which
+# writes the ZIP archive through the streamer, and when you call `each` on the Enumerator
+# it will yield you the bytes the block writes. Since it is an enumerator you can
+# use `next` to take chunks written by the ZipTricks streamer one by one. It can be very
+# convenient when you need to segment your ZIP output into bigger chunks for, say,
+# uploading them to a cloud storage provider such as S3.
+#
+# Another use of the output enumerator is outputting a ZIP archive from Rails or Rack,
+# where an object responding to `each` is required which yields Strings. For instance,
+# you can return a ZIP archive from Rack like so:
+#
+#     iterable_zip_body = ZipTricks::OutputEnumerator.new do | streamer |
+#       streamer.write_deflated_file('big.csv') do |sink|
+#         CSV(sink) do |csv_writer|
+#           csv_writer << Person.column_names
+#           Person.all.find_each do |person|
+#             csv_writer << person.attributes.values
+#           end
+#         end
+#       end
+#     end
+#
+#     [200, {'Content-Type' => 'binary/octet-stream'}, iterable_zip_body]
 class ZipTricks::OutputEnumerator
-  # Prepares a new Rack response body with a Zip output stream.
-  # The block given to the constructor will be called when the response
-  # body will be read by the webserver, and will receive a {ZipTricks::Streamer}
-  # as it's block argument. You can then add entries to the Streamer as usual.
-  # The archive will be automatically closed at the end of the block.
+  DEFAULT_WRITE_BUFFER_SIZE = 64 * 1024
+  # Creates a new OutputEnumerator.
   #
-  #     # Precompute the Content-Length ahead of time
-  #     content_length = ZipTricks::SizeEstimator.estimate do | estimator |
-  #       estimator.add_stored_entry(filename: 'large.tif', size: 1289894)
-  #     end
-  #
-  #     # Prepare the response body.
-  #     # The block will only be called when the
-  #     # response starts to be written.
-  #     body = ZipTricks::OutputEnumerator.new do | streamer |
-  #       streamer.add_stored_entry(filename: 'large.tif', size: 1289894, crc32: 198210)
-  #       streamer << large_file.read(1024*1024) until large_file.eof?
-  #       ...
-  #     end
-  #
-  #     return [200, {'Content-Type' => 'binary/octet-stream',
-  #     'Content-Length' => content_length.to_s}, body]
-  def initialize(**streamer_options, &blk)
+  # @param streamer_options[Hash] options for Streamer, see {ZipTricks::Streamer.new}
+  # @param write_buffer_size[Integer] By default all ZipTricks writes are unbuffered. For output to sockets
+  #     it is beneficial to bulkify those writes so that they are roughly sized to a socket buffer chunk. This
+  #     object will bulkify writes for you in this way (so `each` will yield not on every call to `<<` from the Streamer
+  #     but at block size boundaries or greater). Set it to 0 for unbuffered writes.
+  # @param blk a block that will receive the Streamer object when executing. The block will not be executed
+  #     immediately but only once `each` is called on the OutputEnumerator
+  def initialize(write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, **streamer_options, &blk)
     @streamer_options = streamer_options.to_h
+    @bufsize = write_buffer_size.to_i
     @archiving_block = blk
   end
 
   # Executes the block given to the constructor with a {ZipTricks::Streamer}
   # and passes each written chunk to the block given to the method. This allows one
-  # to "take" output of the ZIP piecewise.
+  # to "take" output of the ZIP piecewise. If called without a block will return an Enumerator
+  # that you can pull data from using `next`.
+  #
+  # **NOTE** Because the `WriteBuffer` inside this object can reuse the buffer, it is important
+  #    that the `String` that is yielded **either** gets consumed eagerly (written byte-by-byte somewhere, or `#dup`-ed)
+  #    since the write buffer will clear it after your block returns. If you expand this Enumerator
+  #    eagerly into an Array you might notice that a lot of the segments of your ZIP output are
+  #    empty - this means that you need to duplicate them.
+  #
+  # @yield [String] a chunk of the ZIP output in binary encoding
   def each
     if block_given?
       block_write = ZipTricks::BlockWrite.new { |chunk| yield(chunk) }
-      ZipTricks::Streamer.open(block_write, **@streamer_options, &@archiving_block)
+      buffer = ZipTricks::WriteBuffer.new(block_write, @bufsize)
+      ZipTricks::Streamer.open(buffer, **@streamer_options, &@archiving_block)
+      buffer.flush
     else
       enum_for(:each)
     end

data/lib/zip_tricks/path_set.rb

@@ -32,6 +32,10 @@
 # conflict is avoided. This is not possible to apply to directories, because when one of the
 # path components is reused in multiple filenames it means those entities should end up in
 # the same directory (subdirectory) once the archive is opened.
+#
+# The `PathSet` keeps track of entries as they get added using 2 Sets (cheap presence checks),
+# one for directories and one for files. It will raise a `Conflict` exception if there are
+# files clobbering one another, or in case files collide with directories.
 class ZipTricks::PathSet
   class Conflict < StandardError
   end

data/lib/zip_tricks/rails_streaming.rb

@@ -1,22 +1,20 @@
 # frozen_string_literal: true
 
-# Should be included into a Rails controller (together with `ActionController::Live`)
-# for easy ZIP output from any action.
+# Should be included into a Rails controller for easy ZIP output from any action.
 module ZipTricks::RailsStreaming
   # Opens a {ZipTricks::Streamer} and yields it to the caller. The output of the streamer
   # gets automatically forwarded to the Rails response stream. When the output completes,
   # the Rails response stream is going to be closed automatically.
+  # @param zip_streamer_options[Hash] options that will be passed to the Streamer.
+  #     See {ZipTricks::Streamer#initialize} for the full list of options.
   # @yield [Streamer] the streamer that can be written to
-  def zip_tricks_stream
+  # @return [ZipTricks::OutputEnumerator] The output enumerator assigned to the response body
+  def zip_tricks_stream(**zip_streamer_options, &zip_streaming_blk)
     # Set a reasonable content type
     response.headers['Content-Type'] = 'application/zip'
     # Make sure nginx buffering is suppressed - see https://github.com/WeTransfer/zip_tricks/issues/48
     response.headers['X-Accel-Buffering'] = 'no'
-    # Create a wrapper for the write call that quacks like something you
-    # can << to, used by ZipTricks
-    w = ZipTricks::BlockWrite.new { |chunk| response.stream.write(chunk) }
-    ZipTricks::Streamer.open(w) { |z| yield(z) }
-  ensure
-    response.stream.close
+    response.sending_file = true
+    self.response_body = ZipTricks::OutputEnumerator.new(**zip_streamer_options, &zip_streaming_blk)
   end
 end

data/lib/zip_tricks/stream_crc32.rb

@@ -27,7 +27,7 @@ class ZipTricks::StreamCRC32
 
   # Creates a new streaming CRC32 calculator
   def initialize
-    @crc = Zlib.crc32('')
+    @crc = Zlib.crc32
   end
 
   # Append data to the CRC32. Updates the contained CRC32 value in place.
@@ -35,7 +35,7 @@ class ZipTricks::StreamCRC32
   # @param blob[String] the string to compute the CRC32 from
   # @return [self]
   def <<(blob)
-    @crc = Zlib.crc32_combine(@crc, Zlib.crc32(blob), blob.bytesize)
+    @crc = Zlib.crc32(blob, @crc)
     self
   end
 

data/lib/zip_tricks/streamer.rb

@@ -91,8 +91,9 @@ class ZipTricks::Streamer
   InvalidOutput = Class.new(ArgumentError)
   Overflow = Class.new(StandardError)
   UnknownMode = Class.new(StandardError)
+  OffsetOutOfSync = Class.new(StandardError)
 
-  private_constant :DeflatedWriter, :StoredWriter, :STORED, :DEFLATED
+  private_constant :STORED, :DEFLATED
 
   # Creates a new Streamer on top of the given IO-ish object and yields it. Once the given block
   # returns, the Streamer will have it's `close` method called, which will write out the central
@@ -130,28 +131,26 @@ class ZipTricks::Streamer
   #     end
   #
   # @param kwargs_for_new [Hash] keyword arguments for {Streamer.new}
-  # @return [Enumerator] the enumerator you can read bytestrings of the ZIP from using `each`
+  # @return [ZipTricks::OutputEnumerator] the enumerator you can read bytestrings of the ZIP from by calling `each`
   def self.output_enum(**kwargs_for_new, &zip_streamer_block)
     ZipTricks::OutputEnumerator.new(**kwargs_for_new, &zip_streamer_block)
   end
 
   # Creates a new Streamer on top of the given IO-ish object.
   #
-  # @param stream[IO] the destination IO for the ZIP. Anything that responds to `<<` can be used.
+  # @param writable[#<<] the destination IO for the ZIP. Anything that responds to `<<` can be used.
   # @param writer[ZipTricks::ZipWriter] the object to be used as the writer.
   #    Defaults to an instance of ZipTricks::ZipWriter, normally you won't need to override it
   # @param auto_rename_duplicate_filenames[Boolean] whether duplicate filenames, when encountered,
   #    should be suffixed with (1), (2) etc. Default value is `false` - if
   #    dupliate names are used an exception will be raised
-  def initialize(stream, writer: create_writer, auto_rename_duplicate_filenames: false)
-    raise InvalidOutput, 'The stream must respond to #<<' unless stream.respond_to?(:<<)
-
-    @dedupe_filenames = auto_rename_duplicate_filenames
-    @out = ZipTricks::WriteAndTell.new(stream)
+  def initialize(writable, writer: create_writer, auto_rename_duplicate_filenames: false)
+    raise InvalidOutput, 'The writable must respond to #<<' unless writable.respond_to?(:<<)
+    @out = ZipTricks::WriteAndTell.new(writable)
     @files = []
-    @local_header_offsets = []
     @path_set = ZipTricks::PathSet.new
     @writer = writer
+    @dedupe_filenames = auto_rename_duplicate_filenames
   end
 
   # Writes a part of a zip entry body (actual binary data of the entry) into the output stream.
@@ -201,14 +200,16 @@ class ZipTricks::Streamer
   # @param uncompressed_size [Integer] the size of the entry when uncompressed, in bytes
   # @param crc32 [Integer] the CRC32 checksum of the entry when uncompressed
   # @param use_data_descriptor [Boolean] whether the entry body will be followed by a data descriptor
+  # @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
   # @return [Integer] the offset the output IO is at after writing the entry header
-  def add_deflated_entry(filename:, modification_time: Time.now.utc, compressed_size: 0, uncompressed_size: 0, crc32: 0, use_data_descriptor: false)
+  def add_deflated_entry(filename:, modification_time: Time.now.utc, compressed_size: 0, uncompressed_size: 0, crc32: 0, unix_permissions: nil, use_data_descriptor: false)
     add_file_and_write_local_header(filename: filename,
                                     modification_time: modification_time,
                                     crc32: crc32,
                                     storage_mode: DEFLATED,
                                     compressed_size: compressed_size,
                                     uncompressed_size: uncompressed_size,
+                                    unix_permissions: unix_permissions,
                                     use_data_descriptor: use_data_descriptor)
     @out.tell
   end
@@ -223,14 +224,16 @@ class ZipTricks::Streamer
   # @param size [Integer] the size of the file when uncompressed, in bytes
   # @param crc32 [Integer] the CRC32 checksum of the entry when uncompressed
   # @param use_data_descriptor [Boolean] whether the entry body will be followed by a data descriptor. When in use
+  # @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
   # @return [Integer] the offset the output IO is at after writing the entry header
-  def add_stored_entry(filename:, modification_time: Time.now.utc,  size: 0, crc32: 0, use_data_descriptor: false)
+  def add_stored_entry(filename:, modification_time: Time.now.utc,  size: 0, crc32: 0, unix_permissions: nil, use_data_descriptor: false)
     add_file_and_write_local_header(filename: filename,
                                     modification_time: modification_time,
                                     crc32: crc32,
                                     storage_mode: STORED,
                                     compressed_size: size,
                                     uncompressed_size: size,
+                                    unix_permissions: unix_permissions,
                                     use_data_descriptor: use_data_descriptor)
     @out.tell
   end
@@ -239,14 +242,16 @@ class ZipTricks::Streamer
   #
   # @param dirname [String] the name of the directory in the archive
   # @param modification_time [Time] the modification time of the directory in the archive
+  # @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
   # @return [Integer] the offset the output IO is at after writing the entry header
-  def add_empty_directory(dirname:, modification_time: Time.now.utc)
+  def add_empty_directory(dirname:, modification_time: Time.now.utc, unix_permissions: nil)
     add_file_and_write_local_header(filename: dirname.to_s + '/',
                                     modification_time: modification_time,
                                     crc32: 0,
                                     storage_mode: STORED,
                                     compressed_size: 0,
                                     uncompressed_size: 0,
+                                    unix_permissions: unix_permissions,
                                     use_data_descriptor: false)
     @out.tell
   end
@@ -284,14 +289,16 @@ class ZipTricks::Streamer
   #
   # @param filename[String] the name of the file in the archive
   # @param modification_time [Time] the modification time of the file in the archive
+  # @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
   # @yield [#<<, #write] an object that the file contents must be written to that will be automatically closed
   # @return [#<<, #write, #close] an object that the file contents must be written to, has to be closed manually
-  def write_stored_file(filename, modification_time: Time.now.utc)
+  def write_stored_file(filename, modification_time: Time.now.utc, unix_permissions: nil)
     add_stored_entry(filename: filename,
                      modification_time: modification_time,
                      use_data_descriptor: true,
                      crc32: 0,
-                     size: 0)
+                     size: 0,
+                     unix_permissions: unix_permissions)
 
     writable = Writable.new(self, StoredWriter.new(@out))
     if block_given?
@@ -336,14 +343,16 @@ class ZipTricks::Streamer
   #
   # @param filename[String] the name of the file in the archive
   # @param modification_time [Time] the modification time of the file in the archive
+  # @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
   # @yield [#<<, #write] an object that the file contents must be written to
-  def write_deflated_file(filename, modification_time: Time.now.utc)
+  def write_deflated_file(filename, modification_time: Time.now.utc, unix_permissions: nil)
     add_deflated_entry(filename: filename,
                        modification_time: modification_time,
                        use_data_descriptor: true,
                        crc32: 0,
                        compressed_size: 0,
-                       uncompressed_size: 0)
+                       uncompressed_size: 0,
+                       unix_permissions: unix_permissions)
 
     writable = Writable.new(self, DeflatedWriter.new(@out))
     if block_given?
@@ -360,21 +369,24 @@ class ZipTricks::Streamer
   #
   # @return [Integer] the offset the output IO is at after closing the archive
   def close
+    # Make sure offsets are in order
+    verify_offsets!
+
     # Record the central directory offset, so that it can be written into the EOCD record
     cdir_starts_at = @out.tell
 
     # Write out the central directory entries, one for each file
-    @files.each_with_index do |entry, i|
-      header_loc = @local_header_offsets.fetch(i)
+    @files.each do |entry|
       @writer.write_central_directory_file_header(io: @out,
-                                                  local_file_header_location: header_loc,
+                                                  local_file_header_location: entry.local_header_offset,
                                                   gp_flags: entry.gp_flags,
                                                   storage_mode: entry.storage_mode,
                                                   compressed_size: entry.compressed_size,
                                                   uncompressed_size: entry.uncompressed_size,
                                                   mtime: entry.mtime,
                                                   crc32: entry.crc32,
-                                                  filename: entry.filename)
+                                                  filename: entry.filename,
+                                                  unix_permissions: entry.unix_permissions)
     end
 
     # Record the central directory size, for the EOCDR
@@ -420,15 +432,40 @@ class ZipTricks::Streamer
     last_entry.compressed_size = compressed_size
     last_entry.uncompressed_size = uncompressed_size
 
+    offset_before_data_descriptor = @out.tell
     @writer.write_data_descriptor(io: @out,
                                   crc32: last_entry.crc32,
                                   compressed_size: last_entry.compressed_size,
                                   uncompressed_size: last_entry.uncompressed_size)
+    last_entry.bytes_used_for_data_descriptor = @out.tell - offset_before_data_descriptor
+
     @out.tell
   end
 
   private
 
+  def verify_offsets!
+    # We need to check whether the offsets noted for the entries actually make sense
+    computed_offset = @files.map(&:total_bytes_used).inject(0, &:+)
+    actual_offset = @out.tell
+    if computed_offset != actual_offset
+      message = <<-EMS
+The offset of the Streamer output IO is out of sync with the expected value. All entries written so far,
+including their compressed bodies, local headers and data descriptors, add up to a certain offset,
+but this offset does not match the actual offset of the IO.
+
+Entries add up to #{computed_offset} bytes and the IO is at #{actual_offset} bytes.
+
+This can happen if you write local headers for an entry, write the "body" of the entry directly to the IO
+object which is your destination, but do not adjust the offset known to the Streamer object. To adjust
+the offfset you need to call `Streamer#simulate_write(body_size)` after outputting the entry. Otherwise
+the local header offsets of the entries you write are going to be incorrect and some ZIP applications
+are going to have problems opening your archive.
+EMS
+      raise OffsetOutOfSync, message
+    end
+  end
+
   def add_file_and_write_local_header(
       filename:,
       modification_time:,
@@ -436,7 +473,8 @@ class ZipTricks::Streamer
       storage_mode:,
       compressed_size:,
       uncompressed_size:,
-      use_data_descriptor:)
+      use_data_descriptor:,
+      unix_permissions:)
 
     # Clean backslashes
     filename = remove_backslash(filename)
@@ -461,16 +499,19 @@ class ZipTricks::Streamer
       uncompressed_size = 0
     end
 
+    local_header_starts_at = @out.tell
+
     e = Entry.new(filename,
                   crc32,
                   compressed_size,
                   uncompressed_size,
                   storage_mode,
                   modification_time,
-                  use_data_descriptor)
-
-    @files << e
-    @local_header_offsets << @out.tell
+                  use_data_descriptor,
+                  _local_file_header_offset = local_header_starts_at,
+                  _bytes_used_for_local_header = 0,
+                  _bytes_used_for_data_descriptor = 0,
+                  unix_permissions)
 
     @writer.write_local_file_header(io: @out,
                                     gp_flags: e.gp_flags,
@@ -480,6 +521,9 @@ class ZipTricks::Streamer
                                     mtime: e.mtime,
                                     filename: e.filename,
                                     storage_mode: e.storage_mode)
+    e.bytes_used_for_local_header = @out.tell - e.local_header_offset
+
+    @files << e
   end
 
   def remove_backslash(filename)

data/lib/zip_tricks/streamer/deflated_writer.rb

@@ -4,13 +4,6 @@
 # registers data passing through it in a CRC32 checksum calculator. Is made to be completely
 # interchangeable with the StoredWriter in terms of interface.
 class ZipTricks::Streamer::DeflatedWriter
-  # After how many bytes of incoming data the deflater for the
-  # contents must be flushed. This is done to prevent unreasonable
-  # memory use when archiving large files, and to ensure we write to
-  # the socket often enough while still maintaining acceptable
-  # compression
-  FLUSH_EVERY_N_BYTES = 1024 * 1024 * 5
-
   # The amount of bytes we will buffer before computing the intermediate
   # CRC32 checksums. Benchmarks show that the optimum is 64KB (see
   # `bench/buffered_crc32_bench.rb), if that is exceeded Zlib is going
@@ -18,11 +11,10 @@ class ZipTricks::Streamer::DeflatedWriter
   CRC32_BUFFER_SIZE = 64 * 1024
 
   def initialize(io)
-    @compressed_io = ZipTricks::WriteAndTell.new(io)
-    @uncompressed_size = 0
+    @compressed_io = io
     @deflater = ::Zlib::Deflate.new(Zlib::DEFAULT_COMPRESSION, -::Zlib::MAX_WBITS)
-    @crc = ZipTricks::WriteBuffer.new(ZipTricks::StreamCRC32.new, CRC32_BUFFER_SIZE)
-    @bytes_since_last_flush = 0
+    @crc = ZipTricks::StreamCRC32.new
+    @crc_buf = ZipTricks::WriteBuffer.new(@crc, CRC32_BUFFER_SIZE)
   end
 
   # Writes the given data into the deflater, and flushes the deflater
@@ -31,13 +23,8 @@ class ZipTricks::Streamer::DeflatedWriter
   # @param data[String] data to be written
   # @return self
   def <<(data)
-    @uncompressed_size += data.bytesize
-    @bytes_since_last_flush += data.bytesize
-    @compressed_io << @deflater.deflate(data)
-    @crc << data
-
-    interim_flush
-
+    @deflater.deflate(data) { |chunk| @compressed_io << chunk }
+    @crc_buf << data
     self
   end
 
@@ -45,18 +32,12 @@ class ZipTricks::Streamer::DeflatedWriter
   # compressed data written and the CRC32 checksum. The return value
   # can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
   #
-  # @param data[String] data to be written
   # @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
   def finish
     @compressed_io << @deflater.finish until @deflater.finished?
-    {crc32: @crc.to_i, compressed_size: @compressed_io.tell, uncompressed_size: @uncompressed_size}
-  end
-
-  private
-
-  def interim_flush
-    return if @bytes_since_last_flush < FLUSH_EVERY_N_BYTES
-    @compressed_io << @deflater.flush
-    @bytes_since_last_flush = 0
+    @crc_buf.flush
+    {crc32: @crc.to_i, compressed_size: @deflater.total_out, uncompressed_size: @deflater.total_in}
+  ensure
+    @deflater.close
   end
 end