zip_tricks 5.2.0 → 5.6.0

data/lib/zip_tricks/streamer/entry.rb

@@ -4,7 +4,7 @@
 # Normally you will not have to use this class directly
 class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_size,
                                               :uncompressed_size, :storage_mode, :mtime,
-                                              :use_data_descriptor)
+                                              :use_data_descriptor, :local_header_offset, :bytes_used_for_local_header, :bytes_used_for_data_descriptor, :unix_permissions)
   def initialize(*)
     super
     filename.force_encoding(Encoding::UTF_8)
@@ -15,6 +15,10 @@ class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_siz
                            end)
   end
 
+  def total_bytes_used
+    bytes_used_for_local_header + compressed_size + bytes_used_for_data_descriptor
+  end
+
   # Set the general purpose flags for the entry. We care about is the EFS
   # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
   # bit so that the unarchiving application knows that the filename in the archive is UTF-8

data/lib/zip_tricks/streamer/stored_writer.rb

@@ -12,7 +12,8 @@ class ZipTricks::Streamer::StoredWriter
 
   def initialize(io)
     @io = ZipTricks::WriteAndTell.new(io)
-    @crc = ZipTricks::WriteBuffer.new(ZipTricks::StreamCRC32.new, CRC32_BUFFER_SIZE)
+    @crc_compute = ZipTricks::StreamCRC32.new
+    @crc = ZipTricks::WriteBuffer.new(@crc_compute, CRC32_BUFFER_SIZE)
   end
 
   # Writes the given data to the contained IO object.
@@ -28,9 +29,9 @@ class ZipTricks::Streamer::StoredWriter
   # Returns the amount of data written and the CRC32 checksum. The return value
   # can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
   #
-  # @param data[String] data to be written
   # @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
   def finish
-    {crc32: @crc.to_i, compressed_size: @io.tell, uncompressed_size: @io.tell}
+    @crc.flush
+    {crc32: @crc_compute.to_i, compressed_size: @io.tell, uncompressed_size: @io.tell}
   end
 end

data/lib/zip_tricks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZipTricks
-  VERSION = '5.2.0'
+  VERSION = '5.6.0'
 end

data/lib/zip_tricks/write_and_tell.rb

@@ -10,9 +10,8 @@ class ZipTricks::WriteAndTell
 
   def <<(bytes)
     return self if bytes.nil?
-    binary_bytes = binary(bytes)
-    @io << binary_bytes
-    @pos += binary_bytes.bytesize
+    @io << bytes.b
+    @pos += bytes.bytesize
     self
   end
 
@@ -23,13 +22,4 @@ class ZipTricks::WriteAndTell
   def tell
     @pos
   end
-
-  private
-
-  def binary(str)
-    return str if str.encoding == Encoding::BINARY
-    str.force_encoding(Encoding::BINARY)
-  rescue RuntimeError # the string is frozen
-    str.dup.force_encoding(Encoding::BINARY)
-  end
 end

data/lib/zip_tricks/write_buffer.rb

@@ -7,13 +7,34 @@
 # CRC32 combine operations - and this adds up. Since the CRC32 value
 # is usually not needed until the complete output has completed
 # we can buffer at least some amount of data before computing CRC32 over it.
+# We also use this buffer for output via Rack, where some amount of buffering
+# helps reduce the number of syscalls made by the webserver. ZipTricks performs
+# lots of very small writes, and some degree of speedup (about 20%) can be achieved
+# with a buffer of a few KB.
+#
+# Note that there is no guarantee that the write buffer is going to flush at or above
+# the given `buffer_size`, because for writes which exceed the buffer size it will
+# first `flush` and then write through the oversized chunk, without buffering it. This
+# helps conserve memory. Also note that the buffer will *not* duplicate strings for you
+# and *will* yield the same buffer String over and over, so if you are storing it in an
+# Array you might need to duplicate it.
+#
+# Note also that the WriteBuffer assumes that the object it `<<`-writes into is going
+# to **consume** in some way the string that it passes in. After the `<<` method returns,
+# the WriteBuffer will be cleared, and it passes the same String reference on every call
+# to `<<`. Therefore, if you need to retain the output of the WriteBuffer in, say, an Array,
+# you might need to `.dup` the `String` it gives you.
 class ZipTricks::WriteBuffer
   # Creates a new WriteBuffer bypassing into a given writable object
   #
-  # @param writable[#<<] An object that responds to `#<<` with string as argument
+  # @param writable[#<<] An object that responds to `#<<` with a String as argument
   # @param buffer_size[Integer] How many bytes to buffer
   def initialize(writable, buffer_size)
-    @buf = StringIO.new
+    # Allocating the buffer using a zero-padded String as a variation
+    # on using capacity:, which JRuby apparently does not like very much. The
+    # desire here is that the buffer doesn't have to be resized during the lifetime
+    # of the object.
+    @buf = ("\0".b * (buffer_size * 2)).clear
     @buffer_size = buffer_size
     @writable = writable
   end
@@ -24,28 +45,27 @@ class ZipTricks::WriteBuffer
   # @param data[String] data to be written
   # @return self
   def <<(data)
-    @buf << data
-    flush! if @buf.size > @buffer_size
+    if data.bytesize >= @buffer_size
+      flush unless @buf.empty? # <- this is were we can output less than @buffer_size
+      @writable << data
+    else
+      @buf << data
+      flush if @buf.bytesize >= @buffer_size
+    end
     self
   end
 
   # Explicitly flushes the buffer if it contains anything
   #
   # @return self
-  def flush!
-    @writable << @buf.string if @buf.size > 0
-    @buf.truncate(0)
-    @buf.rewind
+  def flush
+    unless @buf.empty?
+      @writable << @buf
+      @buf.clear
+    end
     self
   end
 
-  # Flushes the buffer and returns the result of `#to_i` of the contained `writable`.
-  # Primarily facilitates working with StreamCRC32 objects where you finish the
-  # computation by retrieving the CRC as an integer
-  #
-  # @return [Integer] the return value of `writable#to_i`
-  def to_i
-    flush!
-    @writable.to_i
-  end
+  # `flush!` was renamed to `flush` but we preserve this method for backwards compatibility
+  alias_method :flush!, :flush
 end

data/lib/zip_tricks/zip_writer.rb

@@ -31,22 +31,10 @@ class ZipTricks::ZipWriter
   VERSION_MADE_BY                        = 52
   VERSION_NEEDED_TO_EXTRACT              = 20
   VERSION_NEEDED_TO_EXTRACT_ZIP64        = 45
-  DEFAULT_EXTERNAL_ATTRS = begin
-    # These need to be set so that the unarchived files do not become executable on UNIX, for
-    # security purposes. Strictly speaking we would want to make this user-customizable,
-    # but for now just putting in sane defaults will do. For example, Trac with zipinfo does this:
-    # zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--.
-    # We snatch the incantations from Rubyzip for this.
-    unix_perms = 0o644
-    file_type_file = 0o10
-    (file_type_file << 12 | (unix_perms & 0o7777)) << 16
-  end
-  EMPTY_DIRECTORY_EXTERNAL_ATTRS = begin
-    # Applies permissions to an empty directory.
-    unix_perms = 0o755
-    file_type_dir = 0o04
-    (file_type_dir << 12 | (unix_perms & 0o7777)) << 16
-  end
+  DEFAULT_FILE_UNIX_PERMISSIONS = 0o644
+  DEFAULT_DIRECTORY_UNIX_PERMISSIONS = 0o755
+  FILE_TYPE_FILE = 0o10
+  FILE_TYPE_DIRECTORY = 0o04
   MADE_BY_SIGNATURE = begin
     # A combination of the VERSION_MADE_BY low byte and the OS type high byte
     os_type = 3 # UNIX
@@ -57,14 +45,15 @@ class ZipTricks::ZipWriter
   C_UINT2 = 'v'    # Encode a 2-byte unsigned little-endian uint
   C_UINT8 = 'Q<'  # Encode an 8-byte unsigned little-endian uint
   C_CHAR = 'C' # For bit-encoded strings
-  C_INT4 = 'N' # Encode a 4-byte signed little-endian int
+  C_INT4 = 'l<' # Encode a 4-byte signed little-endian int
 
   private_constant :FOUR_BYTE_MAX_UINT,
                    :TWO_BYTE_MAX_UINT,
                    :VERSION_MADE_BY,
                    :VERSION_NEEDED_TO_EXTRACT,
                    :VERSION_NEEDED_TO_EXTRACT_ZIP64,
-                   :DEFAULT_EXTERNAL_ATTRS,
+                   :FILE_TYPE_FILE,
+                   :FILE_TYPE_DIRECTORY,
                    :MADE_BY_SIGNATURE,
                    :C_UINT4,
                    :C_UINT2,
@@ -136,6 +125,7 @@ class ZipTricks::ZipWriter
   # @param crc32[Fixnum] The CRC32 checksum of the file
   # @param mtime[Time]  the modification time to be recorded in the ZIP
   # @param gp_flags[Fixnum] bit-packed general purpose flags
+  # @param unix_permissions[Fixnum?] the permissions for the file, or nil for the default to be used
   # @return [void]
   def write_central_directory_file_header(io:,
                                           local_file_header_location:,
@@ -145,7 +135,9 @@ class ZipTricks::ZipWriter
                                           uncompressed_size:,
                                           mtime:,
                                           crc32:,
-                                          filename:)
+                                          filename:,
+                                          unix_permissions: nil
+                                         )
     # At this point if the header begins somewhere beyound 0xFFFFFFFF we _have_ to record the offset
     # of the local file header as a zip64 extra field, so we give up, give in, you loose, love will always win...
     add_zip64 = (local_file_header_location > FOUR_BYTE_MAX_UINT) ||
@@ -195,16 +187,20 @@ class ZipTricks::ZipWriter
       [TWO_BYTE_MAX_UINT].pack(C_UINT2)
     else
       [0].pack(C_UINT2)
-          end
+    end
     io << [0].pack(C_UINT2)                                # internal file attributes        2 bytes
 
     # Because the add_empty_directory method will create a directory with a trailing "/",
     # this check can be used to assign proper permissions to the created directory.
-    io << if filename.end_with?('/')
-      [EMPTY_DIRECTORY_EXTERNAL_ATTRS].pack(C_UINT4)
+    # external file attributes        4 bytes
+    external_attrs = if filename.end_with?('/')
+      unix_permissions ||= DEFAULT_DIRECTORY_UNIX_PERMISSIONS
+      generate_external_attrs(unix_permissions, FILE_TYPE_DIRECTORY)
     else
-      [DEFAULT_EXTERNAL_ATTRS].pack(C_UINT4)           # external file attributes        4 bytes
+      unix_permissions ||= DEFAULT_FILE_UNIX_PERMISSIONS
+      generate_external_attrs(unix_permissions, FILE_TYPE_FILE)
     end
+    io << [external_attrs].pack(C_UINT4)
 
     io << if add_zip64                                 # relative offset of local header 4 bytes
       [FOUR_BYTE_MAX_UINT].pack(C_UINT4)
@@ -385,7 +381,7 @@ class ZipTricks::ZipWriter
       0x5455, C_UINT2,  # tag for this extra block type ("UT")
       (1 + 4), C_UINT2, # the size of this block (1 byte used for the Flag + 3 longs used for the timestamp)
       flags, C_CHAR,   # encode a single byte
-      mtime.utc.to_i, C_INT4, # Use a signed long, not the unsigned one used by the rest of the ZIP spec.
+      mtime.utc.to_i, C_INT4, # Use a signed int, not the unsigned one used by the rest of the ZIP spec.
     ]
     # The atime and ctime can be omitted if not present
     pack_array(data_and_packspecs)
@@ -434,4 +430,8 @@ class ZipTricks::ZipWriter
     values, packspecs = values_to_packspecs.partition.each_with_index { |_, i| i.even? }
     values.pack(packspecs.join)
   end
+
+  def generate_external_attrs(unix_permissions_int, file_type_int)
+    (file_type_int << 12 | (unix_permissions_int & 0o7777)) << 16
+  end
 end

data/zip_tricks.gemspec

@@ -5,13 +5,13 @@ require 'zip_tricks/version'
 Gem::Specification.new do |spec|
   spec.name           = 'zip_tricks'
   spec.version        = ZipTricks::VERSION
-  spec.authors        = ['Julik Tarkhanov', 'Noah Berman', 'Dmitry Tymchuk', 'David Bosveld']
+  spec.authors        = ['Julik Tarkhanov', 'Noah Berman', 'Dmitry Tymchuk', 'David Bosveld', 'Felix Bünemann']
   spec.email          = ['me@julik.nl']
 
   spec.licenses       = ['MIT (Hippocratic)']
   spec.summary        = 'Stream out ZIP files from Ruby'
   spec.description    = 'Stream out ZIP files from Ruby'
-  spec.homepage       = 'http://github.com/wetransfer/zip_tricks'
+  spec.homepage       = 'https://github.com/wetransfer/zip_tricks'
 
   # Prevent pushing this gem to RubyGems.org.
   # To allow pushes either set the 'allowed_push_host'
@@ -32,10 +32,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rubyzip', '~> 1'
-  spec.add_development_dependency 'terminal-table'
-  spec.add_development_dependency 'range_utils'
 
-  spec.add_development_dependency 'rack', '~> 1.6' # For Jeweler
+  spec.add_development_dependency 'rack', '~> 1.6' # For tests, where we spin up a server
   spec.add_development_dependency 'rake', '~> 12.2'
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'complexity_assert'

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.6.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 - Noah Berman
 - Dmitry Tymchuk
 - David Bosveld
-autorequire: 
+- Felix Bünemann
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2021-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -41,34 +42,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
-- !ruby/object:Gem::Dependency
-  name: terminal-table
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: range_utils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -218,10 +191,10 @@ extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
 - ".document"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -262,22 +235,13 @@ files:
 - lib/zip_tricks/write_and_tell.rb
 - lib/zip_tricks/write_buffer.rb
 - lib/zip_tricks/zip_writer.rb
-- qa/README_QA.md
-- qa/generate_test_files.rb
-- qa/in/VTYL8830.jpg
-- qa/in/war-and-peace.txt
-- qa/support.rb
-- qa/test-report-2016-07-28.txt
-- qa/test-report-2016-12-12.txt
-- qa/test-report-2017-04-2.txt
-- qa/test-report.txt
 - zip_tricks.gemspec
-homepage: http://github.com/wetransfer/zip_tricks
+homepage: https://github.com/wetransfer/zip_tricks
 licenses:
 - MIT (Hippocratic)
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -293,7 +257,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Stream out ZIP files from Ruby
 test_files: []

data/.travis.yml

@@ -1,11 +0,0 @@
-rvm:
-- 2.1.0
-- 2.6.5
-- jruby-9.0
-sudo: false
-cache: bundler
-matrix:
-  allow_failures:
-    - rvm: jruby-9.0
-script:
-  - bundle exec rake

data/qa/README_QA.md

@@ -1,16 +0,0 @@
-## Manual testing harness for ZipTricks
-
-These tests will generate **very large** files that test various edge cases of ZIP generation. The idea is to generate
-these files and to then try to open them with the unarchiver applications we support. The workflow is as follows:
-
-
-1. Configure your storage to have `zip_tricks` directory linked into your virtual machines and to be on a fast volume (SSD RAID0 is recommended)
-2. Run `generate_test_files.rb`. This will take some time and produce a number of large ZIP files.
-3. Open them with the following ZIP unarchivers:
-  * A recent version of `zipinfo` with the `-tlhvz` flags - to see the information about the file
-  * ArchiveUtility on OSX
-  * The Unarchiver on OSX
-  * Built-in Explorer on Windows 7
-  * 7Zip 9.20 on Windows 7
-  * Any other unarchivers you consider necessary
-* Write down your observations in `test-report.txt` and, when cutting a release, timestamp a copy of that file.

data/qa/generate_test_files.rb

@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'support'
-
-build_test 'Two small stored files' do |zip|
-  zip.add_stored_entry(filename: 'text.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-
-  zip.add_stored_entry(filename: 'image.jpg',
-                       size: $image_file.bytesize,
-                       crc32: $image_file_crc)
-  zip << $image_file
-end
-
-build_test 'Two small stored files and an empty directory' do |zip|
-  zip.add_stored_entry(filename: 'text.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-
-  zip.add_stored_entry(filename: 'image.jpg',
-                       size: $image_file.bytesize,
-                       crc32: $image_file_crc)
-  zip << $image_file
-
-  zip.add_empty_directory(dirname: 'Chekov')
-end
-
-build_test 'Filename with diacritics' do |zip|
-  zip.add_stored_entry(filename: 'Kungälv.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-end
-
-build_test 'Purely UTF-8 filename' do |zip|
-  zip.add_stored_entry(filename: 'Война и мир.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-end
-
-# The trick of this test is that each file of the 2, on it's own, does _not_ exceed the
-# size threshold for Zip64. Together, however, they do.
-build_test 'Two entries larger than the overall Zip64 offset' do |zip|
-  big = generate_big_entry((0xFFFFFFFF / 2) + 1_024)
-  zip.add_stored_entry(filename: 'repeated-A.txt',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-
-  zip.add_stored_entry(filename: 'repeated-B.txt',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-end
-
-build_test 'One entry that requires Zip64 and a tiny entry following it' do |zip|
-  big = generate_big_entry(0xFFFFFFFF + 2_048)
-  zip.add_stored_entry(filename: 'large-requires-zip64.bin',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-
-  zip.add_stored_entry(filename: 'tiny-after.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-end
-
-build_test 'One tiny entry followed by second that requires Zip64' do |zip|
-  zip.add_stored_entry(filename: 'tiny-at-start.txt',
-                       size: $war_and_peace.bytesize,
-                       crc32: $war_and_peace_crc)
-  zip << $war_and_peace
-
-  big = generate_big_entry(0xFFFFFFFF + 2_048)
-  zip.add_stored_entry(filename: 'large-requires-zip64.bin',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-end
-
-build_test 'Two entries both requiring Zip64' do |zip|
-  big = generate_big_entry(0xFFFFFFFF + 2_048)
-  zip.add_stored_entry(filename: 'huge-file-1.bin',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-
-  zip.add_stored_entry(filename: 'huge-file-2.bin',
-                       size: big.size,
-                       crc32: big.crc32)
-  big.write_to(zip)
-end
-
-build_test 'Two stored entries using data descriptors' do |zip|
-  zip.write_stored_file('stored.1.bin') do |sink|
-    sink << Random.new.bytes(1_024 * 1_024 * 4)
-  end
-  zip.write_stored_file('stored.2.bin') do |sink|
-    sink << Random.new.bytes(1_024 * 1_024 * 3)
-  end
-end
-
-build_test 'One entry deflated using data descriptors' do |zip|
-  big = generate_big_entry(0xFFFFFFFF / 64)
-  zip.write_deflated_file('war-and-peace-repeated-compressed.txt') do |sink|
-    big.write_to(sink)
-  end
-end
-
-build_test 'Two entries larger than the overall Zip64 offset using data descriptors' do |zip|
-  big = generate_big_entry((0xFFFFFFFF / 2) + 1_024)
-
-  zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
-  zip.write_stored_file('repeated-B.txt') { |sink| big.write_to(sink) }
-end
-
-build_test 'One stored entry larger than Zip64 threshold using data descriptors' do |zip|
-  big = generate_big_entry(0xFFFFFFFF + 64_000)
-
-  zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
-end