zip_tricks 5.2.0 → 5.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b686d9ddb071cdb677559d12879dfa88a77ac63ce6eabad99ce2d8e674c80c28
-  data.tar.gz: fc8d6382de69962cf2cb7585b9a7476e7694abaca77cf969f2cf9b905076d59c
+  metadata.gz: 39403ce8cb20b4f772e5f44420b91be7e120df38b99de7de2a0db46f318760a8
+  data.tar.gz: c1c02357d9e5f322daf6d3ce338092020e753e362b3a70b3caf8d35a782925e8
 SHA512:
-  metadata.gz: 2949e24de0a8bc731ed0ae4d5fc30534e97ad1102f11c07a353ab211c2948bc1957c35527b4cb00eeb5751713d17a7653adbf65ea188b4fa9d0b0f0567926afe
-  data.tar.gz: 54972bf28272c68ce9bcada1f872840de04f7ae9a9e0276abeceb828373639f3ae9a999e788401937454d0354d86b87e462d4b60554e2478e7d8a7e6ac9c1348
+  metadata.gz: 9b16f5a53ee31ea9f1f5ec1316a62d172c897c510df9f1bb9f7fabe12aafe89e2b4e43f1533a01b5507b9d1a0fe438ee3646c7c10eb1117f8053403796167aad
+  data.tar.gz: 3808c09040d549644cb7aba6e8eb4aa0fcbae2123753e406d86db683e4e8d332453112622303be0b8027cff4c43dc60e6678073eb3e7c80cac3359200379210f

data/.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+name: CI
+
+on:
+  - push
+  - pull_request
+
+env:
+  BUNDLE_PATH: vendor/bundle
+
+jobs:
+  lint:
+    name: Code Style
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - '2.7'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'zip_tricks.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'zip_tricks.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: Rubocop Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/rubocop_cache
+          key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-rubocop-
+      - name: Rubocop
+        run: bundle exec rubocop
+  test:
+    name: Specs
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - '2.7'
+          - '2.1'
+          - 'jruby-9.2'
+        experimental: [false]
+        include:
+          - ruby: 3.0
+            experimental: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'zip_tricks.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'zip_tricks.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: RSpec
+        continue-on-error: ${{ matrix.experimental }}
+        run: bundle exec rspec

data/.gitignore

@@ -19,7 +19,7 @@ Gemfile.lock
 # jeweler generated
 pkg
 
-# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore:
 #
 # * Create a file at ~/.gitignore
 # * Include files you want ignored
@@ -36,6 +36,7 @@ pkg
 
 tmp
 testing/*.zip
+.ruby-version
 
 # For TextMate
 #*.tmproj

data/.rubocop.yml

@@ -11,3 +11,5 @@ Style/GlobalVars:
       - qa/*.rb
       - spec/spec_helper.rb
       - spec/support/zip_inspection.rb
+Layout/IndentHeredoc:
+  Enabled: false

data/CHANGELOG.md

@@ -1,4 +1,40 @@
-## 5.2
+## 5.6.0
+
+* Add customisable `unix_permissions` to Streamer and ZipWriter. Beware that customising these permissions can lead to the archive failing to expand with some unarchiving applications, and is especially sensitive for directories.
+
+## 5.5.0
+
+* In `OutputEnumerator` apply some amount of buffering to be within a UNIX socket size for metatada writes. This
+  speeds up usage with Puma by about 20 percent, as there won't be as many `syswrite` calls on the socket.
+* Make `StoredWriter` and `DeflatedWriter` public constants so that standalone tests can be written for them
+
+## 5.4.0
+
+* Use block form for zlib Deflater calls to conserve memory
+* Do not change string encoding in writer wrappers (avoid extra work)
+* Fix a zlib deflater object being leaked per archived file
+* Speed up streaming CRC32 computation
+* When running tests, assign the port for the Puma server dynamically
+* Reduce string allocations in the block deflate spec
+* Make sure RemoteUncap specs run under JRuby correctly
+* Replace Rails::Live streaming with iterable body streaming to avoid issues with Rails::Live across the board
+* Remove `qa/` directory and scripts, as the tests for the library proper should now be sufficient
+* Fix some documentation and sample code omissions and inconsistencies.
+
+## 5.3.1
+
+* Fix extended timestamp timestamp value encoding. Previously we would use an incorrect encoding for the timestamp value, which would output correct but nonsensical timestamps. The pack specifier is now changed to output the correct value.
+
+## 5.3.0
+
+* Raise in `Streamer#close` when the IO offset of the Streamer does not match the size of the written entries. This is a situation which
+  can occur if one adds the local headers, writes the bodies of the files to the socket/output directly, and forgets to adjust the internal
+  Streamer offset. The unadjusted offset would then produce incorrect values in both the local headers which come after the missing
+  offset adjustment _and_ in the central directory headers. Some ZIP unarchivers are able to recover from this (ones that read
+  files "straight-ahead" but others aren't - if the ZIP unarchiver uses central directory entries it would be using incorrect offsets.
+  Instead of producing an invalid ZIP, raise an exception which explains what happened and how it can be resolved.
+
+## 5.2.0
 
 * Remove `Streamer#add_compressed_entry` and `SizeEstimator#add_compressed_entry`
 

data/CONTRIBUTING.md

@@ -27,7 +27,7 @@ The issue tracker is the preferred channel for [bug reports](#bug-reports),
 [feature requests](#feature-requests) and [submitting pull
 requests](#pull-requests), but please respect the following restrictions:
 
-* Please **do not** derail or troll issues. Keep the discussion on topic and respect the opinions of others. Adhere to the principles set out in the [Code of Conduct](https://github.com/WeTransfer/zip_tricks/blob/master/CODE_OF_CONDUCT.md).
+* Please **do not** derail or troll issues. Keep the discussion on topic and respect the opinions of others. Adhere to the principles set out in the [Code of Conduct](https://github.com/WeTransfer/zip_tricks/blob/main/CODE_OF_CONDUCT.md).
 
 ## Bug reports
 
@@ -41,7 +41,7 @@ Guidelines for bug reports:
    reported.
 
 2. **Check if the issue has been fixed** — try to reproduce it using the
-   latest `master` branch in the repository.
+   latest `main` branch in the repository.
 
 3. **Isolate the problem** — create a [reduced test
    case](http://css-tricks.com/reduced-test-cases/) and a live example.
@@ -96,7 +96,7 @@ accurate comments, etc.) and any other requirements (such as test coverage).
 The project uses Rubocop which can be run using `bundle exec rubocop`. The test
 suite can be run with `bundle exec rspec`. You are also encouraged to use the
 script in the `testing` directory to create test files that you can then verify
-with various zip/unzip utilities. Further instructions are [here](https://github.com/WeTransfer/zip_tricks/blob/master/testing/README_TESTING.md).  
+with various zip/unzip utilities. Further instructions are [here](https://github.com/WeTransfer/zip_tricks/blob/main/testing/README_TESTING.md).  
 
 Follow this process if you'd like your work considered for inclusion in the
 project:
@@ -148,4 +148,4 @@ project:
 
 **IMPORTANT**: By submitting a patch, you agree to allow the project owner to
 license your work under the same license as that used by the project, which you
-can see by clicking [here](https://github.com/WeTransfer/zip_tricks/blob/master/LICENSE.txt). 
+can see by clicking [here](https://github.com/WeTransfer/zip_tricks/blob/main/LICENSE.txt). 

data/README.md

@@ -1,6 +1,6 @@
 # zip_tricks
 
-[![Build Status](https://travis-ci.org/WeTransfer/zip_tricks.svg?branch=master)](https://travis-ci.org/WeTransfer/zip_tricks)
+[![Build Status](https://travis-ci.org/WeTransfer/zip_tricks.svg?branch=main)](https://travis-ci.org/WeTransfer/zip_tricks)
 [![Gem Version](https://badge.fury.io/rb/zip_tricks.svg)](https://badge.fury.io/rb/zip_tricks)
 
 Allows streaming, non-rewinding ZIP file output from Ruby.
@@ -24,20 +24,20 @@ to [32 bit sizes.](https://github.com/jruby/jruby/issues/3817)
 
 ## Diving in: send some large CSV reports from Rails
 
-The easiest is to use the Rails' built-in streaming feature:
+The easiest is to include the `ZipTricks::RailsStreaming` module into your
+controller.
 
 ```ruby
 class ZipsController < ActionController::Base
-  include ActionController::Live # required for streaming
   include ZipTricks::RailsStreaming
 
   def download
     zip_tricks_stream do |zip|
       zip.write_deflated_file('report1.csv') do |sink|
         CSV(sink) do |csv_write|
-          csv << Person.column_names
+          csv_write << Person.column_names
           Person.all.find_each do |person|
-            csv << person.attributes.values
+            csv_write << person.attributes.values
           end
         end
       end
@@ -49,6 +49,10 @@ class ZipsController < ActionController::Base
 end
 ```
 
+If you want some more conveniences you can also use [zipline](https://github.com/fringd/zipline) which
+will automatically process and stream attachments (Carrierwave, Shrine, ActiveStorage) and remote objects
+via HTTP.
+
 ## Create a ZIP file without size estimation, compress on-the-fly during writes
 
 Basic use case is compressing on the fly. Some data will be buffered by the Zlib deflater, but
@@ -71,12 +75,15 @@ since you do not know how large the compressed data segments are going to be.
 
 ## Send a ZIP from a Rack response
 
-Create a `RackBody` object and give it's constructor a block that adds files.
-The block will only be called when actually sending the response to the client
+To "pull" data from ZipTricks you can create an `OutputEnumerator` object which will yield the binary chunks piece
+by piece, and apply some amount of buffering as well. Since this `OutputEnumerator` responds to `#each` and yields
+Strings it also can (and should!) be used as a Rack response body. Return it to your webserver and you will
+have your ZIP streamed. The block that you give to the `OutputEnumerator` will only start executing once your
+response body starts getting iterated over - when actually sending the response to the client
 (unless you are using a buffering Rack webserver, such as Webrick).
 
 ```ruby
-body = ZipTricks::RackBody.new do | zip |
+body = ZipTricks::Streamer.output_enum do | zip |
   zip.write_stored_file('mov.mp4') do |sink| # Those MPEG4 files do not compress that well
     File.open('mov.mp4', 'rb'){|source| IO.copy_stream(source, sink) }
   end
@@ -84,7 +91,7 @@ body = ZipTricks::RackBody.new do | zip |
     File.open('novel.txt', 'rb'){|source| IO.copy_stream(source, sink) }
   end
 end
-[200, {'Transfer-Encoding' => 'chunked'}, body]
+[200, {}, body]
 ```
 
 ## Send a ZIP file of known size, with correct headers
@@ -123,11 +130,12 @@ ZipTricks::Streamer.open(io) do | zip |
   # Write the local file header first..
   zip.add_stored_entry(filename: "first-file.bin", size: raw_file.size, crc32: raw_file_crc32)
 
-  # then send the actual file contents bypassing the Streamer interface
+  # Adjust the ZIP offsets within the Streamer
+  zip.simulate_write(my_temp_file.size)
+
+  # ...and then send the actual file contents bypassing the Streamer interface
   io.sendfile(my_temp_file)
 
-  # ...and then adjust the ZIP offsets within the Streamer
-  zip.simulate_write(my_temp_file.size)
 end
 ```
 
@@ -167,16 +175,19 @@ that have not been formally verified (ours hasn't been).
 
 ## Contributing to zip_tricks
 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the latest `main` to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
 * Fork the project.
 * Start a feature/bugfix branch.
 * Commit and push until you are happy with your contribution.
 * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-* If you alter the `ZipWriter`, please take the time to run the test in the `qa/` directory. Ensure that the generated (large) files open manually - see README_QA for more.
 
-## Copyright
+## Copyright and license
+
+Copyright (c) 2020 WeTransfer.
 
-Copyright (c) 2019 WeTransfer. `zip_tricks` is distributed under the conditions of the [Hippocratic License](https://firstdonoharm.dev/version/1/2/license.html)
- - See LICENSE.txt for further details.
+`zip_tricks` is distributed under the conditions of the [Hippocratic License](https://firstdonoharm.dev/version/1/2/license.html)
+See LICENSE.txt for further details. If this license is not acceptable for your use case we still maintain the 4.x version tree
+which remains under the MIT license, see https://rubygems.org/gems/zip_tricks/versions for more information.
+Note that we only backport some performance optimizations and crucial bugfixes but not the new features to that tree.

data/lib/zip_tricks/block_write.rb

@@ -1,11 +1,25 @@
 # frozen_string_literal: true
 
-# Stashes a block given by the Rack webserver when calling each() on a body, and calls
-# that block every time it is written to using :<< (shovel). Poses as an IO for rubyzip.
-
+# Acts as a converter between callers which send data to the `#<<` method (such as all the ZipTricks
+# writer methods, which push onto anything), and a given block. Every time `#<<` gets called on the BlockWrite,
+# the block given to the constructor will be called with the same argument. ZipTricks uses this object
+# when integrating with Rack and in the OutputEnumerator. Normally you wouldn't need to use it manually but
+# you always can. BlockWrite will also ensure the binary string encoding is forced onto any string
+# that passes through it.
+#
+# For example, you can create a Rack response body like so:
+#
+#     class MyRackResponse
+#       def each
+#         writer = ZipTricks::BlockWrite.new {|chunk| yield(chunk) }
+#         writer << "Hello" << "world" << "!"
+#       end
+#     end
+#     [200, {}, MyRackResponse.new]
 class ZipTricks::BlockWrite
-  # The block is the block given to each() of the Rack body, or other block you want
-  # to receive the string chunks written by the zip compressor.
+  # Creates a new BlockWrite.
+  #
+  # @param block The block that will be called when this object receives the `<<` message
   def initialize(&block)
     @block = block
   end
@@ -17,26 +31,17 @@ class ZipTricks::BlockWrite
     end
   end
 
-  # Every time this object gets written to, call the Rack body each() block
-  # with the bytes given instead.
+  # Sends a string through to the block stored in the BlockWrite.
+  #
+  # @param buf[String] the string to write. Note that a zero-length String
+  #    will not be forwarded to the block, as it has special meaning when used
+  #    with chunked encoding (it indicates the end of the stream).
+  # @return self
   def <<(buf)
     # Zero-size output has a special meaning  when using chunked encoding
     return if buf.nil? || buf.bytesize.zero?
 
-    # Ensure we ALWAYS write in binary encoding.
-    encoded =
-      if buf.encoding != Encoding::BINARY
-        # If we got a frozen string we can't force_encoding on it
-        begin
-          buf.force_encoding(Encoding::BINARY)
-        rescue
-          buf.dup.force_encoding(Encoding::BINARY)
-        end
-      else
-        buf
-      end
-
-    @block.call(encoded)
+    @block.call(buf.b)
     self
   end
 end

data/lib/zip_tricks/file_reader.rb

@@ -19,7 +19,7 @@ require 'stringio'
 # ## Usage
 #
 #     File.open('zipfile.zip', 'rb') do |f|
-#       entries = FileReader.read_zip_structure(f)
+#       entries = ZipTricks::FileReader.read_zip_structure(io: f)
 #       entries.each do |e|
 #         File.open(e.filename, 'wb') do |extracted_file|
 #           ex = e.extractor_from(f)
@@ -82,7 +82,9 @@ class ZipTricks::FileReader
 
   private_constant :StoredReader, :InflatingReader
 
-  # Represents a file within the ZIP archive being read
+  # Represents a file within the ZIP archive being read. This is different from
+  # the Entry object used in Streamer for ZIP writing, since during writing more
+  # data can be kept in memory for immediate use.
   class ZipEntry
     # @return [Fixnum] bit-packed version signature of the program that made the archive
     attr_accessor :made_by
@@ -279,7 +281,7 @@ class ZipTricks::FileReader
       seek(io, next_local_header_offset) # Seek to the next entry, and raise if seek is impossible
     end
     entries
-  rescue ReadError
+  rescue ReadError, RangeError # RangeError is raised if offset exceeds int32/int64 range
     log do
       'Got a read/seek error after reaching %<cur_offset>d, no more entries can be recovered' %
         {cur_offset: cur_offset}
@@ -363,7 +365,7 @@ class ZipTricks::FileReader
   # (read starting at this offset to get the data).
   #
   # @param io[#seek, #read] an IO-ish object the ZIP file can be read from
-  # @param local_header_offset[Fixnum] absolute offset (0-based) where the
+  # @param local_file_header_offset[Fixnum] absolute offset (0-based) where the
   # local file header is supposed to begin @return [Fixnum] absolute offset
   # (0-based) of where the compressed data begins for this file within the ZIP
   def get_compressed_data_offset(io:, local_file_header_offset:)
@@ -375,7 +377,7 @@ class ZipTricks::FileReader
   # Parse an IO handle to a ZIP archive into an array of Entry objects, reading from the end
   # of the IO object.
   #
-  # @see {#read_zip_structure}
+  # @see #read_zip_structure
   # @param options[Hash] any options the instance method of the same name accepts
   # @return [Array<ZipEntry>] an array of entries within the ZIP being parsed
   def self.read_zip_structure(**options)
@@ -385,7 +387,7 @@ class ZipTricks::FileReader
   # Parse an IO handle to a ZIP archive into an array of Entry objects, reading from the start of
   # the file and parsing local file headers one-by-one
   #
-  # @see {#read_zip_straight_ahead}
+  # @see #read_zip_straight_ahead
   # @param options[Hash] any options the instance method of the same name accepts
   # @return [Array<ZipEntry>] an array of entries within the ZIP being parsed
   def self.read_zip_straight_ahead(**options)

data/lib/zip_tricks/null_writer.rb

@@ -4,7 +4,7 @@
 # write operations, but want to discard the data (like when
 # estimating the size of a ZIP)
 module ZipTricks::NullWriter
-  # @param data[String] the data to write
+  # @param _[String] the data to write
   # @return [self]
   def self.<<(_)
     self