zip_tricks 4.6.0 → 4.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 387773a063a44c69ac3da83da89fe85731c78331
-  data.tar.gz: 96d6763cc05e7c21f206ecaa013786f822df0d42
+  metadata.gz: ec7428d7634361abe1384db4d39a8c7751dd3cdc
+  data.tar.gz: 3a4b3327e9edc0cc4b84e8ea8d064682104603d1
 SHA512:
-  metadata.gz: 80f07ed5042f76c6e0edf5843a4b51ee468bde57d5fc4ab28b0a53f9f819a1160e339126173ebbe306add30902ac69270d271e61c81aab5a3261098b5ed79b1c
-  data.tar.gz: 8203289c9445f009f65ccaa924a7b07f1ecb5257ac9cb4d7fc6ce7a2204e49ed125cba6c4d1345a933203aa61ab562fe2d40b3bab3db074fccc5141636eb8c2e
+  metadata.gz: a9740445e2b7b646a5e7c5670edf61450543fd33f6e687d2d6adfc07a2a8097a244b9ecc9a8122f80eade94e534193ccdda34ad20ea97289b73f741f7e464a20
+  data.tar.gz: 649ffbc1803e3022092bf7da01245e13a838b0e27bf85f2be40e0cc1792249c4207936f1aa83b33eba74ec1f646962c6eb1cf11ca7677f48978c8721fcdb2c87

data/.travis.yml

@@ -14,5 +14,4 @@ matrix:
   allow_failures:
     - rvm: jruby-9.0
 script:
-  - bundle exec rspec
-  - bundle exec rubocop
+  - bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.7.0
+
+* Replace `RackBody` with `OutputEnumerator` since we want to provide a generic way of deferring ZIP output, also when using enumerators.
+* Remove `RackBody#close` since we got nothing to close 🤷‍♂️
+* Hint nginx that response buffering should be disabled when using Rails zip streaming
+
 ## 4.6.0
 
 * Add `mtime:` option to all Streamer methods for adding files and directories, to permit setting modification time per-entry

data/README.md

@@ -1,6 +1,7 @@
 # zip_tricks
 
 [![Build Status](https://travis-ci.org/WeTransfer/zip_tricks.svg?branch=master)](https://travis-ci.org/WeTransfer/zip_tricks)
+[![Gem Version](https://badge.fury.io/rb/zip_tricks.svg)](https://badge.fury.io/rb/zip_tricks)
 
 Allows streaming, non-rewinding ZIP file output from Ruby.
 

data/Rakefile

@@ -10,6 +10,7 @@ YARD::Rake::YardocTask.new(:doc) do |t|
 end
 
 RSpec::Core::RakeTask.new(:spec)
-task default: :spec
 
-RuboCop::RakeTask.new
+RuboCop::RakeTask.new(:rubocop)
+
+task default: [:spec, :rubocop]

data/examples/deferred_write.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative '../lib/zip_tricks'
+
+# Using deferred writes (when you want to "pull" from a Streamer)
+# is also possible with ZipTricks.
+#
+# The OutputEnumerator class instead of Streamer is very useful for this
+# particular purpose. It does not start the archiving immediately,
+# but waits instead until you start pulling data out of it.
+#
+# Let's make a OutputEnumerator that writes a few files with random content. Note that when you create
+# that body it does not immediately write the ZIP:
+iterable = ZipTricks::Streamer.output_enum do |zip|
+  (1..5).each do |i|
+    zip.write_stored_file('random_%d04d.bin' % i) do |sink|
+      warn "Starting on file #{i}...\n"
+      sink << Random.new.bytes(1024)
+    end
+  end
+end
+
+warn "\n\nOutput using #each"
+
+# Now we can treat the iterable as any Ruby enumerable object, since
+# it supports #each yielding every binary string output by the Streamer.
+# Only when we start using each() will the ZIP start generating. Just using
+# each() like we do here runs the archiving procedure to completion. See how
+# the output of the block within OutputEnumerator is interspersed with the stuff
+# being yielded to each():
+iterable.each do |_binary_string|
+  $stderr << '.'
+end
+
+warn "\n\nOutput Enumerator returned from #each"
+
+# We now have output the entire archive, so using each() again
+# will restart the block we gave it. For example, we can user
+# an Enumerator - via enum_for - to "take" chunks of output when
+# we find necessary:
+enum = iterable.each
+15.times do
+  _bin_str = enum.next  # Obtain the subsequent chunk of the ZIP
+  $stderr << '*'
+end
+
+# ... or a Fiber
+
+warn "\n\nOutput using a Fiber"
+fib = Fiber.new do
+  iterable.each do |binary_string|
+    $stderr << '•'
+    _next_iteration = Fiber.yield(binary_string)
+  end
+end
+15.times do
+  fib.resume # Process the subsequent chunk of the ZIP
+end

data/examples/rack_application.rb

@@ -36,7 +36,7 @@ class ZipDownload
 
     # Create a suitable Rack response body, that will support each(),
     # close() and all the other methods. We can then return it up the stack.
-    zip_response_body = ZipTricks::RackBody.new do |zip|
+    zip_response_body = ZipTricks::Streamer.output_enum do |zip|
       begin
         # We are adding only one file to the ZIP here, but you could do that
         # with an arbitrary number of files of course.

data/lib/zip_tricks/block_write.rb

@@ -20,7 +20,8 @@ class ZipTricks::BlockWrite
   # Every time this object gets written to, call the Rack body each() block
   # with the bytes given instead.
   def <<(buf)
-    return if buf.nil?
+    # Zero-size output has a special meaning  when using chunked encoding
+    return if buf.nil? || buf.bytesize.zero?
 
     # Ensure we ALWAYS write in binary encoding.
     encoded =
@@ -35,16 +36,7 @@ class ZipTricks::BlockWrite
         buf
       end
 
-    #  buf.dup.force_encoding(Encoding::BINARY)
-    # Zero-size output has a special meaning  when using chunked encoding
-    return if encoded.bytesize.zero?
-
     @block.call(encoded)
     self
   end
-
-  # Does nothing
-  def close
-    nil
-  end
 end

data/lib/zip_tricks/output_enumerator.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Can be used as a Rack response body directly. Will yield
+# a {ZipTricks::Streamer} for adding entries to the archive and writing
+# zip entry bodies.
+class ZipTricks::OutputEnumerator
+  # Prepares a new Rack response body with a Zip output stream.
+  # The block given to the constructor will be called when the response
+  # body will be read by the webserver, and will receive a {ZipTricks::Streamer}
+  # as it's block argument. You can then add entries to the Streamer as usual.
+  # The archive will be automatically closed at the end of the block.
+  #
+  #     # Precompute the Content-Length ahead of time
+  #     content_length = ZipTricks::SizeEstimator.estimate do | estimator |
+  #       estimator.add_stored_entry(filename: 'large.tif', size: 1289894)
+  #     end
+  #
+  #     # Prepare the response body.
+  #     # The block will only be called when the
+  #     # response starts to be written.
+  #     body = ZipTricks::OutputEnumerator.new do | streamer |
+  #       streamer.add_stored_entry(filename: 'large.tif', size: 1289894, crc32: 198210)
+  #       streamer << large_file.read(1024*1024) until large_file.eof?
+  #       ...
+  #     end
+  #
+  #     return [200, {'Content-Type' => 'binary/octet-stream',
+  #     'Content-Length' => content_length.to_s}, body]
+  def initialize(**streamer_options, &blk)
+    @streamer_options = streamer_options.to_h
+    @archiving_block = blk
+  end
+
+  # Executes the block given to the constructor with a {ZipTricks::Streamer}
+  # and passes each written chunk to the block given to the method. This allows one
+  # to "take" output of the ZIP piecewise.
+  def each
+    if block_given?
+      block_write = ZipTricks::BlockWrite.new { |chunk| yield(chunk) }
+      ZipTricks::Streamer.open(block_write, **@streamer_options, &@archiving_block)
+    else
+      enum_for(:each)
+    end
+  end
+end

data/lib/zip_tricks/rack_body.rb

@@ -1,44 +1,6 @@
 # frozen_string_literal: true
 
-# Can be used as a Rack response body directly. Will yield
-# a {ZipTricks::Streamer} for adding entries to the archive and writing
-# zip entry bodies.
-class ZipTricks::RackBody
-  # Prepares a new Rack response body with a Zip output stream.
-  # The block given to the constructor will be called when the response
-  # body will be read by the webserver, and will receive a {ZipTricks::Streamer}
-  # as it's block argument. You can then add entries to the Streamer as usual.
-  # The archive will be automatically closed at the end of the block.
-  #
-  #     # Precompute the Content-Length ahead of time
-  #     content_length = ZipTricks::SizeEstimator.estimate do | estimator |
-  #       estimator.add_stored_entry(filename: 'large.tif', size: 1289894)
-  #     end
-  #
-  #     # Prepare the response body. The block will only be called when the
-  #       response starts to be written.
-  #     body = ZipTricks::RackBody.new do | streamer |
-  #       streamer.add_stored_entry(filename: 'large.tif', size: 1289894, crc32: 198210)
-  #       streamer << large_file.read(1024*1024) until large_file.eof?
-  #       ...
-  #     end
-  #
-  #     return [200, {'Content-Type' => 'binary/octet-stream',
-  #     'Content-Length' => content_length.to_s}, body]
-  def initialize(&blk)
-    @archiving_block = blk
-  end
-
-  # Connects a {ZipTricks::BlockWrite} to the Rack webserver output,
-  # and calls the proc given to the constructor with a {ZipTricks::Streamer}
-  # for archive writing.
-  def each(&body_chunk_block)
-    fake_io = ZipTricks::BlockWrite.new(&body_chunk_block)
-    ZipTricks::Streamer.open(fake_io, &@archiving_block)
-  end
-
-  # Does nothing because nothing has to be deallocated or canceled
-  # even if the zip output is incomplete. The archive gets closed
-  # automatically as part of {ZipTricks::Streamer.open}
-  def close; end
+# RackBody is actually just another use of the OutputEnumerator, since a Rack body
+# object must support `#each` yielding successive binary strings.
+class ZipTricks::RackBody < ZipTricks::OutputEnumerator
 end

data/lib/zip_tricks/rails_streaming.rb

@@ -8,7 +8,10 @@ module ZipTricks::RailsStreaming
   # the Rails response stream is going to be closed automatically.
   # @yield [Streamer] the streamer that can be written to
   def zip_tricks_stream
+    # Set a reasonable content type
     response.headers['Content-Type'] = 'application/zip'
+    # Make sure nginx buffering is suppressed - see https://github.com/WeTransfer/zip_tricks/issues/48
+    response.headers['X-Accel-Buffering'] = 'no'
     # Create a wrapper for the write call that quacks like something you
     # can << to, used by ZipTricks
     w = ZipTricks::BlockWrite.new { |chunk| response.stream.write(chunk) }

data/lib/zip_tricks/streamer.rb

@@ -99,6 +99,7 @@ class ZipTricks::Streamer
   # directory of the archive to the output.
   #
   # @param stream [IO] the destination IO for the ZIP (should respond to `tell` and `<<`)
+  # @param kwargs_for_new [Hash] keyword arguments for {Streamer.new}
   # @yield [Streamer] the streamer that can be written to
   def self.open(stream, **kwargs_for_new)
     archive = new(stream, **kwargs_for_new)
@@ -106,6 +107,34 @@ class ZipTricks::Streamer
     archive.close
   end
 
+  # Creates a new Streamer that writes to a buffer. The buffer can be read from using `each`,
+  # and the creation of the ZIP is in lockstep with the caller calling `each` on the returned
+  # output enumerator object. This can be used when the calling program wants to stream the
+  # output of the ZIP archive and throttle that output, or split it into chunks, or use it
+  # as a generator.
+  #
+  # For example:
+  #
+  #     # The block given to {output_enum} won't be executed immediately - rather it
+  #     # will only start to execute when the caller starts to read from the output
+  #     # by calling `each`
+  #     body = ZipTricks::Streamer.output_enum(writer: CustomWriter) do |zip|
+  #       streamer.add_stored_entry(filename: 'large.tif', size: 1289894, crc32: 198210)
+  #       streamer << large_file.read(1024*1024) until large_file.eof?
+  #       ...
+  #     end
+  #
+  #     body.each do |bin_string|
+  #       # Send the output somewhere, buffer it in a file etc.
+  #       ...
+  #     end
+  #
+  # @param kwargs_for_new [Hash] keyword arguments for {Streamer.new}
+  # @return [Enumerator] the enumerator you can read bytestrings of the ZIP from using `each`
+  def self.output_enum(**kwargs_for_new, &zip_streamer_block)
+    ZipTricks::OutputEnumerator.new(**kwargs_for_new, &zip_streamer_block)
+  end
+
   # Creates a new Streamer on top of the given IO-ish object.
   #
   # @param stream[IO] the destination IO for the ZIP. Anything that responds to `<<` can be used.

data/lib/zip_tricks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZipTricks
-  VERSION = '4.6.0'
+  VERSION = '4.7.0'
 end

data/zip_tricks.gemspec

@@ -41,5 +41,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'coderay'
   spec.add_development_dependency 'benchmark-ips'
   spec.add_development_dependency 'yard', '~> 0.9'
-  spec.add_development_dependency 'wetransfer_style', '0.5.0'
+  spec.add_development_dependency 'wetransfer_style', '0.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 4.6.0
+  version: 4.7.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-12 00:00:00.000000000 Z
+date: 2018-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
 description: Stream out ZIP files from Ruby
 email:
 - me@julik.nl
@@ -203,6 +203,7 @@ files:
 - bench/buffered_crc32_bench.rb
 - examples/archive_size_estimate.rb
 - examples/config.ru
+- examples/deferred_write.rb
 - examples/parallel_compression_with_block_deflate.rb
 - examples/rack_application.rb
 - lib/zip_tricks.rb
@@ -212,6 +213,7 @@ files:
 - lib/zip_tricks/file_reader/inflating_reader.rb
 - lib/zip_tricks/file_reader/stored_reader.rb
 - lib/zip_tricks/null_writer.rb
+- lib/zip_tricks/output_enumerator.rb
 - lib/zip_tricks/rack_body.rb
 - lib/zip_tricks/rails_streaming.rb
 - lib/zip_tricks/remote_io.rb