zipline 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8735b5a6429dcffa8672a1a970cc37145eadfd5e5ff13e2cb5b3f08166895d14
-  data.tar.gz: a4555919f140b0661cb12fd0e6f92900e419319ebf7ec7c376ce19034dffd9a9
+  metadata.gz: ca44760a8a5c6019544aa77857692b43d4cd9f429395173a54b779f276f214f2
+  data.tar.gz: c9aec0dc791f84a315658ae9405e88f1d5b7348822befa69e61c92e261ff3a69
 SHA512:
-  metadata.gz: 758292743ee475bc2a276bd00d953fa4f0b1cb1c4ccd2ceae7b2c7dbeced0e1a901a609d183a68c6eeaa898693045f3d5f4ee23bfece0c35b379fb7ce170cdc2
-  data.tar.gz: 7383e439111008fa964a2d762f6855351d3a0372df973d453e0b205aee57f53d9fc8aea54e4f7efed6719b3b8318768f8d125e314ad277f2855bac29fa63c5e2
+  metadata.gz: 2c46de9ba175df42d54c36a37dceddb77cf7816e058528750fe0b05c54313aba1992b8ba12d40b9f029a33c088c79a214d3e50e56cbd0b295c74062fe49c7781
+  data.tar.gz: 5022768afbe367ca08c586b33d16f8a044ad79c7cde80b79be4a3d1c9b18c16d2b0d9cdb9d82f75e3df338d101eda5dce80a099c1261e2e3d828ac637569d790

data/Gemfile

@@ -2,14 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in zipline.gemspec
 gemspec
-
-group :development, :test do
-  gem 'rspec', '~> 3'
-  gem 'fog-aws'
-  gem 'activesupport'
-  gem 'actionpack'
-  gem 'aws-sdk-s3'
-  gem 'carrierwave'
-  gem 'paperclip'
-  gem 'rake'
-end

data/lib/zipline/chunked_body.rb

@@ -0,0 +1,31 @@
+module Zipline
+  # A body wrapper that emits chunked responses, creating valid
+  # "Transfer-Encoding: chunked" HTTP response body. This is copied from Rack::Chunked::Body,
+  # because Rack is not going to include that class after version 3.x
+  # Rails has a substitute class for this inside ActionController::Streaming,
+  # but that module is a private constant in the Rails codebase, and is thus
+  # considered "private" from the Rails standpoint. It is not that much code to
+  # carry, so we copy it into our code.
+  class Chunked
+    TERM = "\r\n"
+    TAIL = "0#{TERM}"
+
+    def initialize(body)
+      @body = body
+    end
+
+    # For each string yielded by the response body, yield
+    # the element in chunked encoding - and finish off with a terminator
+    def each
+      term = TERM
+      @body.each do |chunk|
+        size = chunk.bytesize
+        next if size == 0
+
+        yield [size.to_s(16), term, chunk.b, term].join
+      end
+      yield TAIL
+      yield term
+    end
+  end
+end

data/lib/zipline/tempfile_body.rb

@@ -0,0 +1,52 @@
+module Zipline
+  # Contains a file handle which can be closed once the response finishes sending.
+  # It supports `to_path` so that `Rack::Sendfile` can intercept it
+  class TempfileBody
+    TEMPFILE_NAME_PREFIX = "zipline-tf-body-"
+    attr_reader :tempfile
+
+    # @param body[#each] the enumerable that yields bytes, usually a `RackBody`.
+    #   The `body` will be read in full immediately and closed.
+    def initialize(env, body)
+      @tempfile = Tempfile.new(TEMPFILE_NAME_PREFIX)
+      # Rack::TempfileReaper calls close! on tempfiles which get buffered
+      # We wil assume that it works fine with Rack::Sendfile (i.e. the path
+      # to the file getting served gets used before we unlink the tempfile)
+      env['rack.tempfiles'] ||= []
+      env['rack.tempfiles'] << @tempfile
+
+      @tempfile.binmode
+
+      body.each { |bytes| @tempfile << bytes }
+      body.close if body.respond_to?(:close)
+
+      @tempfile.flush
+    end
+
+    # Returns the size of the contained `Tempfile` so that a correct
+    # Content-Length header can be set
+    #
+    # @return [Integer]
+    def size
+      @tempfile.size
+    end
+
+    # Returns the path to the `Tempfile`, so that Rack::Sendfile can send this response
+    # using the downstream webserver
+    #
+    # @return [String]
+    def to_path
+      @tempfile.to_path
+    end
+
+    # Stream the file's contents if `Rack::Sendfile` isn't present.
+    #
+    # @return [void]
+    def each
+      @tempfile.rewind
+      while chunk = @tempfile.read(16384)
+        yield chunk
+      end
+    end
+  end
+end

data/lib/zipline/version.rb

@@ -1,3 +1,3 @@
 module Zipline
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/lib/zipline/zip_generator.rb

@@ -3,44 +3,35 @@
 module Zipline
   class ZipGenerator
     # takes an array of pairs [[uploader, filename], ... ]
-    def initialize(files,  **kwargs_for_new)
-      @files = files
-      @kwargs_for_new = kwargs_for_new
-    end
-
-    #this is supposed to be streamed!
-    def to_s
-      throw "stop!"
-    end
-
-    def each(&block)
-      fake_io_writer = ZipTricks::BlockWrite.new(&block)
-      # ZipTricks outputs lots of strings in rapid succession, and with
-      # servers it can be beneficial to avoid doing too many tiny writes so that
-      # the number of syscalls is minimized. See https://github.com/WeTransfer/zip_tricks/issues/78
-      # There is a built-in facility for this in ZipTricks which can be used to implement
-      # some cheap buffering here (it exists both in version 4 and version 5). The buffer is really
-      # tiny and roughly equal to the medium Linux socket buffer size (16 KB). Although output
-      # will be not so immediate with this buffering the overall performance will be better,
-      # especially with multiple clients being serviced at the same time.
-      # Note that the WriteBuffer writes the same, retained String object - but the contents
-      # of that object changes between calls. This should work fine with servers where the
-      # contents of the string gets written to a socket immediately before the execution inside
-      # the WriteBuffer resumes), but if the strings get retained somewhere - like in an Array -
-      # this might pose a problem. Unlikely that it will be an issue here though.
-      write_buffer_size = 16 * 1024
-      write_buffer = ZipTricks::WriteBuffer.new(fake_io_writer, write_buffer_size)
-      ZipTricks::Streamer.open(write_buffer, **@kwargs_for_new) do |streamer|
-        @files.each do |file, name, options = {}|
+    def initialize(files,  **kwargs_for_streamer)
+      # Use RackBody as it has buffering built-in in zip_tricks 5.x+
+      @body = ZipTricks::RackBody.new(**kwargs_for_streamer) do |streamer|
+        files.each do |file, name, options = {}|
           handle_file(streamer, file, name.to_s, options)
         end
       end
-      write_buffer.flush! # for any remaining writes
+    end
+
+    def each(&block)
+      return to_enum(:each) unless block_given?
+      @body.each(&block)
+    rescue => e
+      # Since most APM packages do not trace errors occurring within streaming
+      # Rack bodies, it can be helpful to print the error to the Rails log at least
+      error_message = "zipline: an exception (#{e.inspect}) was raised  when serving the ZIP body."
+      error_message += " The error occurred when handling #{@filename.inspect}" if @filename
+      logger.error(error_message)
+      raise
     end
 
     def handle_file(streamer, file, name, options)
       file = normalize(file)
+
+      # Store the filename so that a sensible error message can be displayed in the log
+      # if writing this particular file fails
+      @filename = name
       write_file(streamer, file, name, options)
+      @filename = nil
     end
 
     # This extracts either a url or a local file from the provided file.
@@ -107,6 +98,16 @@ module Zipline
 
     private
 
+    def logger
+      # Rails is not defined in our tests, and might as well not be defined
+      # elsewhere - or the logger might not be configured correctly
+      if defined?(Rails.logger) && Rails.logger
+        Rails.logger
+      else
+        Logger.new($stderr)
+      end
+    end
+
     def is_active_storage_attachment?(file)
       defined?(ActiveStorage::Attachment) && file.is_a?(ActiveStorage::Attachment)
     end

data/lib/zipline.rb

@@ -1,7 +1,9 @@
 require 'content_disposition'
-require "zipline/version"
+require 'zipline/version'
 require 'zip_tricks'
-require "zipline/zip_generator"
+require 'zipline/zip_generator'
+require 'zipline/chunked_body'
+require 'zipline/tempfile_body'
 
 # class MyController < ApplicationController
 #   include Zipline
@@ -18,7 +20,36 @@ module Zipline
     headers['Content-Type'] = Mime::Type.lookup_by_extension('zip').to_s
     response.sending_file = true
     response.cache_control[:public] ||= false
-    self.response_body = zip_generator
+
+    # Disables Rack::ETag if it is enabled (prevent buffering)
+    # see https://github.com/rack/rack/issues/1619#issuecomment-606315714
     self.response.headers['Last-Modified'] = Time.now.httpdate
+
+    if request.get_header("HTTP_VERSION") == "HTTP/1.0"
+      # If HTTP/1.0 is used it is not possible to stream, and if that happens it usually will be
+      # unclear why buffering is happening. Some info in the log is the least one can do.
+      logger.warn { "The downstream HTTP proxy/LB insists on HTTP/1.0 protocol, ZIP response will be buffered." } if logger
+
+      # If we use Rack::ContentLength it would run through our ZIP block twice - once to calculate the content length
+      # of the response, and once - to serve. We can trade performance for disk space and buffer the response into a Tempfile
+      # since we are already buffering.
+      tempfile_body = TempfileBody.new(request.env, zip_generator)
+      headers["Content-Length"] = tempfile_body.size.to_s
+      headers["X-Zipline-Output"] = "buffered"
+      self.response_body = tempfile_body
+    else
+      # Disable buffering for both nginx and Google Load Balancer, see
+      # https://cloud.google.com/appengine/docs/flexible/how-requests-are-handled?tab=python#x-accel-buffering
+      response.headers["X-Accel-Buffering"] = "no"
+
+      # Make sure Rack::ContentLength does not try to compute a content length,
+      # and remove the one already set
+      headers.delete("Content-Length")
+
+      # and send out in chunked encoding
+      headers["Transfer-Encoding"] = "chunked"
+      headers["X-Zipline-Output"] = "streamed"
+      self.response_body = Chunked.new(zip_generator)
+    end
   end
 end

data/spec/lib/zipline/zip_generator_spec.rb

@@ -185,91 +185,4 @@ describe Zipline::ZipGenerator do
       end
     end
   end
-
-  describe '.write_file' do
-    let(:file) { StringIO.new('passthrough') }
-
-    context 'when passing an ActiveStorage::Filename object as filename' do
-      let(:filename) { ActiveStorage::Filename.new('test') }
-
-      let(:generator) do
-        Zipline::ZipGenerator.new([[file, filename]])
-      end
-
-      it 'passes a string as filename to ZipTricks' do
-        allow(file).to receive(:url).and_return('fakeurl')
-        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
-          .with('test')
-        generator.each { |_| 'Test' }
-      end
-    end
-  end
-
-  describe 'passing an options hash' do
-    let(:file) { StringIO.new('passthrough') }
-
-    context 'with optional arguments' do
-      let(:mtime) { 1.day.ago }
-      let(:generator) do
-        Zipline::ZipGenerator.new([[file, 'test', modification_time: mtime]])
-      end
-
-      it 'passes the options hash through handle_file' do
-        expect(generator).to receive(:handle_file)
-          .with(anything, anything, anything, { modification_time: mtime })
-        generator.each { |_| 'Test' }
-      end
-
-      it 'passes the options hash to ZipTricks as kwargs' do
-        allow(file).to receive(:url).and_return('fakeurl')
-        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
-          .with(anything, modification_time: mtime)
-        generator.each { |_| 'Test' }
-      end
-    end
-
-    context 'without optional arguments' do
-      let(:generator) do
-        Zipline::ZipGenerator.new([[file, 'test']])
-      end
-
-      it 'passes the options hash through handle_file' do
-        expect(generator).to receive(:handle_file)
-          .with(anything, anything, anything, {})
-        generator.each { |_| 'Test' }
-      end
-
-      it 'passes the options hash to ZipTricks as kwargs' do
-        allow(file).to receive(:url).and_return('fakeurl')
-        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
-          .with(anything)
-        generator.each { |_| 'Test' }
-      end
-    end
-
-    context 'with extra invalid options' do
-      let(:mtime) { 1.day.ago }
-      let(:generator) do
-        Zipline::ZipGenerator.new([[file, 'test', modification_time: mtime, extra: 'invalid']])
-      end
-
-      it 'passes the whole options hash through handle_file' do
-        expect(generator).to receive(:handle_file)
-          .with(anything, anything, anything, { modification_time: mtime, extra: 'invalid' })
-        generator.each { |_| 'Test' }
-      end
-
-      it 'only passes the kwargs to ZipTricks that it expects (i.e., :modification_time)' do
-        allow(file).to receive(:url).and_return('fakeurl')
-        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
-          .with(anything, modification_time: mtime)
-        generator.each { |_| 'Test' }
-      end
-    end
-  end
-  it 'passes along constructor options to ZipTricks streamer' do
-    expect(ZipTricks::Streamer).to receive(:open).with(anything, { :some => 'options' })
-    generator = Zipline::ZipGenerator.new([file, 'somefile'], :some => 'options')
-    generator.each { |_| 'Test' }
-  end
 end

data/spec/lib/zipline/zipline_spec.rb

@@ -1,29 +1,68 @@
 require 'spec_helper'
-require 'ostruct'
+require 'action_controller'
 
 describe Zipline do
   before { Fog.mock! }
 
-  let (:undertest) {
-    class TestZipline
+  class FakeController < ActionController::Base
+    include Zipline
+    def download_zip
+      files = [
+        [StringIO.new("File content goes here"), "one.txt"],
+        [StringIO.new("Some other content goes here"), "two.txt"]
+      ]
+      zipline(files, 'myfiles.zip', auto_rename_duplicate_filenames: false)
+    end
 
-      attr_accessor :headers
-      attr_accessor :response
-      attr_accessor :response_body
-      def initialize 
-        @headers = {}
-        @response = OpenStruct.new(:cache_control => {}, :headers => {} )
+    class FailingIO < StringIO
+      def read(*)
+        raise "Something wonky"
       end
-      include Zipline
-    end 
-    return TestZipline.new()
-  }
+    end
 
+    def download_zip_with_error_during_streaming
+      files = [
+        [StringIO.new("File content goes here"), "one.txt"],
+        [FailingIO.new("This will fail half-way"), "two.txt"]
+      ]
+      zipline(files, 'myfiles.zip', auto_rename_duplicate_filenames: false)
+    end
+  end
+
+  it 'passes keyword parameters to ZipTricks::Streamer' do
+    fake_rack_env = {
+      "HTTP_VERSION" => "HTTP/1.0",
+      "REQUEST_METHOD" => "GET",
+      "SCRIPT_NAME" => "",
+      "PATH_INFO" => "/download",
+      "QUERY_STRING" => "",
+      "SERVER_NAME" => "host.example",
+      "rack.input" => StringIO.new,
+    }
+    expect(ZipTricks::Streamer).to receive(:new).with(anything, auto_rename_duplicate_filenames: false).and_call_original
+
+    status, headers, body = FakeController.action(:download_zip).call(fake_rack_env)
+
+    expect(headers['Content-Disposition']).to eq("attachment; filename=\"myfiles.zip\"; filename*=UTF-8''myfiles.zip")
+  end
+
+  it 'sends the exception raised in the streaming body to the Rails logger' do
+    fake_rack_env = {
+      "HTTP_VERSION" => "HTTP/1.0",
+      "REQUEST_METHOD" => "GET",
+      "SCRIPT_NAME" => "",
+      "PATH_INFO" => "/download",
+      "QUERY_STRING" => "",
+      "SERVER_NAME" => "host.example",
+      "rack.input" => StringIO.new,
+    }
+    expect(ZipTricks::Streamer).to receive(:new).with(anything, auto_rename_duplicate_filenames: false).and_call_original
+    fake_logger = double()
+    expect(Logger).to receive(:new).and_return(fake_logger)
+    expect(fake_logger).to receive(:error).with(instance_of(String))
 
-  it 'passes arguments along' do
-    expect(Zipline::ZipGenerator).to receive(:new)
-          .with(['some', 'fake', 'files'], { some: 'options' })
-    undertest.zipline(['some', 'fake', 'files'], 'myfiles.zip', some: 'options')
-    expect(undertest.headers['Content-Disposition']).to eq("attachment; filename=\"myfiles.zip\"; filename*=UTF-8''myfiles.zip")
+    expect {
+      FakeController.action(:download_zip_with_error_during_streaming).call(fake_rack_env)
+    }.to raise_error(/Something wonky/)
   end
 end

data/spec/spec_helper.rb

@@ -8,17 +8,16 @@ require 'paperclip'
 require 'fog-aws'
 require 'carrierwave'
 
-Dir["#{File.expand_path('..', __FILE__)}/support/**/*.rb"].each { |f| require f }
+Dir["#{File.expand_path('..', __FILE__)}/support/**/*.rb"].sort.each { |f| require f }
 
 CarrierWave.configure do |config|
   config.fog_provider = 'fog/aws'
   config.fog_credentials = {
-		provider:              'AWS',
-		aws_access_key_id:     'dummy',
-		aws_secret_access_key: 'data',
-		region:                'us-west-2',
-	}
-
+    provider:              'AWS',
+    aws_access_key_id:     'dummy',
+    aws_secret_access_key: 'data',
+    region:                'us-west-2',
+  }
 end
 
 RSpec.configure do |config|

data/zipline.gemspec

@@ -20,8 +20,15 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'actionpack', ['>= 6.0', '< 8.0']
   gem.add_dependency 'content_disposition', '~> 1.0'
-  gem.add_dependency 'zip_tricks', ['>= 4.2.1', '< 6.0']
+  gem.add_dependency 'zip_tricks', ['~> 4.8', '< 6'] # Minimum to 4.8.3 which is the last-released MIT version
+
+  gem.add_development_dependency 'rspec', '~> 3'
+  gem.add_development_dependency 'fog-aws'
+  gem.add_development_dependency 'aws-sdk-s3'
+  gem.add_development_dependency 'carrierwave'
+  gem.add_development_dependency 'paperclip'
+  gem.add_development_dependency 'rake'
 
   # https://github.com/rspec/rspec-mocks/issues/1457
-  gem.add_development_dependency 'rspec-mocks', '3.10.2'
+  gem.add_development_dependency 'rspec-mocks', '~> 3.12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zipline
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Ram Dobson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-24 00:00:00.000000000 Z
+date: 2024-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -48,36 +48,120 @@ dependencies:
   name: zip_tricks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.1
+        version: '4.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.1
+        version: '4.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: carrierwave
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: paperclip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.2
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.2
+        version: '3.12'
 description: a module for streaming dynamically generated zip files
 email:
 - ram.dobson@solsystemscompany.com
@@ -91,6 +175,8 @@ files:
 - README.md
 - Rakefile
 - lib/zipline.rb
+- lib/zipline/chunked_body.rb
+- lib/zipline/tempfile_body.rb
 - lib/zipline/version.rb
 - lib/zipline/zip_generator.rb
 - spec/fakefile.txt
@@ -117,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: stream zip files from rails