zipline 1.6.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca44760a8a5c6019544aa77857692b43d4cd9f429395173a54b779f276f214f2
-  data.tar.gz: c9aec0dc791f84a315658ae9405e88f1d5b7348822befa69e61c92e261ff3a69
+  metadata.gz: 954c769aaf6aa7780a8770fcc9991a1beea5b326f1920b6bfdcc921dcef1a34d
+  data.tar.gz: 7e40d818c5b4e7da8c8167f42e9318d0d7cfee2384657a84fa3ee294184f9819
 SHA512:
-  metadata.gz: 2c46de9ba175df42d54c36a37dceddb77cf7816e058528750fe0b05c54313aba1992b8ba12d40b9f029a33c088c79a214d3e50e56cbd0b295c74062fe49c7781
-  data.tar.gz: 5022768afbe367ca08c586b33d16f8a044ad79c7cde80b79be4a3d1c9b18c16d2b0d9cdb9d82f75e3df338d101eda5dce80a099c1261e2e3d828ac637569d790
+  metadata.gz: 5a39143123fc75b44b94d88ca0e8d31a41e2b63f9368ff70692c502c21171baf406e4eb8bb7a9f9ef0885d2883d8c5e5d40995499068a2a0a9a9906c9135bc68
+  data.tar.gz: 84c7b320023d7195f619638b87acaa14121852a643a727a9ba5c1d848ed3d4bc045832347586ca3b454dfb753ad189f2bfe96d5e97bd6140065a1fa0ef4675b8

data/README.md

@@ -7,7 +7,7 @@ A gem to stream dynamically generated zip files from a rails application. Unlike
 - Removes need for large disk space or memory allocation to generate zips, even huge zips. So it works on Heroku.
 - The user begins downloading immediately, which decreaceses latency, download time, and timeouts on Heroku.
 
-Zipline now depends on [zip tricks](https://github.com/WeTransfer/zip_tricks), and you might want to just use that directly if you have more advanced use cases.
+Zipline now depends on [zip_kit](https://github.com/julik/zip_kit), and you might want to just use that directly if you have more advanced use cases.
 
 ## Installation
 
@@ -43,7 +43,7 @@ class MyController < ApplicationController
     files = users.map{ |user| [user.avatar, "#{user.username}.png", modification_time: 1.day.ago] }
 
     # we can force duplicate file names to be renamed, or raise an error
-    # we can also pass in our own writer if required to conform with the Delegated [ZipTricks::Streamer object](https://github.com/WeTransfer/zip_tricks/blob/main/lib/zip_tricks/streamer.rb#L147) object.
+    # we can also pass in our own writer if required to conform with the delegated [ZipKit::Streamer object](https://github.com/julik/zip_kit/blob/main/lib/zip_kit/streamer.rb#L147) object.
     zipline(files, 'avatars.zip', auto_rename_duplicate_filenames: true) 
   end
 end
@@ -93,7 +93,7 @@ For directories, just give the files names like "directory/file".
 
 ```Ruby
 avatars = [
-  # remote_url                          zip_path             zip_tricks_options
+  # remote_url                          zip_path             write_file options for Streamer
   [ 'http://www.example.com/user1.png', 'avatars/user1.png', modification_time: Time.now.utc ]
   [ 'http://www.example.com/user2.png', 'avatars/user2.png', modification_time: 1.day.ago ]
   [ 'http://www.example.com/user3.png', 'avatars/user3.png' ]

data/lib/zipline/version.rb

@@ -1,3 +1,3 @@
 module Zipline
-  VERSION = "1.6.0"
+  VERSION = "2.0.0"
 end

data/lib/zipline/{zip_generator.rb → zip_handler.rb}

@@ -1,39 +1,23 @@
-# this class acts as a streaming body for rails
-# initialize it with an array of the files you want to zip
 module Zipline
-  class ZipGenerator
+  class ZipHandler
     # takes an array of pairs [[uploader, filename], ... ]
-    def initialize(files,  **kwargs_for_streamer)
-      # Use RackBody as it has buffering built-in in zip_tricks 5.x+
-      @body = ZipTricks::RackBody.new(**kwargs_for_streamer) do |streamer|
-        files.each do |file, name, options = {}|
-          handle_file(streamer, file, name.to_s, options)
-        end
-      end
+    def initialize(streamer, logger)
+      @streamer = streamer
+      @logger = logger
     end
 
-    def each(&block)
-      return to_enum(:each) unless block_given?
-      @body.each(&block)
+    def handle_file(file, name, options)
+      normalized_file = normalize(file)
+      write_file(normalized_file, name, options)
     rescue => e
       # Since most APM packages do not trace errors occurring within streaming
       # Rack bodies, it can be helpful to print the error to the Rails log at least
       error_message = "zipline: an exception (#{e.inspect}) was raised  when serving the ZIP body."
-      error_message += " The error occurred when handling #{@filename.inspect}" if @filename
-      logger.error(error_message)
+      error_message += " The error occurred when handling file #{name.inspect}"
+      @logger.error(error_message) if @logger
       raise
     end
 
-    def handle_file(streamer, file, name, options)
-      file = normalize(file)
-
-      # Store the filename so that a sensible error message can be displayed in the log
-      # if writing this particular file fails
-      @filename = name
-      write_file(streamer, file, name, options)
-      @filename = nil
-    end
-
     # This extracts either a url or a local file from the provided file.
     # Currently support carrierwave and paperclip local and remote storage.
     # returns a hash of the form {url: aUrl} or {file: anIoObject}
@@ -71,8 +55,8 @@ module Zipline
       end
     end
 
-    def write_file(streamer, file, name, options)
-      streamer.write_deflated_file(name, **options.slice(:modification_time)) do |writer_for_file|
+    def write_file(file, name, options)
+      @streamer.write_file(name, **options.slice(:modification_time)) do |writer_for_file|
         if file[:url]
           the_remote_uri = URI(file[:url])
 
@@ -92,20 +76,10 @@ module Zipline
       end
     end
 
-    def is_io?(io_ish)
-      io_ish.respond_to? :read
-    end
-
     private
 
-    def logger
-      # Rails is not defined in our tests, and might as well not be defined
-      # elsewhere - or the logger might not be configured correctly
-      if defined?(Rails.logger) && Rails.logger
-        Rails.logger
-      else
-        Logger.new($stderr)
-      end
+    def is_io?(io_ish)
+      io_ish.respond_to? :read
     end
 
     def is_active_storage_attachment?(file)

data/lib/zipline.rb

@@ -1,9 +1,7 @@
 require 'content_disposition'
+require 'zip_kit'
 require 'zipline/version'
-require 'zip_tricks'
-require 'zipline/zip_generator'
-require 'zipline/chunked_body'
-require 'zipline/tempfile_body'
+require 'zipline/zip_handler'
 
 # class MyController < ApplicationController
 #   include Zipline
@@ -14,42 +12,17 @@ require 'zipline/tempfile_body'
 #   end
 # end
 module Zipline
-  def zipline(files, zipname = 'zipline.zip', **kwargs_for_new)
-    zip_generator = ZipGenerator.new(files, **kwargs_for_new)
-    headers['Content-Disposition'] = ContentDisposition.format(disposition: 'attachment', filename: zipname)
-    headers['Content-Type'] = Mime::Type.lookup_by_extension('zip').to_s
-    response.sending_file = true
-    response.cache_control[:public] ||= false
-
-    # Disables Rack::ETag if it is enabled (prevent buffering)
-    # see https://github.com/rack/rack/issues/1619#issuecomment-606315714
-    self.response.headers['Last-Modified'] = Time.now.httpdate
-
-    if request.get_header("HTTP_VERSION") == "HTTP/1.0"
-      # If HTTP/1.0 is used it is not possible to stream, and if that happens it usually will be
-      # unclear why buffering is happening. Some info in the log is the least one can do.
-      logger.warn { "The downstream HTTP proxy/LB insists on HTTP/1.0 protocol, ZIP response will be buffered." } if logger
-
-      # If we use Rack::ContentLength it would run through our ZIP block twice - once to calculate the content length
-      # of the response, and once - to serve. We can trade performance for disk space and buffer the response into a Tempfile
-      # since we are already buffering.
-      tempfile_body = TempfileBody.new(request.env, zip_generator)
-      headers["Content-Length"] = tempfile_body.size.to_s
-      headers["X-Zipline-Output"] = "buffered"
-      self.response_body = tempfile_body
-    else
-      # Disable buffering for both nginx and Google Load Balancer, see
-      # https://cloud.google.com/appengine/docs/flexible/how-requests-are-handled?tab=python#x-accel-buffering
-      response.headers["X-Accel-Buffering"] = "no"
-
-      # Make sure Rack::ContentLength does not try to compute a content length,
-      # and remove the one already set
-      headers.delete("Content-Length")
+  def self.included(into_controller)
+    into_controller.include(ZipKit::RailsStreaming)
+    super
+  end
 
-      # and send out in chunked encoding
-      headers["Transfer-Encoding"] = "chunked"
-      headers["X-Zipline-Output"] = "streamed"
-      self.response_body = Chunked.new(zip_generator)
+  def zipline(files, zipname = 'zipline.zip', **kwargs_for_zip_kit_stream)
+    zip_kit_stream(filename: zipname, **kwargs_for_zip_kit_stream) do |zip_kit_streamer|
+      handler = Zipline::ZipHandler.new(zip_kit_streamer, logger)
+      files.each do |file, name, options = {}|
+        handler.handle_file(file, name.to_s, options)
+      end
     end
   end
 end

data/spec/lib/zipline/{zip_generator_spec.rb → zip_handler_spec.rb}

@@ -18,7 +18,7 @@ module ActiveStorage
   end
 end
 
-describe Zipline::ZipGenerator do
+describe Zipline::ZipHandler do
   before { Fog.mock! }
   let(:file_attributes){ {
     key: 'fog_file_tests',
@@ -38,21 +38,21 @@ describe Zipline::ZipGenerator do
   let(:file){ directory.files.create(file_attributes) }
 
   describe '.normalize' do
-    let(:generator){ Zipline::ZipGenerator.new([])}
+    let(:handler){ Zipline::ZipHandler.new(_streamer = double(), _logger = nil)}
     context "CarrierWave" do
       context "Remote" do
         let(:file){ CarrierWave::Storage::Fog::File.new(nil,nil,nil) }
         it "extracts the url" do
           allow(file).to receive(:url).and_return('fakeurl')
           expect(File).not_to receive(:open)
-          expect(generator.normalize(file)).to eq({url: 'fakeurl'})
+          expect(handler.normalize(file)).to eq({url: 'fakeurl'})
         end
       end
       context "Local" do
         let(:file){ CarrierWave::SanitizedFile.new(Tempfile.new('t')) }
         it "creates a File" do
           allow(file).to receive(:path).and_return('spec/fakefile.txt')
-          normalized = generator.normalize(file)
+          normalized = handler.normalize(file)
           expect(normalized.keys).to include(:file)
           expect(normalized[:file]).to be_a File
         end
@@ -66,7 +66,7 @@ describe Zipline::ZipGenerator do
             allow(uploader).to receive(:file).and_return(file)
             allow(file).to receive(:url).and_return('fakeurl')
             expect(File).not_to receive(:open)
-            expect(generator.normalize(uploader)).to eq({url: 'fakeurl'})
+            expect(handler.normalize(uploader)).to eq({url: 'fakeurl'})
           end
         end
 
@@ -75,7 +75,7 @@ describe Zipline::ZipGenerator do
           it "creates a File" do
             allow(uploader).to receive(:file).and_return(file)
             allow(file).to receive(:path).and_return('spec/fakefile.txt')
-            normalized = generator.normalize(uploader)
+            normalized = handler.normalize(uploader)
             expect(normalized.keys).to include(:file)
             expect(normalized[:file]).to be_a File
           end
@@ -87,7 +87,7 @@ describe Zipline::ZipGenerator do
         let(:file){ Paperclip::Attachment.new(:name, :instance) }
         it "creates a File" do
           allow(file).to receive(:path).and_return('spec/fakefile.txt')
-          normalized = generator.normalize(file)
+          normalized = handler.normalize(file)
           expect(normalized.keys).to include(:file)
           expect(normalized[:file]).to be_a File
         end
@@ -97,7 +97,7 @@ describe Zipline::ZipGenerator do
         it "creates a URL" do
           allow(file).to receive(:expiring_url).and_return('fakeurl')
           expect(File).to_not receive(:open)
-          expect(generator.normalize(file)).to include(url: 'fakeurl')
+          expect(handler.normalize(file)).to include(url: 'fakeurl')
         end
       end
     end
@@ -107,7 +107,7 @@ describe Zipline::ZipGenerator do
           attached = create_attached_one
           allow_any_instance_of(Object).to receive(:defined?).and_return(true)
 
-          normalized = generator.normalize(attached)
+          normalized = handler.normalize(attached)
 
           expect(normalized.keys).to include(:blob)
           expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
@@ -119,7 +119,7 @@ describe Zipline::ZipGenerator do
           attachment = create_attachment
           allow_any_instance_of(Object).to receive(:defined?).and_return(true)
 
-          normalized = generator.normalize(attachment)
+          normalized = handler.normalize(attachment)
 
           expect(normalized.keys).to include(:blob)
           expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
@@ -131,7 +131,7 @@ describe Zipline::ZipGenerator do
           blob = create_blob
           allow_any_instance_of(Object).to receive(:defined?).and_return(true)
 
-          normalized = generator.normalize(blob)
+          normalized = handler.normalize(blob)
 
           expect(normalized.keys).to include(:blob)
           expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
@@ -169,19 +169,19 @@ describe Zipline::ZipGenerator do
       it "extracts url" do
         allow(file).to receive(:url).and_return('fakeurl')
         expect(File).not_to receive(:open)
-        expect(generator.normalize(file)).to eq(url:  'fakeurl')
+        expect(handler.normalize(file)).to eq(url:  'fakeurl')
       end
     end
     context "IOStream" do
       let(:file){ StringIO.new('passthrough')}
       it "passes through" do
-        expect(generator.normalize(file)).to eq(file: file)
+        expect(handler.normalize(file)).to eq(file: file)
       end
     end
     context "invalid" do
       let(:file){ Thread.new{} }
       it "raises error" do
-        expect{generator.normalize(file)}.to raise_error(ArgumentError)
+        expect{handler.normalize(file)}.to raise_error(ArgumentError)
       end
     end
   end

data/spec/lib/zipline/zipline_spec.rb

@@ -2,7 +2,10 @@ require 'spec_helper'
 require 'action_controller'
 
 describe Zipline do
-  before { Fog.mock! }
+  before do
+    Fog.mock!
+    FakeController.logger = nil
+  end
 
   class FakeController < ActionController::Base
     include Zipline
@@ -29,7 +32,7 @@ describe Zipline do
     end
   end
 
-  it 'passes keyword parameters to ZipTricks::Streamer' do
+  it 'passes keyword parameters to ZipKit::OutputEnumerator' do
     fake_rack_env = {
       "HTTP_VERSION" => "HTTP/1.0",
       "REQUEST_METHOD" => "GET",
@@ -39,7 +42,7 @@ describe Zipline do
       "SERVER_NAME" => "host.example",
       "rack.input" => StringIO.new,
     }
-    expect(ZipTricks::Streamer).to receive(:new).with(anything, auto_rename_duplicate_filenames: false).and_call_original
+    expect(ZipKit::OutputEnumerator).to receive(:new).with(auto_rename_duplicate_filenames: false).and_call_original
 
     status, headers, body = FakeController.action(:download_zip).call(fake_rack_env)
 
@@ -56,13 +59,15 @@ describe Zipline do
       "SERVER_NAME" => "host.example",
       "rack.input" => StringIO.new,
     }
-    expect(ZipTricks::Streamer).to receive(:new).with(anything, auto_rename_duplicate_filenames: false).and_call_original
     fake_logger = double()
-    expect(Logger).to receive(:new).and_return(fake_logger)
-    expect(fake_logger).to receive(:error).with(instance_of(String))
+    allow(fake_logger).to receive(:warn)
+    expect(fake_logger).to receive(:error).with(a_string_matching(/when serving the ZIP/))
+
+    FakeController.logger = fake_logger
 
     expect {
-      FakeController.action(:download_zip_with_error_during_streaming).call(fake_rack_env)
+      status, headers, body = FakeController.action(:download_zip_with_error_during_streaming).call(fake_rack_env)
+      body.each { }
     }.to raise_error(/Something wonky/)
   end
 end

data/zipline.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'actionpack', ['>= 6.0', '< 8.0']
   gem.add_dependency 'content_disposition', '~> 1.0'
-  gem.add_dependency 'zip_tricks', ['~> 4.8', '< 6'] # Minimum to 4.8.3 which is the last-released MIT version
+  gem.add_dependency 'zip_kit', ['~> 6', '>= 6.2.0', '< 7']
 
   gem.add_development_dependency 'rspec', '~> 3'
   gem.add_development_dependency 'fog-aws'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zipline
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Ram Dobson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-12 00:00:00.000000000 Z
+date: 2024-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -45,25 +45,31 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: zip_tricks
+  name: zip_kit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: '6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: '6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -175,12 +181,10 @@ files:
 - README.md
 - Rakefile
 - lib/zipline.rb
-- lib/zipline/chunked_body.rb
-- lib/zipline/tempfile_body.rb
 - lib/zipline/version.rb
-- lib/zipline/zip_generator.rb
+- lib/zipline/zip_handler.rb
 - spec/fakefile.txt
-- spec/lib/zipline/zip_generator_spec.rb
+- spec/lib/zipline/zip_handler_spec.rb
 - spec/lib/zipline/zipline_spec.rb
 - spec/spec_helper.rb
 - zipline.gemspec
@@ -209,6 +213,6 @@ specification_version: 4
 summary: stream zip files from rails
 test_files:
 - spec/fakefile.txt
-- spec/lib/zipline/zip_generator_spec.rb
+- spec/lib/zipline/zip_handler_spec.rb
 - spec/lib/zipline/zipline_spec.rb
 - spec/spec_helper.rb

data/lib/zipline/chunked_body.rb

@@ -1,31 +0,0 @@
-module Zipline
-  # A body wrapper that emits chunked responses, creating valid
-  # "Transfer-Encoding: chunked" HTTP response body. This is copied from Rack::Chunked::Body,
-  # because Rack is not going to include that class after version 3.x
-  # Rails has a substitute class for this inside ActionController::Streaming,
-  # but that module is a private constant in the Rails codebase, and is thus
-  # considered "private" from the Rails standpoint. It is not that much code to
-  # carry, so we copy it into our code.
-  class Chunked
-    TERM = "\r\n"
-    TAIL = "0#{TERM}"
-
-    def initialize(body)
-      @body = body
-    end
-
-    # For each string yielded by the response body, yield
-    # the element in chunked encoding - and finish off with a terminator
-    def each
-      term = TERM
-      @body.each do |chunk|
-        size = chunk.bytesize
-        next if size == 0
-
-        yield [size.to_s(16), term, chunk.b, term].join
-      end
-      yield TAIL
-      yield term
-    end
-  end
-end

data/lib/zipline/tempfile_body.rb

@@ -1,52 +0,0 @@
-module Zipline
-  # Contains a file handle which can be closed once the response finishes sending.
-  # It supports `to_path` so that `Rack::Sendfile` can intercept it
-  class TempfileBody
-    TEMPFILE_NAME_PREFIX = "zipline-tf-body-"
-    attr_reader :tempfile
-
-    # @param body[#each] the enumerable that yields bytes, usually a `RackBody`.
-    #   The `body` will be read in full immediately and closed.
-    def initialize(env, body)
-      @tempfile = Tempfile.new(TEMPFILE_NAME_PREFIX)
-      # Rack::TempfileReaper calls close! on tempfiles which get buffered
-      # We wil assume that it works fine with Rack::Sendfile (i.e. the path
-      # to the file getting served gets used before we unlink the tempfile)
-      env['rack.tempfiles'] ||= []
-      env['rack.tempfiles'] << @tempfile
-
-      @tempfile.binmode
-
-      body.each { |bytes| @tempfile << bytes }
-      body.close if body.respond_to?(:close)
-
-      @tempfile.flush
-    end
-
-    # Returns the size of the contained `Tempfile` so that a correct
-    # Content-Length header can be set
-    #
-    # @return [Integer]
-    def size
-      @tempfile.size
-    end
-
-    # Returns the path to the `Tempfile`, so that Rack::Sendfile can send this response
-    # using the downstream webserver
-    #
-    # @return [String]
-    def to_path
-      @tempfile.to_path
-    end
-
-    # Stream the file's contents if `Rack::Sendfile` isn't present.
-    #
-    # @return [void]
-    def each
-      @tempfile.rewind
-      while chunk = @tempfile.read(16384)
-        yield chunk
-      end
-    end
-  end
-end