zipline 1.0.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9f78131919f81ca6a316cb1f1c7173515a230cee
-  data.tar.gz: 5143037d6d038adf9af45b269f8653f4dd9c1d47
+SHA256:
+  metadata.gz: 8de63c7a8518461b8fdbb249ce857b3f06ffc2bf3a7de69d6d5d6c5111f75015
+  data.tar.gz: edfc26eed9a1d52987a4d47330a015d2ebb0e3129f2c0791199a51b3b5abec70
 SHA512:
-  metadata.gz: 641f00ccfa244d4a5d7e25cb99aaac1356069e9ed1bdf1ea52da7437f613ebf6cc0a31ba0c707777bf0772c899f24d556e657ee29a39a3672a8841ff5d7ef7d7
-  data.tar.gz: bf7592dc54f890f4aadd6628d814048958308c8f20ea51ff3a34088312f497bc4b882b1b4a6fb8e8728dd03b130130cbb4c72617adc86484a738d9af810d22f6
+  metadata.gz: cc54e592ebd7c24cad03669786938a3662430389e81322ceef258898ec5d8e9949db956a7433ab7c4d139b4671100772c062c525f8d20b6782f3a221b80ea9f3
+  data.tar.gz: 2482828adb931c9630c209b7a9a06c913b43d577232dea8d877b88acb1fa3b60834c35a92175608610da0d7e80ad36d6a1f8cd2379b9af5a249ce2b1f7456990

data/.travis.yml

@@ -1,4 +1,8 @@
 language: ruby
+cache: bundler
 rvm:
   - 2.4
-  
+  - 2.5
+  - 2.6
+  - 2.7
+  - 3.0

data/Gemfile

@@ -5,12 +5,10 @@ gemspec
 
 group :development, :test do
   gem 'rspec', '~> 3'
-  gem 'fog'
   gem 'fog-aws'
   gem 'activesupport'
-  gem 'aws-sdk'
+  gem 'aws-sdk-s3'
   gem 'carrierwave'
   gem 'paperclip'
   gem 'rake'
-  gem 'rake-rspec'
 end

data/README.md

@@ -1,4 +1,6 @@
 # Zipline
+[![Build Status](https://travis-ci.org/fringd/zipline.svg?branch=master)](https://travis-ci.org/fringd/zipline)
+[![Gem Version](https://badge.fury.io/rb/zipline.svg)](https://badge.fury.io/rb/zipline)
 
 A gem to stream dynamically generated zip files from a rails application. Unlike other solutions that generate zips for user download, zipline does not wait for the entire zip file to be created (or even for the entire input file in the cloud to be downloaded) before it begins sending the zip file to the user. It does this by never seeking backwards during zip creation, and streaming the zip file over http as it is constructed. The advantages of this are:
 
@@ -28,37 +30,75 @@ supported with [shrine](https://github.com/janko-m/shrine).
 
 You'll need to be using puma or some other server that supports streaming output.
 
-    class MyController < ApplicationController
-      # enable streaming responses
-      include ActionController::Streaming
-      # enable zipline
-      include Zipline
-      
-      def index
-        users = User.all
-        # you can replace user.avatar with any stream or any object that
-        # responds to :url
-        files =  users.map{ |user| [user.avatar, "#{user.username}.png"] }
-        zipline( files, 'avatars.zip')
-      end
-    end
+```Ruby
+class MyController < ApplicationController
+  # enable zipline
+  include Zipline
+
+  def index
+    users = User.all
+    # you can replace user.avatar with any stream or any object that
+    # responds to :url, :path or :file.
+    # :modification_time is an optional third argument you can use.
+    files = users.map{ |user| [user.avatar, "#{user.username}.png", modification_time: 1.day.ago] }
+    zipline(files, 'avatars.zip')
+  end
+end
+```
+
+### ActiveStorage
+
+```Ruby
+users = User.all
+files = users.map{ |user| [user.avatar, user.avatar.filename] }
+zipline(files, 'avatars.zip')
+```
+
+### Carrierwave
+
+```Ruby
+users = User.all
+files = users.map{ |user| [user.avatar, user.avatar_identifier] }
+zipline(files, 'avatars.zip')
+```
+
+### Paperclip ([deprecated](https://thoughtbot.com/blog/closing-the-trombone))
+
+```Ruby
+users = User.all
+files = users.map{ |user| [user.avatar, user.avatar_file_name] }
+zipline(files, 'avatars.zip')
+```
+
+### Url
+
+If you know the URL of the remote file you want to include, you can just pass in the
+URL directly in place of the attachment object.
+```Ruby
+avatars = [
+  ['http://www.example.com/user1.png', 'user1.png']
+  ['http://www.example.com/user2.png', 'user2.png']
+  ['http://www.example.com/user3.png', 'user3.png']
+]
+zipline(avatars, 'avatars.zip')
+```
+
+### Directories
 
 For directories, just give the files names like "directory/file".
 
-To stream files from a remote URL, use open-uri with a [lazy enumerator](http://ruby-doc.org/core-2.0.0/Enumerator/Lazy.html):
-
-    require 'open-uri'
-    avatars = [
-      # remote_url                          zip_path
-      [ 'http://www.example.com/user1.png', 'avatars/user1.png' ]
-      [ 'http://www.example.com/user2.png', 'avatars/user2.png' ]
-      [ 'http://www.example.com/user3.png', 'avatars/user3.png' ]
-    ]
-    file_mappings = avatars
-      .lazy  # Lazy allows us to begin sending the download immediately instead of waiting to download everything
-      .map { |url, path| [open(url), path] }
-    zipline(file_mappings, 'avatars.zip')
-    
+
+```Ruby
+avatars = [
+  # remote_url                          zip_path             zip_tricks_options
+  [ 'http://www.example.com/user1.png', 'avatars/user1.png', modification_time: Time.now.utc ]
+  [ 'http://www.example.com/user2.png', 'avatars/user2.png', modification_time: 1.day.ago ]
+  [ 'http://www.example.com/user3.png', 'avatars/user3.png' ]
+]
+
+zipline(avatars, 'avatars.zip')
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -1,4 +1,12 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
-require "rake/rspec"
-task :default => :spec
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task default: :spec
+rescue LoadError
+  # no rspec available
+end

data/lib/zipline.rb

@@ -1,14 +1,13 @@
 require "zipline/version"
-require 'curb'
 require 'zip_tricks'
 require "zipline/zip_generator"
 
 # class MyController < ApplicationController
 #   include Zipline
 #   def index
-#     users= User.all
-#     files =  users.map{ |user| [user.avatar, "#{user.username}.png"] }
-#     zipline( files, 'avatars.zip')
+#     users = User.all
+#     files = users.map{ |user| [user.avatar, "#{user.username}.png", modification_time: 1.day.ago] }
+#     zipline(files, 'avatars.zip')
 #   end
 # end
 module Zipline

data/lib/zipline/version.rb

@@ -1,3 +1,3 @@
 module Zipline
-  VERSION = "1.0.2"
+  VERSION = "1.3.0"
 end

data/lib/zipline/zip_generator.rb

@@ -14,14 +14,32 @@ module Zipline
 
     def each(&block)
       fake_io_writer = ZipTricks::BlockWrite.new(&block)
-      ZipTricks::Streamer.open(fake_io_writer) do |streamer|
-        @files.each {|file, name| handle_file(streamer, file, name) }
+      # ZipTricks outputs lots of strings in rapid succession, and with
+      # servers it can be beneficial to avoid doing too many tiny writes so that
+      # the number of syscalls is minimized. See https://github.com/WeTransfer/zip_tricks/issues/78
+      # There is a built-in facility for this in ZipTricks which can be used to implement
+      # some cheap buffering here (it exists both in version 4 and version 5). The buffer is really
+      # tiny and roughly equal to the medium Linux socket buffer size (16 KB). Although output
+      # will be not so immediate with this buffering the overall performance will be better,
+      # especially with multiple clients being serviced at the same time.
+      # Note that the WriteBuffer writes the same, retained String object - but the contents
+      # of that object changes between calls. This should work fine with servers where the
+      # contents of the string gets written to a socket immediately before the execution inside
+      # the WriteBuffer resumes), but if the strings get retained somewhere - like in an Array -
+      # this might pose a problem. Unlikely that it will be an issue here though.
+      write_buffer_size = 16 * 1024
+      write_buffer = ZipTricks::WriteBuffer.new(fake_io_writer, write_buffer_size)
+      ZipTricks::Streamer.open(write_buffer) do |streamer|
+        @files.each do |file, name, options = {}|
+          handle_file(streamer, file, name.to_s, options)
+        end
       end
+      write_buffer.flush! # for any remaining writes
     end
 
-    def handle_file(streamer, file, name)
+    def handle_file(streamer, file, name, options)
       file = normalize(file)
-      write_file(streamer, file, name)
+      write_file(streamer, file, name, options)
     end
 
     # This extracts either a url or a local file from the provided file.
@@ -45,32 +63,37 @@ module Zipline
       elsif is_io?(file)
         {file: file}
       elsif defined?(ActiveStorage::Blob) && file.is_a?(ActiveStorage::Blob)
-        {url: file.service_url}
+        {blob: file}
+      elsif is_active_storage_attachment?(file) || is_active_storage_one?(file)
+        {blob: file.blob}
       elsif file.respond_to? :url
         {url: file.url}
       elsif file.respond_to? :path
         {file: File.open(file.path)}
       elsif file.respond_to? :file
         {file: File.open(file.file)}
+      elsif is_url?(file)
+        {url: file}
       else
         raise(ArgumentError, 'Bad File/Stream')
       end
     end
 
-    def write_file(streamer, file, name)
-      streamer.write_deflated_file(name) do |writer_for_file|
+    def write_file(streamer, file, name, options)
+      streamer.write_deflated_file(name, options) do |writer_for_file|
         if file[:url]
-          the_remote_url = file[:url]
-          c = Curl::Easy.new(the_remote_url) do |curl|
-            curl.on_body do |data|
-              writer_for_file << data
-              data.bytesize
+          the_remote_uri = URI(file[:url])
+
+          Net::HTTP.get_response(the_remote_uri) do |response|
+            response.read_body do |chunk|
+              writer_for_file << chunk
             end
           end
-          c.perform
         elsif file[:file]
           IO.copy_stream(file[:file], writer_for_file)
           file[:file].close
+        elsif file[:blob]
+          file[:blob].download { |chunk| writer_for_file << chunk }
         else
           raise(ArgumentError, 'Bad File/Stream')
         end
@@ -80,5 +103,20 @@ module Zipline
     def is_io?(io_ish)
       io_ish.respond_to? :read
     end
+
+    private
+
+    def is_active_storage_attachment?(file)
+      defined?(ActiveStorage::Attachment) && file.is_a?(ActiveStorage::Attachment)
+    end
+
+    def is_active_storage_one?(file)
+      defined?(ActiveStorage::Attached::One) && file.is_a?(ActiveStorage::Attached::One)
+    end
+
+    def is_url?(url)
+      url = URI.parse(url) rescue false
+      url.kind_of?(URI::HTTP) || url.kind_of?(URI::HTTPS)
+    end
   end
 end

data/spec/lib/zipline/zip_generator_spec.rb

@@ -1,6 +1,23 @@
 require 'spec_helper'
 require 'tempfile'
 
+module ActiveStorage
+  class Attached
+    class One < Attached
+    end
+  end
+  class Attachment; end
+  class Blob; end
+  class Filename
+    def initialize(name)
+      @name = name
+    end
+    def to_s
+      @name
+    end
+  end
+end
+
 describe Zipline::ZipGenerator do
   before { Fog.mock! }
   let(:file_attributes){ {
@@ -84,29 +101,68 @@ describe Zipline::ZipGenerator do
         end
       end
     end
-    context "ActiveStorage::Blob" do
-      module ActiveStorage
-        class Filename; end
-        class Blob; end
-      end
-
-      let(:tempfile){ Tempfile.new('t').read }
-      let(:filename) do
-        fn = ActiveStorage::Filename.new()
-        allow(fn).to receive(:to_s).and_return('spec/fakefile.txt')
-        fn
-      end
-      let(:file) do
-        f = ActiveStorage::Blob.new()
-        allow(f).to receive(:filename).and_return(filename)
-        allow(f).to receive(:service_url).and_return('fakeurl')
-        f
-      end
-      it "creates a File" do
-        allow_any_instance_of(Object).to receive(:defined?).and_return(true)
-        normalized = generator.normalize(file)
-        expect(normalized.keys).to include(:url)
-        expect(normalized[:url]).to eq('fakeurl')
+    context "ActiveStorage" do
+      context "Attached::One" do
+        it "get blob" do
+          attached = create_attached_one
+          allow_any_instance_of(Object).to receive(:defined?).and_return(true)
+
+          normalized = generator.normalize(attached)
+
+          expect(normalized.keys).to include(:blob)
+          expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
+        end
+      end
+
+      context "Attachment" do
+        it "get blob" do
+          attachment = create_attachment
+          allow_any_instance_of(Object).to receive(:defined?).and_return(true)
+
+          normalized = generator.normalize(attachment)
+
+          expect(normalized.keys).to include(:blob)
+          expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
+        end
+      end
+
+      context "Blob" do
+        it "get blob" do
+          blob = create_blob
+          allow_any_instance_of(Object).to receive(:defined?).and_return(true)
+
+          normalized = generator.normalize(blob)
+
+          expect(normalized.keys).to include(:blob)
+          expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
+        end
+      end
+
+      def create_attached_one
+        attached = ActiveStorage::Attached::One.new
+        blob = create_blob
+        allow(attached).to receive(:blob).and_return(blob)
+        attached
+      end
+
+      def create_attachment
+        attachment = ActiveStorage::Attachment.new
+        blob = create_blob
+        allow(attachment).to receive(:blob).and_return(blob)
+        attachment
+      end
+
+      def create_blob
+        blob = ActiveStorage::Blob.new
+        allow(blob).to receive(:service_url).and_return('fakeurl')
+        filename = create_filename
+        allow(blob).to receive(:filename).and_return(filename)
+        blob
+      end
+
+      def create_filename
+        # Rails wraps Blob#filname in this class since Rails 5.2
+        ActiveStorage::Filename.new('test')
       end
     end
     context "Fog" do
@@ -130,4 +186,65 @@ describe Zipline::ZipGenerator do
     end
   end
 
+  describe '.write_file' do
+    let(:file) { StringIO.new('passthrough') }
+
+    context 'when passing an ActiveStorage::Filename object as filename' do
+      let(:filename) { ActiveStorage::Filename.new('test') }
+
+      let(:generator) do
+        Zipline::ZipGenerator.new([[file, filename]])
+      end
+
+      it 'passes a string as filename to ZipTricks' do
+        allow(file).to receive(:url).and_return('fakeurl')
+        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
+          .with('test', {})
+        generator.each { |_| 'Test' }
+      end
+    end
+  end
+
+  describe 'passing an options hash' do
+    let(:file) { StringIO.new('passthrough') }
+
+    context 'with optional arguments' do
+      let(:mtime) { 1.day.ago }
+      let(:generator) do
+        Zipline::ZipGenerator.new([[file, 'test', modification_time: mtime]])
+      end
+
+      it 'passes the options hash through handle_file' do
+        expect(generator).to receive(:handle_file)
+          .with(anything, anything, anything, modification_time: mtime)
+        generator.each { |_| 'Test' }
+      end
+
+      it 'passes the options hash to ZipTricks' do
+        allow(file).to receive(:url).and_return('fakeurl')
+        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
+          .with(anything, modification_time: mtime)
+        generator.each { |_| 'Test' }
+      end
+    end
+
+    context 'without optional arguments' do
+      let(:generator) do
+        Zipline::ZipGenerator.new([[file, 'test']])
+      end
+
+      it 'passes the options hash through handle_file' do
+        expect(generator).to receive(:handle_file)
+          .with(anything, anything, anything, {})
+        generator.each { |_| 'Test' }
+      end
+
+      it 'passes the options hash to ZipTricks' do
+        allow(file).to receive(:url).and_return('fakeurl')
+        expect_any_instance_of(ZipTricks::Streamer).to receive(:write_deflated_file)
+          .with(anything, {})
+        generator.each { |_| 'Test' }
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -2,9 +2,7 @@ require 'rspec'
 require 'active_support'
 require 'active_support/core_ext'
 require 'zipline'
-require 'aws-sdk'
 require 'paperclip'
-require 'fog'
 require 'fog-aws'
 require 'carrierwave'
 
@@ -21,8 +19,6 @@ CarrierWave.configure do |config|
 
 end
 
-
-
 RSpec.configure do |config|
   config.color = true
   config.order = :random

data/zipline.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |gem|
   gem.name          = "zipline"
   gem.require_paths = ["lib"]
   gem.version       = Zipline::VERSION
+  gem.licenses      = ['MIT']
 
-  gem.add_dependency 'zip_tricks', ['>= 4.2.1', '<= 5.0.0']
-  gem.add_dependency 'rails', ['>= 3.2.1', '< 5.3']
-  gem.add_dependency 'curb', ['>= 0.8.0', '< 0.10']
+  gem.add_dependency 'actionpack', ['>= 3.2.1', '< 7.0']
+  gem.add_dependency 'zip_tricks', ['>= 4.2.1', '< 6.0']
 end

metadata

@@ -1,37 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: zipline
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Ram Dobson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: zip_tricks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.1
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 5.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.1
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 5.0.0
-- !ruby/object:Gem::Dependency
-  name: rails
+  name: actionpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +19,7 @@ dependencies:
         version: 3.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,27 +29,27 @@ dependencies:
         version: 3.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
-  name: curb
+  name: zip_tricks
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 4.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 4.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '6.0'
 description: a module for streaming dynamically generated zip files
 email:
 - ram.dobson@solsystemscompany.com
@@ -90,9 +70,10 @@ files:
 - spec/spec_helper.rb
 - zipline.gemspec
 homepage: http://github.com/fringd/zipline
-licenses: []
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -107,9 +88,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: stream zip files from rails
 test_files: