zipline 1.6.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/.standard.yml +4 -0
- data/Gemfile +1 -1
- data/README.md +3 -3
- data/Rakefile +4 -8
- data/lib/zipline/version.rb +1 -1
- data/lib/zipline/{zip_generator.rb → zip_handler.rb} +21 -43
- data/lib/zipline.rb +14 -41
- data/spec/lib/zipline/{zip_generator_spec.rb → zip_handler_spec.rb} +61 -51
- data/spec/lib/zipline/zipline_spec.rb +25 -16
- data/spec/spec_helper.rb +14 -14
- data/zipline.gemspec +22 -23
- metadata +34 -22
- data/lib/zipline/chunked_body.rb +0 -31
- data/lib/zipline/tempfile_body.rb +0 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 87ea082c8b6e5b78101eed8383d5851510765a948f5fa836607b14170db32c37
|
4
|
+
data.tar.gz: b7508ab7190eba8dd233d923bc41e3adab5d968f3eee743445a2d7b2f427c26e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 643f217f94d513a1f517bd421117f7ec51b8df45ac6afe23e1ddbb0f4964dedd46f2b895fe0551ce0e3142f911dc5f07c92254a877cd4960506ade02e955a322
|
7
|
+
data.tar.gz: b6a8960a8a8378b46a13b212f4e1713f6b29f08c60c65c48135b765a48e93892063faf2ba983f3a81bb81df9e7d92f875d0341b571d67afb65ed5c8977ec4023
|
data/.github/workflows/ci.yml
CHANGED
data/.standard.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -7,7 +7,7 @@ A gem to stream dynamically generated zip files from a rails application. Unlike
|
|
7
7
|
- Removes need for large disk space or memory allocation to generate zips, even huge zips. So it works on Heroku.
|
8
8
|
- The user begins downloading immediately, which decreaceses latency, download time, and timeouts on Heroku.
|
9
9
|
|
10
|
-
Zipline now depends on [
|
10
|
+
Zipline now depends on [zip_kit](https://github.com/julik/zip_kit), and you might want to just use that directly if you have more advanced use cases.
|
11
11
|
|
12
12
|
## Installation
|
13
13
|
|
@@ -43,7 +43,7 @@ class MyController < ApplicationController
|
|
43
43
|
files = users.map{ |user| [user.avatar, "#{user.username}.png", modification_time: 1.day.ago] }
|
44
44
|
|
45
45
|
# we can force duplicate file names to be renamed, or raise an error
|
46
|
-
# we can also pass in our own writer if required to conform with the
|
46
|
+
# we can also pass in our own writer if required to conform with the delegated [ZipKit::Streamer object](https://github.com/julik/zip_kit/blob/main/lib/zip_kit/streamer.rb#L147) object.
|
47
47
|
zipline(files, 'avatars.zip', auto_rename_duplicate_filenames: true)
|
48
48
|
end
|
49
49
|
end
|
@@ -93,7 +93,7 @@ For directories, just give the files names like "directory/file".
|
|
93
93
|
|
94
94
|
```Ruby
|
95
95
|
avatars = [
|
96
|
-
# remote_url zip_path
|
96
|
+
# remote_url zip_path write_file options for Streamer
|
97
97
|
[ 'http://www.example.com/user1.png', 'avatars/user1.png', modification_time: Time.now.utc ]
|
98
98
|
[ 'http://www.example.com/user2.png', 'avatars/user2.png', modification_time: 1.day.ago ]
|
99
99
|
[ 'http://www.example.com/user3.png', 'avatars/user3.png' ]
|
data/Rakefile
CHANGED
@@ -1,12 +1,8 @@
|
|
1
1
|
#!/usr/bin/env rake
|
2
2
|
require "bundler/gem_tasks"
|
3
|
+
require "standard/rake"
|
4
|
+
require "rspec/core/rake_task"
|
3
5
|
|
4
|
-
|
5
|
-
require 'rspec/core/rake_task'
|
6
|
+
RSpec::Core::RakeTask.new(:spec)
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
task default: :spec
|
10
|
-
rescue LoadError
|
11
|
-
# no rspec available
|
12
|
-
end
|
8
|
+
task default: [:spec, :standard]
|
data/lib/zipline/version.rb
CHANGED
@@ -1,39 +1,23 @@
|
|
1
|
-
# this class acts as a streaming body for rails
|
2
|
-
# initialize it with an array of the files you want to zip
|
3
1
|
module Zipline
|
4
|
-
class
|
2
|
+
class ZipHandler
|
5
3
|
# takes an array of pairs [[uploader, filename], ... ]
|
6
|
-
def initialize(
|
7
|
-
|
8
|
-
@
|
9
|
-
files.each do |file, name, options = {}|
|
10
|
-
handle_file(streamer, file, name.to_s, options)
|
11
|
-
end
|
12
|
-
end
|
4
|
+
def initialize(streamer, logger)
|
5
|
+
@streamer = streamer
|
6
|
+
@logger = logger
|
13
7
|
end
|
14
8
|
|
15
|
-
def
|
16
|
-
|
17
|
-
|
9
|
+
def handle_file(file, name, options)
|
10
|
+
normalized_file = normalize(file)
|
11
|
+
write_file(normalized_file, name, options)
|
18
12
|
rescue => e
|
19
13
|
# Since most APM packages do not trace errors occurring within streaming
|
20
14
|
# Rack bodies, it can be helpful to print the error to the Rails log at least
|
21
15
|
error_message = "zipline: an exception (#{e.inspect}) was raised when serving the ZIP body."
|
22
|
-
error_message += " The error occurred when handling #{
|
23
|
-
logger
|
16
|
+
error_message += " The error occurred when handling file #{name.inspect}"
|
17
|
+
@logger&.error(error_message)
|
24
18
|
raise
|
25
19
|
end
|
26
20
|
|
27
|
-
def handle_file(streamer, file, name, options)
|
28
|
-
file = normalize(file)
|
29
|
-
|
30
|
-
# Store the filename so that a sensible error message can be displayed in the log
|
31
|
-
# if writing this particular file fails
|
32
|
-
@filename = name
|
33
|
-
write_file(streamer, file, name, options)
|
34
|
-
@filename = nil
|
35
|
-
end
|
36
|
-
|
37
21
|
# This extracts either a url or a local file from the provided file.
|
38
22
|
# Currently support carrierwave and paperclip local and remote storage.
|
39
23
|
# returns a hash of the form {url: aUrl} or {file: anIoObject}
|
@@ -67,12 +51,12 @@ module Zipline
|
|
67
51
|
elsif is_url?(file)
|
68
52
|
{url: file}
|
69
53
|
else
|
70
|
-
raise(ArgumentError,
|
54
|
+
raise(ArgumentError, "Bad File/Stream")
|
71
55
|
end
|
72
56
|
end
|
73
57
|
|
74
|
-
def write_file(
|
75
|
-
streamer.
|
58
|
+
def write_file(file, name, options)
|
59
|
+
@streamer.write_file(name, **options.slice(:modification_time)) do |writer_for_file|
|
76
60
|
if file[:url]
|
77
61
|
the_remote_uri = URI(file[:url])
|
78
62
|
|
@@ -87,25 +71,15 @@ module Zipline
|
|
87
71
|
elsif file[:blob]
|
88
72
|
file[:blob].download { |chunk| writer_for_file << chunk }
|
89
73
|
else
|
90
|
-
raise(ArgumentError,
|
74
|
+
raise(ArgumentError, "Bad File/Stream")
|
91
75
|
end
|
92
76
|
end
|
93
77
|
end
|
94
78
|
|
95
|
-
def is_io?(io_ish)
|
96
|
-
io_ish.respond_to? :read
|
97
|
-
end
|
98
|
-
|
99
79
|
private
|
100
80
|
|
101
|
-
def
|
102
|
-
|
103
|
-
# elsewhere - or the logger might not be configured correctly
|
104
|
-
if defined?(Rails.logger) && Rails.logger
|
105
|
-
Rails.logger
|
106
|
-
else
|
107
|
-
Logger.new($stderr)
|
108
|
-
end
|
81
|
+
def is_io?(io_ish)
|
82
|
+
io_ish.respond_to? :read
|
109
83
|
end
|
110
84
|
|
111
85
|
def is_active_storage_attachment?(file)
|
@@ -117,8 +91,12 @@ module Zipline
|
|
117
91
|
end
|
118
92
|
|
119
93
|
def is_url?(url)
|
120
|
-
url =
|
121
|
-
|
94
|
+
url = begin
|
95
|
+
URI.parse(url)
|
96
|
+
rescue
|
97
|
+
false
|
98
|
+
end
|
99
|
+
url.is_a?(URI::HTTP) || url.is_a?(URI::HTTPS)
|
122
100
|
end
|
123
101
|
end
|
124
102
|
end
|
data/lib/zipline.rb
CHANGED
@@ -1,9 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require 'zipline/chunked_body'
|
6
|
-
require 'zipline/tempfile_body'
|
1
|
+
require "content_disposition"
|
2
|
+
require "zip_kit"
|
3
|
+
require "zipline/version"
|
4
|
+
require "zipline/zip_handler"
|
7
5
|
|
8
6
|
# class MyController < ApplicationController
|
9
7
|
# include Zipline
|
@@ -14,42 +12,17 @@ require 'zipline/tempfile_body'
|
|
14
12
|
# end
|
15
13
|
# end
|
16
14
|
module Zipline
|
17
|
-
def
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
response.sending_file = true
|
22
|
-
response.cache_control[:public] ||= false
|
23
|
-
|
24
|
-
# Disables Rack::ETag if it is enabled (prevent buffering)
|
25
|
-
# see https://github.com/rack/rack/issues/1619#issuecomment-606315714
|
26
|
-
self.response.headers['Last-Modified'] = Time.now.httpdate
|
27
|
-
|
28
|
-
if request.get_header("HTTP_VERSION") == "HTTP/1.0"
|
29
|
-
# If HTTP/1.0 is used it is not possible to stream, and if that happens it usually will be
|
30
|
-
# unclear why buffering is happening. Some info in the log is the least one can do.
|
31
|
-
logger.warn { "The downstream HTTP proxy/LB insists on HTTP/1.0 protocol, ZIP response will be buffered." } if logger
|
32
|
-
|
33
|
-
# If we use Rack::ContentLength it would run through our ZIP block twice - once to calculate the content length
|
34
|
-
# of the response, and once - to serve. We can trade performance for disk space and buffer the response into a Tempfile
|
35
|
-
# since we are already buffering.
|
36
|
-
tempfile_body = TempfileBody.new(request.env, zip_generator)
|
37
|
-
headers["Content-Length"] = tempfile_body.size.to_s
|
38
|
-
headers["X-Zipline-Output"] = "buffered"
|
39
|
-
self.response_body = tempfile_body
|
40
|
-
else
|
41
|
-
# Disable buffering for both nginx and Google Load Balancer, see
|
42
|
-
# https://cloud.google.com/appengine/docs/flexible/how-requests-are-handled?tab=python#x-accel-buffering
|
43
|
-
response.headers["X-Accel-Buffering"] = "no"
|
44
|
-
|
45
|
-
# Make sure Rack::ContentLength does not try to compute a content length,
|
46
|
-
# and remove the one already set
|
47
|
-
headers.delete("Content-Length")
|
15
|
+
def self.included(into_controller)
|
16
|
+
into_controller.include(ZipKit::RailsStreaming)
|
17
|
+
super
|
18
|
+
end
|
48
19
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
20
|
+
def zipline(files, zipname = "zipline.zip", **kwargs_for_zip_kit_stream)
|
21
|
+
zip_kit_stream(filename: zipname, **kwargs_for_zip_kit_stream) do |zip_kit_streamer|
|
22
|
+
handler = Zipline::ZipHandler.new(zip_kit_streamer, logger)
|
23
|
+
files.each do |file, name, options = {}|
|
24
|
+
handler.handle_file(file, name.to_s, options)
|
25
|
+
end
|
53
26
|
end
|
54
27
|
end
|
55
28
|
end
|
@@ -1,58 +1,68 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "spec_helper"
|
2
|
+
require "tempfile"
|
3
3
|
|
4
4
|
module ActiveStorage
|
5
5
|
class Attached
|
6
6
|
class One < Attached
|
7
7
|
end
|
8
8
|
end
|
9
|
+
|
9
10
|
class Attachment; end
|
11
|
+
|
10
12
|
class Blob; end
|
13
|
+
|
11
14
|
class Filename
|
12
15
|
def initialize(name)
|
13
16
|
@name = name
|
14
17
|
end
|
18
|
+
|
15
19
|
def to_s
|
16
20
|
@name
|
17
21
|
end
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
21
|
-
describe Zipline::
|
25
|
+
describe Zipline::ZipHandler do
|
22
26
|
before { Fog.mock! }
|
23
|
-
let(:file_attributes)
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
27
|
+
let(:file_attributes) {
|
28
|
+
{
|
29
|
+
key: "fog_file_tests",
|
30
|
+
body: "some body",
|
31
|
+
public: true
|
32
|
+
}
|
33
|
+
}
|
34
|
+
let(:directory_attributes) {
|
35
|
+
{
|
36
|
+
key: "fog_directory"
|
37
|
+
}
|
38
|
+
}
|
39
|
+
let(:storage_attributes) {
|
40
|
+
{
|
41
|
+
aws_access_key_id: "fake_access_key_id",
|
42
|
+
aws_secret_access_key: "fake_secret_access_key",
|
43
|
+
provider: "AWS"
|
44
|
+
}
|
45
|
+
}
|
46
|
+
let(:storage) { Fog::Storage.new(storage_attributes) }
|
47
|
+
let(:directory) { storage.directories.create(directory_attributes) }
|
48
|
+
let(:file) { directory.files.create(file_attributes) }
|
49
|
+
|
50
|
+
describe ".normalize" do
|
51
|
+
let(:handler) { Zipline::ZipHandler.new(_streamer = double, _logger = nil) }
|
42
52
|
context "CarrierWave" do
|
43
53
|
context "Remote" do
|
44
|
-
let(:file){ CarrierWave::Storage::Fog::File.new(nil,nil,nil) }
|
54
|
+
let(:file) { CarrierWave::Storage::Fog::File.new(nil, nil, nil) }
|
45
55
|
it "extracts the url" do
|
46
|
-
allow(file).to receive(:url).and_return(
|
56
|
+
allow(file).to receive(:url).and_return("fakeurl")
|
47
57
|
expect(File).not_to receive(:open)
|
48
|
-
expect(
|
58
|
+
expect(handler.normalize(file)).to eq({url: "fakeurl"})
|
49
59
|
end
|
50
60
|
end
|
51
61
|
context "Local" do
|
52
|
-
let(:file){ CarrierWave::SanitizedFile.new(Tempfile.new(
|
62
|
+
let(:file) { CarrierWave::SanitizedFile.new(Tempfile.new("t")) }
|
53
63
|
it "creates a File" do
|
54
|
-
allow(file).to receive(:path).and_return(
|
55
|
-
normalized =
|
64
|
+
allow(file).to receive(:path).and_return("spec/fakefile.txt")
|
65
|
+
normalized = handler.normalize(file)
|
56
66
|
expect(normalized.keys).to include(:file)
|
57
67
|
expect(normalized[:file]).to be_a File
|
58
68
|
end
|
@@ -61,21 +71,21 @@ describe Zipline::ZipGenerator do
|
|
61
71
|
let(:uploader) { Class.new(CarrierWave::Uploader::Base).new }
|
62
72
|
|
63
73
|
context "Remote" do
|
64
|
-
let(:file){ CarrierWave::Storage::Fog::File.new(nil,nil,nil) }
|
74
|
+
let(:file) { CarrierWave::Storage::Fog::File.new(nil, nil, nil) }
|
65
75
|
it "extracts the url" do
|
66
76
|
allow(uploader).to receive(:file).and_return(file)
|
67
|
-
allow(file).to receive(:url).and_return(
|
77
|
+
allow(file).to receive(:url).and_return("fakeurl")
|
68
78
|
expect(File).not_to receive(:open)
|
69
|
-
expect(
|
79
|
+
expect(handler.normalize(uploader)).to eq({url: "fakeurl"})
|
70
80
|
end
|
71
81
|
end
|
72
82
|
|
73
83
|
context "Local" do
|
74
|
-
let(:file){ CarrierWave::SanitizedFile.new(Tempfile.new(
|
84
|
+
let(:file) { CarrierWave::SanitizedFile.new(Tempfile.new("t")) }
|
75
85
|
it "creates a File" do
|
76
86
|
allow(uploader).to receive(:file).and_return(file)
|
77
|
-
allow(file).to receive(:path).and_return(
|
78
|
-
normalized =
|
87
|
+
allow(file).to receive(:path).and_return("spec/fakefile.txt")
|
88
|
+
normalized = handler.normalize(uploader)
|
79
89
|
expect(normalized.keys).to include(:file)
|
80
90
|
expect(normalized[:file]).to be_a File
|
81
91
|
end
|
@@ -84,20 +94,20 @@ describe Zipline::ZipGenerator do
|
|
84
94
|
end
|
85
95
|
context "Paperclip" do
|
86
96
|
context "Local" do
|
87
|
-
let(:file){ Paperclip::Attachment.new(:name, :instance) }
|
97
|
+
let(:file) { Paperclip::Attachment.new(:name, :instance) }
|
88
98
|
it "creates a File" do
|
89
|
-
allow(file).to receive(:path).and_return(
|
90
|
-
normalized =
|
99
|
+
allow(file).to receive(:path).and_return("spec/fakefile.txt")
|
100
|
+
normalized = handler.normalize(file)
|
91
101
|
expect(normalized.keys).to include(:file)
|
92
102
|
expect(normalized[:file]).to be_a File
|
93
103
|
end
|
94
104
|
end
|
95
105
|
context "Remote" do
|
96
|
-
let(:file){ Paperclip::Attachment.new(:name, :instance, storage: :s3) }
|
106
|
+
let(:file) { Paperclip::Attachment.new(:name, :instance, storage: :s3) }
|
97
107
|
it "creates a URL" do
|
98
|
-
allow(file).to receive(:expiring_url).and_return(
|
108
|
+
allow(file).to receive(:expiring_url).and_return("fakeurl")
|
99
109
|
expect(File).to_not receive(:open)
|
100
|
-
expect(
|
110
|
+
expect(handler.normalize(file)).to include(url: "fakeurl")
|
101
111
|
end
|
102
112
|
end
|
103
113
|
end
|
@@ -107,7 +117,7 @@ describe Zipline::ZipGenerator do
|
|
107
117
|
attached = create_attached_one
|
108
118
|
allow_any_instance_of(Object).to receive(:defined?).and_return(true)
|
109
119
|
|
110
|
-
normalized =
|
120
|
+
normalized = handler.normalize(attached)
|
111
121
|
|
112
122
|
expect(normalized.keys).to include(:blob)
|
113
123
|
expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
|
@@ -119,7 +129,7 @@ describe Zipline::ZipGenerator do
|
|
119
129
|
attachment = create_attachment
|
120
130
|
allow_any_instance_of(Object).to receive(:defined?).and_return(true)
|
121
131
|
|
122
|
-
normalized =
|
132
|
+
normalized = handler.normalize(attachment)
|
123
133
|
|
124
134
|
expect(normalized.keys).to include(:blob)
|
125
135
|
expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
|
@@ -131,7 +141,7 @@ describe Zipline::ZipGenerator do
|
|
131
141
|
blob = create_blob
|
132
142
|
allow_any_instance_of(Object).to receive(:defined?).and_return(true)
|
133
143
|
|
134
|
-
normalized =
|
144
|
+
normalized = handler.normalize(blob)
|
135
145
|
|
136
146
|
expect(normalized.keys).to include(:blob)
|
137
147
|
expect(normalized[:blob]).to be_a(ActiveStorage::Blob)
|
@@ -154,7 +164,7 @@ describe Zipline::ZipGenerator do
|
|
154
164
|
|
155
165
|
def create_blob
|
156
166
|
blob = ActiveStorage::Blob.new
|
157
|
-
allow(blob).to receive(:service_url).and_return(
|
167
|
+
allow(blob).to receive(:service_url).and_return("fakeurl")
|
158
168
|
filename = create_filename
|
159
169
|
allow(blob).to receive(:filename).and_return(filename)
|
160
170
|
blob
|
@@ -162,26 +172,26 @@ describe Zipline::ZipGenerator do
|
|
162
172
|
|
163
173
|
def create_filename
|
164
174
|
# Rails wraps Blob#filname in this class since Rails 5.2
|
165
|
-
ActiveStorage::Filename.new(
|
175
|
+
ActiveStorage::Filename.new("test")
|
166
176
|
end
|
167
177
|
end
|
168
178
|
context "Fog" do
|
169
179
|
it "extracts url" do
|
170
|
-
allow(file).to receive(:url).and_return(
|
180
|
+
allow(file).to receive(:url).and_return("fakeurl")
|
171
181
|
expect(File).not_to receive(:open)
|
172
|
-
expect(
|
182
|
+
expect(handler.normalize(file)).to eq(url: "fakeurl")
|
173
183
|
end
|
174
184
|
end
|
175
185
|
context "IOStream" do
|
176
|
-
let(:file){ StringIO.new(
|
186
|
+
let(:file) { StringIO.new("passthrough") }
|
177
187
|
it "passes through" do
|
178
|
-
expect(
|
188
|
+
expect(handler.normalize(file)).to eq(file: file)
|
179
189
|
end
|
180
190
|
end
|
181
191
|
context "invalid" do
|
182
|
-
let(:file){ Thread.new{} }
|
192
|
+
let(:file) { Thread.new {} }
|
183
193
|
it "raises error" do
|
184
|
-
expect{
|
194
|
+
expect { handler.normalize(file) }.to raise_error(ArgumentError)
|
185
195
|
end
|
186
196
|
end
|
187
197
|
end
|
@@ -1,8 +1,11 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "spec_helper"
|
2
|
+
require "action_controller"
|
3
3
|
|
4
4
|
describe Zipline do
|
5
|
-
before
|
5
|
+
before do
|
6
|
+
Fog.mock!
|
7
|
+
FakeController.logger = nil
|
8
|
+
end
|
6
9
|
|
7
10
|
class FakeController < ActionController::Base
|
8
11
|
include Zipline
|
@@ -11,7 +14,7 @@ describe Zipline do
|
|
11
14
|
[StringIO.new("File content goes here"), "one.txt"],
|
12
15
|
[StringIO.new("Some other content goes here"), "two.txt"]
|
13
16
|
]
|
14
|
-
zipline(files,
|
17
|
+
zipline(files, "myfiles.zip", auto_rename_duplicate_filenames: false)
|
15
18
|
end
|
16
19
|
|
17
20
|
class FailingIO < StringIO
|
@@ -25,11 +28,11 @@ describe Zipline do
|
|
25
28
|
[StringIO.new("File content goes here"), "one.txt"],
|
26
29
|
[FailingIO.new("This will fail half-way"), "two.txt"]
|
27
30
|
]
|
28
|
-
zipline(files,
|
31
|
+
zipline(files, "myfiles.zip", auto_rename_duplicate_filenames: false)
|
29
32
|
end
|
30
33
|
end
|
31
34
|
|
32
|
-
it
|
35
|
+
it "passes keyword parameters to ZipKit::OutputEnumerator" do
|
33
36
|
fake_rack_env = {
|
34
37
|
"HTTP_VERSION" => "HTTP/1.0",
|
35
38
|
"REQUEST_METHOD" => "GET",
|
@@ -37,16 +40,20 @@ describe Zipline do
|
|
37
40
|
"PATH_INFO" => "/download",
|
38
41
|
"QUERY_STRING" => "",
|
39
42
|
"SERVER_NAME" => "host.example",
|
40
|
-
"rack.input" => StringIO.new
|
43
|
+
"rack.input" => StringIO.new
|
41
44
|
}
|
42
|
-
expect(
|
45
|
+
expect(ZipKit::OutputEnumerator).to receive(:new).with(auto_rename_duplicate_filenames: false).and_call_original
|
43
46
|
|
44
47
|
status, headers, body = FakeController.action(:download_zip).call(fake_rack_env)
|
45
48
|
|
46
|
-
expect(
|
49
|
+
expect(status).to eq(200)
|
50
|
+
expect(headers["Content-Disposition"]).to eq("attachment; filename=\"myfiles.zip\"; filename*=UTF-8''myfiles.zip")
|
51
|
+
expect {
|
52
|
+
body.each {}
|
53
|
+
}.not_to raise_error
|
47
54
|
end
|
48
55
|
|
49
|
-
it
|
56
|
+
it "sends the exception raised in the streaming body to the Rails logger" do
|
50
57
|
fake_rack_env = {
|
51
58
|
"HTTP_VERSION" => "HTTP/1.0",
|
52
59
|
"REQUEST_METHOD" => "GET",
|
@@ -54,15 +61,17 @@ describe Zipline do
|
|
54
61
|
"PATH_INFO" => "/download",
|
55
62
|
"QUERY_STRING" => "",
|
56
63
|
"SERVER_NAME" => "host.example",
|
57
|
-
"rack.input" => StringIO.new
|
64
|
+
"rack.input" => StringIO.new
|
58
65
|
}
|
59
|
-
|
60
|
-
fake_logger
|
61
|
-
expect(
|
62
|
-
|
66
|
+
fake_logger = double
|
67
|
+
allow(fake_logger).to receive(:warn)
|
68
|
+
expect(fake_logger).to receive(:error).with(a_string_matching(/when serving the ZIP/))
|
69
|
+
|
70
|
+
FakeController.logger = fake_logger
|
63
71
|
|
64
72
|
expect {
|
65
|
-
FakeController.action(:download_zip_with_error_during_streaming).call(fake_rack_env)
|
73
|
+
_status, _headers, body = FakeController.action(:download_zip_with_error_during_streaming).call(fake_rack_env)
|
74
|
+
body.each {}
|
66
75
|
}.to raise_error(/Something wonky/)
|
67
76
|
end
|
68
77
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,22 +1,22 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require "rspec"
|
2
|
+
require "active_support"
|
3
|
+
require "active_support/core_ext"
|
4
|
+
require "action_dispatch"
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
6
|
+
require "zipline"
|
7
|
+
require "paperclip"
|
8
|
+
require "fog-aws"
|
9
|
+
require "carrierwave"
|
10
10
|
|
11
|
-
Dir["#{File.expand_path(
|
11
|
+
Dir["#{File.expand_path("..", __FILE__)}/support/**/*.rb"].sort.each { |f| require f }
|
12
12
|
|
13
13
|
CarrierWave.configure do |config|
|
14
|
-
config.fog_provider =
|
14
|
+
config.fog_provider = "fog/aws"
|
15
15
|
config.fog_credentials = {
|
16
|
-
provider:
|
17
|
-
aws_access_key_id:
|
18
|
-
aws_secret_access_key:
|
19
|
-
region:
|
16
|
+
provider: "AWS",
|
17
|
+
aws_access_key_id: "dummy",
|
18
|
+
aws_secret_access_key: "data",
|
19
|
+
region: "us-west-2"
|
20
20
|
}
|
21
21
|
end
|
22
22
|
|
data/zipline.gemspec
CHANGED
@@ -1,34 +1,33 @@
|
|
1
|
-
|
2
|
-
require File.expand_path('../lib/zipline/version', __FILE__)
|
1
|
+
require File.expand_path("../lib/zipline/version", __FILE__)
|
3
2
|
|
4
3
|
Gem::Specification.new do |gem|
|
5
|
-
gem.authors
|
6
|
-
gem.email
|
7
|
-
gem.description
|
8
|
-
gem.summary
|
9
|
-
gem.homepage
|
4
|
+
gem.authors = ["Ram Dobson"]
|
5
|
+
gem.email = ["ram.dobson@solsystemscompany.com"]
|
6
|
+
gem.description = "a module for streaming dynamically generated zip files"
|
7
|
+
gem.summary = "stream zip files from rails"
|
8
|
+
gem.homepage = "http://github.com/fringd/zipline"
|
10
9
|
|
11
|
-
gem.files
|
12
|
-
gem.executables
|
13
|
-
gem.
|
14
|
-
gem.name = "zipline"
|
10
|
+
gem.files = `git ls-files`.split($\) - %w[.gitignore]
|
11
|
+
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
12
|
+
gem.name = "zipline"
|
15
13
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version
|
17
|
-
gem.licenses
|
14
|
+
gem.version = Zipline::VERSION
|
15
|
+
gem.licenses = ["MIT"]
|
18
16
|
|
19
17
|
gem.required_ruby_version = ">= 2.7"
|
20
18
|
|
21
|
-
gem.add_dependency
|
22
|
-
gem.add_dependency
|
23
|
-
gem.add_dependency
|
19
|
+
gem.add_dependency "actionpack", [">= 6.0", "< 8.1"]
|
20
|
+
gem.add_dependency "content_disposition", "~> 1.0"
|
21
|
+
gem.add_dependency "zip_kit", ["~> 6", ">= 6.2.0", "< 7"]
|
24
22
|
|
25
|
-
gem.add_development_dependency
|
26
|
-
gem.add_development_dependency
|
27
|
-
gem.add_development_dependency
|
28
|
-
gem.add_development_dependency
|
29
|
-
gem.add_development_dependency
|
30
|
-
gem.add_development_dependency
|
23
|
+
gem.add_development_dependency "rspec", "~> 3"
|
24
|
+
gem.add_development_dependency "fog-aws"
|
25
|
+
gem.add_development_dependency "aws-sdk-s3"
|
26
|
+
gem.add_development_dependency "carrierwave"
|
27
|
+
gem.add_development_dependency "paperclip"
|
28
|
+
gem.add_development_dependency "rake"
|
29
|
+
gem.add_development_dependency "standard", "1.28.5" # Very specific version of standard for 2.6 with _known_ settings
|
31
30
|
|
32
31
|
# https://github.com/rspec/rspec-mocks/issues/1457
|
33
|
-
gem.add_development_dependency
|
32
|
+
gem.add_development_dependency "rspec-mocks", "~> 3.12"
|
34
33
|
end
|
metadata
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zipline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ram Dobson
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: actionpack
|
@@ -19,7 +18,7 @@ dependencies:
|
|
19
18
|
version: '6.0'
|
20
19
|
- - "<"
|
21
20
|
- !ruby/object:Gem::Version
|
22
|
-
version: '8.
|
21
|
+
version: '8.1'
|
23
22
|
type: :runtime
|
24
23
|
prerelease: false
|
25
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -29,7 +28,7 @@ dependencies:
|
|
29
28
|
version: '6.0'
|
30
29
|
- - "<"
|
31
30
|
- !ruby/object:Gem::Version
|
32
|
-
version: '8.
|
31
|
+
version: '8.1'
|
33
32
|
- !ruby/object:Gem::Dependency
|
34
33
|
name: content_disposition
|
35
34
|
requirement: !ruby/object:Gem::Requirement
|
@@ -45,25 +44,31 @@ dependencies:
|
|
45
44
|
- !ruby/object:Gem::Version
|
46
45
|
version: '1.0'
|
47
46
|
- !ruby/object:Gem::Dependency
|
48
|
-
name:
|
47
|
+
name: zip_kit
|
49
48
|
requirement: !ruby/object:Gem::Requirement
|
50
49
|
requirements:
|
51
50
|
- - "~>"
|
52
51
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
52
|
+
version: '6'
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: 6.2.0
|
54
56
|
- - "<"
|
55
57
|
- !ruby/object:Gem::Version
|
56
|
-
version: '
|
58
|
+
version: '7'
|
57
59
|
type: :runtime
|
58
60
|
prerelease: false
|
59
61
|
version_requirements: !ruby/object:Gem::Requirement
|
60
62
|
requirements:
|
61
63
|
- - "~>"
|
62
64
|
- !ruby/object:Gem::Version
|
63
|
-
version: '
|
65
|
+
version: '6'
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 6.2.0
|
64
69
|
- - "<"
|
65
70
|
- !ruby/object:Gem::Version
|
66
|
-
version: '
|
71
|
+
version: '7'
|
67
72
|
- !ruby/object:Gem::Dependency
|
68
73
|
name: rspec
|
69
74
|
requirement: !ruby/object:Gem::Requirement
|
@@ -148,6 +153,20 @@ dependencies:
|
|
148
153
|
- - ">="
|
149
154
|
- !ruby/object:Gem::Version
|
150
155
|
version: '0'
|
156
|
+
- !ruby/object:Gem::Dependency
|
157
|
+
name: standard
|
158
|
+
requirement: !ruby/object:Gem::Requirement
|
159
|
+
requirements:
|
160
|
+
- - '='
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: 1.28.5
|
163
|
+
type: :development
|
164
|
+
prerelease: false
|
165
|
+
version_requirements: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - '='
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: 1.28.5
|
151
170
|
- !ruby/object:Gem::Dependency
|
152
171
|
name: rspec-mocks
|
153
172
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,17 +189,16 @@ extensions: []
|
|
170
189
|
extra_rdoc_files: []
|
171
190
|
files:
|
172
191
|
- ".github/workflows/ci.yml"
|
192
|
+
- ".standard.yml"
|
173
193
|
- Gemfile
|
174
194
|
- LICENSE
|
175
195
|
- README.md
|
176
196
|
- Rakefile
|
177
197
|
- lib/zipline.rb
|
178
|
-
- lib/zipline/chunked_body.rb
|
179
|
-
- lib/zipline/tempfile_body.rb
|
180
198
|
- lib/zipline/version.rb
|
181
|
-
- lib/zipline/
|
199
|
+
- lib/zipline/zip_handler.rb
|
182
200
|
- spec/fakefile.txt
|
183
|
-
- spec/lib/zipline/
|
201
|
+
- spec/lib/zipline/zip_handler_spec.rb
|
184
202
|
- spec/lib/zipline/zipline_spec.rb
|
185
203
|
- spec/spec_helper.rb
|
186
204
|
- zipline.gemspec
|
@@ -188,7 +206,6 @@ homepage: http://github.com/fringd/zipline
|
|
188
206
|
licenses:
|
189
207
|
- MIT
|
190
208
|
metadata: {}
|
191
|
-
post_install_message:
|
192
209
|
rdoc_options: []
|
193
210
|
require_paths:
|
194
211
|
- lib
|
@@ -203,12 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
203
220
|
- !ruby/object:Gem::Version
|
204
221
|
version: '0'
|
205
222
|
requirements: []
|
206
|
-
rubygems_version: 3.
|
207
|
-
signing_key:
|
223
|
+
rubygems_version: 3.6.7
|
208
224
|
specification_version: 4
|
209
225
|
summary: stream zip files from rails
|
210
|
-
test_files:
|
211
|
-
- spec/fakefile.txt
|
212
|
-
- spec/lib/zipline/zip_generator_spec.rb
|
213
|
-
- spec/lib/zipline/zipline_spec.rb
|
214
|
-
- spec/spec_helper.rb
|
226
|
+
test_files: []
|
data/lib/zipline/chunked_body.rb
DELETED
@@ -1,31 +0,0 @@
|
|
1
|
-
module Zipline
|
2
|
-
# A body wrapper that emits chunked responses, creating valid
|
3
|
-
# "Transfer-Encoding: chunked" HTTP response body. This is copied from Rack::Chunked::Body,
|
4
|
-
# because Rack is not going to include that class after version 3.x
|
5
|
-
# Rails has a substitute class for this inside ActionController::Streaming,
|
6
|
-
# but that module is a private constant in the Rails codebase, and is thus
|
7
|
-
# considered "private" from the Rails standpoint. It is not that much code to
|
8
|
-
# carry, so we copy it into our code.
|
9
|
-
class Chunked
|
10
|
-
TERM = "\r\n"
|
11
|
-
TAIL = "0#{TERM}"
|
12
|
-
|
13
|
-
def initialize(body)
|
14
|
-
@body = body
|
15
|
-
end
|
16
|
-
|
17
|
-
# For each string yielded by the response body, yield
|
18
|
-
# the element in chunked encoding - and finish off with a terminator
|
19
|
-
def each
|
20
|
-
term = TERM
|
21
|
-
@body.each do |chunk|
|
22
|
-
size = chunk.bytesize
|
23
|
-
next if size == 0
|
24
|
-
|
25
|
-
yield [size.to_s(16), term, chunk.b, term].join
|
26
|
-
end
|
27
|
-
yield TAIL
|
28
|
-
yield term
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
module Zipline
|
2
|
-
# Contains a file handle which can be closed once the response finishes sending.
|
3
|
-
# It supports `to_path` so that `Rack::Sendfile` can intercept it
|
4
|
-
class TempfileBody
|
5
|
-
TEMPFILE_NAME_PREFIX = "zipline-tf-body-"
|
6
|
-
attr_reader :tempfile
|
7
|
-
|
8
|
-
# @param body[#each] the enumerable that yields bytes, usually a `RackBody`.
|
9
|
-
# The `body` will be read in full immediately and closed.
|
10
|
-
def initialize(env, body)
|
11
|
-
@tempfile = Tempfile.new(TEMPFILE_NAME_PREFIX)
|
12
|
-
# Rack::TempfileReaper calls close! on tempfiles which get buffered
|
13
|
-
# We wil assume that it works fine with Rack::Sendfile (i.e. the path
|
14
|
-
# to the file getting served gets used before we unlink the tempfile)
|
15
|
-
env['rack.tempfiles'] ||= []
|
16
|
-
env['rack.tempfiles'] << @tempfile
|
17
|
-
|
18
|
-
@tempfile.binmode
|
19
|
-
|
20
|
-
body.each { |bytes| @tempfile << bytes }
|
21
|
-
body.close if body.respond_to?(:close)
|
22
|
-
|
23
|
-
@tempfile.flush
|
24
|
-
end
|
25
|
-
|
26
|
-
# Returns the size of the contained `Tempfile` so that a correct
|
27
|
-
# Content-Length header can be set
|
28
|
-
#
|
29
|
-
# @return [Integer]
|
30
|
-
def size
|
31
|
-
@tempfile.size
|
32
|
-
end
|
33
|
-
|
34
|
-
# Returns the path to the `Tempfile`, so that Rack::Sendfile can send this response
|
35
|
-
# using the downstream webserver
|
36
|
-
#
|
37
|
-
# @return [String]
|
38
|
-
def to_path
|
39
|
-
@tempfile.to_path
|
40
|
-
end
|
41
|
-
|
42
|
-
# Stream the file's contents if `Rack::Sendfile` isn't present.
|
43
|
-
#
|
44
|
-
# @return [void]
|
45
|
-
def each
|
46
|
-
@tempfile.rewind
|
47
|
-
while chunk = @tempfile.read(16384)
|
48
|
-
yield chunk
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|