down 5.6.3 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +62 -33
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +32 -2
- data/lib/down/wget.rb +0 -250
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Generated from lib/down.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
self.@backend: _Backend
|
|
5
|
+
|
|
6
|
+
@backend: _Backend
|
|
7
|
+
|
|
8
|
+
# : (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
9
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
10
|
+
def self?.download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
11
|
+
| (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
12
|
+
|
|
13
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
14
|
+
def self?.open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
15
|
+
|
|
16
|
+
# Allows setting a backend via a symbol or a downloader object.
|
|
17
|
+
# : (Symbol | _Backend) -> _Backend
|
|
18
|
+
# : () -> _Backend
|
|
19
|
+
def self?.backend: (Symbol | _Backend) -> _Backend
|
|
20
|
+
| () -> _Backend
|
|
21
|
+
end
|
data/sig/manifest.yaml
ADDED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: down
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 6.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Janko Marohnić
|
|
@@ -37,6 +37,20 @@ dependencies:
|
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0.3'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: tempfile
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 0.3.1
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 0.3.1
|
|
40
54
|
- !ruby/object:Gem::Dependency
|
|
41
55
|
name: minitest
|
|
42
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -160,7 +174,23 @@ files:
|
|
|
160
174
|
- lib/down/net_http.rb
|
|
161
175
|
- lib/down/utils.rb
|
|
162
176
|
- lib/down/version.rb
|
|
163
|
-
-
|
|
177
|
+
- sig/_patches/addressable.rbs
|
|
178
|
+
- sig/_patches/http.rbs
|
|
179
|
+
- sig/_patches/io.rbs
|
|
180
|
+
- sig/_patches/net_http.rbs
|
|
181
|
+
- sig/_patches/open_uri.rbs
|
|
182
|
+
- sig/_patches/stringio.rbs
|
|
183
|
+
- sig/down.rbs
|
|
184
|
+
- sig/generated/down.rbs
|
|
185
|
+
- sig/generated/down/backend.rbs
|
|
186
|
+
- sig/generated/down/chunked_io.rbs
|
|
187
|
+
- sig/generated/down/errors.rbs
|
|
188
|
+
- sig/generated/down/http.rbs
|
|
189
|
+
- sig/generated/down/httpx.rbs
|
|
190
|
+
- sig/generated/down/net_http.rbs
|
|
191
|
+
- sig/generated/down/utils.rbs
|
|
192
|
+
- sig/generated/down/version.rbs
|
|
193
|
+
- sig/manifest.yaml
|
|
164
194
|
homepage: https://github.com/janko/down
|
|
165
195
|
licenses:
|
|
166
196
|
- MIT
|
data/lib/down/wget.rb
DELETED
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
# frozen-string-literal: true
|
|
2
|
-
|
|
3
|
-
warn "The Down::Wget backend has been deprecated and will be removed in the next major version. Please use the Net::HTTP/open-uri, http.rb or HTTPX backend instead."
|
|
4
|
-
|
|
5
|
-
begin
|
|
6
|
-
require "posix-spawn"
|
|
7
|
-
rescue LoadError
|
|
8
|
-
require "open3"
|
|
9
|
-
end
|
|
10
|
-
require "http_parser"
|
|
11
|
-
|
|
12
|
-
require "down/backend"
|
|
13
|
-
|
|
14
|
-
require "tempfile"
|
|
15
|
-
require "uri"
|
|
16
|
-
|
|
17
|
-
module Down
|
|
18
|
-
# Provides streaming downloads implemented with the wget command-line tool.
|
|
19
|
-
# The design is very similar to Down::Http.
|
|
20
|
-
class Wget < Backend
|
|
21
|
-
# Initializes the backend with common defaults.
|
|
22
|
-
def initialize(*arguments)
|
|
23
|
-
@arguments = [
|
|
24
|
-
user_agent: "Down/#{Down::VERSION}",
|
|
25
|
-
max_redirect: 2,
|
|
26
|
-
dns_timeout: 30,
|
|
27
|
-
connect_timeout: 30,
|
|
28
|
-
read_timeout: 30,
|
|
29
|
-
] + arguments
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
# Downlods the remote file to disk. Accepts wget command-line options and
|
|
33
|
-
# some additional options as well.
|
|
34
|
-
def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
|
|
35
|
-
io = open(url, *args, **options, rewindable: false)
|
|
36
|
-
|
|
37
|
-
content_length_proc.call(io.size) if content_length_proc && io.size
|
|
38
|
-
|
|
39
|
-
if max_size && io.size && io.size > max_size
|
|
40
|
-
raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
extname = extension ? ".#{extension}" : File.extname(URI(url).path)
|
|
44
|
-
tempfile = Tempfile.new(["down-wget", extname], binmode: true)
|
|
45
|
-
|
|
46
|
-
until io.eof?
|
|
47
|
-
chunk = io.readpartial(nil, buffer ||= String.new)
|
|
48
|
-
|
|
49
|
-
tempfile.write(chunk)
|
|
50
|
-
|
|
51
|
-
progress_proc.call(tempfile.size) if progress_proc
|
|
52
|
-
|
|
53
|
-
if max_size && tempfile.size > max_size
|
|
54
|
-
raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
tempfile.open # flush written content
|
|
59
|
-
|
|
60
|
-
tempfile.extend Down::Wget::DownloadedFile
|
|
61
|
-
tempfile.url = url
|
|
62
|
-
tempfile.headers = io.data[:headers]
|
|
63
|
-
|
|
64
|
-
download_result(tempfile, destination)
|
|
65
|
-
rescue
|
|
66
|
-
tempfile.close! if tempfile
|
|
67
|
-
raise
|
|
68
|
-
ensure
|
|
69
|
-
io.close if io
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# Starts retrieving the remote file and returns an IO-like object which
|
|
73
|
-
# downloads the response body on-demand. Accepts wget command-line options.
|
|
74
|
-
def open(url, *args, rewindable: true, **options)
|
|
75
|
-
arguments = generate_command(url, *args, **options)
|
|
76
|
-
|
|
77
|
-
command = Down::Wget::Command.execute(arguments)
|
|
78
|
-
# Wrap the wget command output in an IO-like object.
|
|
79
|
-
output = Down::ChunkedIO.new(
|
|
80
|
-
chunks: command.enum_for(:output),
|
|
81
|
-
on_close: command.method(:terminate),
|
|
82
|
-
rewindable: false,
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
# https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363
|
|
86
|
-
header_string = output.readpartial
|
|
87
|
-
header_string << output.readpartial until header_string.include?("\r\n\r\n")
|
|
88
|
-
header_string, first_chunk = header_string.split("\r\n\r\n", 2)
|
|
89
|
-
|
|
90
|
-
# Use an HTTP parser to parse out the response headers.
|
|
91
|
-
parser = HTTP::Parser.new
|
|
92
|
-
parser << header_string
|
|
93
|
-
|
|
94
|
-
if parser.headers.nil?
|
|
95
|
-
output.close
|
|
96
|
-
raise Down::Error, "failed to parse response headers"
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
headers = normalize_headers(parser.headers)
|
|
100
|
-
status = parser.status_code
|
|
101
|
-
|
|
102
|
-
content_length = headers["Content-Length"].to_i if headers["Content-Length"]
|
|
103
|
-
charset = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]
|
|
104
|
-
|
|
105
|
-
# Create an Enumerator which will lazily retrieve chunks of response body.
|
|
106
|
-
chunks = Enumerator.new do |yielder|
|
|
107
|
-
yielder << first_chunk if first_chunk
|
|
108
|
-
yielder << output.readpartial until output.eof?
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
Down::ChunkedIO.new(
|
|
112
|
-
chunks: chunks,
|
|
113
|
-
size: content_length,
|
|
114
|
-
encoding: charset,
|
|
115
|
-
rewindable: rewindable,
|
|
116
|
-
on_close: output.method(:close),
|
|
117
|
-
data: { status: status, headers: headers },
|
|
118
|
-
)
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
private
|
|
122
|
-
|
|
123
|
-
# Generates the wget command.
|
|
124
|
-
def generate_command(url, *args, **options)
|
|
125
|
-
command = %W[wget --no-verbose --save-headers -O -]
|
|
126
|
-
|
|
127
|
-
options = @arguments.grep(Hash).inject({}, :merge).merge(options)
|
|
128
|
-
args = @arguments.grep(->(o){!o.is_a?(Hash)}) + args
|
|
129
|
-
|
|
130
|
-
(args + options.to_a).each do |option, value|
|
|
131
|
-
if option.is_a?(String)
|
|
132
|
-
command << option
|
|
133
|
-
elsif option.length == 1
|
|
134
|
-
command << "-#{option}"
|
|
135
|
-
else
|
|
136
|
-
command << "--#{option.to_s.gsub("_", "-")}"
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
command << value.to_s unless value.nil?
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
command << url
|
|
143
|
-
command
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
# Handles executing the wget command.
|
|
147
|
-
class Command
|
|
148
|
-
PIPE_BUFFER_SIZE = 64*1024
|
|
149
|
-
|
|
150
|
-
def self.execute(arguments)
|
|
151
|
-
# posix-spawn gem has better performance, so we use it if it's available
|
|
152
|
-
if defined?(POSIX::Spawn)
|
|
153
|
-
pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
|
|
154
|
-
status_reaper = Process.detach(pid)
|
|
155
|
-
else
|
|
156
|
-
stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
stdin_pipe.close
|
|
160
|
-
[stdout_pipe, stderr_pipe].each(&:binmode)
|
|
161
|
-
|
|
162
|
-
new(stdout_pipe, stderr_pipe, status_reaper)
|
|
163
|
-
rescue Errno::ENOENT
|
|
164
|
-
raise Down::Error, "wget is not installed"
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def initialize(stdout_pipe, stderr_pipe, status_reaper)
|
|
168
|
-
@status_reaper = status_reaper
|
|
169
|
-
@stdout_pipe = stdout_pipe
|
|
170
|
-
@stderr_pipe = stderr_pipe
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
# Yields chunks of stdout. At the end handles the exit status.
|
|
174
|
-
def output
|
|
175
|
-
# Keep emptying the stderr buffer, to allow the subprocess to send more
|
|
176
|
-
# than 64KB if it wants to.
|
|
177
|
-
stderr_reader = Thread.new { @stderr_pipe.read }
|
|
178
|
-
|
|
179
|
-
yield @stdout_pipe.readpartial(PIPE_BUFFER_SIZE) until @stdout_pipe.eof?
|
|
180
|
-
|
|
181
|
-
status = @status_reaper.value
|
|
182
|
-
stderr = stderr_reader.value
|
|
183
|
-
|
|
184
|
-
close
|
|
185
|
-
|
|
186
|
-
handle_status(status, stderr)
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def terminate
|
|
190
|
-
begin
|
|
191
|
-
Process.kill("TERM", @status_reaper[:pid])
|
|
192
|
-
Process.waitpid(@status_reaper[:pid])
|
|
193
|
-
rescue Errno::ESRCH
|
|
194
|
-
# process has already terminated
|
|
195
|
-
end
|
|
196
|
-
|
|
197
|
-
close
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
def close
|
|
201
|
-
@stdout_pipe.close unless @stdout_pipe.closed?
|
|
202
|
-
@stderr_pipe.close unless @stderr_pipe.closed?
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
private
|
|
206
|
-
|
|
207
|
-
# Translates nonzero wget exit statuses into exceptions.
|
|
208
|
-
def handle_status(status, stderr)
|
|
209
|
-
case status.exitstatus
|
|
210
|
-
when 0 # No problems occurred
|
|
211
|
-
# success
|
|
212
|
-
when 1, # Generic error code
|
|
213
|
-
2, # Parse error---for instance, when parsing command-line options, the .wgetrc or .netrc...
|
|
214
|
-
3 # File I/O error
|
|
215
|
-
raise Down::Error, stderr
|
|
216
|
-
when 4 # Network failure
|
|
217
|
-
raise Down::TimeoutError, stderr if stderr.include?("timed out")
|
|
218
|
-
raise Down::ConnectionError, stderr
|
|
219
|
-
when 5 # SSL verification failure
|
|
220
|
-
raise Down::SSLError, stderr
|
|
221
|
-
when 6 # Username/password authentication failure
|
|
222
|
-
raise Down::ClientError, stderr
|
|
223
|
-
when 7 # Protocol errors
|
|
224
|
-
raise Down::Error, stderr
|
|
225
|
-
when 8 # Server issued an error response
|
|
226
|
-
raise Down::TooManyRedirects, stderr if stderr.include?("redirections exceeded")
|
|
227
|
-
raise Down::ResponseError, stderr
|
|
228
|
-
end
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
# Adds additional attributes to the Tempfile returned in #download.
|
|
233
|
-
module DownloadedFile
|
|
234
|
-
attr_accessor :url, :headers
|
|
235
|
-
|
|
236
|
-
def original_filename
|
|
237
|
-
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
238
|
-
Utils.filename_from_path(URI.parse(url).path)
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
def content_type
|
|
242
|
-
headers["Content-Type"].to_s.split(";").first
|
|
243
|
-
end
|
|
244
|
-
|
|
245
|
-
def charset
|
|
246
|
-
headers["Content-Type"].to_s[/;\s*charset=([^;]+)/i, 1]
|
|
247
|
-
end
|
|
248
|
-
end
|
|
249
|
-
end
|
|
250
|
-
end
|