down 5.6.2 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +146 -56
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +33 -3
- data/lib/down/wget.rb +0 -250
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# Generated from lib/down/net_http.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# Provides streaming downloads implemented with Net::HTTP and open-uri.
|
|
5
|
+
class NetHttp < Backend
|
|
6
|
+
@options: Hash[Symbol, untyped]
|
|
7
|
+
|
|
8
|
+
URI_NORMALIZER: ^(String) -> String
|
|
9
|
+
|
|
10
|
+
# Content encodings that Net::HTTP transparently decodes.
|
|
11
|
+
DECODED_ENCODINGS: Array[String]
|
|
12
|
+
|
|
13
|
+
DEFAULT_OPTIONS: Hash[Symbol, untyped]
|
|
14
|
+
|
|
15
|
+
# Initializes the backend with common defaults.
|
|
16
|
+
# : (**untyped options) -> void
|
|
17
|
+
def initialize: (**untyped options) -> void
|
|
18
|
+
|
|
19
|
+
# Downloads a remote file to disk using open-uri. Accepts any open-uri
|
|
20
|
+
# options, and a few more.
|
|
21
|
+
# : (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
22
|
+
# : (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
23
|
+
def download: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
24
|
+
| (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
25
|
+
|
|
26
|
+
# Starts retrieving the remote file using Net::HTTP and returns an IO-like
|
|
27
|
+
# object which downloads the response body on-demand.
|
|
28
|
+
# : (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
29
|
+
def open: (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
34
|
+
# Receives String-keyed header kwargs alongside the options, which RBS keyword types can't express.
|
|
35
|
+
# : (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
36
|
+
def open_uri: (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
37
|
+
|
|
38
|
+
# Adds name/value pairs from the given Set-Cookie header values to the
|
|
39
|
+
# Cookie request header, ignoring any cookie attributes.
|
|
40
|
+
# : (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
41
|
+
def forward_cookies: (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
42
|
+
|
|
43
|
+
# Parses the Cookie request header value into a hash.
|
|
44
|
+
# : (String? value) -> Hash[String, String?]
|
|
45
|
+
def parse_cookie_header: (String? value) -> Hash[String, String?]
|
|
46
|
+
|
|
47
|
+
# Returns the Cookie request header value from the given headers.
|
|
48
|
+
# : (Hash[untyped, untyped] headers) -> String?
|
|
49
|
+
def cookie_header: (Hash[untyped, untyped] headers) -> String?
|
|
50
|
+
|
|
51
|
+
# Sets the Cookie request header value in the given headers, removing it
|
|
52
|
+
# if the value is nil or empty.
|
|
53
|
+
# : (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
54
|
+
def set_cookie_header: (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
55
|
+
|
|
56
|
+
# Returns whether the given header name is the Cookie header.
|
|
57
|
+
# : (untyped name) -> bool
|
|
58
|
+
def cookie_header?: (untyped name) -> bool
|
|
59
|
+
|
|
60
|
+
# Returns whether the given URIs have the same scheme, host and port.
|
|
61
|
+
# : (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
62
|
+
def same_origin?: (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
63
|
+
|
|
64
|
+
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
65
|
+
# returns a StringIO when there is less than 10KB of content), and gives
|
|
66
|
+
# it the specified file extension.
|
|
67
|
+
# : (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
68
|
+
def ensure_tempfile: (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
69
|
+
|
|
70
|
+
# Makes a Net::HTTP request and follows redirects.
|
|
71
|
+
# : (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
72
|
+
def net_http_request: (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
73
|
+
|
|
74
|
+
# Build a Net::HTTP object for making a request.
|
|
75
|
+
# : (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [String?, String?]?, **untyped) -> [Net::HTTP, Net::HTTP::Get]
|
|
76
|
+
def create_net_http: (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [ String?, String? ]?, **untyped) -> [ Net::HTTP, Net::HTTP::Get ]
|
|
77
|
+
|
|
78
|
+
# Yields chunks of the response body to the block.
|
|
79
|
+
# : (Net::HTTPResponse response) { (String) -> void } -> void
|
|
80
|
+
def stream_body: (Net::HTTPResponse response) { (String) -> void } -> void
|
|
81
|
+
|
|
82
|
+
# Returns the Content-Length, unless a Transfer-Encoding is present or
|
|
83
|
+
# Net::HTTP will decode the body, in which case it doesn't reflect the
|
|
84
|
+
# number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
|
|
85
|
+
# : (Net::HTTPResponse response) -> Integer?
|
|
86
|
+
def content_length: (Net::HTTPResponse response) -> Integer?
|
|
87
|
+
|
|
88
|
+
# Checks that the url is a valid URI and that its scheme is http or https.
|
|
89
|
+
# : (untyped url) -> URI::HTTP
|
|
90
|
+
# : (untyped url, allow_relative: bool) -> URI::Generic
|
|
91
|
+
def ensure_uri: (untyped url) -> URI::HTTP
|
|
92
|
+
| (untyped url, allow_relative: bool) -> URI::Generic
|
|
93
|
+
|
|
94
|
+
# Makes sure that the URL is properly encoded.
|
|
95
|
+
# : (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
96
|
+
def normalize_uri: (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
97
|
+
|
|
98
|
+
# When open-uri raises an exception, it doesn't expose the response object.
|
|
99
|
+
# Fortunately, the exception object holds response data that can be used to
|
|
100
|
+
# rebuild the Net::HTTP response object.
|
|
101
|
+
# : (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
102
|
+
def rebuild_response_from_open_uri_exception: (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
103
|
+
|
|
104
|
+
# Raises non-sucessful response as a Down::ResponseError.
|
|
105
|
+
# : (Net::HTTPResponse response) -> bot
|
|
106
|
+
def response_error!: (Net::HTTPResponse response) -> bot
|
|
107
|
+
|
|
108
|
+
# Re-raise Net::HTTP exceptions as Down::Error exceptions.
|
|
109
|
+
# : (Exception exception) -> bot
|
|
110
|
+
def request_error!: (Exception exception) -> bot
|
|
111
|
+
|
|
112
|
+
# Merge default and ad-hoc options, merging nested headers.
|
|
113
|
+
# : (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
114
|
+
def merge_options: (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
115
|
+
|
|
116
|
+
# Defines some additional attributes for the returned Tempfile (on top of what
|
|
117
|
+
# OpenURI::Meta already defines).
|
|
118
|
+
# @rbs module-self OpenURI::Meta
|
|
119
|
+
module DownloadedFile : OpenURI::Meta
|
|
120
|
+
# : () -> String?
|
|
121
|
+
def original_filename: () -> String?
|
|
122
|
+
|
|
123
|
+
# : () -> String?
|
|
124
|
+
def content_type: () -> String?
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Generated from lib/down/utils.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
module Utils
|
|
5
|
+
# Retrieves potential filename from the "Content-Disposition" header.
|
|
6
|
+
# : (_ToS content_disposition) -> String?
|
|
7
|
+
def self?.filename_from_content_disposition: (_ToS content_disposition) -> String?
|
|
8
|
+
|
|
9
|
+
# Retrieves potential filename from the URL path.
|
|
10
|
+
# : (String path) -> String?
|
|
11
|
+
def self?.filename_from_path: (String path) -> String?
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Generated from lib/down.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
self.@backend: _Backend
|
|
5
|
+
|
|
6
|
+
@backend: _Backend
|
|
7
|
+
|
|
8
|
+
# : (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
9
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
10
|
+
def self?.download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
11
|
+
| (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
12
|
+
|
|
13
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
14
|
+
def self?.open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
15
|
+
|
|
16
|
+
# Allows setting a backend via a symbol or a downloader object.
|
|
17
|
+
# : (Symbol | _Backend) -> _Backend
|
|
18
|
+
# : () -> _Backend
|
|
19
|
+
def self?.backend: (Symbol | _Backend) -> _Backend
|
|
20
|
+
| () -> _Backend
|
|
21
|
+
end
|
data/sig/manifest.yaml
ADDED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: down
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 6.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Janko Marohnić
|
|
@@ -37,6 +37,20 @@ dependencies:
|
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0.3'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: tempfile
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 0.3.1
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 0.3.1
|
|
40
54
|
- !ruby/object:Gem::Dependency
|
|
41
55
|
name: minitest
|
|
42
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -160,7 +174,23 @@ files:
|
|
|
160
174
|
- lib/down/net_http.rb
|
|
161
175
|
- lib/down/utils.rb
|
|
162
176
|
- lib/down/version.rb
|
|
163
|
-
-
|
|
177
|
+
- sig/_patches/addressable.rbs
|
|
178
|
+
- sig/_patches/http.rbs
|
|
179
|
+
- sig/_patches/io.rbs
|
|
180
|
+
- sig/_patches/net_http.rbs
|
|
181
|
+
- sig/_patches/open_uri.rbs
|
|
182
|
+
- sig/_patches/stringio.rbs
|
|
183
|
+
- sig/down.rbs
|
|
184
|
+
- sig/generated/down.rbs
|
|
185
|
+
- sig/generated/down/backend.rbs
|
|
186
|
+
- sig/generated/down/chunked_io.rbs
|
|
187
|
+
- sig/generated/down/errors.rbs
|
|
188
|
+
- sig/generated/down/http.rbs
|
|
189
|
+
- sig/generated/down/httpx.rbs
|
|
190
|
+
- sig/generated/down/net_http.rbs
|
|
191
|
+
- sig/generated/down/utils.rbs
|
|
192
|
+
- sig/generated/down/version.rbs
|
|
193
|
+
- sig/manifest.yaml
|
|
164
194
|
homepage: https://github.com/janko/down
|
|
165
195
|
licenses:
|
|
166
196
|
- MIT
|
|
@@ -179,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
179
209
|
- !ruby/object:Gem::Version
|
|
180
210
|
version: '0'
|
|
181
211
|
requirements: []
|
|
182
|
-
rubygems_version: 4.0.
|
|
212
|
+
rubygems_version: 4.0.13
|
|
183
213
|
specification_version: 4
|
|
184
214
|
summary: Robust streaming downloads using Net::HTTP, http.rb or HTTPX.
|
|
185
215
|
test_files: []
|
data/lib/down/wget.rb
DELETED
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
# frozen-string-literal: true
|
|
2
|
-
|
|
3
|
-
warn "The Down::Wget backend has been deprecated and will be removed in the next major version. Please use the Net::HTTP/open-uri, http.rb or HTTPX backend instead."
|
|
4
|
-
|
|
5
|
-
begin
|
|
6
|
-
require "posix-spawn"
|
|
7
|
-
rescue LoadError
|
|
8
|
-
require "open3"
|
|
9
|
-
end
|
|
10
|
-
require "http_parser"
|
|
11
|
-
|
|
12
|
-
require "down/backend"
|
|
13
|
-
|
|
14
|
-
require "tempfile"
|
|
15
|
-
require "uri"
|
|
16
|
-
|
|
17
|
-
module Down
|
|
18
|
-
# Provides streaming downloads implemented with the wget command-line tool.
|
|
19
|
-
# The design is very similar to Down::Http.
|
|
20
|
-
class Wget < Backend
|
|
21
|
-
# Initializes the backend with common defaults.
|
|
22
|
-
def initialize(*arguments)
|
|
23
|
-
@arguments = [
|
|
24
|
-
user_agent: "Down/#{Down::VERSION}",
|
|
25
|
-
max_redirect: 2,
|
|
26
|
-
dns_timeout: 30,
|
|
27
|
-
connect_timeout: 30,
|
|
28
|
-
read_timeout: 30,
|
|
29
|
-
] + arguments
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
# Downlods the remote file to disk. Accepts wget command-line options and
|
|
33
|
-
# some additional options as well.
|
|
34
|
-
def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
|
|
35
|
-
io = open(url, *args, **options, rewindable: false)
|
|
36
|
-
|
|
37
|
-
content_length_proc.call(io.size) if content_length_proc && io.size
|
|
38
|
-
|
|
39
|
-
if max_size && io.size && io.size > max_size
|
|
40
|
-
raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
extname = extension ? ".#{extension}" : File.extname(URI(url).path)
|
|
44
|
-
tempfile = Tempfile.new(["down-wget", extname], binmode: true)
|
|
45
|
-
|
|
46
|
-
until io.eof?
|
|
47
|
-
chunk = io.readpartial(nil, buffer ||= String.new)
|
|
48
|
-
|
|
49
|
-
tempfile.write(chunk)
|
|
50
|
-
|
|
51
|
-
progress_proc.call(tempfile.size) if progress_proc
|
|
52
|
-
|
|
53
|
-
if max_size && tempfile.size > max_size
|
|
54
|
-
raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
tempfile.open # flush written content
|
|
59
|
-
|
|
60
|
-
tempfile.extend Down::Wget::DownloadedFile
|
|
61
|
-
tempfile.url = url
|
|
62
|
-
tempfile.headers = io.data[:headers]
|
|
63
|
-
|
|
64
|
-
download_result(tempfile, destination)
|
|
65
|
-
rescue
|
|
66
|
-
tempfile.close! if tempfile
|
|
67
|
-
raise
|
|
68
|
-
ensure
|
|
69
|
-
io.close if io
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# Starts retrieving the remote file and returns an IO-like object which
|
|
73
|
-
# downloads the response body on-demand. Accepts wget command-line options.
|
|
74
|
-
def open(url, *args, rewindable: true, **options)
|
|
75
|
-
arguments = generate_command(url, *args, **options)
|
|
76
|
-
|
|
77
|
-
command = Down::Wget::Command.execute(arguments)
|
|
78
|
-
# Wrap the wget command output in an IO-like object.
|
|
79
|
-
output = Down::ChunkedIO.new(
|
|
80
|
-
chunks: command.enum_for(:output),
|
|
81
|
-
on_close: command.method(:terminate),
|
|
82
|
-
rewindable: false,
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
# https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363
|
|
86
|
-
header_string = output.readpartial
|
|
87
|
-
header_string << output.readpartial until header_string.include?("\r\n\r\n")
|
|
88
|
-
header_string, first_chunk = header_string.split("\r\n\r\n", 2)
|
|
89
|
-
|
|
90
|
-
# Use an HTTP parser to parse out the response headers.
|
|
91
|
-
parser = HTTP::Parser.new
|
|
92
|
-
parser << header_string
|
|
93
|
-
|
|
94
|
-
if parser.headers.nil?
|
|
95
|
-
output.close
|
|
96
|
-
raise Down::Error, "failed to parse response headers"
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
headers = normalize_headers(parser.headers)
|
|
100
|
-
status = parser.status_code
|
|
101
|
-
|
|
102
|
-
content_length = headers["Content-Length"].to_i if headers["Content-Length"]
|
|
103
|
-
charset = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]
|
|
104
|
-
|
|
105
|
-
# Create an Enumerator which will lazily retrieve chunks of response body.
|
|
106
|
-
chunks = Enumerator.new do |yielder|
|
|
107
|
-
yielder << first_chunk if first_chunk
|
|
108
|
-
yielder << output.readpartial until output.eof?
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
Down::ChunkedIO.new(
|
|
112
|
-
chunks: chunks,
|
|
113
|
-
size: content_length,
|
|
114
|
-
encoding: charset,
|
|
115
|
-
rewindable: rewindable,
|
|
116
|
-
on_close: output.method(:close),
|
|
117
|
-
data: { status: status, headers: headers },
|
|
118
|
-
)
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
private
|
|
122
|
-
|
|
123
|
-
# Generates the wget command.
|
|
124
|
-
def generate_command(url, *args, **options)
|
|
125
|
-
command = %W[wget --no-verbose --save-headers -O -]
|
|
126
|
-
|
|
127
|
-
options = @arguments.grep(Hash).inject({}, :merge).merge(options)
|
|
128
|
-
args = @arguments.grep(->(o){!o.is_a?(Hash)}) + args
|
|
129
|
-
|
|
130
|
-
(args + options.to_a).each do |option, value|
|
|
131
|
-
if option.is_a?(String)
|
|
132
|
-
command << option
|
|
133
|
-
elsif option.length == 1
|
|
134
|
-
command << "-#{option}"
|
|
135
|
-
else
|
|
136
|
-
command << "--#{option.to_s.gsub("_", "-")}"
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
command << value.to_s unless value.nil?
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
command << url
|
|
143
|
-
command
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
# Handles executing the wget command.
|
|
147
|
-
class Command
|
|
148
|
-
PIPE_BUFFER_SIZE = 64*1024
|
|
149
|
-
|
|
150
|
-
def self.execute(arguments)
|
|
151
|
-
# posix-spawn gem has better performance, so we use it if it's available
|
|
152
|
-
if defined?(POSIX::Spawn)
|
|
153
|
-
pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
|
|
154
|
-
status_reaper = Process.detach(pid)
|
|
155
|
-
else
|
|
156
|
-
stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
stdin_pipe.close
|
|
160
|
-
[stdout_pipe, stderr_pipe].each(&:binmode)
|
|
161
|
-
|
|
162
|
-
new(stdout_pipe, stderr_pipe, status_reaper)
|
|
163
|
-
rescue Errno::ENOENT
|
|
164
|
-
raise Down::Error, "wget is not installed"
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def initialize(stdout_pipe, stderr_pipe, status_reaper)
|
|
168
|
-
@status_reaper = status_reaper
|
|
169
|
-
@stdout_pipe = stdout_pipe
|
|
170
|
-
@stderr_pipe = stderr_pipe
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
# Yields chunks of stdout. At the end handles the exit status.
|
|
174
|
-
def output
|
|
175
|
-
# Keep emptying the stderr buffer, to allow the subprocess to send more
|
|
176
|
-
# than 64KB if it wants to.
|
|
177
|
-
stderr_reader = Thread.new { @stderr_pipe.read }
|
|
178
|
-
|
|
179
|
-
yield @stdout_pipe.readpartial(PIPE_BUFFER_SIZE) until @stdout_pipe.eof?
|
|
180
|
-
|
|
181
|
-
status = @status_reaper.value
|
|
182
|
-
stderr = stderr_reader.value
|
|
183
|
-
|
|
184
|
-
close
|
|
185
|
-
|
|
186
|
-
handle_status(status, stderr)
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def terminate
|
|
190
|
-
begin
|
|
191
|
-
Process.kill("TERM", @status_reaper[:pid])
|
|
192
|
-
Process.waitpid(@status_reaper[:pid])
|
|
193
|
-
rescue Errno::ESRCH
|
|
194
|
-
# process has already terminated
|
|
195
|
-
end
|
|
196
|
-
|
|
197
|
-
close
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
def close
|
|
201
|
-
@stdout_pipe.close unless @stdout_pipe.closed?
|
|
202
|
-
@stderr_pipe.close unless @stderr_pipe.closed?
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
private
|
|
206
|
-
|
|
207
|
-
# Translates nonzero wget exit statuses into exceptions.
|
|
208
|
-
def handle_status(status, stderr)
|
|
209
|
-
case status.exitstatus
|
|
210
|
-
when 0 # No problems occurred
|
|
211
|
-
# success
|
|
212
|
-
when 1, # Generic error code
|
|
213
|
-
2, # Parse error---for instance, when parsing command-line options, the .wgetrc or .netrc...
|
|
214
|
-
3 # File I/O error
|
|
215
|
-
raise Down::Error, stderr
|
|
216
|
-
when 4 # Network failure
|
|
217
|
-
raise Down::TimeoutError, stderr if stderr.include?("timed out")
|
|
218
|
-
raise Down::ConnectionError, stderr
|
|
219
|
-
when 5 # SSL verification failure
|
|
220
|
-
raise Down::SSLError, stderr
|
|
221
|
-
when 6 # Username/password authentication failure
|
|
222
|
-
raise Down::ClientError, stderr
|
|
223
|
-
when 7 # Protocol errors
|
|
224
|
-
raise Down::Error, stderr
|
|
225
|
-
when 8 # Server issued an error response
|
|
226
|
-
raise Down::TooManyRedirects, stderr if stderr.include?("redirections exceeded")
|
|
227
|
-
raise Down::ResponseError, stderr
|
|
228
|
-
end
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
# Adds additional attributes to the Tempfile returned in #download.
|
|
233
|
-
module DownloadedFile
|
|
234
|
-
attr_accessor :url, :headers
|
|
235
|
-
|
|
236
|
-
def original_filename
|
|
237
|
-
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
238
|
-
Utils.filename_from_path(URI.parse(url).path)
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
def content_type
|
|
242
|
-
headers["Content-Type"].to_s.split(";").first
|
|
243
|
-
end
|
|
244
|
-
|
|
245
|
-
def charset
|
|
246
|
-
headers["Content-Type"].to_s[/;\s*charset=([^;]+)/i, 1]
|
|
247
|
-
end
|
|
248
|
-
end
|
|
249
|
-
end
|
|
250
|
-
end
|