down 5.6.3 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +62 -33
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +32 -2
- data/lib/down/wget.rb +0 -250
data/lib/down/http.rb
CHANGED
|
@@ -12,7 +12,11 @@ require "tempfile"
|
|
|
12
12
|
module Down
|
|
13
13
|
# Provides streaming downloads implemented with HTTP.rb.
|
|
14
14
|
class Http < Backend
|
|
15
|
+
# @rbs @method: HTTP::verb | String
|
|
16
|
+
# @rbs @client: HTTP::Session | HTTP::Client
|
|
17
|
+
|
|
15
18
|
# Initializes the backend with common defaults.
|
|
19
|
+
#: (?method: HTTP::verb | String, **untyped options) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> void
|
|
16
20
|
def initialize(method: :get, **options, &block)
|
|
17
21
|
@method = method
|
|
18
22
|
@client = HTTP
|
|
@@ -29,13 +33,17 @@ module Down
|
|
|
29
33
|
|
|
30
34
|
# Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
|
|
31
35
|
# block, and some additional options as well.
|
|
36
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> nil
|
|
37
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> downloaded_file
|
|
32
38
|
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
33
39
|
response = request(url, **, &)
|
|
34
40
|
|
|
35
|
-
|
|
41
|
+
content_length = response.content_length
|
|
42
|
+
|
|
43
|
+
content_length_proc&.call(content_length) if content_length
|
|
36
44
|
|
|
37
|
-
if max_size &&
|
|
38
|
-
raise Down::TooLarge, "file is too large (#{
|
|
45
|
+
if max_size && content_length && content_length > max_size
|
|
46
|
+
raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
39
47
|
end
|
|
40
48
|
|
|
41
49
|
extname = extension ? ".#{extension}" : File.extname(response.uri.path)
|
|
@@ -54,11 +62,11 @@ module Down
|
|
|
54
62
|
|
|
55
63
|
tempfile.open # flush written content
|
|
56
64
|
|
|
57
|
-
tempfile.extend Down::Http::DownloadedFile
|
|
65
|
+
tempfile = tempfile.extend(Down::Http::DownloadedFile) #: Tempfile & Down::Http::DownloadedFile
|
|
58
66
|
tempfile.url = response.uri.to_s
|
|
59
67
|
tempfile.headers = normalize_headers(response.headers.to_h)
|
|
60
68
|
|
|
61
|
-
download_result(tempfile, destination)
|
|
69
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
62
70
|
rescue
|
|
63
71
|
tempfile&.close!
|
|
64
72
|
raise
|
|
@@ -67,6 +75,7 @@ module Down
|
|
|
67
75
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
68
76
|
# downloads the response body on-demand. Accepts HTTP.rb options via a hash
|
|
69
77
|
# or a block.
|
|
78
|
+
#: (url url, ?rewindable: bool, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> ChunkedIO
|
|
70
79
|
def open(url, rewindable: true, **, &)
|
|
71
80
|
response = request(url, **, &)
|
|
72
81
|
|
|
@@ -85,17 +94,19 @@ module Down
|
|
|
85
94
|
|
|
86
95
|
private
|
|
87
96
|
|
|
97
|
+
#: (url url, ?method: HTTP::verb | String, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
88
98
|
def request(url, method: @method, **, &)
|
|
89
99
|
response = send_request(method, url, **, &)
|
|
90
100
|
response_error!(response) unless response.status.success?
|
|
91
101
|
response
|
|
92
102
|
end
|
|
93
103
|
|
|
104
|
+
#: (HTTP::verb | String method, url url, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
94
105
|
def send_request(method, url, **, &block)
|
|
95
106
|
uri = HTTP::URI.parse(url)
|
|
96
107
|
|
|
97
108
|
client = @client
|
|
98
|
-
client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
|
|
109
|
+
client = client.basic_auth(user: uri.user.to_s, pass: uri.password.to_s) if uri.user || uri.password
|
|
99
110
|
client = block.call(client) if block
|
|
100
111
|
|
|
101
112
|
client.request(method, url, **)
|
|
@@ -104,27 +115,30 @@ module Down
|
|
|
104
115
|
end
|
|
105
116
|
|
|
106
117
|
# Yields chunks of the response body to the block.
|
|
118
|
+
#: (HTTP::Response response) { (String) -> void } -> void
|
|
107
119
|
def stream_body(response, &)
|
|
108
120
|
response.body.each(&)
|
|
109
121
|
rescue => exception
|
|
110
122
|
request_error!(exception)
|
|
111
123
|
ensure
|
|
112
|
-
response.connection
|
|
124
|
+
response.connection&.close unless @client.persistent?
|
|
113
125
|
end
|
|
114
126
|
|
|
115
127
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
128
|
+
#: (HTTP::Response response) -> bot
|
|
116
129
|
def response_error!(response)
|
|
117
|
-
|
|
130
|
+
message = response.status.to_s
|
|
118
131
|
|
|
119
132
|
case response.code
|
|
120
|
-
when 404 then raise Down::NotFound.new(
|
|
121
|
-
when 400..499 then raise Down::ClientError.new(
|
|
122
|
-
when 500..599 then raise Down::ServerError.new(
|
|
123
|
-
else raise Down::ResponseError.new(
|
|
133
|
+
when 404 then raise Down::NotFound.new(message, response)
|
|
134
|
+
when 400..499 then raise Down::ClientError.new(message, response)
|
|
135
|
+
when 500..599 then raise Down::ServerError.new(message, response)
|
|
136
|
+
else raise Down::ResponseError.new(message, response)
|
|
124
137
|
end
|
|
125
138
|
end
|
|
126
139
|
|
|
127
140
|
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
141
|
+
#: (Exception exception) -> bot
|
|
128
142
|
def request_error!(exception)
|
|
129
143
|
case exception
|
|
130
144
|
when HTTP::Request::UnsupportedSchemeError, Addressable::URI::InvalidURIError, *invalid_url_errors
|
|
@@ -142,23 +156,28 @@ module Down
|
|
|
142
156
|
end
|
|
143
157
|
end
|
|
144
158
|
|
|
159
|
+
#: () -> Array[singleton(Exception)]
|
|
145
160
|
def invalid_url_errors
|
|
146
161
|
defined?(HTTP::URI::InvalidError) ? [HTTP::URI::InvalidError] : []
|
|
147
162
|
end
|
|
148
163
|
|
|
149
164
|
# Defines some additional attributes for the returned Tempfile.
|
|
150
165
|
module DownloadedFile
|
|
151
|
-
attr_accessor :url
|
|
166
|
+
attr_accessor :url #: String
|
|
167
|
+
attr_accessor :headers #: Hash[String, untyped]
|
|
152
168
|
|
|
169
|
+
#: () -> String?
|
|
153
170
|
def original_filename
|
|
154
171
|
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
155
172
|
Utils.filename_from_path(HTTP::URI.parse(url).path)
|
|
156
173
|
end
|
|
157
174
|
|
|
175
|
+
#: () -> String?
|
|
158
176
|
def content_type
|
|
159
177
|
HTTP::ContentType.parse(headers["Content-Type"]).mime_type
|
|
160
178
|
end
|
|
161
179
|
|
|
180
|
+
#: () -> String?
|
|
162
181
|
def charset
|
|
163
182
|
HTTP::ContentType.parse(headers["Content-Type"]).charset
|
|
164
183
|
end
|
data/lib/down/httpx.rb
CHANGED
|
@@ -11,9 +11,16 @@ module Down
|
|
|
11
11
|
class Httpx < Backend
|
|
12
12
|
USER_AGENT = "Down/#{Down::VERSION}"
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# HTTPX's plugin-composed sessions and stream responses (which delegate via
|
|
15
|
+
# method_missing) aren't expressible with its shipped RBS, so the client and
|
|
16
|
+
# response objects are left untyped here.
|
|
17
|
+
# @rbs @method: Symbol | String
|
|
18
|
+
# @rbs @client: untyped
|
|
19
|
+
|
|
20
|
+
BASIC_AUTH = HTTPX::VERSION >= "1.0.0" ? :basic_auth : :basic_authentication #: Symbol
|
|
15
21
|
|
|
16
22
|
# Initializes the backend with common defaults.
|
|
23
|
+
#: (?method: Symbol | String, ?headers: Hash[String | Symbol, String], **untyped options) ?{ (untyped) -> untyped } -> void
|
|
17
24
|
def initialize(method: :get, headers: {}, **options, &block)
|
|
18
25
|
@method = method
|
|
19
26
|
@client = HTTPX
|
|
@@ -31,6 +38,8 @@ module Down
|
|
|
31
38
|
|
|
32
39
|
# Downloads the remote file to disk. Accepts HTTPX options via a hash or a
|
|
33
40
|
# block, and some additional options as well.
|
|
41
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
42
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
34
43
|
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
35
44
|
response = request(url, **, &)
|
|
36
45
|
|
|
@@ -61,13 +70,13 @@ module Down
|
|
|
61
70
|
|
|
62
71
|
tempfile.open # flush written content
|
|
63
72
|
|
|
64
|
-
tempfile.extend DownloadedFile
|
|
73
|
+
tempfile = tempfile.extend(DownloadedFile) #: Tempfile & DownloadedFile
|
|
65
74
|
tempfile.url = response.uri.to_s
|
|
66
75
|
tempfile.headers = normalize_headers(response.headers.to_h)
|
|
67
76
|
tempfile.content_type = response.content_type.mime_type
|
|
68
77
|
tempfile.charset = response.body.encoding
|
|
69
78
|
|
|
70
|
-
download_result(tempfile, destination)
|
|
79
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
71
80
|
rescue
|
|
72
81
|
tempfile&.close!
|
|
73
82
|
raise
|
|
@@ -76,6 +85,7 @@ module Down
|
|
|
76
85
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
77
86
|
# downloads the response body on-demand. Accepts HTTPX options via a hash
|
|
78
87
|
# or a block.
|
|
88
|
+
#: (url url, ?rewindable: bool, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
79
89
|
def open(url, rewindable: true, **, &)
|
|
80
90
|
response = request(url, stream: true, **, &)
|
|
81
91
|
|
|
@@ -95,6 +105,7 @@ module Down
|
|
|
95
105
|
private
|
|
96
106
|
|
|
97
107
|
# Yields chunks of the response body to the block.
|
|
108
|
+
#: (untyped response) { (String) -> void } -> void
|
|
98
109
|
def stream_body(response, &block)
|
|
99
110
|
response.each do |chunk|
|
|
100
111
|
next if (300..399).include?(response.status)
|
|
@@ -109,6 +120,7 @@ module Down
|
|
|
109
120
|
request_error!(exception)
|
|
110
121
|
end
|
|
111
122
|
|
|
123
|
+
#: (url url, ?method: Symbol | String, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
112
124
|
def request(url, method: @method, **, &)
|
|
113
125
|
response = send_request(method, url, **, &)
|
|
114
126
|
response.raise_for_status
|
|
@@ -120,7 +132,8 @@ module Down
|
|
|
120
132
|
request_error!(error)
|
|
121
133
|
end
|
|
122
134
|
|
|
123
|
-
|
|
135
|
+
#: (Symbol | String method, url url, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
136
|
+
def send_request(method, url, **options, &block)
|
|
124
137
|
uri = URI(url)
|
|
125
138
|
client = @client
|
|
126
139
|
if uri.user || uri.password
|
|
@@ -129,24 +142,26 @@ module Down
|
|
|
129
142
|
end
|
|
130
143
|
client = block.call(client) if block
|
|
131
144
|
|
|
132
|
-
client.request(method.to_s.upcase, uri, stream: true, **)
|
|
145
|
+
client.request(method.to_s.upcase, uri, stream: true, **options) # named splat avoids a Steep 2.1 crash
|
|
133
146
|
rescue => exception
|
|
134
147
|
request_error!(exception)
|
|
135
148
|
end
|
|
136
149
|
|
|
137
150
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
151
|
+
#: (untyped response) -> bot
|
|
138
152
|
def response_error!(response)
|
|
139
|
-
|
|
153
|
+
message = response.status.to_s
|
|
140
154
|
|
|
141
155
|
case response.status
|
|
142
|
-
when 404 then raise Down::NotFound.new(
|
|
143
|
-
when 400..499 then raise Down::ClientError.new(
|
|
144
|
-
when 500..599 then raise Down::ServerError.new(
|
|
145
|
-
else raise Down::ResponseError.new(
|
|
156
|
+
when 404 then raise Down::NotFound.new(message, response)
|
|
157
|
+
when 400..499 then raise Down::ClientError.new(message, response)
|
|
158
|
+
when 500..599 then raise Down::ServerError.new(message, response)
|
|
159
|
+
else raise Down::ResponseError.new(message, response)
|
|
146
160
|
end
|
|
147
161
|
end
|
|
148
162
|
|
|
149
163
|
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
164
|
+
#: (Exception exception) -> bot
|
|
150
165
|
def request_error!(exception)
|
|
151
166
|
case exception
|
|
152
167
|
when URI::Error, HTTPX::UnsupportedSchemeError
|
|
@@ -164,11 +179,15 @@ module Down
|
|
|
164
179
|
|
|
165
180
|
# Defines some additional attributes for the returned Tempfile.
|
|
166
181
|
module DownloadedFile
|
|
167
|
-
attr_accessor :url
|
|
182
|
+
attr_accessor :url #: String
|
|
183
|
+
attr_accessor :headers #: Hash[String, untyped]
|
|
184
|
+
attr_accessor :charset #: Encoding?
|
|
185
|
+
attr_accessor :content_type #: String?
|
|
168
186
|
|
|
187
|
+
#: () -> String?
|
|
169
188
|
def original_filename
|
|
170
189
|
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
171
|
-
Utils.filename_from_path(URI.parse(url).path)
|
|
190
|
+
Utils.filename_from_path(URI.parse(url).path.to_s)
|
|
172
191
|
end
|
|
173
192
|
end
|
|
174
193
|
end
|
data/lib/down/net_http.rb
CHANGED
|
@@ -12,13 +12,15 @@ require "fileutils"
|
|
|
12
12
|
module Down
|
|
13
13
|
# Provides streaming downloads implemented with Net::HTTP and open-uri.
|
|
14
14
|
class NetHttp < Backend
|
|
15
|
+
# @rbs @options: Hash[Symbol, untyped]
|
|
16
|
+
|
|
15
17
|
URI_NORMALIZER = -> (url) do
|
|
16
18
|
addressable_uri = Addressable::URI.parse(url)
|
|
17
19
|
addressable_uri.normalize.to_s
|
|
18
|
-
end
|
|
20
|
+
end #: ^(String) -> String
|
|
19
21
|
|
|
20
22
|
# Content encodings that Net::HTTP transparently decodes.
|
|
21
|
-
DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze
|
|
23
|
+
DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze #: Array[String]
|
|
22
24
|
private_constant :DECODED_ENCODINGS
|
|
23
25
|
|
|
24
26
|
DEFAULT_OPTIONS = {
|
|
@@ -27,15 +29,18 @@ module Down
|
|
|
27
29
|
open_timeout: 30,
|
|
28
30
|
read_timeout: 30,
|
|
29
31
|
uri_normalizer: URI_NORMALIZER,
|
|
30
|
-
}.freeze
|
|
32
|
+
}.freeze #: Hash[Symbol, untyped]
|
|
31
33
|
|
|
32
34
|
# Initializes the backend with common defaults.
|
|
35
|
+
#: (**untyped options) -> void
|
|
33
36
|
def initialize(...)
|
|
34
37
|
@options = merge_options(DEFAULT_OPTIONS, ...)
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
# Downloads a remote file to disk using open-uri. Accepts any open-uri
|
|
38
41
|
# options, and a few more.
|
|
42
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
43
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
39
44
|
def download(url, ...)
|
|
40
45
|
options = merge_options(@options, ...)
|
|
41
46
|
|
|
@@ -68,7 +73,7 @@ module Down
|
|
|
68
73
|
progress_proc&.call(current_size)
|
|
69
74
|
},
|
|
70
75
|
redirect: false,
|
|
71
|
-
}
|
|
76
|
+
} #: Hash[untyped, untyped]
|
|
72
77
|
|
|
73
78
|
# Handle basic authentication in the :proxy option.
|
|
74
79
|
if proxy
|
|
@@ -100,16 +105,17 @@ module Down
|
|
|
100
105
|
open_uri_file = open_uri(uri, **open_uri_options)
|
|
101
106
|
|
|
102
107
|
# Handle the fact that open-uri returns StringIOs for small files.
|
|
103
|
-
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
|
|
108
|
+
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path.to_s)
|
|
104
109
|
tempfile = ensure_tempfile(open_uri_file, extname, tempfile_name)
|
|
105
110
|
OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
|
|
106
111
|
tempfile.extend Down::NetHttp::DownloadedFile
|
|
107
112
|
|
|
108
|
-
download_result(tempfile, destination)
|
|
113
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
109
114
|
end
|
|
110
115
|
|
|
111
116
|
# Starts retrieving the remote file using Net::HTTP and returns an IO-like
|
|
112
117
|
# object which downloads the response body on-demand.
|
|
118
|
+
#: (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
113
119
|
def open(url, ...)
|
|
114
120
|
options = merge_options(@options, ...)
|
|
115
121
|
|
|
@@ -119,7 +125,8 @@ module Down
|
|
|
119
125
|
|
|
120
126
|
# Create a Fiber that halts when response headers are received.
|
|
121
127
|
request = Fiber.new do
|
|
122
|
-
|
|
128
|
+
# :max_redirects comes from DEFAULT_OPTIONS
|
|
129
|
+
net_http_request(uri, **options) do |response| # steep:ignore
|
|
123
130
|
Fiber.yield response
|
|
124
131
|
end
|
|
125
132
|
end
|
|
@@ -146,7 +153,14 @@ module Down
|
|
|
146
153
|
private
|
|
147
154
|
|
|
148
155
|
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
156
|
+
# Receives String-keyed header kwargs alongside the options, which RBS keyword types can't express.
|
|
157
|
+
#: (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
149
158
|
def open_uri(uri, max_redirects:, auth_on_redirect: false, **options)
|
|
159
|
+
# @type var uri: URI::HTTP
|
|
160
|
+
# @type var max_redirects: Integer
|
|
161
|
+
# @type var auth_on_redirect: bool
|
|
162
|
+
# @type var options: Hash[Symbol | String, untyped]
|
|
163
|
+
|
|
150
164
|
# remember cookies provided by the caller, so that we can tell them apart
|
|
151
165
|
# from cookies collected from redirect responses
|
|
152
166
|
user_cookie = cookie_header(options)
|
|
@@ -198,48 +212,59 @@ module Down
|
|
|
198
212
|
|
|
199
213
|
# Adds name/value pairs from the given Set-Cookie header values to the
|
|
200
214
|
# Cookie request header, ignoring any cookie attributes.
|
|
215
|
+
#: (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
201
216
|
def forward_cookies(headers, set_cookies)
|
|
202
217
|
return if set_cookies.nil? || set_cookies.empty?
|
|
203
218
|
|
|
204
219
|
cookies = parse_cookie_header(cookie_header(headers))
|
|
205
220
|
|
|
206
221
|
set_cookies.each do |set_cookie|
|
|
207
|
-
name, value = set_cookie.split(";", 2).first.to_s.split("=", 2)
|
|
208
|
-
next if value.nil? || name.empty?
|
|
222
|
+
name, value = set_cookie.split(";", 2).first.to_s.split("=", 2)
|
|
223
|
+
next if name.nil? || value.nil? || name.strip.empty?
|
|
209
224
|
|
|
210
|
-
cookies[name] = value
|
|
225
|
+
cookies[name.strip] = value.strip
|
|
211
226
|
end
|
|
212
227
|
|
|
213
228
|
set_cookie_header(headers, cookies.map { |name, value| value ? "#{name}=#{value}" : name }.join("; "))
|
|
214
229
|
end
|
|
215
230
|
|
|
216
231
|
# Parses the Cookie request header value into a hash.
|
|
232
|
+
#: (String? value) -> Hash[String, String?]
|
|
217
233
|
def parse_cookie_header(value)
|
|
218
|
-
|
|
234
|
+
cookies = {} #: Hash[String, String?]
|
|
235
|
+
|
|
236
|
+
value.to_s.split(";").each do |pair|
|
|
219
237
|
name, value = pair.split("=", 2).map(&:strip)
|
|
220
238
|
cookies[name] = value unless name.nil? || name.empty?
|
|
221
239
|
end
|
|
240
|
+
|
|
241
|
+
cookies
|
|
222
242
|
end
|
|
223
243
|
|
|
224
244
|
# Returns the Cookie request header value from the given headers.
|
|
245
|
+
#: (Hash[untyped, untyped] headers) -> String?
|
|
225
246
|
def cookie_header(headers)
|
|
226
247
|
headers.find { |name, _| cookie_header?(name) }&.last
|
|
227
248
|
end
|
|
228
249
|
|
|
229
250
|
# Sets the Cookie request header value in the given headers, removing it
|
|
230
251
|
# if the value is nil or empty.
|
|
252
|
+
#: (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
231
253
|
def set_cookie_header(headers, value)
|
|
232
254
|
headers.delete_if { |name, _| cookie_header?(name) }
|
|
233
255
|
headers["Cookie"] = value unless value.nil? || value.empty?
|
|
234
256
|
end
|
|
235
257
|
|
|
258
|
+
# Returns whether the given header name is the Cookie header.
|
|
259
|
+
#: (untyped name) -> bool
|
|
236
260
|
def cookie_header?(name)
|
|
237
261
|
name.is_a?(String) && name.casecmp?("Cookie")
|
|
238
262
|
end
|
|
239
263
|
|
|
240
264
|
# Returns whether the given URIs have the same scheme, host and port.
|
|
265
|
+
#: (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
241
266
|
def same_origin?(uri, other_uri)
|
|
242
|
-
uri.scheme.casecmp?(other_uri.scheme) &&
|
|
267
|
+
uri.scheme.to_s.casecmp?(other_uri.scheme.to_s) &&
|
|
243
268
|
uri.host.to_s.casecmp?(other_uri.host.to_s) &&
|
|
244
269
|
uri.port == other_uri.port
|
|
245
270
|
end
|
|
@@ -247,6 +272,7 @@ module Down
|
|
|
247
272
|
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
248
273
|
# returns a StringIO when there is less than 10KB of content), and gives
|
|
249
274
|
# it the specified file extension.
|
|
275
|
+
#: (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
250
276
|
def ensure_tempfile(io, extension, tempfile_name = nil)
|
|
251
277
|
tempfile = Tempfile.new([tempfile_name || "down-net_http", extension], binmode: true)
|
|
252
278
|
|
|
@@ -254,7 +280,7 @@ module Down
|
|
|
254
280
|
# Windows requires file descriptors to be closed before files are moved
|
|
255
281
|
io.close
|
|
256
282
|
tempfile.close
|
|
257
|
-
FileUtils.mv io.path, tempfile.path
|
|
283
|
+
FileUtils.mv io.path.to_s, tempfile.path.to_s
|
|
258
284
|
else
|
|
259
285
|
IO.copy_stream(io, tempfile)
|
|
260
286
|
io.close
|
|
@@ -265,6 +291,7 @@ module Down
|
|
|
265
291
|
end
|
|
266
292
|
|
|
267
293
|
# Makes a Net::HTTP request and follows redirects.
|
|
294
|
+
#: (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
268
295
|
def net_http_request(uri, max_redirects:, auth_on_redirect: false, **options, &)
|
|
269
296
|
http, request = create_net_http(uri, **options)
|
|
270
297
|
|
|
@@ -319,6 +346,7 @@ module Down
|
|
|
319
346
|
end
|
|
320
347
|
|
|
321
348
|
# Build a Net::HTTP object for making a request.
|
|
349
|
+
#: (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [String?, String?]?, **untyped) -> [Net::HTTP, Net::HTTP::Get]
|
|
322
350
|
def create_net_http(uri, proxy: nil, ssl_verify_mode: nil, ssl_ca_cert: nil, read_timeout: nil, open_timeout: nil, headers: {}, http_basic_authentication: nil, **)
|
|
323
351
|
http_class = Net::HTTP
|
|
324
352
|
|
|
@@ -350,12 +378,13 @@ module Down
|
|
|
350
378
|
get = Net::HTTP::Get.new(uri, headers.to_h)
|
|
351
379
|
|
|
352
380
|
user, password = http_basic_authentication || [uri.user, uri.password]
|
|
353
|
-
get.basic_auth(user, password) if user || password
|
|
381
|
+
get.basic_auth(user.to_s, password.to_s) if user || password
|
|
354
382
|
|
|
355
383
|
[http, get]
|
|
356
384
|
end
|
|
357
385
|
|
|
358
386
|
# Yields chunks of the response body to the block.
|
|
387
|
+
#: (Net::HTTPResponse response) { (String) -> void } -> void
|
|
359
388
|
def stream_body(response, &)
|
|
360
389
|
response.read_body(&)
|
|
361
390
|
rescue => exception
|
|
@@ -365,6 +394,7 @@ module Down
|
|
|
365
394
|
# Returns the Content-Length, unless a Transfer-Encoding is present or
|
|
366
395
|
# Net::HTTP will decode the body, in which case it doesn't reflect the
|
|
367
396
|
# number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
|
|
397
|
+
#: (Net::HTTPResponse response) -> Integer?
|
|
368
398
|
def content_length(response)
|
|
369
399
|
return if response["Transfer-Encoding"]
|
|
370
400
|
|
|
@@ -377,6 +407,8 @@ module Down
|
|
|
377
407
|
end
|
|
378
408
|
|
|
379
409
|
# Checks that the url is a valid URI and that its scheme is http or https.
|
|
410
|
+
#: (untyped url) -> URI::HTTP
|
|
411
|
+
#: (untyped url, allow_relative: bool) -> URI::Generic
|
|
380
412
|
def ensure_uri(url, allow_relative: false)
|
|
381
413
|
begin
|
|
382
414
|
uri = URI(url)
|
|
@@ -392,6 +424,7 @@ module Down
|
|
|
392
424
|
end
|
|
393
425
|
|
|
394
426
|
# Makes sure that the URL is properly encoded.
|
|
427
|
+
#: (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
395
428
|
def normalize_uri(url, uri_normalizer:)
|
|
396
429
|
URI(url)
|
|
397
430
|
rescue URI::InvalidURIError
|
|
@@ -401,6 +434,7 @@ module Down
|
|
|
401
434
|
# When open-uri raises an exception, it doesn't expose the response object.
|
|
402
435
|
# Fortunately, the exception object holds response data that can be used to
|
|
403
436
|
# rebuild the Net::HTTP response object.
|
|
437
|
+
#: (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
404
438
|
def rebuild_response_from_open_uri_exception(exception)
|
|
405
439
|
code, message = exception.io.status
|
|
406
440
|
message ||= "Unknown"
|
|
@@ -420,21 +454,23 @@ module Down
|
|
|
420
454
|
end
|
|
421
455
|
|
|
422
456
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
457
|
+
#: (Net::HTTPResponse response) -> bot
|
|
423
458
|
def response_error!(response)
|
|
424
459
|
code = response.code.to_i
|
|
425
460
|
message = response.message.split(" ").map(&:capitalize).join(" ")
|
|
426
461
|
|
|
427
|
-
|
|
462
|
+
error_message = "#{code} #{message}"
|
|
428
463
|
|
|
429
464
|
case response.code.to_i
|
|
430
|
-
when 404 then raise Down::NotFound.new(
|
|
431
|
-
when 400..499 then raise Down::ClientError.new(
|
|
432
|
-
when 500..599 then raise Down::ServerError.new(
|
|
433
|
-
else raise Down::ResponseError.new(
|
|
465
|
+
when 404 then raise Down::NotFound.new(error_message, response)
|
|
466
|
+
when 400..499 then raise Down::ClientError.new(error_message, response)
|
|
467
|
+
when 500..599 then raise Down::ServerError.new(error_message, response)
|
|
468
|
+
else raise Down::ResponseError.new(error_message, response)
|
|
434
469
|
end
|
|
435
470
|
end
|
|
436
471
|
|
|
437
472
|
# Re-raise Net::HTTP exceptions as Down::Error exceptions.
|
|
473
|
+
#: (Exception exception) -> bot
|
|
438
474
|
def request_error!(exception)
|
|
439
475
|
case exception
|
|
440
476
|
when Net::OpenTimeout
|
|
@@ -451,19 +487,9 @@ module Down
|
|
|
451
487
|
end
|
|
452
488
|
|
|
453
489
|
# Merge default and ad-hoc options, merging nested headers.
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
if headers.any?
|
|
458
|
-
warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
|
|
459
|
-
new_options[:headers] = headers
|
|
460
|
-
elsif new_options.any? { |key, value| key.is_a?(String) }
|
|
461
|
-
warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
|
|
462
|
-
new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
|
|
463
|
-
new_options.reject! { |key, value| key.is_a?(String) }
|
|
464
|
-
else
|
|
465
|
-
new_options[:headers] ||= {}
|
|
466
|
-
end
|
|
490
|
+
#: (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
491
|
+
def merge_options(options, **new_options)
|
|
492
|
+
new_options[:headers] ||= {}
|
|
467
493
|
|
|
468
494
|
options.merge(new_options) do |key, value1, value2|
|
|
469
495
|
key == :headers ? value1.merge(value2) : value2
|
|
@@ -472,12 +498,15 @@ module Down
|
|
|
472
498
|
|
|
473
499
|
# Defines some additional attributes for the returned Tempfile (on top of what
|
|
474
500
|
# OpenURI::Meta already defines).
|
|
501
|
+
# @rbs module-self OpenURI::Meta
|
|
475
502
|
module DownloadedFile
|
|
503
|
+
#: () -> String?
|
|
476
504
|
def original_filename
|
|
477
505
|
Utils.filename_from_content_disposition(meta["content-disposition"]) ||
|
|
478
|
-
Utils.filename_from_path(base_uri.path)
|
|
506
|
+
Utils.filename_from_path(base_uri.path.to_s)
|
|
479
507
|
end
|
|
480
508
|
|
|
509
|
+
#: () -> String?
|
|
481
510
|
def content_type
|
|
482
511
|
super unless meta["content-type"].to_s.empty?
|
|
483
512
|
end
|
data/lib/down/utils.rb
CHANGED
|
@@ -9,6 +9,7 @@ module Down
|
|
|
9
9
|
module_function
|
|
10
10
|
|
|
11
11
|
# Retrieves potential filename from the "Content-Disposition" header.
|
|
12
|
+
#: (_ToS content_disposition) -> String?
|
|
12
13
|
def filename_from_content_disposition(content_disposition)
|
|
13
14
|
content_disposition = content_disposition.to_s
|
|
14
15
|
|
|
@@ -23,6 +24,7 @@ module Down
|
|
|
23
24
|
end
|
|
24
25
|
|
|
25
26
|
# Retrieves potential filename from the URL path.
|
|
27
|
+
#: (String path) -> String?
|
|
26
28
|
def filename_from_path(path)
|
|
27
29
|
filename = path.split("/").last
|
|
28
30
|
CGI.unescape(filename) if filename
|
data/lib/down/version.rb
CHANGED
data/lib/down.rb
CHANGED
|
@@ -4,25 +4,33 @@ require "down/version"
|
|
|
4
4
|
require "down/net_http"
|
|
5
5
|
|
|
6
6
|
module Down
|
|
7
|
+
# @rbs self.@backend: _Backend
|
|
8
|
+
# @rbs @backend: _Backend
|
|
7
9
|
module_function
|
|
8
10
|
|
|
11
|
+
#: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
12
|
+
#: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
9
13
|
def download(...)
|
|
10
14
|
backend.download(...)
|
|
11
15
|
end
|
|
12
16
|
|
|
17
|
+
#: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
13
18
|
def open(...)
|
|
14
19
|
backend.open(...)
|
|
15
20
|
end
|
|
16
21
|
|
|
17
22
|
# Allows setting a backend via a symbol or a downloader object.
|
|
23
|
+
#: (Symbol | _Backend) -> _Backend
|
|
24
|
+
#: () -> _Backend
|
|
18
25
|
def backend(value = nil)
|
|
19
|
-
|
|
26
|
+
case value
|
|
27
|
+
when Symbol
|
|
20
28
|
require "down/#{value}"
|
|
21
29
|
@backend = Down.const_get(value.to_s.split("_").map(&:capitalize).join)
|
|
22
|
-
|
|
23
|
-
@backend = value
|
|
24
|
-
else
|
|
30
|
+
when nil
|
|
25
31
|
@backend
|
|
32
|
+
else
|
|
33
|
+
@backend = value
|
|
26
34
|
end
|
|
27
35
|
end
|
|
28
36
|
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# http.rb's runtime also accepts String verbs (e.g. "POST") and stdlib URI
|
|
2
|
+
# objects, which its shipped signatures don't allow.
|
|
3
|
+
module HTTP
|
|
4
|
+
class Client
|
|
5
|
+
def request: (verb | String verb, String | URI | ::URI::Generic uri, **untyped) -> Response
|
|
6
|
+
| ...
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
class Session
|
|
10
|
+
def request: (verb | String verb, String | URI | ::URI::Generic uri, **untyped) -> Response
|
|
11
|
+
| ...
|
|
12
|
+
end
|
|
13
|
+
end
|
data/sig/_patches/io.rbs
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Net::HTTP.Proxy also accepts a nil address (meaning no proxy), e.g. a
|
|
2
|
+
# URI#hostname, which the shipped signature doesn't allow.
|
|
3
|
+
module Net
|
|
4
|
+
class HTTP
|
|
5
|
+
def self.Proxy: (String? p_addr, ?Integer? p_port, ?String? p_user, ?String? p_pass) -> untyped
|
|
6
|
+
| ...
|
|
7
|
+
end
|
|
8
|
+
end
|