down 5.6.3 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +62 -33
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +32 -2
- data/lib/down/wget.rb +0 -250
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Fills gaps in the stdlib open-uri signatures (rbs 4.2).
|
|
2
|
+
module OpenURI
|
|
3
|
+
module Meta
|
|
4
|
+
def self.init: (Object obj, ?untyped src) -> void
|
|
5
|
+
|
|
6
|
+
attr_reader metas: Hash[String, Array[String]]
|
|
7
|
+
|
|
8
|
+
def charset: () ?{ () -> String } -> String?
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
module OpenRead
|
|
12
|
+
# Without a block it returns the downloaded StringIO/Tempfile extended with
|
|
13
|
+
# OpenURI::Meta, which the shipped signature declares as IO.
|
|
14
|
+
def open: (*untyped) -> ((StringIO | Tempfile) & Meta)
|
|
15
|
+
| ...
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# StringIO#gets also accepts a nil separator (reading all remaining content)
|
|
2
|
+
# and a nil limit (no limit), like IO#gets, which the stdlib signature
|
|
3
|
+
# (rbs 4.2) doesn't allow.
|
|
4
|
+
class StringIO
|
|
5
|
+
def gets: (String? sep, ?Integer? limit, ?chomp: boolish) -> String?
|
|
6
|
+
| ...
|
|
7
|
+
end
|
data/sig/down.rbs
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Hand-written declarations shared across backends. Everything else is
|
|
2
|
+
# generated into sig/generated/ from inline annotations in lib/.
|
|
3
|
+
module Down
|
|
4
|
+
# A URL accepted by the backends.
|
|
5
|
+
type url = String | URI::Generic
|
|
6
|
+
|
|
7
|
+
# Methods added to the Tempfile returned by `download`.
|
|
8
|
+
interface _DownloadedFile
|
|
9
|
+
def original_filename: () -> String?
|
|
10
|
+
def content_type: () -> String?
|
|
11
|
+
def charset: () -> (String | Encoding)? # Down::Httpx returns an Encoding
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Tempfile returned by `download` when no `:destination` is given.
|
|
15
|
+
type downloaded_file = Tempfile & _DownloadedFile
|
|
16
|
+
|
|
17
|
+
# Callback such as ChunkedIO's `:on_close`.
|
|
18
|
+
interface _Callable
|
|
19
|
+
def call: () -> untyped
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Anything that can be assigned via `Down.backend`, typically a backend
|
|
23
|
+
# class such as `Down::NetHttp`.
|
|
24
|
+
interface _Backend
|
|
25
|
+
def download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
26
|
+
| (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
27
|
+
def open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Generated from lib/down/backend.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
class Backend
|
|
5
|
+
def download: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file?
|
|
6
|
+
|
|
7
|
+
def open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
8
|
+
|
|
9
|
+
# : (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
10
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
11
|
+
def self.download: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
12
|
+
| (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
13
|
+
|
|
14
|
+
# : (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
15
|
+
def self.open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
# If destination path is defined, move tempfile to the destination,
|
|
20
|
+
# otherwise return the tempfile unchanged.
|
|
21
|
+
# : (Tempfile tempfile, String? destination) -> Tempfile?
|
|
22
|
+
def download_result: (Tempfile tempfile, String? destination) -> Tempfile?
|
|
23
|
+
|
|
24
|
+
# : [V] (Enumerable[[String, V]] response_headers) -> Hash[String, V]
|
|
25
|
+
def normalize_headers: [V] (Enumerable[[ String, V ]] response_headers) -> Hash[String, V]
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# Generated from lib/down/chunked_io.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# Wraps an enumerator that yields chunks of content into an IO object. It
|
|
5
|
+
# implements some essential IO methods:
|
|
6
|
+
#
|
|
7
|
+
# * IO#read
|
|
8
|
+
# * IO#readpartial
|
|
9
|
+
# * IO#gets
|
|
10
|
+
# * IO#size
|
|
11
|
+
# * IO#pos
|
|
12
|
+
# * IO#seek
|
|
13
|
+
# * IO#eof?
|
|
14
|
+
# * IO#rewind
|
|
15
|
+
# * IO#close
|
|
16
|
+
#
|
|
17
|
+
# Retrieved chunks are appended to a backing store which the reading methods
|
|
18
|
+
# delegate to, so they behave like the corresponding native IO methods. New
|
|
19
|
+
# chunks are retrieved only when the store runs out of content.
|
|
20
|
+
#
|
|
21
|
+
# By default the store is a tempfile which keeps all read content, allowing
|
|
22
|
+
# the Down::ChunkedIO to be rewindable. If rewindability won't be used, it
|
|
23
|
+
# can be disabled by setting `:rewindable` to false, in which case the store
|
|
24
|
+
# is an in-memory buffer that discards read content, eliminating disk I/O.
|
|
25
|
+
#
|
|
26
|
+
# Any cleanup code (i.e. ensure block) that the given enumerator carries is
|
|
27
|
+
# guaranteed to get executed, either when all content has been retrieved or
|
|
28
|
+
# when Down::ChunkedIO is closed. One can also specify an `:on_close`
|
|
29
|
+
# callback that will also get executed in those situations.
|
|
30
|
+
class ChunkedIO
|
|
31
|
+
attr_accessor size: Integer?
|
|
32
|
+
|
|
33
|
+
attr_accessor data: Hash[Symbol, untyped]
|
|
34
|
+
|
|
35
|
+
attr_reader encoding: Encoding
|
|
36
|
+
|
|
37
|
+
@chunks_fiber: Fiber?
|
|
38
|
+
|
|
39
|
+
@trimmed_bytes: Integer
|
|
40
|
+
|
|
41
|
+
@stored_bytes: Integer
|
|
42
|
+
|
|
43
|
+
@store: (File | StringIO)?
|
|
44
|
+
|
|
45
|
+
@closed: bool
|
|
46
|
+
|
|
47
|
+
@next_chunk: String?
|
|
48
|
+
|
|
49
|
+
@rewindable: bool
|
|
50
|
+
|
|
51
|
+
@on_close: (^() -> void | (_Callable & Object))?
|
|
52
|
+
|
|
53
|
+
@chunks: _Each[String] & Object
|
|
54
|
+
|
|
55
|
+
# : (chunks: _Each[String] & Object, ?size: Integer?, ?on_close: (^() -> void | (_Callable & Object))?, ?data: Hash[Symbol, untyped], ?rewindable: bool, ?encoding: (String | Encoding)?) -> void
|
|
56
|
+
def initialize: (chunks: _Each[String] & Object, ?size: Integer?, ?on_close: (^() -> void | (_Callable & Object))?, ?data: Hash[Symbol, untyped], ?rewindable: bool, ?encoding: (String | Encoding)?) -> void
|
|
57
|
+
|
|
58
|
+
# Sets the encoding that read content is returned in.
|
|
59
|
+
# : (String | Encoding encoding) -> void
|
|
60
|
+
def encoding=: (String | Encoding encoding) -> void
|
|
61
|
+
|
|
62
|
+
# For compatibility with multipart-post gem.
|
|
63
|
+
# : () -> Integer?
|
|
64
|
+
def length: () -> Integer?
|
|
65
|
+
|
|
66
|
+
# Yields elements of the underlying enumerator.
|
|
67
|
+
# : () { (String) -> void } -> void
|
|
68
|
+
# : () -> Enumerator[String, void]
|
|
69
|
+
def each_chunk: () { (String) -> void } -> void
|
|
70
|
+
| () -> Enumerator[String, void]
|
|
71
|
+
|
|
72
|
+
# Implements IO#read semantics. Without arguments it retrieves and returns
|
|
73
|
+
# all content in the external encoding.
|
|
74
|
+
#
|
|
75
|
+
# With `length` argument returns exactly that number of bytes if they're
|
|
76
|
+
# available, as a binary string.
|
|
77
|
+
#
|
|
78
|
+
# With `outbuf` argument each call will return that same string object,
|
|
79
|
+
# where the value is replaced with retrieved content.
|
|
80
|
+
#
|
|
81
|
+
# If end of file is reached, returns empty string if called without
|
|
82
|
+
# arguments, or nil if called with arguments. Raises IOError if closed.
|
|
83
|
+
# : (?nil, ?string? outbuf) -> String
|
|
84
|
+
# : (?int? length, ?string? outbuf) -> String?
|
|
85
|
+
def read: (?nil, ?string? outbuf) -> String
|
|
86
|
+
| (?int? length, ?string? outbuf) -> String?
|
|
87
|
+
|
|
88
|
+
# Implements IO#gets semantics. Without arguments it retrieves lines of
|
|
89
|
+
# content separated by newlines.
|
|
90
|
+
#
|
|
91
|
+
# With `separator` argument it does the following:
|
|
92
|
+
#
|
|
93
|
+
# * if `separator` is a nonempty string returns chunks of content
|
|
94
|
+
# surrounded with that sequence of bytes
|
|
95
|
+
# * if `separator` is an empty string returns paragraphs of content
|
|
96
|
+
# (content delimited by two or more newlines, which are skipped)
|
|
97
|
+
# * if `separator` is nil returns all content
|
|
98
|
+
#
|
|
99
|
+
# With `limit` argument returns maximum of that amount of bytes, plus any
|
|
100
|
+
# bytes needed to complete a multibyte character.
|
|
101
|
+
#
|
|
102
|
+
# With `chomp: true` removes the trailing separator from the line.
|
|
103
|
+
#
|
|
104
|
+
# Returns nil if end of file is reached. Raises IOError if closed.
|
|
105
|
+
# : (?String? separator, ?Integer? limit, ?chomp: boolish) -> String?
|
|
106
|
+
# : (Integer limit, ?chomp: boolish) -> String?
|
|
107
|
+
def gets: (?String? separator, ?Integer? limit, ?chomp: boolish) -> String?
|
|
108
|
+
| (Integer limit, ?chomp: boolish) -> String?
|
|
109
|
+
|
|
110
|
+
# Implements IO#readpartial semantics. If there is any content readily
|
|
111
|
+
# available reads from it, otherwise retrieves the next chunk and reads
|
|
112
|
+
# from it.
|
|
113
|
+
#
|
|
114
|
+
# With `maxlen` argument returns maximum of that amount of bytes (default
|
|
115
|
+
# is 16KB).
|
|
116
|
+
#
|
|
117
|
+
# With `outbuf` argument each call will return that same string object,
|
|
118
|
+
# where the value is replaced with retrieved content.
|
|
119
|
+
#
|
|
120
|
+
# Raises EOFError if end of file is reached. Raises IOError if closed.
|
|
121
|
+
# : (?int? maxlen, ?string? outbuf) -> String
|
|
122
|
+
def readpartial: (?int? maxlen, ?string? outbuf) -> String
|
|
123
|
+
|
|
124
|
+
# Implements IO#seek semantics. It retrieves content up to the target
|
|
125
|
+
# position. Raises Errno::ESPIPE if not rewindable.
|
|
126
|
+
# : (int amount, ?(Integer | Symbol) whence) -> Integer
|
|
127
|
+
def seek: (int amount, ?Integer | Symbol whence) -> Integer
|
|
128
|
+
|
|
129
|
+
# Implements IO#pos semantics. Returns the current position of the
|
|
130
|
+
# Down::ChunkedIO. Raises IOError if closed.
|
|
131
|
+
# : () -> Integer
|
|
132
|
+
def pos: () -> Integer
|
|
133
|
+
|
|
134
|
+
alias tell pos
|
|
135
|
+
|
|
136
|
+
# Implements IO#eof? semantics. Returns whether we've reached end of file.
|
|
137
|
+
# It returns true if the store is at the end and there is no more content
|
|
138
|
+
# to retrieve. Raises IOError if closed.
|
|
139
|
+
# : () -> bool
|
|
140
|
+
def eof?: () -> bool
|
|
141
|
+
|
|
142
|
+
alias eof eof?
|
|
143
|
+
|
|
144
|
+
# Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by setting
|
|
145
|
+
# the position to the beginning of the file. Raises IOError if closed or
|
|
146
|
+
# not rewindable.
|
|
147
|
+
# : () -> Integer
|
|
148
|
+
def rewind: () -> Integer
|
|
149
|
+
|
|
150
|
+
# Implements IO#close semantics. Closes the Down::ChunkedIO by terminating
|
|
151
|
+
# chunk retrieval and deleting the retrieved content.
|
|
152
|
+
# : () -> void
|
|
153
|
+
def close: () -> void
|
|
154
|
+
|
|
155
|
+
# Returns whether the Down::ChunkedIO has been closed.
|
|
156
|
+
# : () -> bool
|
|
157
|
+
def closed?: () -> bool
|
|
158
|
+
|
|
159
|
+
# Returns whether the Down::ChunkedIO was specified as rewindable.
|
|
160
|
+
# : () -> bool
|
|
161
|
+
def rewindable?: () -> bool
|
|
162
|
+
|
|
163
|
+
# Returns useful information about the Down::ChunkedIO object.
|
|
164
|
+
# : () -> String
|
|
165
|
+
def inspect: () -> String
|
|
166
|
+
|
|
167
|
+
private
|
|
168
|
+
|
|
169
|
+
# Returns the IO object that retrieved content is appended to and read
|
|
170
|
+
# from. If Down::ChunkedIO is specified as rewindable, it's an anonymous
|
|
171
|
+
# tempfile which keeps all retrieved content, otherwise it's a StringIO
|
|
172
|
+
# from which read content gets discarded.
|
|
173
|
+
# @rbs %a{pure}
|
|
174
|
+
# : () -> (File | StringIO)
|
|
175
|
+
%a{pure}
|
|
176
|
+
def store: () -> (File | StringIO)
|
|
177
|
+
|
|
178
|
+
# Reads a line from the store, retrieving more content if the line might
|
|
179
|
+
# continue in content that wasn't retrieved yet.
|
|
180
|
+
# : (String? separator, Integer? limit, boolish chomp) -> String?
|
|
181
|
+
def read_line: (String? separator, Integer? limit, boolish chomp) -> String?
|
|
182
|
+
|
|
183
|
+
# Reads a paragraph from a StringIO store. StringIO's paragraph mode
|
|
184
|
+
# differs from IO's in how it applies the limit and chomps lines, so we
|
|
185
|
+
# implement it on top of the double-newline separator, like IO does.
|
|
186
|
+
# : (Integer? limit, boolish chomp) -> String?
|
|
187
|
+
def read_paragraph: (Integer? limit, boolish chomp) -> String?
|
|
188
|
+
|
|
189
|
+
# Moves the store position past any consecutive newlines, retrieving more
|
|
190
|
+
# content as needed.
|
|
191
|
+
# : () -> void
|
|
192
|
+
def skip_newlines: () -> void
|
|
193
|
+
|
|
194
|
+
# Returns whether the line just read from the given start position of a
|
|
195
|
+
# StringIO store was truncated. A line read with a positive limit that
|
|
196
|
+
# stops before the end of the store must reach the limit or end with the
|
|
197
|
+
# separator, which JRuby's StringIO violates when the limit cuts a
|
|
198
|
+
# multibyte character that's incomplete at the end of the store.
|
|
199
|
+
# : (String? separator, Integer? limit, Integer start) -> bool
|
|
200
|
+
def truncated_line?: (String? separator, Integer? limit, Integer start) -> bool
|
|
201
|
+
|
|
202
|
+
# Returns whether the line read from the given start position of the store
|
|
203
|
+
# can't continue in content that wasn't retrieved yet, which is when it
|
|
204
|
+
# ends with the separator or reaches the limit with a complete character.
|
|
205
|
+
# In paragraph mode it's never complete, as newlines after it are skipped.
|
|
206
|
+
# : (String? line, String? separator, Integer? limit, Integer start) -> bool
|
|
207
|
+
def line_complete?: (String? line, String? separator, Integer? limit, Integer start) -> bool
|
|
208
|
+
|
|
209
|
+
# Returns the given number of bytes stored at the given store position.
|
|
210
|
+
# : (Integer position, Integer length) -> String
|
|
211
|
+
def store_bytes: (Integer position, Integer length) -> String
|
|
212
|
+
|
|
213
|
+
# Retrieves the next chunk and appends it to the store, without changing
|
|
214
|
+
# the store position. Returns false if there are no more chunks.
|
|
215
|
+
# : () -> bool
|
|
216
|
+
def fill_chunk: () -> bool
|
|
217
|
+
|
|
218
|
+
# Retrieves chunks until the store holds content up to the given
|
|
219
|
+
# position, or until there are no more chunks.
|
|
220
|
+
# : (Integer position) -> void
|
|
221
|
+
def fill_until: (Integer position) -> void
|
|
222
|
+
|
|
223
|
+
# Retrieves all remaining chunks.
|
|
224
|
+
# : () -> void
|
|
225
|
+
def fill_all: () -> void
|
|
226
|
+
|
|
227
|
+
# Returns current chunk and retrieves the next chunk, skipping empty ones.
|
|
228
|
+
# If there are no more chunks, we know we've reached EOF.
|
|
229
|
+
# : () -> String?
|
|
230
|
+
def retrieve_chunk: () -> String?
|
|
231
|
+
|
|
232
|
+
# Returns whether there is any content left to retrieve.
|
|
233
|
+
# : () -> bool
|
|
234
|
+
def chunks_depleted?: () -> bool
|
|
235
|
+
|
|
236
|
+
# Creates a Fiber wrapper around the underlying enumerator. The advantage
|
|
237
|
+
# of using a Fiber here is that we can terminate the chunk retrieval, in a
|
|
238
|
+
# way that executes any cleanup code that the enumerator potentially
|
|
239
|
+
# carries. At the end of iteration the :on_close callback is executed.
|
|
240
|
+
# : () -> Fiber
|
|
241
|
+
def chunks_fiber: () -> Fiber
|
|
242
|
+
|
|
243
|
+
# Finds encoding by name. If the encoding couldn't be find, falls back to
|
|
244
|
+
# the generic binary encoding.
|
|
245
|
+
# : (String | Encoding encoding) -> Encoding
|
|
246
|
+
def find_encoding: (String | Encoding encoding) -> Encoding
|
|
247
|
+
end
|
|
248
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Generated from lib/down/errors.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# generic error which is a superclass to all other errors
|
|
5
|
+
class Error < StandardError
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
# raised when the file is larger than the specified maximum size
|
|
9
|
+
class TooLarge < Error
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# raised when the given URL couldn't be parsed
|
|
13
|
+
class InvalidUrl < Error
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# raised when the number of redirects was larger than the specified maximum
|
|
17
|
+
class TooManyRedirects < Error
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# raised when the requested resource has not been modified
|
|
21
|
+
class NotModified < Error
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# raised when response returned 4xx or 5xx response
|
|
25
|
+
class ResponseError < Error
|
|
26
|
+
# The backend-specific response object (e.g. Net::HTTPResponse, HTTP::Response, HTTPX::Response).
|
|
27
|
+
attr_reader response: untyped
|
|
28
|
+
|
|
29
|
+
# : (String message, ?untyped response) -> void
|
|
30
|
+
def initialize: (String message, ?untyped response) -> void
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# raised when response returned 4xx response
|
|
34
|
+
class ClientError < ResponseError
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# raised when response returned 404 response
|
|
38
|
+
class NotFound < ClientError
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# raised when response returned 5xx response
|
|
42
|
+
class ServerError < ResponseError
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# raised when there was an error connecting to the server
|
|
46
|
+
class ConnectionError < Error
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# raised when connecting to the server too longer than the specified timeout
|
|
50
|
+
class TimeoutError < ConnectionError
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# raised when an SSL error was raised
|
|
54
|
+
class SSLError < Error
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Generated from lib/down/http.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# Provides streaming downloads implemented with HTTP.rb.
|
|
5
|
+
class Http < Backend
|
|
6
|
+
@method: HTTP::verb | String
|
|
7
|
+
|
|
8
|
+
@client: HTTP::Session | HTTP::Client
|
|
9
|
+
|
|
10
|
+
# Initializes the backend with common defaults.
|
|
11
|
+
# : (?method: HTTP::verb | String, **untyped options) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> void
|
|
12
|
+
def initialize: (?method: HTTP::verb | String, **untyped options) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> void
|
|
13
|
+
|
|
14
|
+
# Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
|
|
15
|
+
# block, and some additional options as well.
|
|
16
|
+
# : (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> nil
|
|
17
|
+
# : (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> downloaded_file
|
|
18
|
+
def download: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> nil
|
|
19
|
+
| (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> downloaded_file
|
|
20
|
+
|
|
21
|
+
# Starts retrieving the remote file and returns an IO-like object which
|
|
22
|
+
# downloads the response body on-demand. Accepts HTTP.rb options via a hash
|
|
23
|
+
# or a block.
|
|
24
|
+
# : (url url, ?rewindable: bool, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> ChunkedIO
|
|
25
|
+
def open: (url url, ?rewindable: bool, **untyped) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> ChunkedIO
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# : (url url, ?method: HTTP::verb | String, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
30
|
+
def request: (url url, ?method: HTTP::verb | String, **untyped) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
31
|
+
|
|
32
|
+
# : (HTTP::verb | String method, url url, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
33
|
+
def send_request: (HTTP::verb | String method, url url, **untyped) ?{ (HTTP::Session | HTTP::Client) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
34
|
+
|
|
35
|
+
# Yields chunks of the response body to the block.
|
|
36
|
+
# : (HTTP::Response response) { (String) -> void } -> void
|
|
37
|
+
def stream_body: (HTTP::Response response) { (String) -> void } -> void
|
|
38
|
+
|
|
39
|
+
# Raises non-sucessful response as a Down::ResponseError.
|
|
40
|
+
# : (HTTP::Response response) -> bot
|
|
41
|
+
def response_error!: (HTTP::Response response) -> bot
|
|
42
|
+
|
|
43
|
+
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
44
|
+
# : (Exception exception) -> bot
|
|
45
|
+
def request_error!: (Exception exception) -> bot
|
|
46
|
+
|
|
47
|
+
# : () -> Array[singleton(Exception)]
|
|
48
|
+
def invalid_url_errors: () -> Array[singleton(Exception)]
|
|
49
|
+
|
|
50
|
+
# Defines some additional attributes for the returned Tempfile.
|
|
51
|
+
module DownloadedFile
|
|
52
|
+
attr_accessor url: String
|
|
53
|
+
|
|
54
|
+
attr_accessor headers: Hash[String, untyped]
|
|
55
|
+
|
|
56
|
+
# : () -> String?
|
|
57
|
+
def original_filename: () -> String?
|
|
58
|
+
|
|
59
|
+
# : () -> String?
|
|
60
|
+
def content_type: () -> String?
|
|
61
|
+
|
|
62
|
+
# : () -> String?
|
|
63
|
+
def charset: () -> String?
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Generated from lib/down/httpx.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# Provides streaming downloads implemented with HTTPX.
|
|
5
|
+
class Httpx < Backend
|
|
6
|
+
USER_AGENT: ::String
|
|
7
|
+
|
|
8
|
+
@method: Symbol | String
|
|
9
|
+
|
|
10
|
+
@client: untyped
|
|
11
|
+
|
|
12
|
+
BASIC_AUTH: Symbol
|
|
13
|
+
|
|
14
|
+
# Initializes the backend with common defaults.
|
|
15
|
+
# : (?method: Symbol | String, ?headers: Hash[String | Symbol, String], **untyped options) ?{ (untyped) -> untyped } -> void
|
|
16
|
+
def initialize: (?method: Symbol | String, ?headers: Hash[String | Symbol, String], **untyped options) ?{ (untyped) -> untyped } -> void
|
|
17
|
+
|
|
18
|
+
# Downloads the remote file to disk. Accepts HTTPX options via a hash or a
|
|
19
|
+
# block, and some additional options as well.
|
|
20
|
+
# : (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
21
|
+
# : (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
22
|
+
def download: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
23
|
+
| (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
24
|
+
|
|
25
|
+
# Starts retrieving the remote file and returns an IO-like object which
|
|
26
|
+
# downloads the response body on-demand. Accepts HTTPX options via a hash
|
|
27
|
+
# or a block.
|
|
28
|
+
# : (url url, ?rewindable: bool, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
29
|
+
def open: (url url, ?rewindable: bool, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Yields chunks of the response body to the block.
|
|
34
|
+
# : (untyped response) { (String) -> void } -> void
|
|
35
|
+
def stream_body: (untyped response) { (String) -> void } -> void
|
|
36
|
+
|
|
37
|
+
# : (url url, ?method: Symbol | String, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
38
|
+
def request: (url url, ?method: Symbol | String, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
39
|
+
|
|
40
|
+
# : (Symbol | String method, url url, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
41
|
+
def send_request: (Symbol | String method, url url, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
42
|
+
|
|
43
|
+
# Raises non-sucessful response as a Down::ResponseError.
|
|
44
|
+
# : (untyped response) -> bot
|
|
45
|
+
def response_error!: (untyped response) -> bot
|
|
46
|
+
|
|
47
|
+
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
48
|
+
# : (Exception exception) -> bot
|
|
49
|
+
def request_error!: (Exception exception) -> bot
|
|
50
|
+
|
|
51
|
+
# Defines some additional attributes for the returned Tempfile.
|
|
52
|
+
module DownloadedFile
|
|
53
|
+
attr_accessor url: String
|
|
54
|
+
|
|
55
|
+
attr_accessor headers: Hash[String, untyped]
|
|
56
|
+
|
|
57
|
+
attr_accessor charset: Encoding?
|
|
58
|
+
|
|
59
|
+
attr_accessor content_type: String?
|
|
60
|
+
|
|
61
|
+
# : () -> String?
|
|
62
|
+
def original_filename: () -> String?
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# Generated from lib/down/net_http.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
# Provides streaming downloads implemented with Net::HTTP and open-uri.
|
|
5
|
+
class NetHttp < Backend
|
|
6
|
+
@options: Hash[Symbol, untyped]
|
|
7
|
+
|
|
8
|
+
URI_NORMALIZER: ^(String) -> String
|
|
9
|
+
|
|
10
|
+
# Content encodings that Net::HTTP transparently decodes.
|
|
11
|
+
DECODED_ENCODINGS: Array[String]
|
|
12
|
+
|
|
13
|
+
DEFAULT_OPTIONS: Hash[Symbol, untyped]
|
|
14
|
+
|
|
15
|
+
# Initializes the backend with common defaults.
|
|
16
|
+
# : (**untyped options) -> void
|
|
17
|
+
def initialize: (**untyped options) -> void
|
|
18
|
+
|
|
19
|
+
# Downloads a remote file to disk using open-uri. Accepts any open-uri
|
|
20
|
+
# options, and a few more.
|
|
21
|
+
# : (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
22
|
+
# : (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
23
|
+
def download: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
24
|
+
| (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
25
|
+
|
|
26
|
+
# Starts retrieving the remote file using Net::HTTP and returns an IO-like
|
|
27
|
+
# object which downloads the response body on-demand.
|
|
28
|
+
# : (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
29
|
+
def open: (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
34
|
+
# Receives String-keyed header kwargs alongside the options, which RBS keyword types can't express.
|
|
35
|
+
# : (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
36
|
+
def open_uri: (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
37
|
+
|
|
38
|
+
# Adds name/value pairs from the given Set-Cookie header values to the
|
|
39
|
+
# Cookie request header, ignoring any cookie attributes.
|
|
40
|
+
# : (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
41
|
+
def forward_cookies: (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
42
|
+
|
|
43
|
+
# Parses the Cookie request header value into a hash.
|
|
44
|
+
# : (String? value) -> Hash[String, String?]
|
|
45
|
+
def parse_cookie_header: (String? value) -> Hash[String, String?]
|
|
46
|
+
|
|
47
|
+
# Returns the Cookie request header value from the given headers.
|
|
48
|
+
# : (Hash[untyped, untyped] headers) -> String?
|
|
49
|
+
def cookie_header: (Hash[untyped, untyped] headers) -> String?
|
|
50
|
+
|
|
51
|
+
# Sets the Cookie request header value in the given headers, removing it
|
|
52
|
+
# if the value is nil or empty.
|
|
53
|
+
# : (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
54
|
+
def set_cookie_header: (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
55
|
+
|
|
56
|
+
# Returns whether the given header name is the Cookie header.
|
|
57
|
+
# : (untyped name) -> bool
|
|
58
|
+
def cookie_header?: (untyped name) -> bool
|
|
59
|
+
|
|
60
|
+
# Returns whether the given URIs have the same scheme, host and port.
|
|
61
|
+
# : (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
62
|
+
def same_origin?: (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
63
|
+
|
|
64
|
+
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
65
|
+
# returns a StringIO when there is less than 10KB of content), and gives
|
|
66
|
+
# it the specified file extension.
|
|
67
|
+
# : (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
68
|
+
def ensure_tempfile: (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
69
|
+
|
|
70
|
+
# Makes a Net::HTTP request and follows redirects.
|
|
71
|
+
# : (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
72
|
+
def net_http_request: (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
73
|
+
|
|
74
|
+
# Build a Net::HTTP object for making a request.
|
|
75
|
+
# : (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [String?, String?]?, **untyped) -> [Net::HTTP, Net::HTTP::Get]
|
|
76
|
+
def create_net_http: (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [ String?, String? ]?, **untyped) -> [ Net::HTTP, Net::HTTP::Get ]
|
|
77
|
+
|
|
78
|
+
# Yields chunks of the response body to the block.
|
|
79
|
+
# : (Net::HTTPResponse response) { (String) -> void } -> void
|
|
80
|
+
def stream_body: (Net::HTTPResponse response) { (String) -> void } -> void
|
|
81
|
+
|
|
82
|
+
# Returns the Content-Length, unless a Transfer-Encoding is present or
|
|
83
|
+
# Net::HTTP will decode the body, in which case it doesn't reflect the
|
|
84
|
+
# number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
|
|
85
|
+
# : (Net::HTTPResponse response) -> Integer?
|
|
86
|
+
def content_length: (Net::HTTPResponse response) -> Integer?
|
|
87
|
+
|
|
88
|
+
# Checks that the url is a valid URI and that its scheme is http or https.
|
|
89
|
+
# : (untyped url) -> URI::HTTP
|
|
90
|
+
# : (untyped url, allow_relative: bool) -> URI::Generic
|
|
91
|
+
def ensure_uri: (untyped url) -> URI::HTTP
|
|
92
|
+
| (untyped url, allow_relative: bool) -> URI::Generic
|
|
93
|
+
|
|
94
|
+
# Makes sure that the URL is properly encoded.
|
|
95
|
+
# : (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
96
|
+
def normalize_uri: (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
97
|
+
|
|
98
|
+
# When open-uri raises an exception, it doesn't expose the response object.
|
|
99
|
+
# Fortunately, the exception object holds response data that can be used to
|
|
100
|
+
# rebuild the Net::HTTP response object.
|
|
101
|
+
# : (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
102
|
+
def rebuild_response_from_open_uri_exception: (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
103
|
+
|
|
104
|
+
# Raises non-sucessful response as a Down::ResponseError.
|
|
105
|
+
# : (Net::HTTPResponse response) -> bot
|
|
106
|
+
def response_error!: (Net::HTTPResponse response) -> bot
|
|
107
|
+
|
|
108
|
+
# Re-raise Net::HTTP exceptions as Down::Error exceptions.
|
|
109
|
+
# : (Exception exception) -> bot
|
|
110
|
+
def request_error!: (Exception exception) -> bot
|
|
111
|
+
|
|
112
|
+
# Merge default and ad-hoc options, merging nested headers.
|
|
113
|
+
# : (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
114
|
+
def merge_options: (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
115
|
+
|
|
116
|
+
# Defines some additional attributes for the returned Tempfile (on top of what
|
|
117
|
+
# OpenURI::Meta already defines).
|
|
118
|
+
# @rbs module-self OpenURI::Meta
|
|
119
|
+
module DownloadedFile : OpenURI::Meta
|
|
120
|
+
# : () -> String?
|
|
121
|
+
def original_filename: () -> String?
|
|
122
|
+
|
|
123
|
+
# : () -> String?
|
|
124
|
+
def content_type: () -> String?
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Generated from lib/down/utils.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module Down
|
|
4
|
+
module Utils
|
|
5
|
+
# Retrieves potential filename from the "Content-Disposition" header.
|
|
6
|
+
# : (_ToS content_disposition) -> String?
|
|
7
|
+
def self?.filename_from_content_disposition: (_ToS content_disposition) -> String?
|
|
8
|
+
|
|
9
|
+
# Retrieves potential filename from the URL path.
|
|
10
|
+
# : (String path) -> String?
|
|
11
|
+
def self?.filename_from_path: (String path) -> String?
|
|
12
|
+
end
|
|
13
|
+
end
|