down 5.6.3 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +62 -33
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +32 -2
- data/lib/down/wget.rb +0 -250
data/lib/down/chunked_io.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen-string-literal: true
|
|
2
2
|
|
|
3
3
|
require "tempfile"
|
|
4
|
+
require "stringio"
|
|
4
5
|
require "fiber"
|
|
5
6
|
|
|
6
7
|
module Down
|
|
@@ -12,78 +13,113 @@ module Down
|
|
|
12
13
|
# * IO#gets
|
|
13
14
|
# * IO#size
|
|
14
15
|
# * IO#pos
|
|
16
|
+
# * IO#seek
|
|
15
17
|
# * IO#eof?
|
|
16
18
|
# * IO#rewind
|
|
17
19
|
# * IO#close
|
|
18
20
|
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
21
|
+
# Retrieved chunks are appended to a backing store which the reading methods
|
|
22
|
+
# delegate to, so they behave like the corresponding native IO methods. New
|
|
23
|
+
# chunks are retrieved only when the store runs out of content.
|
|
24
|
+
#
|
|
25
|
+
# By default the store is a tempfile which keeps all read content, allowing
|
|
26
|
+
# the Down::ChunkedIO to be rewindable. If rewindability won't be used, it
|
|
27
|
+
# can be disabled by setting `:rewindable` to false, in which case the store
|
|
28
|
+
# is an in-memory buffer that discards read content, eliminating disk I/O.
|
|
22
29
|
#
|
|
23
30
|
# Any cleanup code (i.e. ensure block) that the given enumerator carries is
|
|
24
31
|
# guaranteed to get executed, either when all content has been retrieved or
|
|
25
32
|
# when Down::ChunkedIO is closed. One can also specify an `:on_close`
|
|
26
33
|
# callback that will also get executed in those situations.
|
|
27
34
|
class ChunkedIO
|
|
28
|
-
attr_accessor :size
|
|
29
|
-
|
|
35
|
+
attr_accessor :size #: Integer?
|
|
36
|
+
attr_accessor :data #: Hash[Symbol, untyped]
|
|
37
|
+
attr_reader :encoding #: Encoding
|
|
38
|
+
|
|
39
|
+
# @rbs @chunks: _Each[String] & Object
|
|
40
|
+
# @rbs @on_close: (^() -> void | (_Callable & Object))?
|
|
41
|
+
# @rbs @rewindable: bool
|
|
42
|
+
# @rbs @next_chunk: String?
|
|
43
|
+
# @rbs @closed: bool
|
|
44
|
+
# @rbs @store: (File | StringIO)?
|
|
45
|
+
# @rbs @stored_bytes: Integer
|
|
46
|
+
# @rbs @trimmed_bytes: Integer
|
|
47
|
+
# @rbs @chunks_fiber: Fiber?
|
|
48
|
+
|
|
49
|
+
#: (chunks: _Each[String] & Object, ?size: Integer?, ?on_close: (^() -> void | (_Callable & Object))?, ?data: Hash[Symbol, untyped], ?rewindable: bool, ?encoding: (String | Encoding)?) -> void
|
|
30
50
|
def initialize(chunks:, size: nil, on_close: nil, data: {}, rewindable: true, encoding: nil)
|
|
31
|
-
@chunks
|
|
32
|
-
@size
|
|
33
|
-
@on_close
|
|
34
|
-
@data
|
|
35
|
-
@encoding
|
|
36
|
-
@rewindable
|
|
37
|
-
@
|
|
38
|
-
@
|
|
39
|
-
@
|
|
40
|
-
@
|
|
51
|
+
@chunks = chunks
|
|
52
|
+
@size = size
|
|
53
|
+
@on_close = on_close
|
|
54
|
+
@data = data
|
|
55
|
+
@encoding = find_encoding(encoding || "binary")
|
|
56
|
+
@rewindable = rewindable
|
|
57
|
+
@next_chunk = nil
|
|
58
|
+
@closed = false
|
|
59
|
+
@store = nil
|
|
60
|
+
@stored_bytes = 0 # number of bytes appended to the store
|
|
61
|
+
@trimmed_bytes = 0 # number of read bytes discarded from the store
|
|
41
62
|
|
|
42
63
|
retrieve_chunk # fetch first chunk so that we know whether the file is empty
|
|
43
64
|
end
|
|
44
65
|
|
|
66
|
+
# Sets the encoding that read content is returned in.
|
|
67
|
+
#: (String | Encoding encoding) -> void
|
|
68
|
+
def encoding=(encoding)
|
|
69
|
+
@encoding = find_encoding(encoding)
|
|
70
|
+
@store&.set_encoding(@encoding, @encoding)
|
|
71
|
+
end
|
|
72
|
+
|
|
45
73
|
# For compatibility with multipart-post gem.
|
|
74
|
+
#: () -> Integer?
|
|
46
75
|
def length
|
|
47
76
|
@size
|
|
48
77
|
end
|
|
49
78
|
|
|
50
79
|
# Yields elements of the underlying enumerator.
|
|
80
|
+
#: () { (String) -> void } -> void
|
|
81
|
+
#: () -> Enumerator[String, void]
|
|
51
82
|
def each_chunk
|
|
52
83
|
fail IOError, "closed stream" if closed?
|
|
53
84
|
|
|
54
|
-
return enum_for(
|
|
85
|
+
return enum_for(:each_chunk) unless block_given?
|
|
55
86
|
|
|
56
|
-
|
|
87
|
+
until chunks_depleted?
|
|
88
|
+
chunk = retrieve_chunk #: String
|
|
89
|
+
yield chunk
|
|
90
|
+
end
|
|
57
91
|
end
|
|
58
92
|
|
|
59
93
|
# Implements IO#read semantics. Without arguments it retrieves and returns
|
|
60
|
-
# all content.
|
|
94
|
+
# all content in the external encoding.
|
|
61
95
|
#
|
|
62
96
|
# With `length` argument returns exactly that number of bytes if they're
|
|
63
|
-
# available.
|
|
97
|
+
# available, as a binary string.
|
|
64
98
|
#
|
|
65
99
|
# With `outbuf` argument each call will return that same string object,
|
|
66
100
|
# where the value is replaced with retrieved content.
|
|
67
101
|
#
|
|
68
102
|
# If end of file is reached, returns empty string if called without
|
|
69
103
|
# arguments, or nil if called with arguments. Raises IOError if closed.
|
|
104
|
+
#: (?nil, ?string? outbuf) -> String
|
|
105
|
+
#: (?int? length, ?string? outbuf) -> String?
|
|
70
106
|
def read(length = nil, outbuf = nil)
|
|
71
107
|
fail IOError, "closed stream" if closed?
|
|
72
108
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
remaining_length = length
|
|
109
|
+
length = length&.to_int # implicit conversion, like IO#read
|
|
110
|
+
outbuf = outbuf&.to_str
|
|
77
111
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
112
|
+
if length
|
|
113
|
+
fill_until(pos + length)
|
|
114
|
+
else
|
|
115
|
+
fill_all
|
|
81
116
|
end
|
|
82
117
|
|
|
83
|
-
|
|
118
|
+
encoding = outbuf&.encoding if length
|
|
119
|
+
data = store.read(length, outbuf)
|
|
120
|
+
data&.force_encoding(encoding) if encoding # StringIO < 3.1.2 changes it to binary
|
|
84
121
|
|
|
85
|
-
data
|
|
86
|
-
data unless data.empty? && length && length > 0
|
|
122
|
+
data
|
|
87
123
|
end
|
|
88
124
|
|
|
89
125
|
# Implements IO#gets semantics. Without arguments it retrieves lines of
|
|
@@ -94,64 +130,35 @@ module Down
|
|
|
94
130
|
# * if `separator` is a nonempty string returns chunks of content
|
|
95
131
|
# surrounded with that sequence of bytes
|
|
96
132
|
# * if `separator` is an empty string returns paragraphs of content
|
|
97
|
-
# (content delimited by two newlines)
|
|
98
|
-
# * if `separator` is nil
|
|
133
|
+
# (content delimited by two or more newlines, which are skipped)
|
|
134
|
+
# * if `separator` is nil returns all content
|
|
99
135
|
#
|
|
100
|
-
# With `limit` argument returns maximum of that amount of bytes
|
|
136
|
+
# With `limit` argument returns maximum of that amount of bytes, plus any
|
|
137
|
+
# bytes needed to complete a multibyte character.
|
|
138
|
+
#
|
|
139
|
+
# With `chomp: true` removes the trailing separator from the line.
|
|
101
140
|
#
|
|
102
141
|
# Returns nil if end of file is reached. Raises IOError if closed.
|
|
103
|
-
|
|
142
|
+
#: (?String? separator, ?Integer? limit, ?chomp: boolish) -> String?
|
|
143
|
+
#: (Integer limit, ?chomp: boolish) -> String?
|
|
144
|
+
def gets(separator_or_limit = $/, limit = nil, chomp: false)
|
|
104
145
|
fail IOError, "closed stream" if closed?
|
|
105
146
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
limit = separator_or_limit
|
|
109
|
-
else
|
|
110
|
-
separator = separator_or_limit
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
return read(limit)&.force_encoding(@encoding) if separator.nil?
|
|
114
|
-
|
|
115
|
-
separator = "\n\n" if separator.empty?
|
|
116
|
-
|
|
117
|
-
data = String.new
|
|
147
|
+
separator = separator_or_limit.is_a?(Integer) ? $/ : separator_or_limit
|
|
148
|
+
limit = separator_or_limit if separator_or_limit.is_a?(Integer)
|
|
118
149
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
buffer.clear if buffer # deallocate buffer
|
|
125
|
-
|
|
126
|
-
line, extra = data.split(separator, 2)
|
|
127
|
-
line << separator if data.include?(separator)
|
|
128
|
-
|
|
129
|
-
data.clear # deallocate data
|
|
130
|
-
|
|
131
|
-
if extra
|
|
132
|
-
@position -= extra.bytesize
|
|
133
|
-
|
|
134
|
-
if cache
|
|
135
|
-
cache.pos -= extra.bytesize
|
|
136
|
-
else
|
|
137
|
-
if @buffer
|
|
138
|
-
@buffer.prepend(extra)
|
|
139
|
-
else
|
|
140
|
-
@buffer = extra
|
|
141
|
-
end
|
|
142
|
-
end
|
|
150
|
+
if separator == "" && store.is_a?(StringIO)
|
|
151
|
+
line = read_paragraph(limit, chomp)
|
|
152
|
+
else
|
|
153
|
+
line = read_line(separator, limit, chomp)
|
|
143
154
|
end
|
|
144
155
|
|
|
145
|
-
line
|
|
156
|
+
line
|
|
146
157
|
end
|
|
147
158
|
|
|
148
159
|
# Implements IO#readpartial semantics. If there is any content readily
|
|
149
|
-
# available reads from it, otherwise
|
|
150
|
-
#
|
|
151
|
-
#
|
|
152
|
-
# Without arguments it either returns all content that's readily available,
|
|
153
|
-
# or the next chunk. This is useful when you don't care about the size of
|
|
154
|
-
# chunks and you want to minimize string allocations.
|
|
160
|
+
# available reads from it, otherwise retrieves the next chunk and reads
|
|
161
|
+
# from it.
|
|
155
162
|
#
|
|
156
163
|
# With `maxlen` argument returns maximum of that amount of bytes (default
|
|
157
164
|
# is 16KB).
|
|
@@ -160,126 +167,111 @@ module Down
|
|
|
160
167
|
# where the value is replaced with retrieved content.
|
|
161
168
|
#
|
|
162
169
|
# Raises EOFError if end of file is reached. Raises IOError if closed.
|
|
170
|
+
#: (?int? maxlen, ?string? outbuf) -> String
|
|
163
171
|
def readpartial(maxlen = nil, outbuf = nil)
|
|
164
172
|
fail IOError, "closed stream" if closed?
|
|
165
173
|
|
|
166
|
-
maxlen
|
|
167
|
-
|
|
168
|
-
data = cache.read(maxlen, outbuf) if cache && !cache.eof?
|
|
169
|
-
data ||= outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
|
|
170
|
-
data ||= "".b
|
|
174
|
+
maxlen = maxlen ? maxlen.to_int : 16*1024 # implicit conversion, like IO#readpartial
|
|
171
175
|
|
|
172
|
-
|
|
176
|
+
fill_chunk while maxlen > 0 && pos >= @stored_bytes && !chunks_depleted?
|
|
173
177
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
remaining_length = maxlen - data.bytesize
|
|
178
|
+
# Reading only stored content allocates a string of that size, instead
|
|
179
|
+
# of maxlen bytes, which also avoids reallocating a reused buffer.
|
|
180
|
+
maxlen = [maxlen, @stored_bytes - pos].min if pos < @stored_bytes
|
|
180
181
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
else
|
|
186
|
-
buffered_data = @buffer
|
|
187
|
-
@buffer = nil
|
|
188
|
-
end
|
|
182
|
+
outbuf = outbuf&.to_str
|
|
183
|
+
encoding = outbuf&.encoding
|
|
184
|
+
data = outbuf ? store.readpartial(maxlen, outbuf) : store.readpartial(maxlen) # JRuby rejects nil outbuf
|
|
185
|
+
data.force_encoding(encoding) if encoding # StringIO < 3.1.2 changes it to binary
|
|
189
186
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
cache.write(buffered_data) if cache
|
|
193
|
-
|
|
194
|
-
buffered_data.clear unless buffered_data.frozen?
|
|
195
|
-
end
|
|
196
|
-
|
|
197
|
-
@position += data.bytesize
|
|
198
|
-
|
|
199
|
-
data.force_encoding(Encoding::BINARY)
|
|
187
|
+
data
|
|
200
188
|
end
|
|
201
189
|
|
|
202
|
-
# Implements IO#seek semantics.
|
|
190
|
+
# Implements IO#seek semantics. It retrieves content up to the target
|
|
191
|
+
# position. Raises Errno::ESPIPE if not rewindable.
|
|
192
|
+
#: (int amount, ?(Integer | Symbol) whence) -> Integer
|
|
203
193
|
def seek(amount, whence = IO::SEEK_SET)
|
|
204
|
-
fail
|
|
194
|
+
fail IOError, "closed stream" if closed?
|
|
195
|
+
fail Errno::ESPIPE unless rewindable?
|
|
196
|
+
|
|
197
|
+
amount = amount.to_int
|
|
205
198
|
|
|
206
199
|
case whence
|
|
207
200
|
when IO::SEEK_SET, :SET
|
|
208
|
-
|
|
201
|
+
whence = IO::SEEK_SET
|
|
202
|
+
fill_until(amount)
|
|
209
203
|
when IO::SEEK_CUR, :CUR
|
|
210
|
-
|
|
204
|
+
whence = IO::SEEK_CUR
|
|
205
|
+
fill_until(pos + amount)
|
|
211
206
|
when IO::SEEK_END, :END
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
IO.copy_stream(self, File::NULL)
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
target_pos = cache.size + amount
|
|
207
|
+
whence = IO::SEEK_END
|
|
208
|
+
fill_all
|
|
218
209
|
else
|
|
219
210
|
fail ArgumentError, "invalid whence: #{whence.inspect}"
|
|
220
211
|
end
|
|
221
212
|
|
|
222
|
-
|
|
223
|
-
cache.seek(target_pos)
|
|
224
|
-
else
|
|
225
|
-
cache.seek(0, IO::SEEK_END)
|
|
226
|
-
IO.copy_stream(self, File::NULL, target_pos - cache.size)
|
|
227
|
-
end
|
|
228
|
-
|
|
229
|
-
@position = cache.pos
|
|
213
|
+
store.seek(amount, whence)
|
|
230
214
|
end
|
|
231
215
|
|
|
232
216
|
# Implements IO#pos semantics. Returns the current position of the
|
|
233
|
-
# Down::ChunkedIO.
|
|
217
|
+
# Down::ChunkedIO. Raises IOError if closed.
|
|
218
|
+
#: () -> Integer
|
|
234
219
|
def pos
|
|
235
|
-
|
|
220
|
+
fail IOError, "closed stream" if closed?
|
|
221
|
+
|
|
222
|
+
@trimmed_bytes + store.pos
|
|
236
223
|
end
|
|
237
224
|
alias tell pos
|
|
238
225
|
|
|
239
226
|
# Implements IO#eof? semantics. Returns whether we've reached end of file.
|
|
240
|
-
# It returns true if
|
|
241
|
-
# retrieve. Raises IOError if closed.
|
|
227
|
+
# It returns true if the store is at the end and there is no more content
|
|
228
|
+
# to retrieve. Raises IOError if closed.
|
|
229
|
+
#: () -> bool
|
|
242
230
|
def eof?
|
|
243
231
|
fail IOError, "closed stream" if closed?
|
|
244
232
|
|
|
245
|
-
|
|
246
|
-
@buffer.nil? && chunks_depleted?
|
|
233
|
+
pos >= @stored_bytes && chunks_depleted?
|
|
247
234
|
end
|
|
248
235
|
alias eof eof?
|
|
249
236
|
|
|
250
|
-
# Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by
|
|
251
|
-
# the
|
|
252
|
-
#
|
|
237
|
+
# Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by setting
|
|
238
|
+
# the position to the beginning of the file. Raises IOError if closed or
|
|
239
|
+
# not rewindable.
|
|
240
|
+
#: () -> Integer
|
|
253
241
|
def rewind
|
|
254
242
|
fail IOError, "closed stream" if closed?
|
|
255
|
-
fail IOError, "this Down::ChunkedIO is not rewindable"
|
|
243
|
+
fail IOError, "this Down::ChunkedIO is not rewindable" unless rewindable?
|
|
256
244
|
|
|
257
|
-
|
|
258
|
-
@position = 0
|
|
245
|
+
store.rewind
|
|
259
246
|
end
|
|
260
247
|
|
|
261
248
|
# Implements IO#close semantics. Closes the Down::ChunkedIO by terminating
|
|
262
|
-
# chunk retrieval and deleting the
|
|
249
|
+
# chunk retrieval and deleting the retrieved content.
|
|
250
|
+
#: () -> void
|
|
263
251
|
def close
|
|
264
252
|
return if @closed
|
|
265
253
|
|
|
266
254
|
chunks_fiber.resume(:terminate) if chunks_fiber.alive?
|
|
267
|
-
|
|
268
|
-
@
|
|
255
|
+
|
|
256
|
+
@store&.close
|
|
257
|
+
@store = nil
|
|
269
258
|
@closed = true
|
|
270
259
|
end
|
|
271
260
|
|
|
272
261
|
# Returns whether the Down::ChunkedIO has been closed.
|
|
262
|
+
#: () -> bool
|
|
273
263
|
def closed?
|
|
274
264
|
!!@closed
|
|
275
265
|
end
|
|
276
266
|
|
|
277
267
|
# Returns whether the Down::ChunkedIO was specified as rewindable.
|
|
268
|
+
#: () -> bool
|
|
278
269
|
def rewindable?
|
|
279
270
|
@rewindable
|
|
280
271
|
end
|
|
281
272
|
|
|
282
273
|
# Returns useful information about the Down::ChunkedIO object.
|
|
274
|
+
#: () -> String
|
|
283
275
|
def inspect
|
|
284
276
|
string = String.new
|
|
285
277
|
string << "#<#{self.class.name}"
|
|
@@ -295,28 +287,221 @@ module Down
|
|
|
295
287
|
|
|
296
288
|
private
|
|
297
289
|
|
|
298
|
-
#
|
|
299
|
-
#
|
|
300
|
-
|
|
301
|
-
|
|
290
|
+
# Returns the IO object that retrieved content is appended to and read
|
|
291
|
+
# from. If Down::ChunkedIO is specified as rewindable, it's an anonymous
|
|
292
|
+
# tempfile which keeps all retrieved content, otherwise it's a StringIO
|
|
293
|
+
# from which read content gets discarded.
|
|
294
|
+
# @rbs %a{pure}
|
|
295
|
+
#: () -> (File | StringIO)
|
|
296
|
+
def store
|
|
297
|
+
@store ||= (
|
|
298
|
+
if rewindable?
|
|
299
|
+
# the file is already deleted (or never named on Linux), so it gets
|
|
300
|
+
# cleaned up once closed, and no other process can open it
|
|
301
|
+
store = Tempfile.create("down-chunked_io", anonymous: true)
|
|
302
|
+
store.binmode # anonymous tempfiles ignore the :binmode option
|
|
303
|
+
else
|
|
304
|
+
store = StringIO.new(String.new(encoding: Encoding::BINARY))
|
|
305
|
+
end
|
|
302
306
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
tempfile.unlink if posix? # remove entry from filesystem if it's POSIX
|
|
307
|
-
tempfile
|
|
307
|
+
# passing the same internal encoding disables Encoding.default_internal
|
|
308
|
+
store.set_encoding(@encoding, @encoding)
|
|
309
|
+
store
|
|
308
310
|
)
|
|
309
311
|
end
|
|
310
312
|
|
|
311
|
-
#
|
|
312
|
-
#
|
|
313
|
+
# Reads a line from the store, retrieving more content if the line might
|
|
314
|
+
# continue in content that wasn't retrieved yet.
|
|
315
|
+
#: (String? separator, Integer? limit, boolish chomp) -> String?
|
|
316
|
+
def read_line(separator, limit, chomp)
|
|
317
|
+
start = store.pos
|
|
318
|
+
line = nil #: String?
|
|
319
|
+
|
|
320
|
+
loop do
|
|
321
|
+
line = store.gets(separator, limit, chomp:)
|
|
322
|
+
|
|
323
|
+
# JRuby's StringIO returns a truncated line and moves to a wrong
|
|
324
|
+
# position when the limit cuts a multibyte character that's incomplete
|
|
325
|
+
# at the end of the store. Like IO, we return the remaining content at
|
|
326
|
+
# the end of file, otherwise we retry with more content.
|
|
327
|
+
if RUBY_ENGINE == "jruby" && line && truncated_line?(separator, limit, start)
|
|
328
|
+
store.pos = start
|
|
329
|
+
line = chunks_depleted? ? store.read : nil
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
# If the store ran out of content, the line might continue in content
|
|
333
|
+
# that wasn't retrieved yet. In that case we retrieve more content and
|
|
334
|
+
# read the line again, at least doubling the content after the line
|
|
335
|
+
# start, so that lines spanning many chunks aren't re-read many times.
|
|
336
|
+
# StringIO doesn't move to the end when only newlines are left in
|
|
337
|
+
# paragraph mode, so we also retrieve more content when there's no line.
|
|
338
|
+
break if chunks_depleted?
|
|
339
|
+
break unless line.nil? || store.eof?
|
|
340
|
+
break if line_complete?(line, separator, limit, start)
|
|
341
|
+
|
|
342
|
+
store.pos = start # keeps the line start in the store when retrieving content
|
|
343
|
+
fill_until(@stored_bytes + [@stored_bytes - @trimmed_bytes - start, 1].max)
|
|
344
|
+
start = store.pos # retrieving content can discard content before it
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
store.seek(0, IO::SEEK_END) if line.nil? # skipped newlines in paragraph mode
|
|
348
|
+
|
|
349
|
+
line
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# Reads a paragraph from a StringIO store. StringIO's paragraph mode
|
|
353
|
+
# differs from IO's in how it applies the limit and chomps lines, so we
|
|
354
|
+
# implement it on top of the double-newline separator, like IO does.
|
|
355
|
+
#: (Integer? limit, boolish chomp) -> String?
|
|
356
|
+
def read_paragraph(limit, chomp)
|
|
357
|
+
skip_newlines
|
|
358
|
+
|
|
359
|
+
line = read_line("\n\n", limit, chomp)
|
|
360
|
+
|
|
361
|
+
skip_newlines if line # IO skips them even when the limit cut the line
|
|
362
|
+
|
|
363
|
+
line
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# Moves the store position past any consecutive newlines, retrieving more
|
|
367
|
+
# content as needed.
|
|
368
|
+
#: () -> void
|
|
369
|
+
def skip_newlines
|
|
370
|
+
loop do
|
|
371
|
+
fill_chunk while store.eof? && !chunks_depleted?
|
|
372
|
+
|
|
373
|
+
byte = store.getbyte
|
|
374
|
+
next if byte == 10 # newline
|
|
375
|
+
|
|
376
|
+
store.pos -= 1 if byte
|
|
377
|
+
break
|
|
378
|
+
end
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
# Returns whether the line just read from the given start position of a
|
|
382
|
+
# StringIO store was truncated. A line read with a positive limit that
|
|
383
|
+
# stops before the end of the store must reach the limit or end with the
|
|
384
|
+
# separator, which JRuby's StringIO violates when the limit cuts a
|
|
385
|
+
# multibyte character that's incomplete at the end of the store.
|
|
386
|
+
#: (String? separator, Integer? limit, Integer start) -> bool
|
|
387
|
+
def truncated_line?(separator, limit, start)
|
|
388
|
+
return false unless store.is_a?(StringIO) && limit && limit > 0
|
|
389
|
+
|
|
390
|
+
line_bytes = store.pos - start
|
|
391
|
+
|
|
392
|
+
return false if line_bytes >= limit || store.eof?
|
|
393
|
+
|
|
394
|
+
if separator.is_a?(String) && !separator.empty? && line_bytes >= separator.bytesize
|
|
395
|
+
return false if store_bytes(store.pos - separator.bytesize, separator.bytesize) == separator.b
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
true
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# Returns whether the line read from the given start position of the store
|
|
402
|
+
# can't continue in content that wasn't retrieved yet, which is when it
|
|
403
|
+
# ends with the separator or reaches the limit with a complete character.
|
|
404
|
+
# In paragraph mode it's never complete, as newlines after it are skipped.
|
|
405
|
+
#: (String? line, String? separator, Integer? limit, Integer start) -> bool
|
|
406
|
+
def line_complete?(line, separator, limit, start)
|
|
407
|
+
return false if line.nil? || separator == ""
|
|
408
|
+
|
|
409
|
+
line_bytes = store.pos - start
|
|
410
|
+
|
|
411
|
+
if separator.is_a?(String) && !separator.empty? && line_bytes >= separator.bytesize
|
|
412
|
+
return true if store_bytes(store.pos - separator.bytesize, separator.bytesize) == separator.b
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
return true if limit && limit >= 0 && line_bytes >= limit && line.valid_encoding?
|
|
416
|
+
|
|
417
|
+
false
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# Returns the given number of bytes stored at the given store position.
|
|
421
|
+
#: (Integer position, Integer length) -> String
|
|
422
|
+
def store_bytes(position, length)
|
|
423
|
+
case store
|
|
424
|
+
when File then store.pread(length, position)
|
|
425
|
+
when StringIO then store.string.byteslice(position, length).to_s.b
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
# Retrieves the next chunk and appends it to the store, without changing
|
|
430
|
+
# the store position. Returns false if there are no more chunks.
|
|
431
|
+
#: () -> bool
|
|
432
|
+
def fill_chunk
|
|
433
|
+
return false if chunks_depleted?
|
|
434
|
+
|
|
435
|
+
chunk = retrieve_chunk #: String
|
|
436
|
+
|
|
437
|
+
case store
|
|
438
|
+
when File
|
|
439
|
+
# unlike #write, #pwrite writes bytes as they are, without transcoding
|
|
440
|
+
store.pwrite(chunk, @stored_bytes)
|
|
441
|
+
when StringIO
|
|
442
|
+
read_bytes = store.pos
|
|
443
|
+
unread_bytes = store.string.bytesize - read_bytes
|
|
444
|
+
|
|
445
|
+
if unread_bytes == 0
|
|
446
|
+
# A copy shares the chunk's memory until either is modified, so all
|
|
447
|
+
# read content is discarded without copying the chunk.
|
|
448
|
+
string = chunk.dup
|
|
449
|
+
elsif read_bytes >= unread_bytes
|
|
450
|
+
# Strings read up to the end of the store share its memory, making
|
|
451
|
+
# appending copy the whole store. Copying only unread content instead
|
|
452
|
+
# discards read content, and copies fewer bytes than were read.
|
|
453
|
+
unread = store.string.force_encoding(Encoding::BINARY).byteslice(read_bytes, unread_bytes).to_s
|
|
454
|
+
string = String.new(capacity: unread_bytes + chunk.bytesize, encoding: Encoding::BINARY)
|
|
455
|
+
string << unread << chunk
|
|
456
|
+
else
|
|
457
|
+
# appending in binary encoding prevents encoding compatibility errors
|
|
458
|
+
store.string.force_encoding(Encoding::BINARY) << chunk
|
|
459
|
+
store.string.force_encoding(@encoding)
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
if string
|
|
463
|
+
@trimmed_bytes += read_bytes
|
|
464
|
+
store.string = string.force_encoding(@encoding)
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
@stored_bytes += chunk.bytesize
|
|
469
|
+
|
|
470
|
+
true
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
# Retrieves chunks until the store holds content up to the given
|
|
474
|
+
# position, or until there are no more chunks.
|
|
475
|
+
#: (Integer position) -> void
|
|
476
|
+
def fill_until(position)
|
|
477
|
+
fill_chunk while @stored_bytes < position && !chunks_depleted?
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
# Retrieves all remaining chunks.
|
|
481
|
+
#: () -> void
|
|
482
|
+
def fill_all
|
|
483
|
+
fill_chunk until chunks_depleted?
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
# Returns current chunk and retrieves the next chunk, skipping empty ones.
|
|
487
|
+
# If there are no more chunks, we know we've reached EOF.
|
|
488
|
+
#: () -> String?
|
|
313
489
|
def retrieve_chunk
|
|
314
490
|
chunk = @next_chunk
|
|
315
|
-
|
|
491
|
+
|
|
492
|
+
loop do
|
|
493
|
+
@next_chunk = chunks_fiber.resume
|
|
494
|
+
break unless chunks_fiber.alive? && @next_chunk == ""
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
# work with raw bytes, as chunks in other encodings can't be combined
|
|
498
|
+
chunk = chunk.b if chunk && chunk.encoding != Encoding::BINARY
|
|
499
|
+
|
|
316
500
|
chunk
|
|
317
501
|
end
|
|
318
502
|
|
|
319
503
|
# Returns whether there is any content left to retrieve.
|
|
504
|
+
#: () -> bool
|
|
320
505
|
def chunks_depleted?
|
|
321
506
|
!chunks_fiber.alive?
|
|
322
507
|
end
|
|
@@ -325,6 +510,7 @@ module Down
|
|
|
325
510
|
# of using a Fiber here is that we can terminate the chunk retrieval, in a
|
|
326
511
|
# way that executes any cleanup code that the enumerator potentially
|
|
327
512
|
# carries. At the end of iteration the :on_close callback is executed.
|
|
513
|
+
#: () -> Fiber
|
|
328
514
|
def chunks_fiber
|
|
329
515
|
@chunks_fiber ||= Fiber.new do
|
|
330
516
|
begin
|
|
@@ -333,22 +519,18 @@ module Down
|
|
|
333
519
|
break if action == :terminate
|
|
334
520
|
end
|
|
335
521
|
ensure
|
|
336
|
-
@on_close
|
|
522
|
+
@on_close&.call
|
|
337
523
|
end
|
|
338
524
|
end
|
|
339
525
|
end
|
|
340
526
|
|
|
341
527
|
# Finds encoding by name. If the encoding couldn't be find, falls back to
|
|
342
528
|
# the generic binary encoding.
|
|
529
|
+
#: (String | Encoding encoding) -> Encoding
|
|
343
530
|
def find_encoding(encoding)
|
|
344
|
-
Encoding.find(encoding)
|
|
531
|
+
Encoding.find(encoding) #: Encoding
|
|
345
532
|
rescue ArgumentError
|
|
346
533
|
Encoding::BINARY
|
|
347
534
|
end
|
|
348
|
-
|
|
349
|
-
# Returns whether the filesystem has POSIX semantics.
|
|
350
|
-
def posix?
|
|
351
|
-
RUBY_PLATFORM !~ /(mswin|mingw|cygwin|java)/
|
|
352
|
-
end
|
|
353
535
|
end
|
|
354
536
|
end
|
data/lib/down/errors.rb
CHANGED
|
@@ -18,8 +18,10 @@ module Down
|
|
|
18
18
|
|
|
19
19
|
# raised when response returned 4xx or 5xx response
|
|
20
20
|
class ResponseError < Error
|
|
21
|
-
|
|
21
|
+
# The backend-specific response object (e.g. Net::HTTPResponse, HTTP::Response, HTTPX::Response).
|
|
22
|
+
attr_reader :response #: untyped
|
|
22
23
|
|
|
24
|
+
#: (String message, ?untyped response) -> void
|
|
23
25
|
def initialize(message, response = nil)
|
|
24
26
|
super(message)
|
|
25
27
|
@response = response
|