down 5.6.2 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  # frozen-string-literal: true
2
2
 
3
3
  require "tempfile"
4
+ require "stringio"
4
5
  require "fiber"
5
6
 
6
7
  module Down
@@ -12,78 +13,113 @@ module Down
12
13
  # * IO#gets
13
14
  # * IO#size
14
15
  # * IO#pos
16
+ # * IO#seek
15
17
  # * IO#eof?
16
18
  # * IO#rewind
17
19
  # * IO#close
18
20
  #
19
- # By default the Down::ChunkedIO caches all read content into a tempfile,
20
- # allowing it to be rewindable. If rewindability won't be used, it can be
21
- # disabled by setting `:rewindable` to false, which eliminates any disk I/O.
21
+ # Retrieved chunks are appended to a backing store which the reading methods
22
+ # delegate to, so they behave like the corresponding native IO methods. New
23
+ # chunks are retrieved only when the store runs out of content.
24
+ #
25
+ # By default the store is a tempfile which keeps all read content, allowing
26
+ # the Down::ChunkedIO to be rewindable. If rewindability won't be used, it
27
+ # can be disabled by setting `:rewindable` to false, in which case the store
28
+ # is an in-memory buffer that discards read content, eliminating disk I/O.
22
29
  #
23
30
  # Any cleanup code (i.e. ensure block) that the given enumerator carries is
24
31
  # guaranteed to get executed, either when all content has been retrieved or
25
32
  # when Down::ChunkedIO is closed. One can also specify an `:on_close`
26
33
  # callback that will also get executed in those situations.
27
34
  class ChunkedIO
28
- attr_accessor :size, :data, :encoding
29
-
35
+ attr_accessor :size #: Integer?
36
+ attr_accessor :data #: Hash[Symbol, untyped]
37
+ attr_reader :encoding #: Encoding
38
+
39
+ # @rbs @chunks: _Each[String] & Object
40
+ # @rbs @on_close: (^() -> void | (_Callable & Object))?
41
+ # @rbs @rewindable: bool
42
+ # @rbs @next_chunk: String?
43
+ # @rbs @closed: bool
44
+ # @rbs @store: (File | StringIO)?
45
+ # @rbs @stored_bytes: Integer
46
+ # @rbs @trimmed_bytes: Integer
47
+ # @rbs @chunks_fiber: Fiber?
48
+
49
+ #: (chunks: _Each[String] & Object, ?size: Integer?, ?on_close: (^() -> void | (_Callable & Object))?, ?data: Hash[Symbol, untyped], ?rewindable: bool, ?encoding: (String | Encoding)?) -> void
30
50
  def initialize(chunks:, size: nil, on_close: nil, data: {}, rewindable: true, encoding: nil)
31
- @chunks = chunks
32
- @size = size
33
- @on_close = on_close
34
- @data = data
35
- @encoding = find_encoding(encoding || "binary")
36
- @rewindable = rewindable
37
- @buffer = nil
38
- @position = 0
39
- @next_chunk = nil
40
- @closed = false
51
+ @chunks = chunks
52
+ @size = size
53
+ @on_close = on_close
54
+ @data = data
55
+ @encoding = find_encoding(encoding || "binary")
56
+ @rewindable = rewindable
57
+ @next_chunk = nil
58
+ @closed = false
59
+ @store = nil
60
+ @stored_bytes = 0 # number of bytes appended to the store
61
+ @trimmed_bytes = 0 # number of read bytes discarded from the store
41
62
 
42
63
  retrieve_chunk # fetch first chunk so that we know whether the file is empty
43
64
  end
44
65
 
66
+ # Sets the encoding that read content is returned in.
67
+ #: (String | Encoding encoding) -> void
68
+ def encoding=(encoding)
69
+ @encoding = find_encoding(encoding)
70
+ @store&.set_encoding(@encoding, @encoding)
71
+ end
72
+
45
73
  # For compatibility with multipart-post gem.
74
+ #: () -> Integer?
46
75
  def length
47
76
  @size
48
77
  end
49
78
 
50
79
  # Yields elements of the underlying enumerator.
80
+ #: () { (String) -> void } -> void
81
+ #: () -> Enumerator[String, void]
51
82
  def each_chunk
52
83
  fail IOError, "closed stream" if closed?
53
84
 
54
- return enum_for(__method__) unless block_given?
85
+ return enum_for(:each_chunk) unless block_given?
55
86
 
56
- yield retrieve_chunk until chunks_depleted?
87
+ until chunks_depleted?
88
+ chunk = retrieve_chunk #: String
89
+ yield chunk
90
+ end
57
91
  end
58
92
 
59
93
  # Implements IO#read semantics. Without arguments it retrieves and returns
60
- # all content.
94
+ # all content in the external encoding.
61
95
  #
62
96
  # With `length` argument returns exactly that number of bytes if they're
63
- # available.
97
+ # available, as a binary string.
64
98
  #
65
99
  # With `outbuf` argument each call will return that same string object,
66
100
  # where the value is replaced with retrieved content.
67
101
  #
68
102
  # If end of file is reached, returns empty string if called without
69
103
  # arguments, or nil if called with arguments. Raises IOError if closed.
104
+ #: (?nil, ?string? outbuf) -> String
105
+ #: (?int? length, ?string? outbuf) -> String?
70
106
  def read(length = nil, outbuf = nil)
71
107
  fail IOError, "closed stream" if closed?
72
108
 
73
- data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
74
- data ||= "".b
75
-
76
- remaining_length = length
109
+ length = length&.to_int # implicit conversion, like IO#read
110
+ outbuf = outbuf&.to_str
77
111
 
78
- until remaining_length == 0 || eof?
79
- data << readpartial(remaining_length, buffer ||= String.new)
80
- remaining_length = length - data.bytesize if length
112
+ if length
113
+ fill_until(pos + length)
114
+ else
115
+ fill_all
81
116
  end
82
117
 
83
- buffer.clear if buffer # deallocate string
118
+ encoding = outbuf&.encoding if length
119
+ data = store.read(length, outbuf)
120
+ data&.force_encoding(encoding) if encoding # StringIO < 3.1.2 changes it to binary
84
121
 
85
- data.force_encoding(@encoding) unless length
86
- data unless data.empty? && length && length > 0
122
+ data
87
123
  end
88
124
 
89
125
  # Implements IO#gets semantics. Without arguments it retrieves lines of
@@ -94,64 +130,35 @@ module Down
94
130
  # * if `separator` is a nonempty string returns chunks of content
95
131
  # surrounded with that sequence of bytes
96
132
  # * if `separator` is an empty string returns paragraphs of content
97
- # (content delimited by two newlines)
98
- # * if `separator` is nil and `limit` is nil returns all content
133
+ # (content delimited by two or more newlines, which are skipped)
134
+ # * if `separator` is nil returns all content
99
135
  #
100
- # With `limit` argument returns maximum of that amount of bytes.
136
+ # With `limit` argument returns maximum of that amount of bytes, plus any
137
+ # bytes needed to complete a multibyte character.
138
+ #
139
+ # With `chomp: true` removes the trailing separator from the line.
101
140
  #
102
141
  # Returns nil if end of file is reached. Raises IOError if closed.
103
- def gets(separator_or_limit = $/, limit = nil)
142
+ #: (?String? separator, ?Integer? limit, ?chomp: boolish) -> String?
143
+ #: (Integer limit, ?chomp: boolish) -> String?
144
+ def gets(separator_or_limit = $/, limit = nil, chomp: false)
104
145
  fail IOError, "closed stream" if closed?
105
146
 
106
- if separator_or_limit.is_a?(Integer)
107
- separator = $/
108
- limit = separator_or_limit
109
- else
110
- separator = separator_or_limit
111
- end
112
-
113
- return read(limit)&.force_encoding(@encoding) if separator.nil?
114
-
115
- separator = "\n\n" if separator.empty?
116
-
117
- data = String.new
147
+ separator = separator_or_limit.is_a?(Integer) ? $/ : separator_or_limit
148
+ limit = separator_or_limit if separator_or_limit.is_a?(Integer)
118
149
 
119
- until data.include?(separator) || data.bytesize == limit || eof?
120
- remaining_length = limit - data.bytesize if limit
121
- data << readpartial(remaining_length, buffer ||= String.new)
122
- end
123
-
124
- buffer.clear if buffer # deallocate buffer
125
-
126
- line, extra = data.split(separator, 2)
127
- line << separator if data.include?(separator)
128
-
129
- data.clear # deallocate data
130
-
131
- if extra
132
- @position -= extra.bytesize
133
-
134
- if cache
135
- cache.pos -= extra.bytesize
136
- else
137
- if @buffer
138
- @buffer.prepend(extra)
139
- else
140
- @buffer = extra
141
- end
142
- end
150
+ if separator == "" && store.is_a?(StringIO)
151
+ line = read_paragraph(limit, chomp)
152
+ else
153
+ line = read_line(separator, limit, chomp)
143
154
  end
144
155
 
145
- line.force_encoding(@encoding) if line
156
+ line
146
157
  end
147
158
 
148
159
  # Implements IO#readpartial semantics. If there is any content readily
149
- # available reads from it, otherwise fetches and reads from the next chunk.
150
- # It writes to and reads from the cache when needed.
151
- #
152
- # Without arguments it either returns all content that's readily available,
153
- # or the next chunk. This is useful when you don't care about the size of
154
- # chunks and you want to minimize string allocations.
160
+ # available reads from it, otherwise retrieves the next chunk and reads
161
+ # from it.
155
162
  #
156
163
  # With `maxlen` argument returns maximum of that amount of bytes (default
157
164
  # is 16KB).
@@ -160,126 +167,111 @@ module Down
160
167
  # where the value is replaced with retrieved content.
161
168
  #
162
169
  # Raises EOFError if end of file is reached. Raises IOError if closed.
170
+ #: (?int? maxlen, ?string? outbuf) -> String
163
171
  def readpartial(maxlen = nil, outbuf = nil)
164
172
  fail IOError, "closed stream" if closed?
165
173
 
166
- maxlen ||= 16*1024
167
-
168
- data = cache.read(maxlen, outbuf) if cache && !cache.eof?
169
- data ||= outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
170
- data ||= "".b
174
+ maxlen = maxlen ? maxlen.to_int : 16*1024 # implicit conversion, like IO#readpartial
171
175
 
172
- return data if maxlen == 0
176
+ fill_chunk while maxlen > 0 && pos >= @stored_bytes && !chunks_depleted?
173
177
 
174
- if @buffer.nil? && data.empty?
175
- fail EOFError, "end of file reached" if chunks_depleted?
176
- @buffer = retrieve_chunk
177
- end
178
-
179
- remaining_length = maxlen - data.bytesize
178
+ # Reading only stored content allocates a string of that size, instead
179
+ # of maxlen bytes, which also avoids reallocating a reused buffer.
180
+ maxlen = [maxlen, @stored_bytes - pos].min if pos < @stored_bytes
180
181
 
181
- unless @buffer.nil? || remaining_length == 0
182
- if remaining_length < @buffer.bytesize
183
- buffered_data = @buffer.byteslice(0, remaining_length)
184
- @buffer = @buffer.byteslice(remaining_length..-1)
185
- else
186
- buffered_data = @buffer
187
- @buffer = nil
188
- end
182
+ outbuf = outbuf&.to_str
183
+ encoding = outbuf&.encoding
184
+ data = outbuf ? store.readpartial(maxlen, outbuf) : store.readpartial(maxlen) # JRuby rejects nil outbuf
185
+ data.force_encoding(encoding) if encoding # StringIO < 3.1.2 changes it to binary
189
186
 
190
- data << buffered_data
191
-
192
- cache.write(buffered_data) if cache
193
-
194
- buffered_data.clear unless buffered_data.frozen?
195
- end
196
-
197
- @position += data.bytesize
198
-
199
- data.force_encoding(Encoding::BINARY)
187
+ data
200
188
  end
201
189
 
202
- # Implements IO#seek semantics.
190
+ # Implements IO#seek semantics. It retrieves content up to the target
191
+ # position. Raises Errno::ESPIPE if not rewindable.
192
+ #: (int amount, ?(Integer | Symbol) whence) -> Integer
203
193
  def seek(amount, whence = IO::SEEK_SET)
204
- fail Errno::ESPIPE, "Illegal seek" if cache.nil?
194
+ fail IOError, "closed stream" if closed?
195
+ fail Errno::ESPIPE unless rewindable?
196
+
197
+ amount = amount.to_int
205
198
 
206
199
  case whence
207
200
  when IO::SEEK_SET, :SET
208
- target_pos = amount
201
+ whence = IO::SEEK_SET
202
+ fill_until(amount)
209
203
  when IO::SEEK_CUR, :CUR
210
- target_pos = @position + amount
204
+ whence = IO::SEEK_CUR
205
+ fill_until(pos + amount)
211
206
  when IO::SEEK_END, :END
212
- unless chunks_depleted?
213
- cache.seek(0, IO::SEEK_END)
214
- IO.copy_stream(self, File::NULL)
215
- end
216
-
217
- target_pos = cache.size + amount
207
+ whence = IO::SEEK_END
208
+ fill_all
218
209
  else
219
210
  fail ArgumentError, "invalid whence: #{whence.inspect}"
220
211
  end
221
212
 
222
- if target_pos <= cache.size
223
- cache.seek(target_pos)
224
- else
225
- cache.seek(0, IO::SEEK_END)
226
- IO.copy_stream(self, File::NULL, target_pos - cache.size)
227
- end
228
-
229
- @position = cache.pos
213
+ store.seek(amount, whence)
230
214
  end
231
215
 
232
216
  # Implements IO#pos semantics. Returns the current position of the
233
- # Down::ChunkedIO.
217
+ # Down::ChunkedIO. Raises IOError if closed.
218
+ #: () -> Integer
234
219
  def pos
235
- @position
220
+ fail IOError, "closed stream" if closed?
221
+
222
+ @trimmed_bytes + store.pos
236
223
  end
237
224
  alias tell pos
238
225
 
239
226
  # Implements IO#eof? semantics. Returns whether we've reached end of file.
240
- # It returns true if cache is at the end and there is no more content to
241
- # retrieve. Raises IOError if closed.
227
+ # It returns true if the store is at the end and there is no more content
228
+ # to retrieve. Raises IOError if closed.
229
+ #: () -> bool
242
230
  def eof?
243
231
  fail IOError, "closed stream" if closed?
244
232
 
245
- return false if cache && !cache.eof?
246
- @buffer.nil? && chunks_depleted?
233
+ pos >= @stored_bytes && chunks_depleted?
247
234
  end
248
235
  alias eof eof?
249
236
 
250
- # Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by rewinding
251
- # the cache and setting the position to the beginning of the file. Raises
252
- # IOError if closed or not rewindable.
237
+ # Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by setting
238
+ # the position to the beginning of the file. Raises IOError if closed or
239
+ # not rewindable.
240
+ #: () -> Integer
253
241
  def rewind
254
242
  fail IOError, "closed stream" if closed?
255
- fail IOError, "this Down::ChunkedIO is not rewindable" if cache.nil?
243
+ fail IOError, "this Down::ChunkedIO is not rewindable" unless rewindable?
256
244
 
257
- cache.rewind
258
- @position = 0
245
+ store.rewind
259
246
  end
260
247
 
261
248
  # Implements IO#close semantics. Closes the Down::ChunkedIO by terminating
262
- # chunk retrieval and deleting the cached content.
249
+ # chunk retrieval and deleting the retrieved content.
250
+ #: () -> void
263
251
  def close
264
252
  return if @closed
265
253
 
266
254
  chunks_fiber.resume(:terminate) if chunks_fiber.alive?
267
- cache.close! if cache
268
- @buffer = nil
255
+
256
+ @store&.close
257
+ @store = nil
269
258
  @closed = true
270
259
  end
271
260
 
272
261
  # Returns whether the Down::ChunkedIO has been closed.
262
+ #: () -> bool
273
263
  def closed?
274
264
  !!@closed
275
265
  end
276
266
 
277
267
  # Returns whether the Down::ChunkedIO was specified as rewindable.
268
+ #: () -> bool
278
269
  def rewindable?
279
270
  @rewindable
280
271
  end
281
272
 
282
273
  # Returns useful information about the Down::ChunkedIO object.
274
+ #: () -> String
283
275
  def inspect
284
276
  string = String.new
285
277
  string << "#<#{self.class.name}"
@@ -295,28 +287,221 @@ module Down
295
287
 
296
288
  private
297
289
 
298
- # If Down::ChunkedIO is specified as rewindable, returns a new Tempfile for
299
- # writing read content to. This allows the Down::ChunkedIO to be rewinded.
300
- def cache
301
- return if !rewindable?
290
+ # Returns the IO object that retrieved content is appended to and read
291
+ # from. If Down::ChunkedIO is specified as rewindable, it's an anonymous
292
+ # tempfile which keeps all retrieved content, otherwise it's a StringIO
293
+ # from which read content gets discarded.
294
+ # @rbs %a{pure}
295
+ #: () -> (File | StringIO)
296
+ def store
297
+ @store ||= (
298
+ if rewindable?
299
+ # the file is already deleted (or never named on Linux), so it gets
300
+ # cleaned up once closed, and no other process can open it
301
+ store = Tempfile.create("down-chunked_io", anonymous: true)
302
+ store.binmode # anonymous tempfiles ignore the :binmode option
303
+ else
304
+ store = StringIO.new(String.new(encoding: Encoding::BINARY))
305
+ end
302
306
 
303
- @cache ||= (
304
- tempfile = Tempfile.new("down-chunked_io", binmode: true)
305
- tempfile.chmod(0000) # make sure nobody else can read or write to it
306
- tempfile.unlink if posix? # remove entry from filesystem if it's POSIX
307
- tempfile
307
+ # passing the same internal encoding disables Encoding.default_internal
308
+ store.set_encoding(@encoding, @encoding)
309
+ store
308
310
  )
309
311
  end
310
312
 
311
- # Returns current chunk and retrieves the next chunk. If next chunk is nil,
312
- # we know we've reached EOF.
313
+ # Reads a line from the store, retrieving more content if the line might
314
+ # continue in content that wasn't retrieved yet.
315
+ #: (String? separator, Integer? limit, boolish chomp) -> String?
316
+ def read_line(separator, limit, chomp)
317
+ start = store.pos
318
+ line = nil #: String?
319
+
320
+ loop do
321
+ line = store.gets(separator, limit, chomp:)
322
+
323
+ # JRuby's StringIO returns a truncated line and moves to a wrong
324
+ # position when the limit cuts a multibyte character that's incomplete
325
+ # at the end of the store. Like IO, we return the remaining content at
326
+ # the end of file, otherwise we retry with more content.
327
+ if RUBY_ENGINE == "jruby" && line && truncated_line?(separator, limit, start)
328
+ store.pos = start
329
+ line = chunks_depleted? ? store.read : nil
330
+ end
331
+
332
+ # If the store ran out of content, the line might continue in content
333
+ # that wasn't retrieved yet. In that case we retrieve more content and
334
+ # read the line again, at least doubling the content after the line
335
+ # start, so that lines spanning many chunks aren't re-read many times.
336
+ # StringIO doesn't move to the end when only newlines are left in
337
+ # paragraph mode, so we also retrieve more content when there's no line.
338
+ break if chunks_depleted?
339
+ break unless line.nil? || store.eof?
340
+ break if line_complete?(line, separator, limit, start)
341
+
342
+ store.pos = start # keeps the line start in the store when retrieving content
343
+ fill_until(@stored_bytes + [@stored_bytes - @trimmed_bytes - start, 1].max)
344
+ start = store.pos # retrieving content can discard content before it
345
+ end
346
+
347
+ store.seek(0, IO::SEEK_END) if line.nil? # skipped newlines in paragraph mode
348
+
349
+ line
350
+ end
351
+
352
+ # Reads a paragraph from a StringIO store. StringIO's paragraph mode
353
+ # differs from IO's in how it applies the limit and chomps lines, so we
354
+ # implement it on top of the double-newline separator, like IO does.
355
+ #: (Integer? limit, boolish chomp) -> String?
356
+ def read_paragraph(limit, chomp)
357
+ skip_newlines
358
+
359
+ line = read_line("\n\n", limit, chomp)
360
+
361
+ skip_newlines if line # IO skips them even when the limit cut the line
362
+
363
+ line
364
+ end
365
+
366
+ # Moves the store position past any consecutive newlines, retrieving more
367
+ # content as needed.
368
+ #: () -> void
369
+ def skip_newlines
370
+ loop do
371
+ fill_chunk while store.eof? && !chunks_depleted?
372
+
373
+ byte = store.getbyte
374
+ next if byte == 10 # newline
375
+
376
+ store.pos -= 1 if byte
377
+ break
378
+ end
379
+ end
380
+
381
+ # Returns whether the line just read from the given start position of a
382
+ # StringIO store was truncated. A line read with a positive limit that
383
+ # stops before the end of the store must reach the limit or end with the
384
+ # separator, which JRuby's StringIO violates when the limit cuts a
385
+ # multibyte character that's incomplete at the end of the store.
386
+ #: (String? separator, Integer? limit, Integer start) -> bool
387
+ def truncated_line?(separator, limit, start)
388
+ return false unless store.is_a?(StringIO) && limit && limit > 0
389
+
390
+ line_bytes = store.pos - start
391
+
392
+ return false if line_bytes >= limit || store.eof?
393
+
394
+ if separator.is_a?(String) && !separator.empty? && line_bytes >= separator.bytesize
395
+ return false if store_bytes(store.pos - separator.bytesize, separator.bytesize) == separator.b
396
+ end
397
+
398
+ true
399
+ end
400
+
401
+ # Returns whether the line read from the given start position of the store
402
+ # can't continue in content that wasn't retrieved yet, which is when it
403
+ # ends with the separator or reaches the limit with a complete character.
404
+ # In paragraph mode it's never complete, as newlines after it are skipped.
405
+ #: (String? line, String? separator, Integer? limit, Integer start) -> bool
406
+ def line_complete?(line, separator, limit, start)
407
+ return false if line.nil? || separator == ""
408
+
409
+ line_bytes = store.pos - start
410
+
411
+ if separator.is_a?(String) && !separator.empty? && line_bytes >= separator.bytesize
412
+ return true if store_bytes(store.pos - separator.bytesize, separator.bytesize) == separator.b
413
+ end
414
+
415
+ return true if limit && limit >= 0 && line_bytes >= limit && line.valid_encoding?
416
+
417
+ false
418
+ end
419
+
420
+ # Returns the given number of bytes stored at the given store position.
421
+ #: (Integer position, Integer length) -> String
422
+ def store_bytes(position, length)
423
+ case store
424
+ when File then store.pread(length, position)
425
+ when StringIO then store.string.byteslice(position, length).to_s.b
426
+ end
427
+ end
428
+
429
+ # Retrieves the next chunk and appends it to the store, without changing
430
+ # the store position. Returns false if there are no more chunks.
431
+ #: () -> bool
432
+ def fill_chunk
433
+ return false if chunks_depleted?
434
+
435
+ chunk = retrieve_chunk #: String
436
+
437
+ case store
438
+ when File
439
+ # unlike #write, #pwrite writes bytes as they are, without transcoding
440
+ store.pwrite(chunk, @stored_bytes)
441
+ when StringIO
442
+ read_bytes = store.pos
443
+ unread_bytes = store.string.bytesize - read_bytes
444
+
445
+ if unread_bytes == 0
446
+ # A copy shares the chunk's memory until either is modified, so all
447
+ # read content is discarded without copying the chunk.
448
+ string = chunk.dup
449
+ elsif read_bytes >= unread_bytes
450
+ # Strings read up to the end of the store share its memory, making
451
+ # appending copy the whole store. Copying only unread content instead
452
+ # discards read content, and copies fewer bytes than were read.
453
+ unread = store.string.force_encoding(Encoding::BINARY).byteslice(read_bytes, unread_bytes).to_s
454
+ string = String.new(capacity: unread_bytes + chunk.bytesize, encoding: Encoding::BINARY)
455
+ string << unread << chunk
456
+ else
457
+ # appending in binary encoding prevents encoding compatibility errors
458
+ store.string.force_encoding(Encoding::BINARY) << chunk
459
+ store.string.force_encoding(@encoding)
460
+ end
461
+
462
+ if string
463
+ @trimmed_bytes += read_bytes
464
+ store.string = string.force_encoding(@encoding)
465
+ end
466
+ end
467
+
468
+ @stored_bytes += chunk.bytesize
469
+
470
+ true
471
+ end
472
+
473
+ # Retrieves chunks until the store holds content up to the given
474
+ # position, or until there are no more chunks.
475
+ #: (Integer position) -> void
476
+ def fill_until(position)
477
+ fill_chunk while @stored_bytes < position && !chunks_depleted?
478
+ end
479
+
480
+ # Retrieves all remaining chunks.
481
+ #: () -> void
482
+ def fill_all
483
+ fill_chunk until chunks_depleted?
484
+ end
485
+
486
+ # Returns current chunk and retrieves the next chunk, skipping empty ones.
487
+ # If there are no more chunks, we know we've reached EOF.
488
+ #: () -> String?
313
489
  def retrieve_chunk
314
490
  chunk = @next_chunk
315
- @next_chunk = chunks_fiber.resume
491
+
492
+ loop do
493
+ @next_chunk = chunks_fiber.resume
494
+ break unless chunks_fiber.alive? && @next_chunk == ""
495
+ end
496
+
497
+ # work with raw bytes, as chunks in other encodings can't be combined
498
+ chunk = chunk.b if chunk && chunk.encoding != Encoding::BINARY
499
+
316
500
  chunk
317
501
  end
318
502
 
319
503
  # Returns whether there is any content left to retrieve.
504
+ #: () -> bool
320
505
  def chunks_depleted?
321
506
  !chunks_fiber.alive?
322
507
  end
@@ -325,6 +510,7 @@ module Down
325
510
  # of using a Fiber here is that we can terminate the chunk retrieval, in a
326
511
  # way that executes any cleanup code that the enumerator potentially
327
512
  # carries. At the end of iteration the :on_close callback is executed.
513
+ #: () -> Fiber
328
514
  def chunks_fiber
329
515
  @chunks_fiber ||= Fiber.new do
330
516
  begin
@@ -333,22 +519,18 @@ module Down
333
519
  break if action == :terminate
334
520
  end
335
521
  ensure
336
- @on_close.call if @on_close
522
+ @on_close&.call
337
523
  end
338
524
  end
339
525
  end
340
526
 
341
527
  # Finds encoding by name. If the encoding couldn't be find, falls back to
342
528
  # the generic binary encoding.
529
+ #: (String | Encoding encoding) -> Encoding
343
530
  def find_encoding(encoding)
344
- Encoding.find(encoding)
531
+ Encoding.find(encoding) #: Encoding
345
532
  rescue ArgumentError
346
533
  Encoding::BINARY
347
534
  end
348
-
349
- # Returns whether the filesystem has POSIX semantics.
350
- def posix?
351
- RUBY_PLATFORM !~ /(mswin|mingw|cygwin|java)/
352
- end
353
535
  end
354
536
  end
data/lib/down/errors.rb CHANGED
@@ -18,8 +18,10 @@ module Down
18
18
 
19
19
  # raised when response returned 4xx or 5xx response
20
20
  class ResponseError < Error
21
- attr_reader :response
21
+ # The backend-specific response object (e.g. Net::HTTPResponse, HTTP::Response, HTTPX::Response).
22
+ attr_reader :response #: untyped
22
23
 
24
+ #: (String message, ?untyped response) -> void
23
25
  def initialize(message, response = nil)
24
26
  super(message)
25
27
  @response = response