ziperu 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/Cargo.lock +1 -1
- data/README.md +56 -5
- data/ext/ziperu/Cargo.toml +1 -1
- data/ext/ziperu/src/archive.rs +85 -54
- data/ext/ziperu/src/errors.rs +76 -14
- data/ext/ziperu/src/extract.rs +92 -33
- data/ext/ziperu/src/lib.rs +12 -9
- data/ext/ziperu/src/reader.rs +211 -13
- data/ext/ziperu/src/sink.rs +48 -8
- data/ext/ziperu/src/source.rs +2 -2
- data/ext/ziperu/src/stream.rs +541 -0
- data/ext/ziperu/src/writer.rs +128 -5
- data/lib/ziperu/entry/source.rb +1 -1
- data/lib/ziperu/entry.rb +8 -1
- data/lib/ziperu/errors.rb +5 -0
- data/lib/ziperu/file.rb +24 -2
- data/lib/ziperu/input_stream.rb +167 -11
- data/lib/ziperu/output_stream.rb +72 -1
- data/lib/ziperu/remote_io.rb +278 -0
- data/lib/ziperu/replace_rubyzip.rb +67 -0
- data/lib/ziperu/version.rb +1 -1
- data/lib/ziperu.rb +4 -0
- metadata +4 -1
data/ext/ziperu/src/extract.rs
CHANGED
|
@@ -210,16 +210,34 @@ fn extract_all(
|
|
|
210
210
|
specs: &[Spec],
|
|
211
211
|
threads: usize,
|
|
212
212
|
) -> Outcome {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
let (dirs, files) = specs.split_at(files_from);
|
|
213
|
+
// Every directory first, wherever it is listed, so that no file has
|
|
214
|
+
// to wait for one and no worker thread meets one. Each spec keeps its
|
|
215
|
+
// place in the batch, which is how a failure names its entry.
|
|
216
|
+
let (dirs, files): (Vec<_>, Vec<_>) = specs.iter().enumerate().partition(|(_, s)| s.directory);
|
|
218
217
|
let mut extractor = Extractor::new(archive, dest, enforce_size, on_exists);
|
|
219
|
-
extractor.extract_each(dirs
|
|
218
|
+
extractor.extract_each(&dirs)?;
|
|
219
|
+
let outcome = extract_files(&mut extractor, dest, &files, threads);
|
|
220
|
+
// A directory's times go on once nothing more is written into it:
|
|
221
|
+
// creating a file inside would move its mtime again.
|
|
222
|
+
let stamped = extractor.stamp_directories();
|
|
223
|
+
let mut notices = outcome?;
|
|
224
|
+
stamped?;
|
|
225
|
+
notices.append(&mut extractor.notices);
|
|
226
|
+
Ok(notices)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/// The files of `extract_all`: in order on this thread, or shared out
|
|
230
|
+
/// among `threads`.
|
|
231
|
+
fn extract_files(
|
|
232
|
+
extractor: &mut Extractor<'_>,
|
|
233
|
+
dest: &str,
|
|
234
|
+
files: &[(usize, &Spec)],
|
|
235
|
+
threads: usize,
|
|
236
|
+
) -> Outcome {
|
|
237
|
+
let (enforce_size, on_exists) = (extractor.enforce_size, extractor.on_exists);
|
|
220
238
|
if threads <= 1 || files.len() <= 1 {
|
|
221
|
-
extractor.extract_each(files
|
|
222
|
-
return Ok(extractor.notices);
|
|
239
|
+
extractor.extract_each(files)?;
|
|
240
|
+
return Ok(std::mem::take(&mut extractor.notices));
|
|
223
241
|
}
|
|
224
242
|
|
|
225
243
|
// Each thread takes every `threads`th file, so a run of large ones
|
|
@@ -241,13 +259,13 @@ fn extract_all(
|
|
|
241
259
|
let token = token.clone();
|
|
242
260
|
scope.spawn(move || {
|
|
243
261
|
crate::gvl::with_token(token, || {
|
|
244
|
-
for (i, spec) in files.iter().
|
|
262
|
+
for &(i, spec) in files.iter().skip(t).step_by(threads) {
|
|
245
263
|
if failed.load(Ordering::Acquire) {
|
|
246
264
|
break;
|
|
247
265
|
}
|
|
248
266
|
if let Err(f) = worker.extract(spec) {
|
|
249
267
|
failed.store(true, Ordering::Release);
|
|
250
|
-
return Err((
|
|
268
|
+
return Err((i, f));
|
|
251
269
|
}
|
|
252
270
|
}
|
|
253
271
|
Ok(worker.notices)
|
|
@@ -260,7 +278,7 @@ fn extract_all(
|
|
|
260
278
|
.map(|w| w.join().expect("extraction thread panicked"))
|
|
261
279
|
.collect()
|
|
262
280
|
});
|
|
263
|
-
let mut notices = extractor.notices;
|
|
281
|
+
let mut notices = std::mem::take(&mut extractor.notices);
|
|
264
282
|
let mut first_failure: Option<(usize, Failure)> = None;
|
|
265
283
|
for result in results {
|
|
266
284
|
match result {
|
|
@@ -292,6 +310,12 @@ fn raise(ruby: &Ruby, entry: Value, failure: Failure) -> Error {
|
|
|
292
310
|
Failure::SizeLimit => exception(ruby, &ENTRY_SIZE_ERROR, (entry,)),
|
|
293
311
|
Failure::Zip(e) => zip_error(ruby, e),
|
|
294
312
|
Failure::Io(e) => read_error(ruby, e),
|
|
313
|
+
// A name the file system cannot take at all (a NUL in it) is
|
|
314
|
+
// what Ruby's own File calls raise ArgumentError for.
|
|
315
|
+
Failure::IoAt(path, e) if e.kind() == io::ErrorKind::InvalidInput => Error::new(
|
|
316
|
+
ruby.exception_arg_error(),
|
|
317
|
+
format!("{e} - {}", path.replace('\0', "\\0")),
|
|
318
|
+
),
|
|
295
319
|
Failure::IoAt(path, e) => io_error(ruby, io::Error::new(e.kind(), format!("{e} - {path}"))),
|
|
296
320
|
}
|
|
297
321
|
}
|
|
@@ -318,6 +342,11 @@ struct Extractor<'a> {
|
|
|
318
342
|
on_exists: OnExists,
|
|
319
343
|
/// Directories known to exist, so each parent is created once.
|
|
320
344
|
made: HashSet<PathBuf>,
|
|
345
|
+
/// Directories made for an entry, with the times and mode to give
|
|
346
|
+
/// them once their files are in.
|
|
347
|
+
stamp: Vec<(usize, PathBuf, Option<SystemTime>, Option<u32>)>,
|
|
348
|
+
/// The place in the batch of the spec being extracted.
|
|
349
|
+
at: usize,
|
|
321
350
|
buf: Vec<u8>,
|
|
322
351
|
notices: Vec<(usize, Notice)>,
|
|
323
352
|
}
|
|
@@ -335,6 +364,8 @@ impl<'a> Extractor<'a> {
|
|
|
335
364
|
enforce_size,
|
|
336
365
|
on_exists,
|
|
337
366
|
made: HashSet::new(),
|
|
367
|
+
stamp: Vec::new(),
|
|
368
|
+
at: 0,
|
|
338
369
|
buf: vec![0; CHUNK],
|
|
339
370
|
notices: Vec::new(),
|
|
340
371
|
}
|
|
@@ -342,9 +373,12 @@ impl<'a> Extractor<'a> {
|
|
|
342
373
|
|
|
343
374
|
/// Extracts `specs` in order; `offset` is the index of the first in
|
|
344
375
|
/// the whole batch, for naming the one that failed.
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
376
|
+
/// Extracts `specs`, each with its place in the whole batch, in
|
|
377
|
+
/// order; a failure names the place.
|
|
378
|
+
fn extract_each(&mut self, specs: &[(usize, &Spec)]) -> Result<(), (usize, Failure)> {
|
|
379
|
+
for &(i, spec) in specs {
|
|
380
|
+
self.at = i;
|
|
381
|
+
self.extract(spec).map_err(|f| (i, f))?;
|
|
348
382
|
}
|
|
349
383
|
Ok(())
|
|
350
384
|
}
|
|
@@ -381,14 +415,29 @@ impl<'a> Extractor<'a> {
|
|
|
381
415
|
}
|
|
382
416
|
fs::create_dir_all(path).map_err(at(path))?;
|
|
383
417
|
self.made.insert(path.to_path_buf());
|
|
384
|
-
if
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
.and_then(|dir| dir.set_times(times))
|
|
388
|
-
.map_err(at(path))?;
|
|
418
|
+
if spec.mtime.is_some() || spec.mode.is_some() {
|
|
419
|
+
self.stamp
|
|
420
|
+
.push((self.at, path.to_path_buf(), spec.mtime, spec.mode));
|
|
389
421
|
}
|
|
390
|
-
|
|
391
|
-
|
|
422
|
+
Ok(())
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/// Gives the directories made their times and modes, deepest first,
|
|
426
|
+
/// so that stamping a parent comes after everything inside it.
|
|
427
|
+
fn stamp_directories(&mut self) -> Result<(), (usize, Failure)> {
|
|
428
|
+
let mut stamp = std::mem::take(&mut self.stamp);
|
|
429
|
+
stamp.sort_by_key(|(_, path, _, _)| std::cmp::Reverse(path.components().count()));
|
|
430
|
+
for (i, path, mtime, mode) in stamp {
|
|
431
|
+
let path = path.as_path();
|
|
432
|
+
if let Some(mtime) = mtime {
|
|
433
|
+
let times = FileTimes::new().set_accessed(mtime).set_modified(mtime);
|
|
434
|
+
open_directory(path)
|
|
435
|
+
.and_then(|dir| dir.set_times(times))
|
|
436
|
+
.map_err(|e| (i, at(path)(e)))?;
|
|
437
|
+
}
|
|
438
|
+
if let Some(mode) = mode {
|
|
439
|
+
set_mode(path, mode).map_err(|e| (i, at(path)(e)))?;
|
|
440
|
+
}
|
|
392
441
|
}
|
|
393
442
|
Ok(())
|
|
394
443
|
}
|
|
@@ -427,18 +476,27 @@ impl<'a> Extractor<'a> {
|
|
|
427
476
|
}
|
|
428
477
|
Err(e) => return Err(at(path)(e)),
|
|
429
478
|
};
|
|
430
|
-
let written = self
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
//
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
479
|
+
let written = match self.copy_entry(spec.index, &mut file) {
|
|
480
|
+
Ok(n) => n,
|
|
481
|
+
Err(failure) => {
|
|
482
|
+
// Interrupted or failed part-way: a half-written file
|
|
483
|
+
// would pass for the entry, and stand in the way of a
|
|
484
|
+
// second run.
|
|
485
|
+
drop(file);
|
|
486
|
+
let _ = fs::remove_file(path);
|
|
487
|
+
return Err(match failure {
|
|
488
|
+
// Reading the entry is the entry's problem; writing is
|
|
489
|
+
// the path's.
|
|
490
|
+
Failure::Io(e)
|
|
491
|
+
if e.kind() == io::ErrorKind::WriteZero
|
|
492
|
+
|| e.kind() == io::ErrorKind::StorageFull =>
|
|
493
|
+
{
|
|
494
|
+
at(path)(e)
|
|
495
|
+
}
|
|
496
|
+
other => other,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
};
|
|
442
500
|
if let Some(mtime) = spec.mtime {
|
|
443
501
|
file.set_times(FileTimes::new().set_accessed(mtime).set_modified(mtime))
|
|
444
502
|
.map_err(at(path))?;
|
|
@@ -495,6 +553,7 @@ impl<'a> Extractor<'a> {
|
|
|
495
553
|
size: raw.size(),
|
|
496
554
|
compressed_size: raw.compressed_size(),
|
|
497
555
|
crc32: raw.crc32(),
|
|
556
|
+
method: Some(crate::archive::compression_code(raw.compression())),
|
|
498
557
|
}
|
|
499
558
|
};
|
|
500
559
|
let mut source = self.archive.clone().into_inner();
|
data/ext/ziperu/src/lib.rs
CHANGED
|
@@ -13,6 +13,7 @@ mod rb;
|
|
|
13
13
|
mod reader;
|
|
14
14
|
mod sink;
|
|
15
15
|
mod source;
|
|
16
|
+
mod stream;
|
|
16
17
|
mod stream_stats;
|
|
17
18
|
mod writer;
|
|
18
19
|
mod zipcrypto;
|
|
@@ -43,15 +44,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
43
44
|
method!(archive::Archive::has_duplicate_names, 0),
|
|
44
45
|
)?;
|
|
45
46
|
archive.define_method("comment", method!(archive::Archive::comment, 0))?;
|
|
46
|
-
archive.define_method("
|
|
47
|
-
archive.define_method(
|
|
48
|
-
"max_entry_size",
|
|
49
|
-
method!(archive::Archive::max_entry_size, 0),
|
|
50
|
-
)?;
|
|
51
|
-
archive.define_method(
|
|
52
|
-
"max_compression_ratio",
|
|
53
|
-
method!(archive::Archive::max_compression_ratio, 0),
|
|
54
|
-
)?;
|
|
47
|
+
archive.define_method("limits", method!(archive::Archive::limits, 0))?;
|
|
55
48
|
archive.define_method("verify_entry", method!(archive::Archive::verify_entry, 2))?;
|
|
56
49
|
archive.define_method(
|
|
57
50
|
"verify_entry_at",
|
|
@@ -73,8 +66,17 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
73
66
|
reader.define_method("eof?", method!(reader::EntryReader::is_eof, 0))?;
|
|
74
67
|
reader.define_method("read", method!(reader::EntryReader::read, 1))?;
|
|
75
68
|
reader.define_method("rewind", method!(reader::EntryReader::rewind, 0))?;
|
|
69
|
+
reader.define_method("seek", method!(reader::EntryReader::seek, 1))?;
|
|
76
70
|
reader.define_method("close", method!(reader::EntryReader::close, 0))?;
|
|
77
71
|
|
|
72
|
+
let stream = native.define_class("StreamReader", ruby.class_object())?;
|
|
73
|
+
stream.define_singleton_method("open", function!(stream::StreamReader::new, 3))?;
|
|
74
|
+
stream.define_method("next_entry", method!(stream::StreamReader::next_entry, 0))?;
|
|
75
|
+
stream.define_method("read", method!(stream::StreamReader::read, 1))?;
|
|
76
|
+
stream.define_method("eof?", method!(stream::StreamReader::is_eof, 0))?;
|
|
77
|
+
stream.define_method("trailer", method!(stream::StreamReader::trailer, 0))?;
|
|
78
|
+
stream.define_method("close", method!(stream::StreamReader::close, 0))?;
|
|
79
|
+
|
|
78
80
|
let writer = native.define_class("Writer", ruby.class_object())?;
|
|
79
81
|
writer.define_singleton_method("open_path", function!(writer::Writer::open_path, 1))?;
|
|
80
82
|
writer.define_singleton_method("open_memory", function!(writer::Writer::open_memory, 0))?;
|
|
@@ -94,6 +96,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
94
96
|
writer.define_method("raw_copy", method!(writer::Writer::raw_copy, 4))?;
|
|
95
97
|
writer.define_method("copy_zipcrypto", method!(writer::Writer::copy_zipcrypto, 4))?;
|
|
96
98
|
writer.define_method("comment=", method!(writer::Writer::set_comment, 1))?;
|
|
99
|
+
writer.define_method("rollback_entry", method!(writer::Writer::rollback_entry, 0))?;
|
|
97
100
|
writer.define_method("finish", method!(writer::Writer::finish, 0))?;
|
|
98
101
|
writer.define_method("closed?", method!(writer::Writer::is_closed, 0))?;
|
|
99
102
|
|
data/ext/ziperu/src/reader.rs
CHANGED
|
@@ -58,6 +58,20 @@ pub(crate) struct EntryInfo {
|
|
|
58
58
|
pub(crate) size: u64,
|
|
59
59
|
pub(crate) compressed_size: u64,
|
|
60
60
|
pub(crate) crc32: u32,
|
|
61
|
+
/// The central directory's compression method, when known. The local
|
|
62
|
+
/// header must agree: reading through it must not decode the data
|
|
63
|
+
/// differently from reading by index.
|
|
64
|
+
pub(crate) method: Option<u16>,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/// The compression method in the local header at the source's position,
|
|
68
|
+
/// which is left where it was.
|
|
69
|
+
fn local_method(source: &mut Source, header_start: u64) -> ZipResult<u16> {
|
|
70
|
+
let mut head = [0u8; 10];
|
|
71
|
+
source.seek(SeekFrom::Start(header_start))?;
|
|
72
|
+
source.read_exact(&mut head)?;
|
|
73
|
+
source.seek(SeekFrom::Start(header_start))?;
|
|
74
|
+
Ok(u16::from_le_bytes([head[8], head[9]]))
|
|
61
75
|
}
|
|
62
76
|
|
|
63
77
|
/// Opens the entry `info` describes by seeking to its local header, with
|
|
@@ -71,6 +85,13 @@ pub(crate) fn open_local_entry<'a>(
|
|
|
71
85
|
password: Option<&'a [u8]>,
|
|
72
86
|
size_cap: u64,
|
|
73
87
|
) -> ZipResult<ZipFile<'a, Source>> {
|
|
88
|
+
if let Some(method) = info.method {
|
|
89
|
+
if local_method(source, info.header_start)? != method {
|
|
90
|
+
return Err(ZipError::InvalidArchive(
|
|
91
|
+
"local header compression method differs from the central directory".into(),
|
|
92
|
+
));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
74
95
|
source.seek(SeekFrom::Start(info.header_start))?;
|
|
75
96
|
let options = ZipReadOptions::new()
|
|
76
97
|
.password(password)
|
|
@@ -150,10 +171,27 @@ struct Inner {
|
|
|
150
171
|
info: EntryInfo,
|
|
151
172
|
/// Declared size, or `None` when reading to the true end of the data.
|
|
152
173
|
size: Option<u64>,
|
|
174
|
+
/// Bytes delivered to Ruby so far.
|
|
153
175
|
consumed: u64,
|
|
154
176
|
finished: bool,
|
|
177
|
+
/// Bytes already decompressed but not yet handed out: what a read
|
|
178
|
+
/// had gathered when it was interrupted, served first by the next.
|
|
179
|
+
pending: Vec<u8>,
|
|
180
|
+
/// Set when a read failed part-way for any reason but an interrupt:
|
|
181
|
+
/// the decoder's state is then unknown, so later reads raise rather
|
|
182
|
+
/// than return data that may have a hole in it. `rewind` clears it.
|
|
183
|
+
failed: Option<String>,
|
|
155
184
|
/// Reads over a Ruby IO must keep the GVL; the rest release it.
|
|
156
185
|
needs_gvl: bool,
|
|
186
|
+
/// For a Stored entry that is not encrypted, which `seek` can move
|
|
187
|
+
/// about in: a source of its own over the archive's bytes.
|
|
188
|
+
raw_source: Option<Source>,
|
|
189
|
+
/// Where the entry's data begins, once a seek has looked it up.
|
|
190
|
+
data_start: Option<u64>,
|
|
191
|
+
/// After a seek, reads come straight from `raw_source`: the absolute
|
|
192
|
+
/// position of the next byte, and of the end of the data. A read
|
|
193
|
+
/// this way checks no CRC (only reading from the start can).
|
|
194
|
+
raw: Option<(u64, u64)>,
|
|
157
195
|
}
|
|
158
196
|
|
|
159
197
|
#[derive(TypedData)]
|
|
@@ -188,20 +226,104 @@ impl EntryReader {
|
|
|
188
226
|
needs_gvl: bool,
|
|
189
227
|
io: Option<Opaque<Value>>,
|
|
190
228
|
) -> Result<Self, Error> {
|
|
229
|
+
let seekable;
|
|
191
230
|
let info = {
|
|
192
231
|
// From the parsed central directory: no seek to the local
|
|
193
232
|
// header, which the reader is about to do anyway.
|
|
194
233
|
let meta = archive.metadata();
|
|
195
234
|
let raw = meta.entry(index).rb(ruby)?;
|
|
235
|
+
seekable = raw.compression() == zip::CompressionMethod::Stored && !raw.encrypted();
|
|
196
236
|
EntryInfo {
|
|
197
237
|
index: Some(index),
|
|
198
238
|
header_start: raw.header_start(),
|
|
199
239
|
size: raw.size(),
|
|
200
240
|
compressed_size: raw.compressed_size(),
|
|
201
241
|
crc32: raw.crc32(),
|
|
242
|
+
method: Some(crate::archive::compression_code(raw.compression())),
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
let raw_source =
|
|
246
|
+
(seekable && enforce_size).then(|| archive.clone().into_inner().clone_small());
|
|
247
|
+
let reader = Self::open_at(ruby, archive, info, password, enforce_size, needs_gvl, io)?;
|
|
248
|
+
reader.inner.borrow_mut().raw_source = raw_source;
|
|
249
|
+
Ok(reader)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/// Moves to byte `pos` of the entry's data (the end, if past it). Only
|
|
253
|
+
/// a Stored entry that is not encrypted has its bytes where they can
|
|
254
|
+
/// be reached directly; any other must be read from the start.
|
|
255
|
+
pub fn seek(ruby: &Ruby, rb_self: &Self, pos: u64) -> Result<(), Error> {
|
|
256
|
+
let mut inner = rb_self.inner(ruby)?;
|
|
257
|
+
if inner.entry.is_none() {
|
|
258
|
+
return Err(Error::new(ruby.exception_io_error(), "closed stream"));
|
|
259
|
+
}
|
|
260
|
+
let (Some(size), true) = (inner.size, inner.raw_source.is_some()) else {
|
|
261
|
+
return Err(Error::new(
|
|
262
|
+
ruby.get_inner(&crate::errors::ERROR),
|
|
263
|
+
"only a Stored entry that is not encrypted can be seeked in; \
|
|
264
|
+
a compressed one must be read from the start (rewind)",
|
|
265
|
+
));
|
|
266
|
+
};
|
|
267
|
+
let data_start = match inner.data_start {
|
|
268
|
+
Some(at) => at,
|
|
269
|
+
None => {
|
|
270
|
+
let header_start = inner.info.header_start;
|
|
271
|
+
let source = inner.raw_source.as_mut().expect("checked above");
|
|
272
|
+
let at = data_offset(source, header_start)
|
|
273
|
+
.map_err(|e| crate::errors::header_error(ruby, e))?;
|
|
274
|
+
inner.data_start = Some(at);
|
|
275
|
+
at
|
|
202
276
|
}
|
|
203
277
|
};
|
|
204
|
-
|
|
278
|
+
let pos = pos.min(size);
|
|
279
|
+
inner.raw = Some((data_start + pos, data_start + size));
|
|
280
|
+
inner.consumed = pos;
|
|
281
|
+
inner.pending.clear();
|
|
282
|
+
inner.failed = None;
|
|
283
|
+
inner.finished = pos >= size;
|
|
284
|
+
Ok(())
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/// `read` after a seek: straight from the archive's bytes.
|
|
288
|
+
fn read_raw(
|
|
289
|
+
ruby: &Ruby,
|
|
290
|
+
inner: &mut Inner,
|
|
291
|
+
maxlen: Option<usize>,
|
|
292
|
+
) -> Result<Option<RString>, Error> {
|
|
293
|
+
let (at, end) = inner.raw.expect("called after a seek");
|
|
294
|
+
let remaining = end - at;
|
|
295
|
+
if remaining == 0 {
|
|
296
|
+
inner.finished = true;
|
|
297
|
+
return Ok(None);
|
|
298
|
+
}
|
|
299
|
+
let want = maxlen.map_or(remaining, |n| (n as u64).min(remaining));
|
|
300
|
+
let keep_gvl = inner.needs_gvl || want < INLINE_READ;
|
|
301
|
+
let source = inner.raw_source.as_mut().expect("seekable");
|
|
302
|
+
let mut buf = Vec::new();
|
|
303
|
+
maybe_without_gvl(keep_gvl, || -> io::Result<()> {
|
|
304
|
+
source.seek_to(at)?;
|
|
305
|
+
while (buf.len() as u64) < want {
|
|
306
|
+
crate::gvl::check()?;
|
|
307
|
+
let step = (want - buf.len() as u64).min(CHUNK as u64) as usize;
|
|
308
|
+
let start = buf.len();
|
|
309
|
+
buf.resize(start + step, 0);
|
|
310
|
+
let n = source.read(&mut buf[start..])?;
|
|
311
|
+
buf.truncate(start + n);
|
|
312
|
+
if n == 0 {
|
|
313
|
+
return Err(io::Error::new(
|
|
314
|
+
io::ErrorKind::UnexpectedEof,
|
|
315
|
+
"the entry's data ends before its declared size",
|
|
316
|
+
));
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
Ok(())
|
|
320
|
+
})
|
|
321
|
+
.map_err(|e| read_error(ruby, e))?;
|
|
322
|
+
let n = buf.len() as u64;
|
|
323
|
+
inner.raw = Some((at + n, end));
|
|
324
|
+
inner.consumed += n;
|
|
325
|
+
inner.finished = at + n >= end;
|
|
326
|
+
Ok(Some(ruby.str_from_slice(&buf)))
|
|
205
327
|
}
|
|
206
328
|
|
|
207
329
|
/// A reader over the entry `info` describes, which need not be one the
|
|
@@ -228,7 +350,12 @@ impl EntryReader {
|
|
|
228
350
|
size,
|
|
229
351
|
consumed: 0,
|
|
230
352
|
finished,
|
|
353
|
+
pending: Vec::new(),
|
|
354
|
+
failed: None,
|
|
231
355
|
needs_gvl,
|
|
356
|
+
raw_source: None,
|
|
357
|
+
data_start: None,
|
|
358
|
+
raw: None,
|
|
232
359
|
}),
|
|
233
360
|
io,
|
|
234
361
|
})
|
|
@@ -270,7 +397,10 @@ impl EntryReader {
|
|
|
270
397
|
maxlen: Option<usize>,
|
|
271
398
|
) -> Result<Option<RString>, Error> {
|
|
272
399
|
let mut inner = rb_self.inner(ruby)?;
|
|
273
|
-
if inner.
|
|
400
|
+
if let Some(reason) = &inner.failed {
|
|
401
|
+
return Err(failed_error(ruby, reason));
|
|
402
|
+
}
|
|
403
|
+
if inner.finished && inner.pending.is_empty() {
|
|
274
404
|
return Ok(None);
|
|
275
405
|
}
|
|
276
406
|
if maxlen == Some(0) {
|
|
@@ -278,6 +408,9 @@ impl EntryReader {
|
|
|
278
408
|
// entry: below, an empty fill means exactly that.
|
|
279
409
|
return Ok(Some(ruby.str_new("")));
|
|
280
410
|
}
|
|
411
|
+
if inner.raw.is_some() {
|
|
412
|
+
return Self::read_raw(ruby, &mut inner, maxlen);
|
|
413
|
+
}
|
|
281
414
|
let remaining = inner.size.map(|s| s - inner.consumed).unwrap_or(u64::MAX);
|
|
282
415
|
let want = match maxlen {
|
|
283
416
|
Some(n) => (n as u64).min(remaining),
|
|
@@ -290,12 +423,43 @@ impl EntryReader {
|
|
|
290
423
|
// Releasing the GVL costs about as much as inflating a few KiB, so
|
|
291
424
|
// tiny reads (`getc`, `each_byte`) keep it.
|
|
292
425
|
let keep_gvl = inner.needs_gvl || want < INLINE_READ;
|
|
293
|
-
let
|
|
294
|
-
|
|
295
|
-
|
|
426
|
+
let mut buf = std::mem::take(&mut inner.pending);
|
|
427
|
+
if !buf.is_empty() && buf.len() as u64 >= want {
|
|
428
|
+
// Served from what an interrupted read left behind. Should
|
|
429
|
+
// this reach the declared size, the next read probes past it.
|
|
430
|
+
inner.pending = buf.split_off(want as usize);
|
|
431
|
+
inner.consumed += want;
|
|
432
|
+
return Ok(Some(ruby.str_from_slice(&buf)));
|
|
433
|
+
}
|
|
434
|
+
let Inner {
|
|
435
|
+
entry, finished, ..
|
|
436
|
+
} = &mut *inner;
|
|
437
|
+
let file = match entry.as_mut().and_then(|e| e.file.as_mut()) {
|
|
438
|
+
Some(file) if !*finished => file,
|
|
439
|
+
// Closed, a rewind failed to reopen it, or the data has ended
|
|
440
|
+
// and only pending bytes were left.
|
|
441
|
+
_ => {
|
|
442
|
+
if buf.is_empty() {
|
|
443
|
+
return Ok(None);
|
|
444
|
+
}
|
|
445
|
+
inner.consumed += buf.len() as u64;
|
|
446
|
+
return Ok(Some(ruby.str_from_slice(&buf)));
|
|
447
|
+
}
|
|
448
|
+
};
|
|
449
|
+
let outcome = maybe_without_gvl(keep_gvl, || fill(file, &mut buf, want, probe));
|
|
450
|
+
let (eof, oversize) = match outcome {
|
|
451
|
+
Ok(flags) => flags,
|
|
452
|
+
Err(e) => {
|
|
453
|
+
if crate::gvl::is_cancelled(&e) {
|
|
454
|
+
// Interrupted between chunks: the decoder is intact,
|
|
455
|
+
// and what it produced so far waits for the next read.
|
|
456
|
+
inner.pending = buf;
|
|
457
|
+
} else {
|
|
458
|
+
inner.failed = Some(e.to_string());
|
|
459
|
+
}
|
|
460
|
+
return Err(read_error(ruby, e));
|
|
461
|
+
}
|
|
296
462
|
};
|
|
297
|
-
let (buf, eof, oversize) = maybe_without_gvl(keep_gvl, || fill(file, want, probe))
|
|
298
|
-
.map_err(|e| read_error(ruby, e))?;
|
|
299
463
|
|
|
300
464
|
let filled = buf.len() as u64;
|
|
301
465
|
inner.consumed += filled;
|
|
@@ -341,19 +505,53 @@ impl EntryReader {
|
|
|
341
505
|
}
|
|
342
506
|
*consumed = 0;
|
|
343
507
|
*finished = *size == Some(0) && info.compressed_size == 0;
|
|
508
|
+
inner.pending.clear();
|
|
509
|
+
inner.failed = None;
|
|
510
|
+
inner.raw = None;
|
|
344
511
|
Ok(())
|
|
345
512
|
}
|
|
346
513
|
}
|
|
347
514
|
|
|
348
|
-
///
|
|
349
|
-
///
|
|
350
|
-
|
|
515
|
+
/// Where the data of the entry whose local header is at `header_start`
|
|
516
|
+
/// begins: after the header's fixed part, the name and the extra field.
|
|
517
|
+
fn data_offset(source: &mut Source, header_start: u64) -> io::Result<u64> {
|
|
518
|
+
const SIGNATURE: u32 = 0x0403_4b50;
|
|
519
|
+
const HEADER_LEN: u64 = 30;
|
|
520
|
+
let mut header = [0u8; HEADER_LEN as usize];
|
|
521
|
+
source.seek_to(header_start)?;
|
|
522
|
+
source.read_exact(&mut header)?;
|
|
523
|
+
if u32::from_le_bytes([header[0], header[1], header[2], header[3]]) != SIGNATURE {
|
|
524
|
+
return Err(io::Error::new(
|
|
525
|
+
io::ErrorKind::InvalidData,
|
|
526
|
+
"local file header signature not found",
|
|
527
|
+
));
|
|
528
|
+
}
|
|
529
|
+
let name_len = u64::from(u16::from_le_bytes([header[26], header[27]]));
|
|
530
|
+
let extra_len = u64::from(u16::from_le_bytes([header[28], header[29]]));
|
|
531
|
+
Ok(header_start + HEADER_LEN + name_len + extra_len)
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/// The error every read raises after one failed part-way.
|
|
535
|
+
fn failed_error(ruby: &Ruby, reason: &str) -> Error {
|
|
536
|
+
Error::new(
|
|
537
|
+
ruby.get_inner(&crate::errors::ERROR),
|
|
538
|
+
format!(
|
|
539
|
+
"entry cannot be read further after an earlier error ({reason}); rewind to start again"
|
|
540
|
+
),
|
|
541
|
+
)
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/// Reads into `buf` until it holds `want` bytes, in `CHUNK` steps, pure
|
|
545
|
+
/// Rust so it can run without the GVL. Returns whether the entry ended
|
|
546
|
+
/// early and (when `probe`) whether data continues past the declared
|
|
547
|
+
/// size. On an error, `buf` keeps what was read before it.
|
|
351
548
|
fn fill(
|
|
352
549
|
file: &mut ZipFile<'static, Source>,
|
|
550
|
+
buf: &mut Vec<u8>,
|
|
353
551
|
want: u64,
|
|
354
552
|
probe: bool,
|
|
355
|
-
) -> io::Result<(
|
|
356
|
-
|
|
553
|
+
) -> io::Result<(bool, bool)> {
|
|
554
|
+
buf.reserve(want.saturating_sub(buf.len() as u64).min(CHUNK as u64) as usize);
|
|
357
555
|
let mut eof = false;
|
|
358
556
|
while (buf.len() as u64) < want {
|
|
359
557
|
crate::gvl::check()?;
|
|
@@ -372,5 +570,5 @@ fn fill(
|
|
|
372
570
|
let mut extra = [0u8; 1];
|
|
373
571
|
oversize = file.read(&mut extra)? != 0;
|
|
374
572
|
}
|
|
375
|
-
Ok((
|
|
573
|
+
Ok((eof, oversize))
|
|
376
574
|
}
|