ziperu 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/Cargo.lock +1 -1
- data/README.md +56 -5
- data/ext/ziperu/Cargo.toml +1 -1
- data/ext/ziperu/src/archive.rs +85 -54
- data/ext/ziperu/src/errors.rs +76 -14
- data/ext/ziperu/src/extract.rs +92 -33
- data/ext/ziperu/src/lib.rs +12 -9
- data/ext/ziperu/src/reader.rs +211 -13
- data/ext/ziperu/src/sink.rs +48 -8
- data/ext/ziperu/src/source.rs +2 -2
- data/ext/ziperu/src/stream.rs +541 -0
- data/ext/ziperu/src/writer.rs +128 -5
- data/lib/ziperu/entry/source.rb +1 -1
- data/lib/ziperu/entry.rb +8 -1
- data/lib/ziperu/errors.rb +5 -0
- data/lib/ziperu/file.rb +24 -2
- data/lib/ziperu/input_stream.rb +167 -11
- data/lib/ziperu/output_stream.rb +72 -1
- data/lib/ziperu/remote_io.rb +278 -0
- data/lib/ziperu/replace_rubyzip.rb +67 -0
- data/lib/ziperu/version.rb +1 -1
- data/lib/ziperu.rb +4 -0
- metadata +4 -1
data/ext/ziperu/src/writer.rs
CHANGED
|
@@ -46,6 +46,10 @@ pub struct Writer {
|
|
|
46
46
|
/// Cleared when this object is freed, before the `ZipWriter` inside
|
|
47
47
|
/// drops, so that its finalizer cannot reach the IO (see `RubyOut`).
|
|
48
48
|
attached: Option<Arc<AtomicBool>>,
|
|
49
|
+
/// Set by the sink once the Ruby IO has raised: the crate and the
|
|
50
|
+
/// Ruby side may no longer agree on what the archive holds (the crate
|
|
51
|
+
/// takes back an entry it was finishing), so nothing more is written.
|
|
52
|
+
broken: Option<Arc<AtomicBool>>,
|
|
49
53
|
}
|
|
50
54
|
|
|
51
55
|
impl DataTypeFunctions for Writer {
|
|
@@ -143,6 +147,11 @@ struct State {
|
|
|
143
147
|
const INLINE_WRITE: usize = 16 * 1024;
|
|
144
148
|
/// How much of a large write goes in between checks for an interrupt.
|
|
145
149
|
const WRITE_CHUNK: usize = 1024 * 1024;
|
|
150
|
+
/// `add_files_parallel` hands the crate this many files per thread at a
|
|
151
|
+
/// time, checking for an interrupt in between.
|
|
152
|
+
const PARALLEL_PART_PER_THREAD: usize = 4;
|
|
153
|
+
/// ...or this many bytes per thread, once each thread has a file.
|
|
154
|
+
const PARALLEL_PART_BYTES_PER_THREAD: u64 = 8 * 1024 * 1024;
|
|
146
155
|
|
|
147
156
|
impl Writer {
|
|
148
157
|
pub fn open_path(ruby: &Ruby, path: String) -> Result<Self, Error> {
|
|
@@ -167,12 +176,14 @@ impl Writer {
|
|
|
167
176
|
/// start, entry by entry, as the stream takes shape.
|
|
168
177
|
pub fn open_io(io: Value) -> Result<Self, Error> {
|
|
169
178
|
let attached = Arc::new(AtomicBool::new(true));
|
|
170
|
-
let
|
|
179
|
+
let broken = Arc::new(AtomicBool::new(false));
|
|
180
|
+
let out = RubyOut::new(io, Arc::clone(&attached), Arc::clone(&broken), true)?;
|
|
171
181
|
let sink = Sink::Ruby(BufWriter::with_capacity(BUF_SIZE, out));
|
|
172
182
|
Ok(Self::over_ruby_io(
|
|
173
183
|
Zip::Seekable(ZipWriter::new(sink)),
|
|
174
184
|
io,
|
|
175
185
|
attached,
|
|
186
|
+
broken,
|
|
176
187
|
))
|
|
177
188
|
}
|
|
178
189
|
|
|
@@ -183,12 +194,14 @@ impl Writer {
|
|
|
183
194
|
/// patched at the end.
|
|
184
195
|
pub fn open_stream(io: Value) -> Result<Self, Error> {
|
|
185
196
|
let attached = Arc::new(AtomicBool::new(true));
|
|
186
|
-
let
|
|
197
|
+
let broken = Arc::new(AtomicBool::new(false));
|
|
198
|
+
let out = RubyOut::new(io, Arc::clone(&attached), Arc::clone(&broken), false)?;
|
|
187
199
|
let (counting, stats) = Counting::new(Sink::Ruby(BufWriter::with_capacity(BUF_SIZE, out)));
|
|
188
200
|
Ok(Self::over_ruby_io(
|
|
189
201
|
Zip::Stream(ZipWriter::new_stream(counting), Box::new(stats)),
|
|
190
202
|
io,
|
|
191
203
|
attached,
|
|
204
|
+
broken,
|
|
192
205
|
))
|
|
193
206
|
}
|
|
194
207
|
|
|
@@ -237,14 +250,32 @@ impl Writer {
|
|
|
237
250
|
inner: RefCell::new(State::new(Zip::Seekable(zip))),
|
|
238
251
|
io: None,
|
|
239
252
|
attached: None,
|
|
253
|
+
broken: None,
|
|
240
254
|
}
|
|
241
255
|
}
|
|
242
256
|
|
|
243
|
-
fn over_ruby_io(
|
|
257
|
+
fn over_ruby_io(
|
|
258
|
+
zip: Zip,
|
|
259
|
+
io: Value,
|
|
260
|
+
attached: Arc<AtomicBool>,
|
|
261
|
+
broken: Arc<AtomicBool>,
|
|
262
|
+
) -> Self {
|
|
244
263
|
Self {
|
|
245
264
|
inner: RefCell::new(State::new(zip)),
|
|
246
265
|
io: Some(Opaque::from(io)),
|
|
247
266
|
attached: Some(attached),
|
|
267
|
+
broken: Some(broken),
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/// An error once the sink's IO has raised (see `broken`).
|
|
272
|
+
fn check_broken(&self, ruby: &Ruby) -> Result<(), Error> {
|
|
273
|
+
match &self.broken {
|
|
274
|
+
Some(b) if b.load(Ordering::Acquire) => Err(Error::new(
|
|
275
|
+
ruby.get_inner(&crate::errors::ERROR),
|
|
276
|
+
"the archive cannot be written further: its IO raised an error earlier",
|
|
277
|
+
)),
|
|
278
|
+
_ => Ok(()),
|
|
248
279
|
}
|
|
249
280
|
}
|
|
250
281
|
|
|
@@ -665,9 +696,47 @@ impl Writer {
|
|
|
665
696
|
let Zip::Seekable(w) = zip else {
|
|
666
697
|
return Err(stream_error(ruby, "entries added in parallel"));
|
|
667
698
|
};
|
|
699
|
+
// The crate cannot be interrupted inside a call, so the batch
|
|
700
|
+
// goes in parts of a few files per thread, with a check for an
|
|
701
|
+
// interrupt (see `gvl`) between them. Should a part fail or the
|
|
702
|
+
// call be interrupted, the parts already in are taken back out
|
|
703
|
+
// again, last first, so the batch is added whole or not at all.
|
|
704
|
+
let part_len = threads.max(1) * PARALLEL_PART_PER_THREAD;
|
|
705
|
+
let part_bytes = threads.max(1) as u64 * PARALLEL_PART_BYTES_PER_THREAD;
|
|
668
706
|
maybe_without_gvl(rb_self.needs_gvl(), || -> ZipResult<()> {
|
|
669
707
|
state.crypto.seal_pending(w)?;
|
|
670
|
-
|
|
708
|
+
let mut added = 0;
|
|
709
|
+
let mut rest = entries.into_iter();
|
|
710
|
+
let outcome = loop {
|
|
711
|
+
// At least a file per thread, then up to `part_len`
|
|
712
|
+
// files or `part_bytes`, whichever comes first.
|
|
713
|
+
let mut part = Vec::new();
|
|
714
|
+
let mut bytes = 0u64;
|
|
715
|
+
while part.len() < part_len
|
|
716
|
+
&& (part.len() < threads.max(1) || bytes < part_bytes)
|
|
717
|
+
{
|
|
718
|
+
let Some(item) = rest.next() else { break };
|
|
719
|
+
bytes += fs::metadata(item.1).map_or(0, |m| m.len());
|
|
720
|
+
part.push(item);
|
|
721
|
+
}
|
|
722
|
+
if part.is_empty() {
|
|
723
|
+
break Ok(());
|
|
724
|
+
}
|
|
725
|
+
if let Err(e) = crate::gvl::check() {
|
|
726
|
+
break Err(ZipError::Io(e));
|
|
727
|
+
}
|
|
728
|
+
let len = part.len();
|
|
729
|
+
if let Err(e) = pool.install(|| w.par_append_from_paths_atomic(part)) {
|
|
730
|
+
break Err(e);
|
|
731
|
+
}
|
|
732
|
+
added += len;
|
|
733
|
+
};
|
|
734
|
+
if outcome.is_err() {
|
|
735
|
+
for _ in 0..added {
|
|
736
|
+
let _ = w.abort_file();
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
outcome
|
|
671
740
|
})
|
|
672
741
|
.map_err(|e| entry_error(ruby, b"", e))?;
|
|
673
742
|
state.count += parsed.len();
|
|
@@ -675,6 +744,40 @@ impl Writer {
|
|
|
675
744
|
})
|
|
676
745
|
}
|
|
677
746
|
|
|
747
|
+
/// Takes the entry last begun back out of the archive: the central
|
|
748
|
+
/// directory will not list it, and the next entry is written over its
|
|
749
|
+
/// bytes. A stream cannot go back over bytes it has sent, so there
|
|
750
|
+
/// only an entry not yet begun can be taken back, which Ruby handles
|
|
751
|
+
/// by deferring the start (see `OutputStream#rollback_entry`).
|
|
752
|
+
pub fn rollback_entry(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
|
|
753
|
+
rb_self.with_state(ruby, |state, zip| {
|
|
754
|
+
if zip.is_stream() {
|
|
755
|
+
return Err(Error::new(
|
|
756
|
+
ruby.get_inner(&crate::errors::ERROR),
|
|
757
|
+
"the entry's data has already gone out on a stream that cannot seek, and cannot be taken back",
|
|
758
|
+
));
|
|
759
|
+
}
|
|
760
|
+
let base = state.append.as_ref().map_or(0, |a| a.base);
|
|
761
|
+
if state.count <= base {
|
|
762
|
+
return Err(Error::new(
|
|
763
|
+
ruby.get_inner(&crate::errors::ERROR),
|
|
764
|
+
"there is no entry to take back",
|
|
765
|
+
));
|
|
766
|
+
}
|
|
767
|
+
let index = state.count - 1;
|
|
768
|
+
// A ZipCrypto entry in progress has nothing to seal now, and
|
|
769
|
+
// no header patch of the entry may land on the next one.
|
|
770
|
+
state.crypto.pending = None;
|
|
771
|
+
state.crypto.patches.retain(|p| p.index != index);
|
|
772
|
+
let Zip::Seekable(w) = zip else {
|
|
773
|
+
unreachable!("a stream was refused above")
|
|
774
|
+
};
|
|
775
|
+
maybe_without_gvl(rb_self.needs_gvl(), || discard_last(w)).rb(ruby)?;
|
|
776
|
+
state.count = index;
|
|
777
|
+
Ok(())
|
|
778
|
+
})
|
|
779
|
+
}
|
|
780
|
+
|
|
678
781
|
pub fn set_comment(ruby: &Ruby, rb_self: &Self, comment: RString) -> Result<(), Error> {
|
|
679
782
|
let bytes = bytes_of(comment).into_boxed_slice();
|
|
680
783
|
rb_self.with_state(ruby, |_, zip| {
|
|
@@ -688,6 +791,7 @@ impl Writer {
|
|
|
688
791
|
/// written (for an append, per entry added), which is how Ruby learns
|
|
689
792
|
/// what a streamed entry came to.
|
|
690
793
|
pub fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(Option<RString>, RArray), Error> {
|
|
794
|
+
rb_self.check_broken(ruby)?;
|
|
691
795
|
let (zip, crypto, base) = {
|
|
692
796
|
let mut state = rb_self.borrow(ruby)?;
|
|
693
797
|
let zip = state.zip.take().ok_or_else(|| closed_error(ruby))?;
|
|
@@ -759,7 +863,10 @@ impl Writer {
|
|
|
759
863
|
/// written over its bytes. Best effort; the error that brought us here
|
|
760
864
|
/// is the one reported.
|
|
761
865
|
fn withdraw(needs_gvl: bool, state: &mut State, zip: &mut Zip) {
|
|
762
|
-
let _ = maybe_without_gvl(needs_gvl, ||
|
|
866
|
+
let _ = maybe_without_gvl(needs_gvl, || match zip {
|
|
867
|
+
Zip::Seekable(w) => discard_last(w),
|
|
868
|
+
Zip::Stream(w, _) => w.abort_file(),
|
|
869
|
+
});
|
|
763
870
|
state.count = state.count.saturating_sub(1);
|
|
764
871
|
}
|
|
765
872
|
|
|
@@ -774,6 +881,7 @@ impl Writer {
|
|
|
774
881
|
ruby: &Ruby,
|
|
775
882
|
f: impl FnOnce(&mut State, &mut Zip) -> Result<T, Error>,
|
|
776
883
|
) -> Result<T, Error> {
|
|
884
|
+
self.check_broken(ruby)?;
|
|
777
885
|
let mut guard = self.borrow(ruby)?;
|
|
778
886
|
let state = &mut *guard;
|
|
779
887
|
let mut zip = state.zip.take().ok_or_else(|| closed_error(ruby))?;
|
|
@@ -794,6 +902,21 @@ impl State {
|
|
|
794
902
|
}
|
|
795
903
|
}
|
|
796
904
|
|
|
905
|
+
/// Takes the last entry back out of a seekable archive and cuts the sink
|
|
906
|
+
/// off where it began. The crate rewinds to the entry's header but cannot
|
|
907
|
+
/// shorten the sink, and at the end it would move the central directory
|
|
908
|
+
/// past the dead bytes rather than leave them after it.
|
|
909
|
+
fn discard_last(w: &mut ZipWriter<Sink>) -> ZipResult<()> {
|
|
910
|
+
w.abort_file()?;
|
|
911
|
+
// SAFETY: the sink is cut at its current position and left there, as
|
|
912
|
+
// `get_mut` requires; nothing the crate has written lies beyond it.
|
|
913
|
+
if let Some(sink) = unsafe { w.get_mut() } {
|
|
914
|
+
let at = sink.stream_position()?;
|
|
915
|
+
sink.end_at(at)?;
|
|
916
|
+
}
|
|
917
|
+
Ok(())
|
|
918
|
+
}
|
|
919
|
+
|
|
797
920
|
/// Seals a ZipCrypto entry still in progress, writes the central
|
|
798
921
|
/// directory, reads the archive back for the per-entry facts, and patches
|
|
799
922
|
/// the headers of ZipCrypto entries.
|
data/lib/ziperu/entry/source.rb
CHANGED
|
@@ -96,7 +96,7 @@ module Ziperu
|
|
|
96
96
|
reopened = @owner.native_archive
|
|
97
97
|
unless still_at_place?(reopened, offset)
|
|
98
98
|
where = @index ? "at index #{@index} of" : 'in'
|
|
99
|
-
raise
|
|
99
|
+
raise EntryMovedError, "entry '#{@raw_name}' is no longer #{where} #{@owner.name}"
|
|
100
100
|
end
|
|
101
101
|
|
|
102
102
|
@native = reopened
|
data/lib/ziperu/entry.rb
CHANGED
|
@@ -86,7 +86,14 @@ module Ziperu
|
|
|
86
86
|
def extra
|
|
87
87
|
if @source.stored? && !@local_extra_merged
|
|
88
88
|
@local_extra_merged = true
|
|
89
|
-
|
|
89
|
+
begin
|
|
90
|
+
parsed_extra.merge(@source.local_extra(@local_header_offset), local: true)
|
|
91
|
+
rescue EntryMovedError
|
|
92
|
+
# Kept across a commit that moved its record (a removed entry,
|
|
93
|
+
# say): the central directory's fields are all there is now,
|
|
94
|
+
# which is what rubyzip reads anyway.
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
90
97
|
# Edits made through this object (`extra.merge`, `extra.delete`)
|
|
91
98
|
# leave no other trace; the bytes are compared at commit.
|
|
92
99
|
@extra_snapshot = extra_bytes
|
data/lib/ziperu/errors.rb
CHANGED
|
@@ -156,6 +156,11 @@ module Ziperu
|
|
|
156
156
|
|
|
157
157
|
# Raised by the Rust core where it lacks the domain object a rubyzip
|
|
158
158
|
# exception wants; the Ruby layer re-raises the rubyzip-shaped error.
|
|
159
|
+
# An entry object outlived a commit that moved its record; what it holds
|
|
160
|
+
# from the central directory is still good, its local header is gone.
|
|
161
|
+
class EntryMovedError < Error # :nodoc:
|
|
162
|
+
end
|
|
163
|
+
|
|
159
164
|
module Native
|
|
160
165
|
class SizeLimitError < Ziperu::Error; end
|
|
161
166
|
class PasswordError < Ziperu::Error; end
|
data/lib/ziperu/file.rb
CHANGED
|
@@ -82,6 +82,11 @@ module Ziperu
|
|
|
82
82
|
# decrypter on `InputStream`; `decrypter:` is accepted here too.
|
|
83
83
|
@password = password || decrypter&.password
|
|
84
84
|
|
|
85
|
+
unless path_or_io.kind_of?(::String) || path_or_io.respond_to?(:to_path) ||
|
|
86
|
+
path_or_io.respond_to?(:read) || path_or_io.respond_to?(:write)
|
|
87
|
+
raise TypeError, "no implicit conversion of #{path_or_io.inspect} into String"
|
|
88
|
+
end
|
|
89
|
+
|
|
85
90
|
@name = if IOExtras.path_like?(path_or_io)
|
|
86
91
|
IOExtras.path_of(path_or_io)
|
|
87
92
|
elsif path_or_io.respond_to?(:path)
|
|
@@ -312,8 +317,9 @@ module Ziperu
|
|
|
312
317
|
compression_level: @compression_level, settings: @settings
|
|
313
318
|
)
|
|
314
319
|
end
|
|
315
|
-
if src_path.respond_to?(:read)
|
|
320
|
+
if src_path.respond_to?(:read) && !IOExtras.path_like?(src_path)
|
|
316
321
|
# An IO (File, StringIO, Tempfile, a socket): streamed at commit.
|
|
322
|
+
# A Pathname has `read` too, but names a file.
|
|
317
323
|
new_entry.gather_fileinfo_from_io(src_path)
|
|
318
324
|
else
|
|
319
325
|
new_entry.gather_fileinfo_from_srcpath(src_path)
|
|
@@ -579,7 +585,7 @@ module Ziperu
|
|
|
579
585
|
# the archive is rewritten.
|
|
580
586
|
def appended_entries
|
|
581
587
|
return nil if @create || @buffer || @native.nil? || @comment_changed || @settings.sort_entries
|
|
582
|
-
return nil unless @settings.append_in_place
|
|
588
|
+
return nil unless @settings.append_in_place && appendable_in_place?
|
|
583
589
|
|
|
584
590
|
all = @cdir.entries
|
|
585
591
|
held = @native.entry_count
|
|
@@ -592,6 +598,18 @@ module Ziperu
|
|
|
592
598
|
end
|
|
593
599
|
private :appended_entries
|
|
594
600
|
|
|
601
|
+
# Whether the archive's file can take an append: one that is read-only
|
|
602
|
+
# can still be replaced (the rewrite renames a new file over it), and
|
|
603
|
+
# one with other hard links must be, or the append would change what
|
|
604
|
+
# they hold too.
|
|
605
|
+
def appendable_in_place?
|
|
606
|
+
stat = ::File.stat(@name)
|
|
607
|
+
stat.file? && stat.writable? && stat.nlink <= 1
|
|
608
|
+
rescue SystemCallError
|
|
609
|
+
false
|
|
610
|
+
end
|
|
611
|
+
private :appendable_in_place?
|
|
612
|
+
|
|
595
613
|
# Writes the archive, with any changes, to `io` and returns it. `io`
|
|
596
614
|
# may be a pipe, socket or response body: entries are streamed to it
|
|
597
615
|
# as they are written, unless the archive holds ZipCrypto entries,
|
|
@@ -835,6 +853,10 @@ module Ziperu
|
|
|
835
853
|
begin
|
|
836
854
|
::File.rename(tmp_filename, name)
|
|
837
855
|
rescue Errno::EACCES
|
|
856
|
+
# A directory (or anything but a file) at the path is not ours to
|
|
857
|
+
# move: the commit fails, as rubyzip's does.
|
|
858
|
+
raise unless ::File.file?(name)
|
|
859
|
+
|
|
838
860
|
# Something else (another Ziperu::File, say) still has the archive
|
|
839
861
|
# open. Renaming an open file is allowed where replacing one is
|
|
840
862
|
# not, so move the old archive aside first; that handle keeps
|
data/lib/ziperu/input_stream.rb
CHANGED
|
@@ -28,8 +28,20 @@ module Ziperu
|
|
|
28
28
|
# `Ziperu.*` setting by its name, for this stream alone (the limits,
|
|
29
29
|
# `validate_entry_sizes:`, `force_entry_names_encoding:`; see
|
|
30
30
|
# Ziperu::Settings).
|
|
31
|
+
#
|
|
32
|
+
# `streaming: true` reads a pipe or socket (an upload, an S3 object's
|
|
33
|
+
# body) in one pass, entry by entry, rather than taking all of it in
|
|
34
|
+
# first to reach the central directory at its end: memory stays at an
|
|
35
|
+
# entry's worth, whatever the archive's size. What the stream gives is
|
|
36
|
+
# what the local headers hold: no comments, no permissions, and an
|
|
37
|
+
# entry whose sizes come after its data has them (`size`, `crc`) only
|
|
38
|
+
# once it has been read to its end. Such an entry must be Deflated,
|
|
39
|
+
# which is how streaming writers write; a Stored one cannot be told
|
|
40
|
+
# apart from what follows it in one pass, and raises. The limits apply
|
|
41
|
+
# to the bytes as they come. A file, a StringIO or a seekable IO is
|
|
42
|
+
# read through its central directory either way. Not in rubyzip.
|
|
31
43
|
def initialize(context, *legacy, offset: 0, decrypter: nil, password: nil,
|
|
32
|
-
enforce_size: nil, metadata_encoding: nil, **opts)
|
|
44
|
+
enforce_size: nil, metadata_encoding: nil, streaming: false, **opts)
|
|
33
45
|
@settings = Settings.take!(opts)
|
|
34
46
|
enforce_size = @settings.validate_entry_sizes if enforce_size.nil?
|
|
35
47
|
@metadata_encoding = metadata_encoding
|
|
@@ -41,6 +53,19 @@ module Ziperu
|
|
|
41
53
|
# AES from ZipCrypto by the entry, so one password serves both.
|
|
42
54
|
decrypter ||= TraditionalDecrypter.new(password) if password
|
|
43
55
|
super(**opts)
|
|
56
|
+
if streaming && self.class.pipe_like?(context)
|
|
57
|
+
IOExtras.binmode(context)
|
|
58
|
+
@stream = Native::StreamReader.open(context, decrypter&.password&.b, enforce_size)
|
|
59
|
+
@stream_entries = 0
|
|
60
|
+
@stream_total = 0
|
|
61
|
+
@decrypter = decrypter
|
|
62
|
+
@enforce_size = enforce_size
|
|
63
|
+
@owns_archive = false
|
|
64
|
+
@archive = nil
|
|
65
|
+
@reader = nil
|
|
66
|
+
@current_entry = nil
|
|
67
|
+
return
|
|
68
|
+
end
|
|
44
69
|
@split = !context.kind_of?(Native::Archive) && self.class.split_archive?(context)
|
|
45
70
|
# An archive handed in (by Entry#get_input_stream) stays open for its
|
|
46
71
|
# owner; one opened here is closed with the stream. A split segment
|
|
@@ -85,6 +110,8 @@ module Ziperu
|
|
|
85
110
|
def close
|
|
86
111
|
@reader&.close
|
|
87
112
|
@reader = nil
|
|
113
|
+
@stream&.close
|
|
114
|
+
@stream = nil
|
|
88
115
|
@archive&.close if @owns_archive
|
|
89
116
|
@archive = nil
|
|
90
117
|
@open_error = nil
|
|
@@ -95,6 +122,8 @@ module Ziperu
|
|
|
95
122
|
def get_next_entry
|
|
96
123
|
raise SplitArchiveError if @split
|
|
97
124
|
check_open!
|
|
125
|
+
return next_stream_entry if @stream
|
|
126
|
+
|
|
98
127
|
return @current_entry = nil if @archive.nil? || @next_index.nil?
|
|
99
128
|
if @next_index >= (@records ? @records.size : @archive.entry_count)
|
|
100
129
|
# Past the last entry there is nothing to read, not the previous
|
|
@@ -154,6 +183,7 @@ module Ziperu
|
|
|
154
183
|
def rewind
|
|
155
184
|
check_open!
|
|
156
185
|
return if @current_entry.nil?
|
|
186
|
+
raise Error, 'an entry read with streaming: true cannot be rewound' if @stream
|
|
157
187
|
|
|
158
188
|
@lineno = 0
|
|
159
189
|
@pos = 0
|
|
@@ -161,6 +191,39 @@ module Ziperu
|
|
|
161
191
|
flush
|
|
162
192
|
end
|
|
163
193
|
|
|
194
|
+
# Moves to byte `offset` of the current entry (from its start, its
|
|
195
|
+
# current position or its end, as `whence` says), as IO#seek does.
|
|
196
|
+
# Only a Stored entry that is not encrypted can be seeked in: its
|
|
197
|
+
# bytes are in the archive as they are. A compressed one raises
|
|
198
|
+
# Ziperu::Error; `rewind` is the way back to its start. Reads after a
|
|
199
|
+
# seek check no CRC, which only reading from the start can do.
|
|
200
|
+
# Not in rubyzip.
|
|
201
|
+
def seek(offset, whence = ::IO::SEEK_SET)
|
|
202
|
+
check_open!
|
|
203
|
+
raise Error, 'no entry to seek in' if @reader.nil?
|
|
204
|
+
raise Error, 'an entry read with streaming: true cannot be seeked in' if @stream
|
|
205
|
+
|
|
206
|
+
target =
|
|
207
|
+
case whence
|
|
208
|
+
when ::IO::SEEK_SET, :SET then offset
|
|
209
|
+
when ::IO::SEEK_CUR, :CUR then pos + offset
|
|
210
|
+
when ::IO::SEEK_END, :END then @current_entry.size + offset
|
|
211
|
+
else raise ArgumentError, "unknown whence: #{whence.inspect}"
|
|
212
|
+
end
|
|
213
|
+
raise Errno::EINVAL, "seek to #{target}" if target.negative?
|
|
214
|
+
|
|
215
|
+
wrap_errors { @reader.seek(target) }
|
|
216
|
+
flush
|
|
217
|
+
@pos = [target, @current_entry.size].min
|
|
218
|
+
0
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# IO#pos=, through #seek.
|
|
222
|
+
# Not in rubyzip.
|
|
223
|
+
def pos=(offset)
|
|
224
|
+
seek(offset)
|
|
225
|
+
end
|
|
226
|
+
|
|
164
227
|
# Modelled after IO#sysread. Reads up to maxlen bytes from the stream,
|
|
165
228
|
# bypassing the line buffer; returns a string (either a new string or the
|
|
166
229
|
# given out_string).
|
|
@@ -202,20 +265,26 @@ module Ziperu
|
|
|
202
265
|
check_limits(open_native(context, settings), settings)
|
|
203
266
|
end
|
|
204
267
|
|
|
205
|
-
# Ziperu.max_entries / max_total_size
|
|
206
|
-
#
|
|
268
|
+
# Ziperu.max_entries / max_total_size / max_entry_size /
|
|
269
|
+
# max_compression_ratio, checked before any entry is read, over every
|
|
270
|
+
# record in the central directory (several under one name included).
|
|
271
|
+
# An archive that fails is closed again.
|
|
207
272
|
def check_limits(native, settings = ::Ziperu)
|
|
208
|
-
|
|
209
|
-
|
|
273
|
+
limits = [settings.max_entries, settings.max_total_size,
|
|
274
|
+
settings.max_entry_size, settings.max_compression_ratio]
|
|
275
|
+
return native if limits.none?
|
|
276
|
+
|
|
277
|
+
count, total, largest, ratio = native.limits
|
|
278
|
+
if (limit = settings.max_entries) && count > limit
|
|
279
|
+
raise LimitError.new('an entry count', count, limit)
|
|
210
280
|
end
|
|
211
|
-
if (limit = settings.max_total_size) &&
|
|
281
|
+
if (limit = settings.max_total_size) && total > limit
|
|
212
282
|
raise LimitError.new('a declared total size', total, limit)
|
|
213
283
|
end
|
|
214
|
-
if (limit = settings.max_entry_size) &&
|
|
284
|
+
if (limit = settings.max_entry_size) && largest > limit
|
|
215
285
|
raise LimitError.new('an entry with a declared size', largest, limit)
|
|
216
286
|
end
|
|
217
|
-
if (limit = settings.max_compression_ratio) &&
|
|
218
|
-
(ratio = native.max_compression_ratio) > limit
|
|
287
|
+
if (limit = settings.max_compression_ratio) && ratio > limit
|
|
219
288
|
raise LimitError.new('an entry with a compression ratio', ratio.round(1), limit)
|
|
220
289
|
end
|
|
221
290
|
|
|
@@ -229,6 +298,12 @@ module Ziperu
|
|
|
229
298
|
IOExtras.path_like?(context)
|
|
230
299
|
end
|
|
231
300
|
|
|
301
|
+
# A pipe, socket or the like: an IO that can only be read forward.
|
|
302
|
+
def pipe_like?(context) # :nodoc:
|
|
303
|
+
!context.kind_of?(Native::Archive) && !path_like?(context) &&
|
|
304
|
+
!context.kind_of?(::StringIO) && context.respond_to?(:read) && !IOExtras.seekable?(context)
|
|
305
|
+
end
|
|
306
|
+
|
|
232
307
|
# Whether what `context` holds begins with a local file header, the
|
|
233
308
|
# one thing rubyzip's `InputStream` needs to yield entries.
|
|
234
309
|
def archive_like?(context) # :nodoc:
|
|
@@ -251,7 +326,15 @@ module Ziperu
|
|
|
251
326
|
end
|
|
252
327
|
|
|
253
328
|
def open_native(context, settings = ::Ziperu)
|
|
254
|
-
|
|
329
|
+
if path_like?(context)
|
|
330
|
+
path = path_of(context)
|
|
331
|
+
begin
|
|
332
|
+
return Native::Archive.open_path(path)
|
|
333
|
+
rescue Errno::ENOENT
|
|
334
|
+
# The native message names no file; Ruby's own does.
|
|
335
|
+
raise Errno::ENOENT, path
|
|
336
|
+
end
|
|
337
|
+
end
|
|
255
338
|
|
|
256
339
|
# A StringIO's bytes are already in memory: snapshot them once rather
|
|
257
340
|
# than paying a method call per chunk (12x faster per entry).
|
|
@@ -338,6 +421,7 @@ module Ziperu
|
|
|
338
421
|
end
|
|
339
422
|
raise ArgumentError, "negative length #{maxlen} given" if maxlen&.negative?
|
|
340
423
|
return nil if @reader.nil?
|
|
424
|
+
return produce_stream_input(maxlen) if @stream
|
|
341
425
|
|
|
342
426
|
wrap_errors { @reader.read(maxlen) }
|
|
343
427
|
end
|
|
@@ -363,7 +447,79 @@ module Ziperu
|
|
|
363
447
|
# archive (`@open_error` holds why), which yields no entries but is
|
|
364
448
|
# not closed.
|
|
365
449
|
def closed?
|
|
366
|
-
@archive.nil? && @open_error.nil?
|
|
450
|
+
@archive.nil? && @open_error.nil? && @stream.nil?
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
# `get_next_entry` with `streaming: true`: the next local header.
|
|
454
|
+
def next_stream_entry
|
|
455
|
+
meta = wrap_errors { @stream.next_entry }
|
|
456
|
+
@reader = nil
|
|
457
|
+
flush
|
|
458
|
+
return @current_entry = nil unless meta
|
|
459
|
+
|
|
460
|
+
@stream_entries += 1
|
|
461
|
+
if (limit = @settings.max_entries) && @stream_entries > limit
|
|
462
|
+
raise LimitError.new('an entry count', @stream_entries, limit)
|
|
463
|
+
end
|
|
464
|
+
if meta[:size]
|
|
465
|
+
check_stream_entry_size(meta[:size])
|
|
466
|
+
if (limit = @settings.max_compression_ratio) && meta[:size].positive?
|
|
467
|
+
ratio = meta[:compressed_size].zero? ? Float::INFINITY : meta[:size].fdiv(meta[:compressed_size])
|
|
468
|
+
raise LimitError.new('an entry with a compression ratio', ratio.round(1), limit) if ratio > limit
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
# Unknown until the entry has been read (see `produce_stream_input`).
|
|
472
|
+
meta[:crc] ||= 0
|
|
473
|
+
meta[:compressed_size] ||= 0
|
|
474
|
+
@current_entry = Entry.from_meta(nil, meta, metadata_encoding: @metadata_encoding, settings: @settings)
|
|
475
|
+
@current_entry.instance_variable_set(:@size, nil) if meta[:size].nil?
|
|
476
|
+
@stream_entry_bytes = 0
|
|
477
|
+
@reader = @stream
|
|
478
|
+
@lineno = 0
|
|
479
|
+
@pos = 0
|
|
480
|
+
@current_entry
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
# Reads from the stream, counting against the limits as the bytes
|
|
484
|
+
# come; nil `maxlen` reads to the end of the entry, in chunks.
|
|
485
|
+
def produce_stream_input(maxlen)
|
|
486
|
+
return stream_chunk(maxlen) if maxlen
|
|
487
|
+
|
|
488
|
+
out = +''.b
|
|
489
|
+
while (chunk = stream_chunk(CHUNK_SIZE))
|
|
490
|
+
out << chunk
|
|
491
|
+
end
|
|
492
|
+
out.empty? ? nil : out
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def stream_chunk(length)
|
|
496
|
+
chunk = wrap_errors { @stream.read(length) }
|
|
497
|
+
if chunk
|
|
498
|
+
@stream_entry_bytes += chunk.bytesize
|
|
499
|
+
@stream_total += chunk.bytesize
|
|
500
|
+
check_stream_entry_size(@stream_entry_bytes)
|
|
501
|
+
if (limit = @settings.max_total_size) && @stream_total > limit
|
|
502
|
+
raise LimitError.new('a total size', @stream_total, limit)
|
|
503
|
+
end
|
|
504
|
+
end
|
|
505
|
+
learn_stream_trailer if @stream.eof?
|
|
506
|
+
chunk
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
def check_stream_entry_size(size)
|
|
510
|
+
return unless (limit = @settings.max_entry_size) && size > limit
|
|
511
|
+
|
|
512
|
+
raise LimitError.new('an entry with a size', size, limit)
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
# Once a streamed entry has been read to its end: what its data came
|
|
516
|
+
# to, for an entry whose header left it to the descriptor.
|
|
517
|
+
def learn_stream_trailer
|
|
518
|
+
crc, size = @stream.trailer
|
|
519
|
+
return unless crc && !@current_entry.size_known?
|
|
520
|
+
|
|
521
|
+
@current_entry.instance_variable_set(:@size, size)
|
|
522
|
+
@current_entry.instance_variable_set(:@crc, crc)
|
|
367
523
|
end
|
|
368
524
|
|
|
369
525
|
# Re-raise the native core's errors as the rubyzip-shaped ones.
|