dsh-ops 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +202 -0
- package/LICENSE +30 -0
- package/NOTICE +106 -0
- package/PROVENANCE.md +435 -0
- package/README.en.md +126 -0
- package/README.md +115 -2
- package/README.zh.md +116 -0
- package/bin/dsh-ops.mjs +1216 -0
- package/cordis.patch.yml +160 -0
- package/docs/manual-validation.md +53 -0
- package/docs/release-0.2.1.md +72 -0
- package/docs/schema-baseline.json +64 -0
- package/docs/schema-current.json +84 -0
- package/docs/schema-measurement.md +17 -0
- package/dsh-plugin.json +88 -0
- package/icon.svg +12 -0
- package/lib/binary.js +409 -0
- package/lib/config.js +198 -0
- package/lib/handshake.js +252 -0
- package/lib/index.js +108 -0
- package/lib/jobs.js +42 -0
- package/lib/policy.js +64 -0
- package/lib/presentation.js +63 -0
- package/lib/profile-install.js +61 -0
- package/lib/rust.js +194 -0
- package/lib/session-shells.js +78 -0
- package/lib/shells.js +998 -0
- package/lib/tools.js +657 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +114 -4
- package/vendor/fastctx/Cargo.lock +3210 -0
- package/vendor/fastctx/Cargo.toml +94 -0
- package/vendor/fastctx/FORK.md +119 -0
- package/vendor/fastctx/LICENSE-APACHE +201 -0
- package/vendor/fastctx/NOTICE +40 -0
- package/vendor/fastctx/README.md +439 -0
- package/vendor/fastctx/THIRD_PARTY_LICENSES.md +17 -0
- package/vendor/fastctx/THIRD_PARTY_LICENSES_RUST.md +7914 -0
- package/vendor/fastctx/UPSTREAM.md +49 -0
- package/vendor/fastctx/build.rs +413 -0
- package/vendor/fastctx/src/background_status.rs +403 -0
- package/vendor/fastctx/src/binary.rs +75 -0
- package/vendor/fastctx/src/bounded_sort.rs +500 -0
- package/vendor/fastctx/src/budget.rs +781 -0
- package/vendor/fastctx/src/cli/mod.rs +110 -0
- package/vendor/fastctx/src/context_guard.rs +289 -0
- package/vendor/fastctx/src/control/mod.rs +6 -0
- package/vendor/fastctx/src/control/paths.rs +49 -0
- package/vendor/fastctx/src/control/settings.rs +753 -0
- package/vendor/fastctx/src/control/transaction.rs +531 -0
- package/vendor/fastctx/src/edit/document.rs +535 -0
- package/vendor/fastctx/src/edit/locks.rs +371 -0
- package/vendor/fastctx/src/edit/mod.rs +213 -0
- package/vendor/fastctx/src/edit/private_storage/unix.rs +315 -0
- package/vendor/fastctx/src/edit/private_storage/windows.rs +793 -0
- package/vendor/fastctx/src/edit/private_storage.rs +234 -0
- package/vendor/fastctx/src/edit/replace.rs +1030 -0
- package/vendor/fastctx/src/edit_server.rs +53 -0
- package/vendor/fastctx/src/encoding/reference_v011.rs +587 -0
- package/vendor/fastctx/src/encoding/snapshot_pipeline.rs +1678 -0
- package/vendor/fastctx/src/encoding.rs +1118 -0
- package/vendor/fastctx/src/file_executor.rs +1151 -0
- package/vendor/fastctx/src/file_snapshot.rs +1491 -0
- package/vendor/fastctx/src/glob_filter.rs +98 -0
- package/vendor/fastctx/src/glob_tool.rs +653 -0
- package/vendor/fastctx/src/grep_sink.rs +1162 -0
- package/vendor/fastctx/src/grep_tool.rs +2449 -0
- package/vendor/fastctx/src/lib.rs +45 -0
- package/vendor/fastctx/src/main.rs +15 -0
- package/vendor/fastctx/src/model.rs +51 -0
- package/vendor/fastctx/src/model_guidance.rs +62 -0
- package/vendor/fastctx/src/operation.rs +356 -0
- package/vendor/fastctx/src/ordered_window.rs +1235 -0
- package/vendor/fastctx/src/os_environment.rs +414 -0
- package/vendor/fastctx/src/path_codec.rs +850 -0
- package/vendor/fastctx/src/paths.rs +244 -0
- package/vendor/fastctx/src/process_identity.rs +763 -0
- package/vendor/fastctx/src/process_policy.rs +74 -0
- package/vendor/fastctx/src/read_tool/batch.rs +496 -0
- package/vendor/fastctx/src/read_tool/hex_file.rs +141 -0
- package/vendor/fastctx/src/read_tool/image_file.rs +88 -0
- package/vendor/fastctx/src/read_tool/mod.rs +245 -0
- package/vendor/fastctx/src/read_tool/pdf.rs +470 -0
- package/vendor/fastctx/src/read_tool/pdf_disabled.rs +47 -0
- package/vendor/fastctx/src/read_tool/pdf_engine.rs +664 -0
- package/vendor/fastctx/src/read_tool/text_file.rs +351 -0
- package/vendor/fastctx/src/render_plan.rs +468 -0
- package/vendor/fastctx/src/runtime/activity.rs +159 -0
- package/vendor/fastctx/src/runtime/hosts.rs +99 -0
- package/vendor/fastctx/src/runtime/journal.rs +556 -0
- package/vendor/fastctx/src/runtime/local_ipc.rs +186 -0
- package/vendor/fastctx/src/runtime/mod.rs +746 -0
- package/vendor/fastctx/src/runtime/protocol.rs +296 -0
- package/vendor/fastctx/src/runtime/session.rs +536 -0
- package/vendor/fastctx/src/runtime/windows_process.rs +66 -0
- package/vendor/fastctx/src/search_parallelism.rs +106 -0
- package/vendor/fastctx/src/search_text.rs +227 -0
- package/vendor/fastctx/src/server.rs +359 -0
- package/vendor/fastctx/src/server_manifest.rs +468 -0
- package/vendor/fastctx/src/server_support.rs +826 -0
- package/vendor/fastctx/src/session.rs +629 -0
- package/vendor/fastctx/src/shell/apply_patch_hint.rs +41 -0
- package/vendor/fastctx/src/shell/bash.rs +263 -0
- package/vendor/fastctx/src/shell/buffer.rs +108 -0
- package/vendor/fastctx/src/shell/encoding.rs +403 -0
- package/vendor/fastctx/src/shell/foreground.rs +115 -0
- package/vendor/fastctx/src/shell/jobs/admission.rs +91 -0
- package/vendor/fastctx/src/shell/jobs/background.rs +146 -0
- package/vendor/fastctx/src/shell/jobs/host.rs +830 -0
- package/vendor/fastctx/src/shell/jobs/identity.rs +29 -0
- package/vendor/fastctx/src/shell/jobs/mod.rs +1513 -0
- package/vendor/fastctx/src/shell/jobs/model.rs +244 -0
- package/vendor/fastctx/src/shell/jobs/output_log.rs +1148 -0
- package/vendor/fastctx/src/shell/jobs/store.rs +1300 -0
- package/vendor/fastctx/src/shell/mod.rs +345 -0
- package/vendor/fastctx/src/shell/normalize.rs +389 -0
- package/vendor/fastctx/src/shell/output.rs +406 -0
- package/vendor/fastctx/src/shell/process.rs +493 -0
- package/vendor/fastctx/src/shell_server.rs +156 -0
- package/vendor/fastctx/src/skip_report.rs +83 -0
- package/vendor/fastctx/src/stdio_transport.rs +177 -0
- package/vendor/fastctx/src/tool_schema.rs +204 -0
- package/vendor/fastctx/src/traversal.rs +846 -0
- package/vendor/fastctx/third-party/pdfium-7763/LICENSE +9 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/abseil.txt +202 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/agg23.txt +14 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/fast_float.txt +27 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/freetype.txt +169 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/icu.txt +542 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/lcms.txt +27 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.ijg +260 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.md +135 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libopenjpeg.txt +32 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libpng.txt +134 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/libtiff.txt +21 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/llvm-libc.txt +278 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/pdfium.txt +230 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/simdutf.txt +18 -0
- package/vendor/fastctx/third-party/pdfium-7763/licenses/zlib.txt +29 -0
|
@@ -0,0 +1,1491 @@
|
|
|
1
|
+
//! Stable, single-open byte snapshots for grep candidates.
|
|
2
|
+
|
|
3
|
+
use crate::binary::detect_binary_type;
|
|
4
|
+
use crate::encoding::{
|
|
5
|
+
BomStreamingValidator, EncodingRejection, StrictStreamingValidator, Utf8ConflictProbe,
|
|
6
|
+
bom_streaming_validator, explicit_streaming_validator,
|
|
7
|
+
};
|
|
8
|
+
use crate::operation::{WorkCheckpoint, WorkStop};
|
|
9
|
+
use crate::path_codec::{FileIdentityHint, PathRecord};
|
|
10
|
+
use std::fs::File;
|
|
11
|
+
use std::io::{self, BufReader, Cursor, Read, Seek, SeekFrom, Write};
|
|
12
|
+
use std::path::Path;
|
|
13
|
+
use std::sync::Arc;
|
|
14
|
+
|
|
15
|
+
const PREFLIGHT_BYTES: usize = 8 * 1024;
|
|
16
|
+
const SNAPSHOT_CHUNK_BYTES: usize = 64 * 1024;
|
|
17
|
+
const MEMORY_SNAPSHOT_LIMIT: usize = 8 * 1024 * 1024;
|
|
18
|
+
|
|
19
|
+
/// A suffix-invariant reason that permits capture to stop before EOF.
|
|
20
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
21
|
+
pub(crate) enum TerminalProof {
|
|
22
|
+
NulWithinFrozenProbe,
|
|
23
|
+
BinaryMagicAfterUtf8Failure,
|
|
24
|
+
ExplicitDecoderMalformed {
|
|
25
|
+
encoding: String,
|
|
26
|
+
},
|
|
27
|
+
BomDecoderMalformed {
|
|
28
|
+
encoding: &'static str,
|
|
29
|
+
},
|
|
30
|
+
AutomaticRejected {
|
|
31
|
+
conflict_hex_offset: usize,
|
|
32
|
+
fallback: FallbackTerminalState,
|
|
33
|
+
},
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/// Why no fallback decoder can overturn an automatic mixed-encoding proof.
|
|
37
|
+
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
38
|
+
pub(crate) enum FallbackTerminalState {
|
|
39
|
+
Absent,
|
|
40
|
+
PermanentlyMalformed,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
impl TerminalProof {
|
|
44
|
+
pub(crate) fn rejection(&self) -> Option<EncodingRejection> {
|
|
45
|
+
match self {
|
|
46
|
+
Self::NulWithinFrozenProbe | Self::BinaryMagicAfterUtf8Failure => None,
|
|
47
|
+
Self::ExplicitDecoderMalformed { encoding } => {
|
|
48
|
+
Some(EncodingRejection::ExplicitMalformed {
|
|
49
|
+
encoding: encoding.clone(),
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
Self::BomDecoderMalformed { encoding } => {
|
|
53
|
+
Some(EncodingRejection::BomMismatch { encoding })
|
|
54
|
+
}
|
|
55
|
+
Self::AutomaticRejected {
|
|
56
|
+
conflict_hex_offset,
|
|
57
|
+
..
|
|
58
|
+
} => Some(EncodingRejection::MixedOrInconsistent {
|
|
59
|
+
conflict_hex_offset: Some(*conflict_hex_offset),
|
|
60
|
+
}),
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/// The stable result of capturing one candidate from its original handle.
|
|
66
|
+
pub(crate) enum CaptureDisposition {
|
|
67
|
+
Searchable(Box<SealedSnapshot>),
|
|
68
|
+
BinarySkipped(TerminalProof),
|
|
69
|
+
EncodingRejected {
|
|
70
|
+
rejection: EncodingRejection,
|
|
71
|
+
proof: TerminalProof,
|
|
72
|
+
},
|
|
73
|
+
FileChanged,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/// Failures that must remain distinct from an actively changing candidate.
|
|
77
|
+
#[derive(Debug)]
|
|
78
|
+
pub(crate) enum CaptureFailure {
|
|
79
|
+
Cancelled,
|
|
80
|
+
EpochRetired,
|
|
81
|
+
InvalidEncoding(EncodingRejection),
|
|
82
|
+
Io(io::Error),
|
|
83
|
+
Snapshot(io::Error),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// A reader over one immutable snapshot backing.
|
|
87
|
+
pub(crate) enum SnapshotReader<'a> {
|
|
88
|
+
Memory(Cursor<&'a [u8]>),
|
|
89
|
+
Temp(BufReader<File>),
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
impl Read for SnapshotReader<'_> {
|
|
93
|
+
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
|
|
94
|
+
match self {
|
|
95
|
+
Self::Memory(reader) => reader.read(output),
|
|
96
|
+
Self::Temp(reader) => reader.read(output),
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
#[derive(Debug)]
|
|
102
|
+
struct TempSnapshot {
|
|
103
|
+
file: tempfile::NamedTempFile,
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
impl TempSnapshot {
|
|
107
|
+
fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'static>> {
|
|
108
|
+
let mut file = self.file.reopen()?;
|
|
109
|
+
if start > 0 {
|
|
110
|
+
file.seek(SeekFrom::Start(start))?;
|
|
111
|
+
}
|
|
112
|
+
Ok(SnapshotReader::Temp(BufReader::new(file)))
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
#[derive(Clone, Debug)]
|
|
117
|
+
enum SnapshotBacking {
|
|
118
|
+
Memory(Arc<[u8]>),
|
|
119
|
+
Temp(Arc<TempSnapshot>),
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
struct CapturedSnapshot {
|
|
123
|
+
len: u64,
|
|
124
|
+
backing: SnapshotBacking,
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/// A cloneable immutable view used when decoded UTF-8 is the snapshot bytes themselves.
|
|
128
|
+
#[derive(Clone, Debug)]
|
|
129
|
+
pub(crate) struct SnapshotByteRange {
|
|
130
|
+
backing: SnapshotBacking,
|
|
131
|
+
start: u64,
|
|
132
|
+
len: u64,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
impl SnapshotByteRange {
|
|
136
|
+
pub(crate) fn len(&self) -> u64 {
|
|
137
|
+
self.len
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
pub(crate) fn memory_bytes(&self) -> Option<&[u8]> {
|
|
141
|
+
let SnapshotBacking::Memory(bytes) = &self.backing else {
|
|
142
|
+
return None;
|
|
143
|
+
};
|
|
144
|
+
let start = usize::try_from(self.start).ok()?;
|
|
145
|
+
let length = usize::try_from(self.len).ok()?;
|
|
146
|
+
let end = start.checked_add(length)?;
|
|
147
|
+
bytes.get(start..end)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
pub(crate) fn open_reader(&self) -> io::Result<SnapshotReader<'_>> {
|
|
151
|
+
match &self.backing {
|
|
152
|
+
SnapshotBacking::Memory(bytes) => {
|
|
153
|
+
let start = usize::try_from(self.start).map_err(|_| snapshot_range_error())?;
|
|
154
|
+
let length = usize::try_from(self.len).map_err(|_| snapshot_range_error())?;
|
|
155
|
+
let end = start.checked_add(length).ok_or_else(snapshot_range_error)?;
|
|
156
|
+
let bytes = bytes.get(start..end).ok_or_else(snapshot_range_error)?;
|
|
157
|
+
Ok(SnapshotReader::Memory(Cursor::new(bytes)))
|
|
158
|
+
}
|
|
159
|
+
SnapshotBacking::Temp(snapshot) => snapshot.open_reader(self.start),
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
pub(crate) fn read_range(&self, range: std::ops::Range<u64>) -> io::Result<Vec<u8>> {
|
|
164
|
+
if range.start > range.end || range.end > self.len {
|
|
165
|
+
return Err(io::Error::new(
|
|
166
|
+
io::ErrorKind::InvalidData,
|
|
167
|
+
"captured text range lies outside its snapshot backing",
|
|
168
|
+
));
|
|
169
|
+
}
|
|
170
|
+
let length = usize::try_from(range.end - range.start).map_err(|_| {
|
|
171
|
+
io::Error::new(
|
|
172
|
+
io::ErrorKind::InvalidData,
|
|
173
|
+
"captured text range does not fit in memory",
|
|
174
|
+
)
|
|
175
|
+
})?;
|
|
176
|
+
let absolute_start = self.start.checked_add(range.start).ok_or_else(|| {
|
|
177
|
+
io::Error::new(io::ErrorKind::InvalidData, "snapshot offset overflow")
|
|
178
|
+
})?;
|
|
179
|
+
let mut reader = match &self.backing {
|
|
180
|
+
SnapshotBacking::Memory(bytes) => {
|
|
181
|
+
let start = usize::try_from(absolute_start).map_err(|_| {
|
|
182
|
+
io::Error::new(io::ErrorKind::InvalidData, "snapshot offset overflow")
|
|
183
|
+
})?;
|
|
184
|
+
let end = start.checked_add(length).ok_or_else(snapshot_range_error)?;
|
|
185
|
+
let bytes = bytes.get(start..end).ok_or_else(snapshot_range_error)?;
|
|
186
|
+
return Ok(bytes.to_vec());
|
|
187
|
+
}
|
|
188
|
+
SnapshotBacking::Temp(snapshot) => snapshot.open_reader(absolute_start)?,
|
|
189
|
+
};
|
|
190
|
+
let mut bytes = vec![0_u8; length];
|
|
191
|
+
reader.read_exact(&mut bytes)?;
|
|
192
|
+
Ok(bytes)
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
fn snapshot_range_error() -> io::Error {
|
|
197
|
+
io::Error::new(
|
|
198
|
+
io::ErrorKind::InvalidData,
|
|
199
|
+
"captured text range lies outside its snapshot backing",
|
|
200
|
+
)
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
impl CapturedSnapshot {
|
|
204
|
+
fn memory_bytes(&self) -> Option<&[u8]> {
|
|
205
|
+
match &self.backing {
|
|
206
|
+
SnapshotBacking::Memory(bytes) => Some(bytes),
|
|
207
|
+
SnapshotBacking::Temp(_) => None,
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'_>> {
|
|
212
|
+
match &self.backing {
|
|
213
|
+
SnapshotBacking::Memory(bytes) => {
|
|
214
|
+
let start = usize::try_from(start)
|
|
215
|
+
.unwrap_or(usize::MAX)
|
|
216
|
+
.min(bytes.len());
|
|
217
|
+
Ok(SnapshotReader::Memory(Cursor::new(&bytes[start..])))
|
|
218
|
+
}
|
|
219
|
+
SnapshotBacking::Temp(snapshot) => snapshot.open_reader(start),
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
fn shared_range(&self, start: u64) -> io::Result<SnapshotByteRange> {
|
|
224
|
+
if start > self.len {
|
|
225
|
+
return Err(io::Error::new(
|
|
226
|
+
io::ErrorKind::InvalidData,
|
|
227
|
+
"decoded text starts beyond the captured snapshot",
|
|
228
|
+
));
|
|
229
|
+
}
|
|
230
|
+
Ok(SnapshotByteRange {
|
|
231
|
+
backing: self.backing.clone(),
|
|
232
|
+
start,
|
|
233
|
+
len: self.len - start,
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
#[cfg(test)]
|
|
238
|
+
fn temp_path(&self) -> Option<&Path> {
|
|
239
|
+
match &self.backing {
|
|
240
|
+
SnapshotBacking::Memory(_) => None,
|
|
241
|
+
SnapshotBacking::Temp(snapshot) => Some(snapshot.file.path()),
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/// Immutable bytes used by every encoding pass and by the regex search.
|
|
247
|
+
pub(crate) struct SealedSnapshot {
|
|
248
|
+
path: PathRecord,
|
|
249
|
+
_before: FileIdentity,
|
|
250
|
+
_after: FileIdentity,
|
|
251
|
+
captured: CapturedSnapshot,
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
impl SealedSnapshot {
|
|
255
|
+
/// Returns the candidate whose original bytes produced this snapshot.
|
|
256
|
+
pub(crate) fn path(&self) -> &PathRecord {
|
|
257
|
+
&self.path
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/// Returns the exact captured length, independent of traversal metadata.
|
|
261
|
+
pub(crate) fn len(&self) -> u64 {
|
|
262
|
+
self.captured.len
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/// Borrows an in-memory backing without copying; temp-backed snapshots return `None`.
|
|
266
|
+
pub(crate) fn memory_bytes(&self) -> Option<&[u8]> {
|
|
267
|
+
self.captured.memory_bytes()
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/// Borrows at most `maximum` leading bytes directly from an in-memory backing.
|
|
271
|
+
pub(crate) fn memory_prefix(&self, maximum: usize) -> Option<&[u8]> {
|
|
272
|
+
self.memory_bytes()
|
|
273
|
+
.map(|bytes| &bytes[..bytes.len().min(maximum)])
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/// Opens an independent reader over the same immutable backing at `start`.
|
|
277
|
+
pub(crate) fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'_>> {
|
|
278
|
+
self.captured.open_reader(start)
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/// Shares the immutable suffix beginning at `start` without reopening the original path.
|
|
282
|
+
pub(crate) fn shared_range(&self, start: u64) -> io::Result<SnapshotByteRange> {
|
|
283
|
+
self.captured.shared_range(start)
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
enum CaptureRead {
|
|
288
|
+
Terminal(TerminalProof),
|
|
289
|
+
Complete(CapturedSnapshot),
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
enum ClassifierMode {
|
|
293
|
+
Explicit {
|
|
294
|
+
encoding: String,
|
|
295
|
+
decoder: StrictStreamingValidator,
|
|
296
|
+
malformed: bool,
|
|
297
|
+
},
|
|
298
|
+
Bom {
|
|
299
|
+
decoder: StrictStreamingValidator,
|
|
300
|
+
encoding: &'static str,
|
|
301
|
+
is_utf8: bool,
|
|
302
|
+
malformed: bool,
|
|
303
|
+
},
|
|
304
|
+
Automatic {
|
|
305
|
+
utf8: StrictStreamingValidator,
|
|
306
|
+
utf8_malformed: bool,
|
|
307
|
+
utf8_conflict: Utf8ConflictProbe,
|
|
308
|
+
conflict_hex_offset: Option<usize>,
|
|
309
|
+
fallback: Option<StrictStreamingValidator>,
|
|
310
|
+
fallback_malformed: bool,
|
|
311
|
+
},
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/// Streaming-only state whose terminal decisions cannot be overturned by unread bytes.
|
|
315
|
+
struct CaptureClassifier {
|
|
316
|
+
explicit_encoding: Option<String>,
|
|
317
|
+
fallback_encoding: Option<String>,
|
|
318
|
+
probe: Vec<u8>,
|
|
319
|
+
mode: Option<ClassifierMode>,
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
impl CaptureClassifier {
|
|
323
|
+
fn new(explicit_encoding: Option<&str>, fallback_encoding: Option<&str>) -> Self {
|
|
324
|
+
Self {
|
|
325
|
+
explicit_encoding: explicit_encoding.map(str::to_string),
|
|
326
|
+
fallback_encoding: fallback_encoding.map(str::to_string),
|
|
327
|
+
probe: Vec::with_capacity(PREFLIGHT_BYTES),
|
|
328
|
+
mode: None,
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
fn feed(&mut self, bytes: &[u8]) -> Result<Option<TerminalProof>, EncodingRejection> {
|
|
333
|
+
let remaining_probe = PREFLIGHT_BYTES.saturating_sub(self.probe.len());
|
|
334
|
+
self.probe
|
|
335
|
+
.extend_from_slice(&bytes[..bytes.len().min(remaining_probe)]);
|
|
336
|
+
|
|
337
|
+
if self.mode.is_none() {
|
|
338
|
+
// Four bytes settle every BOM that can outrank the automatic NUL gate.
|
|
339
|
+
if self.probe.len() < 4 {
|
|
340
|
+
return Ok(None);
|
|
341
|
+
}
|
|
342
|
+
debug_assert!(bytes.len() <= PREFLIGHT_BYTES);
|
|
343
|
+
self.initialize()?;
|
|
344
|
+
} else {
|
|
345
|
+
self.feed_initialized(bytes);
|
|
346
|
+
}
|
|
347
|
+
Ok(self.terminal_proof())
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
fn initialize(&mut self) -> Result<(), EncodingRejection> {
|
|
351
|
+
debug_assert!(self.mode.is_none());
|
|
352
|
+
if let Some(encoding) = self.explicit_encoding.clone() {
|
|
353
|
+
let (mut decoder, bom_len) = explicit_streaming_validator(&encoding, &self.probe)?;
|
|
354
|
+
let malformed = decoder.feed(&self.probe[bom_len..]);
|
|
355
|
+
self.mode = Some(ClassifierMode::Explicit {
|
|
356
|
+
encoding,
|
|
357
|
+
decoder,
|
|
358
|
+
malformed,
|
|
359
|
+
});
|
|
360
|
+
return Ok(());
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
if let Some(BomStreamingValidator {
|
|
364
|
+
mut validator,
|
|
365
|
+
bom_len,
|
|
366
|
+
encoding,
|
|
367
|
+
is_utf8,
|
|
368
|
+
}) = bom_streaming_validator(&self.probe)
|
|
369
|
+
{
|
|
370
|
+
let malformed = validator.feed(&self.probe[bom_len..]);
|
|
371
|
+
self.mode = Some(ClassifierMode::Bom {
|
|
372
|
+
decoder: validator,
|
|
373
|
+
encoding,
|
|
374
|
+
is_utf8,
|
|
375
|
+
malformed,
|
|
376
|
+
});
|
|
377
|
+
return Ok(());
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
let (mut utf8, bom_len) = explicit_streaming_validator("utf-8", &self.probe)
|
|
381
|
+
.expect("the built-in UTF-8 label is valid");
|
|
382
|
+
debug_assert_eq!(bom_len, 0);
|
|
383
|
+
let utf8_malformed = utf8.feed(&self.probe);
|
|
384
|
+
let mut utf8_conflict = Utf8ConflictProbe::new();
|
|
385
|
+
let conflict_hex_offset = utf8_conflict.push(&self.probe);
|
|
386
|
+
let (fallback, fallback_malformed) =
|
|
387
|
+
if let Some(fallback_encoding) = self.fallback_encoding.as_deref() {
|
|
388
|
+
let (mut fallback, bom_len) =
|
|
389
|
+
explicit_streaming_validator(fallback_encoding, &self.probe)?;
|
|
390
|
+
debug_assert_eq!(bom_len, 0, "automatic mode has no BOM to strip");
|
|
391
|
+
let malformed = fallback.feed(&self.probe);
|
|
392
|
+
(Some(fallback), malformed)
|
|
393
|
+
} else {
|
|
394
|
+
(None, false)
|
|
395
|
+
};
|
|
396
|
+
self.mode = Some(ClassifierMode::Automatic {
|
|
397
|
+
utf8,
|
|
398
|
+
utf8_malformed,
|
|
399
|
+
utf8_conflict,
|
|
400
|
+
conflict_hex_offset,
|
|
401
|
+
fallback,
|
|
402
|
+
fallback_malformed,
|
|
403
|
+
});
|
|
404
|
+
Ok(())
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
fn feed_initialized(&mut self, bytes: &[u8]) {
|
|
408
|
+
match self.mode.as_mut().expect("classifier mode is initialized") {
|
|
409
|
+
ClassifierMode::Explicit {
|
|
410
|
+
decoder, malformed, ..
|
|
411
|
+
}
|
|
412
|
+
| ClassifierMode::Bom {
|
|
413
|
+
decoder, malformed, ..
|
|
414
|
+
} => {
|
|
415
|
+
*malformed |= decoder.feed(bytes);
|
|
416
|
+
}
|
|
417
|
+
ClassifierMode::Automatic {
|
|
418
|
+
utf8,
|
|
419
|
+
utf8_malformed,
|
|
420
|
+
utf8_conflict,
|
|
421
|
+
conflict_hex_offset,
|
|
422
|
+
fallback,
|
|
423
|
+
fallback_malformed,
|
|
424
|
+
} => {
|
|
425
|
+
*utf8_malformed |= utf8.feed(bytes);
|
|
426
|
+
if conflict_hex_offset.is_none() {
|
|
427
|
+
*conflict_hex_offset = utf8_conflict.push(bytes);
|
|
428
|
+
}
|
|
429
|
+
if let Some(fallback) = fallback {
|
|
430
|
+
*fallback_malformed |= fallback.feed(bytes);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
fn finish_eof(&self) -> Option<TerminalProof> {
|
|
437
|
+
// EOF-only incomplete sequences are deliberately non-terminal: an appended
|
|
438
|
+
// suffix could complete them, so the sealed full-file validator decides them.
|
|
439
|
+
self.terminal_proof()
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
fn terminal_proof(&self) -> Option<TerminalProof> {
|
|
443
|
+
let mode = self.mode.as_ref()?;
|
|
444
|
+
match mode {
|
|
445
|
+
ClassifierMode::Explicit {
|
|
446
|
+
encoding,
|
|
447
|
+
malformed,
|
|
448
|
+
..
|
|
449
|
+
} => malformed.then(|| TerminalProof::ExplicitDecoderMalformed {
|
|
450
|
+
encoding: encoding.clone(),
|
|
451
|
+
}),
|
|
452
|
+
ClassifierMode::Bom {
|
|
453
|
+
encoding,
|
|
454
|
+
is_utf8,
|
|
455
|
+
malformed,
|
|
456
|
+
..
|
|
457
|
+
} => {
|
|
458
|
+
if *is_utf8 && self.probe.contains(&0) {
|
|
459
|
+
return Some(TerminalProof::NulWithinFrozenProbe);
|
|
460
|
+
}
|
|
461
|
+
(*malformed && (!*is_utf8 || self.probe_is_settled()))
|
|
462
|
+
.then_some(TerminalProof::BomDecoderMalformed { encoding })
|
|
463
|
+
}
|
|
464
|
+
ClassifierMode::Automatic {
|
|
465
|
+
utf8_malformed,
|
|
466
|
+
conflict_hex_offset,
|
|
467
|
+
fallback,
|
|
468
|
+
fallback_malformed,
|
|
469
|
+
..
|
|
470
|
+
} => {
|
|
471
|
+
if self.probe.contains(&0) {
|
|
472
|
+
return Some(TerminalProof::NulWithinFrozenProbe);
|
|
473
|
+
}
|
|
474
|
+
if !self.probe_is_settled() {
|
|
475
|
+
return None;
|
|
476
|
+
}
|
|
477
|
+
if *utf8_malformed && detect_binary_type(&self.probe).is_some() {
|
|
478
|
+
return Some(TerminalProof::BinaryMagicAfterUtf8Failure);
|
|
479
|
+
}
|
|
480
|
+
let conflict_hex_offset = (*conflict_hex_offset)?;
|
|
481
|
+
let fallback = match fallback {
|
|
482
|
+
None => FallbackTerminalState::Absent,
|
|
483
|
+
Some(_) if *fallback_malformed => FallbackTerminalState::PermanentlyMalformed,
|
|
484
|
+
Some(_) => return None,
|
|
485
|
+
};
|
|
486
|
+
Some(TerminalProof::AutomaticRejected {
|
|
487
|
+
conflict_hex_offset,
|
|
488
|
+
fallback,
|
|
489
|
+
})
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
fn probe_is_settled(&self) -> bool {
|
|
495
|
+
self.probe.len() == PREFLIGHT_BYTES
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
enum SnapshotBuilder {
|
|
500
|
+
Memory(Vec<u8>),
|
|
501
|
+
Temp {
|
|
502
|
+
file: tempfile::NamedTempFile,
|
|
503
|
+
len: u64,
|
|
504
|
+
},
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
impl SnapshotBuilder {
|
|
508
|
+
fn new() -> Self {
|
|
509
|
+
Self::Memory(Vec::with_capacity(PREFLIGHT_BYTES))
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
fn append(
|
|
513
|
+
&mut self,
|
|
514
|
+
bytes: &[u8],
|
|
515
|
+
operation: Option<&dyn WorkCheckpoint>,
|
|
516
|
+
) -> Result<(), CaptureFailure> {
|
|
517
|
+
let promote = matches!(
|
|
518
|
+
self,
|
|
519
|
+
Self::Memory(memory)
|
|
520
|
+
if memory.len().saturating_add(bytes.len()) > MEMORY_SNAPSHOT_LIMIT
|
|
521
|
+
);
|
|
522
|
+
if promote {
|
|
523
|
+
checkpoint(operation)?;
|
|
524
|
+
#[cfg(test)]
|
|
525
|
+
if let Some(operation) = operation {
|
|
526
|
+
operation.stage(crate::operation::TestStage::SnapshotPromote);
|
|
527
|
+
}
|
|
528
|
+
checkpoint(operation)?;
|
|
529
|
+
|
|
530
|
+
let memory = match std::mem::replace(self, Self::Memory(Vec::new())) {
|
|
531
|
+
Self::Memory(memory) => memory,
|
|
532
|
+
Self::Temp { .. } => unreachable!("promotion is only possible from memory"),
|
|
533
|
+
};
|
|
534
|
+
let mut file = match tempfile::NamedTempFile::new() {
|
|
535
|
+
Ok(file) => file,
|
|
536
|
+
Err(error) => {
|
|
537
|
+
return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
for chunk in memory.chunks(SNAPSHOT_CHUNK_BYTES) {
|
|
541
|
+
write_snapshot_chunk(&mut file, chunk, operation)?;
|
|
542
|
+
}
|
|
543
|
+
let len = memory.len() as u64;
|
|
544
|
+
*self = Self::Temp { file, len };
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
match self {
|
|
548
|
+
Self::Memory(memory) => memory.extend_from_slice(bytes),
|
|
549
|
+
Self::Temp { file, len } => {
|
|
550
|
+
write_snapshot_chunk(file, bytes, operation)?;
|
|
551
|
+
*len = len.saturating_add(bytes.len() as u64);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
Ok(())
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
fn finish(
|
|
558
|
+
mut self,
|
|
559
|
+
operation: Option<&dyn WorkCheckpoint>,
|
|
560
|
+
) -> Result<CapturedSnapshot, CaptureFailure> {
|
|
561
|
+
match &mut self {
|
|
562
|
+
Self::Memory(_) => {}
|
|
563
|
+
Self::Temp { file, .. } => {
|
|
564
|
+
checkpoint(operation)?;
|
|
565
|
+
if let Err(error) = file.as_file_mut().flush() {
|
|
566
|
+
return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
|
|
567
|
+
}
|
|
568
|
+
checkpoint(operation)?;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
Ok(match self {
|
|
572
|
+
Self::Memory(memory) => CapturedSnapshot {
|
|
573
|
+
len: memory.len() as u64,
|
|
574
|
+
backing: SnapshotBacking::Memory(Arc::from(memory)),
|
|
575
|
+
},
|
|
576
|
+
Self::Temp { file, len } => CapturedSnapshot {
|
|
577
|
+
len,
|
|
578
|
+
backing: SnapshotBacking::Temp(Arc::new(TempSnapshot { file })),
|
|
579
|
+
},
|
|
580
|
+
})
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
fn write_snapshot_chunk(
|
|
585
|
+
file: &mut tempfile::NamedTempFile,
|
|
586
|
+
bytes: &[u8],
|
|
587
|
+
operation: Option<&dyn WorkCheckpoint>,
|
|
588
|
+
) -> Result<(), CaptureFailure> {
|
|
589
|
+
debug_assert!(bytes.len() <= SNAPSHOT_CHUNK_BYTES);
|
|
590
|
+
checkpoint(operation)?;
|
|
591
|
+
if let Err(error) = file.as_file_mut().write_all(bytes) {
|
|
592
|
+
return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
|
|
593
|
+
}
|
|
594
|
+
checkpoint(operation)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
fn capture_reader(
|
|
598
|
+
reader: &mut impl Read,
|
|
599
|
+
explicit_encoding: Option<&str>,
|
|
600
|
+
fallback_encoding: Option<&str>,
|
|
601
|
+
operation: Option<&dyn WorkCheckpoint>,
|
|
602
|
+
) -> Result<CaptureRead, CaptureFailure> {
|
|
603
|
+
let mut builder = SnapshotBuilder::new();
|
|
604
|
+
let mut classifier = CaptureClassifier::new(explicit_encoding, fallback_encoding);
|
|
605
|
+
let mut chunk = [0_u8; SNAPSHOT_CHUNK_BYTES];
|
|
606
|
+
let mut preflight_bytes = 0_usize;
|
|
607
|
+
let mut reached_eof = false;
|
|
608
|
+
|
|
609
|
+
while preflight_bytes < PREFLIGHT_BYTES {
|
|
610
|
+
checkpoint(operation)?;
|
|
611
|
+
#[cfg(test)]
|
|
612
|
+
if let Some(operation) = operation {
|
|
613
|
+
operation.stage(crate::operation::TestStage::CapturePreflightRead);
|
|
614
|
+
}
|
|
615
|
+
checkpoint(operation)?;
|
|
616
|
+
let maximum = PREFLIGHT_BYTES - preflight_bytes;
|
|
617
|
+
let count = match reader.read(&mut chunk[..maximum]) {
|
|
618
|
+
Ok(count) => count,
|
|
619
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
620
|
+
};
|
|
621
|
+
checkpoint(operation)?;
|
|
622
|
+
if count == 0 {
|
|
623
|
+
reached_eof = true;
|
|
624
|
+
break;
|
|
625
|
+
}
|
|
626
|
+
preflight_bytes += count;
|
|
627
|
+
let proof = match classifier.feed(&chunk[..count]) {
|
|
628
|
+
Ok(proof) => proof,
|
|
629
|
+
Err(rejection) => {
|
|
630
|
+
return Err(prefer_stop(
|
|
631
|
+
operation,
|
|
632
|
+
CaptureFailure::InvalidEncoding(rejection),
|
|
633
|
+
));
|
|
634
|
+
}
|
|
635
|
+
};
|
|
636
|
+
checkpoint(operation)?;
|
|
637
|
+
if let Some(proof) = proof {
|
|
638
|
+
return Ok(CaptureRead::Terminal(proof));
|
|
639
|
+
}
|
|
640
|
+
builder.append(&chunk[..count], operation)?;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
if !reached_eof {
|
|
644
|
+
loop {
|
|
645
|
+
checkpoint(operation)?;
|
|
646
|
+
#[cfg(test)]
|
|
647
|
+
if let Some(operation) = operation {
|
|
648
|
+
operation.stage(crate::operation::TestStage::SnapshotChunk);
|
|
649
|
+
}
|
|
650
|
+
checkpoint(operation)?;
|
|
651
|
+
let count = match reader.read(&mut chunk) {
|
|
652
|
+
Ok(count) => count,
|
|
653
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
654
|
+
};
|
|
655
|
+
checkpoint(operation)?;
|
|
656
|
+
if count == 0 {
|
|
657
|
+
break;
|
|
658
|
+
}
|
|
659
|
+
let proof = match classifier.feed(&chunk[..count]) {
|
|
660
|
+
Ok(proof) => proof,
|
|
661
|
+
Err(rejection) => {
|
|
662
|
+
return Err(prefer_stop(
|
|
663
|
+
operation,
|
|
664
|
+
CaptureFailure::InvalidEncoding(rejection),
|
|
665
|
+
));
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
checkpoint(operation)?;
|
|
669
|
+
if let Some(proof) = proof {
|
|
670
|
+
return Ok(CaptureRead::Terminal(proof));
|
|
671
|
+
}
|
|
672
|
+
builder.append(&chunk[..count], operation)?;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
let proof = classifier.finish_eof();
|
|
676
|
+
checkpoint(operation)?;
|
|
677
|
+
if let Some(proof) = proof {
|
|
678
|
+
return Ok(CaptureRead::Terminal(proof));
|
|
679
|
+
}
|
|
680
|
+
builder.finish(operation).map(CaptureRead::Complete)
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
fn checkpoint(operation: Option<&dyn WorkCheckpoint>) -> Result<(), CaptureFailure> {
|
|
684
|
+
match operation.map(WorkCheckpoint::check_work) {
|
|
685
|
+
Some(Err(WorkStop::RequestCancelled)) => Err(CaptureFailure::Cancelled),
|
|
686
|
+
Some(Err(WorkStop::EpochRetired)) => Err(CaptureFailure::EpochRetired),
|
|
687
|
+
Some(Ok(())) | None => Ok(()),
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
fn prefer_stop(operation: Option<&dyn WorkCheckpoint>, failure: CaptureFailure) -> CaptureFailure {
|
|
692
|
+
match checkpoint(operation) {
|
|
693
|
+
Ok(()) => failure,
|
|
694
|
+
Err(stop) => stop,
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
#[cfg(not(windows))]
|
|
699
|
+
fn open_original(path: &Path) -> io::Result<File> {
|
|
700
|
+
File::open(path)
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
#[cfg(windows)]
|
|
704
|
+
fn open_original(path: &Path) -> io::Result<File> {
|
|
705
|
+
use std::os::windows::fs::OpenOptionsExt;
|
|
706
|
+
use windows_sys::Win32::Storage::FileSystem::{
|
|
707
|
+
FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
|
|
708
|
+
};
|
|
709
|
+
|
|
710
|
+
// Replacement must remain observable while the capture handle is live.
|
|
711
|
+
std::fs::OpenOptions::new()
|
|
712
|
+
.read(true)
|
|
713
|
+
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
|
714
|
+
.open(path)
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/// Opens the original candidate once, captures its bytes, and verifies stable identity.
|
|
718
|
+
pub(crate) fn capture_classify(
|
|
719
|
+
candidate: &PathRecord,
|
|
720
|
+
explicit_encoding: Option<&str>,
|
|
721
|
+
fallback_encoding: Option<&str>,
|
|
722
|
+
operation: Option<&dyn WorkCheckpoint>,
|
|
723
|
+
) -> Result<CaptureDisposition, CaptureFailure> {
|
|
724
|
+
checkpoint(operation)?;
|
|
725
|
+
let mut file = match open_original(&candidate.native) {
|
|
726
|
+
Ok(file) => file,
|
|
727
|
+
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
|
728
|
+
checkpoint(operation)?;
|
|
729
|
+
return Ok(CaptureDisposition::FileChanged);
|
|
730
|
+
}
|
|
731
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
732
|
+
};
|
|
733
|
+
let (before, is_regular) = match identity_from_file(&file) {
|
|
734
|
+
Ok(identity) => identity,
|
|
735
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
736
|
+
};
|
|
737
|
+
checkpoint(operation)?;
|
|
738
|
+
if !is_regular || !before.matches_hint(candidate.traversal_fingerprint.as_ref()) {
|
|
739
|
+
return Ok(CaptureDisposition::FileChanged);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
let capture = capture_reader(&mut file, explicit_encoding, fallback_encoding, operation)?;
|
|
743
|
+
checkpoint(operation)?;
|
|
744
|
+
#[cfg(test)]
|
|
745
|
+
if let Some(operation) = operation {
|
|
746
|
+
operation.stage(crate::operation::TestStage::BeforeIdentityPostCheck);
|
|
747
|
+
}
|
|
748
|
+
checkpoint(operation)?;
|
|
749
|
+
|
|
750
|
+
let (after, is_regular) = match identity_from_file(&file) {
|
|
751
|
+
Ok(identity) => identity,
|
|
752
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
753
|
+
};
|
|
754
|
+
let path_identity = match identity_from_path(&candidate.native) {
|
|
755
|
+
Ok(identity) => identity,
|
|
756
|
+
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
|
757
|
+
checkpoint(operation)?;
|
|
758
|
+
return Ok(CaptureDisposition::FileChanged);
|
|
759
|
+
}
|
|
760
|
+
Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
|
|
761
|
+
};
|
|
762
|
+
checkpoint(operation)?;
|
|
763
|
+
if !is_regular
|
|
764
|
+
|| !path_identity.1
|
|
765
|
+
|| !before.same_state(&after)
|
|
766
|
+
|| !after.same_state(&path_identity.0)
|
|
767
|
+
|| !after.matches_hint(candidate.traversal_fingerprint.as_ref())
|
|
768
|
+
{
|
|
769
|
+
return Ok(CaptureDisposition::FileChanged);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
Ok(match capture {
|
|
773
|
+
CaptureRead::Terminal(proof) => match proof.rejection() {
|
|
774
|
+
Some(rejection) => CaptureDisposition::EncodingRejected { rejection, proof },
|
|
775
|
+
None => CaptureDisposition::BinarySkipped(proof),
|
|
776
|
+
},
|
|
777
|
+
CaptureRead::Complete(captured) => {
|
|
778
|
+
CaptureDisposition::Searchable(Box::new(SealedSnapshot {
|
|
779
|
+
path: candidate.clone(),
|
|
780
|
+
_before: before,
|
|
781
|
+
_after: after,
|
|
782
|
+
captured,
|
|
783
|
+
}))
|
|
784
|
+
}
|
|
785
|
+
})
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
#[cfg(unix)]
|
|
789
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
790
|
+
struct FileIdentity {
|
|
791
|
+
device: u64,
|
|
792
|
+
inode: u64,
|
|
793
|
+
len: u64,
|
|
794
|
+
modified_seconds: i64,
|
|
795
|
+
modified_nanoseconds: i64,
|
|
796
|
+
changed_seconds: i64,
|
|
797
|
+
changed_nanoseconds: i64,
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
#[cfg(unix)]
|
|
801
|
+
impl FileIdentity {
|
|
802
|
+
fn from_metadata(metadata: &std::fs::Metadata) -> Self {
|
|
803
|
+
use std::os::unix::fs::MetadataExt;
|
|
804
|
+
Self {
|
|
805
|
+
device: metadata.dev(),
|
|
806
|
+
inode: metadata.ino(),
|
|
807
|
+
len: metadata.len(),
|
|
808
|
+
modified_seconds: metadata.mtime(),
|
|
809
|
+
modified_nanoseconds: metadata.mtime_nsec(),
|
|
810
|
+
changed_seconds: metadata.ctime(),
|
|
811
|
+
changed_nanoseconds: metadata.ctime_nsec(),
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
fn matches_hint(&self, hint: Option<&FileIdentityHint>) -> bool {
|
|
816
|
+
hint.is_none_or(|hint| {
|
|
817
|
+
self.device == hint.device
|
|
818
|
+
&& self.inode == hint.inode
|
|
819
|
+
&& self.len == hint.len
|
|
820
|
+
&& self.modified_seconds == hint.modified_seconds
|
|
821
|
+
&& self.modified_nanoseconds == hint.modified_nanoseconds
|
|
822
|
+
&& self.changed_seconds == hint.changed_seconds
|
|
823
|
+
&& self.changed_nanoseconds == hint.changed_nanoseconds
|
|
824
|
+
})
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
fn same_state(&self, other: &Self) -> bool {
|
|
828
|
+
self == other
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
#[cfg(unix)]
|
|
833
|
+
fn identity_from_file(file: &File) -> io::Result<(FileIdentity, bool)> {
|
|
834
|
+
let metadata = file.metadata()?;
|
|
835
|
+
Ok((FileIdentity::from_metadata(&metadata), metadata.is_file()))
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
#[cfg(unix)]
|
|
839
|
+
fn identity_from_path(path: &Path) -> io::Result<(FileIdentity, bool)> {
|
|
840
|
+
let metadata = std::fs::metadata(path)?;
|
|
841
|
+
Ok((FileIdentity::from_metadata(&metadata), metadata.is_file()))
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
#[cfg(windows)]
|
|
845
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
846
|
+
enum WindowsFileId {
|
|
847
|
+
Extended {
|
|
848
|
+
volume_serial: u64,
|
|
849
|
+
identifier: [u8; 16],
|
|
850
|
+
},
|
|
851
|
+
Legacy {
|
|
852
|
+
volume_serial: u32,
|
|
853
|
+
file_index: u64,
|
|
854
|
+
},
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
#[cfg(windows)]
|
|
858
|
+
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
859
|
+
struct FileIdentity {
|
|
860
|
+
file_id: WindowsFileId,
|
|
861
|
+
len: u64,
|
|
862
|
+
creation_time: u64,
|
|
863
|
+
last_write_time: u64,
|
|
864
|
+
change_time: Option<i64>,
|
|
865
|
+
attributes: u32,
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
#[cfg(windows)]
|
|
869
|
+
impl FileIdentity {
|
|
870
|
+
fn from_file(file: &File) -> io::Result<Self> {
|
|
871
|
+
use std::mem::size_of;
|
|
872
|
+
use std::os::windows::io::AsRawHandle;
|
|
873
|
+
use windows_sys::Win32::Foundation::{ERROR_INVALID_PARAMETER, ERROR_NOT_SUPPORTED};
|
|
874
|
+
use windows_sys::Win32::Storage::FileSystem::{
|
|
875
|
+
BY_HANDLE_FILE_INFORMATION, FILE_BASIC_INFO, FILE_ID_INFO, FileBasicInfo, FileIdInfo,
|
|
876
|
+
GetFileInformationByHandle, GetFileInformationByHandleEx,
|
|
877
|
+
};
|
|
878
|
+
|
|
879
|
+
let handle = file.as_raw_handle();
|
|
880
|
+
let mut info = BY_HANDLE_FILE_INFORMATION::default();
|
|
881
|
+
let ok = unsafe { GetFileInformationByHandle(handle, &mut info) };
|
|
882
|
+
if ok == 0 {
|
|
883
|
+
return Err(io::Error::last_os_error());
|
|
884
|
+
}
|
|
885
|
+
let mut extended_id = FILE_ID_INFO::default();
|
|
886
|
+
let extended_id_ok = unsafe {
|
|
887
|
+
GetFileInformationByHandleEx(
|
|
888
|
+
handle,
|
|
889
|
+
FileIdInfo,
|
|
890
|
+
(&mut extended_id as *mut FILE_ID_INFO).cast(),
|
|
891
|
+
size_of::<FILE_ID_INFO>() as u32,
|
|
892
|
+
)
|
|
893
|
+
};
|
|
894
|
+
let file_id = if extended_id_ok != 0 {
|
|
895
|
+
WindowsFileId::Extended {
|
|
896
|
+
volume_serial: extended_id.VolumeSerialNumber,
|
|
897
|
+
identifier: extended_id.FileId.Identifier,
|
|
898
|
+
}
|
|
899
|
+
} else {
|
|
900
|
+
let error = io::Error::last_os_error();
|
|
901
|
+
match error.raw_os_error() {
|
|
902
|
+
Some(code)
|
|
903
|
+
if code == ERROR_NOT_SUPPORTED as i32
|
|
904
|
+
|| code == ERROR_INVALID_PARAMETER as i32 =>
|
|
905
|
+
{
|
|
906
|
+
WindowsFileId::Legacy {
|
|
907
|
+
volume_serial: info.dwVolumeSerialNumber,
|
|
908
|
+
file_index: (u64::from(info.nFileIndexHigh) << 32)
|
|
909
|
+
| u64::from(info.nFileIndexLow),
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
_ => return Err(error),
|
|
913
|
+
}
|
|
914
|
+
};
|
|
915
|
+
let mut basic = FILE_BASIC_INFO::default();
|
|
916
|
+
let basic_ok = unsafe {
|
|
917
|
+
GetFileInformationByHandleEx(
|
|
918
|
+
handle,
|
|
919
|
+
FileBasicInfo,
|
|
920
|
+
(&mut basic as *mut FILE_BASIC_INFO).cast(),
|
|
921
|
+
size_of::<FILE_BASIC_INFO>() as u32,
|
|
922
|
+
)
|
|
923
|
+
};
|
|
924
|
+
Ok(Self {
|
|
925
|
+
file_id,
|
|
926
|
+
len: (u64::from(info.nFileSizeHigh) << 32) | u64::from(info.nFileSizeLow),
|
|
927
|
+
creation_time: filetime_u64(info.ftCreationTime),
|
|
928
|
+
last_write_time: filetime_u64(info.ftLastWriteTime),
|
|
929
|
+
change_time: (basic_ok != 0).then_some(basic.ChangeTime),
|
|
930
|
+
attributes: info.dwFileAttributes,
|
|
931
|
+
})
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
fn matches_hint(&self, hint: Option<&FileIdentityHint>) -> bool {
|
|
935
|
+
// Stable Windows file IDs are handle-only; traversal metadata can compare
|
|
936
|
+
// only these fields, while the before/after/path handle checks compare IDs.
|
|
937
|
+
hint.is_none_or(|hint| {
|
|
938
|
+
self.len == hint.len
|
|
939
|
+
&& self.creation_time == hint.creation_time
|
|
940
|
+
&& self.last_write_time == hint.last_write_time
|
|
941
|
+
&& self.attributes == hint.attributes
|
|
942
|
+
})
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
fn same_state(&self, other: &Self) -> bool {
|
|
946
|
+
self.file_id == other.file_id
|
|
947
|
+
&& self.len == other.len
|
|
948
|
+
&& self.creation_time == other.creation_time
|
|
949
|
+
&& self.last_write_time == other.last_write_time
|
|
950
|
+
&& self.attributes == other.attributes
|
|
951
|
+
&& match (self.change_time, other.change_time) {
|
|
952
|
+
(Some(left), Some(right)) => left == right,
|
|
953
|
+
_ => true,
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
#[cfg(windows)]
|
|
959
|
+
fn filetime_u64(value: windows_sys::Win32::Foundation::FILETIME) -> u64 {
|
|
960
|
+
(u64::from(value.dwHighDateTime) << 32) | u64::from(value.dwLowDateTime)
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
#[cfg(windows)]
|
|
964
|
+
fn identity_from_file(file: &File) -> io::Result<(FileIdentity, bool)> {
|
|
965
|
+
let metadata = file.metadata()?;
|
|
966
|
+
Ok((FileIdentity::from_file(file)?, metadata.is_file()))
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
#[cfg(windows)]
|
|
970
|
+
fn identity_from_path(path: &Path) -> io::Result<(FileIdentity, bool)> {
|
|
971
|
+
use std::os::windows::fs::OpenOptionsExt;
|
|
972
|
+
use windows_sys::Win32::Storage::FileSystem::{
|
|
973
|
+
FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
|
|
974
|
+
};
|
|
975
|
+
|
|
976
|
+
// Windows path metadata omits the stable file index, so this attributes-only
|
|
977
|
+
// handle is required to prove that the path still names the captured file.
|
|
978
|
+
let file = std::fs::OpenOptions::new()
|
|
979
|
+
.access_mode(FILE_READ_ATTRIBUTES)
|
|
980
|
+
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
|
981
|
+
.open(path)?;
|
|
982
|
+
identity_from_file(&file)
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
#[cfg(test)]
|
|
986
|
+
mod tests {
|
|
987
|
+
use super::{
|
|
988
|
+
CaptureDisposition, CaptureFailure, CaptureRead, FallbackTerminalState,
|
|
989
|
+
MEMORY_SNAPSHOT_LIMIT, PREFLIGHT_BYTES, SNAPSHOT_CHUNK_BYTES, TerminalProof,
|
|
990
|
+
capture_classify, capture_reader,
|
|
991
|
+
};
|
|
992
|
+
use crate::encoding::{
|
|
993
|
+
ByteSource, EncodingDecision, EncodingRejection, validate_source_encoding,
|
|
994
|
+
};
|
|
995
|
+
use crate::operation::{RequestWorkGuard, TestStage};
|
|
996
|
+
use crate::path_codec::PathRecord;
|
|
997
|
+
|
|
998
|
+
use filetime::{FileTime, set_file_mtime};
|
|
999
|
+
use rmcp::model::RequestId;
|
|
1000
|
+
use std::io::{self, Read, Write};
|
|
1001
|
+
use std::sync::Arc;
|
|
1002
|
+
|
|
1003
|
+
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
|
1004
|
+
use tokio_util::sync::CancellationToken;
|
|
1005
|
+
|
|
1006
|
+
struct ChunkedReader {
|
|
1007
|
+
bytes: Vec<u8>,
|
|
1008
|
+
position: usize,
|
|
1009
|
+
maximum: usize,
|
|
1010
|
+
fail_at: Option<usize>,
|
|
1011
|
+
reads: usize,
|
|
1012
|
+
requested: Vec<usize>,
|
|
1013
|
+
position_counter: Option<Arc<AtomicUsize>>,
|
|
1014
|
+
read_limits: Option<Vec<usize>>,
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
impl ChunkedReader {
|
|
1018
|
+
fn new(bytes: Vec<u8>, maximum: usize) -> Self {
|
|
1019
|
+
Self {
|
|
1020
|
+
bytes,
|
|
1021
|
+
position: 0,
|
|
1022
|
+
maximum,
|
|
1023
|
+
fail_at: None,
|
|
1024
|
+
reads: 0,
|
|
1025
|
+
requested: Vec::new(),
|
|
1026
|
+
position_counter: None,
|
|
1027
|
+
read_limits: None,
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
fn with_position_counter(mut self, position: Arc<AtomicUsize>) -> Self {
|
|
1032
|
+
self.position_counter = Some(position);
|
|
1033
|
+
self
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
fn with_read_limits(mut self, read_limits: Vec<usize>) -> Self {
|
|
1037
|
+
assert!(!read_limits.is_empty());
|
|
1038
|
+
assert!(read_limits.iter().all(|limit| *limit > 0));
|
|
1039
|
+
self.read_limits = Some(read_limits);
|
|
1040
|
+
self
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
impl Read for ChunkedReader {
|
|
1045
|
+
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
|
|
1046
|
+
self.reads += 1;
|
|
1047
|
+
self.requested.push(output.len());
|
|
1048
|
+
if self.fail_at.is_some_and(|offset| self.position >= offset) {
|
|
1049
|
+
return Err(io::Error::other("injected read failure"));
|
|
1050
|
+
}
|
|
1051
|
+
if self.position >= self.bytes.len() {
|
|
1052
|
+
return Ok(0);
|
|
1053
|
+
}
|
|
1054
|
+
let read_limit = self.read_limits.as_ref().map_or(self.maximum, |limits| {
|
|
1055
|
+
limits[(self.reads - 1) % limits.len()]
|
|
1056
|
+
});
|
|
1057
|
+
let count = output
|
|
1058
|
+
.len()
|
|
1059
|
+
.min(self.maximum)
|
|
1060
|
+
.min(read_limit)
|
|
1061
|
+
.min(self.bytes.len() - self.position);
|
|
1062
|
+
output[..count].copy_from_slice(&self.bytes[self.position..self.position + count]);
|
|
1063
|
+
self.position += count;
|
|
1064
|
+
if let Some(position) = &self.position_counter {
|
|
1065
|
+
position.store(self.position, Ordering::Release);
|
|
1066
|
+
}
|
|
1067
|
+
Ok(count)
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
#[derive(Debug, Eq, PartialEq)]
|
|
1072
|
+
enum ReferenceDisposition {
|
|
1073
|
+
Text { used_fallback: bool },
|
|
1074
|
+
Binary,
|
|
1075
|
+
Rejected(EncodingRejection),
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
fn reference_disposition(
|
|
1079
|
+
bytes: &[u8],
|
|
1080
|
+
explicit: Option<&str>,
|
|
1081
|
+
fallback: Option<&str>,
|
|
1082
|
+
) -> ReferenceDisposition {
|
|
1083
|
+
let source = ByteSource::Bytes(bytes);
|
|
1084
|
+
match validate_source_encoding(source, explicit).unwrap() {
|
|
1085
|
+
EncodingDecision::Text(_) => ReferenceDisposition::Text {
|
|
1086
|
+
used_fallback: false,
|
|
1087
|
+
},
|
|
1088
|
+
EncodingDecision::Binary => ReferenceDisposition::Binary,
|
|
1089
|
+
EncodingDecision::Rejected(rejection) => {
|
|
1090
|
+
if explicit.is_none()
|
|
1091
|
+
&& !matches!(rejection, EncodingRejection::BomMismatch { .. })
|
|
1092
|
+
&& let Some(fallback) = fallback
|
|
1093
|
+
&& matches!(
|
|
1094
|
+
validate_source_encoding(source, Some(fallback)).unwrap(),
|
|
1095
|
+
EncodingDecision::Text(_)
|
|
1096
|
+
)
|
|
1097
|
+
{
|
|
1098
|
+
ReferenceDisposition::Text {
|
|
1099
|
+
used_fallback: true,
|
|
1100
|
+
}
|
|
1101
|
+
} else {
|
|
1102
|
+
ReferenceDisposition::Rejected(rejection)
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
fn proof_disposition(proof: &TerminalProof) -> ReferenceDisposition {
|
|
1109
|
+
proof
|
|
1110
|
+
.rejection()
|
|
1111
|
+
.map_or(ReferenceDisposition::Binary, ReferenceDisposition::Rejected)
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
fn mixed_prefix() -> Vec<u8> {
|
|
1115
|
+
let mut bytes = "界".repeat(11).into_bytes();
|
|
1116
|
+
debug_assert_eq!(bytes.len(), 33);
|
|
1117
|
+
bytes.push(0xFF);
|
|
1118
|
+
bytes.resize(PREFLIGHT_BYTES, b'a');
|
|
1119
|
+
bytes
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
#[test]
|
|
1123
|
+
fn every_production_terminal_proof_is_suffix_invariant_against_the_reference() {
|
|
1124
|
+
let cases = [
|
|
1125
|
+
(
|
|
1126
|
+
"nul",
|
|
1127
|
+
vec![b'a', 0, b'b', b'c'],
|
|
1128
|
+
None,
|
|
1129
|
+
None,
|
|
1130
|
+
TerminalProof::NulWithinFrozenProbe,
|
|
1131
|
+
),
|
|
1132
|
+
(
|
|
1133
|
+
"binary magic",
|
|
1134
|
+
{
|
|
1135
|
+
let mut bytes = b"\x1F\x8B".to_vec();
|
|
1136
|
+
bytes.resize(PREFLIGHT_BYTES, b'a');
|
|
1137
|
+
bytes
|
|
1138
|
+
},
|
|
1139
|
+
None,
|
|
1140
|
+
None,
|
|
1141
|
+
TerminalProof::BinaryMagicAfterUtf8Failure,
|
|
1142
|
+
),
|
|
1143
|
+
(
|
|
1144
|
+
"explicit malformed",
|
|
1145
|
+
vec![b'a', b'b', b'c', 0xFF],
|
|
1146
|
+
Some("utf-8"),
|
|
1147
|
+
None,
|
|
1148
|
+
TerminalProof::ExplicitDecoderMalformed {
|
|
1149
|
+
encoding: "utf-8".to_string(),
|
|
1150
|
+
},
|
|
1151
|
+
),
|
|
1152
|
+
(
|
|
1153
|
+
"explicit matching BOM malformed",
|
|
1154
|
+
b"\xFF\xFE\x00\x00\x00\x00\x11\x00".to_vec(),
|
|
1155
|
+
Some("utf-32le"),
|
|
1156
|
+
None,
|
|
1157
|
+
TerminalProof::ExplicitDecoderMalformed {
|
|
1158
|
+
encoding: "utf-32le".to_string(),
|
|
1159
|
+
},
|
|
1160
|
+
),
|
|
1161
|
+
(
|
|
1162
|
+
"BOM malformed",
|
|
1163
|
+
b"\xFF\xFE\x00\x00\x00\x00\x11\x00".to_vec(),
|
|
1164
|
+
None,
|
|
1165
|
+
None,
|
|
1166
|
+
TerminalProof::BomDecoderMalformed {
|
|
1167
|
+
encoding: "UTF-32LE",
|
|
1168
|
+
},
|
|
1169
|
+
),
|
|
1170
|
+
(
|
|
1171
|
+
"UTF-8 BOM malformed after NUL gate",
|
|
1172
|
+
{
|
|
1173
|
+
let mut bytes = b"\xEF\xBB\xBF\xFF".to_vec();
|
|
1174
|
+
bytes.resize(PREFLIGHT_BYTES, b'a');
|
|
1175
|
+
bytes
|
|
1176
|
+
},
|
|
1177
|
+
None,
|
|
1178
|
+
None,
|
|
1179
|
+
TerminalProof::BomDecoderMalformed { encoding: "UTF-8" },
|
|
1180
|
+
),
|
|
1181
|
+
(
|
|
1182
|
+
"automatic mixed without fallback",
|
|
1183
|
+
mixed_prefix(),
|
|
1184
|
+
None,
|
|
1185
|
+
None,
|
|
1186
|
+
TerminalProof::AutomaticRejected {
|
|
1187
|
+
conflict_hex_offset: 3,
|
|
1188
|
+
fallback: FallbackTerminalState::Absent,
|
|
1189
|
+
},
|
|
1190
|
+
),
|
|
1191
|
+
(
|
|
1192
|
+
"automatic mixed with permanently malformed fallback",
|
|
1193
|
+
mixed_prefix(),
|
|
1194
|
+
None,
|
|
1195
|
+
Some("utf-8"),
|
|
1196
|
+
TerminalProof::AutomaticRejected {
|
|
1197
|
+
conflict_hex_offset: 3,
|
|
1198
|
+
fallback: FallbackTerminalState::PermanentlyMalformed,
|
|
1199
|
+
},
|
|
1200
|
+
),
|
|
1201
|
+
];
|
|
1202
|
+
let mut suffixes = vec![
|
|
1203
|
+
Vec::new(),
|
|
1204
|
+
b"plain utf-8".to_vec(),
|
|
1205
|
+
vec![0; 257],
|
|
1206
|
+
vec![0x80; 257],
|
|
1207
|
+
(0..4096).map(|index| (index * 73) as u8).collect(),
|
|
1208
|
+
];
|
|
1209
|
+
suffixes.extend((0_u16..=u8::MAX as u16).map(|byte| vec![byte as u8]));
|
|
1210
|
+
for (name, prefix, explicit, fallback, expected_proof) in cases {
|
|
1211
|
+
for suffix in &suffixes {
|
|
1212
|
+
let mut bytes = prefix.clone();
|
|
1213
|
+
bytes.extend_from_slice(suffix);
|
|
1214
|
+
let expected = reference_disposition(&bytes, explicit, fallback);
|
|
1215
|
+
for maximum in [1, 2, 3, 7, 127, PREFLIGHT_BYTES] {
|
|
1216
|
+
let mut reader = ChunkedReader::new(bytes.clone(), maximum);
|
|
1217
|
+
reader.fail_at = Some(prefix.len());
|
|
1218
|
+
let CaptureRead::Terminal(proof) =
|
|
1219
|
+
capture_reader(&mut reader, explicit, fallback, None).unwrap()
|
|
1220
|
+
else {
|
|
1221
|
+
panic!("{name}: expected terminal proof at chunk size {maximum}");
|
|
1222
|
+
};
|
|
1223
|
+
assert_eq!(proof, expected_proof, "{name}, chunk size {maximum}");
|
|
1224
|
+
assert_eq!(
|
|
1225
|
+
proof_disposition(&proof),
|
|
1226
|
+
expected,
|
|
1227
|
+
"{name}, chunk size {maximum}, suffix length {}",
|
|
1228
|
+
suffix.len()
|
|
1229
|
+
);
|
|
1230
|
+
assert!(
|
|
1231
|
+
reader.position
|
|
1232
|
+
<= prefix
|
|
1233
|
+
.len()
|
|
1234
|
+
.saturating_add(maximum.saturating_sub(1))
|
|
1235
|
+
.min(bytes.len()),
|
|
1236
|
+
"{name}, chunk size {maximum}: reader crossed a second read after proof"
|
|
1237
|
+
);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
let suffix = (0..4096)
|
|
1242
|
+
.map(|index| (index * 151 + 17) as u8)
|
|
1243
|
+
.collect::<Vec<_>>();
|
|
1244
|
+
let mut bytes = prefix.clone();
|
|
1245
|
+
bytes.extend_from_slice(&suffix);
|
|
1246
|
+
let expected = reference_disposition(&bytes, explicit, fallback);
|
|
1247
|
+
let mut random_state = 0xA5A5_1F3D_u32;
|
|
1248
|
+
let read_limits = (0..64)
|
|
1249
|
+
.map(|_| {
|
|
1250
|
+
random_state ^= random_state << 13;
|
|
1251
|
+
random_state ^= random_state >> 17;
|
|
1252
|
+
random_state ^= random_state << 5;
|
|
1253
|
+
(random_state as usize % 1024) + 1
|
|
1254
|
+
})
|
|
1255
|
+
.collect();
|
|
1256
|
+
let mut reader = ChunkedReader::new(bytes, usize::MAX).with_read_limits(read_limits);
|
|
1257
|
+
reader.fail_at = Some(prefix.len());
|
|
1258
|
+
let CaptureRead::Terminal(proof) =
|
|
1259
|
+
capture_reader(&mut reader, explicit, fallback, None).unwrap()
|
|
1260
|
+
else {
|
|
1261
|
+
panic!("{name}: expected terminal proof at randomized chunk boundaries");
|
|
1262
|
+
};
|
|
1263
|
+
assert_eq!(proof, expected_proof, "{name}, randomized chunks");
|
|
1264
|
+
assert_eq!(
|
|
1265
|
+
proof_disposition(&proof),
|
|
1266
|
+
expected,
|
|
1267
|
+
"{name}, randomized chunks"
|
|
1268
|
+
);
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
#[test]
|
|
1273
|
+
fn large_capture_promotes_once_reads_repeatably_and_unlinks_on_drop() {
|
|
1274
|
+
let bytes = vec![b'x'; MEMORY_SNAPSHOT_LIMIT + 1];
|
|
1275
|
+
let mut reader = ChunkedReader::new(bytes.clone(), 17 * 1024);
|
|
1276
|
+
let promotions = Arc::new(AtomicUsize::new(0));
|
|
1277
|
+
let promotions_for_hook = Arc::clone(&promotions);
|
|
1278
|
+
let hook = Arc::new(move |stage| {
|
|
1279
|
+
if stage == TestStage::SnapshotPromote {
|
|
1280
|
+
promotions_for_hook.fetch_add(1, Ordering::AcqRel);
|
|
1281
|
+
}
|
|
1282
|
+
});
|
|
1283
|
+
let (mut guard, operation) =
|
|
1284
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(10), CancellationToken::new(), hook);
|
|
1285
|
+
let CaptureRead::Complete(snapshot) =
|
|
1286
|
+
capture_reader(&mut reader, None, None, Some(&operation)).unwrap()
|
|
1287
|
+
else {
|
|
1288
|
+
panic!("expected complete snapshot");
|
|
1289
|
+
};
|
|
1290
|
+
guard.disarm();
|
|
1291
|
+
assert_eq!(promotions.load(Ordering::Acquire), 1);
|
|
1292
|
+
assert_eq!(reader.requested.first(), Some(&PREFLIGHT_BYTES));
|
|
1293
|
+
assert!(
|
|
1294
|
+
reader
|
|
1295
|
+
.requested
|
|
1296
|
+
.iter()
|
|
1297
|
+
.all(|requested| *requested <= SNAPSHOT_CHUNK_BYTES)
|
|
1298
|
+
);
|
|
1299
|
+
let temp_path = snapshot.temp_path().unwrap().to_path_buf();
|
|
1300
|
+
assert!(temp_path.exists());
|
|
1301
|
+
assert_eq!(snapshot.len, bytes.len() as u64);
|
|
1302
|
+
for _ in 0..2 {
|
|
1303
|
+
let mut reopened = snapshot.open_reader(0).unwrap();
|
|
1304
|
+
let mut actual = Vec::new();
|
|
1305
|
+
reopened.read_to_end(&mut actual).unwrap();
|
|
1306
|
+
assert_eq!(actual, bytes);
|
|
1307
|
+
}
|
|
1308
|
+
drop(snapshot);
|
|
1309
|
+
assert!(!temp_path.exists());
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
#[test]
|
|
1313
|
+
fn cancellation_at_capture_stages_stops_within_the_declared_read_boundary() {
|
|
1314
|
+
let preflight_token = CancellationToken::new();
|
|
1315
|
+
let token_for_hook = preflight_token.clone();
|
|
1316
|
+
let hook = Arc::new(move |stage| {
|
|
1317
|
+
if stage == TestStage::CapturePreflightRead {
|
|
1318
|
+
token_for_hook.cancel();
|
|
1319
|
+
}
|
|
1320
|
+
});
|
|
1321
|
+
let (_guard, operation) =
|
|
1322
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(11), preflight_token, hook);
|
|
1323
|
+
let mut reader = ChunkedReader::new(vec![b'a'; PREFLIGHT_BYTES + 1], usize::MAX);
|
|
1324
|
+
assert!(matches!(
|
|
1325
|
+
capture_reader(&mut reader, None, None, Some(&operation)),
|
|
1326
|
+
Err(CaptureFailure::Cancelled)
|
|
1327
|
+
));
|
|
1328
|
+
assert_eq!(reader.position, 0);
|
|
1329
|
+
|
|
1330
|
+
let stream_token = CancellationToken::new();
|
|
1331
|
+
let token_for_hook = stream_token.clone();
|
|
1332
|
+
let hook = Arc::new(move |stage| {
|
|
1333
|
+
if stage == TestStage::SnapshotChunk {
|
|
1334
|
+
token_for_hook.cancel();
|
|
1335
|
+
}
|
|
1336
|
+
});
|
|
1337
|
+
let (_guard, operation) =
|
|
1338
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(12), stream_token, hook);
|
|
1339
|
+
let mut reader = ChunkedReader::new(vec![b'a'; PREFLIGHT_BYTES + 1], usize::MAX);
|
|
1340
|
+
assert!(matches!(
|
|
1341
|
+
capture_reader(&mut reader, None, None, Some(&operation)),
|
|
1342
|
+
Err(CaptureFailure::Cancelled)
|
|
1343
|
+
));
|
|
1344
|
+
assert_eq!(reader.position, PREFLIGHT_BYTES);
|
|
1345
|
+
|
|
1346
|
+
let promote_token = CancellationToken::new();
|
|
1347
|
+
let token_for_hook = promote_token.clone();
|
|
1348
|
+
let promotions = Arc::new(AtomicUsize::new(0));
|
|
1349
|
+
let promotions_for_hook = Arc::clone(&promotions);
|
|
1350
|
+
let hook = Arc::new(move |stage| {
|
|
1351
|
+
if stage == TestStage::SnapshotPromote {
|
|
1352
|
+
promotions_for_hook.fetch_add(1, Ordering::AcqRel);
|
|
1353
|
+
token_for_hook.cancel();
|
|
1354
|
+
}
|
|
1355
|
+
});
|
|
1356
|
+
let (_guard, operation) =
|
|
1357
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(13), promote_token, hook);
|
|
1358
|
+
let position = Arc::new(AtomicUsize::new(0));
|
|
1359
|
+
let mut reader = ChunkedReader::new(vec![b'a'; MEMORY_SNAPSHOT_LIMIT + 1], usize::MAX)
|
|
1360
|
+
.with_position_counter(Arc::clone(&position));
|
|
1361
|
+
assert!(matches!(
|
|
1362
|
+
capture_reader(&mut reader, None, None, Some(&operation)),
|
|
1363
|
+
Err(CaptureFailure::Cancelled)
|
|
1364
|
+
));
|
|
1365
|
+
assert_eq!(promotions.load(Ordering::Acquire), 1);
|
|
1366
|
+
assert!(position.load(Ordering::Acquire) <= MEMORY_SNAPSHOT_LIMIT + SNAPSHOT_CHUNK_BYTES);
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
#[test]
|
|
1370
|
+
fn sealed_temp_snapshot_survives_original_removal_for_all_later_passes() {
|
|
1371
|
+
let temp = tempfile::tempdir().unwrap();
|
|
1372
|
+
let path = temp.path().join("large-searchable.txt");
|
|
1373
|
+
let mut bytes = vec![b'a'; MEMORY_SNAPSHOT_LIMIT + 1];
|
|
1374
|
+
bytes.extend_from_slice(b"\nneedle\n");
|
|
1375
|
+
std::fs::write(&path, &bytes).unwrap();
|
|
1376
|
+
let metadata = std::fs::metadata(&path).unwrap();
|
|
1377
|
+
let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
|
|
1378
|
+
let CaptureDisposition::Searchable(snapshot) =
|
|
1379
|
+
capture_classify(&candidate, None, None, None).unwrap()
|
|
1380
|
+
else {
|
|
1381
|
+
panic!("expected searchable snapshot");
|
|
1382
|
+
};
|
|
1383
|
+
let temp_path = snapshot.captured.temp_path().unwrap().to_path_buf();
|
|
1384
|
+
std::fs::remove_file(&path).unwrap();
|
|
1385
|
+
|
|
1386
|
+
let source = ByteSource::Snapshot(&snapshot);
|
|
1387
|
+
let EncodingDecision::Text(validated) = validate_source_encoding(source, None).unwrap()
|
|
1388
|
+
else {
|
|
1389
|
+
panic!("expected trusted UTF-8 snapshot");
|
|
1390
|
+
};
|
|
1391
|
+
let mut reader = validated.open_source_reader(source).unwrap();
|
|
1392
|
+
let mut actual = Vec::new();
|
|
1393
|
+
reader.read_to_end(&mut actual).unwrap();
|
|
1394
|
+
assert_eq!(actual, bytes);
|
|
1395
|
+
assert!(temp_path.exists());
|
|
1396
|
+
drop(reader);
|
|
1397
|
+
let shared = snapshot.shared_range(0).unwrap();
|
|
1398
|
+
drop(snapshot);
|
|
1399
|
+
assert!(temp_path.exists());
|
|
1400
|
+
let mut shared_reader = shared.open_reader().unwrap();
|
|
1401
|
+
let mut shared_bytes = Vec::new();
|
|
1402
|
+
shared_reader.read_to_end(&mut shared_bytes).unwrap();
|
|
1403
|
+
assert_eq!(shared_bytes, bytes);
|
|
1404
|
+
drop(shared_reader);
|
|
1405
|
+
assert_eq!(
|
|
1406
|
+
shared
|
|
1407
|
+
.read_range((bytes.len() - 7) as u64..bytes.len() as u64)
|
|
1408
|
+
.unwrap(),
|
|
1409
|
+
b"needle\n"
|
|
1410
|
+
);
|
|
1411
|
+
drop(shared);
|
|
1412
|
+
assert!(!temp_path.exists());
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
#[test]
|
|
1416
|
+
fn traversal_mismatch_and_post_capture_mutation_are_file_changed() {
|
|
1417
|
+
let temp = tempfile::tempdir().unwrap();
|
|
1418
|
+
let path = temp.path().join("changing.txt");
|
|
1419
|
+
std::fs::write(&path, b"before").unwrap();
|
|
1420
|
+
let metadata = std::fs::metadata(&path).unwrap();
|
|
1421
|
+
let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
|
|
1422
|
+
std::fs::write(&path, b"different length").unwrap();
|
|
1423
|
+
assert!(matches!(
|
|
1424
|
+
capture_classify(&candidate, None, None, None).unwrap(),
|
|
1425
|
+
CaptureDisposition::FileChanged
|
|
1426
|
+
));
|
|
1427
|
+
|
|
1428
|
+
std::fs::write(&path, b"same-one").unwrap();
|
|
1429
|
+
set_file_mtime(&path, FileTime::from_unix_time(1_700_000_000, 0)).unwrap();
|
|
1430
|
+
let metadata = std::fs::metadata(&path).unwrap();
|
|
1431
|
+
let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
|
|
1432
|
+
std::fs::write(&path, b"same-two").unwrap();
|
|
1433
|
+
set_file_mtime(&path, FileTime::from_unix_time(1_700_000_010, 0)).unwrap();
|
|
1434
|
+
assert!(matches!(
|
|
1435
|
+
capture_classify(&candidate, None, None, None).unwrap(),
|
|
1436
|
+
CaptureDisposition::FileChanged
|
|
1437
|
+
));
|
|
1438
|
+
|
|
1439
|
+
std::fs::write(&path, b"stable bytes").unwrap();
|
|
1440
|
+
let metadata = std::fs::metadata(&path).unwrap();
|
|
1441
|
+
let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
|
|
1442
|
+
let changed = Arc::new(AtomicBool::new(false));
|
|
1443
|
+
let changed_for_hook = Arc::clone(&changed);
|
|
1444
|
+
let path_for_hook = path.clone();
|
|
1445
|
+
let hook = Arc::new(move |stage| {
|
|
1446
|
+
if stage == TestStage::BeforeIdentityPostCheck
|
|
1447
|
+
&& !changed_for_hook.swap(true, Ordering::AcqRel)
|
|
1448
|
+
{
|
|
1449
|
+
let mut file = std::fs::OpenOptions::new()
|
|
1450
|
+
.write(true)
|
|
1451
|
+
.truncate(true)
|
|
1452
|
+
.open(&path_for_hook)
|
|
1453
|
+
.unwrap();
|
|
1454
|
+
file.write_all(b"mutated to another length").unwrap();
|
|
1455
|
+
file.flush().unwrap();
|
|
1456
|
+
}
|
|
1457
|
+
});
|
|
1458
|
+
let (mut guard, operation) =
|
|
1459
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(1), CancellationToken::new(), hook);
|
|
1460
|
+
assert!(matches!(
|
|
1461
|
+
capture_classify(&candidate, None, None, Some(&operation)).unwrap(),
|
|
1462
|
+
CaptureDisposition::FileChanged
|
|
1463
|
+
));
|
|
1464
|
+
guard.disarm();
|
|
1465
|
+
assert!(changed.load(Ordering::Acquire));
|
|
1466
|
+
|
|
1467
|
+
std::fs::write(&path, b"stable replacement bytes").unwrap();
|
|
1468
|
+
let metadata = std::fs::metadata(&path).unwrap();
|
|
1469
|
+
let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
|
|
1470
|
+
let replaced = Arc::new(AtomicBool::new(false));
|
|
1471
|
+
let replaced_for_hook = Arc::clone(&replaced);
|
|
1472
|
+
let path_for_hook = path.clone();
|
|
1473
|
+
let old_path = temp.path().join("captured-old.txt");
|
|
1474
|
+
let hook = Arc::new(move |stage| {
|
|
1475
|
+
if stage == TestStage::BeforeIdentityPostCheck
|
|
1476
|
+
&& !replaced_for_hook.swap(true, Ordering::AcqRel)
|
|
1477
|
+
{
|
|
1478
|
+
std::fs::rename(&path_for_hook, &old_path).unwrap();
|
|
1479
|
+
std::fs::write(&path_for_hook, b"stable replacement bytes").unwrap();
|
|
1480
|
+
}
|
|
1481
|
+
});
|
|
1482
|
+
let (mut guard, operation) =
|
|
1483
|
+
RequestWorkGuard::new_with_hook(RequestId::Number(2), CancellationToken::new(), hook);
|
|
1484
|
+
assert!(matches!(
|
|
1485
|
+
capture_classify(&candidate, None, None, Some(&operation)).unwrap(),
|
|
1486
|
+
CaptureDisposition::FileChanged
|
|
1487
|
+
));
|
|
1488
|
+
guard.disarm();
|
|
1489
|
+
assert!(replaced.load(Ordering::Acquire));
|
|
1490
|
+
}
|
|
1491
|
+
}
|