dsh-ops 0.0.0-stage → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +202 -0
  2. package/LICENSE +30 -0
  3. package/NOTICE +106 -0
  4. package/PROVENANCE.md +435 -0
  5. package/README.en.md +126 -0
  6. package/README.md +115 -2
  7. package/README.zh.md +116 -0
  8. package/bin/dsh-ops.mjs +1216 -0
  9. package/cordis.patch.yml +160 -0
  10. package/docs/manual-validation.md +53 -0
  11. package/docs/release-0.2.1.md +72 -0
  12. package/docs/schema-baseline.json +64 -0
  13. package/docs/schema-current.json +84 -0
  14. package/docs/schema-measurement.md +17 -0
  15. package/dsh-plugin.json +88 -0
  16. package/icon.svg +12 -0
  17. package/lib/binary.js +409 -0
  18. package/lib/config.js +198 -0
  19. package/lib/handshake.js +252 -0
  20. package/lib/index.js +108 -0
  21. package/lib/jobs.js +42 -0
  22. package/lib/policy.js +64 -0
  23. package/lib/presentation.js +63 -0
  24. package/lib/profile-install.js +61 -0
  25. package/lib/rust.js +194 -0
  26. package/lib/session-shells.js +78 -0
  27. package/lib/shells.js +998 -0
  28. package/lib/tools.js +657 -0
  29. package/locale/en.json +6 -0
  30. package/locale/zh.json +6 -0
  31. package/package.json +114 -4
  32. package/vendor/fastctx/Cargo.lock +3210 -0
  33. package/vendor/fastctx/Cargo.toml +94 -0
  34. package/vendor/fastctx/FORK.md +119 -0
  35. package/vendor/fastctx/LICENSE-APACHE +201 -0
  36. package/vendor/fastctx/NOTICE +40 -0
  37. package/vendor/fastctx/README.md +439 -0
  38. package/vendor/fastctx/THIRD_PARTY_LICENSES.md +17 -0
  39. package/vendor/fastctx/THIRD_PARTY_LICENSES_RUST.md +7914 -0
  40. package/vendor/fastctx/UPSTREAM.md +49 -0
  41. package/vendor/fastctx/build.rs +413 -0
  42. package/vendor/fastctx/src/background_status.rs +403 -0
  43. package/vendor/fastctx/src/binary.rs +75 -0
  44. package/vendor/fastctx/src/bounded_sort.rs +500 -0
  45. package/vendor/fastctx/src/budget.rs +781 -0
  46. package/vendor/fastctx/src/cli/mod.rs +110 -0
  47. package/vendor/fastctx/src/context_guard.rs +289 -0
  48. package/vendor/fastctx/src/control/mod.rs +6 -0
  49. package/vendor/fastctx/src/control/paths.rs +49 -0
  50. package/vendor/fastctx/src/control/settings.rs +753 -0
  51. package/vendor/fastctx/src/control/transaction.rs +531 -0
  52. package/vendor/fastctx/src/edit/document.rs +535 -0
  53. package/vendor/fastctx/src/edit/locks.rs +371 -0
  54. package/vendor/fastctx/src/edit/mod.rs +213 -0
  55. package/vendor/fastctx/src/edit/private_storage/unix.rs +315 -0
  56. package/vendor/fastctx/src/edit/private_storage/windows.rs +793 -0
  57. package/vendor/fastctx/src/edit/private_storage.rs +234 -0
  58. package/vendor/fastctx/src/edit/replace.rs +1030 -0
  59. package/vendor/fastctx/src/edit_server.rs +53 -0
  60. package/vendor/fastctx/src/encoding/reference_v011.rs +587 -0
  61. package/vendor/fastctx/src/encoding/snapshot_pipeline.rs +1678 -0
  62. package/vendor/fastctx/src/encoding.rs +1118 -0
  63. package/vendor/fastctx/src/file_executor.rs +1151 -0
  64. package/vendor/fastctx/src/file_snapshot.rs +1491 -0
  65. package/vendor/fastctx/src/glob_filter.rs +98 -0
  66. package/vendor/fastctx/src/glob_tool.rs +653 -0
  67. package/vendor/fastctx/src/grep_sink.rs +1162 -0
  68. package/vendor/fastctx/src/grep_tool.rs +2449 -0
  69. package/vendor/fastctx/src/lib.rs +45 -0
  70. package/vendor/fastctx/src/main.rs +15 -0
  71. package/vendor/fastctx/src/model.rs +51 -0
  72. package/vendor/fastctx/src/model_guidance.rs +62 -0
  73. package/vendor/fastctx/src/operation.rs +356 -0
  74. package/vendor/fastctx/src/ordered_window.rs +1235 -0
  75. package/vendor/fastctx/src/os_environment.rs +414 -0
  76. package/vendor/fastctx/src/path_codec.rs +850 -0
  77. package/vendor/fastctx/src/paths.rs +244 -0
  78. package/vendor/fastctx/src/process_identity.rs +763 -0
  79. package/vendor/fastctx/src/process_policy.rs +74 -0
  80. package/vendor/fastctx/src/read_tool/batch.rs +496 -0
  81. package/vendor/fastctx/src/read_tool/hex_file.rs +141 -0
  82. package/vendor/fastctx/src/read_tool/image_file.rs +88 -0
  83. package/vendor/fastctx/src/read_tool/mod.rs +245 -0
  84. package/vendor/fastctx/src/read_tool/pdf.rs +470 -0
  85. package/vendor/fastctx/src/read_tool/pdf_disabled.rs +47 -0
  86. package/vendor/fastctx/src/read_tool/pdf_engine.rs +664 -0
  87. package/vendor/fastctx/src/read_tool/text_file.rs +351 -0
  88. package/vendor/fastctx/src/render_plan.rs +468 -0
  89. package/vendor/fastctx/src/runtime/activity.rs +159 -0
  90. package/vendor/fastctx/src/runtime/hosts.rs +99 -0
  91. package/vendor/fastctx/src/runtime/journal.rs +556 -0
  92. package/vendor/fastctx/src/runtime/local_ipc.rs +186 -0
  93. package/vendor/fastctx/src/runtime/mod.rs +746 -0
  94. package/vendor/fastctx/src/runtime/protocol.rs +296 -0
  95. package/vendor/fastctx/src/runtime/session.rs +536 -0
  96. package/vendor/fastctx/src/runtime/windows_process.rs +66 -0
  97. package/vendor/fastctx/src/search_parallelism.rs +106 -0
  98. package/vendor/fastctx/src/search_text.rs +227 -0
  99. package/vendor/fastctx/src/server.rs +359 -0
  100. package/vendor/fastctx/src/server_manifest.rs +468 -0
  101. package/vendor/fastctx/src/server_support.rs +826 -0
  102. package/vendor/fastctx/src/session.rs +629 -0
  103. package/vendor/fastctx/src/shell/apply_patch_hint.rs +41 -0
  104. package/vendor/fastctx/src/shell/bash.rs +263 -0
  105. package/vendor/fastctx/src/shell/buffer.rs +108 -0
  106. package/vendor/fastctx/src/shell/encoding.rs +403 -0
  107. package/vendor/fastctx/src/shell/foreground.rs +115 -0
  108. package/vendor/fastctx/src/shell/jobs/admission.rs +91 -0
  109. package/vendor/fastctx/src/shell/jobs/background.rs +146 -0
  110. package/vendor/fastctx/src/shell/jobs/host.rs +830 -0
  111. package/vendor/fastctx/src/shell/jobs/identity.rs +29 -0
  112. package/vendor/fastctx/src/shell/jobs/mod.rs +1513 -0
  113. package/vendor/fastctx/src/shell/jobs/model.rs +244 -0
  114. package/vendor/fastctx/src/shell/jobs/output_log.rs +1148 -0
  115. package/vendor/fastctx/src/shell/jobs/store.rs +1300 -0
  116. package/vendor/fastctx/src/shell/mod.rs +345 -0
  117. package/vendor/fastctx/src/shell/normalize.rs +389 -0
  118. package/vendor/fastctx/src/shell/output.rs +406 -0
  119. package/vendor/fastctx/src/shell/process.rs +493 -0
  120. package/vendor/fastctx/src/shell_server.rs +156 -0
  121. package/vendor/fastctx/src/skip_report.rs +83 -0
  122. package/vendor/fastctx/src/stdio_transport.rs +177 -0
  123. package/vendor/fastctx/src/tool_schema.rs +204 -0
  124. package/vendor/fastctx/src/traversal.rs +846 -0
  125. package/vendor/fastctx/third-party/pdfium-7763/LICENSE +9 -0
  126. package/vendor/fastctx/third-party/pdfium-7763/licenses/abseil.txt +202 -0
  127. package/vendor/fastctx/third-party/pdfium-7763/licenses/agg23.txt +14 -0
  128. package/vendor/fastctx/third-party/pdfium-7763/licenses/fast_float.txt +27 -0
  129. package/vendor/fastctx/third-party/pdfium-7763/licenses/freetype.txt +169 -0
  130. package/vendor/fastctx/third-party/pdfium-7763/licenses/icu.txt +542 -0
  131. package/vendor/fastctx/third-party/pdfium-7763/licenses/lcms.txt +27 -0
  132. package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.ijg +260 -0
  133. package/vendor/fastctx/third-party/pdfium-7763/licenses/libjpeg_turbo.md +135 -0
  134. package/vendor/fastctx/third-party/pdfium-7763/licenses/libopenjpeg.txt +32 -0
  135. package/vendor/fastctx/third-party/pdfium-7763/licenses/libpng.txt +134 -0
  136. package/vendor/fastctx/third-party/pdfium-7763/licenses/libtiff.txt +21 -0
  137. package/vendor/fastctx/third-party/pdfium-7763/licenses/llvm-libc.txt +278 -0
  138. package/vendor/fastctx/third-party/pdfium-7763/licenses/pdfium.txt +230 -0
  139. package/vendor/fastctx/third-party/pdfium-7763/licenses/simdutf.txt +18 -0
  140. package/vendor/fastctx/third-party/pdfium-7763/licenses/zlib.txt +29 -0
@@ -0,0 +1,1491 @@
1
+ //! Stable, single-open byte snapshots for grep candidates.
2
+
3
+ use crate::binary::detect_binary_type;
4
+ use crate::encoding::{
5
+ BomStreamingValidator, EncodingRejection, StrictStreamingValidator, Utf8ConflictProbe,
6
+ bom_streaming_validator, explicit_streaming_validator,
7
+ };
8
+ use crate::operation::{WorkCheckpoint, WorkStop};
9
+ use crate::path_codec::{FileIdentityHint, PathRecord};
10
+ use std::fs::File;
11
+ use std::io::{self, BufReader, Cursor, Read, Seek, SeekFrom, Write};
12
+ use std::path::Path;
13
+ use std::sync::Arc;
14
+
15
+ const PREFLIGHT_BYTES: usize = 8 * 1024;
16
+ const SNAPSHOT_CHUNK_BYTES: usize = 64 * 1024;
17
+ const MEMORY_SNAPSHOT_LIMIT: usize = 8 * 1024 * 1024;
18
+
19
+ /// A suffix-invariant reason that permits capture to stop before EOF.
20
+ #[derive(Clone, Debug, Eq, PartialEq)]
21
+ pub(crate) enum TerminalProof {
22
+ NulWithinFrozenProbe,
23
+ BinaryMagicAfterUtf8Failure,
24
+ ExplicitDecoderMalformed {
25
+ encoding: String,
26
+ },
27
+ BomDecoderMalformed {
28
+ encoding: &'static str,
29
+ },
30
+ AutomaticRejected {
31
+ conflict_hex_offset: usize,
32
+ fallback: FallbackTerminalState,
33
+ },
34
+ }
35
+
36
+ /// Why no fallback decoder can overturn an automatic mixed-encoding proof.
37
+ #[derive(Clone, Copy, Debug, Eq, PartialEq)]
38
+ pub(crate) enum FallbackTerminalState {
39
+ Absent,
40
+ PermanentlyMalformed,
41
+ }
42
+
43
+ impl TerminalProof {
44
+ pub(crate) fn rejection(&self) -> Option<EncodingRejection> {
45
+ match self {
46
+ Self::NulWithinFrozenProbe | Self::BinaryMagicAfterUtf8Failure => None,
47
+ Self::ExplicitDecoderMalformed { encoding } => {
48
+ Some(EncodingRejection::ExplicitMalformed {
49
+ encoding: encoding.clone(),
50
+ })
51
+ }
52
+ Self::BomDecoderMalformed { encoding } => {
53
+ Some(EncodingRejection::BomMismatch { encoding })
54
+ }
55
+ Self::AutomaticRejected {
56
+ conflict_hex_offset,
57
+ ..
58
+ } => Some(EncodingRejection::MixedOrInconsistent {
59
+ conflict_hex_offset: Some(*conflict_hex_offset),
60
+ }),
61
+ }
62
+ }
63
+ }
64
+
65
+ /// The stable result of capturing one candidate from its original handle.
66
+ pub(crate) enum CaptureDisposition {
67
+ Searchable(Box<SealedSnapshot>),
68
+ BinarySkipped(TerminalProof),
69
+ EncodingRejected {
70
+ rejection: EncodingRejection,
71
+ proof: TerminalProof,
72
+ },
73
+ FileChanged,
74
+ }
75
+
76
+ /// Failures that must remain distinct from an actively changing candidate.
77
+ #[derive(Debug)]
78
+ pub(crate) enum CaptureFailure {
79
+ Cancelled,
80
+ EpochRetired,
81
+ InvalidEncoding(EncodingRejection),
82
+ Io(io::Error),
83
+ Snapshot(io::Error),
84
+ }
85
+
86
+ /// A reader over one immutable snapshot backing.
87
+ pub(crate) enum SnapshotReader<'a> {
88
+ Memory(Cursor<&'a [u8]>),
89
+ Temp(BufReader<File>),
90
+ }
91
+
92
+ impl Read for SnapshotReader<'_> {
93
+ fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
94
+ match self {
95
+ Self::Memory(reader) => reader.read(output),
96
+ Self::Temp(reader) => reader.read(output),
97
+ }
98
+ }
99
+ }
100
+
101
+ #[derive(Debug)]
102
+ struct TempSnapshot {
103
+ file: tempfile::NamedTempFile,
104
+ }
105
+
106
+ impl TempSnapshot {
107
+ fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'static>> {
108
+ let mut file = self.file.reopen()?;
109
+ if start > 0 {
110
+ file.seek(SeekFrom::Start(start))?;
111
+ }
112
+ Ok(SnapshotReader::Temp(BufReader::new(file)))
113
+ }
114
+ }
115
+
116
+ #[derive(Clone, Debug)]
117
+ enum SnapshotBacking {
118
+ Memory(Arc<[u8]>),
119
+ Temp(Arc<TempSnapshot>),
120
+ }
121
+
122
+ struct CapturedSnapshot {
123
+ len: u64,
124
+ backing: SnapshotBacking,
125
+ }
126
+
127
+ /// A cloneable immutable view used when decoded UTF-8 is the snapshot bytes themselves.
128
+ #[derive(Clone, Debug)]
129
+ pub(crate) struct SnapshotByteRange {
130
+ backing: SnapshotBacking,
131
+ start: u64,
132
+ len: u64,
133
+ }
134
+
135
+ impl SnapshotByteRange {
136
+ pub(crate) fn len(&self) -> u64 {
137
+ self.len
138
+ }
139
+
140
+ pub(crate) fn memory_bytes(&self) -> Option<&[u8]> {
141
+ let SnapshotBacking::Memory(bytes) = &self.backing else {
142
+ return None;
143
+ };
144
+ let start = usize::try_from(self.start).ok()?;
145
+ let length = usize::try_from(self.len).ok()?;
146
+ let end = start.checked_add(length)?;
147
+ bytes.get(start..end)
148
+ }
149
+
150
+ pub(crate) fn open_reader(&self) -> io::Result<SnapshotReader<'_>> {
151
+ match &self.backing {
152
+ SnapshotBacking::Memory(bytes) => {
153
+ let start = usize::try_from(self.start).map_err(|_| snapshot_range_error())?;
154
+ let length = usize::try_from(self.len).map_err(|_| snapshot_range_error())?;
155
+ let end = start.checked_add(length).ok_or_else(snapshot_range_error)?;
156
+ let bytes = bytes.get(start..end).ok_or_else(snapshot_range_error)?;
157
+ Ok(SnapshotReader::Memory(Cursor::new(bytes)))
158
+ }
159
+ SnapshotBacking::Temp(snapshot) => snapshot.open_reader(self.start),
160
+ }
161
+ }
162
+
163
+ pub(crate) fn read_range(&self, range: std::ops::Range<u64>) -> io::Result<Vec<u8>> {
164
+ if range.start > range.end || range.end > self.len {
165
+ return Err(io::Error::new(
166
+ io::ErrorKind::InvalidData,
167
+ "captured text range lies outside its snapshot backing",
168
+ ));
169
+ }
170
+ let length = usize::try_from(range.end - range.start).map_err(|_| {
171
+ io::Error::new(
172
+ io::ErrorKind::InvalidData,
173
+ "captured text range does not fit in memory",
174
+ )
175
+ })?;
176
+ let absolute_start = self.start.checked_add(range.start).ok_or_else(|| {
177
+ io::Error::new(io::ErrorKind::InvalidData, "snapshot offset overflow")
178
+ })?;
179
+ let mut reader = match &self.backing {
180
+ SnapshotBacking::Memory(bytes) => {
181
+ let start = usize::try_from(absolute_start).map_err(|_| {
182
+ io::Error::new(io::ErrorKind::InvalidData, "snapshot offset overflow")
183
+ })?;
184
+ let end = start.checked_add(length).ok_or_else(snapshot_range_error)?;
185
+ let bytes = bytes.get(start..end).ok_or_else(snapshot_range_error)?;
186
+ return Ok(bytes.to_vec());
187
+ }
188
+ SnapshotBacking::Temp(snapshot) => snapshot.open_reader(absolute_start)?,
189
+ };
190
+ let mut bytes = vec![0_u8; length];
191
+ reader.read_exact(&mut bytes)?;
192
+ Ok(bytes)
193
+ }
194
+ }
195
+
196
+ fn snapshot_range_error() -> io::Error {
197
+ io::Error::new(
198
+ io::ErrorKind::InvalidData,
199
+ "captured text range lies outside its snapshot backing",
200
+ )
201
+ }
202
+
203
+ impl CapturedSnapshot {
204
+ fn memory_bytes(&self) -> Option<&[u8]> {
205
+ match &self.backing {
206
+ SnapshotBacking::Memory(bytes) => Some(bytes),
207
+ SnapshotBacking::Temp(_) => None,
208
+ }
209
+ }
210
+
211
+ fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'_>> {
212
+ match &self.backing {
213
+ SnapshotBacking::Memory(bytes) => {
214
+ let start = usize::try_from(start)
215
+ .unwrap_or(usize::MAX)
216
+ .min(bytes.len());
217
+ Ok(SnapshotReader::Memory(Cursor::new(&bytes[start..])))
218
+ }
219
+ SnapshotBacking::Temp(snapshot) => snapshot.open_reader(start),
220
+ }
221
+ }
222
+
223
+ fn shared_range(&self, start: u64) -> io::Result<SnapshotByteRange> {
224
+ if start > self.len {
225
+ return Err(io::Error::new(
226
+ io::ErrorKind::InvalidData,
227
+ "decoded text starts beyond the captured snapshot",
228
+ ));
229
+ }
230
+ Ok(SnapshotByteRange {
231
+ backing: self.backing.clone(),
232
+ start,
233
+ len: self.len - start,
234
+ })
235
+ }
236
+
237
+ #[cfg(test)]
238
+ fn temp_path(&self) -> Option<&Path> {
239
+ match &self.backing {
240
+ SnapshotBacking::Memory(_) => None,
241
+ SnapshotBacking::Temp(snapshot) => Some(snapshot.file.path()),
242
+ }
243
+ }
244
+ }
245
+
246
+ /// Immutable bytes used by every encoding pass and by the regex search.
247
+ pub(crate) struct SealedSnapshot {
248
+ path: PathRecord,
249
+ _before: FileIdentity,
250
+ _after: FileIdentity,
251
+ captured: CapturedSnapshot,
252
+ }
253
+
254
+ impl SealedSnapshot {
255
+ /// Returns the candidate whose original bytes produced this snapshot.
256
+ pub(crate) fn path(&self) -> &PathRecord {
257
+ &self.path
258
+ }
259
+
260
+ /// Returns the exact captured length, independent of traversal metadata.
261
+ pub(crate) fn len(&self) -> u64 {
262
+ self.captured.len
263
+ }
264
+
265
+ /// Borrows an in-memory backing without copying; temp-backed snapshots return `None`.
266
+ pub(crate) fn memory_bytes(&self) -> Option<&[u8]> {
267
+ self.captured.memory_bytes()
268
+ }
269
+
270
+ /// Borrows at most `maximum` leading bytes directly from an in-memory backing.
271
+ pub(crate) fn memory_prefix(&self, maximum: usize) -> Option<&[u8]> {
272
+ self.memory_bytes()
273
+ .map(|bytes| &bytes[..bytes.len().min(maximum)])
274
+ }
275
+
276
+ /// Opens an independent reader over the same immutable backing at `start`.
277
+ pub(crate) fn open_reader(&self, start: u64) -> io::Result<SnapshotReader<'_>> {
278
+ self.captured.open_reader(start)
279
+ }
280
+
281
+ /// Shares the immutable suffix beginning at `start` without reopening the original path.
282
+ pub(crate) fn shared_range(&self, start: u64) -> io::Result<SnapshotByteRange> {
283
+ self.captured.shared_range(start)
284
+ }
285
+ }
286
+
287
+ enum CaptureRead {
288
+ Terminal(TerminalProof),
289
+ Complete(CapturedSnapshot),
290
+ }
291
+
292
+ enum ClassifierMode {
293
+ Explicit {
294
+ encoding: String,
295
+ decoder: StrictStreamingValidator,
296
+ malformed: bool,
297
+ },
298
+ Bom {
299
+ decoder: StrictStreamingValidator,
300
+ encoding: &'static str,
301
+ is_utf8: bool,
302
+ malformed: bool,
303
+ },
304
+ Automatic {
305
+ utf8: StrictStreamingValidator,
306
+ utf8_malformed: bool,
307
+ utf8_conflict: Utf8ConflictProbe,
308
+ conflict_hex_offset: Option<usize>,
309
+ fallback: Option<StrictStreamingValidator>,
310
+ fallback_malformed: bool,
311
+ },
312
+ }
313
+
314
+ /// Streaming-only state whose terminal decisions cannot be overturned by unread bytes.
315
+ struct CaptureClassifier {
316
+ explicit_encoding: Option<String>,
317
+ fallback_encoding: Option<String>,
318
+ probe: Vec<u8>,
319
+ mode: Option<ClassifierMode>,
320
+ }
321
+
322
+ impl CaptureClassifier {
323
+ fn new(explicit_encoding: Option<&str>, fallback_encoding: Option<&str>) -> Self {
324
+ Self {
325
+ explicit_encoding: explicit_encoding.map(str::to_string),
326
+ fallback_encoding: fallback_encoding.map(str::to_string),
327
+ probe: Vec::with_capacity(PREFLIGHT_BYTES),
328
+ mode: None,
329
+ }
330
+ }
331
+
332
+ fn feed(&mut self, bytes: &[u8]) -> Result<Option<TerminalProof>, EncodingRejection> {
333
+ let remaining_probe = PREFLIGHT_BYTES.saturating_sub(self.probe.len());
334
+ self.probe
335
+ .extend_from_slice(&bytes[..bytes.len().min(remaining_probe)]);
336
+
337
+ if self.mode.is_none() {
338
+ // Four bytes settle every BOM that can outrank the automatic NUL gate.
339
+ if self.probe.len() < 4 {
340
+ return Ok(None);
341
+ }
342
+ debug_assert!(bytes.len() <= PREFLIGHT_BYTES);
343
+ self.initialize()?;
344
+ } else {
345
+ self.feed_initialized(bytes);
346
+ }
347
+ Ok(self.terminal_proof())
348
+ }
349
+
350
+ fn initialize(&mut self) -> Result<(), EncodingRejection> {
351
+ debug_assert!(self.mode.is_none());
352
+ if let Some(encoding) = self.explicit_encoding.clone() {
353
+ let (mut decoder, bom_len) = explicit_streaming_validator(&encoding, &self.probe)?;
354
+ let malformed = decoder.feed(&self.probe[bom_len..]);
355
+ self.mode = Some(ClassifierMode::Explicit {
356
+ encoding,
357
+ decoder,
358
+ malformed,
359
+ });
360
+ return Ok(());
361
+ }
362
+
363
+ if let Some(BomStreamingValidator {
364
+ mut validator,
365
+ bom_len,
366
+ encoding,
367
+ is_utf8,
368
+ }) = bom_streaming_validator(&self.probe)
369
+ {
370
+ let malformed = validator.feed(&self.probe[bom_len..]);
371
+ self.mode = Some(ClassifierMode::Bom {
372
+ decoder: validator,
373
+ encoding,
374
+ is_utf8,
375
+ malformed,
376
+ });
377
+ return Ok(());
378
+ }
379
+
380
+ let (mut utf8, bom_len) = explicit_streaming_validator("utf-8", &self.probe)
381
+ .expect("the built-in UTF-8 label is valid");
382
+ debug_assert_eq!(bom_len, 0);
383
+ let utf8_malformed = utf8.feed(&self.probe);
384
+ let mut utf8_conflict = Utf8ConflictProbe::new();
385
+ let conflict_hex_offset = utf8_conflict.push(&self.probe);
386
+ let (fallback, fallback_malformed) =
387
+ if let Some(fallback_encoding) = self.fallback_encoding.as_deref() {
388
+ let (mut fallback, bom_len) =
389
+ explicit_streaming_validator(fallback_encoding, &self.probe)?;
390
+ debug_assert_eq!(bom_len, 0, "automatic mode has no BOM to strip");
391
+ let malformed = fallback.feed(&self.probe);
392
+ (Some(fallback), malformed)
393
+ } else {
394
+ (None, false)
395
+ };
396
+ self.mode = Some(ClassifierMode::Automatic {
397
+ utf8,
398
+ utf8_malformed,
399
+ utf8_conflict,
400
+ conflict_hex_offset,
401
+ fallback,
402
+ fallback_malformed,
403
+ });
404
+ Ok(())
405
+ }
406
+
407
+ fn feed_initialized(&mut self, bytes: &[u8]) {
408
+ match self.mode.as_mut().expect("classifier mode is initialized") {
409
+ ClassifierMode::Explicit {
410
+ decoder, malformed, ..
411
+ }
412
+ | ClassifierMode::Bom {
413
+ decoder, malformed, ..
414
+ } => {
415
+ *malformed |= decoder.feed(bytes);
416
+ }
417
+ ClassifierMode::Automatic {
418
+ utf8,
419
+ utf8_malformed,
420
+ utf8_conflict,
421
+ conflict_hex_offset,
422
+ fallback,
423
+ fallback_malformed,
424
+ } => {
425
+ *utf8_malformed |= utf8.feed(bytes);
426
+ if conflict_hex_offset.is_none() {
427
+ *conflict_hex_offset = utf8_conflict.push(bytes);
428
+ }
429
+ if let Some(fallback) = fallback {
430
+ *fallback_malformed |= fallback.feed(bytes);
431
+ }
432
+ }
433
+ }
434
+ }
435
+
436
+ fn finish_eof(&self) -> Option<TerminalProof> {
437
+ // EOF-only incomplete sequences are deliberately non-terminal: an appended
438
+ // suffix could complete them, so the sealed full-file validator decides them.
439
+ self.terminal_proof()
440
+ }
441
+
442
+ fn terminal_proof(&self) -> Option<TerminalProof> {
443
+ let mode = self.mode.as_ref()?;
444
+ match mode {
445
+ ClassifierMode::Explicit {
446
+ encoding,
447
+ malformed,
448
+ ..
449
+ } => malformed.then(|| TerminalProof::ExplicitDecoderMalformed {
450
+ encoding: encoding.clone(),
451
+ }),
452
+ ClassifierMode::Bom {
453
+ encoding,
454
+ is_utf8,
455
+ malformed,
456
+ ..
457
+ } => {
458
+ if *is_utf8 && self.probe.contains(&0) {
459
+ return Some(TerminalProof::NulWithinFrozenProbe);
460
+ }
461
+ (*malformed && (!*is_utf8 || self.probe_is_settled()))
462
+ .then_some(TerminalProof::BomDecoderMalformed { encoding })
463
+ }
464
+ ClassifierMode::Automatic {
465
+ utf8_malformed,
466
+ conflict_hex_offset,
467
+ fallback,
468
+ fallback_malformed,
469
+ ..
470
+ } => {
471
+ if self.probe.contains(&0) {
472
+ return Some(TerminalProof::NulWithinFrozenProbe);
473
+ }
474
+ if !self.probe_is_settled() {
475
+ return None;
476
+ }
477
+ if *utf8_malformed && detect_binary_type(&self.probe).is_some() {
478
+ return Some(TerminalProof::BinaryMagicAfterUtf8Failure);
479
+ }
480
+ let conflict_hex_offset = (*conflict_hex_offset)?;
481
+ let fallback = match fallback {
482
+ None => FallbackTerminalState::Absent,
483
+ Some(_) if *fallback_malformed => FallbackTerminalState::PermanentlyMalformed,
484
+ Some(_) => return None,
485
+ };
486
+ Some(TerminalProof::AutomaticRejected {
487
+ conflict_hex_offset,
488
+ fallback,
489
+ })
490
+ }
491
+ }
492
+ }
493
+
494
+ fn probe_is_settled(&self) -> bool {
495
+ self.probe.len() == PREFLIGHT_BYTES
496
+ }
497
+ }
498
+
499
+ enum SnapshotBuilder {
500
+ Memory(Vec<u8>),
501
+ Temp {
502
+ file: tempfile::NamedTempFile,
503
+ len: u64,
504
+ },
505
+ }
506
+
507
+ impl SnapshotBuilder {
508
+ fn new() -> Self {
509
+ Self::Memory(Vec::with_capacity(PREFLIGHT_BYTES))
510
+ }
511
+
512
+ fn append(
513
+ &mut self,
514
+ bytes: &[u8],
515
+ operation: Option<&dyn WorkCheckpoint>,
516
+ ) -> Result<(), CaptureFailure> {
517
+ let promote = matches!(
518
+ self,
519
+ Self::Memory(memory)
520
+ if memory.len().saturating_add(bytes.len()) > MEMORY_SNAPSHOT_LIMIT
521
+ );
522
+ if promote {
523
+ checkpoint(operation)?;
524
+ #[cfg(test)]
525
+ if let Some(operation) = operation {
526
+ operation.stage(crate::operation::TestStage::SnapshotPromote);
527
+ }
528
+ checkpoint(operation)?;
529
+
530
+ let memory = match std::mem::replace(self, Self::Memory(Vec::new())) {
531
+ Self::Memory(memory) => memory,
532
+ Self::Temp { .. } => unreachable!("promotion is only possible from memory"),
533
+ };
534
+ let mut file = match tempfile::NamedTempFile::new() {
535
+ Ok(file) => file,
536
+ Err(error) => {
537
+ return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
538
+ }
539
+ };
540
+ for chunk in memory.chunks(SNAPSHOT_CHUNK_BYTES) {
541
+ write_snapshot_chunk(&mut file, chunk, operation)?;
542
+ }
543
+ let len = memory.len() as u64;
544
+ *self = Self::Temp { file, len };
545
+ }
546
+
547
+ match self {
548
+ Self::Memory(memory) => memory.extend_from_slice(bytes),
549
+ Self::Temp { file, len } => {
550
+ write_snapshot_chunk(file, bytes, operation)?;
551
+ *len = len.saturating_add(bytes.len() as u64);
552
+ }
553
+ }
554
+ Ok(())
555
+ }
556
+
557
+ fn finish(
558
+ mut self,
559
+ operation: Option<&dyn WorkCheckpoint>,
560
+ ) -> Result<CapturedSnapshot, CaptureFailure> {
561
+ match &mut self {
562
+ Self::Memory(_) => {}
563
+ Self::Temp { file, .. } => {
564
+ checkpoint(operation)?;
565
+ if let Err(error) = file.as_file_mut().flush() {
566
+ return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
567
+ }
568
+ checkpoint(operation)?;
569
+ }
570
+ }
571
+ Ok(match self {
572
+ Self::Memory(memory) => CapturedSnapshot {
573
+ len: memory.len() as u64,
574
+ backing: SnapshotBacking::Memory(Arc::from(memory)),
575
+ },
576
+ Self::Temp { file, len } => CapturedSnapshot {
577
+ len,
578
+ backing: SnapshotBacking::Temp(Arc::new(TempSnapshot { file })),
579
+ },
580
+ })
581
+ }
582
+ }
583
+
584
+ fn write_snapshot_chunk(
585
+ file: &mut tempfile::NamedTempFile,
586
+ bytes: &[u8],
587
+ operation: Option<&dyn WorkCheckpoint>,
588
+ ) -> Result<(), CaptureFailure> {
589
+ debug_assert!(bytes.len() <= SNAPSHOT_CHUNK_BYTES);
590
+ checkpoint(operation)?;
591
+ if let Err(error) = file.as_file_mut().write_all(bytes) {
592
+ return Err(prefer_stop(operation, CaptureFailure::Snapshot(error)));
593
+ }
594
+ checkpoint(operation)
595
+ }
596
+
597
+ fn capture_reader(
598
+ reader: &mut impl Read,
599
+ explicit_encoding: Option<&str>,
600
+ fallback_encoding: Option<&str>,
601
+ operation: Option<&dyn WorkCheckpoint>,
602
+ ) -> Result<CaptureRead, CaptureFailure> {
603
+ let mut builder = SnapshotBuilder::new();
604
+ let mut classifier = CaptureClassifier::new(explicit_encoding, fallback_encoding);
605
+ let mut chunk = [0_u8; SNAPSHOT_CHUNK_BYTES];
606
+ let mut preflight_bytes = 0_usize;
607
+ let mut reached_eof = false;
608
+
609
+ while preflight_bytes < PREFLIGHT_BYTES {
610
+ checkpoint(operation)?;
611
+ #[cfg(test)]
612
+ if let Some(operation) = operation {
613
+ operation.stage(crate::operation::TestStage::CapturePreflightRead);
614
+ }
615
+ checkpoint(operation)?;
616
+ let maximum = PREFLIGHT_BYTES - preflight_bytes;
617
+ let count = match reader.read(&mut chunk[..maximum]) {
618
+ Ok(count) => count,
619
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
620
+ };
621
+ checkpoint(operation)?;
622
+ if count == 0 {
623
+ reached_eof = true;
624
+ break;
625
+ }
626
+ preflight_bytes += count;
627
+ let proof = match classifier.feed(&chunk[..count]) {
628
+ Ok(proof) => proof,
629
+ Err(rejection) => {
630
+ return Err(prefer_stop(
631
+ operation,
632
+ CaptureFailure::InvalidEncoding(rejection),
633
+ ));
634
+ }
635
+ };
636
+ checkpoint(operation)?;
637
+ if let Some(proof) = proof {
638
+ return Ok(CaptureRead::Terminal(proof));
639
+ }
640
+ builder.append(&chunk[..count], operation)?;
641
+ }
642
+
643
+ if !reached_eof {
644
+ loop {
645
+ checkpoint(operation)?;
646
+ #[cfg(test)]
647
+ if let Some(operation) = operation {
648
+ operation.stage(crate::operation::TestStage::SnapshotChunk);
649
+ }
650
+ checkpoint(operation)?;
651
+ let count = match reader.read(&mut chunk) {
652
+ Ok(count) => count,
653
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
654
+ };
655
+ checkpoint(operation)?;
656
+ if count == 0 {
657
+ break;
658
+ }
659
+ let proof = match classifier.feed(&chunk[..count]) {
660
+ Ok(proof) => proof,
661
+ Err(rejection) => {
662
+ return Err(prefer_stop(
663
+ operation,
664
+ CaptureFailure::InvalidEncoding(rejection),
665
+ ));
666
+ }
667
+ };
668
+ checkpoint(operation)?;
669
+ if let Some(proof) = proof {
670
+ return Ok(CaptureRead::Terminal(proof));
671
+ }
672
+ builder.append(&chunk[..count], operation)?;
673
+ }
674
+ }
675
+ let proof = classifier.finish_eof();
676
+ checkpoint(operation)?;
677
+ if let Some(proof) = proof {
678
+ return Ok(CaptureRead::Terminal(proof));
679
+ }
680
+ builder.finish(operation).map(CaptureRead::Complete)
681
+ }
682
+
683
+ fn checkpoint(operation: Option<&dyn WorkCheckpoint>) -> Result<(), CaptureFailure> {
684
+ match operation.map(WorkCheckpoint::check_work) {
685
+ Some(Err(WorkStop::RequestCancelled)) => Err(CaptureFailure::Cancelled),
686
+ Some(Err(WorkStop::EpochRetired)) => Err(CaptureFailure::EpochRetired),
687
+ Some(Ok(())) | None => Ok(()),
688
+ }
689
+ }
690
+
691
+ fn prefer_stop(operation: Option<&dyn WorkCheckpoint>, failure: CaptureFailure) -> CaptureFailure {
692
+ match checkpoint(operation) {
693
+ Ok(()) => failure,
694
+ Err(stop) => stop,
695
+ }
696
+ }
697
+
698
+ #[cfg(not(windows))]
699
+ fn open_original(path: &Path) -> io::Result<File> {
700
+ File::open(path)
701
+ }
702
+
703
+ #[cfg(windows)]
704
+ fn open_original(path: &Path) -> io::Result<File> {
705
+ use std::os::windows::fs::OpenOptionsExt;
706
+ use windows_sys::Win32::Storage::FileSystem::{
707
+ FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
708
+ };
709
+
710
+ // Replacement must remain observable while the capture handle is live.
711
+ std::fs::OpenOptions::new()
712
+ .read(true)
713
+ .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
714
+ .open(path)
715
+ }
716
+
717
+ /// Opens the original candidate once, captures its bytes, and verifies stable identity.
718
+ pub(crate) fn capture_classify(
719
+ candidate: &PathRecord,
720
+ explicit_encoding: Option<&str>,
721
+ fallback_encoding: Option<&str>,
722
+ operation: Option<&dyn WorkCheckpoint>,
723
+ ) -> Result<CaptureDisposition, CaptureFailure> {
724
+ checkpoint(operation)?;
725
+ let mut file = match open_original(&candidate.native) {
726
+ Ok(file) => file,
727
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {
728
+ checkpoint(operation)?;
729
+ return Ok(CaptureDisposition::FileChanged);
730
+ }
731
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
732
+ };
733
+ let (before, is_regular) = match identity_from_file(&file) {
734
+ Ok(identity) => identity,
735
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
736
+ };
737
+ checkpoint(operation)?;
738
+ if !is_regular || !before.matches_hint(candidate.traversal_fingerprint.as_ref()) {
739
+ return Ok(CaptureDisposition::FileChanged);
740
+ }
741
+
742
+ let capture = capture_reader(&mut file, explicit_encoding, fallback_encoding, operation)?;
743
+ checkpoint(operation)?;
744
+ #[cfg(test)]
745
+ if let Some(operation) = operation {
746
+ operation.stage(crate::operation::TestStage::BeforeIdentityPostCheck);
747
+ }
748
+ checkpoint(operation)?;
749
+
750
+ let (after, is_regular) = match identity_from_file(&file) {
751
+ Ok(identity) => identity,
752
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
753
+ };
754
+ let path_identity = match identity_from_path(&candidate.native) {
755
+ Ok(identity) => identity,
756
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {
757
+ checkpoint(operation)?;
758
+ return Ok(CaptureDisposition::FileChanged);
759
+ }
760
+ Err(error) => return Err(prefer_stop(operation, CaptureFailure::Io(error))),
761
+ };
762
+ checkpoint(operation)?;
763
+ if !is_regular
764
+ || !path_identity.1
765
+ || !before.same_state(&after)
766
+ || !after.same_state(&path_identity.0)
767
+ || !after.matches_hint(candidate.traversal_fingerprint.as_ref())
768
+ {
769
+ return Ok(CaptureDisposition::FileChanged);
770
+ }
771
+
772
+ Ok(match capture {
773
+ CaptureRead::Terminal(proof) => match proof.rejection() {
774
+ Some(rejection) => CaptureDisposition::EncodingRejected { rejection, proof },
775
+ None => CaptureDisposition::BinarySkipped(proof),
776
+ },
777
+ CaptureRead::Complete(captured) => {
778
+ CaptureDisposition::Searchable(Box::new(SealedSnapshot {
779
+ path: candidate.clone(),
780
+ _before: before,
781
+ _after: after,
782
+ captured,
783
+ }))
784
+ }
785
+ })
786
+ }
787
+
788
+ #[cfg(unix)]
789
+ #[derive(Clone, Debug, Eq, PartialEq)]
790
+ struct FileIdentity {
791
+ device: u64,
792
+ inode: u64,
793
+ len: u64,
794
+ modified_seconds: i64,
795
+ modified_nanoseconds: i64,
796
+ changed_seconds: i64,
797
+ changed_nanoseconds: i64,
798
+ }
799
+
800
+ #[cfg(unix)]
801
+ impl FileIdentity {
802
+ fn from_metadata(metadata: &std::fs::Metadata) -> Self {
803
+ use std::os::unix::fs::MetadataExt;
804
+ Self {
805
+ device: metadata.dev(),
806
+ inode: metadata.ino(),
807
+ len: metadata.len(),
808
+ modified_seconds: metadata.mtime(),
809
+ modified_nanoseconds: metadata.mtime_nsec(),
810
+ changed_seconds: metadata.ctime(),
811
+ changed_nanoseconds: metadata.ctime_nsec(),
812
+ }
813
+ }
814
+
815
+ fn matches_hint(&self, hint: Option<&FileIdentityHint>) -> bool {
816
+ hint.is_none_or(|hint| {
817
+ self.device == hint.device
818
+ && self.inode == hint.inode
819
+ && self.len == hint.len
820
+ && self.modified_seconds == hint.modified_seconds
821
+ && self.modified_nanoseconds == hint.modified_nanoseconds
822
+ && self.changed_seconds == hint.changed_seconds
823
+ && self.changed_nanoseconds == hint.changed_nanoseconds
824
+ })
825
+ }
826
+
827
+ fn same_state(&self, other: &Self) -> bool {
828
+ self == other
829
+ }
830
+ }
831
+
832
+ #[cfg(unix)]
833
+ fn identity_from_file(file: &File) -> io::Result<(FileIdentity, bool)> {
834
+ let metadata = file.metadata()?;
835
+ Ok((FileIdentity::from_metadata(&metadata), metadata.is_file()))
836
+ }
837
+
838
+ #[cfg(unix)]
839
+ fn identity_from_path(path: &Path) -> io::Result<(FileIdentity, bool)> {
840
+ let metadata = std::fs::metadata(path)?;
841
+ Ok((FileIdentity::from_metadata(&metadata), metadata.is_file()))
842
+ }
843
+
844
+ #[cfg(windows)]
845
+ #[derive(Clone, Debug, Eq, PartialEq)]
846
+ enum WindowsFileId {
847
+ Extended {
848
+ volume_serial: u64,
849
+ identifier: [u8; 16],
850
+ },
851
+ Legacy {
852
+ volume_serial: u32,
853
+ file_index: u64,
854
+ },
855
+ }
856
+
857
+ #[cfg(windows)]
858
+ #[derive(Clone, Debug, Eq, PartialEq)]
859
+ struct FileIdentity {
860
+ file_id: WindowsFileId,
861
+ len: u64,
862
+ creation_time: u64,
863
+ last_write_time: u64,
864
+ change_time: Option<i64>,
865
+ attributes: u32,
866
+ }
867
+
868
+ #[cfg(windows)]
869
+ impl FileIdentity {
870
+ fn from_file(file: &File) -> io::Result<Self> {
871
+ use std::mem::size_of;
872
+ use std::os::windows::io::AsRawHandle;
873
+ use windows_sys::Win32::Foundation::{ERROR_INVALID_PARAMETER, ERROR_NOT_SUPPORTED};
874
+ use windows_sys::Win32::Storage::FileSystem::{
875
+ BY_HANDLE_FILE_INFORMATION, FILE_BASIC_INFO, FILE_ID_INFO, FileBasicInfo, FileIdInfo,
876
+ GetFileInformationByHandle, GetFileInformationByHandleEx,
877
+ };
878
+
879
+ let handle = file.as_raw_handle();
880
+ let mut info = BY_HANDLE_FILE_INFORMATION::default();
881
+ let ok = unsafe { GetFileInformationByHandle(handle, &mut info) };
882
+ if ok == 0 {
883
+ return Err(io::Error::last_os_error());
884
+ }
885
+ let mut extended_id = FILE_ID_INFO::default();
886
+ let extended_id_ok = unsafe {
887
+ GetFileInformationByHandleEx(
888
+ handle,
889
+ FileIdInfo,
890
+ (&mut extended_id as *mut FILE_ID_INFO).cast(),
891
+ size_of::<FILE_ID_INFO>() as u32,
892
+ )
893
+ };
894
+ let file_id = if extended_id_ok != 0 {
895
+ WindowsFileId::Extended {
896
+ volume_serial: extended_id.VolumeSerialNumber,
897
+ identifier: extended_id.FileId.Identifier,
898
+ }
899
+ } else {
900
+ let error = io::Error::last_os_error();
901
+ match error.raw_os_error() {
902
+ Some(code)
903
+ if code == ERROR_NOT_SUPPORTED as i32
904
+ || code == ERROR_INVALID_PARAMETER as i32 =>
905
+ {
906
+ WindowsFileId::Legacy {
907
+ volume_serial: info.dwVolumeSerialNumber,
908
+ file_index: (u64::from(info.nFileIndexHigh) << 32)
909
+ | u64::from(info.nFileIndexLow),
910
+ }
911
+ }
912
+ _ => return Err(error),
913
+ }
914
+ };
915
+ let mut basic = FILE_BASIC_INFO::default();
916
+ let basic_ok = unsafe {
917
+ GetFileInformationByHandleEx(
918
+ handle,
919
+ FileBasicInfo,
920
+ (&mut basic as *mut FILE_BASIC_INFO).cast(),
921
+ size_of::<FILE_BASIC_INFO>() as u32,
922
+ )
923
+ };
924
+ Ok(Self {
925
+ file_id,
926
+ len: (u64::from(info.nFileSizeHigh) << 32) | u64::from(info.nFileSizeLow),
927
+ creation_time: filetime_u64(info.ftCreationTime),
928
+ last_write_time: filetime_u64(info.ftLastWriteTime),
929
+ change_time: (basic_ok != 0).then_some(basic.ChangeTime),
930
+ attributes: info.dwFileAttributes,
931
+ })
932
+ }
933
+
934
+ fn matches_hint(&self, hint: Option<&FileIdentityHint>) -> bool {
935
+ // Stable Windows file IDs are handle-only; traversal metadata can compare
936
+ // only these fields, while the before/after/path handle checks compare IDs.
937
+ hint.is_none_or(|hint| {
938
+ self.len == hint.len
939
+ && self.creation_time == hint.creation_time
940
+ && self.last_write_time == hint.last_write_time
941
+ && self.attributes == hint.attributes
942
+ })
943
+ }
944
+
945
+ fn same_state(&self, other: &Self) -> bool {
946
+ self.file_id == other.file_id
947
+ && self.len == other.len
948
+ && self.creation_time == other.creation_time
949
+ && self.last_write_time == other.last_write_time
950
+ && self.attributes == other.attributes
951
+ && match (self.change_time, other.change_time) {
952
+ (Some(left), Some(right)) => left == right,
953
+ _ => true,
954
+ }
955
+ }
956
+ }
957
+
958
+ #[cfg(windows)]
959
+ fn filetime_u64(value: windows_sys::Win32::Foundation::FILETIME) -> u64 {
960
+ (u64::from(value.dwHighDateTime) << 32) | u64::from(value.dwLowDateTime)
961
+ }
962
+
963
+ #[cfg(windows)]
964
+ fn identity_from_file(file: &File) -> io::Result<(FileIdentity, bool)> {
965
+ let metadata = file.metadata()?;
966
+ Ok((FileIdentity::from_file(file)?, metadata.is_file()))
967
+ }
968
+
969
+ #[cfg(windows)]
970
+ fn identity_from_path(path: &Path) -> io::Result<(FileIdentity, bool)> {
971
+ use std::os::windows::fs::OpenOptionsExt;
972
+ use windows_sys::Win32::Storage::FileSystem::{
973
+ FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
974
+ };
975
+
976
+ // Windows path metadata omits the stable file index, so this attributes-only
977
+ // handle is required to prove that the path still names the captured file.
978
+ let file = std::fs::OpenOptions::new()
979
+ .access_mode(FILE_READ_ATTRIBUTES)
980
+ .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
981
+ .open(path)?;
982
+ identity_from_file(&file)
983
+ }
984
+
985
+ #[cfg(test)]
986
+ mod tests {
987
+ use super::{
988
+ CaptureDisposition, CaptureFailure, CaptureRead, FallbackTerminalState,
989
+ MEMORY_SNAPSHOT_LIMIT, PREFLIGHT_BYTES, SNAPSHOT_CHUNK_BYTES, TerminalProof,
990
+ capture_classify, capture_reader,
991
+ };
992
+ use crate::encoding::{
993
+ ByteSource, EncodingDecision, EncodingRejection, validate_source_encoding,
994
+ };
995
+ use crate::operation::{RequestWorkGuard, TestStage};
996
+ use crate::path_codec::PathRecord;
997
+
998
+ use filetime::{FileTime, set_file_mtime};
999
+ use rmcp::model::RequestId;
1000
+ use std::io::{self, Read, Write};
1001
+ use std::sync::Arc;
1002
+
1003
+ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
1004
+ use tokio_util::sync::CancellationToken;
1005
+
1006
+ struct ChunkedReader {
1007
+ bytes: Vec<u8>,
1008
+ position: usize,
1009
+ maximum: usize,
1010
+ fail_at: Option<usize>,
1011
+ reads: usize,
1012
+ requested: Vec<usize>,
1013
+ position_counter: Option<Arc<AtomicUsize>>,
1014
+ read_limits: Option<Vec<usize>>,
1015
+ }
1016
+
1017
+ impl ChunkedReader {
1018
+ fn new(bytes: Vec<u8>, maximum: usize) -> Self {
1019
+ Self {
1020
+ bytes,
1021
+ position: 0,
1022
+ maximum,
1023
+ fail_at: None,
1024
+ reads: 0,
1025
+ requested: Vec::new(),
1026
+ position_counter: None,
1027
+ read_limits: None,
1028
+ }
1029
+ }
1030
+
1031
+ fn with_position_counter(mut self, position: Arc<AtomicUsize>) -> Self {
1032
+ self.position_counter = Some(position);
1033
+ self
1034
+ }
1035
+
1036
+ fn with_read_limits(mut self, read_limits: Vec<usize>) -> Self {
1037
+ assert!(!read_limits.is_empty());
1038
+ assert!(read_limits.iter().all(|limit| *limit > 0));
1039
+ self.read_limits = Some(read_limits);
1040
+ self
1041
+ }
1042
+ }
1043
+
1044
+ impl Read for ChunkedReader {
1045
+ fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
1046
+ self.reads += 1;
1047
+ self.requested.push(output.len());
1048
+ if self.fail_at.is_some_and(|offset| self.position >= offset) {
1049
+ return Err(io::Error::other("injected read failure"));
1050
+ }
1051
+ if self.position >= self.bytes.len() {
1052
+ return Ok(0);
1053
+ }
1054
+ let read_limit = self.read_limits.as_ref().map_or(self.maximum, |limits| {
1055
+ limits[(self.reads - 1) % limits.len()]
1056
+ });
1057
+ let count = output
1058
+ .len()
1059
+ .min(self.maximum)
1060
+ .min(read_limit)
1061
+ .min(self.bytes.len() - self.position);
1062
+ output[..count].copy_from_slice(&self.bytes[self.position..self.position + count]);
1063
+ self.position += count;
1064
+ if let Some(position) = &self.position_counter {
1065
+ position.store(self.position, Ordering::Release);
1066
+ }
1067
+ Ok(count)
1068
+ }
1069
+ }
1070
+
1071
+ #[derive(Debug, Eq, PartialEq)]
1072
+ enum ReferenceDisposition {
1073
+ Text { used_fallback: bool },
1074
+ Binary,
1075
+ Rejected(EncodingRejection),
1076
+ }
1077
+
1078
+ fn reference_disposition(
1079
+ bytes: &[u8],
1080
+ explicit: Option<&str>,
1081
+ fallback: Option<&str>,
1082
+ ) -> ReferenceDisposition {
1083
+ let source = ByteSource::Bytes(bytes);
1084
+ match validate_source_encoding(source, explicit).unwrap() {
1085
+ EncodingDecision::Text(_) => ReferenceDisposition::Text {
1086
+ used_fallback: false,
1087
+ },
1088
+ EncodingDecision::Binary => ReferenceDisposition::Binary,
1089
+ EncodingDecision::Rejected(rejection) => {
1090
+ if explicit.is_none()
1091
+ && !matches!(rejection, EncodingRejection::BomMismatch { .. })
1092
+ && let Some(fallback) = fallback
1093
+ && matches!(
1094
+ validate_source_encoding(source, Some(fallback)).unwrap(),
1095
+ EncodingDecision::Text(_)
1096
+ )
1097
+ {
1098
+ ReferenceDisposition::Text {
1099
+ used_fallback: true,
1100
+ }
1101
+ } else {
1102
+ ReferenceDisposition::Rejected(rejection)
1103
+ }
1104
+ }
1105
+ }
1106
+ }
1107
+
1108
+ fn proof_disposition(proof: &TerminalProof) -> ReferenceDisposition {
1109
+ proof
1110
+ .rejection()
1111
+ .map_or(ReferenceDisposition::Binary, ReferenceDisposition::Rejected)
1112
+ }
1113
+
1114
+ fn mixed_prefix() -> Vec<u8> {
1115
+ let mut bytes = "界".repeat(11).into_bytes();
1116
+ debug_assert_eq!(bytes.len(), 33);
1117
+ bytes.push(0xFF);
1118
+ bytes.resize(PREFLIGHT_BYTES, b'a');
1119
+ bytes
1120
+ }
1121
+
1122
+ #[test]
1123
+ fn every_production_terminal_proof_is_suffix_invariant_against_the_reference() {
1124
+ let cases = [
1125
+ (
1126
+ "nul",
1127
+ vec![b'a', 0, b'b', b'c'],
1128
+ None,
1129
+ None,
1130
+ TerminalProof::NulWithinFrozenProbe,
1131
+ ),
1132
+ (
1133
+ "binary magic",
1134
+ {
1135
+ let mut bytes = b"\x1F\x8B".to_vec();
1136
+ bytes.resize(PREFLIGHT_BYTES, b'a');
1137
+ bytes
1138
+ },
1139
+ None,
1140
+ None,
1141
+ TerminalProof::BinaryMagicAfterUtf8Failure,
1142
+ ),
1143
+ (
1144
+ "explicit malformed",
1145
+ vec![b'a', b'b', b'c', 0xFF],
1146
+ Some("utf-8"),
1147
+ None,
1148
+ TerminalProof::ExplicitDecoderMalformed {
1149
+ encoding: "utf-8".to_string(),
1150
+ },
1151
+ ),
1152
+ (
1153
+ "explicit matching BOM malformed",
1154
+ b"\xFF\xFE\x00\x00\x00\x00\x11\x00".to_vec(),
1155
+ Some("utf-32le"),
1156
+ None,
1157
+ TerminalProof::ExplicitDecoderMalformed {
1158
+ encoding: "utf-32le".to_string(),
1159
+ },
1160
+ ),
1161
+ (
1162
+ "BOM malformed",
1163
+ b"\xFF\xFE\x00\x00\x00\x00\x11\x00".to_vec(),
1164
+ None,
1165
+ None,
1166
+ TerminalProof::BomDecoderMalformed {
1167
+ encoding: "UTF-32LE",
1168
+ },
1169
+ ),
1170
+ (
1171
+ "UTF-8 BOM malformed after NUL gate",
1172
+ {
1173
+ let mut bytes = b"\xEF\xBB\xBF\xFF".to_vec();
1174
+ bytes.resize(PREFLIGHT_BYTES, b'a');
1175
+ bytes
1176
+ },
1177
+ None,
1178
+ None,
1179
+ TerminalProof::BomDecoderMalformed { encoding: "UTF-8" },
1180
+ ),
1181
+ (
1182
+ "automatic mixed without fallback",
1183
+ mixed_prefix(),
1184
+ None,
1185
+ None,
1186
+ TerminalProof::AutomaticRejected {
1187
+ conflict_hex_offset: 3,
1188
+ fallback: FallbackTerminalState::Absent,
1189
+ },
1190
+ ),
1191
+ (
1192
+ "automatic mixed with permanently malformed fallback",
1193
+ mixed_prefix(),
1194
+ None,
1195
+ Some("utf-8"),
1196
+ TerminalProof::AutomaticRejected {
1197
+ conflict_hex_offset: 3,
1198
+ fallback: FallbackTerminalState::PermanentlyMalformed,
1199
+ },
1200
+ ),
1201
+ ];
1202
+ let mut suffixes = vec![
1203
+ Vec::new(),
1204
+ b"plain utf-8".to_vec(),
1205
+ vec![0; 257],
1206
+ vec![0x80; 257],
1207
+ (0..4096).map(|index| (index * 73) as u8).collect(),
1208
+ ];
1209
+ suffixes.extend((0_u16..=u8::MAX as u16).map(|byte| vec![byte as u8]));
1210
+ for (name, prefix, explicit, fallback, expected_proof) in cases {
1211
+ for suffix in &suffixes {
1212
+ let mut bytes = prefix.clone();
1213
+ bytes.extend_from_slice(suffix);
1214
+ let expected = reference_disposition(&bytes, explicit, fallback);
1215
+ for maximum in [1, 2, 3, 7, 127, PREFLIGHT_BYTES] {
1216
+ let mut reader = ChunkedReader::new(bytes.clone(), maximum);
1217
+ reader.fail_at = Some(prefix.len());
1218
+ let CaptureRead::Terminal(proof) =
1219
+ capture_reader(&mut reader, explicit, fallback, None).unwrap()
1220
+ else {
1221
+ panic!("{name}: expected terminal proof at chunk size {maximum}");
1222
+ };
1223
+ assert_eq!(proof, expected_proof, "{name}, chunk size {maximum}");
1224
+ assert_eq!(
1225
+ proof_disposition(&proof),
1226
+ expected,
1227
+ "{name}, chunk size {maximum}, suffix length {}",
1228
+ suffix.len()
1229
+ );
1230
+ assert!(
1231
+ reader.position
1232
+ <= prefix
1233
+ .len()
1234
+ .saturating_add(maximum.saturating_sub(1))
1235
+ .min(bytes.len()),
1236
+ "{name}, chunk size {maximum}: reader crossed a second read after proof"
1237
+ );
1238
+ }
1239
+ }
1240
+
1241
+ let suffix = (0..4096)
1242
+ .map(|index| (index * 151 + 17) as u8)
1243
+ .collect::<Vec<_>>();
1244
+ let mut bytes = prefix.clone();
1245
+ bytes.extend_from_slice(&suffix);
1246
+ let expected = reference_disposition(&bytes, explicit, fallback);
1247
+ let mut random_state = 0xA5A5_1F3D_u32;
1248
+ let read_limits = (0..64)
1249
+ .map(|_| {
1250
+ random_state ^= random_state << 13;
1251
+ random_state ^= random_state >> 17;
1252
+ random_state ^= random_state << 5;
1253
+ (random_state as usize % 1024) + 1
1254
+ })
1255
+ .collect();
1256
+ let mut reader = ChunkedReader::new(bytes, usize::MAX).with_read_limits(read_limits);
1257
+ reader.fail_at = Some(prefix.len());
1258
+ let CaptureRead::Terminal(proof) =
1259
+ capture_reader(&mut reader, explicit, fallback, None).unwrap()
1260
+ else {
1261
+ panic!("{name}: expected terminal proof at randomized chunk boundaries");
1262
+ };
1263
+ assert_eq!(proof, expected_proof, "{name}, randomized chunks");
1264
+ assert_eq!(
1265
+ proof_disposition(&proof),
1266
+ expected,
1267
+ "{name}, randomized chunks"
1268
+ );
1269
+ }
1270
+ }
1271
+
1272
+ #[test]
1273
+ fn large_capture_promotes_once_reads_repeatably_and_unlinks_on_drop() {
1274
+ let bytes = vec![b'x'; MEMORY_SNAPSHOT_LIMIT + 1];
1275
+ let mut reader = ChunkedReader::new(bytes.clone(), 17 * 1024);
1276
+ let promotions = Arc::new(AtomicUsize::new(0));
1277
+ let promotions_for_hook = Arc::clone(&promotions);
1278
+ let hook = Arc::new(move |stage| {
1279
+ if stage == TestStage::SnapshotPromote {
1280
+ promotions_for_hook.fetch_add(1, Ordering::AcqRel);
1281
+ }
1282
+ });
1283
+ let (mut guard, operation) =
1284
+ RequestWorkGuard::new_with_hook(RequestId::Number(10), CancellationToken::new(), hook);
1285
+ let CaptureRead::Complete(snapshot) =
1286
+ capture_reader(&mut reader, None, None, Some(&operation)).unwrap()
1287
+ else {
1288
+ panic!("expected complete snapshot");
1289
+ };
1290
+ guard.disarm();
1291
+ assert_eq!(promotions.load(Ordering::Acquire), 1);
1292
+ assert_eq!(reader.requested.first(), Some(&PREFLIGHT_BYTES));
1293
+ assert!(
1294
+ reader
1295
+ .requested
1296
+ .iter()
1297
+ .all(|requested| *requested <= SNAPSHOT_CHUNK_BYTES)
1298
+ );
1299
+ let temp_path = snapshot.temp_path().unwrap().to_path_buf();
1300
+ assert!(temp_path.exists());
1301
+ assert_eq!(snapshot.len, bytes.len() as u64);
1302
+ for _ in 0..2 {
1303
+ let mut reopened = snapshot.open_reader(0).unwrap();
1304
+ let mut actual = Vec::new();
1305
+ reopened.read_to_end(&mut actual).unwrap();
1306
+ assert_eq!(actual, bytes);
1307
+ }
1308
+ drop(snapshot);
1309
+ assert!(!temp_path.exists());
1310
+ }
1311
+
1312
+ #[test]
1313
+ fn cancellation_at_capture_stages_stops_within_the_declared_read_boundary() {
1314
+ let preflight_token = CancellationToken::new();
1315
+ let token_for_hook = preflight_token.clone();
1316
+ let hook = Arc::new(move |stage| {
1317
+ if stage == TestStage::CapturePreflightRead {
1318
+ token_for_hook.cancel();
1319
+ }
1320
+ });
1321
+ let (_guard, operation) =
1322
+ RequestWorkGuard::new_with_hook(RequestId::Number(11), preflight_token, hook);
1323
+ let mut reader = ChunkedReader::new(vec![b'a'; PREFLIGHT_BYTES + 1], usize::MAX);
1324
+ assert!(matches!(
1325
+ capture_reader(&mut reader, None, None, Some(&operation)),
1326
+ Err(CaptureFailure::Cancelled)
1327
+ ));
1328
+ assert_eq!(reader.position, 0);
1329
+
1330
+ let stream_token = CancellationToken::new();
1331
+ let token_for_hook = stream_token.clone();
1332
+ let hook = Arc::new(move |stage| {
1333
+ if stage == TestStage::SnapshotChunk {
1334
+ token_for_hook.cancel();
1335
+ }
1336
+ });
1337
+ let (_guard, operation) =
1338
+ RequestWorkGuard::new_with_hook(RequestId::Number(12), stream_token, hook);
1339
+ let mut reader = ChunkedReader::new(vec![b'a'; PREFLIGHT_BYTES + 1], usize::MAX);
1340
+ assert!(matches!(
1341
+ capture_reader(&mut reader, None, None, Some(&operation)),
1342
+ Err(CaptureFailure::Cancelled)
1343
+ ));
1344
+ assert_eq!(reader.position, PREFLIGHT_BYTES);
1345
+
1346
+ let promote_token = CancellationToken::new();
1347
+ let token_for_hook = promote_token.clone();
1348
+ let promotions = Arc::new(AtomicUsize::new(0));
1349
+ let promotions_for_hook = Arc::clone(&promotions);
1350
+ let hook = Arc::new(move |stage| {
1351
+ if stage == TestStage::SnapshotPromote {
1352
+ promotions_for_hook.fetch_add(1, Ordering::AcqRel);
1353
+ token_for_hook.cancel();
1354
+ }
1355
+ });
1356
+ let (_guard, operation) =
1357
+ RequestWorkGuard::new_with_hook(RequestId::Number(13), promote_token, hook);
1358
+ let position = Arc::new(AtomicUsize::new(0));
1359
+ let mut reader = ChunkedReader::new(vec![b'a'; MEMORY_SNAPSHOT_LIMIT + 1], usize::MAX)
1360
+ .with_position_counter(Arc::clone(&position));
1361
+ assert!(matches!(
1362
+ capture_reader(&mut reader, None, None, Some(&operation)),
1363
+ Err(CaptureFailure::Cancelled)
1364
+ ));
1365
+ assert_eq!(promotions.load(Ordering::Acquire), 1);
1366
+ assert!(position.load(Ordering::Acquire) <= MEMORY_SNAPSHOT_LIMIT + SNAPSHOT_CHUNK_BYTES);
1367
+ }
1368
+
1369
+ #[test]
1370
+ fn sealed_temp_snapshot_survives_original_removal_for_all_later_passes() {
1371
+ let temp = tempfile::tempdir().unwrap();
1372
+ let path = temp.path().join("large-searchable.txt");
1373
+ let mut bytes = vec![b'a'; MEMORY_SNAPSHOT_LIMIT + 1];
1374
+ bytes.extend_from_slice(b"\nneedle\n");
1375
+ std::fs::write(&path, &bytes).unwrap();
1376
+ let metadata = std::fs::metadata(&path).unwrap();
1377
+ let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
1378
+ let CaptureDisposition::Searchable(snapshot) =
1379
+ capture_classify(&candidate, None, None, None).unwrap()
1380
+ else {
1381
+ panic!("expected searchable snapshot");
1382
+ };
1383
+ let temp_path = snapshot.captured.temp_path().unwrap().to_path_buf();
1384
+ std::fs::remove_file(&path).unwrap();
1385
+
1386
+ let source = ByteSource::Snapshot(&snapshot);
1387
+ let EncodingDecision::Text(validated) = validate_source_encoding(source, None).unwrap()
1388
+ else {
1389
+ panic!("expected trusted UTF-8 snapshot");
1390
+ };
1391
+ let mut reader = validated.open_source_reader(source).unwrap();
1392
+ let mut actual = Vec::new();
1393
+ reader.read_to_end(&mut actual).unwrap();
1394
+ assert_eq!(actual, bytes);
1395
+ assert!(temp_path.exists());
1396
+ drop(reader);
1397
+ let shared = snapshot.shared_range(0).unwrap();
1398
+ drop(snapshot);
1399
+ assert!(temp_path.exists());
1400
+ let mut shared_reader = shared.open_reader().unwrap();
1401
+ let mut shared_bytes = Vec::new();
1402
+ shared_reader.read_to_end(&mut shared_bytes).unwrap();
1403
+ assert_eq!(shared_bytes, bytes);
1404
+ drop(shared_reader);
1405
+ assert_eq!(
1406
+ shared
1407
+ .read_range((bytes.len() - 7) as u64..bytes.len() as u64)
1408
+ .unwrap(),
1409
+ b"needle\n"
1410
+ );
1411
+ drop(shared);
1412
+ assert!(!temp_path.exists());
1413
+ }
1414
+
1415
+ #[test]
1416
+ fn traversal_mismatch_and_post_capture_mutation_are_file_changed() {
1417
+ let temp = tempfile::tempdir().unwrap();
1418
+ let path = temp.path().join("changing.txt");
1419
+ std::fs::write(&path, b"before").unwrap();
1420
+ let metadata = std::fs::metadata(&path).unwrap();
1421
+ let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
1422
+ std::fs::write(&path, b"different length").unwrap();
1423
+ assert!(matches!(
1424
+ capture_classify(&candidate, None, None, None).unwrap(),
1425
+ CaptureDisposition::FileChanged
1426
+ ));
1427
+
1428
+ std::fs::write(&path, b"same-one").unwrap();
1429
+ set_file_mtime(&path, FileTime::from_unix_time(1_700_000_000, 0)).unwrap();
1430
+ let metadata = std::fs::metadata(&path).unwrap();
1431
+ let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
1432
+ std::fs::write(&path, b"same-two").unwrap();
1433
+ set_file_mtime(&path, FileTime::from_unix_time(1_700_000_010, 0)).unwrap();
1434
+ assert!(matches!(
1435
+ capture_classify(&candidate, None, None, None).unwrap(),
1436
+ CaptureDisposition::FileChanged
1437
+ ));
1438
+
1439
+ std::fs::write(&path, b"stable bytes").unwrap();
1440
+ let metadata = std::fs::metadata(&path).unwrap();
1441
+ let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
1442
+ let changed = Arc::new(AtomicBool::new(false));
1443
+ let changed_for_hook = Arc::clone(&changed);
1444
+ let path_for_hook = path.clone();
1445
+ let hook = Arc::new(move |stage| {
1446
+ if stage == TestStage::BeforeIdentityPostCheck
1447
+ && !changed_for_hook.swap(true, Ordering::AcqRel)
1448
+ {
1449
+ let mut file = std::fs::OpenOptions::new()
1450
+ .write(true)
1451
+ .truncate(true)
1452
+ .open(&path_for_hook)
1453
+ .unwrap();
1454
+ file.write_all(b"mutated to another length").unwrap();
1455
+ file.flush().unwrap();
1456
+ }
1457
+ });
1458
+ let (mut guard, operation) =
1459
+ RequestWorkGuard::new_with_hook(RequestId::Number(1), CancellationToken::new(), hook);
1460
+ assert!(matches!(
1461
+ capture_classify(&candidate, None, None, Some(&operation)).unwrap(),
1462
+ CaptureDisposition::FileChanged
1463
+ ));
1464
+ guard.disarm();
1465
+ assert!(changed.load(Ordering::Acquire));
1466
+
1467
+ std::fs::write(&path, b"stable replacement bytes").unwrap();
1468
+ let metadata = std::fs::metadata(&path).unwrap();
1469
+ let candidate = PathRecord::from_metadata(&path, temp.path(), &metadata, true).unwrap();
1470
+ let replaced = Arc::new(AtomicBool::new(false));
1471
+ let replaced_for_hook = Arc::clone(&replaced);
1472
+ let path_for_hook = path.clone();
1473
+ let old_path = temp.path().join("captured-old.txt");
1474
+ let hook = Arc::new(move |stage| {
1475
+ if stage == TestStage::BeforeIdentityPostCheck
1476
+ && !replaced_for_hook.swap(true, Ordering::AcqRel)
1477
+ {
1478
+ std::fs::rename(&path_for_hook, &old_path).unwrap();
1479
+ std::fs::write(&path_for_hook, b"stable replacement bytes").unwrap();
1480
+ }
1481
+ });
1482
+ let (mut guard, operation) =
1483
+ RequestWorkGuard::new_with_hook(RequestId::Number(2), CancellationToken::new(), hook);
1484
+ assert!(matches!(
1485
+ capture_classify(&candidate, None, None, Some(&operation)).unwrap(),
1486
+ CaptureDisposition::FileChanged
1487
+ ));
1488
+ guard.disarm();
1489
+ assert!(replaced.load(Ordering::Acquire));
1490
+ }
1491
+ }