agent-sanitizer 2.19.1 → 2.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/THREAT-MODEL.md +6 -2
- package/bin/sanitize-cli.mjs +62 -27
- package/package.json +4 -4
- package/src/output.mjs +28 -11
- package/src/rehydrate.mjs +13 -20
- package/src/view-map.mjs +59 -22
- package/types/output.d.mts +15 -2
- package/types/view-map.d.mts +47 -3
package/README.md
CHANGED
|
@@ -94,6 +94,11 @@ owns each message, and any value outside the enum makes `sanitizeText` **throw**
|
|
|
94
94
|
| `filter-flagged` | The filter flagged the output as a possible injection without deleting (content intact) |
|
|
95
95
|
| `filter-error` | The filter reported a non-fatal internal error while scanning (a fatal filter throws) |
|
|
96
96
|
|
|
97
|
+
Every span is matched against the **original** text and the deletions applied in
|
|
98
|
+
a single ordered pass, so the bytes a filter can remove are exactly the bytes its
|
|
99
|
+
spans matched in the input — an earlier deletion can never manufacture a match
|
|
100
|
+
for a later span (overlapping spans resolve first-match-wins).
|
|
101
|
+
|
|
97
102
|
## What installing entails
|
|
98
103
|
|
|
99
104
|
Installing the plugin puts four hooks on every session, and this is what they
|
package/THREAT-MODEL.md
CHANGED
|
@@ -169,8 +169,12 @@ fail-closed path: a redactor that throws makes the pipeline rethrow, so the
|
|
|
169
169
|
caller suppresses the output rather than emit an unvetted value. Layer 5 is a
|
|
170
170
|
deliberately thin, safe slot: the injected filter returns **verbatim spans to
|
|
171
171
|
delete** (never replacement text), so even a compromised filter can only remove
|
|
172
|
-
legitimate content—it can never inject bytes into the model’s view.
|
|
173
|
-
|
|
172
|
+
legitimate content—it can never inject bytes into the model’s view. That removal
|
|
173
|
+
is bounded to the spans the filter actually named: every span is matched against
|
|
174
|
+
the **original** text and the deletions applied in a single ordered pass, so an
|
|
175
|
+
earlier deletion cannot join two kept regions into a match for a later span and
|
|
176
|
+
erase text neither span occurred in. A live second-LLM injection filter is the
|
|
177
|
+
caller’s to wire behind that contract.
|
|
174
178
|
|
|
175
179
|
The same "never inject" property governs the filter’s `warning`: it is a
|
|
176
180
|
**closed enum code** (`FILTER_WARNING`: `spans-removed` / `filter-flagged` /
|
package/bin/sanitize-cli.mjs
CHANGED
|
@@ -70,8 +70,16 @@ function maxInputBytes() {
|
|
|
70
70
|
return Math.floor(parsed);
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/** The one-line reason reported for a caught throw. A thrown non-Error that
|
|
74
|
+
* still carries a `message` keeps reporting it (the historical behaviour);
|
|
75
|
+
* anything else stringifies.
|
|
76
|
+
* @param {unknown} err */
|
|
77
|
+
const errorMessage = (err) =>
|
|
78
|
+
/** @type {{ message?: string }} */ (err)?.message ?? String(err);
|
|
79
|
+
|
|
73
80
|
/** Throw if `text` exceeds the configured byte cap. The message names the limit
|
|
74
|
-
* and the env var so a caller can act on it.
|
|
81
|
+
* and the env var so a caller can act on it.
|
|
82
|
+
* @param {string} text */
|
|
75
83
|
function enforceSizeLimit(text) {
|
|
76
84
|
const limit = maxInputBytes();
|
|
77
85
|
const size = Buffer.byteLength(text, "utf8");
|
|
@@ -82,43 +90,49 @@ function enforceSizeLimit(text) {
|
|
|
82
90
|
);
|
|
83
91
|
}
|
|
84
92
|
|
|
85
|
-
/**
|
|
93
|
+
/** Read a required string field, throwing when it is absent or the wrong type.
|
|
94
|
+
* Returns the value (rather than only asserting) so the caller carries the
|
|
95
|
+
* narrowed `string` forward instead of re-reading an untyped bag.
|
|
96
|
+
* @param {Record<string, unknown>} req @param {string} key
|
|
97
|
+
* @returns {string} */
|
|
86
98
|
function requireString(req, key) {
|
|
87
|
-
|
|
99
|
+
const value = req[key];
|
|
100
|
+
if (typeof value !== "string")
|
|
88
101
|
throw new Error(`request.${key} must be a string`);
|
|
102
|
+
return value;
|
|
89
103
|
}
|
|
90
104
|
|
|
91
105
|
/** Operations the CLI exposes. Each takes the parsed request, returns the JSON
|
|
92
106
|
* payload object. Non-`sanitize` modules are imported lazily so a caller that
|
|
93
107
|
* only ever sanitizes never loads prompt/output/instructions code. */
|
|
94
108
|
const OPS = {
|
|
109
|
+
/** @param {Record<string, unknown>} req */
|
|
95
110
|
async sanitize(req) {
|
|
96
|
-
requireString(req, "text");
|
|
97
|
-
const { cleaned, found, warnings } = await sanitize(
|
|
111
|
+
const text = requireString(req, "text");
|
|
112
|
+
const { cleaned, found, warnings } = await sanitize(text, {
|
|
98
113
|
html: Boolean(req.html),
|
|
99
114
|
});
|
|
100
115
|
return { cleaned, found, warnings };
|
|
101
116
|
},
|
|
102
117
|
|
|
118
|
+
/** @param {Record<string, unknown>} req */
|
|
103
119
|
async sanitizeText(req) {
|
|
104
|
-
requireString(req, "text");
|
|
120
|
+
const text = requireString(req, "text");
|
|
105
121
|
// Layers 1–3 only: redact (Layer 4) and filterInjection (Layer 5) are
|
|
106
122
|
// injected JS callbacks with no wire form, so they're never set here.
|
|
107
123
|
const { sanitizeText } = await import("../src/output.mjs");
|
|
108
|
-
const { cleaned, warnings, modified, sgrNote } = await sanitizeText(
|
|
109
|
-
req.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
exfilScan: Boolean(req.exfilScan),
|
|
113
|
-
},
|
|
114
|
-
);
|
|
124
|
+
const { cleaned, warnings, modified, sgrNote } = await sanitizeText(text, {
|
|
125
|
+
html: Boolean(req.html),
|
|
126
|
+
exfilScan: Boolean(req.exfilScan),
|
|
127
|
+
});
|
|
115
128
|
return { cleaned, warnings, modified, sgrNote };
|
|
116
129
|
},
|
|
117
130
|
|
|
131
|
+
/** @param {Record<string, unknown>} req */
|
|
118
132
|
async classifyPrompt(req) {
|
|
119
|
-
requireString(req, "text");
|
|
133
|
+
const text = requireString(req, "text");
|
|
120
134
|
const { classifyPrompt } = await import("../src/prompt.mjs");
|
|
121
|
-
return classifyPrompt(
|
|
135
|
+
return classifyPrompt(text);
|
|
122
136
|
},
|
|
123
137
|
|
|
124
138
|
// SECURITY (R8): `scanInstructionFiles` and `cleanFile` take filesystem
|
|
@@ -130,10 +144,12 @@ const OPS = {
|
|
|
130
144
|
// this CLI's stdin to untrusted/model-controlled input for these ops. If you
|
|
131
145
|
// must accept untrusted callers, add opt-in root confinement (reject paths
|
|
132
146
|
// that resolve outside an allow-listed root) before exposing them.
|
|
147
|
+
/** @param {Record<string, unknown>} req */
|
|
133
148
|
async scanInstructionFiles(req) {
|
|
149
|
+
const globs = req.globs;
|
|
134
150
|
if (
|
|
135
|
-
!Array.isArray(
|
|
136
|
-
|
|
151
|
+
!Array.isArray(globs) ||
|
|
152
|
+
globs.some((/** @type {unknown} */ g) => typeof g !== "string")
|
|
137
153
|
)
|
|
138
154
|
throw new Error("request.globs must be an array of strings");
|
|
139
155
|
// Fail loud on a present-but-non-string cwd rather than silently dropping it
|
|
@@ -143,13 +159,14 @@ const OPS = {
|
|
|
143
159
|
throw new Error("request.cwd must be a string");
|
|
144
160
|
const { scanInstructionFiles } = await import("../src/instructions.mjs");
|
|
145
161
|
const opts = typeof req.cwd === "string" ? { cwd: req.cwd } : {};
|
|
146
|
-
return { findings: scanInstructionFiles(
|
|
162
|
+
return { findings: scanInstructionFiles(globs, opts) };
|
|
147
163
|
},
|
|
148
164
|
|
|
165
|
+
/** @param {Record<string, unknown>} req */
|
|
149
166
|
async cleanFile(req) {
|
|
150
|
-
requireString(req, "path");
|
|
167
|
+
const path = requireString(req, "path");
|
|
151
168
|
const { cleanFile } = await import("../src/instructions.mjs");
|
|
152
|
-
return { changed: cleanFile(
|
|
169
|
+
return { changed: cleanFile(path) };
|
|
153
170
|
},
|
|
154
171
|
};
|
|
155
172
|
|
|
@@ -162,9 +179,13 @@ const OPS = {
|
|
|
162
179
|
*/
|
|
163
180
|
async function handle(payload) {
|
|
164
181
|
enforceSizeLimit(payload);
|
|
165
|
-
const request = JSON.parse(payload);
|
|
166
|
-
|
|
167
|
-
|
|
182
|
+
const request = /** @type {Record<string, unknown>} */ (JSON.parse(payload));
|
|
183
|
+
// Stringified so a non-string `op` (a number, an object) still reaches the
|
|
184
|
+
// hasOwnProperty guard and the same "unknown op" error it always did.
|
|
185
|
+
const op = String(request.op ?? "sanitize");
|
|
186
|
+
const run = Object.prototype.hasOwnProperty.call(OPS, op)
|
|
187
|
+
? OPS[/** @type {keyof typeof OPS} */ (op)]
|
|
188
|
+
: null;
|
|
168
189
|
if (!run) throw new Error(`unknown op: ${op}`);
|
|
169
190
|
return JSON.stringify(await run(request));
|
|
170
191
|
}
|
|
@@ -189,6 +210,11 @@ async function readAll(stream) {
|
|
|
189
210
|
return text;
|
|
190
211
|
}
|
|
191
212
|
|
|
213
|
+
/**
|
|
214
|
+
* One unit of the worker's one-response-per-input-line framing.
|
|
215
|
+
* @typedef {{ kind: "line", text: string } | { kind: "oversize" }} SplitEvent
|
|
216
|
+
*/
|
|
217
|
+
|
|
192
218
|
/**
|
|
193
219
|
* Streaming newline-splitter that never buffers a line past the byte `limit`.
|
|
194
220
|
*
|
|
@@ -211,6 +237,7 @@ async function readAll(stream) {
|
|
|
211
237
|
* the one-response-per-input-line framing holds even for a dropped line.
|
|
212
238
|
*
|
|
213
239
|
* @param {number} limit per-line byte cap (`maxInputBytes()`)
|
|
240
|
+
* @returns {((chunk: Buffer) => SplitEvent[]) & { end: () => SplitEvent[] }}
|
|
214
241
|
*/
|
|
215
242
|
function createLineSplitter(limit) {
|
|
216
243
|
// The bytes of the current line are held as a LIST of chunk slices plus their
|
|
@@ -240,7 +267,8 @@ function createLineSplitter(limit) {
|
|
|
240
267
|
return buf;
|
|
241
268
|
};
|
|
242
269
|
|
|
243
|
-
/** Strip one trailing `\r` so CRLF input frames identically to LF.
|
|
270
|
+
/** Strip one trailing `\r` so CRLF input frames identically to LF.
|
|
271
|
+
* @param {Buffer} buf */
|
|
244
272
|
const toLine = (buf) => {
|
|
245
273
|
const stripCr = buf.length > 0 && buf[buf.length - 1] === 0x0d;
|
|
246
274
|
return buf.toString("utf8", 0, stripCr ? buf.length - 1 : buf.length);
|
|
@@ -250,6 +278,7 @@ function createLineSplitter(limit) {
|
|
|
250
278
|
// into the pending list, flipping to `discarding` if it would breach the cap.
|
|
251
279
|
// Applies identically to a newline-terminated segment and to the unterminated
|
|
252
280
|
// tail, so an oversize line is caught WITHIN a chunk, not only at its edge.
|
|
281
|
+
/** @param {Buffer} segment */
|
|
253
282
|
const accumulate = (segment) => {
|
|
254
283
|
if (discarding) return;
|
|
255
284
|
if (pendingLen + segment.length > limit) {
|
|
@@ -263,8 +292,11 @@ function createLineSplitter(limit) {
|
|
|
263
292
|
}
|
|
264
293
|
};
|
|
265
294
|
|
|
266
|
-
/** Feed one chunk, returning the events it completes (newline-terminated).
|
|
295
|
+
/** Feed one chunk, returning the events it completes (newline-terminated).
|
|
296
|
+
* @param {Buffer} chunk
|
|
297
|
+
* @returns {SplitEvent[]} */
|
|
267
298
|
const push = (chunk) => {
|
|
299
|
+
/** @type {SplitEvent[]} */
|
|
268
300
|
const events = [];
|
|
269
301
|
let start = 0;
|
|
270
302
|
for (let i = 0; i < chunk.length; i++) {
|
|
@@ -287,6 +319,7 @@ function createLineSplitter(limit) {
|
|
|
287
319
|
* Flush at EOF. A final line with no trailing `\n` is still a request, so it
|
|
288
320
|
* gets a response — matching `readline`, which emits its last line on `close`.
|
|
289
321
|
* An empty tail (stream ended on a `\n`, or was empty) yields nothing.
|
|
322
|
+
* @returns {SplitEvent[]}
|
|
290
323
|
*/
|
|
291
324
|
push.end = () => {
|
|
292
325
|
if (discarding) {
|
|
@@ -301,6 +334,7 @@ function createLineSplitter(limit) {
|
|
|
301
334
|
return push;
|
|
302
335
|
}
|
|
303
336
|
|
|
337
|
+
/** @param {number} limit */
|
|
304
338
|
const OVERSIZE_ERROR = (limit) =>
|
|
305
339
|
JSON.stringify({
|
|
306
340
|
error:
|
|
@@ -323,6 +357,7 @@ async function runWorker() {
|
|
|
323
357
|
// "request too large" error a one-shot caller sees. `response` never holds a
|
|
324
358
|
// newline: `JSON.stringify` of the result or of a one-key error object is
|
|
325
359
|
// single-line, so the one-line-per-request framing holds.
|
|
360
|
+
/** @param {SplitEvent} event */
|
|
326
361
|
const respond = async (event) => {
|
|
327
362
|
if (event.kind === "oversize") {
|
|
328
363
|
process.stdout.write(`${OVERSIZE_ERROR(limit)}\n`);
|
|
@@ -332,7 +367,7 @@ async function runWorker() {
|
|
|
332
367
|
try {
|
|
333
368
|
response = await handle(event.text);
|
|
334
369
|
} catch (err) {
|
|
335
|
-
response = JSON.stringify({ error:
|
|
370
|
+
response = JSON.stringify({ error: errorMessage(err) });
|
|
336
371
|
}
|
|
337
372
|
process.stdout.write(`${response}\n`);
|
|
338
373
|
};
|
|
@@ -353,7 +388,7 @@ async function runOneShot() {
|
|
|
353
388
|
try {
|
|
354
389
|
response = await handle(await readAll(process.stdin));
|
|
355
390
|
} catch (err) {
|
|
356
|
-
process.stderr.write(`sanitize CLI: ${
|
|
391
|
+
process.stderr.write(`sanitize CLI: ${errorMessage(err)}\n`);
|
|
357
392
|
process.exitCode = 1;
|
|
358
393
|
return;
|
|
359
394
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-sanitizer",
|
|
3
|
-
"version": "2.19.
|
|
3
|
+
"version": "2.19.3",
|
|
4
4
|
"description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
@@ -207,14 +207,14 @@
|
|
|
207
207
|
"unist-util-visit": "5.1.0"
|
|
208
208
|
},
|
|
209
209
|
"scripts": {
|
|
210
|
-
"test": "
|
|
211
|
-
"coverage": "
|
|
210
|
+
"test": "node scripts/coverage.mjs",
|
|
211
|
+
"coverage": "node scripts/coverage.mjs",
|
|
212
212
|
"check": "tsc --noEmit && tsc -p tsconfig.hooks.json --noEmit",
|
|
213
213
|
"typecheck": "tsc --noEmit && tsc -p tsconfig.hooks.json --noEmit",
|
|
214
214
|
"build:types": "tsc -p tsconfig.build.json && tsc -p tsconfig.build-hooks.json",
|
|
215
215
|
"gen:joining-type": "node scripts/gen-joining-type.mjs",
|
|
216
216
|
"lint": "eslint .",
|
|
217
|
-
"test:mutation": "
|
|
217
|
+
"test:mutation": "node scripts/mutate.mjs",
|
|
218
218
|
"format": "prettier --write .",
|
|
219
219
|
"format:check": "prettier --check ."
|
|
220
220
|
}
|
package/src/output.mjs
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
import { CATEGORY, describeStripped, isSgrOnly } from "./invisible.mjs";
|
|
28
28
|
import { HTML_TAG_PRESENT, MD_LINK_HINT } from "./gates.mjs";
|
|
29
29
|
import { applyLayer1, LONE_SURROGATE_RE } from "./layer1.mjs";
|
|
30
|
+
import { orderedMatches, spliceOrdered } from "./view-map.mjs";
|
|
30
31
|
|
|
31
32
|
/**
|
|
32
33
|
* Closed enum of LIBRARY-OWNED Layer-5 warning codes — the ONLY warning values
|
|
@@ -206,22 +207,38 @@ export function describeWarned(warned) {
|
|
|
206
207
|
/**
|
|
207
208
|
* Delete each verbatim span in `spans` from `text`. The secure Layer-5
|
|
208
209
|
* primitive: a filter can only ask for deletions, so this can never inject
|
|
209
|
-
* bytes. Returns the new text and how many
|
|
210
|
-
*
|
|
210
|
+
* bytes. Returns the new text and how many span occurrences were removed (0
|
|
211
|
+
* when no span was present).
|
|
212
|
+
*
|
|
213
|
+
* Every occurrence is located in the ORIGINAL `text` and the deletions applied
|
|
214
|
+
* in one ordered pass ({@link spliceOrdered}), so every removed byte lies inside
|
|
215
|
+
* a match some span had in the INPUT. Deleting span-by-span with a
|
|
216
|
+
* chained `split`/`join` would not hold that line: an earlier deletion joins the
|
|
217
|
+
* bytes on either side of it and can CREATE a match for a later span that never
|
|
218
|
+
* occurred in the input — `deleteVerbatimSpans("PRE-XX-POST", ["-XX-", "PREPOST"])`
|
|
219
|
+
* then deletes the whole document. That would widen the Layer-5 seam's blast
|
|
220
|
+
* radius (see the module doc) from "a compromised filter can at most remove the
|
|
221
|
+
* content it named" to "it can remove content it never named".
|
|
222
|
+
*
|
|
223
|
+
* Overlapping spans are resolved first-match-wins, so `removed` counts the
|
|
224
|
+
* occurrences actually spliced out, never a double-count of the same bytes.
|
|
211
225
|
* @param {string} text
|
|
212
226
|
* @param {string[]} spans
|
|
213
227
|
* @returns {{ text: string, removed: number }}
|
|
214
228
|
*/
|
|
215
229
|
export function deleteVerbatimSpans(text, spans) {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
230
|
+
// Keep only non-empty STRING spans. The filter is untrusted JS, not a
|
|
231
|
+
// type-checked caller, so the array can hold anything: `indexOf(123)` would
|
|
232
|
+
// silently match the literal text "123" (deleting content the filter never
|
|
233
|
+
// named), and `occurrences` steps by `needle.length` — `undefined` for a
|
|
234
|
+
// number — making `indexOf(needle, NaN)` clamp back to the same index and
|
|
235
|
+
// loop forever. Fail open on a malformed entry rather than mangle bytes or
|
|
236
|
+
// hang the pipeline.
|
|
237
|
+
const usable = spans.filter(
|
|
238
|
+
(span) => typeof span === "string" && span !== "",
|
|
239
|
+
);
|
|
240
|
+
const spliced = spliceOrdered(text, orderedMatches(text, usable), () => "");
|
|
241
|
+
return { text: spliced.text, removed: spliced.spans.length };
|
|
225
242
|
}
|
|
226
243
|
|
|
227
244
|
/**
|
package/src/rehydrate.mjs
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
occurrences,
|
|
52
52
|
overlapAwareCount,
|
|
53
53
|
orderedMatches,
|
|
54
|
+
spliceOrdered,
|
|
54
55
|
alignDeletions,
|
|
55
56
|
resolveSpan,
|
|
56
57
|
rehydrateNewString,
|
|
@@ -415,9 +416,9 @@ async function rehydrateWrite(ti, view, io, hint) {
|
|
|
415
416
|
};
|
|
416
417
|
|
|
417
418
|
// Resolve each of this file's placeholder texts to its single secret first,
|
|
418
|
-
// then splice in ONE ordered pass (R6)
|
|
419
|
-
//
|
|
420
|
-
//
|
|
419
|
+
// then splice in ONE ordered pass (R6) via the shared `spliceOrdered` — see
|
|
420
|
+
// its doc for why a chained `out.split(ph).join(secret)` per placeholder is
|
|
421
|
+
// unsound.
|
|
421
422
|
const valueByPh = new Map();
|
|
422
423
|
for (const phText of texts) {
|
|
423
424
|
const produced = view.pairs.filter((pair) => pair.placeholder === phText);
|
|
@@ -438,23 +439,15 @@ async function rehydrateWrite(ti, view, io, hint) {
|
|
|
438
439
|
};
|
|
439
440
|
valueByPh.set(phText, values[0]);
|
|
440
441
|
}
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
//
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
const secret = valueByPh.get(match.text);
|
|
451
|
-
out += ti.content.slice(last, match.index);
|
|
452
|
-
const secretStart = out.length;
|
|
453
|
-
out += secret;
|
|
454
|
-
secretSpans.push({ start: secretStart, end: out.length });
|
|
455
|
-
last = match.index + match.text.length;
|
|
456
|
-
}
|
|
457
|
-
out += ti.content.slice(last);
|
|
442
|
+
// `secretSpans`: byte ranges in `out` occupied by the substituted secret
|
|
443
|
+
// values. A hint occurrence inside one of these is a pathological secret whose
|
|
444
|
+
// bytes contain the hint prefix, NOT a placeholder the model pasted — so it is
|
|
445
|
+
// excluded from the foreign-placeholder scan below.
|
|
446
|
+
const { text: out, spans: secretSpans } = spliceOrdered(
|
|
447
|
+
ti.content,
|
|
448
|
+
orderedMatches(ti.content, texts),
|
|
449
|
+
(match) => valueByPh.get(match.text),
|
|
450
|
+
);
|
|
458
451
|
const secrets = [...valueByPh.values()];
|
|
459
452
|
|
|
460
453
|
// R3: the new content may mix a valid same-file placeholder (substituted
|
package/src/view-map.mjs
CHANGED
|
@@ -223,9 +223,15 @@ export function resolveSpan(
|
|
|
223
223
|
}
|
|
224
224
|
|
|
225
225
|
/**
|
|
226
|
-
* All occurrences of any needle in `text`, ordered by position.
|
|
227
|
-
*
|
|
228
|
-
*
|
|
226
|
+
* All occurrences of any needle in `text`, ordered by position. Every index is
|
|
227
|
+
* computed against the ORIGINAL `text`, so the caller can splice them in one
|
|
228
|
+
* pass ({@link spliceOrdered}). Redaction placeholder texts never
|
|
229
|
+
* substring-overlap one another (each ends in "]" right after its
|
|
230
|
+
* distinguishing label), so for that caller the sorted matches are also
|
|
231
|
+
* non-overlapping; needles from an untrusted source (a Layer-5 filter's
|
|
232
|
+
* removeSpans) can overlap, which spliceOrdered resolves first-match-wins.
|
|
233
|
+
* Distinct needles matching at the SAME index keep `needles` order (Array#sort
|
|
234
|
+
* is stable), so first-match-wins is deterministic.
|
|
229
235
|
* @param {string} text
|
|
230
236
|
* @param {string[]} needles
|
|
231
237
|
* @returns {{text: string, index: number}[]}
|
|
@@ -238,6 +244,47 @@ export function orderedMatches(text, needles) {
|
|
|
238
244
|
return out.sort((left, right) => left.index - right.index);
|
|
239
245
|
}
|
|
240
246
|
|
|
247
|
+
/**
|
|
248
|
+
* Replace every match in `matches` with `replacementFor(match, i)` in a SINGLE
|
|
249
|
+
* ordered pass over `text`. THE splice primitive for this codebase — the sole
|
|
250
|
+
* sound way to substitute several needles at once.
|
|
251
|
+
*
|
|
252
|
+
* A chained `text.split(needle).join(value)` per needle is unsound in both
|
|
253
|
+
* directions, which is why no caller may hand-roll one:
|
|
254
|
+
* - substitution: an inserted value whose bytes contain a LATER needle is
|
|
255
|
+
* re-matched by the next split and corrupted (or partially exposed);
|
|
256
|
+
* - deletion: an earlier deletion joins the bytes on either side of it and
|
|
257
|
+
* can CREATE a later needle's match, deleting text that needle never
|
|
258
|
+
* matched in the input ("PRE-XX-POST" minus "-XX-" yields "PREPOST").
|
|
259
|
+
* Because every index in `matches` is measured against the original `text`,
|
|
260
|
+
* this pass only ever touches bytes the caller actually matched.
|
|
261
|
+
*
|
|
262
|
+
* Overlapping matches are resolved first-match-wins: a match starting before
|
|
263
|
+
* the previous one ended is skipped, never spliced at a shifted offset.
|
|
264
|
+
* `i` is the match's index in `matches` (stable across skips) so a caller
|
|
265
|
+
* pairing matches positionally with its own array stays aligned.
|
|
266
|
+
* @param {string} text
|
|
267
|
+
* @param {{text: string, index: number}[]} matches ordered by index, indices into `text`
|
|
268
|
+
* @param {(match: {text: string, index: number}, i: number) => string} replacementFor
|
|
269
|
+
* @returns {{text: string, spans: {start: number, end: number}[]}} spliced text
|
|
270
|
+
* and the [start, end) range each replacement occupies in it
|
|
271
|
+
*/
|
|
272
|
+
export function spliceOrdered(text, matches, replacementFor) {
|
|
273
|
+
let out = "";
|
|
274
|
+
let last = 0;
|
|
275
|
+
/** @type {{start: number, end: number}[]} */
|
|
276
|
+
const spans = [];
|
|
277
|
+
matches.forEach((match, i) => {
|
|
278
|
+
if (match.index < last) return;
|
|
279
|
+
out += text.slice(last, match.index);
|
|
280
|
+
const start = out.length;
|
|
281
|
+
out += replacementFor(match, i);
|
|
282
|
+
spans.push({ start, end: out.length });
|
|
283
|
+
last = match.index + match.text.length;
|
|
284
|
+
});
|
|
285
|
+
return { text: out + text.slice(last), spans };
|
|
286
|
+
}
|
|
287
|
+
|
|
241
288
|
/**
|
|
242
289
|
* On-disk [start, end) span of every redaction pair, mapped from its view
|
|
243
290
|
* offset through placeholder expansion (view → cleaned) and stripped invisible
|
|
@@ -308,24 +355,16 @@ export function rehydrateNewString(oldS, newS, spanPairs, filePairs) {
|
|
|
308
355
|
newSeq.length === spanPairs.length &&
|
|
309
356
|
newSeq.every((match, i) => match.text === spanPairs[i].placeholder)
|
|
310
357
|
) {
|
|
311
|
-
let out = "";
|
|
312
|
-
let last = 0;
|
|
313
|
-
newSeq.forEach((match, i) => {
|
|
314
|
-
out += newS.slice(last, match.index) + spanPairs[i].original;
|
|
315
|
-
last = match.index + match.text.length;
|
|
316
|
-
});
|
|
317
358
|
return {
|
|
318
|
-
text:
|
|
359
|
+
text: spliceOrdered(newS, newSeq, (_match, i) => spanPairs[i].original)
|
|
360
|
+
.text,
|
|
319
361
|
secrets: spanPairs.map((pair) => pair.original),
|
|
320
362
|
};
|
|
321
363
|
}
|
|
322
364
|
|
|
323
365
|
// Each placeholder text must name exactly one secret; resolve that mapping
|
|
324
|
-
// first, then splice in a SINGLE ordered pass
|
|
325
|
-
// `out.split(ph).join(secret)` per placeholder is unsound
|
|
326
|
-
// whose bytes happen to contain a later placeholder text would be re-matched
|
|
327
|
-
// and corrupted (or partially exposed) by the next split. One pass over the
|
|
328
|
-
// ordered match positions only ever touches the original new_string bytes.
|
|
366
|
+
// first, then splice in a SINGLE ordered pass (see spliceOrdered for why a
|
|
367
|
+
// chained `out.split(ph).join(secret)` per placeholder is unsound).
|
|
329
368
|
const valueByPh = new Map();
|
|
330
369
|
for (const phText of new Set(newSeq.map((match) => match.text))) {
|
|
331
370
|
const values = [
|
|
@@ -344,11 +383,9 @@ export function rehydrateNewString(oldS, newS, spanPairs, filePairs) {
|
|
|
344
383
|
};
|
|
345
384
|
valueByPh.set(phText, values[0]);
|
|
346
385
|
}
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
}
|
|
353
|
-
return { text: out + newS.slice(last), secrets: [...valueByPh.values()] };
|
|
386
|
+
return {
|
|
387
|
+
text: spliceOrdered(newS, newSeq, (match) => valueByPh.get(match.text))
|
|
388
|
+
.text,
|
|
389
|
+
secrets: [...valueByPh.values()],
|
|
390
|
+
};
|
|
354
391
|
}
|
package/types/output.d.mts
CHANGED
|
@@ -26,8 +26,21 @@ export function describeWarned(warned: {
|
|
|
26
26
|
/**
|
|
27
27
|
* Delete each verbatim span in `spans` from `text`. The secure Layer-5
|
|
28
28
|
* primitive: a filter can only ask for deletions, so this can never inject
|
|
29
|
-
* bytes. Returns the new text and how many
|
|
30
|
-
*
|
|
29
|
+
* bytes. Returns the new text and how many span occurrences were removed (0
|
|
30
|
+
* when no span was present).
|
|
31
|
+
*
|
|
32
|
+
* Every occurrence is located in the ORIGINAL `text` and the deletions applied
|
|
33
|
+
* in one ordered pass ({@link spliceOrdered}), so every removed byte lies inside
|
|
34
|
+
* a match some span had in the INPUT. Deleting span-by-span with a
|
|
35
|
+
* chained `split`/`join` would not hold that line: an earlier deletion joins the
|
|
36
|
+
* bytes on either side of it and can CREATE a match for a later span that never
|
|
37
|
+
* occurred in the input — `deleteVerbatimSpans("PRE-XX-POST", ["-XX-", "PREPOST"])`
|
|
38
|
+
* then deletes the whole document. That would widen the Layer-5 seam's blast
|
|
39
|
+
* radius (see the module doc) from "a compromised filter can at most remove the
|
|
40
|
+
* content it named" to "it can remove content it never named".
|
|
41
|
+
*
|
|
42
|
+
* Overlapping spans are resolved first-match-wins, so `removed` counts the
|
|
43
|
+
* occurrences actually spliced out, never a double-count of the same bytes.
|
|
31
44
|
* @param {string} text
|
|
32
45
|
* @param {string[]} spans
|
|
33
46
|
* @returns {{ text: string, removed: number }}
|
package/types/view-map.d.mts
CHANGED
|
@@ -106,9 +106,15 @@ export function resolveSpan(content: string, cleaned: string, view: {
|
|
|
106
106
|
}[];
|
|
107
107
|
} | null;
|
|
108
108
|
/**
|
|
109
|
-
* All occurrences of any needle in `text`, ordered by position.
|
|
110
|
-
*
|
|
111
|
-
*
|
|
109
|
+
* All occurrences of any needle in `text`, ordered by position. Every index is
|
|
110
|
+
* computed against the ORIGINAL `text`, so the caller can splice them in one
|
|
111
|
+
* pass ({@link spliceOrdered}). Redaction placeholder texts never
|
|
112
|
+
* substring-overlap one another (each ends in "]" right after its
|
|
113
|
+
* distinguishing label), so for that caller the sorted matches are also
|
|
114
|
+
* non-overlapping; needles from an untrusted source (a Layer-5 filter's
|
|
115
|
+
* removeSpans) can overlap, which spliceOrdered resolves first-match-wins.
|
|
116
|
+
* Distinct needles matching at the SAME index keep `needles` order (Array#sort
|
|
117
|
+
* is stable), so first-match-wins is deterministic.
|
|
112
118
|
* @param {string} text
|
|
113
119
|
* @param {string[]} needles
|
|
114
120
|
* @returns {{text: string, index: number}[]}
|
|
@@ -117,6 +123,44 @@ export function orderedMatches(text: string, needles: string[]): {
|
|
|
117
123
|
text: string;
|
|
118
124
|
index: number;
|
|
119
125
|
}[];
|
|
126
|
+
/**
|
|
127
|
+
* Replace every match in `matches` with `replacementFor(match, i)` in a SINGLE
|
|
128
|
+
* ordered pass over `text`. THE splice primitive for this codebase — the sole
|
|
129
|
+
* sound way to substitute several needles at once.
|
|
130
|
+
*
|
|
131
|
+
* A chained `text.split(needle).join(value)` per needle is unsound in both
|
|
132
|
+
* directions, which is why no caller may hand-roll one:
|
|
133
|
+
* - substitution: an inserted value whose bytes contain a LATER needle is
|
|
134
|
+
* re-matched by the next split and corrupted (or partially exposed);
|
|
135
|
+
* - deletion: an earlier deletion joins the bytes on either side of it and
|
|
136
|
+
* can CREATE a later needle's match, deleting text that needle never
|
|
137
|
+
* matched in the input ("PRE-XX-POST" minus "-XX-" yields "PREPOST").
|
|
138
|
+
* Because every index in `matches` is measured against the original `text`,
|
|
139
|
+
* this pass only ever touches bytes the caller actually matched.
|
|
140
|
+
*
|
|
141
|
+
* Overlapping matches are resolved first-match-wins: a match starting before
|
|
142
|
+
* the previous one ended is skipped, never spliced at a shifted offset.
|
|
143
|
+
* `i` is the match's index in `matches` (stable across skips) so a caller
|
|
144
|
+
* pairing matches positionally with its own array stays aligned.
|
|
145
|
+
* @param {string} text
|
|
146
|
+
* @param {{text: string, index: number}[]} matches ordered by index, indices into `text`
|
|
147
|
+
* @param {(match: {text: string, index: number}, i: number) => string} replacementFor
|
|
148
|
+
* @returns {{text: string, spans: {start: number, end: number}[]}} spliced text
|
|
149
|
+
* and the [start, end) range each replacement occupies in it
|
|
150
|
+
*/
|
|
151
|
+
export function spliceOrdered(text: string, matches: {
|
|
152
|
+
text: string;
|
|
153
|
+
index: number;
|
|
154
|
+
}[], replacementFor: (match: {
|
|
155
|
+
text: string;
|
|
156
|
+
index: number;
|
|
157
|
+
}, i: number) => string): {
|
|
158
|
+
text: string;
|
|
159
|
+
spans: {
|
|
160
|
+
start: number;
|
|
161
|
+
end: number;
|
|
162
|
+
}[];
|
|
163
|
+
};
|
|
120
164
|
/**
|
|
121
165
|
* On-disk [start, end) span of every redaction pair, mapped from its view
|
|
122
166
|
* offset through placeholder expansion (view → cleaned) and stripped invisible
|