agent-sanitizer 2.30.0 → 2.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/THREAT-MODEL.md +40 -19
- package/claude-hooks/lib/placeholder-grammar.mjs +162 -84
- package/claude-hooks/lib/reveal.mjs +129 -5
- package/claude-hooks/plugin-hooks.mjs +1 -0
- package/claude-hooks/pretooluse-sanitize.mjs +159 -2
- package/claude-hooks/sanitize-output.mjs +76 -23
- package/package.json +1 -1
- package/src/gates.mjs +5 -3
- package/src/html.mjs +123 -30
- package/src/index.mjs +32 -18
- package/src/output.mjs +73 -17
- package/types/claude-hooks/lib/placeholder-grammar.d.mts +75 -29
- package/types/claude-hooks/lib/reveal.d.mts +46 -0
- package/types/claude-hooks/pretooluse-sanitize.d.mts +35 -0
- package/types/claude-hooks/sanitize-output.d.mts +17 -3
- package/types/gates.d.mts +5 -3
- package/types/html.d.mts +74 -24
- package/types/index.d.mts +19 -10
- package/types/output.d.mts +24 -3
package/src/html.mjs
CHANGED
|
@@ -10,6 +10,16 @@
|
|
|
10
10
|
* and `data:` URI resources are REPORTED in the result's `warned` counts but
|
|
11
11
|
* never removed, so fetched page source stays inspectable.
|
|
12
12
|
*
|
|
13
|
+
* Every splice is ROUND-TRIPPABLE: the placeholder carries a content-addressed
|
|
14
|
+
* key (see {@link layer2Placeholder}) and `sanitizeHtml` returns a `splices`
|
|
15
|
+
* array pairing each placeholder with the original bytes it replaced, so a
|
|
16
|
+
* caller that must write the text back (an agent editing a PR body whose
|
|
17
|
+
* comments were spliced) can rehydrate instead of persisting the loss —
|
|
18
|
+
* comments are ubiquitous legitimate content (PR templates, tooling markers),
|
|
19
|
+
* and an earlier lossy splice corrupted real documents (#244). The splice
|
|
20
|
+
* itself stays: comments hide content from a human reading the rendered page,
|
|
21
|
+
* which is the exact channel this layer exists to close.
|
|
22
|
+
*
|
|
13
23
|
* Layer 3 reports data-exfil-shaped URLs (suspicious query params, oversized
|
|
14
24
|
* payloads, embedded credentials) without modifying them; the caller surfaces
|
|
15
25
|
* the report as a warning.
|
|
@@ -18,6 +28,7 @@
|
|
|
18
28
|
* remark/rehype/unified graph costs ~200ms of module-load time, so the main
|
|
19
29
|
* entry `await import()`s this module only when its cheap regex gates match.
|
|
20
30
|
*/
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
21
32
|
// @ts-ignore -- css-tree ships no bundled types and @types/css-tree lags the 3.x
|
|
22
33
|
// API (e.g. `ident.decode`); the value AST is walked with local `any` types.
|
|
23
34
|
import * as csstree from "css-tree";
|
|
@@ -1284,8 +1295,65 @@ export function closingTagName(htmlValue) {
|
|
|
1284
1295
|
|
|
1285
1296
|
// ─── Layer 2: splice engine ──────────────────────────────────────────────────
|
|
1286
1297
|
|
|
1287
|
-
|
|
1288
|
-
|
|
1298
|
+
/**
|
|
1299
|
+
* @typedef {"comment" | "hidden"} SpliceKind
|
|
1300
|
+
* @typedef {{ start: number, end: number, kind: SpliceKind }} SpliceRange
|
|
1301
|
+
* @typedef {{ placeholder: string, original: string, start: number }} SplicePair
|
|
1302
|
+
* One splice: the keyed placeholder now in the output text, the ORIGINAL
|
|
1303
|
+
* bytes it replaced, and the placeholder's start offset in the RETURNED text.
|
|
1304
|
+
* All offsets in this module — unist positions and these — are plain JS
|
|
1305
|
+
* string indices, i.e. UTF-16 code units.
|
|
1306
|
+
*/
|
|
1307
|
+
|
|
1308
|
+
// The kind-specific prose of a Layer-2 placeholder. The full placeholder is
|
|
1309
|
+
// built by {@link layer2Placeholder} and matched by {@link LAYER2_PLACEHOLDER_RE};
|
|
1310
|
+
// keep all three in sync — they are ONE grammar shared with the rehydrating
|
|
1311
|
+
// hooks and the tests.
|
|
1312
|
+
const PLACEHOLDER_LABEL = Object.freeze({
|
|
1313
|
+
hidden: "hidden HTML",
|
|
1314
|
+
comment: "HTML comment",
|
|
1315
|
+
});
|
|
1316
|
+
|
|
1317
|
+
// How many lowercase-hex chars of the sha256 make the placeholder key. 48 bits
|
|
1318
|
+
// is far past accidental-collision range for the handful of splices one
|
|
1319
|
+
// document carries, while keeping the placeholder short enough to read.
|
|
1320
|
+
const PLACEHOLDER_KEY_LEN = 12;
|
|
1321
|
+
|
|
1322
|
+
/**
|
|
1323
|
+
* The keyed, content-addressed placeholder for one Layer-2 splice:
|
|
1324
|
+
* `[hidden HTML removed #<key>]` / `[HTML comment removed #<key>]`, where
|
|
1325
|
+
* `<key>` is the first 12 lowercase-hex chars of sha256 over the UTF-8
|
|
1326
|
+
* encoding of the ORIGINAL spliced text. Content-addressed on purpose:
|
|
1327
|
+
* identical spliced content yields the identical placeholder, so a rehydrator
|
|
1328
|
+
* can match placeholder → original by key alone, and duplicated content never
|
|
1329
|
+
* produces conflicting keys.
|
|
1330
|
+
* @param {SpliceKind} kind
|
|
1331
|
+
* @param {string} original the exact text the splice removed
|
|
1332
|
+
* @returns {string}
|
|
1333
|
+
*/
|
|
1334
|
+
export function layer2Placeholder(kind, original) {
|
|
1335
|
+
const key = createHash("sha256")
|
|
1336
|
+
.update(original, "utf8")
|
|
1337
|
+
.digest("hex")
|
|
1338
|
+
.slice(0, PLACEHOLDER_KEY_LEN);
|
|
1339
|
+
return `[${PLACEHOLDER_LABEL[kind]} removed #${key}]`;
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
/**
|
|
1343
|
+
* The single grammar definition for keyed Layer-2 placeholders — the exact
|
|
1344
|
+
* output of {@link layer2Placeholder}, capture group 1 = the key. Global so
|
|
1345
|
+
* callers can scan a document for every placeholder; reset `lastIndex` (or
|
|
1346
|
+
* use `matchAll`) between uses.
|
|
1347
|
+
*/
|
|
1348
|
+
export const LAYER2_PLACEHOLDER_RE =
|
|
1349
|
+
/\[(?:hidden HTML|HTML comment) removed #([0-9a-f]{12})\]/g;
|
|
1350
|
+
|
|
1351
|
+
// DEPRECATED un-keyed placeholder PREFIXES, kept exported for callers that
|
|
1352
|
+
// match "some Layer-2 placeholder of this kind" without knowing the key. The
|
|
1353
|
+
// full placeholder is keyed — build it with {@link layer2Placeholder}, match it
|
|
1354
|
+
// with {@link LAYER2_PLACEHOLDER_RE}.
|
|
1355
|
+
export const HIDDEN_PLACEHOLDER = "[hidden HTML removed";
|
|
1356
|
+
export const COMMENT_PLACEHOLDER = "[HTML comment removed";
|
|
1289
1357
|
// Shown when the remark/rehype parse itself fails (e.g. pathologically nested
|
|
1290
1358
|
// markup overflows the recursive tree walk with a RangeError). The top-level
|
|
1291
1359
|
// `sanitize`/`sanitizeText` contract is "never throws, `cleaned` is always a
|
|
@@ -1297,18 +1365,23 @@ export const HIDDEN_PLACEHOLDER = "[hidden HTML removed]";
|
|
|
1297
1365
|
export const UNPARSEABLE_PLACEHOLDER = "[HTML unparseable — withheld]";
|
|
1298
1366
|
|
|
1299
1367
|
/**
|
|
1300
|
-
* Replace each range of `text` with its kind's placeholder, preserving
|
|
1301
|
-
* byte outside the ranges verbatim. Overlapping/nested ranges are merged
|
|
1368
|
+
* Replace each range of `text` with its kind's keyed placeholder, preserving
|
|
1369
|
+
* every byte outside the ranges verbatim. Overlapping/nested ranges are merged
|
|
1302
1370
|
* (defense-in-depth — the scanners emit disjoint ranges).
|
|
1371
|
+
*
|
|
1372
|
+
* Returns the spliced text plus `pairs`, one per emitted placeholder in output
|
|
1373
|
+
* order, each pairing the placeholder with the ORIGINAL bytes it replaced and
|
|
1374
|
+
* its start offset in the RETURNED text (UTF-16 code-unit string indices, the
|
|
1375
|
+
* same space as `ranges`) — everything a rehydrator needs to undo the splice.
|
|
1303
1376
|
* @param {string} text
|
|
1304
|
-
* @param {
|
|
1305
|
-
* @returns {string}
|
|
1377
|
+
* @param {SpliceRange[]} ranges
|
|
1378
|
+
* @returns {{ text: string, pairs: SplicePair[] }}
|
|
1306
1379
|
*/
|
|
1307
1380
|
export function spliceRanges(text, ranges) {
|
|
1308
1381
|
const sorted = [...ranges].sort(
|
|
1309
1382
|
(left, right) => left.start - right.start || left.end - right.end,
|
|
1310
1383
|
);
|
|
1311
|
-
/** @type {
|
|
1384
|
+
/** @type {SpliceRange[]} */
|
|
1312
1385
|
const merged = [];
|
|
1313
1386
|
for (const range of sorted) {
|
|
1314
1387
|
const last = merged[merged.length - 1];
|
|
@@ -1316,8 +1389,8 @@ export function spliceRanges(text, ranges) {
|
|
|
1316
1389
|
if (range.end > last.end) last.end = range.end;
|
|
1317
1390
|
// A hidden range absorbed into a comment range (the comment sorts first
|
|
1318
1391
|
// on a tie) must keep the hidden label — hidden content placeholdered as
|
|
1319
|
-
// "[HTML comment removed]" would understate what was stripped.
|
|
1320
|
-
// dominates: if either side is hidden, the union is hidden.
|
|
1392
|
+
// an "[HTML comment removed …]" would understate what was stripped.
|
|
1393
|
+
// Hidden dominates: if either side is hidden, the union is hidden.
|
|
1321
1394
|
if (range.kind === "hidden") last.kind = "hidden";
|
|
1322
1395
|
} else {
|
|
1323
1396
|
merged.push({ ...range });
|
|
@@ -1325,13 +1398,17 @@ export function spliceRanges(text, ranges) {
|
|
|
1325
1398
|
}
|
|
1326
1399
|
let out = "";
|
|
1327
1400
|
let cursor = 0;
|
|
1401
|
+
/** @type {SplicePair[]} */
|
|
1402
|
+
const pairs = [];
|
|
1328
1403
|
for (const range of merged) {
|
|
1329
|
-
out +=
|
|
1330
|
-
|
|
1331
|
-
|
|
1404
|
+
out += text.slice(cursor, range.start);
|
|
1405
|
+
const original = text.slice(range.start, range.end);
|
|
1406
|
+
const placeholder = layer2Placeholder(range.kind, original);
|
|
1407
|
+
pairs.push({ placeholder, original, start: out.length });
|
|
1408
|
+
out += placeholder;
|
|
1332
1409
|
cursor = range.end;
|
|
1333
1410
|
}
|
|
1334
|
-
return out + text.slice(cursor);
|
|
1411
|
+
return { text: out + text.slice(cursor), pairs };
|
|
1335
1412
|
}
|
|
1336
1413
|
|
|
1337
1414
|
/** @returns {{ tags: Record<string, number>, dataSrc: number }} */
|
|
@@ -1369,7 +1446,7 @@ function hasWarned(warned) {
|
|
|
1369
1446
|
* through matching close, and parse5 extends an unclosed element to the end
|
|
1370
1447
|
* of the fragment — fail-closed for truncated markup).
|
|
1371
1448
|
* @param {string} html
|
|
1372
|
-
* @returns {{ ranges:
|
|
1449
|
+
* @returns {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }}
|
|
1373
1450
|
*/
|
|
1374
1451
|
export function scanHtmlFragment(html) {
|
|
1375
1452
|
return scanFragmentTree(html, parseFragment(html));
|
|
@@ -1382,10 +1459,10 @@ export function scanHtmlFragment(html) {
|
|
|
1382
1459
|
* the ranges are offsets into `html`, read from that tree's positions.
|
|
1383
1460
|
* @param {string} html
|
|
1384
1461
|
* @param {any} tree
|
|
1385
|
-
* @returns {{ ranges:
|
|
1462
|
+
* @returns {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }}
|
|
1386
1463
|
*/
|
|
1387
1464
|
function scanFragmentTree(html, tree) {
|
|
1388
|
-
/** @type {
|
|
1465
|
+
/** @type {SpliceRange[]} */
|
|
1389
1466
|
const ranges = [];
|
|
1390
1467
|
const warned = newWarned();
|
|
1391
1468
|
// @ts-ignore -- visit callback returns EXIT/SKIP only on matches; implicit undefined return is intentional
|
|
@@ -1491,7 +1568,7 @@ function commentSpans(value) {
|
|
|
1491
1568
|
* @param {string} value
|
|
1492
1569
|
* @param {number} base absolute offset of the start of `value`
|
|
1493
1570
|
* @param {number} nodeEnd absolute offset of the end of the containing node
|
|
1494
|
-
* @param {
|
|
1571
|
+
* @param {SpliceRange[]} ranges
|
|
1495
1572
|
*/
|
|
1496
1573
|
function collectCommentRanges(value, base, nodeEnd, ranges) {
|
|
1497
1574
|
BOGUS_COMMENT_OPEN_RE.lastIndex = 0;
|
|
@@ -1541,7 +1618,7 @@ function collectCommentRanges(value, base, nodeEnd, ranges) {
|
|
|
1541
1618
|
* @param {{ tag: string | null, depth: number, regionStart: number }} state
|
|
1542
1619
|
* @param {string} value
|
|
1543
1620
|
* @param {number} nodeEnd absolute end offset of this node
|
|
1544
|
-
* @param {
|
|
1621
|
+
* @param {SpliceRange[]} ranges
|
|
1545
1622
|
*/
|
|
1546
1623
|
function updateHiddenState(state, value, nodeEnd, ranges) {
|
|
1547
1624
|
if (value.startsWith("</")) {
|
|
@@ -1603,7 +1680,7 @@ function hasHtmlLeaf(node) {
|
|
|
1603
1680
|
* sibling/nested node the way it does in the flat token stream.
|
|
1604
1681
|
* @param {any} node
|
|
1605
1682
|
* @param {string} text the full document source, for raw-slice absorb folding
|
|
1606
|
-
* @param {
|
|
1683
|
+
* @param {SpliceRange[]} ranges
|
|
1607
1684
|
* @param {ReturnType<typeof newWarned>} warned
|
|
1608
1685
|
*/
|
|
1609
1686
|
function scanInlineChildren(node, text, ranges, warned) {
|
|
@@ -1697,11 +1774,11 @@ const FLOW_HTML_PARENTS = new Set([
|
|
|
1697
1774
|
|
|
1698
1775
|
/**
|
|
1699
1776
|
* @param {string} text
|
|
1700
|
-
* @returns {{ ranges:
|
|
1777
|
+
* @returns {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }}
|
|
1701
1778
|
*/
|
|
1702
1779
|
function scanMarkdown(text) {
|
|
1703
1780
|
const tree = mdParser.parse(text);
|
|
1704
|
-
/** @type {
|
|
1781
|
+
/** @type {SpliceRange[]} */
|
|
1705
1782
|
const ranges = [];
|
|
1706
1783
|
const warned = newWarned();
|
|
1707
1784
|
|
|
@@ -1812,18 +1889,30 @@ export function looksLikeHtmlSource(text) {
|
|
|
1812
1889
|
|
|
1813
1890
|
/**
|
|
1814
1891
|
* Layer 2 over web-ingress text: splice out HTML comments and hidden elements
|
|
1815
|
-
* (placeholders mark the cuts; all other bytes are preserved verbatim)
|
|
1816
|
-
* count preserved scripting/resource tags for the caller's warning. Returns
|
|
1817
|
-
* null when there is nothing to strip and nothing to report.
|
|
1818
|
-
*
|
|
1819
|
-
*
|
|
1820
|
-
*
|
|
1892
|
+
* (keyed placeholders mark the cuts; all other bytes are preserved verbatim)
|
|
1893
|
+
* and count preserved scripting/resource tags for the caller's warning. Returns
|
|
1894
|
+
* null when there is nothing to strip and nothing to report.
|
|
1895
|
+
*
|
|
1896
|
+
* `splices` pairs every emitted placeholder with the original bytes it
|
|
1897
|
+
* replaced (see {@link spliceRanges}), so a caller can rehydrate the text —
|
|
1898
|
+
* nothing is lost, only hidden behind an identity-carrying placeholder.
|
|
1899
|
+
*
|
|
1900
|
+
* `unparseable` is set (true) only on the fail-closed path below, where the
|
|
1901
|
+
* whole input was withheld behind {@link UNPARSEABLE_PLACEHOLDER} rather than
|
|
1902
|
+
* spliced — the caller's warning must describe a whole-output withhold, not a
|
|
1903
|
+
* splice. There `splices` is `[]`: the parser blew up before any span could be
|
|
1904
|
+
* located, so nothing is recoverable per-splice (the caller's pre-splice
|
|
1905
|
+
* `reveal` is the only copy).
|
|
1906
|
+
*
|
|
1907
|
+
* Idempotent over its own output: a keyed placeholder contains no `<`, so a
|
|
1908
|
+
* re-run neither gates on it (HTML_TAG_PRESENT needs a tag) nor reads it as
|
|
1909
|
+
* markup — placeholders already in the text pass through byte-identical.
|
|
1821
1910
|
* @param {string} text
|
|
1822
|
-
* @returns {{ text: string, removed: { comments: number, hidden: number }, warned: { tags: Record<string, number>, dataSrc: number }, unparseable?: true } | null}
|
|
1911
|
+
* @returns {{ text: string, removed: { comments: number, hidden: number }, warned: { tags: Record<string, number>, dataSrc: number }, splices: SplicePair[], unparseable?: true } | null}
|
|
1823
1912
|
*/
|
|
1824
1913
|
export function sanitizeHtml(text) {
|
|
1825
1914
|
if (!HTML_TAG_PRESENT.test(text)) return null;
|
|
1826
|
-
/** @type {{ ranges:
|
|
1915
|
+
/** @type {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }} */
|
|
1827
1916
|
let scan;
|
|
1828
1917
|
try {
|
|
1829
1918
|
// One parse decides the branch AND feeds it, so the source branch does not
|
|
@@ -1839,6 +1928,7 @@ export function sanitizeHtml(text) {
|
|
|
1839
1928
|
text: UNPARSEABLE_PLACEHOLDER,
|
|
1840
1929
|
removed: { comments: 0, hidden: 1 },
|
|
1841
1930
|
warned: newWarned(),
|
|
1931
|
+
splices: [],
|
|
1842
1932
|
unparseable: true,
|
|
1843
1933
|
};
|
|
1844
1934
|
}
|
|
@@ -1847,10 +1937,13 @@ export function sanitizeHtml(text) {
|
|
|
1847
1937
|
const removed = { comments: 0, hidden: 0 };
|
|
1848
1938
|
for (const range of ranges)
|
|
1849
1939
|
removed[range.kind === "comment" ? "comments" : "hidden"]++;
|
|
1940
|
+
const spliced =
|
|
1941
|
+
ranges.length > 0 ? spliceRanges(text, ranges) : { text, pairs: [] };
|
|
1850
1942
|
return {
|
|
1851
|
-
text:
|
|
1943
|
+
text: spliced.text,
|
|
1852
1944
|
removed,
|
|
1853
1945
|
warned,
|
|
1946
|
+
splices: spliced.pairs,
|
|
1854
1947
|
};
|
|
1855
1948
|
}
|
|
1856
1949
|
|
package/src/index.mjs
CHANGED
|
@@ -84,29 +84,43 @@ export {
|
|
|
84
84
|
*
|
|
85
85
|
* The layer bodies live in `./output.mjs`; this is a facade over them, not a
|
|
86
86
|
* second implementation (see the module doc). It narrows `sanitizeText`'s result
|
|
87
|
-
* to the
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
* Layer
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
87
|
+
* to the fields this entry promises — `modified`/`sgrNote` describe the
|
|
88
|
+
* tool-output pipeline's banner, and `reveal` is produced only by options this
|
|
89
|
+
* facade does not expose. `splices` IS passed through (present only when Layer 2
|
|
90
|
+
* spliced): the placeholder→original pairs a caller needs to rehydrate keyed
|
|
91
|
+
* Layer-2 placeholders — same field, same shape as `sanitizeText`'s, since this
|
|
92
|
+
* facade wraps the same layers (grammar in `./html.mjs`: `layer2Placeholder` /
|
|
93
|
+
* `LAYER2_PLACEHOLDER_RE`). `html` selects Layers 2 AND 3 together here, which
|
|
94
|
+
* is the surface this entry has always had; `exfilScan` exposes Layer 3's
|
|
95
|
+
* non-destructive detection on its own (unconditionally implied by `html`,
|
|
96
|
+
* which it can add to but never switch off) for callers that must keep the
|
|
97
|
+
* visible bytes intact — e.g. a PR diff where the Layer-2 splice would corrupt
|
|
98
|
+
* legitimate markup — matching the separate flags `sanitizeText` takes for the
|
|
99
|
+
* tool-output pipeline, which needs Layer 3's detection without Layer 2's
|
|
100
|
+
* splice.
|
|
96
101
|
* @param {string} text
|
|
97
102
|
* @param {{ html?: boolean, exfilScan?: boolean } | null} [options]
|
|
98
|
-
* @returns {Promise<{ cleaned: string, found: string[], warnings: string[], notes: string[] }>}
|
|
103
|
+
* @returns {Promise<{ cleaned: string, found: string[], warnings: string[], notes: string[], splices?: Array<{ placeholder: string, original: string }> }>}
|
|
99
104
|
*/
|
|
100
105
|
export async function sanitize(text, options) {
|
|
101
106
|
if (typeof text !== "string")
|
|
102
107
|
throw new TypeError("sanitize(text, options): text must be a string");
|
|
103
108
|
const { html = false, exfilScan = false } = options ?? {};
|
|
104
|
-
const { cleaned, found, warnings, notes } = await sanitizeText(
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
const { cleaned, found, warnings, notes, splices } = await sanitizeText(
|
|
110
|
+
text,
|
|
111
|
+
{
|
|
112
|
+
html,
|
|
113
|
+
// `html` implies the scan unconditionally, and `exfilScan` can only ADD it:
|
|
114
|
+
// an opt-OUT would make `{ html: true, exfilScan: false }` splice Layer 2
|
|
115
|
+
// while silently dropping Layer 3's report — a fail-open the docs deny.
|
|
116
|
+
exfilScan: exfilScan || html,
|
|
117
|
+
},
|
|
118
|
+
);
|
|
119
|
+
return {
|
|
120
|
+
cleaned,
|
|
121
|
+
found,
|
|
122
|
+
warnings,
|
|
123
|
+
notes,
|
|
124
|
+
...(splices !== undefined && { splices }),
|
|
125
|
+
};
|
|
112
126
|
}
|
package/src/output.mjs
CHANGED
|
@@ -386,19 +386,25 @@ function processLayer1(text, sgrCarveOut) {
|
|
|
386
386
|
* folded into `state`. Returns the pre-splice text when Layer 2 removed bytes so
|
|
387
387
|
* the caller can hand it back for later inspection of what the splice hid (the
|
|
388
388
|
* model cannot otherwise tell a benign `<!-- TODO -->` from an injection
|
|
389
|
-
* payload), and `undefined` otherwise
|
|
390
|
-
*
|
|
391
|
-
*
|
|
389
|
+
* payload), and `undefined` otherwise — plus Layer 2's `splices`, the
|
|
390
|
+
* placeholder→original pairs a rehydrator needs (in document order; the
|
|
391
|
+
* per-splice offsets are dropped here because later layers may mutate the text,
|
|
392
|
+
* making offsets into this stage's text meaningless). Both are STAGE VALUES,
|
|
393
|
+
* not results: neither has been through Layer 4, so {@link sanitizeText} must
|
|
394
|
+
* vet them before they leave. The transform itself stays pure — the caller owns
|
|
395
|
+
* any persistence.
|
|
392
396
|
* @param {PipelineState} state
|
|
393
397
|
* @param {{ html?: boolean, exfilScan?: boolean }} options
|
|
394
|
-
* @returns {Promise<string | undefined
|
|
398
|
+
* @returns {Promise<{ reveal: string | undefined, splices: Array<{ placeholder: string, original: string }> }>}
|
|
395
399
|
*/
|
|
396
400
|
async function applyMarkdownPipeline(state, { html, exfilScan }) {
|
|
397
401
|
const inputText = state.text;
|
|
398
402
|
/** @type {string | undefined} */
|
|
399
403
|
let reveal;
|
|
404
|
+
/** @type {Array<{ placeholder: string, original: string }>} */
|
|
405
|
+
const splices = [];
|
|
400
406
|
if ((!html && !exfilScan) || !needsMarkdownPipeline(inputText))
|
|
401
|
-
return undefined;
|
|
407
|
+
return { reveal: undefined, splices };
|
|
402
408
|
let sanitizeHtml, detectExfil;
|
|
403
409
|
/* c8 ignore start -- a rejected dynamic import of a module that ships in
|
|
404
410
|
this very package (not an optional peer dep) requires corrupting
|
|
@@ -417,12 +423,17 @@ async function applyMarkdownPipeline(state, { html, exfilScan }) {
|
|
|
417
423
|
}
|
|
418
424
|
/* c8 ignore stop */
|
|
419
425
|
// Layer 2 — strips what a rendered page would not show (comments, hidden
|
|
420
|
-
// elements); scripting/resource tags preserved+reported.
|
|
426
|
+
// elements); scripting/resource tags preserved+reported. Each cut leaves a
|
|
427
|
+
// keyed placeholder whose original bytes ride out in `splices`.
|
|
421
428
|
if (html) {
|
|
422
429
|
const layer2 = sanitizeHtml(state.text);
|
|
423
430
|
if (layer2) {
|
|
424
431
|
if (layer2.text !== state.text) {
|
|
425
432
|
reveal = state.text;
|
|
433
|
+
// Keep only {placeholder, original}: the offsets sanitizeHtml returns
|
|
434
|
+
// point into THIS stage's text, which Layers 4/5 may still mutate.
|
|
435
|
+
for (const { placeholder, original } of layer2.splices)
|
|
436
|
+
splices.push({ placeholder, original });
|
|
426
437
|
applyMutation(state, layer2.text);
|
|
427
438
|
if (layer2.removed.comments > 0)
|
|
428
439
|
state.found.push(CATEGORY.HTML_COMMENTS);
|
|
@@ -472,18 +483,19 @@ async function applyMarkdownPipeline(state, { html, exfilScan }) {
|
|
|
472
483
|
);
|
|
473
484
|
}
|
|
474
485
|
}
|
|
475
|
-
return reveal;
|
|
486
|
+
return { reveal, splices };
|
|
476
487
|
}
|
|
477
488
|
|
|
478
489
|
/**
|
|
479
490
|
* Vet a pipeline STAGE value on its way out of {@link sanitizeText}. Only
|
|
480
491
|
* `cleaned` traverses every layer; anything else a caller is handed (today the
|
|
481
492
|
* Layer-2 `reveal`) is a snapshot from the middle of the pipeline and still
|
|
482
|
-
* carries whatever the layers after it would have removed. `reveal`
|
|
483
|
-
*
|
|
484
|
-
* and Layer 4 only ever saw the POST-splice text, meaning a
|
|
485
|
-
* spliced-out HTML comment has never been redacted. The
|
|
486
|
-
*
|
|
493
|
+
* carries whatever the layers after it would have removed. `reveal` and each
|
|
494
|
+
* splice's `original` are PRE-splice text, so they hold exactly the bytes
|
|
495
|
+
* Layer 2 hid — and Layer 4 only ever saw the POST-splice text, meaning a
|
|
496
|
+
* secret inside a spliced-out HTML comment has never been redacted. The
|
|
497
|
+
* documented use of these fields is to persist them, i.e. to write that secret
|
|
498
|
+
* to a log, sidecar, or rehydrated file.
|
|
487
499
|
*
|
|
488
500
|
* Fails CLOSED by WITHHOLDING rather than throwing: a redactor failure here must
|
|
489
501
|
* not discard the already-vetted `cleaned` the caller needs, and dropping the
|
|
@@ -542,7 +554,13 @@ async function vetStageValue(text, redact, findings, label) {
|
|
|
542
554
|
* `reveal` is the pre-Layer-2 text, present only when the HTML splice removed
|
|
543
555
|
* bytes, so a caller can persist what was hidden for later inspection (see
|
|
544
556
|
* {@link applyMarkdownPipeline}); the field is omitted otherwise, and also when
|
|
545
|
-
* it could not be vetted (see {@link vetStageValue}).
|
|
557
|
+
* it could not be vetted (see {@link vetStageValue}). `splices` is its
|
|
558
|
+
* per-placeholder twin — Layer 2's placeholder→original pairs, in document
|
|
559
|
+
* order, so a hook can rehydrate individual splices (the keyed-placeholder
|
|
560
|
+
* grammar lives in ./html.mjs: `layer2Placeholder`/`LAYER2_PLACEHOLDER_RE`).
|
|
561
|
+
* Present only when Layer 2 spliced; each `original` is vetted like `reveal`,
|
|
562
|
+
* and one that cannot be vetted is WITHHELD (dropped from the array) under the
|
|
563
|
+
* same doctrine — Layer 4 never saw pre-splice text.
|
|
546
564
|
*
|
|
547
565
|
* Every byte mutation goes through {@link applyMutation} and every Layer-4 call
|
|
548
566
|
* through {@link runRedact}, so a layer cannot re-establish some of the
|
|
@@ -562,7 +580,7 @@ async function vetStageValue(text, redact, findings, label) {
|
|
|
562
580
|
* vocabulary), so they contribute findings only.
|
|
563
581
|
* @param {string} text
|
|
564
582
|
* @param {SanitizeTextOptions} [options]
|
|
565
|
-
* @returns {Promise<{ cleaned: string, found: string[], warnings: string[], notes: string[], modified: boolean, sgrNote: boolean, reveal?: string }>}
|
|
583
|
+
* @returns {Promise<{ cleaned: string, found: string[], warnings: string[], notes: string[], modified: boolean, sgrNote: boolean, reveal?: string, splices?: Array<{ placeholder: string, original: string }> }>}
|
|
566
584
|
*/
|
|
567
585
|
export async function sanitizeText(text, options = {}) {
|
|
568
586
|
const { redact, filterInjection, sgrCarveOut = false } = options;
|
|
@@ -579,7 +597,8 @@ export async function sanitizeText(text, options = {}) {
|
|
|
579
597
|
unreportedChange: false,
|
|
580
598
|
};
|
|
581
599
|
|
|
582
|
-
const revealText
|
|
600
|
+
const { reveal: revealText, splices: stageSplices } =
|
|
601
|
+
await applyMarkdownPipeline(state, options);
|
|
583
602
|
|
|
584
603
|
// Layer 4 — fail closed (see runRedact).
|
|
585
604
|
if (redact) await runRedact(state, redact);
|
|
@@ -626,6 +645,22 @@ export async function sanitizeText(text, options = {}) {
|
|
|
626
645
|
state.findings,
|
|
627
646
|
"pre-splice copy of the removed HTML",
|
|
628
647
|
);
|
|
648
|
+
// Each splice `original` skipped Layer 4 the same way `reveal` did, so it
|
|
649
|
+
// gets the same exit vetting. A splice whose original cannot be vetted is
|
|
650
|
+
// WITHHELD — dropped from the array — mirroring the reveal doctrine: better
|
|
651
|
+
// an unrecoverable splice than an unvetted secret handed out for persistence.
|
|
652
|
+
/** @type {Array<{ placeholder: string, original: string }>} */
|
|
653
|
+
const splices = [];
|
|
654
|
+
for (const splice of stageSplices) {
|
|
655
|
+
const vetted = await vetStageValue(
|
|
656
|
+
splice.original,
|
|
657
|
+
redact,
|
|
658
|
+
state.findings,
|
|
659
|
+
"original text of a removed-HTML splice",
|
|
660
|
+
);
|
|
661
|
+
if (vetted !== undefined)
|
|
662
|
+
splices.push({ placeholder: splice.placeholder, original: vetted });
|
|
663
|
+
}
|
|
629
664
|
const warnings = warningMessages(state.findings);
|
|
630
665
|
const notes = noteMessages(state.findings);
|
|
631
666
|
return {
|
|
@@ -642,6 +677,10 @@ export async function sanitizeText(text, options = {}) {
|
|
|
642
677
|
sgrNote:
|
|
643
678
|
notes.length > 0 && warnings.length === 0 && !state.unreportedChange,
|
|
644
679
|
...(reveal !== undefined && { reveal }),
|
|
680
|
+
// Presence-gated like `reveal`: the field exists only when Layer 2 spliced
|
|
681
|
+
// and at least one original survived vetting, so the common-case result
|
|
682
|
+
// shape stays minimal.
|
|
683
|
+
...(splices.length > 0 && { splices }),
|
|
645
684
|
};
|
|
646
685
|
}
|
|
647
686
|
|
|
@@ -749,6 +788,12 @@ function depthMemo() {
|
|
|
749
788
|
* the HTML splice removed bytes) so a caller can persist what was hidden — the
|
|
750
789
|
* structured-output analogue of {@link sanitizeText}'s `reveal`. Same
|
|
751
790
|
* mutated-accumulator contract as `warnings`.
|
|
791
|
+
*
|
|
792
|
+
* `splices` accumulates each string leaf's Layer-2 placeholder→original pairs
|
|
793
|
+
* (each `original` already vetted, withheld entries dropped — see
|
|
794
|
+
* {@link sanitizeText}) — the per-placeholder twin of `reveals`, so a hook
|
|
795
|
+
* caller gets them for object-shaped tool output too. Same mutated-accumulator
|
|
796
|
+
* contract as `reveals`.
|
|
752
797
|
* @param {any} value
|
|
753
798
|
* @param {SanitizeTextOptions} options
|
|
754
799
|
* @param {string[]} warnings
|
|
@@ -756,6 +801,8 @@ function depthMemo() {
|
|
|
756
801
|
* @param {string[]} [notes] the NOTE-severity counterpart of `warnings`;
|
|
757
802
|
* appended last so an existing positional caller keeps working (it simply
|
|
758
803
|
* discards the notes, which is exactly as loud as before the split)
|
|
804
|
+
* @param {Array<{ placeholder: string, original: string }>} [splices] appended
|
|
805
|
+
* after `notes` for the same positional-compatibility reason
|
|
759
806
|
* @returns {Promise<{ value: any, modified: boolean, sgrNote: boolean }>}
|
|
760
807
|
*/
|
|
761
808
|
export async function sanitizeValue(
|
|
@@ -764,6 +811,7 @@ export async function sanitizeValue(
|
|
|
764
811
|
warnings,
|
|
765
812
|
reveals = [],
|
|
766
813
|
notes = [],
|
|
814
|
+
splices = [],
|
|
767
815
|
) {
|
|
768
816
|
return sanitizeValueAt(
|
|
769
817
|
value,
|
|
@@ -771,6 +819,7 @@ export async function sanitizeValue(
|
|
|
771
819
|
warnings,
|
|
772
820
|
notes,
|
|
773
821
|
reveals,
|
|
822
|
+
splices,
|
|
774
823
|
0,
|
|
775
824
|
new WeakSet(),
|
|
776
825
|
depthMemo(),
|
|
@@ -788,6 +837,8 @@ export async function sanitizeValue(
|
|
|
788
837
|
* @param {string[]} warnings
|
|
789
838
|
* @param {string[]} notes accumulates each string leaf's NOTE-severity findings
|
|
790
839
|
* @param {string[]} reveals accumulates each string leaf's pre-Layer-2 text
|
|
840
|
+
* @param {Array<{ placeholder: string, original: string }>} splices accumulates
|
|
841
|
+
* each string leaf's Layer-2 placeholder→original pairs (already vetted)
|
|
791
842
|
* @param {number} depth
|
|
792
843
|
* @param {WeakSet<object>} seen
|
|
793
844
|
* @param {ReturnType<typeof depthMemo<{ value: any, modified: boolean, sgrNote: boolean }>>} memo
|
|
@@ -798,8 +849,9 @@ export async function sanitizeValue(
|
|
|
798
849
|
* ~25-object diamond measured at 68 s, far under MAX_DEPTH) — since the path-
|
|
799
850
|
* scoped `seen` set only guards cycles, not repeated work. Because warnings
|
|
800
851
|
* dedup in composeContext, skipping a cached node's duplicate warnings is
|
|
801
|
-
* harmless. A cached node's `reveals` are likewise not
|
|
802
|
-
* for the same reason (the caller dedups reveals by
|
|
852
|
+
* harmless. A cached node's `reveals` and `splices` are likewise not
|
|
853
|
+
* re-emitted, harmless for the same reason (the caller dedups reveals by
|
|
854
|
+
* content and splices by their content-addressed key).
|
|
803
855
|
* @returns {Promise<{ value: any, modified: boolean, sgrNote: boolean }>}
|
|
804
856
|
*/
|
|
805
857
|
async function sanitizeValueAt(
|
|
@@ -808,6 +860,7 @@ async function sanitizeValueAt(
|
|
|
808
860
|
warnings,
|
|
809
861
|
notes,
|
|
810
862
|
reveals,
|
|
863
|
+
splices,
|
|
811
864
|
depth,
|
|
812
865
|
seen,
|
|
813
866
|
memo,
|
|
@@ -817,6 +870,7 @@ async function sanitizeValueAt(
|
|
|
817
870
|
warnings.push(...result.warnings);
|
|
818
871
|
notes.push(...result.notes);
|
|
819
872
|
if (result.reveal !== undefined) reveals.push(result.reveal);
|
|
873
|
+
if (result.splices !== undefined) splices.push(...result.splices);
|
|
820
874
|
return {
|
|
821
875
|
value: result.cleaned,
|
|
822
876
|
modified: result.modified,
|
|
@@ -906,6 +960,7 @@ async function sanitizeValueAt(
|
|
|
906
960
|
warnings,
|
|
907
961
|
notes,
|
|
908
962
|
reveals,
|
|
963
|
+
splices,
|
|
909
964
|
depth + 1,
|
|
910
965
|
seen,
|
|
911
966
|
memo,
|
|
@@ -942,6 +997,7 @@ async function sanitizeValueAt(
|
|
|
942
997
|
warnings,
|
|
943
998
|
notes,
|
|
944
999
|
reveals,
|
|
1000
|
+
splices,
|
|
945
1001
|
depth + 1,
|
|
946
1002
|
seen,
|
|
947
1003
|
memo,
|