agent-sanitizer 2.23.1 → 2.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/THREAT-MODEL.md +37 -10
- package/claude-hooks/lib/control-plane.mjs +17 -2
- package/claude-hooks/lib/hook-io.mjs +39 -0
- package/claude-hooks/lib/hook-timing.mjs +170 -0
- package/claude-hooks/lib/redactor-client.mjs +11 -2
- package/claude-hooks/sanitize-output.mjs +11 -6
- package/claude-hooks/sanitize-user-prompt.mjs +1 -1
- package/claude-hooks/scan-invisible-chars.mjs +107 -28
- package/package.json +1 -1
- package/src/ansi.mjs +66 -5
- package/src/index.mjs +7 -1
- package/src/invisible.mjs +96 -13
- package/src/layer1.mjs +105 -20
- package/src/output.mjs +21 -10
- package/src/prompt.mjs +16 -26
- package/types/ansi.d.mts +54 -4
- package/types/claude-hooks/lib/control-plane.d.mts +6 -1
- package/types/claude-hooks/lib/hook-timing.d.mts +106 -0
- package/types/claude-hooks/lib/redactor-client.d.mts +2 -1
- package/types/claude-hooks/scan-invisible-chars.d.mts +31 -14
- package/types/index.d.mts +1 -1
- package/types/invisible.d.mts +2 -0
- package/types/layer1.d.mts +54 -2
package/src/ansi.mjs
CHANGED
|
@@ -80,7 +80,7 @@ const ST_C1 = 0x9c;
|
|
|
80
80
|
const OSC_C1 = 0x9d;
|
|
81
81
|
const BEL = 0x07;
|
|
82
82
|
|
|
83
|
-
/** The
|
|
83
|
+
/** The six things an introducer can turn out to be. */
|
|
84
84
|
export const TOKEN_KIND = Object.freeze({
|
|
85
85
|
/** A display-only `ESC[…m` / `U+009B…m` colour sequence. */
|
|
86
86
|
SGR: "sgr",
|
|
@@ -88,10 +88,70 @@ export const TOKEN_KIND = Object.freeze({
|
|
|
88
88
|
CSI: "csi",
|
|
89
89
|
/** An OSC string: introducer, body and terminator as one unit. */
|
|
90
90
|
OSC: "osc",
|
|
91
|
-
/**
|
|
91
|
+
/**
|
|
92
|
+
* A 7-bit `ESC` that starts no sequence the grammar recognizes — a truncated
|
|
93
|
+
* write, a log fragment cut mid-escape, a stray byte living in a file.
|
|
94
|
+
*/
|
|
92
95
|
ORPHAN: "orphan-introducer",
|
|
96
|
+
/**
|
|
97
|
+
* A 7-bit `ESC` that OPENS a CSI (`ESC [`) it never completes. Split from
|
|
98
|
+
* {@link TOKEN_KIND.ORPHAN} because a terminal's CSI parser is STATEFUL: it
|
|
99
|
+
* keeps consuming what follows as parameters and intermediates until a final
|
|
100
|
+
* byte (0x40-0x7E) arrives, so `hello ESC[12 world` renders as `hello orld`
|
|
101
|
+
* — the ` w` is eaten as the sequence's intermediate and final. That is the
|
|
102
|
+
* model-sees/human-sees divergence the gate exists for, so consumers that
|
|
103
|
+
* downgrade an inert strip to a note must keep warning on this one; only a
|
|
104
|
+
* lone `ESC` that opens nothing is inert.
|
|
105
|
+
*/
|
|
106
|
+
ORPHAN_CSI: "orphan-csi-introducer",
|
|
107
|
+
/**
|
|
108
|
+
* A RAW C1 byte (U+0080-U+009F) that starts no sequence the grammar
|
|
109
|
+
* recognizes. Split from {@link TOKEN_KIND.ORPHAN} because the two carry very
|
|
110
|
+
* different weight: a lone `ESC` is ordinary debris in terminal output, while
|
|
111
|
+
* a raw C1 byte is not something legitimate UTF-8 text produces, and the
|
|
112
|
+
* block includes the string introducers DCS/SOS/PM/APC (U+0090/0098/009E/
|
|
113
|
+
* 009F) — which this grammar does not consume, so an unrecognized one here
|
|
114
|
+
* means a terminal WOULD have swallowed the following text as a control
|
|
115
|
+
* payload. Consumers that downgrade an inert strip to a note (see
|
|
116
|
+
* `isBenignAnsiKinds` in ./layer1.mjs) must keep warning on this one.
|
|
117
|
+
*/
|
|
118
|
+
ORPHAN_C1: "orphan-c1-introducer",
|
|
93
119
|
});
|
|
94
120
|
|
|
121
|
+
/**
|
|
122
|
+
* True for either orphan kind — the tokens {@link scanAnsi} emits for an
|
|
123
|
+
* introducer that completes no sequence, which the stripper must leave in place
|
|
124
|
+
* for the residual sweep rather than splice (see stripAnsiOnce).
|
|
125
|
+
* @param {string} kind one of {@link TOKEN_KIND}
|
|
126
|
+
* @returns {boolean}
|
|
127
|
+
*/
|
|
128
|
+
export function isOrphanKind(kind) {
|
|
129
|
+
return (
|
|
130
|
+
kind === TOKEN_KIND.ORPHAN ||
|
|
131
|
+
kind === TOKEN_KIND.ORPHAN_CSI ||
|
|
132
|
+
kind === TOKEN_KIND.ORPHAN_C1
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The orphan kind for the introducer character `ch`, given the character `next`
|
|
138
|
+
* that follows it: a raw C1 byte, an `ESC` that opened an incomplete CSI, or a
|
|
139
|
+
* lone `ESC`. The one place that split is decided, shared by the tokenizer and
|
|
140
|
+
* by Layer 1's residual sweep (which sees bare characters, not tokens).
|
|
141
|
+
*
|
|
142
|
+
* `[` is the only lookahead that matters. `ESC ]` is an OSC, which the scanner
|
|
143
|
+
* consumes to the end of input if unterminated (so it never reaches here), and
|
|
144
|
+
* every other second byte — `ESC (`, `ESC #`, `ESC P` — bounds what a terminal
|
|
145
|
+
* swallows to a byte or two rather than running until a final byte arrives.
|
|
146
|
+
* @param {string} ch
|
|
147
|
+
* @param {string} [next] the following character, or undefined at end of input
|
|
148
|
+
* @returns {string}
|
|
149
|
+
*/
|
|
150
|
+
export function orphanKindFor(ch, next) {
|
|
151
|
+
if (ch.charCodeAt(0) !== ESC) return TOKEN_KIND.ORPHAN_C1;
|
|
152
|
+
return next === "[" ? TOKEN_KIND.ORPHAN_CSI : TOKEN_KIND.ORPHAN;
|
|
153
|
+
}
|
|
154
|
+
|
|
95
155
|
/**
|
|
96
156
|
* @typedef {object} AnsiToken
|
|
97
157
|
* @property {number} start Index of the introducer.
|
|
@@ -168,8 +228,9 @@ const INTRODUCER_SCAN_RE = new RegExp(CONTROL_INTRODUCER_SOURCE, "g");
|
|
|
168
228
|
/**
|
|
169
229
|
* Tokenize every raw control introducer in `text`.
|
|
170
230
|
*
|
|
171
|
-
* Every introducer yields exactly one token — an
|
|
172
|
-
*
|
|
231
|
+
* Every introducer yields exactly one token — an orphan kind (see
|
|
232
|
+
* {@link orphanKindFor}) when it starts nothing the grammar recognizes — so
|
|
233
|
+
* "which introducers are in this text" and "which
|
|
173
234
|
* sequences are in this text" are answered by the same scan. That is what lets
|
|
174
235
|
* the stripper (splice every non-orphan token, then sweep) and the SGR-only
|
|
175
236
|
* predicate (every token is SGR) agree by construction.
|
|
@@ -190,7 +251,7 @@ export function scanAnsi(text) {
|
|
|
190
251
|
const csiEnd = oscEnd < 0 ? scanCsi(text, start) : -1;
|
|
191
252
|
let end = start + 1;
|
|
192
253
|
/** @type {string} */
|
|
193
|
-
let kind =
|
|
254
|
+
let kind = orphanKindFor(text[start], text[start + 1]);
|
|
194
255
|
if (oscEnd >= 0) {
|
|
195
256
|
end = oscEnd;
|
|
196
257
|
kind = TOKEN_KIND.OSC;
|
package/src/index.mjs
CHANGED
|
@@ -23,7 +23,13 @@ import { sanitizeText } from "./output.mjs";
|
|
|
23
23
|
// Layer 1 lives in the zero-dependency `./layer1.mjs`, shared verbatim with the
|
|
24
24
|
// tool-output pipeline (`./output`) and the Edit-repair rehydrator
|
|
25
25
|
// (`./rehydrate`) so every consumer derives the identical model-facing view.
|
|
26
|
-
export {
|
|
26
|
+
export {
|
|
27
|
+
applyLayer1,
|
|
28
|
+
isBenignAnsi,
|
|
29
|
+
isBenignAnsiKinds,
|
|
30
|
+
stripAnsiFully,
|
|
31
|
+
LONE_SURROGATE_RE,
|
|
32
|
+
} from "./layer1.mjs";
|
|
27
33
|
|
|
28
34
|
export {
|
|
29
35
|
stripInvisible,
|
package/src/invisible.mjs
CHANGED
|
@@ -228,9 +228,10 @@ export const CONSECUTIVE_JOINER_CAP = 8;
|
|
|
228
228
|
// visible characters in a row), exactly like CONSECUTIVE_JOINER_CAP.
|
|
229
229
|
export const CONSECUTIVE_SELECTOR_CAP = 8;
|
|
230
230
|
|
|
231
|
-
// Floor on the document-wide preserve budget
|
|
232
|
-
// (see `kind` in analyzeCarve
|
|
233
|
-
//
|
|
231
|
+
// Floor on the document-wide preserve budget for joiners, selectors and tag
|
|
232
|
+
// sequences (see `kind` in analyzeCarve — blank fillers are NOT charged here;
|
|
233
|
+
// they have their own allowance, see TOTAL_PRESERVED_BLANK_BUDGET). The
|
|
234
|
+
// Joining_Type gate strips joiners that do no
|
|
234
235
|
// rendering work regardless of count, so the bulk covert channel (ZWNJ
|
|
235
236
|
// scattered through Latin/ASCII/mixed text) is closed by shape, not by
|
|
236
237
|
// counting. What remains is the residual channel of MEANINGFUL joiners/
|
|
@@ -260,6 +261,31 @@ export const PRESERVED_JOINER_PER_VISIBLE = 8;
|
|
|
260
261
|
// text can widen.
|
|
261
262
|
export const PRESERVE_HARD_CAP = 64;
|
|
262
263
|
|
|
264
|
+
// Floor on the document-wide allowance for PRESERVED blank fillers (the
|
|
265
|
+
// Braille blank and the Hangul fillers — see the blank-filler carve-out). Kept
|
|
266
|
+
// separate from the joiner/selector budget because the two have completely
|
|
267
|
+
// different legitimate densities, and short blank-dense strings (a one-line
|
|
268
|
+
// Braille phrase, a lone archaic syllable) must stay un-clipped.
|
|
269
|
+
export const TOTAL_PRESERVED_BLANK_BUDGET = 16;
|
|
270
|
+
|
|
271
|
+
// Visible ANCHOR-script code points (a non-blank Braille cell, a non-filler
|
|
272
|
+
// Hangul jamo/syllable) required per preserved blank filler above the floor. A
|
|
273
|
+
// blank is only ever preservable next to one of these, so this ratio is what
|
|
274
|
+
// separates real text from the degenerate channel: U+2800 separates WORDS and a
|
|
275
|
+
// filler completes a syllable, so genuine text spends several anchor characters
|
|
276
|
+
// per blank and stays under one blank per two anchors, while the alternation an
|
|
277
|
+
// attacker needs to stuff a bit per character (`가ᅟ가ᅟ…`, `⠃⠀⠃⠀…`) is exactly
|
|
278
|
+
// 1:1 and fails.
|
|
279
|
+
//
|
|
280
|
+
// Deliberately NOT capped by PRESERVE_HARD_CAP: an absolute ceiling is what
|
|
281
|
+
// truncated long Braille documents, and unlike the joiner channel this one
|
|
282
|
+
// cannot scale on invisible cover text — every additional bit costs the
|
|
283
|
+
// attacker two VISIBLE anchor-script characters, and the blanks themselves
|
|
284
|
+
// render as spacing a reader can see. Over the ratio, NO blank in the document
|
|
285
|
+
// is preserved (all-or-nothing, so a document never comes out half-spaced) and
|
|
286
|
+
// every one of them becomes payload — which then also feeds the scatter floor.
|
|
287
|
+
export const PRESERVED_BLANK_PER_ANCHOR = 2;
|
|
288
|
+
|
|
263
289
|
// Scripts whose orthography uses ZWNJ/ZWJ between letters as a rendering
|
|
264
290
|
// control. The runtime gate is now script-agnostic (it reads Joining_Type, so it
|
|
265
291
|
// covers every cursive/Brahmic script, not just these), but this list remains
|
|
@@ -402,6 +428,17 @@ function isBrahmicConsonantChar(ch) {
|
|
|
402
428
|
// the anchor and is stripped — the run-length gate falls out of the anchor. The
|
|
403
429
|
// zero-width Mn marks in BLANK_NON_CF (U+034F/17B4/17B5) have no such benign
|
|
404
430
|
// standalone use, so they are never preserved.
|
|
431
|
+
//
|
|
432
|
+
// Blank fillers are NOT charged against the joiner/selector preserve budget:
|
|
433
|
+
// that budget's density model is "~1 preserved invisible per 8 visible chars"
|
|
434
|
+
// (PRESERVED_JOINER_PER_VISIBLE), measured on Persian ZWNJ prose, with a fixed
|
|
435
|
+
// PRESERVE_HARD_CAP ceiling. Blanks are an order of magnitude denser in genuine
|
|
436
|
+
// text — U+2800 IS the word space of Unicode Braille, and a Hangul filler
|
|
437
|
+
// completes a defective syllable — so charging them there mangled real content:
|
|
438
|
+
// a 40-word Braille passage lost 14 of its 39 word spaces (words run together)
|
|
439
|
+
// and a 200-word one kept 64 of 199, with `found` reporting a strip on a
|
|
440
|
+
// perfectly legitimate document. They draw on the anchor-proportional allowance
|
|
441
|
+
// below instead (see TOTAL_PRESERVED_BLANK_BUDGET).
|
|
405
442
|
const BRAILLE_BLANK = 0x2800;
|
|
406
443
|
const HANGUL_FILLERS = new Set([0x115f, 0x1160, 0x3164, 0xffa0]);
|
|
407
444
|
// Code points that trigger the carve-out path for blank fillers (see
|
|
@@ -600,6 +637,44 @@ function analyzeCarve(cps) {
|
|
|
600
637
|
if (isPreservedBlankFiller(cps, i)) return "blank";
|
|
601
638
|
return null;
|
|
602
639
|
});
|
|
640
|
+
// Blank fillers are budgeted here, document-wide and all-or-nothing, against
|
|
641
|
+
// the visible anchor-script text rather than against the joiner/selector
|
|
642
|
+
// counter in carveStrip (see PRESERVED_BLANK_PER_ANCHOR for why the two
|
|
643
|
+
// cannot share a density model). Deciding it in analyzeCarve rather than in
|
|
644
|
+
// the emit loop keeps countPayloadInvisible and payloadInvisibleView honest:
|
|
645
|
+
// a blank the stripper will remove is payload to every consumer, so the
|
|
646
|
+
// prompt layer's scatter gate sees it without re-deriving the budget.
|
|
647
|
+
//
|
|
648
|
+
// Budgeted PER SCRIPT, because a blank never anchors cross-script: pooling the
|
|
649
|
+
// two anchor counts would let one script's cover text fund the other's
|
|
650
|
+
// channel, so 400 chars of ordinary Korean prose would buy an unreported
|
|
651
|
+
// `⠃⠀⠃⠀…` alternation of 200 Braille blanks. The anchor scan is skipped below
|
|
652
|
+
// the floor: it costs a script regex per visible character, and analyzeCarve
|
|
653
|
+
// runs on every prompt and tool output.
|
|
654
|
+
const blankScript = kind.map((k, i) =>
|
|
655
|
+
k !== "blank"
|
|
656
|
+
? null
|
|
657
|
+
: cps[i].codePointAt(0) === BRAILLE_BLANK
|
|
658
|
+
? "braille"
|
|
659
|
+
: "hangul",
|
|
660
|
+
);
|
|
661
|
+
for (const [
|
|
662
|
+
script,
|
|
663
|
+
isAnchor,
|
|
664
|
+
] of /** @type {[string, (ch: string) => boolean][]} */ ([
|
|
665
|
+
["braille", isBrailleCell],
|
|
666
|
+
["hangul", isHangul],
|
|
667
|
+
])) {
|
|
668
|
+
const blanks = blankScript.filter((s) => s === script).length;
|
|
669
|
+
if (blanks <= TOTAL_PRESERVED_BLANK_BUDGET) continue;
|
|
670
|
+
const anchors = cps.reduce(
|
|
671
|
+
(n, ch, i) => n + (codes[i] === null && isAnchor(ch) ? 1 : 0),
|
|
672
|
+
0,
|
|
673
|
+
);
|
|
674
|
+
if (blanks > Math.floor(anchors / PRESERVED_BLANK_PER_ANCHOR))
|
|
675
|
+
for (let i = 0; i < kind.length; i++)
|
|
676
|
+
if (blankScript[i] === script) kind[i] = null;
|
|
677
|
+
}
|
|
603
678
|
let payloadInvis = 0;
|
|
604
679
|
let visibleLen = 0;
|
|
605
680
|
for (let i = 0; i < cps.length; i++) {
|
|
@@ -790,12 +865,15 @@ function clusterEnds(body) {
|
|
|
790
865
|
|
|
791
866
|
/**
|
|
792
867
|
* Carve-out strip (an invisible the carve-out might preserve is present): walk
|
|
793
|
-
* GRAPHEME CLUSTERS, preserving a cluster's joiners/selectors/tags
|
|
794
|
-
*
|
|
795
|
-
*
|
|
796
|
-
*
|
|
797
|
-
*
|
|
798
|
-
*
|
|
868
|
+
* GRAPHEME CLUSTERS, preserving a cluster's joiners/selectors/tags only where
|
|
869
|
+
* each has its `kind` set AND the text stays under the scatter floor AND the
|
|
870
|
+
* whole cluster fits inside the remaining per-run (CONSECUTIVE_JOINER_CAP /
|
|
871
|
+
* CONSECUTIVE_SELECTOR_CAP) and document-wide (TOTAL_PRESERVED_JOINER_BUDGET)
|
|
872
|
+
* preserve allowance — otherwise every preservable char in that cluster is
|
|
873
|
+
* stripped like any other payload byte. Blank fillers are the exception: their
|
|
874
|
+
* allowance is anchor-proportional and already spent document-wide in
|
|
875
|
+
* analyzeCarve (see PRESERVED_BLANK_PER_ANCHOR), so here they answer only to
|
|
876
|
+
* the scatter floor and do not draw on the joiner/selector budget.
|
|
799
877
|
*
|
|
800
878
|
* The budget is charged against the CLUSTER, not the code point, because the
|
|
801
879
|
* cluster is the indivisible unit: charging per code point let a limit fall due
|
|
@@ -862,7 +940,10 @@ function carveStrip(body) {
|
|
|
862
940
|
let joiners = 0;
|
|
863
941
|
let selectors = 0;
|
|
864
942
|
for (let k = start; k < end; k++) {
|
|
865
|
-
|
|
943
|
+
// "blank" is exempt: analyzeCarve already decided it against the
|
|
944
|
+
// anchor-proportional allowance, so it neither draws on this budget nor
|
|
945
|
+
// is stripped by it (only by the scatter floor, via allowCarveOut).
|
|
946
|
+
if (kind[k] === null || kind[k] === "blank") continue;
|
|
866
947
|
need++;
|
|
867
948
|
if (kind[k] === "joiner") joiners++;
|
|
868
949
|
if (kind[k] === "ivs" || kind[k] === "stdvs") selectors++;
|
|
@@ -904,11 +985,13 @@ function carveStrip(body) {
|
|
|
904
985
|
out += cps[k]; // ordinary visible character
|
|
905
986
|
continue;
|
|
906
987
|
}
|
|
907
|
-
|
|
988
|
+
// A blank filler rides on allowCarveOut alone (its own allowance is
|
|
989
|
+
// already spent in analyzeCarve); everything else rides on `fits`.
|
|
990
|
+
if (kind[k] === "blank" ? allowCarveOut : fits && kind[k] !== null) {
|
|
908
991
|
if (kind[k] === "joiner") joinerRun++;
|
|
909
992
|
if (kind[k] === "ivs" || kind[k] === "stdvs") selectorRun++;
|
|
910
|
-
preservedTotal++;
|
|
911
|
-
prevVisible = false; // a joiner/selector/tag keeps the cluster open
|
|
993
|
+
if (kind[k] !== "blank") preservedTotal++;
|
|
994
|
+
prevVisible = false; // a joiner/selector/tag/blank keeps the cluster open
|
|
912
995
|
out += cps[k];
|
|
913
996
|
continue;
|
|
914
997
|
}
|
package/src/layer1.mjs
CHANGED
|
@@ -13,7 +13,13 @@
|
|
|
13
13
|
* point where a lone surrogate would otherwise corrupt a match or a parse.
|
|
14
14
|
*/
|
|
15
15
|
import { stripInvisibleWithReport, CATEGORY } from "./invisible.mjs";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
CONTROL_INTRODUCER_SOURCE,
|
|
18
|
+
isOrphanKind,
|
|
19
|
+
orphanKindFor,
|
|
20
|
+
scanAnsi,
|
|
21
|
+
TOKEN_KIND,
|
|
22
|
+
} from "./ansi.mjs";
|
|
17
23
|
|
|
18
24
|
// The ANSI grammar and the introducer charset live in ./ansi.mjs so this module
|
|
19
25
|
// and invisible.mjs (which owns the public isSgrOnly / SGR_RE and cannot import
|
|
@@ -24,6 +30,23 @@ import { CONTROL_INTRODUCER_SOURCE, scanAnsi, TOKEN_KIND } from "./ansi.mjs";
|
|
|
24
30
|
// survives Layer 1.
|
|
25
31
|
const CONTROL_INTRODUCER_RE = new RegExp(CONTROL_INTRODUCER_SOURCE, "g");
|
|
26
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Run the residual sweep, recording the orphan kind of every introducer it
|
|
35
|
+
* removes. The sweep sees bare characters rather than tokens, so the kind comes
|
|
36
|
+
* from {@link orphanKindFor} — the same decision the tokenizer makes, not a
|
|
37
|
+
* second spelling of it — fed the following character from the text being
|
|
38
|
+
* swept, which is the context a terminal reading this introducer would have.
|
|
39
|
+
* @param {string} text
|
|
40
|
+
* @param {Set<string>} kinds
|
|
41
|
+
* @returns {string}
|
|
42
|
+
*/
|
|
43
|
+
function sweepIntroducers(text, kinds) {
|
|
44
|
+
return text.replace(CONTROL_INTRODUCER_RE, (ch, offset) => {
|
|
45
|
+
kinds.add(orphanKindFor(ch, text[offset + 1]));
|
|
46
|
+
return "";
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
27
50
|
// Unpaired UTF-16 surrogates (high not followed by low, or low not preceded by
|
|
28
51
|
// high). Normalized before any HTML parser, which throws on a stray byte —
|
|
29
52
|
// which would otherwise let a single malformed code unit suppress all output.
|
|
@@ -41,13 +64,18 @@ const MAX_ANSI_PASSES = 3;
|
|
|
41
64
|
* view. Orphans are removed by applyLayer1's residual sweep, once no
|
|
42
65
|
* reconstitution is possible.
|
|
43
66
|
* @param {string} text
|
|
67
|
+
* @param {Set<string>} [kinds] collects the {@link TOKEN_KIND} of every
|
|
68
|
+
* sequence this pass actually removed, so a caller can tell a display-only
|
|
69
|
+
* colour strip from a cursor/erase/OSC one without re-scanning (see
|
|
70
|
+
* {@link isBenignAnsiKinds}).
|
|
44
71
|
* @returns {string}
|
|
45
72
|
*/
|
|
46
|
-
function stripAnsiOnce(text) {
|
|
73
|
+
function stripAnsiOnce(text, kinds) {
|
|
47
74
|
let out = "";
|
|
48
75
|
let last = 0;
|
|
49
76
|
for (const token of scanAnsi(text)) {
|
|
50
|
-
if (token.kind
|
|
77
|
+
if (isOrphanKind(token.kind)) continue;
|
|
78
|
+
kinds?.add(token.kind);
|
|
51
79
|
out += text.slice(last, token.start);
|
|
52
80
|
last = token.end;
|
|
53
81
|
}
|
|
@@ -68,18 +96,71 @@ function stripAnsiOnce(text) {
|
|
|
68
96
|
* survives here. Past the bound a reconstituted sequence therefore degrades to
|
|
69
97
|
* VISIBLE text rather than a hidden control, which is the fail-open direction.
|
|
70
98
|
* @param {string} input
|
|
99
|
+
* @param {Set<string>} [kinds] see {@link stripAnsiOnce}; accumulates across passes
|
|
71
100
|
* @returns {string}
|
|
72
101
|
*/
|
|
73
|
-
export function stripAnsiFully(input) {
|
|
102
|
+
export function stripAnsiFully(input, kinds) {
|
|
74
103
|
let prev = input;
|
|
75
|
-
let out = stripAnsiOnce(prev);
|
|
104
|
+
let out = stripAnsiOnce(prev, kinds);
|
|
76
105
|
for (let pass = 1; pass < MAX_ANSI_PASSES && out !== prev; pass++) {
|
|
77
106
|
prev = out;
|
|
78
|
-
out = stripAnsiOnce(prev);
|
|
107
|
+
out = stripAnsiOnce(prev, kinds);
|
|
79
108
|
}
|
|
80
109
|
return out;
|
|
81
110
|
}
|
|
82
111
|
|
|
112
|
+
/**
|
|
113
|
+
* True when the ANSI a Layer-1 strip removed was INERT: every removed sequence
|
|
114
|
+
* was either a display-only SGR colour token or a LONE 7-bit `ESC` that opened
|
|
115
|
+
* nothing at all (a stray byte in a file, a truncated write, a log fragment cut
|
|
116
|
+
* mid-escape).
|
|
117
|
+
*
|
|
118
|
+
* The two other orphan kinds are deliberately NOT inert. A raw C1 orphan
|
|
119
|
+
* (TOKEN_KIND.ORPHAN_C1): legit UTF-8 text does not carry raw C1 bytes, and the
|
|
120
|
+
* block holds the DCS/SOS/PM/APC string introducers this grammar does not
|
|
121
|
+
* consume — so an unrecognized one means a terminal would have eaten the
|
|
122
|
+
* following text as a control payload. An incomplete CSI (TOKEN_KIND.ORPHAN_CSI)
|
|
123
|
+
* for the same reason at 7 bits: the CSI parser keeps consuming until a final
|
|
124
|
+
* byte, so `hello ESC[12 world` hides ` w` from the human while the model reads
|
|
125
|
+
* the whole prompt.
|
|
126
|
+
*
|
|
127
|
+
* This draws a severity line, not a presence line: the bytes are stripped
|
|
128
|
+
* either way, so all that rides on the answer is whether the operator sees a
|
|
129
|
+
* WARNING or a terse note. An orphan introducer cannot move the cursor, erase
|
|
130
|
+
* the screen, relabel a window, or open an OSC string — every one of those needs
|
|
131
|
+
* a COMPLETE token, which {@link scanAnsi} classifies as CSI or OSC and this
|
|
132
|
+
* rejects. Warning on a lone `ESC` is the false positive that costs the most: one
|
|
133
|
+
* pre-existing `ESC` in a markdown file, echoed back in an Edit result, raises
|
|
134
|
+
* the same alarm as a cursor-spoofing payload, and an alarm that fires on inert
|
|
135
|
+
* bytes is the one operators learn to scroll past.
|
|
136
|
+
*
|
|
137
|
+
* It takes the kinds the STRIP recorded, never a fresh scan of the raw text,
|
|
138
|
+
* and that is the whole point: a scan of the raw text answers about sequences
|
|
139
|
+
* that have not been reconstituted yet, so `ESC` + `ESC[m` + `[2J` (a bare ESC,
|
|
140
|
+
* an SGR, then plain text) reads as orphan-only there while the strip's second
|
|
141
|
+
* pass actually removes a CSI erase. Recording what each pass removed reports
|
|
142
|
+
* the sequences that really existed at Layer 1's fixed point.
|
|
143
|
+
* @param {readonly string[] | Set<string>} kinds {@link TOKEN_KIND} values removed
|
|
144
|
+
* @returns {boolean}
|
|
145
|
+
*/
|
|
146
|
+
export function isBenignAnsiKinds(kinds) {
|
|
147
|
+
return [...kinds].every(
|
|
148
|
+
(kind) => kind === TOKEN_KIND.SGR || kind === TOKEN_KIND.ORPHAN,
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* {@link isBenignAnsiKinds} for callers that hold only the text — it runs the
|
|
154
|
+
* full Layer-1 composition to get the fixed-point view. Callers that already
|
|
155
|
+
* ran {@link applyLayer1} must read its `ansiKinds` instead of paying for a
|
|
156
|
+
* second strip.
|
|
157
|
+
* @param {string} text
|
|
158
|
+
* @returns {boolean}
|
|
159
|
+
*/
|
|
160
|
+
export function isBenignAnsi(text) {
|
|
161
|
+
return isBenignAnsiKinds(applyLayer1(text).ansiKinds);
|
|
162
|
+
}
|
|
163
|
+
|
|
83
164
|
// How many times the {ANSI strip, invisible strip} composition may re-run before
|
|
84
165
|
// the sweep is forced. Each iteration deletes at least one character, so the
|
|
85
166
|
// loop terminates on its own; the bound is a DoS guard on the same quadratic
|
|
@@ -119,19 +200,25 @@ const MAX_LAYER1_PASSES = 4;
|
|
|
119
200
|
*
|
|
120
201
|
* `deAnsi` is the ANSI strip of the ORIGINAL text (invisible runs intact), the
|
|
121
202
|
* scope a LONG_RUN payload check needs — not an intermediate of the loop.
|
|
203
|
+
*
|
|
204
|
+
* `ansiKinds` is the {@link TOKEN_KIND} of every ANSI sequence the composition
|
|
205
|
+
* removed, deduped — the severity detail `found`'s single ANSI category cannot
|
|
206
|
+
* carry (see {@link isBenignAnsiKinds}). It is also what DERIVES that category:
|
|
207
|
+
* a kind is recorded exactly when bytes were removed, so "we reported ANSI" and
|
|
208
|
+
* "here is what the ANSI was" can no longer disagree.
|
|
122
209
|
* @param {string} text
|
|
123
|
-
* @returns {{ cleaned: string, deAnsi: string, found: string[] }}
|
|
210
|
+
* @returns {{ cleaned: string, deAnsi: string, found: string[], ansiKinds: string[] }}
|
|
124
211
|
*/
|
|
125
212
|
export function applyLayer1(text) {
|
|
126
|
-
|
|
213
|
+
/** @type {Set<string>} TOKEN_KINDs removed by every ANSI pass below. */
|
|
214
|
+
const ansiKinds = new Set();
|
|
215
|
+
const deAnsi = stripAnsiFully(text, ansiKinds);
|
|
127
216
|
/** @type {Set<string>} Union of the categories every iteration reported. */
|
|
128
217
|
const found = new Set();
|
|
129
218
|
let cleaned = text;
|
|
130
|
-
let ansiFound = false;
|
|
131
219
|
|
|
132
220
|
for (let pass = 0; pass < MAX_LAYER1_PASSES; pass++) {
|
|
133
|
-
const afterAnsi = pass === 0 ? deAnsi : stripAnsiFully(cleaned);
|
|
134
|
-
if (afterAnsi !== cleaned) ansiFound = true;
|
|
221
|
+
const afterAnsi = pass === 0 ? deAnsi : stripAnsiFully(cleaned, ansiKinds);
|
|
135
222
|
// stripInvisibleWithReport returns `found` for exactly the categories it
|
|
136
223
|
// removed — so a ZWNJ/ZWJ the carve-out PRESERVES never registers as a
|
|
137
224
|
// strip. The second argument stays the ORIGINAL `text` on every iteration:
|
|
@@ -152,18 +239,16 @@ export function applyLayer1(text) {
|
|
|
152
239
|
// pass. A sweep that changes the text feeds one more round (removing an
|
|
153
240
|
// introducer can make invisibles adjacent, exactly as removing a sequence
|
|
154
241
|
// can); one that changes nothing means the whole composition has converged.
|
|
155
|
-
const swept = cleaned
|
|
242
|
+
const swept = sweepIntroducers(cleaned, ansiKinds);
|
|
156
243
|
if (swept === cleaned) break;
|
|
157
244
|
cleaned = swept;
|
|
158
|
-
ansiFound = true;
|
|
159
245
|
}
|
|
160
246
|
|
|
161
|
-
|
|
162
|
-
if (swept !== cleaned) {
|
|
163
|
-
cleaned = swept;
|
|
164
|
-
ansiFound = true;
|
|
165
|
-
}
|
|
247
|
+
cleaned = sweepIntroducers(cleaned, ansiKinds);
|
|
166
248
|
|
|
167
|
-
|
|
168
|
-
|
|
249
|
+
// Derived, not tracked in parallel: a kind is recorded exactly when an ANSI
|
|
250
|
+
// pass or the sweep removed bytes, so the category and the severity detail
|
|
251
|
+
// are two readings of one fact.
|
|
252
|
+
if (ansiKinds.size > 0) found.add(CATEGORY.ANSI);
|
|
253
|
+
return { cleaned, deAnsi, found: [...found], ansiKinds: [...ansiKinds] };
|
|
169
254
|
}
|
package/src/output.mjs
CHANGED
|
@@ -24,9 +24,13 @@
|
|
|
24
24
|
* actually removes something, so a secret that a deletion reconstitutes is
|
|
25
25
|
* still caught before this function returns.
|
|
26
26
|
*/
|
|
27
|
-
import { CATEGORY, describeStripped
|
|
27
|
+
import { CATEGORY, describeStripped } from "./invisible.mjs";
|
|
28
28
|
import { needsMarkdownPipeline } from "./gates.mjs";
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
applyLayer1,
|
|
31
|
+
isBenignAnsiKinds,
|
|
32
|
+
LONE_SURROGATE_RE,
|
|
33
|
+
} from "./layer1.mjs";
|
|
30
34
|
import {
|
|
31
35
|
describeExfil,
|
|
32
36
|
describeHtmlSanitized,
|
|
@@ -283,9 +287,10 @@ export function deleteVerbatimSpans(text, spans) {
|
|
|
283
287
|
|
|
284
288
|
/**
|
|
285
289
|
* Layer 1 + surrogate normalisation: invisible chars, ANSI, lone surrogates.
|
|
286
|
-
* `sgrNote` is true when the ONLY change was display-only SGR
|
|
287
|
-
*
|
|
288
|
-
*
|
|
290
|
+
* `sgrNote` is true when the ONLY change was INERT ANSI — display-only SGR
|
|
291
|
+
* colour and/or a stray orphan introducer that formed no sequence — AND the
|
|
292
|
+
* caller opted into the carve-out (`sgrCarveOut`); the caller reports that with
|
|
293
|
+
* a terse note, not the WARNING prefix.
|
|
289
294
|
* @param {string} text
|
|
290
295
|
* @param {boolean} sgrCarveOut
|
|
291
296
|
* @returns {{ cleaned: string, found: string[], warnings: string[], modified: boolean, sgrNote: boolean }}
|
|
@@ -297,18 +302,24 @@ function processLayer1(text, sgrCarveOut) {
|
|
|
297
302
|
const found = [];
|
|
298
303
|
let modified = false;
|
|
299
304
|
let sgrNote = false;
|
|
300
|
-
const {
|
|
305
|
+
const {
|
|
306
|
+
cleaned: layer1,
|
|
307
|
+
deAnsi,
|
|
308
|
+
found: invisFound,
|
|
309
|
+
ansiKinds,
|
|
310
|
+
} = applyLayer1(text);
|
|
301
311
|
let cleaned = layer1;
|
|
302
312
|
if (invisFound.length > 0) {
|
|
303
313
|
found.push(...invisFound);
|
|
304
314
|
modified = true;
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
// chars were present
|
|
315
|
+
// Inert ANSI with the carve-out enabled: the strip removed cosmetic styling
|
|
316
|
+
// and/or a stray escape byte, and nothing else (found is exactly [ANSI], so
|
|
317
|
+
// zero invisible chars were present). Report it as a note — a cursor-move,
|
|
318
|
+
// erase or OSC token lands in ansiKinds as CSI/OSC and keeps the WARNING.
|
|
308
319
|
sgrNote =
|
|
309
320
|
invisFound.length === 1 &&
|
|
310
321
|
invisFound[0] === CATEGORY.ANSI &&
|
|
311
|
-
|
|
322
|
+
isBenignAnsiKinds(ansiKinds) &&
|
|
312
323
|
sgrCarveOut;
|
|
313
324
|
if (!sgrNote) warnings.push(describeStripped(invisFound, deAnsi));
|
|
314
325
|
}
|
package/src/prompt.mjs
CHANGED
|
@@ -8,12 +8,14 @@
|
|
|
8
8
|
* the only way to neutralize a payload is to block. This is the pure decision;
|
|
9
9
|
* a host wraps it in whatever its agent's prompt-submission hook expects.
|
|
10
10
|
*
|
|
11
|
-
* One carve-out: a prompt whose only escape content is SGR color/style
|
|
12
|
-
* (`ESC [ params m`)
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
11
|
+
* One carve-out: a prompt whose only escape content is INERT — SGR color/style
|
|
12
|
+
* codes (`ESC [ params m`) and/or an orphan introducer that completes no
|
|
13
|
+
* sequence — passes with a note instead of blocking. Pasting colored terminal
|
|
14
|
+
* output (test runs, build logs) is the single most common debugging action,
|
|
15
|
+
* and SGR is display-only by the ECMA-48 grammar; an orphan `ESC` is not a
|
|
16
|
+
* sequence at all. Neither can move the cursor, erase the screen, or carry an
|
|
17
|
+
* OSC payload. Anything that IS a complete CSI/OSC token still blocks, as do
|
|
18
|
+
* the invisible-char thresholds.
|
|
17
19
|
*/
|
|
18
20
|
import {
|
|
19
21
|
CHECKS,
|
|
@@ -24,9 +26,8 @@ import {
|
|
|
24
26
|
SCATTERED_THRESHOLD,
|
|
25
27
|
countPayloadInvisible,
|
|
26
28
|
stripInvisible,
|
|
27
|
-
isSgrOnly,
|
|
28
29
|
} from "./invisible.mjs";
|
|
29
|
-
import { stripAnsiFully } from "./layer1.mjs";
|
|
30
|
+
import { isBenignAnsi, stripAnsiFully } from "./layer1.mjs";
|
|
30
31
|
import { CONTROL_INTRODUCER_SOURCE } from "./ansi.mjs";
|
|
31
32
|
|
|
32
33
|
// Every raw ANSI control a prompt can carry: 7-bit ESC (U+001B) and the entire
|
|
@@ -44,21 +45,6 @@ import { CONTROL_INTRODUCER_SOURCE } from "./ansi.mjs";
|
|
|
44
45
|
// differently, so even a grep-based drift check would have missed a divergence.
|
|
45
46
|
const ANSI_INTRODUCER = new RegExp(CONTROL_INTRODUCER_SOURCE);
|
|
46
47
|
|
|
47
|
-
/**
|
|
48
|
-
* True when every ANSI introducer in `prompt` belongs to a display-only SGR
|
|
49
|
-
* color sequence -- the note carve-out's precondition. `isSgrOnly` already tests
|
|
50
|
-
* the SGR-stripped prompt against the WHOLE C1 control block (U+0080-U+009F,
|
|
51
|
-
* which includes the C1 OSC introducer U+009D and DCS/SOS/PM/APC) plus the 7-bit
|
|
52
|
-
* ESC, so a residual C1-OSC or any non-SGR escape already denies it — no
|
|
53
|
-
* separate re-check is needed (an earlier `&& !ANSI_INTRODUCER.test(...)` here
|
|
54
|
-
* was an exact duplicate of that gate over the same stripped string).
|
|
55
|
-
* @param {string} prompt
|
|
56
|
-
* @returns {boolean}
|
|
57
|
-
*/
|
|
58
|
-
function isSgrColorOnly(prompt) {
|
|
59
|
-
return isSgrOnly(prompt);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
48
|
/**
|
|
63
49
|
* Human-facing block reason: what was detected, the thresholds, a code-point
|
|
64
50
|
* sample of the long run (if any), and how to recover.
|
|
@@ -139,9 +125,13 @@ export function classifyPrompt(prompt, strip = stripAnsiFully) {
|
|
|
139
125
|
|
|
140
126
|
if (!hasAnsi && invisiblesBelowThreshold) return { action: "pass" };
|
|
141
127
|
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
|
|
128
|
+
// Inert escapes in an otherwise clean prompt — display-only colour and/or an
|
|
129
|
+
// orphan introducer that forms no sequence: pass with a note instead of
|
|
130
|
+
// blocking, so pasted colored logs and log fragments cut mid-escape remain
|
|
131
|
+
// usable. isBenignAnsi judges from what Layer 1's strip actually removed, so
|
|
132
|
+
// it covers the whole C1 block and any sequence that only reconstitutes
|
|
133
|
+
// during the strip; a complete CSI/OSC token falls through to the block.
|
|
134
|
+
if (hasAnsi && invisiblesBelowThreshold && isBenignAnsi(prompt))
|
|
145
135
|
return { action: "note" };
|
|
146
136
|
|
|
147
137
|
// CHECKS pairs a machine-readable category code with its detector; map each
|