agent-sanitizer 2.61.0 → 2.61.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THREAT-MODEL.md +43 -14
- package/claude-hooks/lib/authored-content.mjs +60 -43
- package/package.json +4 -1
- package/python/agent_sanitizer/data/invisible-charset.json +2 -2
- package/src/ansi.mjs +105 -26
- package/src/cf-charset.mjs +6 -6
- package/src/index.mjs +2 -0
- package/src/instructions.mjs +20 -25
- package/src/invisible.mjs +722 -175
- package/src/joining-type.mjs +57 -10
- package/src/layer1.mjs +134 -31
- package/src/prompt.mjs +20 -22
- package/src/standardized-variants.mjs +69 -9
- package/src/unicode-properties.mjs +365 -0
- package/types/ansi.d.mts +45 -16
- package/types/cf-charset.d.mts +5 -5
- package/types/index.d.mts +1 -1
- package/types/instructions.d.mts +6 -2
- package/types/invisible.d.mts +61 -6
- package/types/joining-type.d.mts +5 -2
- package/types/layer1.d.mts +39 -9
- package/types/src/ansi.d.mts +45 -16
- package/types/src/cf-charset.d.mts +5 -5
- package/types/src/instructions.d.mts +6 -2
- package/types/src/invisible.d.mts +61 -6
- package/types/src/joining-type.d.mts +5 -2
- package/types/src/standardized-variants.d.mts +8 -8
- package/types/src/unicode-properties.d.mts +29 -0
- package/types/standardized-variants.d.mts +8 -8
- package/types/unicode-properties.d.mts +29 -0
package/src/joining-type.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* GENERATED by scripts/gen-joining-type.mjs from
|
|
2
|
+
* GENERATED by scripts/gen-joining-type.mjs from the pinned UCD 17.0.0 — DO NOT EDIT.
|
|
3
3
|
*
|
|
4
4
|
* Unicode Joining_Type, Indic virama and Brahmic consonant range tables backing
|
|
5
5
|
* the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`;
|
|
@@ -10,7 +10,7 @@ export const UNICODE_VERSION = "17.0.0";
|
|
|
10
10
|
|
|
11
11
|
// The two range tables below are GENERATED UCD data, not hand-written logic.
|
|
12
12
|
// Their correctness is pinned by the SSOT round-trip in
|
|
13
|
-
// test/joining-type.test.mjs (re-derive from the
|
|
13
|
+
// test/joining-type.test.mjs (re-derive from the pinned UCD, then assert
|
|
14
14
|
// every covered code point and range boundary matches) — a far stronger and
|
|
15
15
|
// faster guard than mutation. Mutating ~550 numeric literals is pathologically
|
|
16
16
|
// slow (the shard was cancelled at ~19 min) and redundant, so Stryker skips
|
|
@@ -126,6 +126,7 @@ const JOINING_RANGES = [
|
|
|
126
126
|
[0x886, 0x886, "D"],
|
|
127
127
|
[0x889, 0x88d, "D"],
|
|
128
128
|
[0x88e, 0x88e, "R"],
|
|
129
|
+
[0x88f, 0x88f, "D"],
|
|
129
130
|
[0x897, 0x89f, "T"],
|
|
130
131
|
[0x8a0, 0x8a9, "D"],
|
|
131
132
|
[0x8aa, 0x8ac, "R"],
|
|
@@ -256,7 +257,8 @@ const JOINING_RANGES = [
|
|
|
256
257
|
[0x1a65, 0x1a6c, "T"],
|
|
257
258
|
[0x1a73, 0x1a7c, "T"],
|
|
258
259
|
[0x1a7f, 0x1a7f, "T"],
|
|
259
|
-
[0x1ab0,
|
|
260
|
+
[0x1ab0, 0x1add, "T"],
|
|
261
|
+
[0x1ae0, 0x1aeb, "T"],
|
|
260
262
|
[0x1b00, 0x1b03, "T"],
|
|
261
263
|
[0x1b34, 0x1b34, "T"],
|
|
262
264
|
[0x1b36, 0x1b3a, "T"],
|
|
@@ -381,7 +383,8 @@ const JOINING_RANGES = [
|
|
|
381
383
|
[0x10eab, 0x10eac, "T"],
|
|
382
384
|
[0x10ec2, 0x10ec2, "R"],
|
|
383
385
|
[0x10ec3, 0x10ec4, "D"],
|
|
384
|
-
[
|
|
386
|
+
[0x10ec6, 0x10ec7, "D"],
|
|
387
|
+
[0x10efa, 0x10eff, "T"],
|
|
385
388
|
[0x10f30, 0x10f32, "D"],
|
|
386
389
|
[0x10f33, 0x10f33, "R"],
|
|
387
390
|
[0x10f34, 0x10f44, "D"],
|
|
@@ -478,6 +481,9 @@ const JOINING_RANGES = [
|
|
|
478
481
|
[0x11a59, 0x11a5b, "T"],
|
|
479
482
|
[0x11a8a, 0x11a96, "T"],
|
|
480
483
|
[0x11a98, 0x11a99, "T"],
|
|
484
|
+
[0x11b60, 0x11b60, "T"],
|
|
485
|
+
[0x11b62, 0x11b64, "T"],
|
|
486
|
+
[0x11b66, 0x11b66, "T"],
|
|
481
487
|
[0x11c30, 0x11c36, "T"],
|
|
482
488
|
[0x11c38, 0x11c3d, "T"],
|
|
483
489
|
[0x11c3f, 0x11c3f, "T"],
|
|
@@ -534,6 +540,10 @@ const JOINING_RANGES = [
|
|
|
534
540
|
[0x1e2ec, 0x1e2ef, "T"],
|
|
535
541
|
[0x1e4ec, 0x1e4ef, "T"],
|
|
536
542
|
[0x1e5ee, 0x1e5ef, "T"],
|
|
543
|
+
[0x1e6e3, 0x1e6e3, "T"],
|
|
544
|
+
[0x1e6e6, 0x1e6e6, "T"],
|
|
545
|
+
[0x1e6ee, 0x1e6ef, "T"],
|
|
546
|
+
[0x1e6f5, 0x1e6f5, "T"],
|
|
537
547
|
[0x1e8d0, 0x1e8d6, "T"],
|
|
538
548
|
[0x1e900, 0x1e943, "D"],
|
|
539
549
|
[0x1e944, 0x1e94b, "T"],
|
|
@@ -575,9 +585,9 @@ const VIRAMA_RANGES = [
|
|
|
575
585
|
];
|
|
576
586
|
|
|
577
587
|
// [start, end, Script] Indic_Syllabic_Category = Consonant ranges restricted to
|
|
578
|
-
// the
|
|
579
|
-
//
|
|
580
|
-
//
|
|
588
|
+
// the scripts that own a virama — the only bases a virama attaches to. Keyed
|
|
589
|
+
// by script name so the contract test can check each span against the script
|
|
590
|
+
// it claims.
|
|
581
591
|
/** @type {ReadonlyArray<readonly [string, number, number]>} */
|
|
582
592
|
export const BRAHMIC_CONSONANT_RANGES = [
|
|
583
593
|
["Devanagari", 0x0915, 0x0939],
|
|
@@ -628,6 +638,33 @@ export const BRAHMIC_CONSONANT_RANGES = [
|
|
|
628
638
|
["Sinhala", 0x0db3, 0x0dbb],
|
|
629
639
|
["Sinhala", 0x0dbd, 0x0dbd],
|
|
630
640
|
["Sinhala", 0x0dc0, 0x0dc6],
|
|
641
|
+
["Balinese", 0x1b13, 0x1b33],
|
|
642
|
+
["Balinese", 0x1b45, 0x1b4c],
|
|
643
|
+
["Syloti_Nagri", 0xa807, 0xa80a],
|
|
644
|
+
["Syloti_Nagri", 0xa80c, 0xa822],
|
|
645
|
+
["Saurashtra", 0xa892, 0xa8b3],
|
|
646
|
+
["Javanese", 0xa989, 0xa98b],
|
|
647
|
+
["Javanese", 0xa98f, 0xa9b2],
|
|
648
|
+
["Brahmi", 0x11013, 0x11037],
|
|
649
|
+
["Brahmi", 0x11075, 0x11075],
|
|
650
|
+
["Kaithi", 0x1108d, 0x110af],
|
|
651
|
+
["Sharada", 0x11191, 0x111b2],
|
|
652
|
+
["Khojki", 0x11208, 0x11211],
|
|
653
|
+
["Khojki", 0x11213, 0x1122b],
|
|
654
|
+
["Khojki", 0x1123f, 0x1123f],
|
|
655
|
+
["Grantha", 0x11315, 0x11328],
|
|
656
|
+
["Grantha", 0x1132a, 0x11330],
|
|
657
|
+
["Grantha", 0x11332, 0x11333],
|
|
658
|
+
["Grantha", 0x11335, 0x11339],
|
|
659
|
+
["Newa", 0x1140e, 0x11434],
|
|
660
|
+
["Tirhuta", 0x1148f, 0x114af],
|
|
661
|
+
["Siddham", 0x1158e, 0x115ae],
|
|
662
|
+
["Modi", 0x1160e, 0x1162f],
|
|
663
|
+
["Takri", 0x1168a, 0x116aa],
|
|
664
|
+
["Takri", 0x116b8, 0x116b8],
|
|
665
|
+
["Dogra", 0x1180a, 0x1182b],
|
|
666
|
+
["Nandinagari", 0x119ae, 0x119d0],
|
|
667
|
+
["Bhaiksuki", 0x11c0e, 0x11c2e],
|
|
631
668
|
];
|
|
632
669
|
// Stryker restore all
|
|
633
670
|
|
|
@@ -674,12 +711,22 @@ export function isVirama(cp) {
|
|
|
674
711
|
/**
|
|
675
712
|
* True when `cp` is a Brahmic consonant — the only base a virama attaches to,
|
|
676
713
|
* and therefore the only base after which a ZWJ/ZWNJ is a real conjunct
|
|
677
|
-
* request rather than a zero-width payload.
|
|
714
|
+
* request rather than a zero-width payload. A binary search, like
|
|
715
|
+
* `lookup`: the spans are sorted and disjoint, and the carve analysis asks
|
|
716
|
+
* this of every visible code point of a long document, where a scan of all the
|
|
717
|
+
* spans per code point was the dominant cost.
|
|
678
718
|
* @param {number} cp
|
|
679
719
|
* @returns {boolean}
|
|
680
720
|
*/
|
|
681
721
|
export function isBrahmicConsonant(cp) {
|
|
682
|
-
|
|
683
|
-
|
|
722
|
+
let lo = 0;
|
|
723
|
+
let hi = BRAHMIC_CONSONANT_RANGES.length - 1;
|
|
724
|
+
while (lo <= hi) {
|
|
725
|
+
const mid = (lo + hi) >> 1;
|
|
726
|
+
const [, start, end] = BRAHMIC_CONSONANT_RANGES[mid];
|
|
727
|
+
if (cp < start) hi = mid - 1;
|
|
728
|
+
else if (cp > end) lo = mid + 1;
|
|
729
|
+
else return true;
|
|
730
|
+
}
|
|
684
731
|
return false;
|
|
685
732
|
}
|
package/src/layer1.mjs
CHANGED
|
@@ -24,14 +24,18 @@
|
|
|
24
24
|
import { stripInvisibleWithReport, CATEGORY, STRIP } from "./invisible.mjs";
|
|
25
25
|
import {
|
|
26
26
|
CONTROL_INTRODUCER_SOURCE,
|
|
27
|
+
DEFAULT_RENDITION,
|
|
28
|
+
foregroundMatchesBackground,
|
|
27
29
|
isCancelledOrphan,
|
|
28
30
|
isOrphanKind,
|
|
29
31
|
orphanKindFor,
|
|
30
32
|
scanAnsi,
|
|
31
|
-
|
|
33
|
+
sgrRendition,
|
|
32
34
|
TOKEN_KIND,
|
|
33
35
|
} from "./ansi.mjs";
|
|
34
36
|
|
|
37
|
+
/** @typedef {import("./ansi.mjs").SgrRendition} SgrRendition */
|
|
38
|
+
|
|
35
39
|
// The ANSI grammar and the introducer charset live in ./ansi.mjs so this module
|
|
36
40
|
// and invisible.mjs (which owns the public isSgrOnly / SGR_RE and cannot import
|
|
37
41
|
// this one) scan with the SAME tokenizer.
|
|
@@ -178,16 +182,24 @@ function rendersNothing(gap) {
|
|
|
178
182
|
/**
|
|
179
183
|
* True when the SGR sequences in `text` do more than style visible text — the
|
|
180
184
|
* question a consumer must answer before PRESERVING escapes rather than
|
|
181
|
-
* stripping them.
|
|
185
|
+
* stripping them. Three arms, all model-sees/human-sees divergences:
|
|
182
186
|
*
|
|
183
|
-
* 1. CONCEAL is on while text that would otherwise render goes by
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
187
|
+
* 1. CONCEAL is on while text that would otherwise render goes by. Only SGR
|
|
188
|
+
* moves that state (see {@link sgrRendition}) — a cursor move or an erase
|
|
189
|
+
* leaves a terminal exactly as concealed as it was.
|
|
190
|
+
* 2. The foreground is the background while a glyph goes by (`ESC[30;40m`):
|
|
191
|
+
* concealment by colour, judged only where the match is unambiguous (see
|
|
192
|
+
* {@link foregroundMatchesBackground}). Block Elements are the one
|
|
193
|
+
* exception judged across the whole text rather than per glyph (see
|
|
194
|
+
* {@link drawsSecondBlockElement}).
|
|
195
|
+
* 3. {@link SGR_RUN_THRESHOLD} sequences with nothing that renders between
|
|
188
196
|
* them: the run puts no glyph on the screen, so it carries data rather than
|
|
189
197
|
* styling.
|
|
190
198
|
*
|
|
199
|
+
* Either still in force when the text ENDS is a refusal too: rendition
|
|
200
|
+
* outlives its bytes, so a file ending in `ESC[8m` blanks whatever the terminal
|
|
201
|
+
* prints after it.
|
|
202
|
+
*
|
|
191
203
|
* What it does NOT claim: a message spread thinly through legitimately coloured
|
|
192
204
|
* text — one sequence per line, say — is indistinguishable from styling, and
|
|
193
205
|
* this answers no for it. That is the same bargain the invisible layer's
|
|
@@ -201,24 +213,71 @@ function rendersNothing(gap) {
|
|
|
201
213
|
* @returns {boolean}
|
|
202
214
|
*/
|
|
203
215
|
export function sgrCarriesPayload(text) {
|
|
204
|
-
|
|
216
|
+
/** @type {Readonly<SgrRendition>} */
|
|
217
|
+
let rendition = DEFAULT_RENDITION;
|
|
205
218
|
let run = 0;
|
|
206
219
|
let previousEnd = 0;
|
|
220
|
+
/** @type {Set<string>} */
|
|
221
|
+
const solidBlocks = new Set();
|
|
207
222
|
for (const token of scanAnsi(text)) {
|
|
208
|
-
const
|
|
223
|
+
const gap = text.slice(previousEnd, token.start);
|
|
224
|
+
const blankGap = rendersNothing(gap);
|
|
209
225
|
previousEnd = token.end;
|
|
226
|
+
// Judged ahead of the non-SGR exit: the gap before an erase or cursor move
|
|
227
|
+
// is drawn in the last SGR's state, so `ESC[30;40m` + text + `ESC[2J` +
|
|
228
|
+
// reset would otherwise hide the text unchecked.
|
|
229
|
+
if (hidesText(rendition, gap, solidBlocks)) return true;
|
|
210
230
|
if (token.kind !== TOKEN_KIND.SGR) {
|
|
211
231
|
run = 0;
|
|
212
232
|
continue;
|
|
213
233
|
}
|
|
214
|
-
// This refusal is what blocks `ESC[8m` from hiding text from the human
|
|
215
|
-
// while leaving its bytes for the model.
|
|
216
|
-
if (concealed && !blankGap) return true;
|
|
217
234
|
run = blankGap ? run + 1 : 1;
|
|
218
235
|
if (run >= SGR_RUN_THRESHOLD) return true;
|
|
219
|
-
|
|
236
|
+
rendition = sgrRendition(text.slice(token.start, token.end), rendition);
|
|
237
|
+
}
|
|
238
|
+
return rendition.concealed || foregroundMatchesBackground(rendition);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// A glyph whose identity a foreground painted in its own background hides:
|
|
242
|
+
// anything but whitespace (which looks the same either way) and the Block
|
|
243
|
+
// Elements, which drawsSecondBlockElement judges instead.
|
|
244
|
+
const COLOUR_HIDDEN_GLYPH = /[^\s\u2580-\u259F]/u;
|
|
245
|
+
const BLOCK_ELEMENT = /[\u2580-\u259F]/gu;
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* True when `gap`, drawn in a foreground matching its background, brings the
|
|
249
|
+
* text's count of distinct Block Elements (U+2580-U+259F) so drawn past one.
|
|
250
|
+
* In matching colours every one paints a solid cell, hiding the glyph CHOICE
|
|
251
|
+
* from a human; a single glyph throughout is what a half-block image renderer
|
|
252
|
+
* (catimg's `▀`) emits, so only a second one is refused. `seen` spans the whole
|
|
253
|
+
* text so the choice cannot be spread one glyph per sequence.
|
|
254
|
+
* @param {string} gap @param {Set<string>} seen
|
|
255
|
+
* @returns {boolean}
|
|
256
|
+
*/
|
|
257
|
+
function drawsSecondBlockElement(gap, seen) {
|
|
258
|
+
for (const [glyph] of gap.matchAll(BLOCK_ELEMENT)) {
|
|
259
|
+
seen.add(glyph);
|
|
260
|
+
if (seen.size > 1) return true;
|
|
220
261
|
}
|
|
221
|
-
return
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* True when `rendition` hides some of `gap` from a human: CONCEAL blanks every
|
|
267
|
+
* glyph, and a foreground matching its background hides every
|
|
268
|
+
* {@link COLOUR_HIDDEN_GLYPH} and the choice among Block Elements.
|
|
269
|
+
* @param {Readonly<SgrRendition>} rendition @param {string} gap
|
|
270
|
+
* @param {Set<string>} solidBlocks the Block Elements already drawn in
|
|
271
|
+
* matching colours, across the whole text
|
|
272
|
+
* @returns {boolean}
|
|
273
|
+
*/
|
|
274
|
+
function hidesText(rendition, gap, solidBlocks) {
|
|
275
|
+
if (rendition.concealed) return !rendersNothing(gap);
|
|
276
|
+
if (!foregroundMatchesBackground(rendition)) return false;
|
|
277
|
+
return (
|
|
278
|
+
COLOUR_HIDDEN_GLYPH.test(gap.replace(STRIP, "")) ||
|
|
279
|
+
drawsSecondBlockElement(gap, solidBlocks)
|
|
280
|
+
);
|
|
222
281
|
}
|
|
223
282
|
|
|
224
283
|
/**
|
|
@@ -233,21 +292,58 @@ export function isBenignAnsi(text) {
|
|
|
233
292
|
return isBenignAnsiKinds(applyLayer1(text).ansiKinds);
|
|
234
293
|
}
|
|
235
294
|
|
|
295
|
+
/**
|
|
296
|
+
* True when the escapes in `text` may be let through: every removed kind is
|
|
297
|
+
* inert ({@link isBenignAnsiKinds}) AND the SGR is just styling
|
|
298
|
+
* ({@link sgrCarriesPayload}). THE answer for every surface that preserves
|
|
299
|
+
* escapes, so `ESC[8m` cannot be inert to one and hiding text in another.
|
|
300
|
+
* @param {string} text the raw text, as a terminal would receive it
|
|
301
|
+
* @param {Iterable<string>} [kinds] the {@link TOKEN_KIND}s a strip of `text`
|
|
302
|
+
* removed, when the caller already has them; defaults to Layer 1's
|
|
303
|
+
* @returns {boolean}
|
|
304
|
+
*/
|
|
305
|
+
export function escapesPreservable(text, kinds = applyLayer1(text).ansiKinds) {
|
|
306
|
+
return isBenignAnsiKinds([...kinds]) && !sgrCarriesPayload(text);
|
|
307
|
+
}
|
|
308
|
+
|
|
236
309
|
// How many times the {ANSI strip, invisible strip} composition may re-run before
|
|
237
|
-
//
|
|
238
|
-
// loop terminates on its own; the bound is a DoS guard: n
|
|
239
|
-
// on adversarial input is quadratic.
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
//
|
|
243
|
-
//
|
|
244
|
-
// emoji neighbours on BOTH sides, so a preserved joiner can never sit inside an
|
|
245
|
-
// escape sequence, and every joiner that hides one is therefore stripped by the
|
|
246
|
-
// FIRST invisible pass. Past the bound the composition simply stops early — the
|
|
247
|
-
// final sweep still holds the no-introducer guarantee, and what is left degrades
|
|
248
|
-
// to visible text (the fail-open direction).
|
|
310
|
+
// it falls back to {@link stripLayer1Monotone}. Each iteration deletes at least
|
|
311
|
+
// one character, so the loop terminates on its own; the bound is a DoS guard: n
|
|
312
|
+
// rounds of O(n) scans on adversarial input is quadratic. Three rounds is what
|
|
313
|
+
// the deepest REAL reconstitution needs (strip invisibles, remove the ANSI they
|
|
314
|
+
// hid, strip the joiners that removal made adjacent, confirm). Adversarial input
|
|
315
|
+
// can need more — a registered variation selector preserved INSIDE an escape,
|
|
316
|
+
// whose preserve budget shrinks as each removal shortens the text.
|
|
249
317
|
const MAX_LAYER1_PASSES = 4;
|
|
250
318
|
|
|
319
|
+
/**
|
|
320
|
+
* Remove every complete ANSI sequence and then every raw control introducer
|
|
321
|
+
* left over. The output holds no introducer, so no later pass can find an
|
|
322
|
+
* escape in it: a fixed point by construction.
|
|
323
|
+
* @param {string} text
|
|
324
|
+
* @param {Set<string>} [kinds] collects the {@link TOKEN_KIND} of every removal
|
|
325
|
+
* @returns {string}
|
|
326
|
+
*/
|
|
327
|
+
export function stripTerminalFully(text, kinds = new Set()) {
|
|
328
|
+
return sweepIntroducers(stripAnsiFully(text, kinds), kinds);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* The carve-out-free Layer 1 a non-converging composition falls back to: EVERY
|
|
333
|
+
* invisible, then every escape and introducer, so its output is a fixed point.
|
|
334
|
+
* It starts from what the caller SENT: an earlier sweep may already have turned
|
|
335
|
+
* an escape's body into text no later strip can tell from prose.
|
|
336
|
+
* @param {string} text what the caller sent
|
|
337
|
+
* @param {Set<string>} found collects the invisible CATEGORY codes removed
|
|
338
|
+
* @param {Set<string>} kinds collects the {@link TOKEN_KIND} of every removal
|
|
339
|
+
* @returns {string}
|
|
340
|
+
*/
|
|
341
|
+
function stripLayer1Monotone(text, found, kinds) {
|
|
342
|
+
const bulk = stripInvisibleWithReport(text, text, { carveOut: false });
|
|
343
|
+
for (const category of bulk.found) found.add(category);
|
|
344
|
+
return stripTerminalFully(bulk.cleaned, kinds);
|
|
345
|
+
}
|
|
346
|
+
|
|
251
347
|
/**
|
|
252
348
|
* Layer 1: ANSI + invisible-char strip with a result guaranteed free of every
|
|
253
349
|
* raw ANSI control introducer (7-bit ESC U+001B and the whole 8-bit C1 control
|
|
@@ -266,9 +362,10 @@ const MAX_LAYER1_PASSES = 4;
|
|
|
266
362
|
*
|
|
267
363
|
* The residual sweep runs only once the composition is STABLE, because sweeping
|
|
268
364
|
* an introducer early destroys the sequence a later ANSI pass would have removed
|
|
269
|
-
* whole, promoting a hidden control to visible text. A
|
|
270
|
-
*
|
|
271
|
-
*
|
|
365
|
+
* whole, promoting a hidden control to visible text. A composition that has not
|
|
366
|
+
* converged within the pass bound falls back to a carve-out-free strip whose
|
|
367
|
+
* output holds no invisible and no introducer, so both the no-raw-introducer
|
|
368
|
+
* guarantee and idempotence hold whatever the bound.
|
|
272
369
|
*
|
|
273
370
|
* `deAnsi` is the ANSI strip of the ORIGINAL text (invisible runs intact), the
|
|
274
371
|
* scope a LONG_RUN payload check needs — not an intermediate of the loop.
|
|
@@ -288,6 +385,7 @@ export function applyLayer1(text) {
|
|
|
288
385
|
/** @type {Set<string>} Union of the categories every iteration reported. */
|
|
289
386
|
const found = new Set();
|
|
290
387
|
let cleaned = text;
|
|
388
|
+
let converged = false;
|
|
291
389
|
|
|
292
390
|
for (let pass = 0; pass < MAX_LAYER1_PASSES; pass++) {
|
|
293
391
|
const afterAnsi = pass === 0 ? deAnsi : stripAnsiFully(cleaned, ansiKinds);
|
|
@@ -312,11 +410,16 @@ export function applyLayer1(text) {
|
|
|
312
410
|
// introducer can make invisibles adjacent, exactly as removing a sequence
|
|
313
411
|
// can); one that changes nothing means the whole composition has converged.
|
|
314
412
|
const swept = sweepIntroducers(cleaned, ansiKinds);
|
|
315
|
-
if (swept === cleaned)
|
|
413
|
+
if (swept === cleaned) {
|
|
414
|
+
converged = true;
|
|
415
|
+
break;
|
|
416
|
+
}
|
|
316
417
|
cleaned = swept;
|
|
317
418
|
}
|
|
318
419
|
|
|
319
|
-
|
|
420
|
+
// Keeps Layer 1 a FIXED POINT past the pass bound: an unconverged state
|
|
421
|
+
// re-cleans to something else and may still hold an escape.
|
|
422
|
+
if (!converged) cleaned = stripLayer1Monotone(text, found, ansiKinds);
|
|
320
423
|
|
|
321
424
|
// Derived, not tracked in parallel: a kind is recorded exactly when an ANSI
|
|
322
425
|
// pass or the sweep removed bytes, so the category and the severity detail
|
package/src/prompt.mjs
CHANGED
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
* output (test runs, build logs) is the single most common debugging action,
|
|
15
15
|
* and SGR is display-only by the ECMA-48 grammar; an orphan `ESC` is not a
|
|
16
16
|
* sequence at all. Neither can move the cursor, erase the screen, or carry an
|
|
17
|
-
* OSC payload. Anything that IS a complete CSI/OSC token still blocks, as
|
|
18
|
-
*
|
|
17
|
+
* OSC payload. Anything that IS a complete CSI/OSC token still blocks, as does
|
|
18
|
+
* SGR that hides text (CONCEAL, a foreground in its own background, a run that
|
|
19
|
+
* renders nothing) and the invisible-char verdict.
|
|
19
20
|
*/
|
|
20
21
|
import {
|
|
21
22
|
CHECKS,
|
|
@@ -23,10 +24,9 @@ import {
|
|
|
23
24
|
CATEGORY_LABELS,
|
|
24
25
|
LONG_RUN_THRESHOLD,
|
|
25
26
|
SCATTERED_THRESHOLD,
|
|
26
|
-
|
|
27
|
-
payloadLongRunSample,
|
|
27
|
+
payloadVerdict,
|
|
28
28
|
} from "./invisible.mjs";
|
|
29
|
-
import {
|
|
29
|
+
import { escapesPreservable, stripAnsiFully } from "./layer1.mjs";
|
|
30
30
|
import { CONTROL_INTRODUCER_SOURCE } from "./ansi.mjs";
|
|
31
31
|
|
|
32
32
|
// Every raw ANSI control a prompt can carry: 7-bit ESC (U+001B) and the entire
|
|
@@ -95,27 +95,21 @@ export function classifyPrompt(prompt, strip = stripAnsiFully) {
|
|
|
95
95
|
const hasAnsi = ANSI_INTRODUCER.test(prompt);
|
|
96
96
|
const deAnsi = strip(prompt);
|
|
97
97
|
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
|
|
102
|
-
// countEffectiveInvisible — including why the run probe masks the invisibles
|
|
103
|
-
// the strip layer would PRESERVE, so a legitimate emoji sequence cannot block
|
|
104
|
-
// a prompt that the strip layer would not even have flagged.
|
|
105
|
-
const longRunSample = payloadLongRunSample(deAnsi);
|
|
106
|
-
const invisibleCount = countEffectiveInvisible(deAnsi);
|
|
107
|
-
const invisiblesBelowThreshold =
|
|
108
|
-
longRunSample === null && invisibleCount < SCATTERED_THRESHOLD;
|
|
98
|
+
// The invisible half is payloadVerdict — the one definition the instruction
|
|
99
|
+
// scanner and the write path also gate on — so a legitimate emoji sequence
|
|
100
|
+
// cannot block a prompt that the other surfaces would not even have flagged.
|
|
101
|
+
const verdict = payloadVerdict(deAnsi);
|
|
109
102
|
|
|
110
|
-
if (!hasAnsi &&
|
|
103
|
+
if (!hasAnsi && !verdict.payloadCapable) return { action: "pass" };
|
|
111
104
|
|
|
112
105
|
// Inert escapes in an otherwise clean prompt — display-only colour and/or an
|
|
113
106
|
// orphan introducer that forms no sequence: pass with a note instead of
|
|
114
107
|
// blocking, so pasted colored logs and log fragments cut mid-escape remain
|
|
115
|
-
// usable.
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
|
|
108
|
+
// usable. escapesPreservable judges from what Layer 1's strip actually
|
|
109
|
+
// removed (the whole C1 block, and any sequence that only reconstitutes during
|
|
110
|
+
// the strip) AND refuses SGR that hides text; anything else falls through to
|
|
111
|
+
// the block.
|
|
112
|
+
if (hasAnsi && !verdict.payloadCapable && escapesPreservable(prompt))
|
|
119
113
|
return { action: "note" };
|
|
120
114
|
|
|
121
115
|
// CHECKS pairs a machine-readable category code with its detector; map each
|
|
@@ -126,6 +120,10 @@ export function classifyPrompt(prompt, strip = stripAnsiFully) {
|
|
|
126
120
|
if (hasAnsi) categories.push(CATEGORY_LABELS[CATEGORY.ANSI]);
|
|
127
121
|
return {
|
|
128
122
|
action: "block",
|
|
129
|
-
reason: formatReason(
|
|
123
|
+
reason: formatReason(
|
|
124
|
+
categories,
|
|
125
|
+
verdict.invisibleCount,
|
|
126
|
+
verdict.firstRun?.text ?? null,
|
|
127
|
+
),
|
|
130
128
|
};
|
|
131
129
|
}
|
|
@@ -2,19 +2,19 @@
|
|
|
2
2
|
* GENERATED by scripts/gen-standardized-variants.mjs from UCD
|
|
3
3
|
* StandardizedVariants.txt (Unicode 17.0.0) — DO NOT EDIT.
|
|
4
4
|
*
|
|
5
|
-
* Registered standardized variation sequences: a base code point followed by
|
|
6
|
-
* variation selector
|
|
7
|
-
* only when the immediately preceding base appears here; every other
|
|
8
|
-
*
|
|
9
|
-
* `node scripts/gen-standardized-variants.mjs`;
|
|
10
|
-
* fails if this drifts from the pinned UCD
|
|
5
|
+
* Registered standardized variation sequences: a base code point followed by the
|
|
6
|
+
* variation selector the UCD registers for it. invisible.mjs preserves such a
|
|
7
|
+
* selector only when the immediately preceding base appears here; every other
|
|
8
|
+
* one is stripped as a hidden-payload byte. Regenerate with
|
|
9
|
+
* `node scripts/gen-standardized-variants.mjs`;
|
|
10
|
+
* test/invisible-charset.test.mjs fails if this drifts from the pinned UCD.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
export const UNICODE_VERSION = "17.0.0";
|
|
14
14
|
|
|
15
15
|
// The table below is GENERATED UCD data, not hand-written logic. Its correctness
|
|
16
16
|
// is pinned by the SSOT round-trip in test/invisible-charset.test.mjs (re-parse
|
|
17
|
-
// the vendored
|
|
17
|
+
// the vendored file, then assert every pair matches) — a stronger and faster
|
|
18
18
|
// guard than mutating a few hundred numeric literals, so Stryker skips them.
|
|
19
19
|
// Stryker disable all
|
|
20
20
|
/** @type {Array<[number, number]>} [base, selector], sorted by base then selector. */
|
|
@@ -36,6 +36,66 @@ export const STANDARDIZED_VARIANTS = [
|
|
|
36
36
|
[0x1078, 0xfe00],
|
|
37
37
|
[0x107a, 0xfe00],
|
|
38
38
|
[0x1080, 0xfe00],
|
|
39
|
+
[0x1820, 0x180b],
|
|
40
|
+
[0x1820, 0x180c],
|
|
41
|
+
[0x1821, 0x180b],
|
|
42
|
+
[0x1822, 0x180b],
|
|
43
|
+
[0x1823, 0x180b],
|
|
44
|
+
[0x1824, 0x180b],
|
|
45
|
+
[0x1825, 0x180b],
|
|
46
|
+
[0x1825, 0x180c],
|
|
47
|
+
[0x1826, 0x180b],
|
|
48
|
+
[0x1826, 0x180c],
|
|
49
|
+
[0x1828, 0x180b],
|
|
50
|
+
[0x1828, 0x180c],
|
|
51
|
+
[0x1828, 0x180d],
|
|
52
|
+
[0x182a, 0x180b],
|
|
53
|
+
[0x182c, 0x180b],
|
|
54
|
+
[0x182c, 0x180c],
|
|
55
|
+
[0x182c, 0x180d],
|
|
56
|
+
[0x182d, 0x180b],
|
|
57
|
+
[0x182d, 0x180c],
|
|
58
|
+
[0x182d, 0x180d],
|
|
59
|
+
[0x1830, 0x180b],
|
|
60
|
+
[0x1830, 0x180c],
|
|
61
|
+
[0x1832, 0x180b],
|
|
62
|
+
[0x1833, 0x180b],
|
|
63
|
+
[0x1835, 0x180b],
|
|
64
|
+
[0x1836, 0x180b],
|
|
65
|
+
[0x1836, 0x180c],
|
|
66
|
+
[0x1838, 0x180b],
|
|
67
|
+
[0x1844, 0x180b],
|
|
68
|
+
[0x1845, 0x180b],
|
|
69
|
+
[0x1846, 0x180b],
|
|
70
|
+
[0x1847, 0x180b],
|
|
71
|
+
[0x1847, 0x180c],
|
|
72
|
+
[0x1848, 0x180b],
|
|
73
|
+
[0x1849, 0x180b],
|
|
74
|
+
[0x184d, 0x180b],
|
|
75
|
+
[0x184e, 0x180b],
|
|
76
|
+
[0x185d, 0x180b],
|
|
77
|
+
[0x185e, 0x180b],
|
|
78
|
+
[0x185e, 0x180c],
|
|
79
|
+
[0x1860, 0x180b],
|
|
80
|
+
[0x1863, 0x180b],
|
|
81
|
+
[0x1868, 0x180b],
|
|
82
|
+
[0x1868, 0x180c],
|
|
83
|
+
[0x1869, 0x180b],
|
|
84
|
+
[0x186f, 0x180b],
|
|
85
|
+
[0x1873, 0x180b],
|
|
86
|
+
[0x1873, 0x180c],
|
|
87
|
+
[0x1873, 0x180d],
|
|
88
|
+
[0x1874, 0x180b],
|
|
89
|
+
[0x1874, 0x180c],
|
|
90
|
+
[0x1874, 0x180d],
|
|
91
|
+
[0x1876, 0x180b],
|
|
92
|
+
[0x1880, 0x180b],
|
|
93
|
+
[0x1881, 0x180b],
|
|
94
|
+
[0x1887, 0x180b],
|
|
95
|
+
[0x1887, 0x180c],
|
|
96
|
+
[0x1887, 0x180d],
|
|
97
|
+
[0x1888, 0x180b],
|
|
98
|
+
[0x188a, 0x180b],
|
|
39
99
|
[0x2018, 0xfe00],
|
|
40
100
|
[0x2018, 0xfe01],
|
|
41
101
|
[0x2018, 0xfe02],
|
|
@@ -1324,8 +1384,8 @@ const VARIANT_KEYS = new Set(
|
|
|
1324
1384
|
);
|
|
1325
1385
|
|
|
1326
1386
|
/**
|
|
1327
|
-
* True when `base` followed by `selector`
|
|
1328
|
-
*
|
|
1387
|
+
* True when `base` followed by `selector` is a registered standardized
|
|
1388
|
+
* variation sequence.
|
|
1329
1389
|
* @param {number} base
|
|
1330
1390
|
* @param {number} selector
|
|
1331
1391
|
* @returns {boolean}
|