agent-sanitizer 2.34.7 → 2.34.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/package.json +1 -1
  2. package/src/html.mjs +108 -14
  3. package/src/invisible.mjs +548 -238
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-sanitizer",
3
- "version": "2.34.7",
3
+ "version": "2.34.8",
4
4
  "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
5
5
  "type": "module",
6
6
  "repository": {
package/src/html.mjs CHANGED
@@ -36,7 +36,7 @@ import { unified } from "unified";
36
36
  import remarkParse from "remark-parse";
37
37
  import remarkGfm from "remark-gfm";
38
38
  import rehypeParse from "rehype-parse";
39
- import { visit, SKIP, EXIT } from "unist-util-visit";
39
+ import { SKIP, EXIT } from "unist-util-visit";
40
40
  import {
41
41
  HTML_TAG_PRESENT,
42
42
  MD_LINK_HINT,
@@ -1374,14 +1374,93 @@ function hasDataSrc(el) {
1374
1374
  // there is exactly one parser configuration to reason about.
1375
1375
  const htmlParser = unified().use(rehypeParse, { fragment: true });
1376
1376
 
1377
+ /**
1378
+ * Preorder depth-first walk of a unist tree, calling `visitor(node, index,
1379
+ * parent)` on every node whose `type` is `test` (or on every node when `test` is
1380
+ * null). `EXIT` ends the walk, `SKIP` leaves the node's children unvisited —
1381
+ * the `unist-util-visit` contract, which is what the call sites here are
1382
+ * written against.
1383
+ *
1384
+ * Spelled out rather than imported because `unist-util-visit` allocates a fresh
1385
+ * ancestors array and a closure per node, and on a document that is one long
1386
+ * flat sibling list — 32k `<p>` elements per megabyte of ordinary HTML — that
1387
+ * turns a linear walk super-linear: 3480 ms against this walk's 62 ms over the
1388
+ * same 4 MB tree. The three parallel arrays are the stack, so the walk itself
1389
+ * allocates nothing per node.
1390
+ * @param {any} tree
1391
+ * @param {string | null} test
1392
+ * @param {(node: any, index: number | undefined, parent: any) => unknown} visitor
1393
+ */
1394
+ function walk(tree, test, visitor) {
1395
+ /** @type {any[]} */
1396
+ const nodes = [tree];
1397
+ /** @type {Array<number | undefined>} */
1398
+ const indices = [undefined];
1399
+ /** @type {any[]} */
1400
+ const parents = [undefined];
1401
+ while (nodes.length > 0) {
1402
+ const node = nodes.pop();
1403
+ const index = indices.pop();
1404
+ const parent = parents.pop();
1405
+ const result =
1406
+ test === null || node.type === test
1407
+ ? visitor(node, index, parent)
1408
+ : undefined;
1409
+ if (result === EXIT) return;
1410
+ if (result === SKIP) continue;
1411
+ const children = node.children;
1412
+ if (children === undefined) continue;
1413
+ for (let i = children.length - 1; i >= 0; i--) {
1414
+ nodes.push(children[i]);
1415
+ indices.push(i);
1416
+ parents.push(node);
1417
+ }
1418
+ }
1419
+ }
1420
+
1421
+ /**
1422
+ * `parse` with its most recent (input, tree) pair remembered.
1423
+ *
1424
+ * Layers 2 and 3 each parse the SAME tool output: `sanitizeHtml` tokenizes it to
1425
+ * decide the source-vs-markdown branch, and `detectExfil` tokenizes it again to
1426
+ * read `src`/`href` off the elements — two full parse5 runs and two full
1427
+ * micromark runs over one document, which is most of what the HTML layer costs
1428
+ * on ordinary prose. Every consumer only READS the tree (the sole AST mutation
1429
+ * in this module is over a css-tree value parsed per style string), so one tree
1430
+ * is safe to hand to both.
1431
+ *
1432
+ * One entry, replaced on every miss, so the retained footprint is one tree for
1433
+ * the document most recently sanitized — the tree that call had allocated
1434
+ * anyway. A miss re-parses and answers identically, so the cache can never
1435
+ * change a verdict, only what one costs.
1436
+ * @param {(text: string) => any} parse
1437
+ * @returns {(text: string) => any}
1438
+ */
1439
+ function lastParseCached(parse) {
1440
+ /** @type {string | null} */
1441
+ let cachedText = null;
1442
+ /** @type {any} */
1443
+ let cachedTree = null;
1444
+ return (text) => {
1445
+ if (cachedText === text) return cachedTree;
1446
+ // The key is recorded only AFTER the parse returns. A parse that throws —
1447
+ // which is how a pathologically nested fragment reaches the fail-closed
1448
+ // withhold — must leave the entry untouched, or the next call for that same
1449
+ // text hits a key whose tree came from a DIFFERENT document and gets a
1450
+ // verdict about the wrong input instead of the withhold.
1451
+ const tree = parse(text);
1452
+ cachedText = text;
1453
+ cachedTree = tree;
1454
+ return tree;
1455
+ };
1456
+ }
1457
+
1377
1458
  /**
1378
1459
  * Parse `html` as an HTML fragment with the real tokenizer (parse5, via rehype).
1379
1460
  * @param {string} html
1380
1461
  * @returns {any}
1381
1462
  */
1382
- function parseFragment(html) {
1383
- return htmlParser.parse(html);
1384
- }
1463
+ const parseFragment = lastParseCached((html) => htmlParser.parse(html));
1385
1464
 
1386
1465
  /**
1387
1466
  * @param {string} htmlValue
@@ -1391,7 +1470,7 @@ function parseHtmlTag(htmlValue) {
1391
1470
  const tree = parseFragment(htmlValue);
1392
1471
  /** @type {any} */
1393
1472
  let firstElement = null;
1394
- visit(tree, "element", (node) => {
1473
+ walk(tree, "element", (node) => {
1395
1474
  firstElement = node;
1396
1475
  return EXIT;
1397
1476
  });
@@ -1606,7 +1685,7 @@ function scanFragmentTree(html, tree) {
1606
1685
  const warned = newWarned();
1607
1686
  // @ts-ignore -- visit callback returns EXIT/SKIP only on matches; implicit undefined return is intentional
1608
1687
  // eslint-disable-next-line consistent-return
1609
- visit(tree, (/** @type {any} */ node) => {
1688
+ walk(tree, null, (/** @type {any} */ node) => {
1610
1689
  const isComment = node.type === "comment";
1611
1690
  if (isComment || isHiddenElement(node)) {
1612
1691
  /* c8 ignore start -- parse5 omits positions only on recovery-synthesized
@@ -1635,6 +1714,20 @@ function scanFragmentTree(html, tree) {
1635
1714
 
1636
1715
  const mdParser = unified().use(remarkParse).use(remarkGfm);
1637
1716
 
1717
+ /** The markdown tree for `text`, cached the same way {@link parseFragment} is.
1718
+ * @type {(text: string) => any} */
1719
+ const parseMarkdown = lastParseCached((text) => mdParser.parse(text));
1720
+
1721
+ // A `code` node is a FENCED or INDENTED block and nothing else, so either a
1722
+ // three-run of backticks/tildes or a line opening on a four-column indent must
1723
+ // appear somewhere in the text for one to exist. CommonMark expands a tab to
1724
+ // the next four-column tab stop, so fewer than four spaces followed by a tab is
1725
+ // an indent too (" \tfoo" is a code block) — hence ` *\t` rather than `\t`.
1726
+ // Read over the whole document and deliberately loose (a stray ``` anywhere is
1727
+ // enough to parse), because the only sound direction to be wrong in here is
1728
+ // towards parsing.
1729
+ const MARKDOWN_CODE_HINT = /```|~~~|^(?: {4}| *\t)/m;
1730
+
1638
1731
  // A markup-declaration-open (`<!`) or processing-instruction-ish (`<?`) start.
1639
1732
  // Inside an inline html node these begin a *bogus comment* unless they open a
1640
1733
  // proper `<!--…-->` comment (handled on the fast path) — `<!bogus>`, `<?php?>`,
@@ -1688,7 +1781,7 @@ function commentSpans(value) {
1688
1781
  const tree = parseFragment(value);
1689
1782
  /** @type {Map<number, number>} */
1690
1783
  const spans = new Map();
1691
- visit(tree, "comment", (/** @type {any} */ node) => {
1784
+ walk(tree, "comment", (/** @type {any} */ node) => {
1692
1785
  if (node.position)
1693
1786
  spans.set(node.position.start.offset, node.position.end.offset);
1694
1787
  });
@@ -1916,7 +2009,7 @@ const FLOW_HTML_PARENTS = new Set([
1916
2009
  * @returns {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }}
1917
2010
  */
1918
2011
  function scanMarkdown(text) {
1919
- const tree = mdParser.parse(text);
2012
+ const tree = parseMarkdown(text);
1920
2013
  /** @type {SpliceRange[]} */
1921
2014
  const ranges = [];
1922
2015
  const warned = newWarned();
@@ -1924,7 +2017,7 @@ function scanMarkdown(text) {
1924
2017
  // Flow html blocks carry complete markup, so rehype locates comments/hidden
1925
2018
  // elements precisely within them; block-local offsets are shifted to
1926
2019
  // document coordinates.
1927
- visit(tree, "html", (/** @type {any} */ node, _index, parent) => {
2020
+ walk(tree, "html", (/** @type {any} */ node, _index, parent) => {
1928
2021
  if (!FLOW_HTML_PARENTS.has(parent?.type)) return;
1929
2022
  const base = node.position.start.offset;
1930
2023
  const sub = scanHtmlFragment(text.slice(base, node.position.end.offset));
@@ -1944,7 +2037,7 @@ function scanMarkdown(text) {
1944
2037
  // are walked as part of their root in document order, so the walk is skipped
1945
2038
  // for them here to avoid double-scanning and to keep the absorb state flowing
1946
2039
  // across those boundaries.
1947
- visit(tree, (/** @type {any} */ node) => {
2040
+ walk(tree, null, (/** @type {any} */ node) => {
1948
2041
  if (!PHRASING_ROOTS.has(node.type)) return;
1949
2042
  if (!hasHtmlLeaf(node)) return;
1950
2043
  scanInlineChildren(node, text, ranges, warned);
@@ -1969,8 +2062,9 @@ function scanMarkdown(text) {
1969
2062
  * @returns {boolean}
1970
2063
  */
1971
2064
  function hasMarkdownCode(text) {
2065
+ if (!MARKDOWN_CODE_HINT.test(text)) return false;
1972
2066
  let found = false;
1973
- visit(mdParser.parse(text), "code", () => {
2067
+ walk(parseMarkdown(text), "code", () => {
1974
2068
  found = true;
1975
2069
  return EXIT;
1976
2070
  });
@@ -2564,7 +2658,7 @@ function extractHtmlUrls(text) {
2564
2658
  const tree = parseFragment(text);
2565
2659
  /** @type {Array<{ url: string, isImage: boolean, autoFetched: boolean, context: "resource" | "form" | "refresh" }>} */
2566
2660
  const urls = [];
2567
- visit(tree, "element", (/** @type {any} */ node) => {
2661
+ walk(tree, "element", (/** @type {any} */ node) => {
2568
2662
  // hast element nodes always carry a `properties` object (parse5 sets it).
2569
2663
  const props = node.properties;
2570
2664
  const isImage = node.tagName === "img";
@@ -2640,8 +2734,8 @@ export function detectExfil(text) {
2640
2734
  try {
2641
2735
  // Remark AST handles markdown links/images/definitions (balanced parens,
2642
2736
  // reference links) correctly, unlike a hand-rolled regex.
2643
- const tree = mdParser.parse(text);
2644
- visit(tree, (node) => {
2737
+ const tree = parseMarkdown(text);
2738
+ walk(tree, null, (node) => {
2645
2739
  if (
2646
2740
  node.type !== "link" &&
2647
2741
  node.type !== "image" &&
package/src/invisible.mjs CHANGED
@@ -405,6 +405,13 @@ const EMOJI_BASE = /\p{Extended_Pictographic}/u;
405
405
  // following ZWJ (🏳️‍🌈 = flag base, VS16, ZWJ, rainbow; 👁️‍🗨️), so the joiner's real
406
406
  // left neighbor for the emoji test is the pictograph, not the selector.
407
407
  const VARIATION_SELECTOR = new RegExp(`[${VS}]`, "u");
408
+ // Code-point twins of the four single-character classes above, memoized: the
409
+ // carve analysis asks these per invisible (and per neighbour), and a Unicode
410
+ // property class costs far more than a table hit. See memoizedCpPredicate.
411
+ const isEmojiLeft = memoizedCpPredicate(EMOJI_LEFT);
412
+ const isEmojiBase = memoizedCpPredicate(EMOJI_BASE);
413
+ const isKeycapBase = memoizedCpPredicate(KEYCAP_BASE);
414
+ const isVariationSelectorCp = memoizedCpPredicate(VARIATION_SELECTOR);
408
415
  // U+FE0F (VS16) forces emoji presentation, U+FE0E (VS15) forces text
409
416
  // presentation (☺︎ vs ☺); either one directly after a pictograph is part of a
410
417
  // visible glyph, not a hidden variation-selector run.
@@ -462,11 +469,10 @@ const IVS_MAX = 0xe01ef;
462
469
  const CJK_IDEOGRAPH_RE =
463
470
  /[\p{Unified_Ideograph}\u{F900}-\u{FAFF}\u{2F800}-\u{2FA1F}]/u;
464
471
 
465
- /** True when `ch` is a CJK ideograph (the only base an ideographic variation
466
- * selector legitimately follows). @param {string} ch @returns {boolean} */
467
- function isCjkIdeograph(ch) {
468
- return CJK_IDEOGRAPH_RE.test(ch);
469
- }
472
+ /** True when `cp` is a CJK ideograph (the only base an ideographic variation
473
+ * selector legitimately follows). -1 (past a string boundary) is not.
474
+ * @type {(cp: number) => boolean} */
475
+ const isCjkIdeograph = memoizedCpPredicate(CJK_IDEOGRAPH_RE);
470
476
 
471
477
  // Brahmic consonants: the only base a virama does half-form/conjunct work on.
472
478
  // A bare or base-less halant + ZWJ carries no rendering and is a smuggling
@@ -485,15 +491,11 @@ function isCjkIdeograph(ch) {
485
491
  // against the script it claims.
486
492
  export { BRAHMIC_CONSONANT_RANGES } from "./joining-type.mjs";
487
493
 
488
- /** True when `ch` is a Brahmic consonant. Takes a CHARACTER, like its sibling
489
- * predicates here (`isCjkIdeograph`, `isJoinControl`), over the code-point
490
- * `isBrahmicConsonant` it wraps in ./joining-type.mjs, where every predicate
491
- * takes a code point.
492
- * @param {string} ch @returns {boolean} */
493
- function isBrahmicConsonantChar(ch) {
494
- return (
495
- ch !== "" && isBrahmicConsonant(/** @type {number} */ (ch.codePointAt(0)))
496
- );
494
+ /** True when the code point `cp` is a Brahmic consonant; -1 (past a string
495
+ * boundary) is not. A thin sign-guard over ./joining-type.mjs's table lookup.
496
+ * @param {number} cp @returns {boolean} */
497
+ function isBrahmicConsonantCp(cp) {
498
+ return cp >= 0 && isBrahmicConsonant(cp);
497
499
  }
498
500
 
499
501
  // ─── Blank-filler carve-out (Braille / archaic Hangul) ───────────────────────
@@ -528,11 +530,12 @@ const GATED_BLANK_RE = new RegExp("[\\u115F\\u1160\\u2800\\u3164\\uFFA0]", "u");
528
530
  // Script_Extensions=Braille — no character is shared with another script).
529
531
  const BRAILLE_RE = /\p{Script=Braille}/u;
530
532
 
533
+ const isBrailleScript = memoizedCpPredicate(BRAILLE_RE);
534
+
531
535
  /** A real (non-blank) Braille cell — the anchoring neighbour for a U+2800 blank.
532
- * @param {string} ch @returns {boolean} */
533
- function isBrailleCell(ch) {
534
- const cp = ch ? ch.codePointAt(0) : -1;
535
- return cp !== BRAILLE_BLANK && BRAILLE_RE.test(ch);
536
+ * @param {number} cp @returns {boolean} */
537
+ function isBrailleCell(cp) {
538
+ return cp !== BRAILLE_BLANK && isBrailleScript(cp);
536
539
  }
537
540
 
538
541
  // Script=Hangul, straight from the runtime's Unicode data. NOT
@@ -542,28 +545,61 @@ function isBrailleCell(ch) {
542
545
  // thereby preserve — a Hangul filler in text with no Hangul in it at all.
543
546
  const HANGUL_RE = /\p{Script=Hangul}/u;
544
547
 
548
+ const isHangulScript = memoizedCpPredicate(HANGUL_RE);
549
+
545
550
  /** A Hangul jamo/syllable (NOT itself one of the fillers) — the anchoring
546
- * neighbour for a Hangul filler. @param {string} ch @returns {boolean} */
547
- function isHangul(ch) {
548
- const cp = ch ? /** @type {number} */ (ch.codePointAt(0)) : -1;
551
+ * neighbour for a Hangul filler. @param {number} cp @returns {boolean} */
552
+ function isHangul(cp) {
549
553
  if (HANGUL_FILLERS.has(cp)) return false; // a filler cannot anchor another filler
550
- return HANGUL_RE.test(ch);
554
+ return isHangulScript(cp);
551
555
  }
552
556
 
553
- // code point -> CHECKS category, projected from the SAME three code-point sets
554
- // the CHECKS regexes are built from, in CHECKS order so a code point in two sets
555
- // gets the category the regexes would report first. Every CHECKS class holds
557
+ // The carve analysis runs once per code point of every prompt and every tool
558
+ // output, so its per-code-point state is held in TYPED arrays of small integer
559
+ // codes rather than arrays of category strings: `Uint8Array` costs one byte per
560
+ // code point with no pointer to trace, where a `(string|null)[]` costs a machine
561
+ // word per entry and hands the collector one more object graph to walk on every
562
+ // blocking hook call. These are the codes; CODE_CATEGORY projects them back onto
563
+ // the public CATEGORY strings at the boundary, so nothing outside this module
564
+ // sees them.
565
+ const CODE_VISIBLE = 0;
566
+ const CODE_CF = 1;
567
+ const CODE_VS = 2;
568
+ const CODE_BLANK = 3;
569
+
570
+ // Preserve kinds, in the same one-byte-per-code-point spirit. KIND_NONE is the
571
+ // zero value, so a freshly allocated `kind` array already reads "payload" —
572
+ // which is the fail-closed default this analysis wants.
573
+ const KIND_NONE = 0;
574
+ const KIND_JOINER = 1;
575
+ const KIND_EMOJIVS = 2;
576
+ const KIND_TAG = 3;
577
+ const KIND_STDVS = 4;
578
+ const KIND_IVS = 5;
579
+ const KIND_BLANK = 6;
580
+
581
+ /** Small code -> CHECKS category, indexed by the CODE_* values above. In CHECKS
582
+ * order, so a code point in two sets reports the category the regexes would
583
+ * report first. @type {(string|null)[]} */
584
+ const CODE_CATEGORY = [
585
+ null,
586
+ CATEGORY.CF,
587
+ CATEGORY.VARIATION_SELECTORS,
588
+ CATEGORY.BLANK_FILLERS,
589
+ ];
590
+
591
+ // code point -> CODE_* above, projected from the SAME three code-point sets the
592
+ // CHECKS regexes are built from, in CHECKS order. Every CHECKS class holds
556
593
  // single code points under the `u` flag, so a lookup and a `.test` answer
557
594
  // identically; test/invisible-fast-path.test.mjs pins that over the whole
558
595
  // code-point space, because a code point dropped here is one the scatter gate
559
- // stops counting. A lookup rather than the regexes because this runs once per
560
- // code point of every prompt and tool output.
561
- /** @type {Map<number, string>} */
596
+ // stops counting.
597
+ /** @type {Map<number, number>} */
562
598
  const TRACKED_INVISIBLE = new Map();
563
- for (const [code, codepoints] of /** @type {[string, Iterable<number>][]} */ ([
564
- [CATEGORY.CF, CF_CODEPOINTS],
565
- [CATEGORY.VARIATION_SELECTORS, [...VS].map((ch) => ch.codePointAt(0))],
566
- [CATEGORY.BLANK_FILLERS, [...BLANK_NON_CF].map((ch) => ch.codePointAt(0))],
599
+ for (const [code, codepoints] of /** @type {[number, Iterable<number>][]} */ ([
600
+ [CODE_CF, CF_CODEPOINTS],
601
+ [CODE_VS, [...VS].map((ch) => ch.codePointAt(0))],
602
+ [CODE_BLANK, [...BLANK_NON_CF].map((ch) => ch.codePointAt(0))],
567
603
  ]))
568
604
  for (const cp of codepoints)
569
605
  if (!TRACKED_INVISIBLE.has(cp)) TRACKED_INVISIBLE.set(cp, code);
@@ -573,32 +609,87 @@ for (const [code, codepoints] of /** @type {[string, Iterable<number>][]} */ ([
573
609
  const MIN_TRACKED_CP = Math.min(...TRACKED_INVISIBLE.keys());
574
610
 
575
611
  /**
576
- * The CHECKS category code (a CATEGORY value) a code point belongs to, or null
577
- * when it is not payload-capable (an ordinary visible character).
612
+ * The CODE_* class a code point belongs to, or CODE_VISIBLE when it is not
613
+ * payload-capable (an ordinary visible character).
578
614
  * @param {number} cp
579
- * @returns {string | null}
615
+ * @returns {number}
580
616
  */
581
617
  function classifyCp(cp) {
582
- if (cp < MIN_TRACKED_CP) return null;
583
- return TRACKED_INVISIBLE.get(cp) ?? null;
618
+ if (cp < MIN_TRACKED_CP) return CODE_VISIBLE;
619
+ return TRACKED_INVISIBLE.get(cp) ?? CODE_VISIBLE;
584
620
  }
585
621
 
622
+ // Distinct code points a memoizedCpPredicate will remember. Comfortably above
623
+ // the character repertoire of any real document (and of every test corpus here)
624
+ // while bounding what a single hostile paste can make one predicate retain.
625
+ const CP_PREDICATE_MEMO_CAP = 4096;
626
+
586
627
  /**
587
- * The CHECKS category code (a CATEGORY value) a single code point belongs to,
588
- * or null when it is not payload-capable (an ordinary visible character).
589
- * @param {string} ch one code point
590
- * @returns {string | null}
628
+ * A code-point predicate backed by a single-code-point RegExp, memoized on the
629
+ * code point.
630
+ *
631
+ * The carve analysis asks these of the same handful of code points over and
632
+ * over — a document is a few hundred distinct characters repeated — and each
633
+ * ask otherwise costs a `String.fromCodePoint` allocation plus a Unicode
634
+ * property-class match. The memo is a pure accelerator: it answers exactly what
635
+ * the RegExp answers, for the same input, forever (a code point's Unicode
636
+ * properties do not change inside a process).
637
+ *
638
+ * The entry cap is a MEMORY policy, not a scan bound: past it the predicate
639
+ * still answers every code point, just without recording new ones, so no input
640
+ * can change a verdict by overflowing the table — only by giving up the speed-up
641
+ * it would otherwise have had.
642
+ * @param {RegExp} re single-code-point matcher
643
+ * @returns {(cp: number) => boolean}
644
+ */
645
+ function memoizedCpPredicate(re) {
646
+ /** @type {Map<number, boolean>} */
647
+ const memo = new Map();
648
+ return (cp) => {
649
+ if (cp < 0) return false; // past a string boundary: the "" neighbour
650
+ const hit = memo.get(cp);
651
+ if (hit !== undefined) return hit;
652
+ const answer = re.test(String.fromCodePoint(cp));
653
+ if (memo.size < CP_PREDICATE_MEMO_CAP) memo.set(cp, answer);
654
+ return answer;
655
+ };
656
+ }
657
+
658
+ /**
659
+ * `text` as its code points, in the units the String iterator yields — the same
660
+ * sequence `Array.from(text)` produces (an unpaired surrogate stands alone as
661
+ * itself), as numbers rather than one heap-allocated one-character string each.
662
+ * Every carve-analysis predicate reads a code point, so the numbers are the
663
+ * whole requirement: a 256 KB paste that would cost a quarter of a million
664
+ * short-lived strings per pass costs one buffer, and the collector stays out of
665
+ * a blocking hook.
666
+ * @param {string} text
667
+ * @returns {Int32Array}
591
668
  */
592
- function classify(ch) {
593
- return classifyCp(/** @type {number} */ (ch.codePointAt(0)));
669
+ function codePointArray(text) {
670
+ const cps = new Int32Array(text.length);
671
+ let n = 0;
672
+ for (let i = 0; i < text.length; i++) {
673
+ const unit = text.charCodeAt(i);
674
+ let cp = unit;
675
+ if (unit >= 0xd800 && unit <= 0xdbff && i + 1 < text.length) {
676
+ const low = text.charCodeAt(i + 1);
677
+ if (low >= 0xdc00 && low <= 0xdfff) {
678
+ cp = (unit - 0xd800) * 0x400 + (low - 0xdc00) + 0x10000;
679
+ i++;
680
+ }
681
+ }
682
+ cps[n++] = cp;
683
+ }
684
+ return n === cps.length ? cps : cps.subarray(0, n);
594
685
  }
595
686
 
596
687
  /**
597
- * True when `text` holds at least one code point {@link classify} calls
598
- * invisible. Walks UTF-16 units directly rather than iterating code-point
599
- * strings, so a clean multi-MB paste is answered without allocating one string
600
- * per character; surrogates are paired exactly as the String iterator pairs
601
- * them, an unpaired one classifying as the lone surrogate it is.
688
+ * True when `text` holds at least one code point {@link classifyCp} calls
689
+ * invisible. Walks UTF-16 units directly rather than building the code-point
690
+ * array, so a clean multi-MB paste is answered without allocating anything;
691
+ * surrogates are paired exactly as the String iterator pairs them, an unpaired
692
+ * one classifying as the lone surrogate it is.
602
693
  * @param {string} text
603
694
  * @returns {boolean}
604
695
  */
@@ -612,7 +703,7 @@ function hasInvisibleCodePoint(text) {
612
703
  i++;
613
704
  }
614
705
  }
615
- if (classifyCp(cp) !== null) return true;
706
+ if (classifyCp(cp) !== CODE_VISIBLE) return true;
616
707
  }
617
708
  return false;
618
709
  }
@@ -636,15 +727,13 @@ function codePointLength(text) {
636
727
  * @param {string} jt @returns {boolean} */
637
728
  const isCursiveLetter = (jt) => jt === "D" || jt === "R" || jt === "L";
638
729
 
639
- /** The Joining_Type of a single-code-point string, or "U" for "" (boundary).
640
- * @param {string} ch @returns {string} */
641
- const jtOf = (ch) =>
642
- ch ? joiningType(/** @type {number} */ (ch.codePointAt(0))) : "U";
730
+ /** The Joining_Type of a code point, or "U" for -1 (past a string boundary).
731
+ * @param {number} cp @returns {string} */
732
+ const jtOf = (cp) => (cp < 0 ? "U" : joiningType(cp));
643
733
 
644
- /** True when `ch` is itself a ZWNJ/ZWJ (used to reject joiner runs).
645
- * @param {string} ch @returns {boolean} */
646
- function isJoinControl(ch) {
647
- const cp = ch ? ch.codePointAt(0) : -1;
734
+ /** True when `cp` is itself a ZWNJ/ZWJ (used to reject joiner runs).
735
+ * @param {number} cp @returns {boolean} */
736
+ function isJoinControl(cp) {
648
737
  return cp === ZWNJ || cp === ZWJ;
649
738
  }
650
739
 
@@ -658,18 +747,18 @@ function isJoinControl(ch) {
658
747
  * between two joiners hides the joiner-RUN until a first pass removes it, after
659
748
  * which a second pass strips the now-adjacent joiners — non-idempotent. Join
660
749
  * controls (ZWNJ/ZWJ) are deliberately NOT skipped: they are the run signal.
661
- * Returns "" past the string boundary.
662
- * @param {string[]} cps @param {number} i @param {number} dir -1 or +1
663
- * @returns {string}
750
+ * Returns -1 past the string boundary (the old "" neighbour).
751
+ * @param {Int32Array} cps @param {number} i @param {number} dir -1 or +1
752
+ * @returns {number}
664
753
  */
665
754
  function effectiveNeighbor(cps, i, dir) {
666
755
  for (let j = i + dir; j >= 0 && j < cps.length; j += dir) {
667
- const ch = cps[j];
668
- if (jtOf(ch) === "T") continue;
669
- if (!isJoinControl(ch) && classify(ch) !== null) continue;
670
- return ch;
756
+ const cp = cps[j];
757
+ if (jtOf(cp) === "T") continue;
758
+ if (!isJoinControl(cp) && classifyCp(cp) !== CODE_VISIBLE) continue;
759
+ return cp;
671
760
  }
672
- return "";
761
+ return -1;
673
762
  }
674
763
 
675
764
  /**
@@ -680,15 +769,19 @@ function effectiveNeighbor(cps, i, dir) {
680
769
  * virama (which is Joining_Type Transparent, so it is the anchor, never skipped),
681
770
  * then uses effectiveNeighbor to find the consonant base past any transparent
682
771
  * marks / invisibles between it and the virama.
683
- * @param {string[]} cps @param {number} i
772
+ * @param {Int32Array} cps @param {number} i
684
773
  * @returns {boolean}
685
774
  */
686
775
  function followsBrahmicConjunct(cps, i) {
687
776
  let j = i - 1;
688
- while (j >= 0 && !isJoinControl(cps[j]) && classify(cps[j]) !== null) j--;
689
- if (j < 0 || !isVirama(/** @type {number} */ (cps[j].codePointAt(0))))
690
- return false;
691
- return isBrahmicConsonantChar(effectiveNeighbor(cps, j, -1));
777
+ while (
778
+ j >= 0 &&
779
+ !isJoinControl(cps[j]) &&
780
+ classifyCp(cps[j]) !== CODE_VISIBLE
781
+ )
782
+ j--;
783
+ if (j < 0 || !isVirama(cps[j])) return false;
784
+ return isBrahmicConsonantCp(effectiveNeighbor(cps, j, -1));
692
785
  }
693
786
 
694
787
  /**
@@ -700,12 +793,12 @@ function followsBrahmicConjunct(cps, i) {
700
793
  * - Arabic-family joiner: between two cursive letters (ZWNJ needs both, ZWJ at
701
794
  * least one — it forces a connected form). A joiner whose effective neighbour
702
795
  * is ANOTHER joiner is a run (a zero-width payload channel) and is rejected.
703
- * Leading/trailing joiners fall out because "" has Joining_Type U.
704
- * @param {string[]} cps @param {number} i
796
+ * Leading/trailing joiners fall out because the -1 boundary has Joining_Type U.
797
+ * @param {Int32Array} cps @param {number} i
705
798
  * @returns {boolean}
706
799
  */
707
800
  function isPreservedJoiner(cps, i) {
708
- const cp = /** @type {number} */ (cps[i].codePointAt(0));
801
+ const cp = cps[i];
709
802
  if (cp !== ZWNJ && cp !== ZWJ) return false;
710
803
  // Emoji ZWJ sequences use ZWJ only; the real neighbours are the pictographs,
711
804
  // which may each sit behind a variation selector (🏳️‍🌈 = base VS16 ZWJ rainbow;
@@ -713,8 +806,8 @@ function isPreservedJoiner(cps, i) {
713
806
  // selectors on BOTH sides — leftNonSelector and its mirror rightNonSelector.
714
807
  if (
715
808
  cp === ZWJ &&
716
- EMOJI_LEFT.test(leftNonSelector(cps, i)) &&
717
- EMOJI_BASE.test(rightNonSelector(cps, i))
809
+ isEmojiLeft(leftNonSelector(cps, i)) &&
810
+ isEmojiBase(rightNonSelector(cps, i))
718
811
  )
719
812
  return true;
720
813
  // Indic: meaningful only after a virama sitting on a real consonant base.
@@ -737,62 +830,65 @@ function isPreservedJoiner(cps, i) {
737
830
  * A keycap base + selector with NO trailing U+20E3 is a bare hidden presentation
738
831
  * selector, so it fails closed and is stripped. A longer selector run still
739
832
  * surfaces: the next selector's left neighbour is itself a selector.
740
- * @param {string[]} cps @param {number} i
833
+ * @param {Int32Array} cps @param {number} i
741
834
  * @returns {boolean}
742
835
  */
743
836
  function isEmojiPresentationSelector(cps, i) {
744
- if (
745
- !PRESENTATION_SELECTORS.has(/** @type {number} */ (cps[i].codePointAt(0)))
746
- )
747
- return false;
748
- const prev = cps[i - 1] ?? "";
749
- if (EMOJI_LEFT.test(prev)) return true;
837
+ if (!PRESENTATION_SELECTORS.has(cps[i])) return false;
838
+ const prev = i > 0 ? cps[i - 1] : -1;
839
+ if (isEmojiLeft(prev)) return true;
750
840
  return (
751
- KEYCAP_BASE.test(prev) &&
752
- (cps[i + 1]?.codePointAt(0) ?? -1) === COMBINING_KEYCAP
841
+ isKeycapBase(prev) &&
842
+ (i + 1 < cps.length ? cps[i + 1] : -1) === COMBINING_KEYCAP
753
843
  );
754
844
  }
755
845
 
756
846
  /**
757
847
  * Per-invisible carve-out analysis, shared by carveStrip and
758
- * countPayloadInvisible: for each code point, its CHECKS category (null when
759
- * visible) and its preserve `kind` ("joiner" | "emojivs" | "tag" | "stdvs" |
760
- * "ivs" | "blank" | null). Everything invisible that is NOT preserve-eligible is
761
- * payload; the scatter floor counts only that, so meaningful joiners/selectors
762
- * never push honest prose over the threshold.
763
- * @param {string[]} cps
764
- * @returns {{ codes: (string|null)[], kind: (string|null)[], payloadInvis: number, visibleLen: number }}
848
+ * countPayloadInvisible: for each code point, its CODE_* class (CODE_VISIBLE
849
+ * when visible) and its preserve KIND_* (KIND_NONE when the char is payload).
850
+ * Everything invisible that is NOT preserve-eligible is payload; the scatter
851
+ * floor counts only that, so meaningful joiners/selectors never push honest
852
+ * prose over the threshold.
853
+ * @param {Int32Array} cps
854
+ * @returns {{ codes: Uint8Array, kind: Uint8Array, payloadInvis: number, visibleLen: number }}
765
855
  */
766
856
  function analyzeCarve(cps) {
767
- const codes = new Array(cps.length);
857
+ const codes = new Uint8Array(cps.length);
768
858
  // Indices of the invisibles, so every pass below is O(invisibles) rather than
769
859
  // O(length): in ordinary text they are a vanishing fraction of a paste, and a
770
860
  // full-length pass per preserve rule is what made the analysis linear in the
771
861
  // WHOLE prompt several times over.
772
- const invisible = [];
862
+ let invisCount = 0;
773
863
  let visibleLen = 0;
774
864
  let hasTagBase = false;
775
865
  for (let i = 0; i < cps.length; i++) {
776
- const code = classify(cps[i]);
866
+ const code = classifyCp(cps[i]);
777
867
  codes[i] = code;
778
- if (code !== null) {
779
- invisible.push(i);
868
+ if (code !== CODE_VISIBLE) {
869
+ invisCount++;
780
870
  continue;
781
871
  }
782
872
  visibleLen++;
783
- hasTagBase ||= cps[i].codePointAt(0) === TAG_BASE;
873
+ hasTagBase ||= cps[i] === TAG_BASE;
784
874
  }
875
+ // Sized exactly, from the count the pass above already produced: a growable
876
+ // array reallocates its way up through a document that is mostly invisible,
877
+ // and that churn is the collector's whole share of this function.
878
+ const invisible = new Int32Array(invisCount);
879
+ for (let i = 0, at = 0; at < invisCount; i++)
880
+ if (codes[i] !== CODE_VISIBLE) invisible[at++] = i;
785
881
  // A tag sequence is preservable only when it opens on a TAG_BASE pictograph,
786
882
  // so with no base in the text markTagSequences returns all-false.
787
883
  const tagKeep = hasTagBase ? markTagSequences(cps) : null;
788
- const kind = new Array(cps.length).fill(null);
884
+ const kind = new Uint8Array(cps.length);
789
885
  for (const i of invisible) {
790
- if (tagKeep !== null && tagKeep[i]) kind[i] = "tag";
791
- else if (isPreservedJoiner(cps, i)) kind[i] = "joiner";
792
- else if (isEmojiPresentationSelector(cps, i)) kind[i] = "emojivs";
793
- else if (isStandardizedVariationSelector(cps, i)) kind[i] = "stdvs";
794
- else if (isIdeographicVariationSelector(cps, i)) kind[i] = "ivs";
795
- else if (isPreservedBlankFiller(cps, i)) kind[i] = "blank";
886
+ if (tagKeep !== null && tagKeep[i]) kind[i] = KIND_TAG;
887
+ else if (isPreservedJoiner(cps, i)) kind[i] = KIND_JOINER;
888
+ else if (isEmojiPresentationSelector(cps, i)) kind[i] = KIND_EMOJIVS;
889
+ else if (isStandardizedVariationSelector(cps, i)) kind[i] = KIND_STDVS;
890
+ else if (isIdeographicVariationSelector(cps, i)) kind[i] = KIND_IVS;
891
+ else if (isPreservedBlankFiller(cps, i)) kind[i] = KIND_BLANK;
796
892
  }
797
893
  // Blank fillers are budgeted here, document-wide and all-or-nothing, against
798
894
  // the visible anchor-script text rather than against the joiner/selector
@@ -811,28 +907,25 @@ function analyzeCarve(cps) {
811
907
  /** @type {Record<string, number[]>} */
812
908
  const blankIndices = { braille: [], hangul: [] };
813
909
  for (const i of invisible)
814
- if (kind[i] === "blank")
815
- blankIndices[
816
- cps[i].codePointAt(0) === BRAILLE_BLANK ? "braille" : "hangul"
817
- ].push(i);
910
+ if (kind[i] === KIND_BLANK)
911
+ blankIndices[cps[i] === BRAILLE_BLANK ? "braille" : "hangul"].push(i);
818
912
  for (const [
819
913
  script,
820
914
  isAnchor,
821
- ] of /** @type {[string, (ch: string) => boolean][]} */ ([
915
+ ] of /** @type {[string, (cp: number) => boolean][]} */ ([
822
916
  ["braille", isBrailleCell],
823
917
  ["hangul", isHangul],
824
918
  ])) {
825
919
  const indices = blankIndices[script];
826
920
  if (indices.length <= TOTAL_PRESERVED_BLANK_BUDGET) continue;
827
- const anchors = cps.reduce(
828
- (n, ch, i) => n + (codes[i] === null && isAnchor(ch) ? 1 : 0),
829
- 0,
830
- );
921
+ let anchors = 0;
922
+ for (let i = 0; i < cps.length; i++)
923
+ if (codes[i] === CODE_VISIBLE && isAnchor(cps[i])) anchors++;
831
924
  if (indices.length > Math.floor(anchors / PRESERVED_BLANK_PER_ANCHOR))
832
- for (const i of indices) kind[i] = null;
925
+ for (const i of indices) kind[i] = KIND_NONE;
833
926
  }
834
927
  let payloadInvis = 0;
835
- for (const i of invisible) if (kind[i] === null) payloadInvis++;
928
+ for (const i of invisible) if (kind[i] === KIND_NONE) payloadInvis++;
836
929
  return { codes, kind, payloadInvis, visibleLen };
837
930
  }
838
931
 
@@ -847,12 +940,12 @@ function analyzeCarve(cps) {
847
940
  */
848
941
  export function countPayloadInvisible(text) {
849
942
  // Payload is a SUBSET of the invisibles (analyzeCarve counts only positions
850
- // classify marks invisible), so a text with none has a payload count of zero
943
+ // classifyCp marks invisible), so a text with none has a payload count of zero
851
944
  // and needs neither the code-point array nor the carve analysis. This reads
852
945
  // every code point — it is a necessary condition checked in full, not a scan
853
946
  // bound an attacker could paste past.
854
947
  if (!hasInvisibleCodePoint(text)) return 0;
855
- return analyzeCarve(Array.from(text)).payloadInvis;
948
+ return analyzeCarve(codePointArray(text)).payloadInvis;
856
949
  }
857
950
 
858
951
  /**
@@ -872,33 +965,33 @@ function bulkStrip(body) {
872
965
 
873
966
  /**
874
967
  * The nearest code point left of index `i` that is not a variation selector, or
875
- * "" at the string start. An emoji ZWJ sequence can place a VS16 between the base
968
+ * -1 at the string start. An emoji ZWJ sequence can place a VS16 between the base
876
969
  * pictograph and the ZWJ, so the joiner's real left neighbor is found by stepping
877
970
  * over any variation selector(s).
878
- * @param {string[]} cps
971
+ * @param {Int32Array} cps
879
972
  * @param {number} i
880
- * @returns {string}
973
+ * @returns {number}
881
974
  */
882
975
  function leftNonSelector(cps, i) {
883
976
  let p = i - 1;
884
- while (p >= 0 && VARIATION_SELECTOR.test(cps[p])) p--;
885
- return cps[p] ?? "";
977
+ while (p >= 0 && isVariationSelectorCp(cps[p])) p--;
978
+ return p >= 0 ? cps[p] : -1;
886
979
  }
887
980
 
888
981
  /**
889
982
  * The nearest code point right of index `i` that is not a variation selector, or
890
- * "" at the string end. Mirror of {@link leftNonSelector}: an emoji ZWJ can be
983
+ * -1 at the string end. Mirror of {@link leftNonSelector}: an emoji ZWJ can be
891
984
  * followed by a selector before the next component's base pictograph, so the
892
985
  * joiner's real right neighbor is found by stepping over any variation
893
986
  * selector(s).
894
- * @param {string[]} cps
987
+ * @param {Int32Array} cps
895
988
  * @param {number} i
896
- * @returns {string}
989
+ * @returns {number}
897
990
  */
898
991
  function rightNonSelector(cps, i) {
899
992
  let p = i + 1;
900
- while (p < cps.length && VARIATION_SELECTOR.test(cps[p])) p++;
901
- return cps[p] ?? "";
993
+ while (p < cps.length && isVariationSelectorCp(cps[p])) p++;
994
+ return p < cps.length ? cps[p] : -1;
902
995
  }
903
996
 
904
997
  /**
@@ -910,23 +1003,17 @@ function rightNonSelector(cps, i) {
910
1003
  * payload is not a registered subdivision is left unmarked so it is stripped —
911
1004
  * grammatical validity alone is NOT enough, since a valid run spells arbitrary
912
1005
  * ASCII (fail closed on the top ASCII-smuggling vector).
913
- * @param {string[]} cps
914
- * @returns {boolean[]} keep[i] true iff `cps[i]` is inside a preservable tag sequence
1006
+ * @param {Int32Array} cps
1007
+ * @returns {Uint8Array} keep[i] non-zero iff `cps[i]` is inside a preservable tag sequence
915
1008
  */
916
1009
  function markTagSequences(cps) {
917
- const keep = new Array(cps.length).fill(false);
918
- /** @param {number} k @returns {number} */
919
- const cpAt = (k) => /** @type {number} */ (cps[k].codePointAt(0));
1010
+ const keep = new Uint8Array(cps.length);
920
1011
  for (let i = 0; i < cps.length; i++) {
921
- if (cpAt(i) !== TAG_BASE) continue;
1012
+ if (cps[i] !== TAG_BASE) continue;
922
1013
  let j = i + 1;
923
1014
  let payload = "";
924
- while (
925
- j < cps.length &&
926
- cpAt(j) >= TAG_SPEC_MIN &&
927
- cpAt(j) <= TAG_SPEC_MAX
928
- ) {
929
- payload += String.fromCharCode(cpAt(j) - 0xe0000);
1015
+ while (j < cps.length && cps[j] >= TAG_SPEC_MIN && cps[j] <= TAG_SPEC_MAX) {
1016
+ payload += String.fromCharCode(cps[j] - 0xe0000);
930
1017
  j++;
931
1018
  }
932
1019
  const tagLen = j - (i + 1);
@@ -936,10 +1023,10 @@ function markTagSequences(cps) {
936
1023
  tagLen >= 1 &&
937
1024
  tagLen <= MAX_TAG_SPEC_CHARS &&
938
1025
  j < cps.length &&
939
- cpAt(j) === TAG_CANCEL &&
1026
+ cps[j] === TAG_CANCEL &&
940
1027
  REGISTERED_TAG_PAYLOADS.has(payload)
941
1028
  ) {
942
- for (let k = i + 1; k <= j; k++) keep[k] = true;
1029
+ for (let k = i + 1; k <= j; k++) keep[k] = 1;
943
1030
  i = j; // resume after the consumed sequence
944
1031
  }
945
1032
  }
@@ -951,29 +1038,26 @@ function markTagSequences(cps) {
951
1038
  * immediately preceding code point forms a REGISTERED standardized variation
952
1039
  * sequence (per the generated UCD table). Every unregistered FE00–FE0D selector
953
1040
  * stays payload.
954
- * @param {string[]} cps @param {number} i
1041
+ * @param {Int32Array} cps @param {number} i
955
1042
  * @returns {boolean}
956
1043
  */
957
1044
  function isStandardizedVariationSelector(cps, i) {
958
- const cp = /** @type {number} */ (cps[i].codePointAt(0));
1045
+ const cp = cps[i];
959
1046
  if (cp < 0xfe00 || cp > 0xfe0d) return false;
960
- const prev = cps[i - 1];
961
- return prev
962
- ? isStandardizedVariant(/** @type {number} */ (prev.codePointAt(0)), cp)
963
- : false;
1047
+ return i > 0 ? isStandardizedVariant(cps[i - 1], cp) : false;
964
1048
  }
965
1049
 
966
1050
  /**
967
1051
  * True when `cps[i]` is an ideographic variation selector (VS17–VS256,
968
1052
  * U+E0100–U+E01EF) immediately after a CJK ideograph — the registry-faithful
969
1053
  * structural gate for an ideographic variation sequence.
970
- * @param {string[]} cps @param {number} i
1054
+ * @param {Int32Array} cps @param {number} i
971
1055
  * @returns {boolean}
972
1056
  */
973
1057
  function isIdeographicVariationSelector(cps, i) {
974
- const cp = /** @type {number} */ (cps[i].codePointAt(0));
1058
+ const cp = cps[i];
975
1059
  if (cp < IVS_MIN || cp > IVS_MAX) return false;
976
- return isCjkIdeograph(cps[i - 1] ?? "");
1060
+ return isCjkIdeograph(i > 0 ? cps[i - 1] : -1);
977
1061
  }
978
1062
 
979
1063
  /**
@@ -981,13 +1065,13 @@ function isIdeographicVariationSelector(cps, i) {
981
1065
  * next to a real, script-appropriate visible neighbour — a genuine empty Braille
982
1066
  * cell or archaic-Korean filler, not a hidden run. The zero-width Mn marks in
983
1067
  * BLANK_NON_CF have no such anchored use and are never preserved here.
984
- * @param {string[]} cps @param {number} i
1068
+ * @param {Int32Array} cps @param {number} i
985
1069
  * @returns {boolean}
986
1070
  */
987
1071
  function isPreservedBlankFiller(cps, i) {
988
- const cp = /** @type {number} */ (cps[i].codePointAt(0));
989
- const prev = cps[i - 1] ?? "";
990
- const next = cps[i + 1] ?? "";
1072
+ const cp = cps[i];
1073
+ const prev = i > 0 ? cps[i - 1] : -1;
1074
+ const next = i + 1 < cps.length ? cps[i + 1] : -1;
991
1075
  if (cp === BRAILLE_BLANK) return isBrailleCell(prev) || isBrailleCell(next);
992
1076
  if (HANGUL_FILLERS.has(cp)) return isHangul(prev) || isHangul(next);
993
1077
  return false;
@@ -1003,22 +1087,99 @@ const GRAPHEME_SEGMENTER = new Intl.Segmenter("en", {
1003
1087
  });
1004
1088
 
1005
1089
  /**
1006
- * The code-point index one past the end of each grapheme cluster of `body`, in
1007
- * order the segmenter's UTF-16 boundaries restated in code-point space, which
1008
- * is the space `carveStrip`'s per-code-point arrays live in. ECMA-402 guarantees
1009
- * the segments partition the input, so the last entry is always the code-point
1010
- * length of `body`.
1011
- * @param {string} body
1012
- * @returns {number[]}
1090
+ * The UTF-16 index at which each code point of `cps` starts, plus one final
1091
+ * entry for the end of the string the bridge between the code-point space the
1092
+ * carve analysis works in and the UTF-16 space `body.slice` and the segmenter
1093
+ * speak.
1094
+ * @param {Int32Array} cps
1095
+ * @returns {Int32Array}
1013
1096
  */
1014
- function clusterEnds(body) {
1015
- const ends = [];
1016
- let end = 0;
1017
- for (const { segment } of GRAPHEME_SEGMENTER.segment(body)) {
1018
- end += codePointLength(segment);
1019
- ends.push(end);
1097
+ function u16Offsets(cps) {
1098
+ const offsets = new Int32Array(cps.length + 1);
1099
+ let at = 0;
1100
+ for (let i = 0; i < cps.length; i++) {
1101
+ offsets[i] = at;
1102
+ at += cps[i] > 0xffff ? 2 : 1;
1020
1103
  }
1021
- return ends;
1104
+ offsets[cps.length] = at;
1105
+ return offsets;
1106
+ }
1107
+
1108
+ /**
1109
+ * The code-point index of the UTF-16 index `u16`, by binary search over
1110
+ * {@link u16Offsets}. `u16` is always a code-point (indeed a cluster) boundary,
1111
+ * so the search always lands exactly.
1112
+ * @param {Int32Array} offsets @param {number} u16
1113
+ * @returns {number}
1114
+ */
1115
+ function cpIndexOf(offsets, u16) {
1116
+ let lo = 0;
1117
+ let hi = offsets.length - 1;
1118
+ while (lo < hi) {
1119
+ const mid = (lo + hi) >> 1;
1120
+ if (offsets[mid] < u16) lo = mid + 1;
1121
+ else hi = mid;
1122
+ }
1123
+ return lo;
1124
+ }
1125
+
1126
+ // Above this share of the code points being carve-out CANDIDATES, resolving each
1127
+ // candidate's cluster with `Segments.containing` costs more than segmenting the
1128
+ // document once from the start (a `containing` call is a few times dearer than
1129
+ // one step of the sequential iterator). Both strategies read the SAME segmenter,
1130
+ // so they return the same boundaries; this only picks the cheaper way to ask.
1131
+ const CONTAINING_CANDIDATE_SHARE = 4;
1132
+
1133
+ /**
1134
+ * A source of the grapheme cluster containing a given code-point index, over
1135
+ * indices asked for in increasing order.
1136
+ *
1137
+ * Cluster boundaries matter only where a preservable invisible sits: a cluster
1138
+ * with none leaves every counter in {@link carveStrip} untouched, and the emit
1139
+ * loop treats each of its code points the same however the boundaries fall. So
1140
+ * the clusters are resolved on demand around the candidates rather than by
1141
+ * segmenting the whole document — the difference between reading one emoji at
1142
+ * the end of an 8 MB log and segmenting the 8 MB.
1143
+ * @param {string} body @param {Int32Array} offsets @param {number} candidates
1144
+ * @returns {(cp: number) => { start: number, end: number }}
1145
+ */
1146
+ function clusterResolver(body, offsets, candidates) {
1147
+ const cpCount = offsets.length - 1;
1148
+ if (candidates * CONTAINING_CANDIDATE_SHARE < cpCount) {
1149
+ const segments = GRAPHEME_SEGMENTER.segment(body);
1150
+ return (cp) => {
1151
+ // `offsets[cp]` is inside the string for every code-point index the walk
1152
+ // asks about, so the segment always exists.
1153
+ const found = /** @type {{ index: number, segment: string }} */ (
1154
+ segments.containing(offsets[cp])
1155
+ );
1156
+ return {
1157
+ start: cpIndexOf(offsets, found.index),
1158
+ end: cpIndexOf(offsets, found.index + found.segment.length),
1159
+ };
1160
+ };
1161
+ }
1162
+ // Dense candidates: walk the segmenter forward instead, keeping a code-point
1163
+ // cursor in step with it so no boundary needs searching for. The iterator is
1164
+ // abandoned wherever the last candidate leaves it, so a document whose
1165
+ // preserve budget is spent in its first line is never segmented past it.
1166
+ const iterator = GRAPHEME_SEGMENTER.segment(body)[Symbol.iterator]();
1167
+ let start = 0;
1168
+ let end = 0;
1169
+ return (cp) => {
1170
+ while (end <= cp) {
1171
+ // ECMA-402 guarantees the segments partition the input, so every code
1172
+ // point inside it has a cluster and the iterator cannot run out first; if
1173
+ // it ever did, reading `.segment` off `undefined` throws here rather than
1174
+ // silently answering with a boundary nobody computed.
1175
+ const step = /** @type {{ value: { segment: string } }} */ (
1176
+ iterator.next()
1177
+ );
1178
+ start = end;
1179
+ end = cpIndexOf(offsets, offsets[start] + step.value.segment.length);
1180
+ }
1181
+ return { start, end };
1182
+ };
1022
1183
  }
1023
1184
 
1024
1185
  /**
@@ -1046,14 +1207,19 @@ function clusterEnds(body) {
1046
1207
  * makes the caller claim a strip that did not happen, and a stuffed channel
1047
1208
  * surfaces as its category once it overruns the budget.
1048
1209
  * @param {string} body
1210
+ * @param {Int32Array} [cps] `body`'s code points, when the caller already has them
1211
+ * @param {ReturnType<typeof analyzeCarve>} [analysis] likewise the analysis
1049
1212
  * @returns {{ cleaned: string, found: string[] }}
1050
1213
  */
1051
- function carveStrip(body) {
1052
- const cps = Array.from(body);
1053
- // Pass 1: classify + evaluate the gate once (see analyzeCarve). Only PAYLOAD
1054
- // invisibles count toward the scatter floor, so a meaningful-joiner-dense text
1055
- // (formal Persian, a long Devanagari conjunct run) stays under it.
1056
- const { codes, kind, payloadInvis, visibleLen } = analyzeCarve(cps);
1214
+ function carveStrip(
1215
+ body,
1216
+ cps = codePointArray(body),
1217
+ analysis = analyzeCarve(cps),
1218
+ ) {
1219
+ // Only PAYLOAD invisibles count toward the scatter floor, so a
1220
+ // meaningful-joiner-dense text (formal Persian, a long Devanagari conjunct
1221
+ // run) stays under it.
1222
+ const { codes, kind, payloadInvis, visibleLen } = analysis;
1057
1223
  // SCATTERED_THRESHOLD is the floor on payload invisibles: past it the document
1058
1224
  // is drowning in hidden bytes, so the carve-out is off and even a meaningful
1059
1225
  // joiner is stripped (threshold-evasion catch — over-strip beats under).
@@ -1070,8 +1236,25 @@ function carveStrip(body) {
1070
1236
  ),
1071
1237
  );
1072
1238
 
1239
+ const n = cps.length;
1240
+ // Every code point that draws on the joiner/selector budget, in order. Blank
1241
+ // fillers are exempt (their allowance is already spent in analyzeCarve), so a
1242
+ // cluster holding only those needs no boundary of its own.
1243
+ /** @type {number[]} */
1244
+ const candidates = [];
1245
+ if (allowCarveOut)
1246
+ for (let k = 0; k < n; k++)
1247
+ if (kind[k] !== KIND_NONE && kind[k] !== KIND_BLANK) candidates.push(k);
1248
+
1073
1249
  const foundCodes = new Set();
1074
- let out = "";
1250
+ /** @type {string[]} */
1251
+ const kept = [];
1252
+ const offsets = u16Offsets(cps);
1253
+ // Start of the kept run currently being accumulated, in UTF-16 units. Output
1254
+ // is assembled from slices of `body` — one per uninterrupted kept run — rather
1255
+ // than a character at a time, so a document with nothing to strip costs one
1256
+ // slice however long it is.
1257
+ let runStart = 0;
1075
1258
  // Preserved JOINERS in the current uninterrupted cluster (tags/blank fillers
1076
1259
  // and presentation selectors don't chain, so they are exempt). A genuine gap
1077
1260
  // (two visible chars in a row — see prevVisible) resets it; past the cap the
@@ -1088,80 +1271,151 @@ function carveStrip(body) {
1088
1271
  // char is stripped and its category reported.
1089
1272
  let preservedTotal = 0;
1090
1273
  let prevVisible = false;
1091
- let start = 0;
1092
- for (const end of clusterEnds(body)) {
1274
+
1275
+ /** Emit `[from, to)` under a settled `fits`, updating every counter.
1276
+ * @param {number} from @param {number} to @param {boolean} fits */
1277
+ const emit = (from, to, fits) => {
1278
+ for (let k = from; k < to; k++) {
1279
+ const code = codes[k];
1280
+ if (code === CODE_VISIBLE) {
1281
+ // A visible char following another visible char is a real word/segment
1282
+ // boundary, not a join — the joined cluster (if any) ended here.
1283
+ if (prevVisible) {
1284
+ joinerRun = 0;
1285
+ selectorRun = 0;
1286
+ }
1287
+ prevVisible = true;
1288
+ continue; // ordinary visible character: stays inside the kept run
1289
+ }
1290
+ // A blank filler rides on allowCarveOut alone (its own allowance is
1291
+ // already spent in analyzeCarve); everything else rides on `fits`.
1292
+ if (
1293
+ kind[k] === KIND_BLANK ? allowCarveOut : fits && kind[k] !== KIND_NONE
1294
+ ) {
1295
+ if (kind[k] === KIND_JOINER) joinerRun++;
1296
+ if (kind[k] === KIND_IVS || kind[k] === KIND_STDVS) selectorRun++;
1297
+ if (kind[k] !== KIND_BLANK) preservedTotal++;
1298
+ prevVisible = false; // a joiner/selector/tag/blank keeps the cluster open
1299
+ continue;
1300
+ }
1301
+ foundCodes.add(CODE_CATEGORY[code]);
1302
+ prevVisible = false; // a stripped invisible neither opens nor closes a gap
1303
+ if (offsets[k] > runStart) kept.push(body.slice(runStart, offsets[k]));
1304
+ runStart = offsets[k + 1];
1305
+ }
1306
+ };
1307
+
1308
+ const clusterOf =
1309
+ candidates.length > 0
1310
+ ? clusterResolver(body, offsets, candidates.length)
1311
+ : null;
1312
+ // The last index at which the emit loop resets the run counters — a visible
1313
+ // code point directly after another visible one. The condition reads only
1314
+ // `codes`, never a preserve decision, so the whole set of reset points is
1315
+ // known before the walk starts. Past this index the run counters can only
1316
+ // ever be incremented, which is what makes the exhaustion test below final.
1317
+ let lastGap = -1;
1318
+ // Remaining candidates by the limit each one draws on, decremented as the
1319
+ // walk passes them.
1320
+ let leftJoiner = 0;
1321
+ let leftSelector = 0;
1322
+ let leftOther = 0;
1323
+ if (clusterOf !== null) {
1324
+ for (let i = n - 1; i > 0; i--)
1325
+ if (codes[i] === CODE_VISIBLE && codes[i - 1] === CODE_VISIBLE) {
1326
+ lastGap = i;
1327
+ break;
1328
+ }
1329
+ for (const c of candidates) {
1330
+ if (kind[c] === KIND_JOINER) leftJoiner++;
1331
+ else if (kind[c] === KIND_IVS || kind[c] === KIND_STDVS) leftSelector++;
1332
+ else leftOther++;
1333
+ }
1334
+ }
1335
+
1336
+ /** True when no cluster from here on can be preserved, so the rest of the
1337
+ * document is settled and needs no boundaries. Every term is monotone:
1338
+ * `preservedTotal` only grows, and past `lastGap` the run counters only grow
1339
+ * too, so a limit that is spent here is spent for good.
1340
+ * @param {number} at */
1341
+ const nothingLeftFits = (at) => {
1342
+ if (preservedTotal >= maxPreserved) return true;
1343
+ if (at <= lastGap) return false; // a reset ahead can reopen a spent run
1344
+ if (leftOther > 0) return false; // a tag/presentation selector draws on neither run
1345
+ if (leftJoiner > 0 && joinerRun < CONSECUTIVE_JOINER_CAP) return false;
1346
+ if (leftSelector > 0 && selectorRun < CONSECUTIVE_SELECTOR_CAP)
1347
+ return false;
1348
+ return true;
1349
+ };
1350
+
1351
+ let k = 0;
1352
+ let next = 0;
1353
+ while (k < n) {
1354
+ while (next < candidates.length && candidates[next] < k) next++;
1355
+ if (
1356
+ clusterOf === null ||
1357
+ next === candidates.length ||
1358
+ nothingLeftFits(k)
1359
+ ) {
1360
+ emit(k, n, false);
1361
+ break;
1362
+ }
1363
+ const { start, end } = clusterOf(candidates[next]);
1364
+ if (start > k) emit(k, start, false);
1093
1365
  // Charge the WHOLE cluster's preservables against every limit at once: if
1094
1366
  // any one of them would fall due part-way through, none of the cluster is
1095
- // preserved. `need === 0` (the common case: a cluster with no preservable
1096
- // invisible) leaves every counter untouched.
1367
+ // preserved.
1097
1368
  let need = 0;
1098
1369
  let joiners = 0;
1099
1370
  let selectors = 0;
1100
- for (let k = start; k < end; k++) {
1101
- // "blank" is exempt: analyzeCarve already decided it against the
1102
- // anchor-proportional allowance, so it neither draws on this budget nor
1103
- // is stripped by it (only by the scatter floor, via allowCarveOut).
1104
- if (kind[k] === null || kind[k] === "blank") continue;
1371
+ for (let j = start; j < end; j++) {
1372
+ if (kind[j] === KIND_NONE || kind[j] === KIND_BLANK) continue;
1105
1373
  need++;
1106
- if (kind[k] === "joiner") joiners++;
1107
- if (kind[k] === "ivs" || kind[k] === "stdvs") selectors++;
1374
+ if (kind[j] === KIND_JOINER) joiners++;
1375
+ if (kind[j] === KIND_IVS || kind[j] === KIND_STDVS) selectors++;
1108
1376
  }
1109
1377
  // The run counters as they stand at the cluster's FIRST preservable char.
1110
1378
  // A cluster can OPEN with a genuine gap — two visible code points in a row,
1111
1379
  // e.g. the second of two adjacent emoji ZWJ sequences, or a letter and its
1112
- // harakat — which the emit loop resets on a few lines below. Judging the
1113
- // caps on the stale pre-reset count would strip joiners the cap never meant
1114
- // to catch: a false positive on legitimate joined text. This mirrors the
1115
- // emit loop's gap rule exactly (only a visible char after another visible
1116
- // char closes a run; any invisible, payload included, does not) and stops at
1117
- // the first preservable, since resets past it are the emit loop's business.
1380
+ // harakat — which the emit loop resets on. Judging the caps on the stale
1381
+ // pre-reset count would strip joiners the cap never meant to catch: a false
1382
+ // positive on legitimate joined text. This mirrors the emit loop's gap rule
1383
+ // exactly (only a visible char after another visible char closes a run; any
1384
+ // invisible, payload included, does not) and stops at the first preservable,
1385
+ // since resets past it are the emit loop's business.
1118
1386
  let runJoiner = joinerRun;
1119
1387
  let runSelector = selectorRun;
1120
1388
  let seenVisible = prevVisible;
1121
- for (let k = start; k < end && kind[k] === null; k++) {
1122
- if (codes[k] === null && seenVisible) {
1389
+ for (let j = start; j < end && kind[j] === KIND_NONE; j++) {
1390
+ if (codes[j] === CODE_VISIBLE && seenVisible) {
1123
1391
  runJoiner = 0;
1124
1392
  runSelector = 0;
1125
1393
  }
1126
- seenVisible = codes[k] === null;
1394
+ seenVisible = codes[j] === CODE_VISIBLE;
1127
1395
  }
1128
- const fits =
1129
- allowCarveOut &&
1130
- preservedTotal + need <= maxPreserved &&
1131
- runJoiner + joiners <= CONSECUTIVE_JOINER_CAP &&
1132
- runSelector + selectors <= CONSECUTIVE_SELECTOR_CAP;
1133
- for (let k = start; k < end; k++) {
1134
- const code = codes[k];
1135
- if (code === null) {
1136
- // A visible char following another visible char is a real word/segment
1137
- // boundary, not a join — the joined cluster (if any) ended here.
1138
- if (prevVisible) {
1139
- joinerRun = 0;
1140
- selectorRun = 0;
1141
- }
1142
- prevVisible = true;
1143
- out += cps[k]; // ordinary visible character
1144
- continue;
1145
- }
1146
- // A blank filler rides on allowCarveOut alone (its own allowance is
1147
- // already spent in analyzeCarve); everything else rides on `fits`.
1148
- if (kind[k] === "blank" ? allowCarveOut : fits && kind[k] !== null) {
1149
- if (kind[k] === "joiner") joinerRun++;
1150
- if (kind[k] === "ivs" || kind[k] === "stdvs") selectorRun++;
1151
- if (kind[k] !== "blank") preservedTotal++;
1152
- prevVisible = false; // a joiner/selector/tag/blank keeps the cluster open
1153
- out += cps[k];
1154
- continue;
1155
- }
1156
- foundCodes.add(code);
1157
- prevVisible = false; // a stripped invisible neither opens nor closes a gap
1396
+ for (let j = next; j < candidates.length && candidates[j] < end; j++) {
1397
+ if (kind[candidates[j]] === KIND_JOINER) leftJoiner--;
1398
+ else if (
1399
+ kind[candidates[j]] === KIND_IVS ||
1400
+ kind[candidates[j]] === KIND_STDVS
1401
+ )
1402
+ leftSelector--;
1403
+ else leftOther--;
1158
1404
  }
1159
- start = end;
1405
+ emit(
1406
+ start,
1407
+ end,
1408
+ preservedTotal + need <= maxPreserved &&
1409
+ runJoiner + joiners <= CONSECUTIVE_JOINER_CAP &&
1410
+ runSelector + selectors <= CONSECUTIVE_SELECTOR_CAP,
1411
+ );
1412
+ k = end;
1160
1413
  }
1414
+ kept.push(body.slice(runStart));
1161
1415
  const found = CHECKS.filter(([code]) => foundCodes.has(code)).map(
1162
1416
  ([code]) => code,
1163
1417
  );
1164
- return { cleaned: out, found };
1418
+ return { cleaned: kept.join(""), found };
1165
1419
  }
1166
1420
 
1167
1421
  // Any tag char (the whole U+E0000–U+E007F block is category Cf).
@@ -1198,12 +1452,28 @@ function needsCarveOut(body) {
1198
1452
  * @returns {string}
1199
1453
  */
1200
1454
  export function payloadInvisibleView(text) {
1201
- const cps = Array.from(text);
1455
+ const cps = codePointArray(text);
1202
1456
  const { codes, kind } = analyzeCarve(cps);
1203
- let out = "";
1204
- for (let i = 0; i < cps.length; i++)
1205
- out += codes[i] !== null && kind[i] === null ? cps[i] : " ";
1206
- return out;
1457
+ const offsets = u16Offsets(cps);
1458
+ /** @param {number} i */
1459
+ const isPayload = (i) => codes[i] !== CODE_VISIBLE && kind[i] === KIND_NONE;
1460
+ // Written a RUN at a time — one `" ".repeat(n)` per masked stretch, one slice
1461
+ // of `text` per payload stretch — because the two stretches alternate a
1462
+ // handful of times in a real document and once per character in none of them.
1463
+ /** @type {string[]} */
1464
+ const parts = [];
1465
+ let i = 0;
1466
+ while (i < cps.length) {
1467
+ const maskFrom = i;
1468
+ while (i < cps.length && !isPayload(i)) i++;
1469
+ // One space per masked CODE POINT, so the view stays code-point-aligned with
1470
+ // `text` even where an astral character is two UTF-16 units wide.
1471
+ if (i > maskFrom) parts.push(" ".repeat(i - maskFrom));
1472
+ const keepFrom = i;
1473
+ while (i < cps.length && isPayload(i)) i++;
1474
+ if (i > keepFrom) parts.push(text.slice(offsets[keepFrom], offsets[i]));
1475
+ }
1476
+ return parts.join("");
1207
1477
  }
1208
1478
 
1209
1479
  /**
@@ -1259,13 +1529,40 @@ export function payloadLongRunSample(text) {
1259
1529
  * @returns {number}
1260
1530
  */
1261
1531
  export function countEffectiveInvisible(text) {
1262
- const payload = countPayloadInvisible(text);
1263
- const stripped = stripInvisible(text);
1532
+ // A text with no invisible code point has no payload and nothing to strip, so
1533
+ // it needs neither the code-point array nor the carve analysis. This reads
1534
+ // every code point — a necessary condition checked in full, not a scan bound
1535
+ // an attacker could paste past.
1536
+ if (!hasInvisibleCodePoint(text)) return 0;
1537
+ // A leading BOM is sliced off before the strip but counted by the analysis, so
1538
+ // the two would read different strings; that case pays for both separately.
1539
+ if (text.charCodeAt(0) === 0xfeff)
1540
+ return surplusOver(countPayloadInvisible(text), text, stripInvisible(text));
1541
+ // Otherwise both terms are readings of ONE carve analysis of ONE string, so
1542
+ // the analysis is done once and handed to the strip.
1543
+ const cps = codePointArray(text);
1544
+ const analysis = analyzeCarve(cps);
1545
+ const { cleaned } = stripBody(text, cps, analysis);
1546
+ return surplusOver(analysis.payloadInvis, text, cleaned, cps.length);
1547
+ }
1548
+
1549
+ /**
1550
+ * The payload count plus the code points the strip removed BEYOND it — the
1551
+ * carve-out's over-budget surplus (see {@link countEffectiveInvisible}). A
1552
+ * leading BOM is preserved by the strip but counted as payload, so the
1553
+ * difference can go slightly negative; hence the clamp.
1554
+ * @param {number} payload @param {string} text @param {string} stripped
1555
+ * @param {number} [cpLen] `text`'s code-point length, when already known
1556
+ * @returns {number}
1557
+ */
1558
+ function surplusOver(payload, text, stripped, cpLen) {
1264
1559
  // Two identical strings differ by zero code points, so the common clean paste
1265
1560
  // skips both counting passes; codePointLength is the String iterator's count,
1266
1561
  // the same unit `[...text].length` measures, without the array.
1267
1562
  const removed =
1268
- stripped === text ? 0 : codePointLength(text) - codePointLength(stripped);
1563
+ stripped === text
1564
+ ? 0
1565
+ : (cpLen ?? codePointLength(text)) - codePointLength(stripped);
1269
1566
  return payload + Math.max(0, removed - payload);
1270
1567
  }
1271
1568
 
@@ -1320,12 +1617,25 @@ export function stripInvisibleWithReport(text, originalText = text) {
1320
1617
  const hasLeadingBom =
1321
1618
  originalText.charCodeAt(0) === 0xfeff && text.charCodeAt(0) === 0xfeff;
1322
1619
  const body = hasLeadingBom ? text.slice(1) : text;
1323
- const { cleaned, found } = needsCarveOut(body)
1324
- ? carveStrip(body)
1325
- : bulkStrip(body);
1620
+ const { cleaned, found } = stripBody(body);
1326
1621
  return { cleaned: hasLeadingBom ? BOM + cleaned : cleaned, found };
1327
1622
  }
1328
1623
 
1624
+ /**
1625
+ * The strip of a BOM-resolved body: the carve-out walk when anything in it could
1626
+ * be preserved, the single bulk regex pass otherwise. `cps` and `analysis` let a
1627
+ * caller that has already analyzed `body` hand the work over instead of paying
1628
+ * for it twice.
1629
+ * @param {string} body
1630
+ * @param {Int32Array} [cps]
1631
+ * @param {ReturnType<typeof analyzeCarve>} [analysis]
1632
+ * @returns {{ cleaned: string, found: string[] }}
1633
+ */
1634
+ function stripBody(body, cps, analysis) {
1635
+ if (!needsCarveOut(body)) return bulkStrip(body);
1636
+ return carveStrip(body, cps, analysis);
1637
+ }
1638
+
1329
1639
  /**
1330
1640
  * Strip payload-capable invisible chars (cleaned text only). See
1331
1641
  * stripInvisibleWithReport for the BOM and ZWNJ/ZWJ carve-out semantics.