agent-sanitizer 2.34.7 → 2.34.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/html.mjs +108 -14
- package/src/invisible.mjs +548 -238
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-sanitizer",
|
|
3
|
-
"version": "2.34.
|
|
3
|
+
"version": "2.34.8",
|
|
4
4
|
"description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
package/src/html.mjs
CHANGED
|
@@ -36,7 +36,7 @@ import { unified } from "unified";
|
|
|
36
36
|
import remarkParse from "remark-parse";
|
|
37
37
|
import remarkGfm from "remark-gfm";
|
|
38
38
|
import rehypeParse from "rehype-parse";
|
|
39
|
-
import {
|
|
39
|
+
import { SKIP, EXIT } from "unist-util-visit";
|
|
40
40
|
import {
|
|
41
41
|
HTML_TAG_PRESENT,
|
|
42
42
|
MD_LINK_HINT,
|
|
@@ -1374,14 +1374,93 @@ function hasDataSrc(el) {
|
|
|
1374
1374
|
// there is exactly one parser configuration to reason about.
|
|
1375
1375
|
const htmlParser = unified().use(rehypeParse, { fragment: true });
|
|
1376
1376
|
|
|
1377
|
+
/**
|
|
1378
|
+
* Preorder depth-first walk of a unist tree, calling `visitor(node, index,
|
|
1379
|
+
* parent)` on every node whose `type` is `test` (or on every node when `test` is
|
|
1380
|
+
* null). `EXIT` ends the walk, `SKIP` leaves the node's children unvisited —
|
|
1381
|
+
* the `unist-util-visit` contract, which is what the call sites here are
|
|
1382
|
+
* written against.
|
|
1383
|
+
*
|
|
1384
|
+
* Spelled out rather than imported because `unist-util-visit` allocates a fresh
|
|
1385
|
+
* ancestors array and a closure per node, and on a document that is one long
|
|
1386
|
+
* flat sibling list — 32k `<p>` elements per megabyte of ordinary HTML — that
|
|
1387
|
+
* turns a linear walk super-linear: 3480 ms against this walk's 62 ms over the
|
|
1388
|
+
* same 4 MB tree. The three parallel arrays are the stack, so the walk itself
|
|
1389
|
+
* allocates nothing per node.
|
|
1390
|
+
* @param {any} tree
|
|
1391
|
+
* @param {string | null} test
|
|
1392
|
+
* @param {(node: any, index: number | undefined, parent: any) => unknown} visitor
|
|
1393
|
+
*/
|
|
1394
|
+
function walk(tree, test, visitor) {
|
|
1395
|
+
/** @type {any[]} */
|
|
1396
|
+
const nodes = [tree];
|
|
1397
|
+
/** @type {Array<number | undefined>} */
|
|
1398
|
+
const indices = [undefined];
|
|
1399
|
+
/** @type {any[]} */
|
|
1400
|
+
const parents = [undefined];
|
|
1401
|
+
while (nodes.length > 0) {
|
|
1402
|
+
const node = nodes.pop();
|
|
1403
|
+
const index = indices.pop();
|
|
1404
|
+
const parent = parents.pop();
|
|
1405
|
+
const result =
|
|
1406
|
+
test === null || node.type === test
|
|
1407
|
+
? visitor(node, index, parent)
|
|
1408
|
+
: undefined;
|
|
1409
|
+
if (result === EXIT) return;
|
|
1410
|
+
if (result === SKIP) continue;
|
|
1411
|
+
const children = node.children;
|
|
1412
|
+
if (children === undefined) continue;
|
|
1413
|
+
for (let i = children.length - 1; i >= 0; i--) {
|
|
1414
|
+
nodes.push(children[i]);
|
|
1415
|
+
indices.push(i);
|
|
1416
|
+
parents.push(node);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
/**
|
|
1422
|
+
* `parse` with its most recent (input, tree) pair remembered.
|
|
1423
|
+
*
|
|
1424
|
+
* Layers 2 and 3 each parse the SAME tool output: `sanitizeHtml` tokenizes it to
|
|
1425
|
+
* decide the source-vs-markdown branch, and `detectExfil` tokenizes it again to
|
|
1426
|
+
* read `src`/`href` off the elements — two full parse5 runs and two full
|
|
1427
|
+
* micromark runs over one document, which is most of what the HTML layer costs
|
|
1428
|
+
* on ordinary prose. Every consumer only READS the tree (the sole AST mutation
|
|
1429
|
+
* in this module is over a css-tree value parsed per style string), so one tree
|
|
1430
|
+
* is safe to hand to both.
|
|
1431
|
+
*
|
|
1432
|
+
* One entry, replaced on every miss, so the retained footprint is one tree for
|
|
1433
|
+
* the document most recently sanitized — the tree that call had allocated
|
|
1434
|
+
* anyway. A miss re-parses and answers identically, so the cache can never
|
|
1435
|
+
* change a verdict, only what one costs.
|
|
1436
|
+
* @param {(text: string) => any} parse
|
|
1437
|
+
* @returns {(text: string) => any}
|
|
1438
|
+
*/
|
|
1439
|
+
function lastParseCached(parse) {
|
|
1440
|
+
/** @type {string | null} */
|
|
1441
|
+
let cachedText = null;
|
|
1442
|
+
/** @type {any} */
|
|
1443
|
+
let cachedTree = null;
|
|
1444
|
+
return (text) => {
|
|
1445
|
+
if (cachedText === text) return cachedTree;
|
|
1446
|
+
// The key is recorded only AFTER the parse returns. A parse that throws —
|
|
1447
|
+
// which is how a pathologically nested fragment reaches the fail-closed
|
|
1448
|
+
// withhold — must leave the entry untouched, or the next call for that same
|
|
1449
|
+
// text hits a key whose tree came from a DIFFERENT document and gets a
|
|
1450
|
+
// verdict about the wrong input instead of the withhold.
|
|
1451
|
+
const tree = parse(text);
|
|
1452
|
+
cachedText = text;
|
|
1453
|
+
cachedTree = tree;
|
|
1454
|
+
return tree;
|
|
1455
|
+
};
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1377
1458
|
/**
|
|
1378
1459
|
* Parse `html` as an HTML fragment with the real tokenizer (parse5, via rehype).
|
|
1379
1460
|
* @param {string} html
|
|
1380
1461
|
* @returns {any}
|
|
1381
1462
|
*/
|
|
1382
|
-
|
|
1383
|
-
return htmlParser.parse(html);
|
|
1384
|
-
}
|
|
1463
|
+
const parseFragment = lastParseCached((html) => htmlParser.parse(html));
|
|
1385
1464
|
|
|
1386
1465
|
/**
|
|
1387
1466
|
* @param {string} htmlValue
|
|
@@ -1391,7 +1470,7 @@ function parseHtmlTag(htmlValue) {
|
|
|
1391
1470
|
const tree = parseFragment(htmlValue);
|
|
1392
1471
|
/** @type {any} */
|
|
1393
1472
|
let firstElement = null;
|
|
1394
|
-
|
|
1473
|
+
walk(tree, "element", (node) => {
|
|
1395
1474
|
firstElement = node;
|
|
1396
1475
|
return EXIT;
|
|
1397
1476
|
});
|
|
@@ -1606,7 +1685,7 @@ function scanFragmentTree(html, tree) {
|
|
|
1606
1685
|
const warned = newWarned();
|
|
1607
1686
|
// @ts-ignore -- visit callback returns EXIT/SKIP only on matches; implicit undefined return is intentional
|
|
1608
1687
|
// eslint-disable-next-line consistent-return
|
|
1609
|
-
|
|
1688
|
+
walk(tree, null, (/** @type {any} */ node) => {
|
|
1610
1689
|
const isComment = node.type === "comment";
|
|
1611
1690
|
if (isComment || isHiddenElement(node)) {
|
|
1612
1691
|
/* c8 ignore start -- parse5 omits positions only on recovery-synthesized
|
|
@@ -1635,6 +1714,20 @@ function scanFragmentTree(html, tree) {
|
|
|
1635
1714
|
|
|
1636
1715
|
const mdParser = unified().use(remarkParse).use(remarkGfm);
|
|
1637
1716
|
|
|
1717
|
+
/** The markdown tree for `text`, cached the same way {@link parseFragment} is.
|
|
1718
|
+
* @type {(text: string) => any} */
|
|
1719
|
+
const parseMarkdown = lastParseCached((text) => mdParser.parse(text));
|
|
1720
|
+
|
|
1721
|
+
// A `code` node is a FENCED or INDENTED block and nothing else, so either a
|
|
1722
|
+
// three-run of backticks/tildes or a line opening on a four-column indent must
|
|
1723
|
+
// appear somewhere in the text for one to exist. CommonMark expands a tab to
|
|
1724
|
+
// the next four-column tab stop, so fewer than four spaces followed by a tab is
|
|
1725
|
+
// an indent too (" \tfoo" is a code block) — hence ` *\t` rather than `\t`.
|
|
1726
|
+
// Read over the whole document and deliberately loose (a stray ``` anywhere is
|
|
1727
|
+
// enough to parse), because the only sound direction to be wrong in here is
|
|
1728
|
+
// towards parsing.
|
|
1729
|
+
const MARKDOWN_CODE_HINT = /```|~~~|^(?: {4}| *\t)/m;
|
|
1730
|
+
|
|
1638
1731
|
// A markup-declaration-open (`<!`) or processing-instruction-ish (`<?`) start.
|
|
1639
1732
|
// Inside an inline html node these begin a *bogus comment* unless they open a
|
|
1640
1733
|
// proper `<!--…-->` comment (handled on the fast path) — `<!bogus>`, `<?php?>`,
|
|
@@ -1688,7 +1781,7 @@ function commentSpans(value) {
|
|
|
1688
1781
|
const tree = parseFragment(value);
|
|
1689
1782
|
/** @type {Map<number, number>} */
|
|
1690
1783
|
const spans = new Map();
|
|
1691
|
-
|
|
1784
|
+
walk(tree, "comment", (/** @type {any} */ node) => {
|
|
1692
1785
|
if (node.position)
|
|
1693
1786
|
spans.set(node.position.start.offset, node.position.end.offset);
|
|
1694
1787
|
});
|
|
@@ -1916,7 +2009,7 @@ const FLOW_HTML_PARENTS = new Set([
|
|
|
1916
2009
|
* @returns {{ ranges: SpliceRange[], warned: ReturnType<typeof newWarned> }}
|
|
1917
2010
|
*/
|
|
1918
2011
|
function scanMarkdown(text) {
|
|
1919
|
-
const tree =
|
|
2012
|
+
const tree = parseMarkdown(text);
|
|
1920
2013
|
/** @type {SpliceRange[]} */
|
|
1921
2014
|
const ranges = [];
|
|
1922
2015
|
const warned = newWarned();
|
|
@@ -1924,7 +2017,7 @@ function scanMarkdown(text) {
|
|
|
1924
2017
|
// Flow html blocks carry complete markup, so rehype locates comments/hidden
|
|
1925
2018
|
// elements precisely within them; block-local offsets are shifted to
|
|
1926
2019
|
// document coordinates.
|
|
1927
|
-
|
|
2020
|
+
walk(tree, "html", (/** @type {any} */ node, _index, parent) => {
|
|
1928
2021
|
if (!FLOW_HTML_PARENTS.has(parent?.type)) return;
|
|
1929
2022
|
const base = node.position.start.offset;
|
|
1930
2023
|
const sub = scanHtmlFragment(text.slice(base, node.position.end.offset));
|
|
@@ -1944,7 +2037,7 @@ function scanMarkdown(text) {
|
|
|
1944
2037
|
// are walked as part of their root in document order, so the walk is skipped
|
|
1945
2038
|
// for them here to avoid double-scanning and to keep the absorb state flowing
|
|
1946
2039
|
// across those boundaries.
|
|
1947
|
-
|
|
2040
|
+
walk(tree, null, (/** @type {any} */ node) => {
|
|
1948
2041
|
if (!PHRASING_ROOTS.has(node.type)) return;
|
|
1949
2042
|
if (!hasHtmlLeaf(node)) return;
|
|
1950
2043
|
scanInlineChildren(node, text, ranges, warned);
|
|
@@ -1969,8 +2062,9 @@ function scanMarkdown(text) {
|
|
|
1969
2062
|
* @returns {boolean}
|
|
1970
2063
|
*/
|
|
1971
2064
|
function hasMarkdownCode(text) {
|
|
2065
|
+
if (!MARKDOWN_CODE_HINT.test(text)) return false;
|
|
1972
2066
|
let found = false;
|
|
1973
|
-
|
|
2067
|
+
walk(parseMarkdown(text), "code", () => {
|
|
1974
2068
|
found = true;
|
|
1975
2069
|
return EXIT;
|
|
1976
2070
|
});
|
|
@@ -2564,7 +2658,7 @@ function extractHtmlUrls(text) {
|
|
|
2564
2658
|
const tree = parseFragment(text);
|
|
2565
2659
|
/** @type {Array<{ url: string, isImage: boolean, autoFetched: boolean, context: "resource" | "form" | "refresh" }>} */
|
|
2566
2660
|
const urls = [];
|
|
2567
|
-
|
|
2661
|
+
walk(tree, "element", (/** @type {any} */ node) => {
|
|
2568
2662
|
// hast element nodes always carry a `properties` object (parse5 sets it).
|
|
2569
2663
|
const props = node.properties;
|
|
2570
2664
|
const isImage = node.tagName === "img";
|
|
@@ -2640,8 +2734,8 @@ export function detectExfil(text) {
|
|
|
2640
2734
|
try {
|
|
2641
2735
|
// Remark AST handles markdown links/images/definitions (balanced parens,
|
|
2642
2736
|
// reference links) correctly, unlike a hand-rolled regex.
|
|
2643
|
-
const tree =
|
|
2644
|
-
|
|
2737
|
+
const tree = parseMarkdown(text);
|
|
2738
|
+
walk(tree, null, (node) => {
|
|
2645
2739
|
if (
|
|
2646
2740
|
node.type !== "link" &&
|
|
2647
2741
|
node.type !== "image" &&
|
package/src/invisible.mjs
CHANGED
|
@@ -405,6 +405,13 @@ const EMOJI_BASE = /\p{Extended_Pictographic}/u;
|
|
|
405
405
|
// following ZWJ (🏳️🌈 = flag base, VS16, ZWJ, rainbow; 👁️🗨️), so the joiner's real
|
|
406
406
|
// left neighbor for the emoji test is the pictograph, not the selector.
|
|
407
407
|
const VARIATION_SELECTOR = new RegExp(`[${VS}]`, "u");
|
|
408
|
+
// Code-point twins of the four single-character classes above, memoized: the
|
|
409
|
+
// carve analysis asks these per invisible (and per neighbour), and a Unicode
|
|
410
|
+
// property class costs far more than a table hit. See memoizedCpPredicate.
|
|
411
|
+
const isEmojiLeft = memoizedCpPredicate(EMOJI_LEFT);
|
|
412
|
+
const isEmojiBase = memoizedCpPredicate(EMOJI_BASE);
|
|
413
|
+
const isKeycapBase = memoizedCpPredicate(KEYCAP_BASE);
|
|
414
|
+
const isVariationSelectorCp = memoizedCpPredicate(VARIATION_SELECTOR);
|
|
408
415
|
// U+FE0F (VS16) forces emoji presentation, U+FE0E (VS15) forces text
|
|
409
416
|
// presentation (☺︎ vs ☺); either one directly after a pictograph is part of a
|
|
410
417
|
// visible glyph, not a hidden variation-selector run.
|
|
@@ -462,11 +469,10 @@ const IVS_MAX = 0xe01ef;
|
|
|
462
469
|
const CJK_IDEOGRAPH_RE =
|
|
463
470
|
/[\p{Unified_Ideograph}\u{F900}-\u{FAFF}\u{2F800}-\u{2FA1F}]/u;
|
|
464
471
|
|
|
465
|
-
/** True when `
|
|
466
|
-
* selector legitimately follows).
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
}
|
|
472
|
+
/** True when `cp` is a CJK ideograph (the only base an ideographic variation
|
|
473
|
+
* selector legitimately follows). -1 (past a string boundary) is not.
|
|
474
|
+
* @type {(cp: number) => boolean} */
|
|
475
|
+
const isCjkIdeograph = memoizedCpPredicate(CJK_IDEOGRAPH_RE);
|
|
470
476
|
|
|
471
477
|
// Brahmic consonants: the only base a virama does half-form/conjunct work on.
|
|
472
478
|
// A bare or base-less halant + ZWJ carries no rendering and is a smuggling
|
|
@@ -485,15 +491,11 @@ function isCjkIdeograph(ch) {
|
|
|
485
491
|
// against the script it claims.
|
|
486
492
|
export { BRAHMIC_CONSONANT_RANGES } from "./joining-type.mjs";
|
|
487
493
|
|
|
488
|
-
/** True when `
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
function isBrahmicConsonantChar(ch) {
|
|
494
|
-
return (
|
|
495
|
-
ch !== "" && isBrahmicConsonant(/** @type {number} */ (ch.codePointAt(0)))
|
|
496
|
-
);
|
|
494
|
+
/** True when the code point `cp` is a Brahmic consonant; -1 (past a string
|
|
495
|
+
* boundary) is not. A thin sign-guard over ./joining-type.mjs's table lookup.
|
|
496
|
+
* @param {number} cp @returns {boolean} */
|
|
497
|
+
function isBrahmicConsonantCp(cp) {
|
|
498
|
+
return cp >= 0 && isBrahmicConsonant(cp);
|
|
497
499
|
}
|
|
498
500
|
|
|
499
501
|
// ─── Blank-filler carve-out (Braille / archaic Hangul) ───────────────────────
|
|
@@ -528,11 +530,12 @@ const GATED_BLANK_RE = new RegExp("[\\u115F\\u1160\\u2800\\u3164\\uFFA0]", "u");
|
|
|
528
530
|
// Script_Extensions=Braille — no character is shared with another script).
|
|
529
531
|
const BRAILLE_RE = /\p{Script=Braille}/u;
|
|
530
532
|
|
|
533
|
+
const isBrailleScript = memoizedCpPredicate(BRAILLE_RE);
|
|
534
|
+
|
|
531
535
|
/** A real (non-blank) Braille cell — the anchoring neighbour for a U+2800 blank.
|
|
532
|
-
* @param {
|
|
533
|
-
function isBrailleCell(
|
|
534
|
-
|
|
535
|
-
return cp !== BRAILLE_BLANK && BRAILLE_RE.test(ch);
|
|
536
|
+
* @param {number} cp @returns {boolean} */
|
|
537
|
+
function isBrailleCell(cp) {
|
|
538
|
+
return cp !== BRAILLE_BLANK && isBrailleScript(cp);
|
|
536
539
|
}
|
|
537
540
|
|
|
538
541
|
// Script=Hangul, straight from the runtime's Unicode data. NOT
|
|
@@ -542,28 +545,61 @@ function isBrailleCell(ch) {
|
|
|
542
545
|
// thereby preserve — a Hangul filler in text with no Hangul in it at all.
|
|
543
546
|
const HANGUL_RE = /\p{Script=Hangul}/u;
|
|
544
547
|
|
|
548
|
+
const isHangulScript = memoizedCpPredicate(HANGUL_RE);
|
|
549
|
+
|
|
545
550
|
/** A Hangul jamo/syllable (NOT itself one of the fillers) — the anchoring
|
|
546
|
-
* neighbour for a Hangul filler. @param {
|
|
547
|
-
function isHangul(
|
|
548
|
-
const cp = ch ? /** @type {number} */ (ch.codePointAt(0)) : -1;
|
|
551
|
+
* neighbour for a Hangul filler. @param {number} cp @returns {boolean} */
|
|
552
|
+
function isHangul(cp) {
|
|
549
553
|
if (HANGUL_FILLERS.has(cp)) return false; // a filler cannot anchor another filler
|
|
550
|
-
return
|
|
554
|
+
return isHangulScript(cp);
|
|
551
555
|
}
|
|
552
556
|
|
|
553
|
-
//
|
|
554
|
-
//
|
|
555
|
-
//
|
|
557
|
+
// The carve analysis runs once per code point of every prompt and every tool
|
|
558
|
+
// output, so its per-code-point state is held in TYPED arrays of small integer
|
|
559
|
+
// codes rather than arrays of category strings: `Uint8Array` costs one byte per
|
|
560
|
+
// code point with no pointer to trace, where a `(string|null)[]` costs a machine
|
|
561
|
+
// word per entry and hands the collector one more object graph to walk on every
|
|
562
|
+
// blocking hook call. These are the codes; CODE_CATEGORY projects them back onto
|
|
563
|
+
// the public CATEGORY strings at the boundary, so nothing outside this module
|
|
564
|
+
// sees them.
|
|
565
|
+
const CODE_VISIBLE = 0;
|
|
566
|
+
const CODE_CF = 1;
|
|
567
|
+
const CODE_VS = 2;
|
|
568
|
+
const CODE_BLANK = 3;
|
|
569
|
+
|
|
570
|
+
// Preserve kinds, in the same one-byte-per-code-point spirit. KIND_NONE is the
|
|
571
|
+
// zero value, so a freshly allocated `kind` array already reads "payload" —
|
|
572
|
+
// which is the fail-closed default this analysis wants.
|
|
573
|
+
const KIND_NONE = 0;
|
|
574
|
+
const KIND_JOINER = 1;
|
|
575
|
+
const KIND_EMOJIVS = 2;
|
|
576
|
+
const KIND_TAG = 3;
|
|
577
|
+
const KIND_STDVS = 4;
|
|
578
|
+
const KIND_IVS = 5;
|
|
579
|
+
const KIND_BLANK = 6;
|
|
580
|
+
|
|
581
|
+
/** Small code -> CHECKS category, indexed by the CODE_* values above. In CHECKS
|
|
582
|
+
* order, so a code point in two sets reports the category the regexes would
|
|
583
|
+
* report first. @type {(string|null)[]} */
|
|
584
|
+
const CODE_CATEGORY = [
|
|
585
|
+
null,
|
|
586
|
+
CATEGORY.CF,
|
|
587
|
+
CATEGORY.VARIATION_SELECTORS,
|
|
588
|
+
CATEGORY.BLANK_FILLERS,
|
|
589
|
+
];
|
|
590
|
+
|
|
591
|
+
// code point -> CODE_* above, projected from the SAME three code-point sets the
|
|
592
|
+
// CHECKS regexes are built from, in CHECKS order. Every CHECKS class holds
|
|
556
593
|
// single code points under the `u` flag, so a lookup and a `.test` answer
|
|
557
594
|
// identically; test/invisible-fast-path.test.mjs pins that over the whole
|
|
558
595
|
// code-point space, because a code point dropped here is one the scatter gate
|
|
559
|
-
// stops counting.
|
|
560
|
-
|
|
561
|
-
/** @type {Map<number, string>} */
|
|
596
|
+
// stops counting.
|
|
597
|
+
/** @type {Map<number, number>} */
|
|
562
598
|
const TRACKED_INVISIBLE = new Map();
|
|
563
|
-
for (const [code, codepoints] of /** @type {[
|
|
564
|
-
[
|
|
565
|
-
[
|
|
566
|
-
[
|
|
599
|
+
for (const [code, codepoints] of /** @type {[number, Iterable<number>][]} */ ([
|
|
600
|
+
[CODE_CF, CF_CODEPOINTS],
|
|
601
|
+
[CODE_VS, [...VS].map((ch) => ch.codePointAt(0))],
|
|
602
|
+
[CODE_BLANK, [...BLANK_NON_CF].map((ch) => ch.codePointAt(0))],
|
|
567
603
|
]))
|
|
568
604
|
for (const cp of codepoints)
|
|
569
605
|
if (!TRACKED_INVISIBLE.has(cp)) TRACKED_INVISIBLE.set(cp, code);
|
|
@@ -573,32 +609,87 @@ for (const [code, codepoints] of /** @type {[string, Iterable<number>][]} */ ([
|
|
|
573
609
|
const MIN_TRACKED_CP = Math.min(...TRACKED_INVISIBLE.keys());
|
|
574
610
|
|
|
575
611
|
/**
|
|
576
|
-
* The
|
|
577
|
-
*
|
|
612
|
+
* The CODE_* class a code point belongs to, or CODE_VISIBLE when it is not
|
|
613
|
+
* payload-capable (an ordinary visible character).
|
|
578
614
|
* @param {number} cp
|
|
579
|
-
* @returns {
|
|
615
|
+
* @returns {number}
|
|
580
616
|
*/
|
|
581
617
|
function classifyCp(cp) {
|
|
582
|
-
if (cp < MIN_TRACKED_CP) return
|
|
583
|
-
return TRACKED_INVISIBLE.get(cp) ??
|
|
618
|
+
if (cp < MIN_TRACKED_CP) return CODE_VISIBLE;
|
|
619
|
+
return TRACKED_INVISIBLE.get(cp) ?? CODE_VISIBLE;
|
|
584
620
|
}
|
|
585
621
|
|
|
622
|
+
// Distinct code points a memoizedCpPredicate will remember. Comfortably above
|
|
623
|
+
// the character repertoire of any real document (and of every test corpus here)
|
|
624
|
+
// while bounding what a single hostile paste can make one predicate retain.
|
|
625
|
+
const CP_PREDICATE_MEMO_CAP = 4096;
|
|
626
|
+
|
|
586
627
|
/**
|
|
587
|
-
*
|
|
588
|
-
*
|
|
589
|
-
*
|
|
590
|
-
*
|
|
628
|
+
* A code-point predicate backed by a single-code-point RegExp, memoized on the
|
|
629
|
+
* code point.
|
|
630
|
+
*
|
|
631
|
+
* The carve analysis asks these of the same handful of code points over and
|
|
632
|
+
* over — a document is a few hundred distinct characters repeated — and each
|
|
633
|
+
* ask otherwise costs a `String.fromCodePoint` allocation plus a Unicode
|
|
634
|
+
* property-class match. The memo is a pure accelerator: it answers exactly what
|
|
635
|
+
* the RegExp answers, for the same input, forever (a code point's Unicode
|
|
636
|
+
* properties do not change inside a process).
|
|
637
|
+
*
|
|
638
|
+
* The entry cap is a MEMORY policy, not a scan bound: past it the predicate
|
|
639
|
+
* still answers every code point, just without recording new ones, so no input
|
|
640
|
+
* can change a verdict by overflowing the table — only by giving up the speed-up
|
|
641
|
+
* it would otherwise have had.
|
|
642
|
+
* @param {RegExp} re single-code-point matcher
|
|
643
|
+
* @returns {(cp: number) => boolean}
|
|
644
|
+
*/
|
|
645
|
+
function memoizedCpPredicate(re) {
|
|
646
|
+
/** @type {Map<number, boolean>} */
|
|
647
|
+
const memo = new Map();
|
|
648
|
+
return (cp) => {
|
|
649
|
+
if (cp < 0) return false; // past a string boundary: the "" neighbour
|
|
650
|
+
const hit = memo.get(cp);
|
|
651
|
+
if (hit !== undefined) return hit;
|
|
652
|
+
const answer = re.test(String.fromCodePoint(cp));
|
|
653
|
+
if (memo.size < CP_PREDICATE_MEMO_CAP) memo.set(cp, answer);
|
|
654
|
+
return answer;
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* `text` as its code points, in the units the String iterator yields — the same
|
|
660
|
+
* sequence `Array.from(text)` produces (an unpaired surrogate stands alone as
|
|
661
|
+
* itself), as numbers rather than one heap-allocated one-character string each.
|
|
662
|
+
* Every carve-analysis predicate reads a code point, so the numbers are the
|
|
663
|
+
* whole requirement: a 256 KB paste that would cost a quarter of a million
|
|
664
|
+
* short-lived strings per pass costs one buffer, and the collector stays out of
|
|
665
|
+
* a blocking hook.
|
|
666
|
+
* @param {string} text
|
|
667
|
+
* @returns {Int32Array}
|
|
591
668
|
*/
|
|
592
|
-
function
|
|
593
|
-
|
|
669
|
+
function codePointArray(text) {
|
|
670
|
+
const cps = new Int32Array(text.length);
|
|
671
|
+
let n = 0;
|
|
672
|
+
for (let i = 0; i < text.length; i++) {
|
|
673
|
+
const unit = text.charCodeAt(i);
|
|
674
|
+
let cp = unit;
|
|
675
|
+
if (unit >= 0xd800 && unit <= 0xdbff && i + 1 < text.length) {
|
|
676
|
+
const low = text.charCodeAt(i + 1);
|
|
677
|
+
if (low >= 0xdc00 && low <= 0xdfff) {
|
|
678
|
+
cp = (unit - 0xd800) * 0x400 + (low - 0xdc00) + 0x10000;
|
|
679
|
+
i++;
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
cps[n++] = cp;
|
|
683
|
+
}
|
|
684
|
+
return n === cps.length ? cps : cps.subarray(0, n);
|
|
594
685
|
}
|
|
595
686
|
|
|
596
687
|
/**
|
|
597
|
-
* True when `text` holds at least one code point {@link
|
|
598
|
-
* invisible. Walks UTF-16 units directly rather than
|
|
599
|
-
*
|
|
600
|
-
*
|
|
601
|
-
*
|
|
688
|
+
* True when `text` holds at least one code point {@link classifyCp} calls
|
|
689
|
+
* invisible. Walks UTF-16 units directly rather than building the code-point
|
|
690
|
+
* array, so a clean multi-MB paste is answered without allocating anything;
|
|
691
|
+
* surrogates are paired exactly as the String iterator pairs them, an unpaired
|
|
692
|
+
* one classifying as the lone surrogate it is.
|
|
602
693
|
* @param {string} text
|
|
603
694
|
* @returns {boolean}
|
|
604
695
|
*/
|
|
@@ -612,7 +703,7 @@ function hasInvisibleCodePoint(text) {
|
|
|
612
703
|
i++;
|
|
613
704
|
}
|
|
614
705
|
}
|
|
615
|
-
if (classifyCp(cp) !==
|
|
706
|
+
if (classifyCp(cp) !== CODE_VISIBLE) return true;
|
|
616
707
|
}
|
|
617
708
|
return false;
|
|
618
709
|
}
|
|
@@ -636,15 +727,13 @@ function codePointLength(text) {
|
|
|
636
727
|
* @param {string} jt @returns {boolean} */
|
|
637
728
|
const isCursiveLetter = (jt) => jt === "D" || jt === "R" || jt === "L";
|
|
638
729
|
|
|
639
|
-
/** The Joining_Type of a
|
|
640
|
-
* @param {
|
|
641
|
-
const jtOf = (
|
|
642
|
-
ch ? joiningType(/** @type {number} */ (ch.codePointAt(0))) : "U";
|
|
730
|
+
/** The Joining_Type of a code point, or "U" for -1 (past a string boundary).
|
|
731
|
+
* @param {number} cp @returns {string} */
|
|
732
|
+
const jtOf = (cp) => (cp < 0 ? "U" : joiningType(cp));
|
|
643
733
|
|
|
644
|
-
/** True when `
|
|
645
|
-
* @param {
|
|
646
|
-
function isJoinControl(
|
|
647
|
-
const cp = ch ? ch.codePointAt(0) : -1;
|
|
734
|
+
/** True when `cp` is itself a ZWNJ/ZWJ (used to reject joiner runs).
|
|
735
|
+
* @param {number} cp @returns {boolean} */
|
|
736
|
+
function isJoinControl(cp) {
|
|
648
737
|
return cp === ZWNJ || cp === ZWJ;
|
|
649
738
|
}
|
|
650
739
|
|
|
@@ -658,18 +747,18 @@ function isJoinControl(ch) {
|
|
|
658
747
|
* between two joiners hides the joiner-RUN until a first pass removes it, after
|
|
659
748
|
* which a second pass strips the now-adjacent joiners — non-idempotent. Join
|
|
660
749
|
* controls (ZWNJ/ZWJ) are deliberately NOT skipped: they are the run signal.
|
|
661
|
-
* Returns
|
|
662
|
-
* @param {
|
|
663
|
-
* @returns {
|
|
750
|
+
* Returns -1 past the string boundary (the old "" neighbour).
|
|
751
|
+
* @param {Int32Array} cps @param {number} i @param {number} dir -1 or +1
|
|
752
|
+
* @returns {number}
|
|
664
753
|
*/
|
|
665
754
|
function effectiveNeighbor(cps, i, dir) {
|
|
666
755
|
for (let j = i + dir; j >= 0 && j < cps.length; j += dir) {
|
|
667
|
-
const
|
|
668
|
-
if (jtOf(
|
|
669
|
-
if (!isJoinControl(
|
|
670
|
-
return
|
|
756
|
+
const cp = cps[j];
|
|
757
|
+
if (jtOf(cp) === "T") continue;
|
|
758
|
+
if (!isJoinControl(cp) && classifyCp(cp) !== CODE_VISIBLE) continue;
|
|
759
|
+
return cp;
|
|
671
760
|
}
|
|
672
|
-
return
|
|
761
|
+
return -1;
|
|
673
762
|
}
|
|
674
763
|
|
|
675
764
|
/**
|
|
@@ -680,15 +769,19 @@ function effectiveNeighbor(cps, i, dir) {
|
|
|
680
769
|
* virama (which is Joining_Type Transparent, so it is the anchor, never skipped),
|
|
681
770
|
* then uses effectiveNeighbor to find the consonant base past any transparent
|
|
682
771
|
* marks / invisibles between it and the virama.
|
|
683
|
-
* @param {
|
|
772
|
+
* @param {Int32Array} cps @param {number} i
|
|
684
773
|
* @returns {boolean}
|
|
685
774
|
*/
|
|
686
775
|
function followsBrahmicConjunct(cps, i) {
|
|
687
776
|
let j = i - 1;
|
|
688
|
-
while (
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
777
|
+
while (
|
|
778
|
+
j >= 0 &&
|
|
779
|
+
!isJoinControl(cps[j]) &&
|
|
780
|
+
classifyCp(cps[j]) !== CODE_VISIBLE
|
|
781
|
+
)
|
|
782
|
+
j--;
|
|
783
|
+
if (j < 0 || !isVirama(cps[j])) return false;
|
|
784
|
+
return isBrahmicConsonantCp(effectiveNeighbor(cps, j, -1));
|
|
692
785
|
}
|
|
693
786
|
|
|
694
787
|
/**
|
|
@@ -700,12 +793,12 @@ function followsBrahmicConjunct(cps, i) {
|
|
|
700
793
|
* - Arabic-family joiner: between two cursive letters (ZWNJ needs both, ZWJ at
|
|
701
794
|
* least one — it forces a connected form). A joiner whose effective neighbour
|
|
702
795
|
* is ANOTHER joiner is a run (a zero-width payload channel) and is rejected.
|
|
703
|
-
* Leading/trailing joiners fall out because
|
|
704
|
-
* @param {
|
|
796
|
+
* Leading/trailing joiners fall out because the -1 boundary has Joining_Type U.
|
|
797
|
+
* @param {Int32Array} cps @param {number} i
|
|
705
798
|
* @returns {boolean}
|
|
706
799
|
*/
|
|
707
800
|
function isPreservedJoiner(cps, i) {
|
|
708
|
-
const cp =
|
|
801
|
+
const cp = cps[i];
|
|
709
802
|
if (cp !== ZWNJ && cp !== ZWJ) return false;
|
|
710
803
|
// Emoji ZWJ sequences use ZWJ only; the real neighbours are the pictographs,
|
|
711
804
|
// which may each sit behind a variation selector (🏳️🌈 = base VS16 ZWJ rainbow;
|
|
@@ -713,8 +806,8 @@ function isPreservedJoiner(cps, i) {
|
|
|
713
806
|
// selectors on BOTH sides — leftNonSelector and its mirror rightNonSelector.
|
|
714
807
|
if (
|
|
715
808
|
cp === ZWJ &&
|
|
716
|
-
|
|
717
|
-
|
|
809
|
+
isEmojiLeft(leftNonSelector(cps, i)) &&
|
|
810
|
+
isEmojiBase(rightNonSelector(cps, i))
|
|
718
811
|
)
|
|
719
812
|
return true;
|
|
720
813
|
// Indic: meaningful only after a virama sitting on a real consonant base.
|
|
@@ -737,62 +830,65 @@ function isPreservedJoiner(cps, i) {
|
|
|
737
830
|
* A keycap base + selector with NO trailing U+20E3 is a bare hidden presentation
|
|
738
831
|
* selector, so it fails closed and is stripped. A longer selector run still
|
|
739
832
|
* surfaces: the next selector's left neighbour is itself a selector.
|
|
740
|
-
* @param {
|
|
833
|
+
* @param {Int32Array} cps @param {number} i
|
|
741
834
|
* @returns {boolean}
|
|
742
835
|
*/
|
|
743
836
|
function isEmojiPresentationSelector(cps, i) {
|
|
744
|
-
if (
|
|
745
|
-
|
|
746
|
-
)
|
|
747
|
-
return false;
|
|
748
|
-
const prev = cps[i - 1] ?? "";
|
|
749
|
-
if (EMOJI_LEFT.test(prev)) return true;
|
|
837
|
+
if (!PRESENTATION_SELECTORS.has(cps[i])) return false;
|
|
838
|
+
const prev = i > 0 ? cps[i - 1] : -1;
|
|
839
|
+
if (isEmojiLeft(prev)) return true;
|
|
750
840
|
return (
|
|
751
|
-
|
|
752
|
-
(cps[i + 1]
|
|
841
|
+
isKeycapBase(prev) &&
|
|
842
|
+
(i + 1 < cps.length ? cps[i + 1] : -1) === COMBINING_KEYCAP
|
|
753
843
|
);
|
|
754
844
|
}
|
|
755
845
|
|
|
756
846
|
/**
|
|
757
847
|
* Per-invisible carve-out analysis, shared by carveStrip and
|
|
758
|
-
* countPayloadInvisible: for each code point, its
|
|
759
|
-
* visible) and its preserve
|
|
760
|
-
*
|
|
761
|
-
*
|
|
762
|
-
*
|
|
763
|
-
* @param {
|
|
764
|
-
* @returns {{ codes:
|
|
848
|
+
* countPayloadInvisible: for each code point, its CODE_* class (CODE_VISIBLE
|
|
849
|
+
* when visible) and its preserve KIND_* (KIND_NONE when the char is payload).
|
|
850
|
+
* Everything invisible that is NOT preserve-eligible is payload; the scatter
|
|
851
|
+
* floor counts only that, so meaningful joiners/selectors never push honest
|
|
852
|
+
* prose over the threshold.
|
|
853
|
+
* @param {Int32Array} cps
|
|
854
|
+
* @returns {{ codes: Uint8Array, kind: Uint8Array, payloadInvis: number, visibleLen: number }}
|
|
765
855
|
*/
|
|
766
856
|
function analyzeCarve(cps) {
|
|
767
|
-
const codes = new
|
|
857
|
+
const codes = new Uint8Array(cps.length);
|
|
768
858
|
// Indices of the invisibles, so every pass below is O(invisibles) rather than
|
|
769
859
|
// O(length): in ordinary text they are a vanishing fraction of a paste, and a
|
|
770
860
|
// full-length pass per preserve rule is what made the analysis linear in the
|
|
771
861
|
// WHOLE prompt several times over.
|
|
772
|
-
|
|
862
|
+
let invisCount = 0;
|
|
773
863
|
let visibleLen = 0;
|
|
774
864
|
let hasTagBase = false;
|
|
775
865
|
for (let i = 0; i < cps.length; i++) {
|
|
776
|
-
const code =
|
|
866
|
+
const code = classifyCp(cps[i]);
|
|
777
867
|
codes[i] = code;
|
|
778
|
-
if (code !==
|
|
779
|
-
|
|
868
|
+
if (code !== CODE_VISIBLE) {
|
|
869
|
+
invisCount++;
|
|
780
870
|
continue;
|
|
781
871
|
}
|
|
782
872
|
visibleLen++;
|
|
783
|
-
hasTagBase ||= cps[i]
|
|
873
|
+
hasTagBase ||= cps[i] === TAG_BASE;
|
|
784
874
|
}
|
|
875
|
+
// Sized exactly, from the count the pass above already produced: a growable
|
|
876
|
+
// array reallocates its way up through a document that is mostly invisible,
|
|
877
|
+
// and that churn is the collector's whole share of this function.
|
|
878
|
+
const invisible = new Int32Array(invisCount);
|
|
879
|
+
for (let i = 0, at = 0; at < invisCount; i++)
|
|
880
|
+
if (codes[i] !== CODE_VISIBLE) invisible[at++] = i;
|
|
785
881
|
// A tag sequence is preservable only when it opens on a TAG_BASE pictograph,
|
|
786
882
|
// so with no base in the text markTagSequences returns all-false.
|
|
787
883
|
const tagKeep = hasTagBase ? markTagSequences(cps) : null;
|
|
788
|
-
const kind = new
|
|
884
|
+
const kind = new Uint8Array(cps.length);
|
|
789
885
|
for (const i of invisible) {
|
|
790
|
-
if (tagKeep !== null && tagKeep[i]) kind[i] =
|
|
791
|
-
else if (isPreservedJoiner(cps, i)) kind[i] =
|
|
792
|
-
else if (isEmojiPresentationSelector(cps, i)) kind[i] =
|
|
793
|
-
else if (isStandardizedVariationSelector(cps, i)) kind[i] =
|
|
794
|
-
else if (isIdeographicVariationSelector(cps, i)) kind[i] =
|
|
795
|
-
else if (isPreservedBlankFiller(cps, i)) kind[i] =
|
|
886
|
+
if (tagKeep !== null && tagKeep[i]) kind[i] = KIND_TAG;
|
|
887
|
+
else if (isPreservedJoiner(cps, i)) kind[i] = KIND_JOINER;
|
|
888
|
+
else if (isEmojiPresentationSelector(cps, i)) kind[i] = KIND_EMOJIVS;
|
|
889
|
+
else if (isStandardizedVariationSelector(cps, i)) kind[i] = KIND_STDVS;
|
|
890
|
+
else if (isIdeographicVariationSelector(cps, i)) kind[i] = KIND_IVS;
|
|
891
|
+
else if (isPreservedBlankFiller(cps, i)) kind[i] = KIND_BLANK;
|
|
796
892
|
}
|
|
797
893
|
// Blank fillers are budgeted here, document-wide and all-or-nothing, against
|
|
798
894
|
// the visible anchor-script text rather than against the joiner/selector
|
|
@@ -811,28 +907,25 @@ function analyzeCarve(cps) {
|
|
|
811
907
|
/** @type {Record<string, number[]>} */
|
|
812
908
|
const blankIndices = { braille: [], hangul: [] };
|
|
813
909
|
for (const i of invisible)
|
|
814
|
-
if (kind[i] ===
|
|
815
|
-
blankIndices[
|
|
816
|
-
cps[i].codePointAt(0) === BRAILLE_BLANK ? "braille" : "hangul"
|
|
817
|
-
].push(i);
|
|
910
|
+
if (kind[i] === KIND_BLANK)
|
|
911
|
+
blankIndices[cps[i] === BRAILLE_BLANK ? "braille" : "hangul"].push(i);
|
|
818
912
|
for (const [
|
|
819
913
|
script,
|
|
820
914
|
isAnchor,
|
|
821
|
-
] of /** @type {[string, (
|
|
915
|
+
] of /** @type {[string, (cp: number) => boolean][]} */ ([
|
|
822
916
|
["braille", isBrailleCell],
|
|
823
917
|
["hangul", isHangul],
|
|
824
918
|
])) {
|
|
825
919
|
const indices = blankIndices[script];
|
|
826
920
|
if (indices.length <= TOTAL_PRESERVED_BLANK_BUDGET) continue;
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
);
|
|
921
|
+
let anchors = 0;
|
|
922
|
+
for (let i = 0; i < cps.length; i++)
|
|
923
|
+
if (codes[i] === CODE_VISIBLE && isAnchor(cps[i])) anchors++;
|
|
831
924
|
if (indices.length > Math.floor(anchors / PRESERVED_BLANK_PER_ANCHOR))
|
|
832
|
-
for (const i of indices) kind[i] =
|
|
925
|
+
for (const i of indices) kind[i] = KIND_NONE;
|
|
833
926
|
}
|
|
834
927
|
let payloadInvis = 0;
|
|
835
|
-
for (const i of invisible) if (kind[i] ===
|
|
928
|
+
for (const i of invisible) if (kind[i] === KIND_NONE) payloadInvis++;
|
|
836
929
|
return { codes, kind, payloadInvis, visibleLen };
|
|
837
930
|
}
|
|
838
931
|
|
|
@@ -847,12 +940,12 @@ function analyzeCarve(cps) {
|
|
|
847
940
|
*/
|
|
848
941
|
export function countPayloadInvisible(text) {
|
|
849
942
|
// Payload is a SUBSET of the invisibles (analyzeCarve counts only positions
|
|
850
|
-
//
|
|
943
|
+
// classifyCp marks invisible), so a text with none has a payload count of zero
|
|
851
944
|
// and needs neither the code-point array nor the carve analysis. This reads
|
|
852
945
|
// every code point — it is a necessary condition checked in full, not a scan
|
|
853
946
|
// bound an attacker could paste past.
|
|
854
947
|
if (!hasInvisibleCodePoint(text)) return 0;
|
|
855
|
-
return analyzeCarve(
|
|
948
|
+
return analyzeCarve(codePointArray(text)).payloadInvis;
|
|
856
949
|
}
|
|
857
950
|
|
|
858
951
|
/**
|
|
@@ -872,33 +965,33 @@ function bulkStrip(body) {
|
|
|
872
965
|
|
|
873
966
|
/**
|
|
874
967
|
* The nearest code point left of index `i` that is not a variation selector, or
|
|
875
|
-
*
|
|
968
|
+
* -1 at the string start. An emoji ZWJ sequence can place a VS16 between the base
|
|
876
969
|
* pictograph and the ZWJ, so the joiner's real left neighbor is found by stepping
|
|
877
970
|
* over any variation selector(s).
|
|
878
|
-
* @param {
|
|
971
|
+
* @param {Int32Array} cps
|
|
879
972
|
* @param {number} i
|
|
880
|
-
* @returns {
|
|
973
|
+
* @returns {number}
|
|
881
974
|
*/
|
|
882
975
|
function leftNonSelector(cps, i) {
|
|
883
976
|
let p = i - 1;
|
|
884
|
-
while (p >= 0 &&
|
|
885
|
-
return cps[p]
|
|
977
|
+
while (p >= 0 && isVariationSelectorCp(cps[p])) p--;
|
|
978
|
+
return p >= 0 ? cps[p] : -1;
|
|
886
979
|
}
|
|
887
980
|
|
|
888
981
|
/**
|
|
889
982
|
* The nearest code point right of index `i` that is not a variation selector, or
|
|
890
|
-
*
|
|
983
|
+
* -1 at the string end. Mirror of {@link leftNonSelector}: an emoji ZWJ can be
|
|
891
984
|
* followed by a selector before the next component's base pictograph, so the
|
|
892
985
|
* joiner's real right neighbor is found by stepping over any variation
|
|
893
986
|
* selector(s).
|
|
894
|
-
* @param {
|
|
987
|
+
* @param {Int32Array} cps
|
|
895
988
|
* @param {number} i
|
|
896
|
-
* @returns {
|
|
989
|
+
* @returns {number}
|
|
897
990
|
*/
|
|
898
991
|
function rightNonSelector(cps, i) {
|
|
899
992
|
let p = i + 1;
|
|
900
|
-
while (p < cps.length &&
|
|
901
|
-
return cps[p]
|
|
993
|
+
while (p < cps.length && isVariationSelectorCp(cps[p])) p++;
|
|
994
|
+
return p < cps.length ? cps[p] : -1;
|
|
902
995
|
}
|
|
903
996
|
|
|
904
997
|
/**
|
|
@@ -910,23 +1003,17 @@ function rightNonSelector(cps, i) {
|
|
|
910
1003
|
* payload is not a registered subdivision is left unmarked so it is stripped —
|
|
911
1004
|
* grammatical validity alone is NOT enough, since a valid run spells arbitrary
|
|
912
1005
|
* ASCII (fail closed on the top ASCII-smuggling vector).
|
|
913
|
-
* @param {
|
|
914
|
-
* @returns {
|
|
1006
|
+
* @param {Int32Array} cps
|
|
1007
|
+
* @returns {Uint8Array} keep[i] non-zero iff `cps[i]` is inside a preservable tag sequence
|
|
915
1008
|
*/
|
|
916
1009
|
function markTagSequences(cps) {
|
|
917
|
-
const keep = new
|
|
918
|
-
/** @param {number} k @returns {number} */
|
|
919
|
-
const cpAt = (k) => /** @type {number} */ (cps[k].codePointAt(0));
|
|
1010
|
+
const keep = new Uint8Array(cps.length);
|
|
920
1011
|
for (let i = 0; i < cps.length; i++) {
|
|
921
|
-
if (
|
|
1012
|
+
if (cps[i] !== TAG_BASE) continue;
|
|
922
1013
|
let j = i + 1;
|
|
923
1014
|
let payload = "";
|
|
924
|
-
while (
|
|
925
|
-
|
|
926
|
-
cpAt(j) >= TAG_SPEC_MIN &&
|
|
927
|
-
cpAt(j) <= TAG_SPEC_MAX
|
|
928
|
-
) {
|
|
929
|
-
payload += String.fromCharCode(cpAt(j) - 0xe0000);
|
|
1015
|
+
while (j < cps.length && cps[j] >= TAG_SPEC_MIN && cps[j] <= TAG_SPEC_MAX) {
|
|
1016
|
+
payload += String.fromCharCode(cps[j] - 0xe0000);
|
|
930
1017
|
j++;
|
|
931
1018
|
}
|
|
932
1019
|
const tagLen = j - (i + 1);
|
|
@@ -936,10 +1023,10 @@ function markTagSequences(cps) {
|
|
|
936
1023
|
tagLen >= 1 &&
|
|
937
1024
|
tagLen <= MAX_TAG_SPEC_CHARS &&
|
|
938
1025
|
j < cps.length &&
|
|
939
|
-
|
|
1026
|
+
cps[j] === TAG_CANCEL &&
|
|
940
1027
|
REGISTERED_TAG_PAYLOADS.has(payload)
|
|
941
1028
|
) {
|
|
942
|
-
for (let k = i + 1; k <= j; k++) keep[k] =
|
|
1029
|
+
for (let k = i + 1; k <= j; k++) keep[k] = 1;
|
|
943
1030
|
i = j; // resume after the consumed sequence
|
|
944
1031
|
}
|
|
945
1032
|
}
|
|
@@ -951,29 +1038,26 @@ function markTagSequences(cps) {
|
|
|
951
1038
|
* immediately preceding code point forms a REGISTERED standardized variation
|
|
952
1039
|
* sequence (per the generated UCD table). Every unregistered FE00–FE0D selector
|
|
953
1040
|
* stays payload.
|
|
954
|
-
* @param {
|
|
1041
|
+
* @param {Int32Array} cps @param {number} i
|
|
955
1042
|
* @returns {boolean}
|
|
956
1043
|
*/
|
|
957
1044
|
function isStandardizedVariationSelector(cps, i) {
|
|
958
|
-
const cp =
|
|
1045
|
+
const cp = cps[i];
|
|
959
1046
|
if (cp < 0xfe00 || cp > 0xfe0d) return false;
|
|
960
|
-
|
|
961
|
-
return prev
|
|
962
|
-
? isStandardizedVariant(/** @type {number} */ (prev.codePointAt(0)), cp)
|
|
963
|
-
: false;
|
|
1047
|
+
return i > 0 ? isStandardizedVariant(cps[i - 1], cp) : false;
|
|
964
1048
|
}
|
|
965
1049
|
|
|
966
1050
|
/**
|
|
967
1051
|
* True when `cps[i]` is an ideographic variation selector (VS17–VS256,
|
|
968
1052
|
* U+E0100–U+E01EF) immediately after a CJK ideograph — the registry-faithful
|
|
969
1053
|
* structural gate for an ideographic variation sequence.
|
|
970
|
-
* @param {
|
|
1054
|
+
* @param {Int32Array} cps @param {number} i
|
|
971
1055
|
* @returns {boolean}
|
|
972
1056
|
*/
|
|
973
1057
|
function isIdeographicVariationSelector(cps, i) {
|
|
974
|
-
const cp =
|
|
1058
|
+
const cp = cps[i];
|
|
975
1059
|
if (cp < IVS_MIN || cp > IVS_MAX) return false;
|
|
976
|
-
return isCjkIdeograph(cps[i - 1]
|
|
1060
|
+
return isCjkIdeograph(i > 0 ? cps[i - 1] : -1);
|
|
977
1061
|
}
|
|
978
1062
|
|
|
979
1063
|
/**
|
|
@@ -981,13 +1065,13 @@ function isIdeographicVariationSelector(cps, i) {
|
|
|
981
1065
|
* next to a real, script-appropriate visible neighbour — a genuine empty Braille
|
|
982
1066
|
* cell or archaic-Korean filler, not a hidden run. The zero-width Mn marks in
|
|
983
1067
|
* BLANK_NON_CF have no such anchored use and are never preserved here.
|
|
984
|
-
* @param {
|
|
1068
|
+
* @param {Int32Array} cps @param {number} i
|
|
985
1069
|
* @returns {boolean}
|
|
986
1070
|
*/
|
|
987
1071
|
function isPreservedBlankFiller(cps, i) {
|
|
988
|
-
const cp =
|
|
989
|
-
const prev = cps[i - 1]
|
|
990
|
-
const next = cps[i + 1]
|
|
1072
|
+
const cp = cps[i];
|
|
1073
|
+
const prev = i > 0 ? cps[i - 1] : -1;
|
|
1074
|
+
const next = i + 1 < cps.length ? cps[i + 1] : -1;
|
|
991
1075
|
if (cp === BRAILLE_BLANK) return isBrailleCell(prev) || isBrailleCell(next);
|
|
992
1076
|
if (HANGUL_FILLERS.has(cp)) return isHangul(prev) || isHangul(next);
|
|
993
1077
|
return false;
|
|
@@ -1003,22 +1087,99 @@ const GRAPHEME_SEGMENTER = new Intl.Segmenter("en", {
|
|
|
1003
1087
|
});
|
|
1004
1088
|
|
|
1005
1089
|
/**
|
|
1006
|
-
* The
|
|
1007
|
-
*
|
|
1008
|
-
*
|
|
1009
|
-
*
|
|
1010
|
-
*
|
|
1011
|
-
* @
|
|
1012
|
-
* @returns {number[]}
|
|
1090
|
+
* The UTF-16 index at which each code point of `cps` starts, plus one final
|
|
1091
|
+
* entry for the end of the string — the bridge between the code-point space the
|
|
1092
|
+
* carve analysis works in and the UTF-16 space `body.slice` and the segmenter
|
|
1093
|
+
* speak.
|
|
1094
|
+
* @param {Int32Array} cps
|
|
1095
|
+
* @returns {Int32Array}
|
|
1013
1096
|
*/
|
|
1014
|
-
function
|
|
1015
|
-
const
|
|
1016
|
-
let
|
|
1017
|
-
for (
|
|
1018
|
-
|
|
1019
|
-
|
|
1097
|
+
function u16Offsets(cps) {
|
|
1098
|
+
const offsets = new Int32Array(cps.length + 1);
|
|
1099
|
+
let at = 0;
|
|
1100
|
+
for (let i = 0; i < cps.length; i++) {
|
|
1101
|
+
offsets[i] = at;
|
|
1102
|
+
at += cps[i] > 0xffff ? 2 : 1;
|
|
1020
1103
|
}
|
|
1021
|
-
|
|
1104
|
+
offsets[cps.length] = at;
|
|
1105
|
+
return offsets;
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
/**
|
|
1109
|
+
* The code-point index of the UTF-16 index `u16`, by binary search over
|
|
1110
|
+
* {@link u16Offsets}. `u16` is always a code-point (indeed a cluster) boundary,
|
|
1111
|
+
* so the search always lands exactly.
|
|
1112
|
+
* @param {Int32Array} offsets @param {number} u16
|
|
1113
|
+
* @returns {number}
|
|
1114
|
+
*/
|
|
1115
|
+
function cpIndexOf(offsets, u16) {
|
|
1116
|
+
let lo = 0;
|
|
1117
|
+
let hi = offsets.length - 1;
|
|
1118
|
+
while (lo < hi) {
|
|
1119
|
+
const mid = (lo + hi) >> 1;
|
|
1120
|
+
if (offsets[mid] < u16) lo = mid + 1;
|
|
1121
|
+
else hi = mid;
|
|
1122
|
+
}
|
|
1123
|
+
return lo;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
// Above this share of the code points being carve-out CANDIDATES, resolving each
|
|
1127
|
+
// candidate's cluster with `Segments.containing` costs more than segmenting the
|
|
1128
|
+
// document once from the start (a `containing` call is a few times dearer than
|
|
1129
|
+
// one step of the sequential iterator). Both strategies read the SAME segmenter,
|
|
1130
|
+
// so they return the same boundaries; this only picks the cheaper way to ask.
|
|
1131
|
+
const CONTAINING_CANDIDATE_SHARE = 4;
|
|
1132
|
+
|
|
1133
|
+
/**
|
|
1134
|
+
* A source of the grapheme cluster containing a given code-point index, over
|
|
1135
|
+
* indices asked for in increasing order.
|
|
1136
|
+
*
|
|
1137
|
+
* Cluster boundaries matter only where a preservable invisible sits: a cluster
|
|
1138
|
+
* with none leaves every counter in {@link carveStrip} untouched, and the emit
|
|
1139
|
+
* loop treats each of its code points the same however the boundaries fall. So
|
|
1140
|
+
* the clusters are resolved on demand around the candidates rather than by
|
|
1141
|
+
* segmenting the whole document — the difference between reading one emoji at
|
|
1142
|
+
* the end of an 8 MB log and segmenting the 8 MB.
|
|
1143
|
+
* @param {string} body @param {Int32Array} offsets @param {number} candidates
|
|
1144
|
+
* @returns {(cp: number) => { start: number, end: number }}
|
|
1145
|
+
*/
|
|
1146
|
+
function clusterResolver(body, offsets, candidates) {
|
|
1147
|
+
const cpCount = offsets.length - 1;
|
|
1148
|
+
if (candidates * CONTAINING_CANDIDATE_SHARE < cpCount) {
|
|
1149
|
+
const segments = GRAPHEME_SEGMENTER.segment(body);
|
|
1150
|
+
return (cp) => {
|
|
1151
|
+
// `offsets[cp]` is inside the string for every code-point index the walk
|
|
1152
|
+
// asks about, so the segment always exists.
|
|
1153
|
+
const found = /** @type {{ index: number, segment: string }} */ (
|
|
1154
|
+
segments.containing(offsets[cp])
|
|
1155
|
+
);
|
|
1156
|
+
return {
|
|
1157
|
+
start: cpIndexOf(offsets, found.index),
|
|
1158
|
+
end: cpIndexOf(offsets, found.index + found.segment.length),
|
|
1159
|
+
};
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
// Dense candidates: walk the segmenter forward instead, keeping a code-point
|
|
1163
|
+
// cursor in step with it so no boundary needs searching for. The iterator is
|
|
1164
|
+
// abandoned wherever the last candidate leaves it, so a document whose
|
|
1165
|
+
// preserve budget is spent in its first line is never segmented past it.
|
|
1166
|
+
const iterator = GRAPHEME_SEGMENTER.segment(body)[Symbol.iterator]();
|
|
1167
|
+
let start = 0;
|
|
1168
|
+
let end = 0;
|
|
1169
|
+
return (cp) => {
|
|
1170
|
+
while (end <= cp) {
|
|
1171
|
+
// ECMA-402 guarantees the segments partition the input, so every code
|
|
1172
|
+
// point inside it has a cluster and the iterator cannot run out first; if
|
|
1173
|
+
// it ever did, reading `.segment` off `undefined` throws here rather than
|
|
1174
|
+
// silently answering with a boundary nobody computed.
|
|
1175
|
+
const step = /** @type {{ value: { segment: string } }} */ (
|
|
1176
|
+
iterator.next()
|
|
1177
|
+
);
|
|
1178
|
+
start = end;
|
|
1179
|
+
end = cpIndexOf(offsets, offsets[start] + step.value.segment.length);
|
|
1180
|
+
}
|
|
1181
|
+
return { start, end };
|
|
1182
|
+
};
|
|
1022
1183
|
}
|
|
1023
1184
|
|
|
1024
1185
|
/**
|
|
@@ -1046,14 +1207,19 @@ function clusterEnds(body) {
|
|
|
1046
1207
|
* makes the caller claim a strip that did not happen, and a stuffed channel
|
|
1047
1208
|
* surfaces as its category once it overruns the budget.
|
|
1048
1209
|
* @param {string} body
|
|
1210
|
+
* @param {Int32Array} [cps] `body`'s code points, when the caller already has them
|
|
1211
|
+
* @param {ReturnType<typeof analyzeCarve>} [analysis] likewise the analysis
|
|
1049
1212
|
* @returns {{ cleaned: string, found: string[] }}
|
|
1050
1213
|
*/
|
|
1051
|
-
function carveStrip(
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1214
|
+
function carveStrip(
|
|
1215
|
+
body,
|
|
1216
|
+
cps = codePointArray(body),
|
|
1217
|
+
analysis = analyzeCarve(cps),
|
|
1218
|
+
) {
|
|
1219
|
+
// Only PAYLOAD invisibles count toward the scatter floor, so a
|
|
1220
|
+
// meaningful-joiner-dense text (formal Persian, a long Devanagari conjunct
|
|
1221
|
+
// run) stays under it.
|
|
1222
|
+
const { codes, kind, payloadInvis, visibleLen } = analysis;
|
|
1057
1223
|
// SCATTERED_THRESHOLD is the floor on payload invisibles: past it the document
|
|
1058
1224
|
// is drowning in hidden bytes, so the carve-out is off and even a meaningful
|
|
1059
1225
|
// joiner is stripped (threshold-evasion catch — over-strip beats under).
|
|
@@ -1070,8 +1236,25 @@ function carveStrip(body) {
|
|
|
1070
1236
|
),
|
|
1071
1237
|
);
|
|
1072
1238
|
|
|
1239
|
+
const n = cps.length;
|
|
1240
|
+
// Every code point that draws on the joiner/selector budget, in order. Blank
|
|
1241
|
+
// fillers are exempt (their allowance is already spent in analyzeCarve), so a
|
|
1242
|
+
// cluster holding only those needs no boundary of its own.
|
|
1243
|
+
/** @type {number[]} */
|
|
1244
|
+
const candidates = [];
|
|
1245
|
+
if (allowCarveOut)
|
|
1246
|
+
for (let k = 0; k < n; k++)
|
|
1247
|
+
if (kind[k] !== KIND_NONE && kind[k] !== KIND_BLANK) candidates.push(k);
|
|
1248
|
+
|
|
1073
1249
|
const foundCodes = new Set();
|
|
1074
|
-
|
|
1250
|
+
/** @type {string[]} */
|
|
1251
|
+
const kept = [];
|
|
1252
|
+
const offsets = u16Offsets(cps);
|
|
1253
|
+
// Start of the kept run currently being accumulated, in UTF-16 units. Output
|
|
1254
|
+
// is assembled from slices of `body` — one per uninterrupted kept run — rather
|
|
1255
|
+
// than a character at a time, so a document with nothing to strip costs one
|
|
1256
|
+
// slice however long it is.
|
|
1257
|
+
let runStart = 0;
|
|
1075
1258
|
// Preserved JOINERS in the current uninterrupted cluster (tags/blank fillers
|
|
1076
1259
|
// and presentation selectors don't chain, so they are exempt). A genuine gap
|
|
1077
1260
|
// (two visible chars in a row — see prevVisible) resets it; past the cap the
|
|
@@ -1088,80 +1271,151 @@ function carveStrip(body) {
|
|
|
1088
1271
|
// char is stripped and its category reported.
|
|
1089
1272
|
let preservedTotal = 0;
|
|
1090
1273
|
let prevVisible = false;
|
|
1091
|
-
|
|
1092
|
-
|
|
1274
|
+
|
|
1275
|
+
/** Emit `[from, to)` under a settled `fits`, updating every counter.
|
|
1276
|
+
* @param {number} from @param {number} to @param {boolean} fits */
|
|
1277
|
+
const emit = (from, to, fits) => {
|
|
1278
|
+
for (let k = from; k < to; k++) {
|
|
1279
|
+
const code = codes[k];
|
|
1280
|
+
if (code === CODE_VISIBLE) {
|
|
1281
|
+
// A visible char following another visible char is a real word/segment
|
|
1282
|
+
// boundary, not a join — the joined cluster (if any) ended here.
|
|
1283
|
+
if (prevVisible) {
|
|
1284
|
+
joinerRun = 0;
|
|
1285
|
+
selectorRun = 0;
|
|
1286
|
+
}
|
|
1287
|
+
prevVisible = true;
|
|
1288
|
+
continue; // ordinary visible character: stays inside the kept run
|
|
1289
|
+
}
|
|
1290
|
+
// A blank filler rides on allowCarveOut alone (its own allowance is
|
|
1291
|
+
// already spent in analyzeCarve); everything else rides on `fits`.
|
|
1292
|
+
if (
|
|
1293
|
+
kind[k] === KIND_BLANK ? allowCarveOut : fits && kind[k] !== KIND_NONE
|
|
1294
|
+
) {
|
|
1295
|
+
if (kind[k] === KIND_JOINER) joinerRun++;
|
|
1296
|
+
if (kind[k] === KIND_IVS || kind[k] === KIND_STDVS) selectorRun++;
|
|
1297
|
+
if (kind[k] !== KIND_BLANK) preservedTotal++;
|
|
1298
|
+
prevVisible = false; // a joiner/selector/tag/blank keeps the cluster open
|
|
1299
|
+
continue;
|
|
1300
|
+
}
|
|
1301
|
+
foundCodes.add(CODE_CATEGORY[code]);
|
|
1302
|
+
prevVisible = false; // a stripped invisible neither opens nor closes a gap
|
|
1303
|
+
if (offsets[k] > runStart) kept.push(body.slice(runStart, offsets[k]));
|
|
1304
|
+
runStart = offsets[k + 1];
|
|
1305
|
+
}
|
|
1306
|
+
};
|
|
1307
|
+
|
|
1308
|
+
const clusterOf =
|
|
1309
|
+
candidates.length > 0
|
|
1310
|
+
? clusterResolver(body, offsets, candidates.length)
|
|
1311
|
+
: null;
|
|
1312
|
+
// The last index at which the emit loop resets the run counters — a visible
|
|
1313
|
+
// code point directly after another visible one. The condition reads only
|
|
1314
|
+
// `codes`, never a preserve decision, so the whole set of reset points is
|
|
1315
|
+
// known before the walk starts. Past this index the run counters can only
|
|
1316
|
+
// ever be incremented, which is what makes the exhaustion test below final.
|
|
1317
|
+
let lastGap = -1;
|
|
1318
|
+
// Remaining candidates by the limit each one draws on, decremented as the
|
|
1319
|
+
// walk passes them.
|
|
1320
|
+
let leftJoiner = 0;
|
|
1321
|
+
let leftSelector = 0;
|
|
1322
|
+
let leftOther = 0;
|
|
1323
|
+
if (clusterOf !== null) {
|
|
1324
|
+
for (let i = n - 1; i > 0; i--)
|
|
1325
|
+
if (codes[i] === CODE_VISIBLE && codes[i - 1] === CODE_VISIBLE) {
|
|
1326
|
+
lastGap = i;
|
|
1327
|
+
break;
|
|
1328
|
+
}
|
|
1329
|
+
for (const c of candidates) {
|
|
1330
|
+
if (kind[c] === KIND_JOINER) leftJoiner++;
|
|
1331
|
+
else if (kind[c] === KIND_IVS || kind[c] === KIND_STDVS) leftSelector++;
|
|
1332
|
+
else leftOther++;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
/** True when no cluster from here on can be preserved, so the rest of the
|
|
1337
|
+
* document is settled and needs no boundaries. Every term is monotone:
|
|
1338
|
+
* `preservedTotal` only grows, and past `lastGap` the run counters only grow
|
|
1339
|
+
* too, so a limit that is spent here is spent for good.
|
|
1340
|
+
* @param {number} at */
|
|
1341
|
+
const nothingLeftFits = (at) => {
|
|
1342
|
+
if (preservedTotal >= maxPreserved) return true;
|
|
1343
|
+
if (at <= lastGap) return false; // a reset ahead can reopen a spent run
|
|
1344
|
+
if (leftOther > 0) return false; // a tag/presentation selector draws on neither run
|
|
1345
|
+
if (leftJoiner > 0 && joinerRun < CONSECUTIVE_JOINER_CAP) return false;
|
|
1346
|
+
if (leftSelector > 0 && selectorRun < CONSECUTIVE_SELECTOR_CAP)
|
|
1347
|
+
return false;
|
|
1348
|
+
return true;
|
|
1349
|
+
};
|
|
1350
|
+
|
|
1351
|
+
let k = 0;
|
|
1352
|
+
let next = 0;
|
|
1353
|
+
while (k < n) {
|
|
1354
|
+
while (next < candidates.length && candidates[next] < k) next++;
|
|
1355
|
+
if (
|
|
1356
|
+
clusterOf === null ||
|
|
1357
|
+
next === candidates.length ||
|
|
1358
|
+
nothingLeftFits(k)
|
|
1359
|
+
) {
|
|
1360
|
+
emit(k, n, false);
|
|
1361
|
+
break;
|
|
1362
|
+
}
|
|
1363
|
+
const { start, end } = clusterOf(candidates[next]);
|
|
1364
|
+
if (start > k) emit(k, start, false);
|
|
1093
1365
|
// Charge the WHOLE cluster's preservables against every limit at once: if
|
|
1094
1366
|
// any one of them would fall due part-way through, none of the cluster is
|
|
1095
|
-
// preserved.
|
|
1096
|
-
// invisible) leaves every counter untouched.
|
|
1367
|
+
// preserved.
|
|
1097
1368
|
let need = 0;
|
|
1098
1369
|
let joiners = 0;
|
|
1099
1370
|
let selectors = 0;
|
|
1100
|
-
for (let
|
|
1101
|
-
|
|
1102
|
-
// anchor-proportional allowance, so it neither draws on this budget nor
|
|
1103
|
-
// is stripped by it (only by the scatter floor, via allowCarveOut).
|
|
1104
|
-
if (kind[k] === null || kind[k] === "blank") continue;
|
|
1371
|
+
for (let j = start; j < end; j++) {
|
|
1372
|
+
if (kind[j] === KIND_NONE || kind[j] === KIND_BLANK) continue;
|
|
1105
1373
|
need++;
|
|
1106
|
-
if (kind[
|
|
1107
|
-
if (kind[
|
|
1374
|
+
if (kind[j] === KIND_JOINER) joiners++;
|
|
1375
|
+
if (kind[j] === KIND_IVS || kind[j] === KIND_STDVS) selectors++;
|
|
1108
1376
|
}
|
|
1109
1377
|
// The run counters as they stand at the cluster's FIRST preservable char.
|
|
1110
1378
|
// A cluster can OPEN with a genuine gap — two visible code points in a row,
|
|
1111
1379
|
// e.g. the second of two adjacent emoji ZWJ sequences, or a letter and its
|
|
1112
|
-
// harakat — which the emit loop resets on
|
|
1113
|
-
//
|
|
1114
|
-
//
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1117
|
-
//
|
|
1380
|
+
// harakat — which the emit loop resets on. Judging the caps on the stale
|
|
1381
|
+
// pre-reset count would strip joiners the cap never meant to catch: a false
|
|
1382
|
+
// positive on legitimate joined text. This mirrors the emit loop's gap rule
|
|
1383
|
+
// exactly (only a visible char after another visible char closes a run; any
|
|
1384
|
+
// invisible, payload included, does not) and stops at the first preservable,
|
|
1385
|
+
// since resets past it are the emit loop's business.
|
|
1118
1386
|
let runJoiner = joinerRun;
|
|
1119
1387
|
let runSelector = selectorRun;
|
|
1120
1388
|
let seenVisible = prevVisible;
|
|
1121
|
-
for (let
|
|
1122
|
-
if (codes[
|
|
1389
|
+
for (let j = start; j < end && kind[j] === KIND_NONE; j++) {
|
|
1390
|
+
if (codes[j] === CODE_VISIBLE && seenVisible) {
|
|
1123
1391
|
runJoiner = 0;
|
|
1124
1392
|
runSelector = 0;
|
|
1125
1393
|
}
|
|
1126
|
-
seenVisible = codes[
|
|
1394
|
+
seenVisible = codes[j] === CODE_VISIBLE;
|
|
1127
1395
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
// A visible char following another visible char is a real word/segment
|
|
1137
|
-
// boundary, not a join — the joined cluster (if any) ended here.
|
|
1138
|
-
if (prevVisible) {
|
|
1139
|
-
joinerRun = 0;
|
|
1140
|
-
selectorRun = 0;
|
|
1141
|
-
}
|
|
1142
|
-
prevVisible = true;
|
|
1143
|
-
out += cps[k]; // ordinary visible character
|
|
1144
|
-
continue;
|
|
1145
|
-
}
|
|
1146
|
-
// A blank filler rides on allowCarveOut alone (its own allowance is
|
|
1147
|
-
// already spent in analyzeCarve); everything else rides on `fits`.
|
|
1148
|
-
if (kind[k] === "blank" ? allowCarveOut : fits && kind[k] !== null) {
|
|
1149
|
-
if (kind[k] === "joiner") joinerRun++;
|
|
1150
|
-
if (kind[k] === "ivs" || kind[k] === "stdvs") selectorRun++;
|
|
1151
|
-
if (kind[k] !== "blank") preservedTotal++;
|
|
1152
|
-
prevVisible = false; // a joiner/selector/tag/blank keeps the cluster open
|
|
1153
|
-
out += cps[k];
|
|
1154
|
-
continue;
|
|
1155
|
-
}
|
|
1156
|
-
foundCodes.add(code);
|
|
1157
|
-
prevVisible = false; // a stripped invisible neither opens nor closes a gap
|
|
1396
|
+
for (let j = next; j < candidates.length && candidates[j] < end; j++) {
|
|
1397
|
+
if (kind[candidates[j]] === KIND_JOINER) leftJoiner--;
|
|
1398
|
+
else if (
|
|
1399
|
+
kind[candidates[j]] === KIND_IVS ||
|
|
1400
|
+
kind[candidates[j]] === KIND_STDVS
|
|
1401
|
+
)
|
|
1402
|
+
leftSelector--;
|
|
1403
|
+
else leftOther--;
|
|
1158
1404
|
}
|
|
1159
|
-
|
|
1405
|
+
emit(
|
|
1406
|
+
start,
|
|
1407
|
+
end,
|
|
1408
|
+
preservedTotal + need <= maxPreserved &&
|
|
1409
|
+
runJoiner + joiners <= CONSECUTIVE_JOINER_CAP &&
|
|
1410
|
+
runSelector + selectors <= CONSECUTIVE_SELECTOR_CAP,
|
|
1411
|
+
);
|
|
1412
|
+
k = end;
|
|
1160
1413
|
}
|
|
1414
|
+
kept.push(body.slice(runStart));
|
|
1161
1415
|
const found = CHECKS.filter(([code]) => foundCodes.has(code)).map(
|
|
1162
1416
|
([code]) => code,
|
|
1163
1417
|
);
|
|
1164
|
-
return { cleaned:
|
|
1418
|
+
return { cleaned: kept.join(""), found };
|
|
1165
1419
|
}
|
|
1166
1420
|
|
|
1167
1421
|
// Any tag char (the whole U+E0000–U+E007F block is category Cf).
|
|
@@ -1198,12 +1452,28 @@ function needsCarveOut(body) {
|
|
|
1198
1452
|
* @returns {string}
|
|
1199
1453
|
*/
|
|
1200
1454
|
export function payloadInvisibleView(text) {
|
|
1201
|
-
const cps =
|
|
1455
|
+
const cps = codePointArray(text);
|
|
1202
1456
|
const { codes, kind } = analyzeCarve(cps);
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1457
|
+
const offsets = u16Offsets(cps);
|
|
1458
|
+
/** @param {number} i */
|
|
1459
|
+
const isPayload = (i) => codes[i] !== CODE_VISIBLE && kind[i] === KIND_NONE;
|
|
1460
|
+
// Written a RUN at a time — one `" ".repeat(n)` per masked stretch, one slice
|
|
1461
|
+
// of `text` per payload stretch — because the two stretches alternate a
|
|
1462
|
+
// handful of times in a real document and once per character in none of them.
|
|
1463
|
+
/** @type {string[]} */
|
|
1464
|
+
const parts = [];
|
|
1465
|
+
let i = 0;
|
|
1466
|
+
while (i < cps.length) {
|
|
1467
|
+
const maskFrom = i;
|
|
1468
|
+
while (i < cps.length && !isPayload(i)) i++;
|
|
1469
|
+
// One space per masked CODE POINT, so the view stays code-point-aligned with
|
|
1470
|
+
// `text` even where an astral character is two UTF-16 units wide.
|
|
1471
|
+
if (i > maskFrom) parts.push(" ".repeat(i - maskFrom));
|
|
1472
|
+
const keepFrom = i;
|
|
1473
|
+
while (i < cps.length && isPayload(i)) i++;
|
|
1474
|
+
if (i > keepFrom) parts.push(text.slice(offsets[keepFrom], offsets[i]));
|
|
1475
|
+
}
|
|
1476
|
+
return parts.join("");
|
|
1207
1477
|
}
|
|
1208
1478
|
|
|
1209
1479
|
/**
|
|
@@ -1259,13 +1529,40 @@ export function payloadLongRunSample(text) {
|
|
|
1259
1529
|
* @returns {number}
|
|
1260
1530
|
*/
|
|
1261
1531
|
export function countEffectiveInvisible(text) {
|
|
1262
|
-
|
|
1263
|
-
|
|
1532
|
+
// A text with no invisible code point has no payload and nothing to strip, so
|
|
1533
|
+
// it needs neither the code-point array nor the carve analysis. This reads
|
|
1534
|
+
// every code point — a necessary condition checked in full, not a scan bound
|
|
1535
|
+
// an attacker could paste past.
|
|
1536
|
+
if (!hasInvisibleCodePoint(text)) return 0;
|
|
1537
|
+
// A leading BOM is sliced off before the strip but counted by the analysis, so
|
|
1538
|
+
// the two would read different strings; that case pays for both separately.
|
|
1539
|
+
if (text.charCodeAt(0) === 0xfeff)
|
|
1540
|
+
return surplusOver(countPayloadInvisible(text), text, stripInvisible(text));
|
|
1541
|
+
// Otherwise both terms are readings of ONE carve analysis of ONE string, so
|
|
1542
|
+
// the analysis is done once and handed to the strip.
|
|
1543
|
+
const cps = codePointArray(text);
|
|
1544
|
+
const analysis = analyzeCarve(cps);
|
|
1545
|
+
const { cleaned } = stripBody(text, cps, analysis);
|
|
1546
|
+
return surplusOver(analysis.payloadInvis, text, cleaned, cps.length);
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
/**
|
|
1550
|
+
* The payload count plus the code points the strip removed BEYOND it — the
|
|
1551
|
+
* carve-out's over-budget surplus (see {@link countEffectiveInvisible}). A
|
|
1552
|
+
* leading BOM is preserved by the strip but counted as payload, so the
|
|
1553
|
+
* difference can go slightly negative; hence the clamp.
|
|
1554
|
+
* @param {number} payload @param {string} text @param {string} stripped
|
|
1555
|
+
* @param {number} [cpLen] `text`'s code-point length, when already known
|
|
1556
|
+
* @returns {number}
|
|
1557
|
+
*/
|
|
1558
|
+
function surplusOver(payload, text, stripped, cpLen) {
|
|
1264
1559
|
// Two identical strings differ by zero code points, so the common clean paste
|
|
1265
1560
|
// skips both counting passes; codePointLength is the String iterator's count,
|
|
1266
1561
|
// the same unit `[...text].length` measures, without the array.
|
|
1267
1562
|
const removed =
|
|
1268
|
-
stripped === text
|
|
1563
|
+
stripped === text
|
|
1564
|
+
? 0
|
|
1565
|
+
: (cpLen ?? codePointLength(text)) - codePointLength(stripped);
|
|
1269
1566
|
return payload + Math.max(0, removed - payload);
|
|
1270
1567
|
}
|
|
1271
1568
|
|
|
@@ -1320,12 +1617,25 @@ export function stripInvisibleWithReport(text, originalText = text) {
|
|
|
1320
1617
|
const hasLeadingBom =
|
|
1321
1618
|
originalText.charCodeAt(0) === 0xfeff && text.charCodeAt(0) === 0xfeff;
|
|
1322
1619
|
const body = hasLeadingBom ? text.slice(1) : text;
|
|
1323
|
-
const { cleaned, found } =
|
|
1324
|
-
? carveStrip(body)
|
|
1325
|
-
: bulkStrip(body);
|
|
1620
|
+
const { cleaned, found } = stripBody(body);
|
|
1326
1621
|
return { cleaned: hasLeadingBom ? BOM + cleaned : cleaned, found };
|
|
1327
1622
|
}
|
|
1328
1623
|
|
|
1624
|
+
/**
|
|
1625
|
+
* The strip of a BOM-resolved body: the carve-out walk when anything in it could
|
|
1626
|
+
* be preserved, the single bulk regex pass otherwise. `cps` and `analysis` let a
|
|
1627
|
+
* caller that has already analyzed `body` hand the work over instead of paying
|
|
1628
|
+
* for it twice.
|
|
1629
|
+
* @param {string} body
|
|
1630
|
+
* @param {Int32Array} [cps]
|
|
1631
|
+
* @param {ReturnType<typeof analyzeCarve>} [analysis]
|
|
1632
|
+
* @returns {{ cleaned: string, found: string[] }}
|
|
1633
|
+
*/
|
|
1634
|
+
function stripBody(body, cps, analysis) {
|
|
1635
|
+
if (!needsCarveOut(body)) return bulkStrip(body);
|
|
1636
|
+
return carveStrip(body, cps, analysis);
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1329
1639
|
/**
|
|
1330
1640
|
* Strip payload-capable invisible chars (cleaned text only). See
|
|
1331
1641
|
* stripInvisibleWithReport for the BOM and ZWNJ/ZWJ carve-out semantics.
|