agent-sanitizer 2.63.0 → 2.64.0

package/types/gates.d.mts

@@ -20,9 +20,20 @@ export function needsMarkdownPipeline(text: string): boolean;
  */
 export function needsUrlScan(text: string): boolean;
 /**
- * True when either pre-gate alternation shape-matches `text`. Split into two
- * literals (see SECRET_HINT) and OR'd so neither grows into a
- * polynomial-backtracking shape.
+ * POLICY, not a gate: true when `text` reads as HTML rather than data that
+ * merely holds a `<` (a comment open, or a tag open with a `>` after it). The
+ * output hook uses it to decide whether an MCP result earns the Layer-2 splice,
+ * so `a<b` stays verbatim; kept apart from {@link HTML_TAG_PRESENT} so widening
+ * the gate never widens the policy. Linear, unlike a `<…[^>]*>` regex.
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function isHtmlShaped(text: string): boolean;
+/**
+ * True when either pre-gate alternation shape-matches `text`, or the same text
+ * with every `\n` removed (the view the engine's `_redact_cross_line` scans, so
+ * this stays a superset of the redactor). Two literals OR'd so neither grows
+ * into a polynomial-backtracking shape (see SECRET_HINT).
  * @param {string} text
  * @returns {boolean}
  */
@@ -36,29 +47,31 @@ export function matchesSecretHint(text: string): boolean;
  * `export … from "./html.mjs"` would eagerly evaluate that ~200ms module on
  * every root import, defeating the lazy-load design. This module imports
  * nothing, so re-exporting it is free.
+ *
+ * INVARIANT: every gate is a SUPERSET of its parser (a narrower one silently
+ * turns its layer off; a needless fire only costs a parse), checked by
+ * test/gates-superset.test.mjs.
  */
 /**
- * Matches any HTML tag-like construct: opening tags, closing tags (`</`),
- * comments and bogus declarations (`<!`), and processing instructions / bogus
- * comments (`<?…?>`, which the HTML tokenizer hides exactly like a comment).
- * The `<!`/`<?` arms carry a comment-only document into the pipeline at all:
- * without them it would skip both Layer 2's splice of the comment and Layer 3's
- * exfil scan over the comment interior.
- * Gate for Layer 2 (HTML sanitization) and the HTML img/a exfil path in
- * Layer 3.
+ * Matches every point where the HTML tokenizer leaves the data state: `<` then
+ * an ASCII letter, `/`, `!` or `?`. No closing `>` on purpose: a comment holding
+ * `1 < 2`, or left unterminated, still hides everything after it. Gate for
+ * Layer 2 and the HTML img/a exfil path in Layer 3.
  */
 export const HTML_TAG_PRESENT: RegExp;
 /**
  * Matches markdown link/image syntax (`](`, `![`) and reference link
- * definitions (`[label]: url` at line start). Gate for Layer 3 (markdown
- * exfiltration detection).
+ * definitions (`[label]: url` at a line start, also nested in a blockquote or
+ * list item). The label admits backslash escapes and line endings, as
+ * CommonMark labels do, but no unescaped `[`, which also keeps the scan linear.
+ * Gate for Layer 3 (markdown exfiltration detection).
  */
 export const MD_LINK_HINT: RegExp;
 /**
- * Matches an absolute http(s) URL anywhere in the text. Gate for the Layer-3
- * URL detectors, which read the URLs a GFM autolink literal yields and so find
- * a look-alike host or an exfil-shaped query in bare prose — no tag and no
- * link syntax required.
+ * Matches the start of every URL a GFM autolink literal yields (`http://`,
+ * `https://`, scheme-less `www.`). Gate for the Layer-3 URL detectors, which
+ * find a look-alike host or exfil-shaped query in bare prose. (A bare e-mail
+ * becomes a host-less `mailto:` URL that neither detector can flag.)
  */
 export const URL_PRESENT: RegExp;
 /** @type {RegExp} */

package/types/html.d.mts

@@ -130,7 +130,7 @@ export function urlHost(url: string): string;
  * how loudly (see the exfil tier in ./output.mjs).
  * @param {string} text
  * @param {{ flagDigestValues?: boolean }} [options] see {@link checkExfilUrl}
- * @returns {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string }> | null}
+ * @returns {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string, unscannable?: true }> | null}
  */
 export function detectExfil(text: string, options?: {
     flagDigestValues?: boolean;
@@ -139,6 +139,7 @@ export function detectExfil(text: string, options?: {
     autoFetched: boolean;
     reason: string;
     target: string;
+    unscannable?: true;
 }> | null;
 /**
  * Layer 3, second detector: report URLs whose HOST is a confusable of an ASCII
@@ -149,11 +150,12 @@ export function detectExfil(text: string, options?: {
  *
  * Fails CLOSED on a parse blow-up for the same reason detectExfil does.
  * @param {string} text
- * @returns {Array<{ severity: string, description: string }> | null}
+ * @returns {Array<{ severity: string, description: string, unscannable?: true }> | null}
  */
 export function detectConfusableHosts(text: string): Array<{
     severity: string;
     description: string;
+    unscannable?: true;
 }> | null;
 export const REPORTED_TAGS: Set<string>;
 export const HIDDEN_PLACEHOLDER: "[hidden HTML removed";

package/types/index.d.mts

@@ -60,4 +60,4 @@ export function sanitize(text: string, options?: {
 }>;
 export { applyLayer1, applyLayer1WellFormed, escapesPreservable, isBenignAnsi, isBenignAnsiKinds, normalizeLoneSurrogates, stripAnsiFully, stripTerminalFully, sgrCarriesPayload, SGR_RUN_THRESHOLD, LONE_SURROGATE_RE } from "./layer1.mjs";
 export { stripInvisible, stripInvisibleWithReport, isSgrOnly, STRIP, SGR_RE, CHECKS, CATEGORY, CATEGORY_LABELS, LINGUISTIC_SCRIPTS, VS, BLANK_NON_CF, LONG_RUN_RE, LONG_RUN_THRESHOLD, SCATTERED_THRESHOLD, findLongRuns, hasLongRun } from "./invisible.mjs";
-export { HTML_TAG_PRESENT, MD_LINK_HINT, SECRET_HINT, SECRET_HINT_EXT, matchesSecretHint } from "./gates.mjs";
+export { HTML_TAG_PRESENT, isHtmlShaped, MD_LINK_HINT, SECRET_HINT, SECRET_HINT_EXT, matchesSecretHint } from "./gates.mjs";

package/types/invisible.d.mts

@@ -212,6 +212,7 @@ export const CATEGORY: Readonly<{
     HIDDEN_HTML: "hidden-html";
     EXFIL_URLS: "exfil-urls";
     CONFUSABLE_HOST: "confusable-host";
+    UNSCANNABLE: "unscannable";
 }>;
 /** @type {Readonly<Record<string, string>>} */
 export const CATEGORY_LABELS: Readonly<Record<string, string>>;

package/types/joining-type.d.mts

@@ -24,11 +24,12 @@ export function isVirama(cp: number): boolean;
  */
 export function isBrahmicConsonant(cp: number): boolean;
 /**
- * GENERATED by scripts/gen-joining-type.mjs from the pinned UCD 17.0.0 — DO NOT EDIT.
+ * GENERATED by scripts/gen-joining-type.mjs from the pinned UCD 17.0.0 at install
+ * time; git-ignored, never committed — DO NOT EDIT.
  *
  * Unicode Joining_Type, Indic virama and Brahmic consonant range tables backing
- * the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`;
- * test/joining-type.test.mjs fails if this drifts from the pinned UCD.
+ * the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`
+ * (`pnpm install` does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {ReadonlyArray<readonly [string, number, number]>} */

package/types/output.d.mts

@@ -71,7 +71,7 @@ export function deleteVerbatimSpans(text: string, spans: string[]): {
  * and one that cannot be vetted is WITHHELD (dropped from the array) under the
  * same doctrine — Layer 4 never saw pre-splice text.
  *
- * Every byte mutation goes through {@link applyMutation} and every Layer-4 call
+ * Every byte mutation goes through {@link mutate} and every Layer-4 call
  * through {@link runRedact}, so a layer cannot re-establish some of the
  * post-mutation invariants and forget the rest, and every string in the returned
  * object has traversed Layer 4.
@@ -118,59 +118,68 @@ export function sanitizeText(text: string, options?: SanitizeTextOptions): Promi
  */
 export function isWalkableContainer(value: any): boolean;
 /**
- * Sanitize every string leaf of a tool-output value, preserving its shape (a
- * structured tool output whose shape changes would be ignored by a harness,
- * leaking the raw value). Non-string leaves pass through; `warnings` and
- * `notes` accumulate across leaves, split by severity (see ./severity.mjs).
- * `sgrNote` is the OR across leaves — true when SOME leaf was note-only — so a
- * caller can still pick the quiet banner when no leaf raised a warning.
- *
- * Fails CLOSED on two hostile shapes that would otherwise throw a `RangeError`
- * as an unhandled async rejection (a DoS that leaves the output un-sanitized):
- * nesting past {@link MAX_DEPTH}, and a reference cycle. Either replaces the
- * offending subtree with a placeholder string + a warning, never passing the
- * raw subtree through. Keys are also screened for hidden chars (see below).
- *
- * `reveals` accumulates each string leaf's pre-Layer-2 text (present only when
- * the HTML splice removed bytes) so a caller can persist what was hidden — the
- * structured-output analogue of {@link sanitizeText}'s `reveal`. Same
- * mutated-accumulator contract as `warnings`.
- *
- * `splices` accumulates each string leaf's Layer-2 placeholder→original pairs
- * (each `original` already vetted, withheld entries dropped — see
- * {@link sanitizeText}) — the per-placeholder twin of `reveals`, so a hook
- * caller gets them for object-shaped tool output too. Same mutated-accumulator
- * contract as `reveals`.
+ * Sanitize every string of a tool-output value with `options` — each string
+ * leaf AND each object key — preserving its shape. The walk itself, and every
+ * rule it applies, is {@link sanitizeValueWith}'s; this binds the leaf and key
+ * sanitizers to {@link sanitizeText}. A key runs every layer a leaf does except
+ * Layer 5: a span deletion in a NAME can collapse two fields into one, which
+ * costs both their values (see the collision rule).
  * @param {any} value
  * @param {SanitizeTextOptions} options
  * @param {string[]} warnings
  * @param {string[]} [reveals]
- * @param {string[]} [notes]  the NOTE-severity counterpart of `warnings`;
- *   appended last so an existing positional caller keeps working (it simply
- *   discards the notes, which is exactly as loud as before the split)
- * @param {Array<{ placeholder: string, original: string }>} [splices]  appended
- *   after `notes` for the same positional-compatibility reason
- * @returns {Promise<{ value: any, modified: boolean, sgrNote: boolean }>}
+ * @param {string[]} [notes]  the NOTE-severity counterpart of `warnings`
+ * @param {Array<{ placeholder: string, original: string }>} [splices]
+ * @returns {Promise<WalkResult>}
  */
 export function sanitizeValue(value: any, options: SanitizeTextOptions, warnings: string[], reveals?: string[], notes?: string[], splices?: Array<{
     placeholder: string;
     original: string;
-}>): Promise<{
-    value: any;
-    modified: boolean;
-    sgrNote: boolean;
-}>;
+}>): Promise<WalkResult>;
+/**
+ * THE structured-output walk, shared by {@link sanitizeValue} and the output
+ * hook so the two cannot disagree about a value's strings. Runs `leaf` on every
+ * string leaf and `key` on every object key (a field name is as
+ * attacker-controlled as a leaf) and rebuilds the same shape (a harness ignores
+ * a reshaped output and shows the RAW one). Each string's findings, `reveal` and
+ * `splices` fold into `sinks`, whose caller owns dedup.
+ *
+ * Keys that sanitize to the same name fail closed: every value under that name
+ * becomes {@link COLLISION_WITHHELD_MESSAGE}, the later ones under a
+ * disambiguated key so the field count holds. Nesting past {@link MAX_DEPTH}
+ * and a reference cycle become a placeholder plus a warning, not a `RangeError`.
+ * `sgrNote` holds only when the walk added no warning.
+ * @param {any} value
+ * @param {StringSanitizers} sanitizers
+ * @param {WalkSinks} sinks
+ * @returns {Promise<WalkResult>}
+ */
+export function sanitizeValueWith(value: any, sanitizers: StringSanitizers, sinks: WalkSinks): Promise<WalkResult>;
+/**
+ * The warning for a post-sanitization key collision, naming where it sits.
+ * Exported so tests assert by reference rather than re-typing the prose.
+ * @param {string} name  the collapsed-to name
+ * @param {string} path  dotted location of the owning object ("" = top level)
+ * @returns {string}
+ */
+export function collisionWarning(name: string, path: string): string;
 /**
  * Compose the model-facing context line for a sanitized/flagged tool output.
  * `injectionAlert` is the caller's optional trailing alert (e.g. appended only
  * for untrusted-ingress tools where a semantic-injection filter actually ran).
+ *
+ * `notes` follow the WARNING text: a note can carry the instruction a
+ * detect-only finding rests on ("do not fetch them"), which an unrelated
+ * warning does not make redundant. {@link INERT_ANSI_NOTE} alone is dropped
+ * beside a warning (matched by the constant, not its wording).
  * @param {boolean} modified  output bytes were changed (vs. flagged only)
  * @param {string[]} warnings
- * @param {{ injectionAlert?: string }} [options]
+ * @param {{ injectionAlert?: string, notes?: string[] }} [options]
  * @returns {string}
  */
-export function composeContext(modified: boolean, warnings: string[], { injectionAlert }?: {
+export function composeContext(modified: boolean, warnings: string[], { injectionAlert, notes }?: {
     injectionAlert?: string;
+    notes?: string[];
 }): string;
 /**
  * Replace every string leaf of `value` with `message`, preserving shape so a
@@ -233,6 +242,8 @@ export { needsMarkdownPipeline };
  * caller still emits a sanitized, flagged result instead of crashing.
  */
 export const MAX_DEPTH: 200;
+/** A colliding field's WHOLE value, so no scalar passes under a legitimate name (the JSON type change is accepted). */
+export const COLLISION_WITHHELD_MESSAGE: "[WITHHELD \u2014 this field's name collided with another after sanitization]";
 /**
  * Layer-4 result: the redacted text, the category labels redacted, and an
  * optional caller-supplied annotation appended to the warning.
@@ -258,7 +269,7 @@ export type Layer5Result = {
 };
 /**
  * The running state of one {@link sanitizeText} call. Layers read `text` and
- * mutate it ONLY through {@link applyMutation}. Findings carry their own
+ * mutate it ONLY through {@link mutate}. Findings carry their own
  * severity (see ./severity.mjs) and are split into `warnings`/`notes` at the
  * single exit, so no layer can push into the wrong list. `found` is the
  * machine-readable twin, unaffected by the split: the {@link CATEGORY} codes
@@ -269,7 +280,6 @@ export type PipelineState = {
     found: string[];
     findings: import("./severity.mjs").Finding[];
     modified: boolean;
-    unreportedChange: boolean;
 };
 export type SanitizeTextOptions = {
     html?: boolean;
@@ -290,6 +300,47 @@ export type SanitizeTextOptions = {
 export type Deadline = {
     remainingMs: () => number;
 };
+export type WalkResult = {
+    value: any;
+    modified: boolean;
+    sgrNote: boolean;
+};
+export type StringVerdict = {
+    cleaned: string;
+    warnings: string[];
+    notes: string[];
+    modified: boolean;
+    sgrNote: boolean;
+    reveal?: string;
+    splices?: Array<{
+        placeholder: string;
+        original: string;
+    }>;
+};
+export type StringSanitizers = {
+    leaf: (text: string) => Promise<StringVerdict>;
+    key: (text: string) => Promise<StringVerdict>;
+};
+export type WalkSinks = {
+    warnings: string[];
+    notes: string[];
+    reveals: string[];
+    splices: Array<{
+        placeholder: string;
+        original: string;
+    }>;
+};
+/**
+ * `seen` holds the ancestors on the active path only (a value shared across
+ * sibling branches is not a cycle). `memo` caches each FULLY-PROCESSED result
+ * by `(node, depth)` (see {@link depthMemo}); without it a shared-substructure
+ * DAG is re-sanitized once per PATH — a ~25-object diamond measured 68 s.
+ * Skipping a cached node's duplicate findings is harmless: callers dedup.
+ */
+export type WalkContext = WalkSinks & StringSanitizers & {
+    seen: WeakSet<object>;
+    memo: ReturnType<typeof depthMemo<WalkResult>>;
+};
 /**
  * a content-block field's value test
  */
@@ -299,4 +350,45 @@ export type BlockSchema = {
     optional: Record<string, FieldShape>;
 };
 import { needsMarkdownPipeline } from "./gates.mjs";
+/**
+ * Cache of a walked subtree keyed by `(node, depth)` — the pair the walk's
+ * result actually depends on. Both walkers below truncate past
+ * {@link MAX_DEPTH}, so the SAME node yields different output at different
+ * depths: withheld on a long path, walked on a short one. Keying by node
+ * identity alone therefore caches a path-dependent answer, and a shared node
+ * first reached deep withholds real content everywhere it is reached later —
+ * the module's own "a node withheld for depth on a long path must still be
+ * walked on a shorter one" rule, silently violated.
+ *
+ * Taking `depth` as a mandatory argument is the point: there is no API here that
+ * can key by identity alone. Refusing to cache truncated subtrees instead would
+ * NOT work — truncation propagates to every ancestor, so a hostile diamond DAG
+ * with a deep tail would go uncached and re-walk once per path, re-opening the
+ * exponential blow-up the memo exists to prevent. Work stays bounded at
+ * O(nodes × distinct depths), i.e. at most {@link MAX_DEPTH} entries per node.
+ *
+ * Residual, unchanged from before: the CYCLE placeholder also depends on the
+ * ancestor `seen` set, which is not part of the key. Two paths reaching a node
+ * at the same depth with different ancestors can therefore share a cached
+ * result. Encoding `seen` in the key means storing every node a subtree walked
+ * and re-checking it on lookup — the memo's cost becomes the walk it replaces —
+ * and refusing to cache cyclic subtrees hits the same exponential wall as
+ * above. A cycle is already a fail-closed pathological shape, so this trades a
+ * placeholder's exact placement for a hard work bound.
+ * @template T
+ */
+declare function depthMemo<T>(): {
+    /**
+     * @param {object} value
+     * @param {number} depth
+     * @returns {T | undefined}
+     */
+    get(value: object, depth: number): T | undefined;
+    /**
+     * @param {object} value
+     * @param {number} depth
+     * @param {T} result
+     */
+    set(value: object, depth: number, result: T): void;
+};
 export { describeExfil, describeRemoved, describeWarned } from "./warnings.mjs";

package/types/severity.d.mts

@@ -1,7 +1,3 @@
-/**
- * @typedef {{ severity: string, message: string }} Finding
- *   A single reportable outcome and how loudly to report it.
- */
 /**
  * A WARNING-severity finding: injection-shaped, worth the banner.
  * @param {string} message

package/types/src/cf-charset.d.mts

@@ -1,13 +1,14 @@
 /**
- * GENERATED by scripts/gen-invisible-charset.mjs from the pinned UCD
- * (General_Category=Cf, Unicode 17.0.0) — DO NOT EDIT.
+ * GENERATED by scripts/gen-cf-charset.mjs from the pinned UCD
+ * (General_Category=Cf, Unicode 17.0.0) at install, pack and build
+ * time; git-ignored, never committed — DO NOT EDIT.
  *
  * The general-category Cf code points, PINNED at generation time. invisible.mjs
  * strips exactly this set instead of asking the runtime for Cf, and the Python port
  * reads the SAME set from data/invisible-charset.json's `cf_codepoints`, so both
  * layers strip an identical Cf set regardless of each runtime's own Unicode
- * version. Regenerate with `node scripts/gen-invisible-charset.mjs`;
- * test/invisible-charset.test.mjs fails if this drifts from the pinned UCD.
+ * version. Regenerate with `node scripts/gen-cf-charset.mjs` (`pnpm install`
+ * does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {readonly number[]} Sorted ascending. */

package/types/src/invisible.d.mts

@@ -212,6 +212,7 @@ export const CATEGORY: Readonly<{
     HIDDEN_HTML: "hidden-html";
     EXFIL_URLS: "exfil-urls";
     CONFUSABLE_HOST: "confusable-host";
+    UNSCANNABLE: "unscannable";
 }>;
 /** @type {Readonly<Record<string, string>>} */
 export const CATEGORY_LABELS: Readonly<Record<string, string>>;

package/types/src/joining-type.d.mts

@@ -24,11 +24,12 @@ export function isVirama(cp: number): boolean;
  */
 export function isBrahmicConsonant(cp: number): boolean;
 /**
- * GENERATED by scripts/gen-joining-type.mjs from the pinned UCD 17.0.0 — DO NOT EDIT.
+ * GENERATED by scripts/gen-joining-type.mjs from the pinned UCD 17.0.0 at install
+ * time; git-ignored, never committed — DO NOT EDIT.
  *
  * Unicode Joining_Type, Indic virama and Brahmic consonant range tables backing
- * the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`;
- * test/joining-type.test.mjs fails if this drifts from the pinned UCD.
+ * the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`
+ * (`pnpm install` does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {ReadonlyArray<readonly [string, number, number]>} */

package/types/src/standardized-variants.d.mts

@@ -8,14 +8,14 @@
 export function isStandardizedVariant(base: number, selector: number): boolean;
 /**
  * GENERATED by scripts/gen-standardized-variants.mjs from UCD
- * StandardizedVariants.txt (Unicode 17.0.0) — DO NOT EDIT.
+ * StandardizedVariants.txt (Unicode 17.0.0) at install time; git-ignored,
+ * never committed — DO NOT EDIT.
  *
  * Registered standardized variation sequences: a base code point followed by the
  * variation selector the UCD registers for it. invisible.mjs preserves such a
  * selector only when the immediately preceding base appears here; every other
  * one is stripped as a hidden-payload byte. Regenerate with
- * `node scripts/gen-standardized-variants.mjs`;
- * test/invisible-charset.test.mjs fails if this drifts from the pinned UCD.
+ * `node scripts/gen-standardized-variants.mjs` (`pnpm install` does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {Array<[number, number]>} [base, selector], sorted by base then selector. */

package/types/src/unicode-properties.d.mts

@@ -7,12 +7,13 @@
  */
 export function inSpans(spans: ReadonlyArray<readonly [number, number]>, cp: number): boolean;
 /**
- * GENERATED by scripts/gen-unicode-properties.mjs from the pinned UCD 17.0.0 — DO NOT EDIT.
+ * GENERATED by scripts/gen-unicode-properties.mjs from the pinned UCD 17.0.0 at
+ * install time; git-ignored, never committed — DO NOT EDIT.
  *
  * The Unicode properties src/invisible.mjs's carve-out reads, PINNED so the
  * carve-out answers the same on every Node whatever its ICU's Unicode version.
- * Regenerate with `node scripts/gen-unicode-properties.mjs`;
- * test/unicode-properties.test.mjs fails if this drifts from the pinned package.
+ * Regenerate with `node scripts/gen-unicode-properties.mjs` (`pnpm install`
+ * does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {ReadonlyArray<readonly [number, number]>} */
@@ -27,3 +28,5 @@ export const UNIFIED_IDEOGRAPH: ReadonlyArray<readonly [number, number]>;
 export const SCRIPT_BRAILLE: ReadonlyArray<readonly [number, number]>;
 /** @type {ReadonlyArray<readonly [number, number]>} */
 export const SCRIPT_HANGUL: ReadonlyArray<readonly [number, number]>;
+/** @type {ReadonlyArray<readonly [number, number]>} */
+export const GENERAL_CATEGORY_MARK: ReadonlyArray<readonly [number, number]>;

package/types/standardized-variants.d.mts

@@ -8,14 +8,14 @@
 export function isStandardizedVariant(base: number, selector: number): boolean;
 /**
  * GENERATED by scripts/gen-standardized-variants.mjs from UCD
- * StandardizedVariants.txt (Unicode 17.0.0) — DO NOT EDIT.
+ * StandardizedVariants.txt (Unicode 17.0.0) at install time; git-ignored,
+ * never committed — DO NOT EDIT.
  *
  * Registered standardized variation sequences: a base code point followed by the
  * variation selector the UCD registers for it. invisible.mjs preserves such a
  * selector only when the immediately preceding base appears here; every other
  * one is stripped as a hidden-payload byte. Regenerate with
- * `node scripts/gen-standardized-variants.mjs`;
- * test/invisible-charset.test.mjs fails if this drifts from the pinned UCD.
+ * `node scripts/gen-standardized-variants.mjs` (`pnpm install` does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {Array<[number, number]>} [base, selector], sorted by base then selector. */

package/types/unicode-properties.d.mts

@@ -7,12 +7,13 @@
  */
 export function inSpans(spans: ReadonlyArray<readonly [number, number]>, cp: number): boolean;
 /**
- * GENERATED by scripts/gen-unicode-properties.mjs from the pinned UCD 17.0.0 — DO NOT EDIT.
+ * GENERATED by scripts/gen-unicode-properties.mjs from the pinned UCD 17.0.0 at
+ * install time; git-ignored, never committed — DO NOT EDIT.
  *
  * The Unicode properties src/invisible.mjs's carve-out reads, PINNED so the
  * carve-out answers the same on every Node whatever its ICU's Unicode version.
- * Regenerate with `node scripts/gen-unicode-properties.mjs`;
- * test/unicode-properties.test.mjs fails if this drifts from the pinned package.
+ * Regenerate with `node scripts/gen-unicode-properties.mjs` (`pnpm install`
+ * does).
  */
 export const UNICODE_VERSION: "17.0.0";
 /** @type {ReadonlyArray<readonly [number, number]>} */
@@ -27,3 +28,5 @@ export const UNIFIED_IDEOGRAPH: ReadonlyArray<readonly [number, number]>;
 export const SCRIPT_BRAILLE: ReadonlyArray<readonly [number, number]>;
 /** @type {ReadonlyArray<readonly [number, number]>} */
 export const SCRIPT_HANGUL: ReadonlyArray<readonly [number, number]>;
+/** @type {ReadonlyArray<readonly [number, number]>} */
+export const GENERAL_CATEGORY_MARK: ReadonlyArray<readonly [number, number]>;

package/types/view-map.d.mts

@@ -68,6 +68,8 @@ export function overlapAwareCount(haystack: string, needle: string): number;
  * The character runs Layer 1 deleted, located by greedy subsequence alignment
  * (stripping only deletes, so `cleaned` is always a subsequence of `content`).
  * Throws if the subsequence property does not hold — the caller fails closed.
+ * Aligned per CODE POINT (Layer 1 deletes whole ones): a code-unit walk matches
+ * U+1D173's high surrogate against U+1D11E's and splits both. Offsets stay UTF-16.
  * @param {string} content disk bytes
  * @param {string} cleaned Layer-1 view of the same bytes
  * @returns {{start: number, deleted: string}[]}

package/types/warnings.d.mts

@@ -74,3 +74,8 @@ export const LONE_SURROGATE_WARNING: "Normalized lone UTF-16 surrogates";
  * be false.
  */
 export const HTML_UNPARSEABLE_WARNING: "HTML unparseable \u2014 the entire output was withheld behind a placeholder; nothing was spliced";
+/**
+ * The Layer-3 warning when the URL scan overflowed the stack. It names no URL
+ * (none was found), yet stays a warning: unread is not vetted.
+ */
+export const UNSCANNABLE_WARNING: "Input too deeply nested to scan for exfil URLs or look-alike hosts \u2014 any URL in it is unvetted; do not fetch, relay, or embed it";