agent-sanitizer 2.64.1 → 2.64.2

package/LICENSE-THIRD-PARTY

@@ -1,8 +1,9 @@
 Third-party code redistributed in this package.
 
-src/vendor/gfm-autolink-literal.mjs
-  Copied from micromark-extension-gfm-autolink-literal@2.1.0, with one
-  function changed (see that file's header). Original licence follows.
+src/vendor/gfm-autolink-literal.mjs (generated, not tracked)
+  Generated at install and pack time from micromark-extension-gfm-autolink-literal@2.1.0
+  with one function changed by scripts/patches/ (see scripts/gen-gfm-autolink.mjs).
+  Original licence follows.
 
 (The MIT License)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-sanitizer",
-  "version": "2.64.1",
+  "version": "2.64.2",
   "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
   "type": "module",
   "repository": {
@@ -58,12 +58,14 @@
     "agent-sanitizer": "link:.",
     "bun": "1.3.11",
     "c8": "11.0.0",
+    "diff": "9.0.0",
     "esbuild": "0.28.1",
     "eslint": "10.4.0",
     "eslint-plugin-redos": "4.5.0",
     "fast-check": "4.8.0",
     "globals": "17.6.0",
     "lint-staged": "^17.0.5",
+    "micromark-extension-gfm-autolink-literal": "2.1.0",
     "prettier": "^3.0.0",
     "rehype-parse": "9.0.1",
     "remark-gfm": "4.0.1",
@@ -249,6 +251,7 @@
     "build:types": "tsc -p tsconfig.build.json && tsc -p tsconfig.build-hooks.json",
     "gen:joining-type": "node scripts/gen-joining-type.mjs",
     "gen:unicode": "node scripts/gen-unicode.mjs",
+    "gen:gfm-autolink": "node scripts/gen-gfm-autolink.mjs",
     "gen:unicode-properties": "node scripts/gen-unicode-properties.mjs",
     "gen:fail-open-lib": "node scripts/gen-fail-open-lib.mjs",
     "gen:node-floor-lib": "node scripts/gen-node-floor-lib.mjs",

package/src/gfm.mjs

@@ -4,8 +4,9 @@
  * This is `remark-gfm@4.0.1`'s own plugin body and
  * `micromark-extension-gfm@3.0.0`'s own `gfm()` composition, inlined so that
  * `gfmAutolinkLiteral` can come from `./vendor/gfm-autolink-literal.mjs`
- * instead of the published package — see that file for what differs and when
- * to delete both. Every other extension, and the mdast layer, are the upstream
+ * instead of the published package. That module is generated, not committed:
+ * `scripts/gen-gfm-autolink.mjs` patches the pinned upstream at install time,
+ * and its header says what differs and when to delete both. Every other extension, and the mdast layer, are the upstream
  * ones at the versions `remark-gfm` pins.
  *
  * Assembling the list here rather than adding an extension beside `remark-gfm`

package/src/html-tree-adapter.mjs

@@ -168,6 +168,30 @@ export const CONVERT_DEPTH = 128;
  */
 export const MAX_HTML_DEPTH = 4096;
 
+/**
+ * Which elements `hast-util-from-parse5` converts as a template: hastscript reads
+ * the tag name as a selector (`template.x` is a `template` with class `x`), so
+ * the name up to its first `#` or `.`, lowercased.
+ *
+ * WHY IT MATTERS — the converter then recurses into the parse5 node's `content`,
+ * which only an HTML-namespace `<template>` has. A `template` inside SVG or
+ * MathML, or an HTML `template.x` tag, has none, the converter throws,
+ * and the whole document would be withheld over a benign element.
+ */
+const CONVERTS_AS_TEMPLATE = /^template(?:[#.]|$)/i;
+
+/**
+ * Prefixed to such an element's name for the conversion, and stripped after.
+ *
+ * A space, because no parsed element's name can hold one (the tokenizer ends a
+ * tag name at whitespace), so a masked name collides with nothing, and the
+ * converter neither drops it (its selector parse splits on `#` and `.` only) nor
+ * changes it (lowercasing, the SVG case map). The element then converts the way
+ * any element without template content does: its children as `children`, where
+ * every scan of the hast tree reads them, which is also where the DOM has them.
+ */
+const TEMPLATE_MASK = " ";
+
 /**
  * `fromParse5(tree, { file })`, recursing at most `chunkDepth` levels at a time.
  *
@@ -177,7 +201,10 @@ export const MAX_HTML_DEPTH = 4096;
  * child its parent's conversion reads (an unclosed element ends where its last
  * child does). One walk then swaps each placeholder for its subtree. The result
  * is the tree a single `fromParse5` call returns, which
- * `html-tree-adapter.test` holds it to.
+ * `html-tree-adapter.test` holds it to — except that an element the converter
+ * would take for a template with no content to take (see
+ * {@link CONVERTS_AS_TEMPLATE}) converts as an ordinary element instead of
+ * throwing.
  *
  * Mutates `tree`: {@link parseHtmlFragment} owns the parse and drops it after.
  * @param {any} tree
@@ -189,6 +216,7 @@ export function fromParse5Bounded(tree, file, chunkDepth = CONVERT_DEPTH) {
   /** @type {Array<{ siblings: any[], index: number }>} */
   const cuts = [];
   const stack = [{ node: tree, depth: 0 }];
+  let masked = false;
   while (stack.length > 0) {
     const { node, depth } = /** @type {{ node: any, depth: number }} */ (
       stack.pop()
@@ -197,6 +225,13 @@ export function fromParse5Bounded(tree, file, chunkDepth = CONVERT_DEPTH) {
       throw new RangeError(`HTML nested deeper than ${MAX_HTML_DEPTH} levels`);
     // A template's content is a root its template's conversion recurses into.
     if (node.content) stack.push({ node: node.content, depth: depth + 1 });
+    else if (
+      node.tagName !== undefined &&
+      CONVERTS_AS_TEMPLATE.test(node.tagName)
+    ) {
+      node.tagName = TEMPLATE_MASK + node.tagName;
+      masked = true;
+    }
     const siblings = node.childNodes ?? [];
     const childDepth = depth + 1;
     for (let index = 0; index < siblings.length; index++) {
@@ -222,8 +257,10 @@ export function fromParse5Bounded(tree, file, chunkDepth = CONVERT_DEPTH) {
   const root = fromParse5(tree, { file });
   /** @type {any[]} */
   const pending = [root];
-  while (grafts.size > 0 && pending.length > 0) {
+  while ((grafts.size > 0 || masked) && pending.length > 0) {
     const node = pending.pop();
+    if (masked && node.tagName?.startsWith(TEMPLATE_MASK))
+      node.tagName = node.tagName.slice(TEMPLATE_MASK.length);
     if (node.content) pending.push(node.content);
     const children = node.children ?? [];
     for (let i = 0; i < children.length; i++) {

package/src/vendor/gfm-autolink-literal.mjs

@@ -1,81 +1,52 @@
 /**
- * GFM autolink literals for micromark, vendored from
- * `micromark-extension-gfm-autolink-literal@2.1.0` (MIT, (c) 2020 Titus
- * Wormer). The licence is in `LICENSE-THIRD-PARTY` at the repository root.
+ * GENERATED by scripts/gen-gfm-autolink.mjs at install, pack and build time;
+ * git-ignored, never committed — DO NOT EDIT.
  *
- * WHY A COPY. `previousUnbalanced` below decides whether an autolink literal
- * may start at this position, and upstream answers it by walking back over
- * every event the document has produced so far. It memoizes only the "no"
- * answer, so one unclosed `[` — a JSON array is the everyday shape, since its
- * `]` is the last byte — makes the answer "yes" for the rest of the document
- * and costs one full walk per word character. 172 KB of prose behind a single
- * `[` took 20.2s to parse; the same text takes 0.17s here. Layer 3 reads its
- * URLs off this parse, so the cost lands on every `detectExfil` call.
- *
- * WHAT CHANGED. One function. `previousUnbalanced` now also records where the
- * walk stopped, and a later walk resumes from that index rather than re-reading
- * the run before it. The record is used only while `events[index]` still holds
- * the same token, so a resolver that splices events invalidates it and the walk
- * falls back to reading everything. Nothing else in this file differs from
- * upstream — diff it against the published package to confirm.
- *
- * WHY NOT A PATCH. A `pnpm` patch fixes this repository's own installs and the
- * plugin bundle built from them, and reaches no consumer of the published npm
- * package (see the npm-consumer gap in the project history). A copy under
- * `src/` ships.
- *
- * WHEN TO DELETE IT. When a published release of the upstream package carries
- * an equivalent fix: drop this file and the assembly in `./gfm.mjs`, and go
- * back to `remark-gfm`. `test/gfm-autolink-parity.test.mjs` pins the behaviour
- * either way.
+ * `micromark-extension-gfm-autolink-literal@2.1.0`'s `lib/syntax.js` (MIT, (c) 2020
+ * Titus Wormer; licence in LICENSE-THIRD-PARTY) with the one-function
+ * `previousUnbalanced` performance fix from
+ * `scripts/patches/micromark-extension-gfm-autolink-literal@2.1.0.patch`. The
+ * generator's header says why it is patched and when to delete it.
  */
-
 /**
  * @import {Code, ConstructRecord, Event, Extension, Previous, State, TokenizeContext, Tokenizer} from 'micromark-util-types'
  */
 
-import {
-  asciiAlpha,
-  asciiAlphanumeric,
-  asciiControl,
-  markdownLineEndingOrSpace,
-  unicodePunctuation,
-  unicodeWhitespace,
-} from "micromark-util-character";
+import { asciiAlpha, asciiAlphanumeric, asciiControl, markdownLineEndingOrSpace, unicodePunctuation, unicodeWhitespace } from 'micromark-util-character';
 const wwwPrefix = {
   tokenize: tokenizeWwwPrefix,
-  partial: true,
+  partial: true
 };
 const domain = {
   tokenize: tokenizeDomain,
-  partial: true,
+  partial: true
 };
 const path = {
   tokenize: tokenizePath,
-  partial: true,
+  partial: true
 };
 const trail = {
   tokenize: tokenizeTrail,
-  partial: true,
+  partial: true
 };
 const emailDomainDotTrail = {
   tokenize: tokenizeEmailDomainDotTrail,
-  partial: true,
+  partial: true
 };
 const wwwAutolink = {
-  name: "wwwAutolink",
+  name: 'wwwAutolink',
   tokenize: tokenizeWwwAutolink,
-  previous: previousWww,
+  previous: previousWww
 };
 const protocolAutolink = {
-  name: "protocolAutolink",
+  name: 'protocolAutolink',
   tokenize: tokenizeProtocolAutolink,
-  previous: previousProtocol,
+  previous: previousProtocol
 };
 const emailAutolink = {
-  name: "emailAutolink",
+  name: 'emailAutolink',
   tokenize: tokenizeEmailAutolink,
-  previous: previousEmail,
+  previous: previousEmail
 };
 
 /** @type {ConstructRecord} */
@@ -91,7 +62,7 @@ const text = {};
  */
 export function gfmAutolinkLiteral() {
   return {
-    text,
+    text
   };
 }
 
@@ -102,8 +73,7 @@ let code = 48;
 while (code < 123) {
   text[code] = emailAutolink;
   code++;
-  if (code === 58) code = 65;
-  else if (code === 91) code = 97;
+  if (code === 58) code = 65;else if (code === 91) code = 97;
 }
 text[43] = emailAutolink;
 text[45] = emailAutolink;
@@ -150,15 +120,11 @@ function tokenizeEmailAutolink(effects, ok, nok) {
    * @type {State}
    */
   function start(code) {
-    if (
-      !gfmAtext(code) ||
-      !previousEmail.call(self, self.previous) ||
-      previousUnbalanced(self.events)
-    ) {
+    if (!gfmAtext(code) || !previousEmail.call(self, self.previous) || previousUnbalanced(self.events)) {
       return nok(code);
     }
-    effects.enter("literalAutolink");
-    effects.enter("literalAutolinkEmail");
+    effects.enter('literalAutolink');
+    effects.enter('literalAutolinkEmail');
     return atext(code);
   }
 
@@ -201,11 +167,7 @@ function tokenizeEmailAutolink(effects, ok, nok) {
   function emailDomain(code) {
     // Dot followed by alphanumerical (not `-` or `_`).
     if (code === 46) {
-      return effects.check(
-        emailDomainDotTrail,
-        emailDomainAfter,
-        emailDomainDot,
-      )(code);
+      return effects.check(emailDomainDotTrail, emailDomainAfter, emailDomainDot)(code);
     }
 
     // Alphanumerical, `-`, and `_`.
@@ -254,8 +216,8 @@ function tokenizeEmailAutolink(effects, ok, nok) {
     // Domain must not be empty, must include a dot, and must end in alphabetical.
     // Source: <https://github.com/github/cmark-gfm/blob/ef1cfcb/extensions/autolink.c#L332>.
     if (data && dot && asciiAlpha(self.previous)) {
-      effects.exit("literalAutolinkEmail");
-      effects.exit("literalAutolink");
+      effects.exit('literalAutolinkEmail');
+      effects.exit('literalAutolink');
       return ok(code);
     }
     return nok(code);
@@ -288,22 +250,14 @@ function tokenizeWwwAutolink(effects, ok, nok) {
    * @type {State}
    */
   function wwwStart(code) {
-    if (
-      (code !== 87 && code !== 119) ||
-      !previousWww.call(self, self.previous) ||
-      previousUnbalanced(self.events)
-    ) {
+    if (code !== 87 && code !== 119 || !previousWww.call(self, self.previous) || previousUnbalanced(self.events)) {
       return nok(code);
     }
-    effects.enter("literalAutolink");
-    effects.enter("literalAutolinkWww");
+    effects.enter('literalAutolink');
+    effects.enter('literalAutolinkWww');
     // Note: we *check*, so we can discard the `www.` we parsed.
     // If it worked, we consider it as a part of the domain.
-    return effects.check(
-      wwwPrefix,
-      effects.attempt(domain, effects.attempt(path, wwwAfter), nok),
-      nok,
-    )(code);
+    return effects.check(wwwPrefix, effects.attempt(domain, effects.attempt(path, wwwAfter), nok), nok)(code);
   }
 
   /**
@@ -317,8 +271,8 @@ function tokenizeWwwAutolink(effects, ok, nok) {
    * @type {State}
    */
   function wwwAfter(code) {
-    effects.exit("literalAutolinkWww");
-    effects.exit("literalAutolink");
+    effects.exit('literalAutolinkWww');
+    effects.exit('literalAutolink');
     return ok(code);
   }
 }
@@ -336,7 +290,7 @@ function tokenizeWwwAutolink(effects, ok, nok) {
  */
 function tokenizeProtocolAutolink(effects, ok, nok) {
   const self = this;
-  let buffer = "";
+  let buffer = '';
   let seen = false;
   return protocolStart;
 
@@ -351,13 +305,9 @@ function tokenizeProtocolAutolink(effects, ok, nok) {
    * @type {State}
    */
   function protocolStart(code) {
-    if (
-      (code === 72 || code === 104) &&
-      previousProtocol.call(self, self.previous) &&
-      !previousUnbalanced(self.events)
-    ) {
-      effects.enter("literalAutolink");
-      effects.enter("literalAutolinkHttp");
+    if ((code === 72 || code === 104) && previousProtocol.call(self, self.previous) && !previousUnbalanced(self.events)) {
+      effects.enter('literalAutolink');
+      effects.enter('literalAutolinkHttp');
       buffer += String.fromCodePoint(code);
       effects.consume(code);
       return protocolPrefixInside;
@@ -385,7 +335,7 @@ function tokenizeProtocolAutolink(effects, ok, nok) {
     }
     if (code === 58) {
       const protocol = buffer.toLowerCase();
-      if (protocol === "http" || protocol === "https") {
+      if (protocol === 'http' || protocol === 'https') {
         effects.consume(code);
         return protocolSlashesInside;
       }
@@ -428,17 +378,7 @@ function tokenizeProtocolAutolink(effects, ok, nok) {
   function afterProtocol(code) {
     // To do: this is different from `markdown-rs`:
     // https://github.com/wooorm/markdown-rs/blob/b3a921c761309ae00a51fe348d8a43adbc54b518/src/construct/gfm_autolink_literal.rs#L172-L182
-    return code === null ||
-      asciiControl(code) ||
-      markdownLineEndingOrSpace(code) ||
-      unicodeWhitespace(code) ||
-      unicodePunctuation(code)
-      ? nok(code)
-      : effects.attempt(
-          domain,
-          effects.attempt(path, protocolAfter),
-          nok,
-        )(code);
+    return code === null || asciiControl(code) || markdownLineEndingOrSpace(code) || unicodeWhitespace(code) || unicodePunctuation(code) ? nok(code) : effects.attempt(domain, effects.attempt(path, protocolAfter), nok)(code);
   }
 
   /**
@@ -452,8 +392,8 @@ function tokenizeProtocolAutolink(effects, ok, nok) {
    * @type {State}
    */
   function protocolAfter(code) {
-    effects.exit("literalAutolinkHttp");
-    effects.exit("literalAutolink");
+    effects.exit('literalAutolinkHttp');
+    effects.exit('literalAutolink');
     return ok(code);
   }
 }
@@ -556,12 +496,7 @@ function tokenizeDomain(effects, ok, nok) {
     // Instead of some new production for Unicode alphanumerics, markdown
     // already has that for Unicode punctuation and whitespace, so use those.
     // Source: <https://github.com/github/cmark-gfm/blob/ef1cfcb/extensions/autolink.c#L12>.
-    if (
-      code === null ||
-      markdownLineEndingOrSpace(code) ||
-      unicodeWhitespace(code) ||
-      (code !== 45 && unicodePunctuation(code))
-    ) {
+    if (code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code) || code !== 45 && unicodePunctuation(code)) {
       return domainAfter(code);
     }
     seen = true;
@@ -656,30 +591,10 @@ function tokenizePath(effects, ok) {
     // Check whether this trailing punctuation marker is optionally
     // followed by more trailing markers, and then followed
     // by an end.
-    if (
-      code === 33 ||
-      code === 34 ||
-      code === 38 ||
-      code === 39 ||
-      code === 41 ||
-      code === 42 ||
-      code === 44 ||
-      code === 46 ||
-      code === 58 ||
-      code === 59 ||
-      code === 60 ||
-      code === 63 ||
-      code === 93 ||
-      code === 95 ||
-      code === 126
-    ) {
+    if (code === 33 || code === 34 || code === 38 || code === 39 || code === 41 || code === 42 || code === 44 || code === 46 || code === 58 || code === 59 || code === 60 || code === 63 || code === 93 || code === 95 || code === 126) {
       return effects.check(trail, ok, pathAtPunctuation)(code);
     }
-    if (
-      code === null ||
-      markdownLineEndingOrSpace(code) ||
-      unicodeWhitespace(code)
-    ) {
+    if (code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code)) {
       return ok(code);
     }
     effects.consume(code);
@@ -736,20 +651,7 @@ function tokenizeTrail(effects, ok, nok) {
    */
   function trail(code) {
     // Regular trailing punctuation.
-    if (
-      code === 33 ||
-      code === 34 ||
-      code === 39 ||
-      code === 41 ||
-      code === 42 ||
-      code === 44 ||
-      code === 46 ||
-      code === 58 ||
-      code === 59 ||
-      code === 63 ||
-      code === 95 ||
-      code === 126
-    ) {
+    if (code === 33 || code === 34 || code === 39 || code === 41 || code === 42 || code === 44 || code === 46 || code === 58 || code === 59 || code === 63 || code === 95 || code === 126) {
       effects.consume(code);
       return trail;
     }
@@ -770,13 +672,10 @@ function tokenizeTrail(effects, ok, nok) {
       return trailBracketAfter;
     }
     if (
-      // `<` is an end.
-      code === 60 ||
-      // So is whitespace.
-      code === null ||
-      markdownLineEndingOrSpace(code) ||
-      unicodeWhitespace(code)
-    ) {
+    // `<` is an end.
+    code === 60 ||
+    // So is whitespace.
+    code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code)) {
       return ok(code);
     }
     return nok(code);
@@ -798,13 +697,7 @@ function tokenizeTrail(effects, ok, nok) {
   function trailBracketAfter(code) {
     // Whitespace or something that could start a resource or reference is the end.
     // Switch back to trail otherwise.
-    if (
-      code === null ||
-      code === 40 ||
-      code === 91 ||
-      markdownLineEndingOrSpace(code) ||
-      unicodeWhitespace(code)
-    ) {
+    if (code === null || code === 40 || code === 91 || markdownLineEndingOrSpace(code) || unicodeWhitespace(code)) {
       return ok(code);
     }
     return trail(code);
@@ -908,16 +801,7 @@ function tokenizeEmailDomainDotTrail(effects, ok, nok) {
  * @type {Previous}
  */
 function previousWww(code) {
-  return (
-    code === null ||
-    code === 40 ||
-    code === 42 ||
-    code === 95 ||
-    code === 91 ||
-    code === 93 ||
-    code === 126 ||
-    markdownLineEndingOrSpace(code)
-  );
+  return code === null || code === 40 || code === 42 || code === 95 || code === 91 || code === 93 || code === 126 || markdownLineEndingOrSpace(code);
 }
 
 /**
@@ -947,13 +831,7 @@ function previousEmail(code) {
  * @returns {boolean}
  */
 function gfmAtext(code) {
-  return (
-    code === 43 ||
-    code === 45 ||
-    code === 46 ||
-    code === 95 ||
-    asciiAlphanumeric(code)
-  );
+  return code === 43 || code === 45 || code === 46 || code === 95 || asciiAlphanumeric(code);
 }
 
 /**
@@ -965,10 +843,7 @@ function previousUnbalanced(events) {
   let result = false;
   while (index--) {
     const token = events[index][1];
-    if (
-      (token.type === "labelLink" || token.type === "labelImage") &&
-      !token._balanced
-    ) {
+    if ((token.type === 'labelLink' || token.type === 'labelImage') && !token._balanced) {
       result = true;
       break;
     }

package/types/html-tree-adapter.d.mts

@@ -30,7 +30,10 @@ export function parseHtmlFragment(html: string): any;
  * child its parent's conversion reads (an unclosed element ends where its last
  * child does). One walk then swaps each placeholder for its subtree. The result
  * is the tree a single `fromParse5` call returns, which
- * `html-tree-adapter.test` holds it to.
+ * `html-tree-adapter.test` holds it to — except that an element the converter
+ * would take for a template with no content to take (see
+ * {@link CONVERTS_AS_TEMPLATE}) converts as an ordinary element instead of
+ * throwing.
  *
  * Mutates `tree`: {@link parseHtmlFragment} owns the parse and drops it after.
  * @param {any} tree