agent-sanitizer 2.47.9 → 2.47.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-sanitizer",
-  "version": "2.47.9",
+  "version": "2.47.11",
   "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
   "type": "module",
   "repository": {
@@ -63,6 +63,7 @@
     "globals": "17.6.0",
     "lint-staged": "^17.0.5",
     "prettier": "^3.0.0",
+    "rehype-parse": "9.0.1",
     "smol-toml": "^1.7.1",
     "typescript": "6.0.3",
     "typescript-eslint": "8.61.0",
@@ -214,12 +215,14 @@
   "dependencies": {
     "agent-control-plane-core": "0.3.0",
     "css-tree": "^3.2.1",
+    "hast-util-from-parse5": "8.0.3",
     "namespace-guard": "0.20.0",
-    "rehype-parse": "9.0.1",
+    "parse5": "7.3.0",
     "remark-gfm": "4.0.1",
     "remark-parse": "11.0.0",
     "unified": "11.0.5",
-    "unist-util-visit": "5.1.0"
+    "unist-util-visit": "5.1.0",
+    "vfile": "6.0.3"
   },
   "scripts": {
     "test": "node scripts/coverage.mjs",

package/src/html-tree-adapter.mjs

@@ -0,0 +1,143 @@
+/**
+ * The parse5 tree adapter this package parses HTML fragments with, and the
+ * fragment parse itself.
+ *
+ * WHY IT EXISTS — parse5 ends a fragment parse by moving every child of the
+ * parsed `html` element into the fragment it returns, one at a time, always
+ * taking the first (`_adoptNodes`). The default tree adapter removes each one
+ * with `childNodes.splice(0, 1)`, which shifts every remaining sibling: for a
+ * document with N top-level nodes that is N^2/2 element moves. 256 KB of
+ * ordinary `<p>` prose is ~8600 of them, and the removal alone was a quarter of
+ * the whole HTML layer's cost. The cursor below turns each removal into an
+ * increment, so the drain is linear in N.
+ *
+ * It is a separate module, and not in the package's `exports` map, because it
+ * is an implementation detail of `./html` that its own test parses against.
+ */
+import {
+  defaultTreeAdapter,
+  parseFragment as parse5ParseFragment,
+} from "parse5";
+import { fromParse5 } from "hast-util-from-parse5";
+import { VFile } from "vfile";
+
+/**
+ * A tree adapter for ONE parse, plus the settle step that ends it.
+ *
+ * The pending state is per-parse rather than module-level so that a parse which
+ * throws cannot leave a deferred removal for the next one to inherit.
+ * @returns {{ adapter: any, settle: () => void }}
+ */
+export function createTreeAdapter() {
+  /**
+   * Where a parent's un-detached children start, for every parent with a
+   * removal still deferred.
+   * @type {Map<any, number>}
+   */
+  const drainedUpTo = new Map();
+
+  /**
+   * Put `parent.childNodes` in the state the default adapter's splices would
+   * have left it in.
+   *
+   * INVARIANT — every adapter method that reads or writes a parent's children
+   * BY POSITION calls this first, so a deferred removal is never observable to
+   * parse5, which reaches every child through this adapter. `setDocumentType`
+   * is the one method that scans a node's children without going through those:
+   * it scans the DOCUMENT's, and a fragment parse detaches nothing from the
+   * document.
+   * @param {any} parent
+   */
+  function flush(parent) {
+    const at = drainedUpTo.get(parent);
+    if (at === undefined) return;
+    drainedUpTo.delete(parent);
+    parent.childNodes.splice(0, at);
+  }
+
+  /**
+   * Flush every parent still holding a deferred removal.
+   *
+   * This is what makes the deferral invisible to the tree's READERS, which get
+   * the nodes directly and not through the adapter: `hast-util-from-parse5`
+   * here, and any consumer of the parse5 AST. Call it once the parse has
+   * returned, before anything reads the tree.
+   */
+  function settle() {
+    for (const parent of [...drainedUpTo.keys()]) flush(parent);
+  }
+
+  const adapter = {
+    ...defaultTreeAdapter,
+    /** @param {any} node */
+    detachNode(node) {
+      const parent = node.parentNode;
+      if (!parent) return;
+      const at = drainedUpTo.get(parent) ?? 0;
+      // The front of the un-drained region: record the removal and move on. Any
+      // other position is a mid-tree detach (the adoption agency algorithm,
+      // foster parenting), which is rare and takes the default adapter's splice.
+      if (parent.childNodes[at] === node) {
+        drainedUpTo.set(parent, at + 1);
+        node.parentNode = null;
+        return;
+      }
+      flush(parent);
+      defaultTreeAdapter.detachNode(node);
+    },
+    /** @param {any} node */
+    getFirstChild(node) {
+      return node.childNodes[drainedUpTo.get(node) ?? 0];
+    },
+    /** @param {any} node */
+    getChildNodes(node) {
+      flush(node);
+      return defaultTreeAdapter.getChildNodes(node);
+    },
+    /** @param {any} parent @param {any} child */
+    appendChild(parent, child) {
+      flush(parent);
+      defaultTreeAdapter.appendChild(parent, child);
+    },
+    /** @param {any} parent @param {any} child @param {any} reference */
+    insertBefore(parent, child, reference) {
+      flush(parent);
+      defaultTreeAdapter.insertBefore(parent, child, reference);
+    },
+    /** @param {any} parent @param {string} text */
+    insertText(parent, text) {
+      flush(parent);
+      defaultTreeAdapter.insertText(parent, text);
+    },
+    /** @param {any} parent @param {string} text @param {any} reference */
+    insertTextBefore(parent, text, reference) {
+      flush(parent);
+      defaultTreeAdapter.insertTextBefore(parent, text, reference);
+    },
+  };
+
+  return { adapter, settle };
+}
+
+/**
+ * Parse `html` as an HTML fragment and return the hast tree.
+ *
+ * This is `hast-util-from-html`'s fragment path with {@link createTreeAdapter}'s
+ * adapter in place of the default one — the option `rehype-parse` does not
+ * forward. The settings are that path's own: positions on, parse errors ignored
+ * (this package reports on the tree, never on the tokenizer's complaints), and
+ * scripting off, so `noscript` content parses as markup.
+ * @param {string} html
+ * @returns {any}
+ */
+export function parseHtmlFragment(html) {
+  const { adapter, settle } = createTreeAdapter();
+  const fragment = parse5ParseFragment(html, {
+    sourceCodeLocationInfo: true,
+    onParseError: null,
+    scriptingEnabled: false,
+    treeAdapter: adapter,
+  });
+  settle();
+  return fromParse5(fragment, { file: new VFile(html) });
+}

package/src/html.mjs

@@ -49,7 +49,7 @@ import { ident as cssIdent } from "css-tree/utils";
 import { unified } from "unified";
 import remarkParse from "remark-parse";
 import remarkGfm from "remark-gfm";
-import rehypeParse from "rehype-parse";
+import { parseHtmlFragment } from "./html-tree-adapter.mjs";
 import { SKIP, EXIT } from "unist-util-visit";
 import {
   HTML_TAG_PRESENT,
@@ -1420,11 +1420,6 @@ function hasDataSrc(el) {
   );
 }
 
-// One shared fragment parser for every HTML parse in this module (mirroring
-// `mdParser` below): all of them must agree on the tokenizer's verdict, so
-// there is exactly one parser configuration to reason about.
-const htmlParser = unified().use(rehypeParse, { fragment: true });
-
 /**
  * Preorder depth-first walk of a unist tree, calling `visitor(node, index,
  * parent)` on every node whose `type` is `test` (or on every node when `test` is
@@ -1507,11 +1502,12 @@ function lastParseCached(parse) {
 }
 
 /**
- * Parse `html` as an HTML fragment with the real tokenizer (parse5, via rehype).
- * @param {string} html
- * @returns {any}
+ * The one HTML parse in this module: the real tokenizer (parse5, through
+ * `./html-tree-adapter.mjs`), with the last (input, tree) pair remembered. Every
+ * parse here must agree on the tokenizer's verdict, so there is exactly one
+ * parser configuration to reason about.
  */
-const parseFragment = lastParseCached((html) => htmlParser.parse(html));
+const parseFragment = lastParseCached(parseHtmlFragment);
 
 /**
  * @param {string} htmlValue
@@ -1722,7 +1718,7 @@ function hasWarned(warned) {
 /**
  * Scan raw HTML for hidden content to strip and preserved tags to report.
  * Returned ranges are offsets into `html`; comments and hidden elements span
- * the whole element including its content (rehype positions cover open tag
+ * the whole element including its content (hast positions cover open tag
  * through matching close, and parse5 extends an unclosed element to the end
  * of the fragment — fail-closed for truncated markup).
  * @param {string} html
@@ -1803,8 +1799,8 @@ const BOGUS_COMMENT_OPEN_RE = /<[!?]/g;
 // comment / declaration (`<!`, `<?`). Per the HTML tokenizer such a construct
 // keeps consuming the input stream until the next `>`, so it absorbs the
 // following inline-html node (an open tag swallows it as bogus attributes; a
-// bogus end tag / `<!…` opens a bogus comment). parse5 (the flow/source branch,
-// via rehype) models this; the per-tag balance walk below does not, so without
+// bogus end tag / `<!…` opens a bogus comment). parse5 (the flow/source branch)
+// models this; the per-tag balance walk below does not, so without
 // this a fragment parses differently as a flow block than as a paragraph —
 // breaking idempotency once a first pass demotes a block to phrasing (see
 // html-property "second pass changes nothing"). An open/end tag requires a
@@ -1830,7 +1826,7 @@ function foldAbsorb(absorbing, raw) {
 
 /**
  * Map of comment start-offset -> end-offset (exclusive) for EVERY comment the
- * HTML tokenizer finds in `value`, from a SINGLE rehype parse. Validated against
+ * HTML tokenizer finds in `value`, from a SINGLE parse5 parse. Validated against
  * the real tokenizer (parse5) rather than a hand-rolled bogus-comment state
  * machine, so a bogus comment (`<!bogus>`, `<?php?>`, `<![CDATA[…]]>`) is spliced
  * to exactly the span a browser hides and a `<Foo>` element, a `<!doctype>`, or
@@ -1964,7 +1960,7 @@ function hasHtmlLeaf(node) {
  * the container's end when unbalanced — fail-closed), comments become
  * single-node ranges, and preserved tags are counted. Inline html is tokenized
  * per TAG (an element's content sits in sibling text nodes), which is why this
- * walk exists instead of handing the value to rehype.
+ * walk exists instead of handing the value to parse5.
  *
  * The absorb state is folded from the RAW source between html nodes (not from
  * mdast node values), so markdown constructs that reshuffle the character
@@ -2076,7 +2072,7 @@ function scanMarkdown(text) {
   const ranges = [];
   const warned = newWarned();
 
-  // Flow html blocks carry complete markup, so rehype locates comments/hidden
+  // Flow html blocks carry complete markup, so parse5 locates comments/hidden
   // elements precisely within them; block-local offsets are shifted to
   // document coordinates.
   walk(tree, "html", (/** @type {any} */ node, _index, parent) => {
@@ -2137,7 +2133,7 @@ function hasMarkdownCode(text) {
  * The parsed fragment tree for `text` when `text` is HTML *source*, else null.
  *
  * "HTML source" means the markup accounts for the WHOLE document: the real
- * tokenizer (parse5, via rehype) places every element there is, and the only
+ * tokenizer (parse5) places every element there is, and the only
  * character data it leaves OUTSIDE all of them is whitespace. That is exactly
  * the property the source branch needs — it hands the whole input to
  * `scanHtmlFragment` as one fragment, which is faithful only when there is no
@@ -2847,7 +2843,7 @@ function parseSrcset(value) {
 
 /**
  * Candidate URLs of a `srcset` (a "url descriptor" string parsed per the HTML
- * grammar) or `ping` (a space-separated url list rehype delivers as an array)
+ * grammar) or `ping` (a space-separated url list hast delivers as an array)
  * attribute. An absent attribute (neither string nor array) yields none.
  * @param {unknown} value
  * @returns {string[]}
@@ -2862,8 +2858,9 @@ function multiUrlAttr(value) {
 }
 
 /**
- * URL-bearing attributes of every HTML element in `text`, parsed with rehype so
- * quoting/casing/entities are handled correctly (no hand-rolled tag regex).
+ * URL-bearing attributes of every HTML element in `text`, parsed with the real
+ * HTML tokenizer so quoting/casing/entities are handled correctly (no
+ * hand-rolled tag regex).
  * `context` selects the per-URL check the caller applies: resource URLs get the
  * exfil-shape test; form-submission and meta-refresh targets additionally flag
  * any absolute off-origin destination.
@@ -2906,7 +2903,7 @@ function extractHtmlUrls(text) {
           autoFetched: true,
           context: "form",
         });
-    // rehype delivers `http-equiv` as an array (comma-separated); join it back
+    // hast delivers `http-equiv` as an array (comma-separated); join it back
     // so a `refresh` directive is matched regardless of how it was tokenized.
     const httpEquiv = Array.isArray(props.httpEquiv)
       ? props.httpEquiv.join(",").toLowerCase()

package/types/html-tree-adapter.d.mts

@@ -0,0 +1,23 @@
+/**
+ * A tree adapter for ONE parse, plus the settle step that ends it.
+ *
+ * The pending state is per-parse rather than module-level so that a parse which
+ * throws cannot leave a deferred removal for the next one to inherit.
+ * @returns {{ adapter: any, settle: () => void }}
+ */
+export function createTreeAdapter(): {
+    adapter: any;
+    settle: () => void;
+};
+/**
+ * Parse `html` as an HTML fragment and return the hast tree.
+ *
+ * This is `hast-util-from-html`'s fragment path with {@link createTreeAdapter}'s
+ * adapter in place of the default one — the option `rehype-parse` does not
+ * forward. The settings are that path's own: positions on, parse errors ignored
+ * (this package reports on the tree, never on the tokenizer's complaints), and
+ * scripting off, so `noscript` content parses as markup.
+ * @param {string} html
+ * @returns {any}
+ */
+export function parseHtmlFragment(html: string): any;

package/types/html.d.mts

@@ -53,7 +53,7 @@ export function spliceRanges(text: string, ranges: SpliceRange[]): {
 /**
  * Scan raw HTML for hidden content to strip and preserved tags to report.
  * Returned ranges are offsets into `html`; comments and hidden elements span
- * the whole element including its content (rehype positions cover open tag
+ * the whole element including its content (hast positions cover open tag
  * through matching close, and parse5 extends an unclosed element to the end
  * of the fragment — fail-closed for truncated markup).
  * @param {string} html