agent-sanitizer 2.52.0 → 2.54.0

package/LICENSE-THIRD-PARTY

@@ -0,0 +1,28 @@
+Third-party code redistributed in this package.
+
+src/vendor/gfm-autolink-literal.mjs
+  Copied from micromark-extension-gfm-autolink-literal@2.1.0, with one
+  function changed (see that file's header). Original licence follows.
+
+(The MIT License)
+
+Copyright (c) 2020 Titus Wormer <tituswormer@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -40,6 +40,15 @@ const result = await sanitize(pageSource, { html: true });
 // Layer 3 alone: flag exfil-shaped URLs without splicing anything (for text
 // that must stay byte-faithful, e.g. a PR diff). Implied by `html: true`.
 const scanned = await sanitize(diffText, { exfilScan: true });
+
+// Layer 3 reads an exact-digest-length hex value under a generic parameter
+// name (`?v=<md5>`, an ETag, a commit id) as a fingerprint. `flagDigestValues`
+// reports it as payload instead — more false positives, no 16-to-64-byte
+// channel under a name the caller picks. For monitors, not for splicing.
+const strict = await sanitize(logText, {
+  exfilScan: true,
+  flagDigestValues: true,
+});
 ```
 
 `sanitize` never throws and never silently drops content—any change comes with

package/THREAT-MODEL.md

@@ -140,6 +140,18 @@ attributes (`src`/`href`/`background`/`srcset`/`ping`, form `action`/`formaction
 - off-origin form actions and `meta refresh` redirects
 - `javascript:` / `vbscript:` targets
 
+**The digest exemption, and the switch that lifts it.** A value that is
+exactly one digest width of hex (32/40/56/64/96/128) under a generic parameter
+name reads as a fingerprint, not a payload: a cache-buster `?v=<md5>`, an ETag,
+a request id, a git commit, imgix's `?s=`. Under a name that already says
+credential the same characters read as payload and still flag. That leaves a
+residual, because the caller writing the URL picks the name: a payload padded to
+exactly one digest length rides under a generic one, buying 16 to 64 bytes per
+parameter. `flagDigestValues` moves the trade-off — it drops the exemption
+entirely, at the cost of flagging every real fingerprint — for a caller whose
+job is monitoring rather than presenting text to a model. Like every Layer 3
+option it can only ADD detection; there is no switch that turns a report off.
+
 Each threat carries a `reason` and the destination `target` (never the
 payload-bearing query/fragment) — the finding is shown to the operator with the
 target named and the payload withheld, since re-presenting the exfil payload in

package/bin/sanitize-cli.mjs

@@ -15,7 +15,7 @@
  * bare `{ text, html }` keeps working). Per op:
  *
  *   sanitize           { text, html? }            -> { cleaned, found, warnings, notes, splices? }
- *   sanitizeText       { text, html?, exfilScan? } -> { cleaned, warnings, notes, modified, sgrNote }
+ *   sanitizeText       { text, html?, exfilScan?, flagDigestValues? } -> { cleaned, warnings, notes, modified, sgrNote }
  *   classifyPrompt     { text }                    -> { action, reason? }
  *   scanInstructionFiles { globs, cwd? }           -> { findings: [{ file, findings }] }
  *   cleanFile          { path }                    -> { changed }
@@ -135,6 +135,7 @@ export const OPS = {
       {
         html: Boolean(req.html),
         exfilScan: Boolean(req.exfilScan),
+        flagDigestValues: Boolean(req.flagDigestValues),
       },
     );
     return { cleaned, warnings, notes, modified, sgrNote };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-sanitizer",
-  "version": "2.52.0",
+  "version": "2.54.0",
   "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
   "type": "module",
   "repository": {
@@ -65,6 +65,7 @@
     "lint-staged": "^17.0.5",
     "prettier": "^3.0.0",
     "rehype-parse": "9.0.1",
+    "remark-gfm": "4.0.1",
     "smol-toml": "^1.7.1",
     "typescript": "6.0.3",
     "typescript-eslint": "8.61.0",
@@ -204,6 +205,7 @@
   },
   "files": [
     "src/*.mjs",
+    "src/vendor/*.mjs",
     "python/agent_sanitizer/data/invisible-charset.json",
     "python/agent_sanitizer/secrets/data/credential-names.json",
     "python/agent_sanitizer/secrets/data/redaction-floor.json",
@@ -213,6 +215,7 @@
     "bin/sanitize-cli.mjs",
     "types",
     "LICENSE",
+    "LICENSE-THIRD-PARTY",
     "README.md",
     "THREAT-MODEL.md",
     "SECURITY.md"
@@ -221,9 +224,16 @@
     "agent-control-plane-core": "0.3.0",
     "css-tree": "^3.2.1",
     "hast-util-from-parse5": "8.0.3",
+    "mdast-util-gfm": "3.1.0",
+    "micromark-extension-gfm-footnote": "2.1.0",
+    "micromark-extension-gfm-strikethrough": "2.1.0",
+    "micromark-extension-gfm-table": "2.1.1",
+    "micromark-extension-gfm-task-list-item": "2.1.0",
+    "micromark-util-character": "2.1.1",
+    "micromark-util-combine-extensions": "2.0.1",
+    "micromark-util-types": "2.0.2",
     "namespace-guard": "0.20.0",
     "parse5": "7.3.0",
-    "remark-gfm": "4.0.1",
     "remark-parse": "11.0.0",
     "unified": "11.0.5",
     "unist-util-visit": "5.1.0",

package/src/gfm.mjs

@@ -0,0 +1,54 @@
+/**
+ * `remark-gfm` with one extension swapped for a corrected copy.
+ *
+ * This is `remark-gfm@4.0.1`'s own plugin body and
+ * `micromark-extension-gfm@3.0.0`'s own `gfm()` composition, inlined so that
+ * `gfmAutolinkLiteral` can come from `./vendor/gfm-autolink-literal.mjs`
+ * instead of the published package — see that file for what differs and when
+ * to delete both. Every other extension, and the mdast layer, are the upstream
+ * ones at the versions `remark-gfm` pins.
+ *
+ * Assembling the list here rather than adding an extension beside `remark-gfm`
+ * is deliberate: micromark tries the constructs registered for a character in
+ * order, so a second autolink extension would run AFTER the upstream one and
+ * change nothing.
+ *
+ * `test/gfm-autolink-parity.test.mjs` pins this against upstream's answers.
+ */
+import { gfmFromMarkdown, gfmToMarkdown } from "mdast-util-gfm";
+import { gfmFootnote } from "micromark-extension-gfm-footnote";
+import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough";
+import { gfmTable } from "micromark-extension-gfm-table";
+import { gfmTaskListItem } from "micromark-extension-gfm-task-list-item";
+import { combineExtensions } from "micromark-util-combine-extensions";
+
+import { gfmAutolinkLiteral } from "./vendor/gfm-autolink-literal.mjs";
+
+/**
+ * A unified plugin adding GFM support: autolink literals, footnotes,
+ * strikethrough, tables and task lists — the same set, in the same order, as
+ * `remark-gfm`.
+ * @this {any} unified processor
+ * @returns {undefined}
+ */
+export default function remarkGfmFixed() {
+  const data = this.data();
+  const micromarkExtensions =
+    data.micromarkExtensions || (data.micromarkExtensions = []);
+  const fromMarkdownExtensions =
+    data.fromMarkdownExtensions || (data.fromMarkdownExtensions = []);
+  const toMarkdownExtensions =
+    data.toMarkdownExtensions || (data.toMarkdownExtensions = []);
+
+  micromarkExtensions.push(
+    combineExtensions([
+      gfmAutolinkLiteral(),
+      gfmFootnote(),
+      gfmStrikethrough(),
+      gfmTable(),
+      gfmTaskListItem(),
+    ]),
+  );
+  fromMarkdownExtensions.push(gfmFromMarkdown());
+  toMarkdownExtensions.push(gfmToMarkdown());
+}

package/src/html.mjs

@@ -48,7 +48,7 @@ import cssGenerate from "css-tree/generator";
 import { ident as cssIdent } from "css-tree/utils";
 import { unified } from "unified";
 import remarkParse from "remark-parse";
-import remarkGfm from "remark-gfm";
+import remarkGfm from "./gfm.mjs";
 import { parseHtmlFragment } from "./html-tree-adapter.mjs";
 import { SKIP, EXIT } from "unist-util-visit";
 import {
@@ -2720,9 +2720,13 @@ function rawParams(qs) {
  * @param {string} name    lowercased parameter name, for the allowlist gate
  * @param {string} value   RAW (un-decoded) value
  * @param {string} rawName RAW (case-preserved, un-decoded) name
+ * @param {boolean} flagDigestValues  drop the digest exemption entirely. The
+ *   name is the weak half of the test above, because the caller writing the
+ *   URL picks it: a payload padded to exactly one digest length otherwise
+ *   rides under any generic name.
  * @returns {string | null}
  */
-function paramExfilReason(name, value, rawName) {
+function paramExfilReason(name, value, rawName, flagDigestValues) {
   if (BENIGN_BLOB_PARAM_RE.test(name)) return null;
   const publicKeyId =
     PUBLIC_KEY_ID_PARAM_RE.test(name) && value.length < BLOB_VALUE_MIN_LEN;
@@ -2730,9 +2734,9 @@ function paramExfilReason(name, value, rawName) {
   // — a cache-buster `?v=`, an ETag, imgix's `?s=`, a commit id. Under a name
   // that already says credential the same 64 hex characters read as 32 bytes of
   // payload, so the exemption stops there.
-  const digestIsBenign = !(
-    KEYWORD_PARAM_NAME_RE.test(name) || matchesSecretHint(name)
-  );
+  const digestIsBenign =
+    !flagDigestValues &&
+    !(KEYWORD_PARAM_NAME_RE.test(name) || matchesSecretHint(name));
   for (const candidate of [rawName, value]) {
     if (!candidate) continue;
     // A leaked credential is an OPAQUE, separator-free token. Gate the
@@ -2784,7 +2788,10 @@ function rawUrlKeywordExfil(url) {
   for (const segment of url.slice(qIdx + 1).split("#")) {
     for (const [name, value, rawName] of rawParams(segment)) {
       if (!KEYWORD_PARAM_NAME_RE.test(name)) continue;
-      const reason = paramExfilReason(name, value, rawName);
+      // Only credential-named params reach here, and the digest exemption never
+      // applies to those, so there is no exemption for `flagDigestValues` to
+      // lift on this path.
+      const reason = paramExfilReason(name, value, rawName, false);
       if (reason) return reason;
     }
   }
@@ -2826,19 +2833,18 @@ function allParamsBenign(parsed) {
 /**
  * Walk the query and fragment parameters of a parsed URL for an exfil reason.
  * @param {URL} parsed
+ * @param {boolean} flagDigestValues
  * @returns {string | null}
  */
-function checkUrlParams(parsed) {
-  for (const [name, value, rawName] of rawParams(parsed.search.slice(1))) {
-    const reason = paramExfilReason(name, value, rawName);
-    if (reason) return reason;
-  }
-  // The fragment carries the same `key=value` channel (`#token=…`); a bare
-  // anchor (`#section-2`) yields one empty-value param that trips nothing.
-  for (const [name, value, rawName] of rawParams(parsed.hash.slice(1))) {
-    const reason = paramExfilReason(name, value, rawName);
-    if (reason) return reason;
-  }
+function checkUrlParams(parsed, flagDigestValues) {
+  // Query and fragment are one channel: `#token=…` carries what `?token=…` does,
+  // and a bare anchor (`#section-2`) yields one empty-value param that trips
+  // nothing.
+  for (const segment of [parsed.search.slice(1), parsed.hash.slice(1)])
+    for (const [name, value, rawName] of rawParams(segment)) {
+      const reason = paramExfilReason(name, value, rawName, flagDigestValues);
+      if (reason) return reason;
+    }
   return null;
 }
 
@@ -2861,10 +2867,18 @@ function checkUrlPath(parsed) {
 }
 
 /**
+ * `flagDigestValues` drops the digest exemption: an exact-digest-length hex
+ * value under a generic parameter name is reported as payload rather than read
+ * as a fingerprint. Off by default because the exemption is what keeps a
+ * cache-buster, an ETag and a commit id quiet; on for a caller whose cost of a
+ * missed 16-to-64-byte channel beats its cost of those false positives. Like
+ * every option this module takes, it can only ADD detection.
  * @param {string} url
+ * @param {{ flagDigestValues?: boolean }} [options]
  * @returns {string | null}
  */
-export function checkExfilUrl(url) {
+export function checkExfilUrl(url, options = {}) {
+  const { flagDigestValues = false } = options;
   // A browser strips tab/newline/CR ANYWHERE in a URL before resolving its
   // scheme, so `java\tscript:alert(1)` navigates as `javascript:`. Strip them
   // for the scheme tests (the payload/length checks below keep the raw string).
@@ -2913,7 +2927,7 @@ export function checkExfilUrl(url) {
     return "unusually long query string";
   if (parsed.hash.length > LONG_QUERY_THRESHOLD)
     return "unusually long fragment";
-  return checkUrlParams(parsed) || checkUrlPath(parsed);
+  return checkUrlParams(parsed, flagDigestValues) || checkUrlPath(parsed);
 }
 
 /**
@@ -3177,9 +3191,10 @@ function collectUrls(text) {
  * link somebody has to follow. Both are reported; the caller uses it to decide
  * how loudly (see the exfil tier in ./output.mjs).
  * @param {string} text
+ * @param {{ flagDigestValues?: boolean }} [options] see {@link checkExfilUrl}
  * @returns {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string }> | null}
  */
-export function detectExfil(text) {
+export function detectExfil(text, options = {}) {
   if (!needsUrlScan(text)) return null;
 
   /** @type {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string }>} */
@@ -3188,7 +3203,7 @@ export function detectExfil(text) {
   try {
     for (const { url, isImage, autoFetched, context } of collectUrls(text)) {
       const reason =
-        checkExfilUrl(url) ||
+        checkExfilUrl(url, options) ||
         (context !== "resource" && isOffOrigin(url)
           ? OFF_ORIGIN_REASON[context]
           : null);

package/src/index.mjs

@@ -102,14 +102,23 @@ export {
  * legitimate markup — matching the separate flags `sanitizeText` takes for the
  * tool-output pipeline, which needs Layer 3's detection without Layer 2's
  * splice.
+ *
+ * `flagDigestValues` widens Layer 3 only: it drops the digest exemption, so an
+ * exact-digest-length hex value under a generic parameter name is reported as
+ * payload rather than read as a cache-buster or an ETag. Off by default, and
+ * like `exfilScan` it can only ADD detection.
  * @param {string} text
- * @param {{ html?: boolean, exfilScan?: boolean } | null} [options]
+ * @param {{ html?: boolean, exfilScan?: boolean, flagDigestValues?: boolean } | null} [options]
  * @returns {Promise<{ cleaned: string, found: string[], warnings: string[], notes: string[], splices?: Array<{ placeholder: string, original: string }> }>}
  */
 export async function sanitize(text, options) {
   if (typeof text !== "string")
     throw new TypeError("sanitize(text, options): text must be a string");
-  const { html = false, exfilScan = false } = options ?? {};
+  const {
+    html = false,
+    exfilScan = false,
+    flagDigestValues = false,
+  } = options ?? {};
   const { cleaned, found, warnings, notes, splices } = await sanitizeText(
     text,
     {
@@ -118,6 +127,7 @@ export async function sanitize(text, options) {
       // an opt-OUT would make `{ html: true, exfilScan: false }` splice Layer 2
       // while silently dropping Layer 3's report — a fail-open the docs deny.
       exfilScan: exfilScan || html,
+      flagDigestValues,
     },
   );
   return {

package/src/output.mjs

@@ -393,10 +393,13 @@ function processLayer1(text, sgrCarveOut) {
  * vet them before they leave. The transform itself stays pure — the caller owns
  * any persistence.
  * @param {PipelineState} state
- * @param {{ html?: boolean, exfilScan?: boolean, deadline?: Deadline }} options
+ * @param {{ html?: boolean, exfilScan?: boolean, flagDigestValues?: boolean, deadline?: Deadline }} options
  * @returns {Promise<{ reveal: string | undefined, splices: Array<{ placeholder: string, original: string }> }>}
  */
-async function applyMarkdownPipeline(state, { html, exfilScan, deadline }) {
+async function applyMarkdownPipeline(
+  state,
+  { html, exfilScan, flagDigestValues, deadline },
+) {
   const inputText = state.text;
   /** @type {string | undefined} */
   let reveal;
@@ -485,7 +488,7 @@ async function applyMarkdownPipeline(state, { html, exfilScan, deadline }) {
   // suspicious, not less, yet Layer 2 has already removed it from `cleaned`.
   if (runLayer3) {
     refuseIfSpent();
-    const threats = detectExfil(inputText);
+    const threats = detectExfil(inputText, { flagDigestValues });
     // Severity tracks who does the fetching. An auto-fetched target — an image,
     // a stylesheet, a form action, a meta refresh — exfiltrates the moment the
     // content renders, with nobody deciding anything: a WARNING. A plain LINK
@@ -571,6 +574,7 @@ async function vetStageValue(text, redact, findings, label) {
  * @typedef {{
  *   html?: boolean,
  *   exfilScan?: boolean,
+ *   flagDigestValues?: boolean,
  *   redact?: (text: string) => Promise<RedactResult|null> | (RedactResult|null),
  *   filterInjection?: (text: string) => Promise<Layer5Result|null> | (Layer5Result|null),
  *   sgrCarveOut?: boolean,