@sema-agent/server 7.69.0 → 7.71.0

package/dist/trace/redact.js

@@ -1,23 +1,40 @@
+import { stripFormatCharacters } from "@sema-agent/core";
 import { recordFailOpen } from "../observability/fail-open.js";
+export const normalizeForRedaction = stripFormatCharacters;
 const REDACTION_TOKEN_RE_SRC = String.raw `«redacted(?::[a-z-]{1,16}){0,2}»`;
-const FULL_SECRET_PATTERNS = [
-    [/-----BEGIN[^-](?:[^-](?:[^-]|-(?!-))*)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END[^-](?:[^-](?:[^-]|-(?!-))*)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
+const ENTROPY_SCAN_MAX = 96;
+const twoStrongAhead = (valueClass) => String.raw `(?=${valueClass}{0,${ENTROPY_SCAN_MAX}}[0-9A-Z]${valueClass}{0,${ENTROPY_SCAN_MAX}}[0-9A-Z])`;
+const IDENT_CONNECTORS = "[_./-]";
+const noConnectorAhead = (valueClass) => String.raw `(?!${valueClass}{0,${ENTROPY_SCAN_MAX}}${IDENT_CONNECTORS}${valueClass})`;
+const valueArm = (prefixSrc, valueClass, min) => new RegExp(`${prefixSrc}${twoStrongAhead(valueClass)}${valueClass}{${min},}`, "g");
+const WORD_SEGMENT_CLASS = "[A-Za-z0-9._~+/-]";
+const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => new RegExp(`(?:${keywordSrc}\\s+${valueClass}{${min},}${tailSrc}` +
+    `|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${valueClass}{${min},}${tailSrc})`, "g");
+const STRUCTURAL_SECRET_PATTERNS = [
+    [/-----BEGIN(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
     [/---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]{0,8192}?---- END SSH2 ENCRYPTED PRIVATE KEY ----/g, "«redacted:private-key»"],
-    [/\bsk-[A-Za-z0-9_-]{16,}\b/g, "«redacted:key»"],
-    [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}\b/g, "«redacted:gh-token»"],
-    [/\bgithub_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59}\b/g, "«redacted:gh-token»"],
-    [/\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, "«redacted:slack-token»"],
-    [/\bAKIA[0-9A-Z]{16}\b/g, "«redacted:aws-key»"],
-    [/\bASIA[0-9A-Z]{16}\b/g, "«redacted:aws-key»"],
-    [/\b(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test|org)_[A-Za-z0-9]{24,}|pypi-[A-Za-z0-9_-]{85,})\b/g, "«redacted:key»"],
-    [/\bxapp-\d-[A-Za-z0-9-]{20,}\b/g, "«redacted:slack-token»"],
-    [/\b[Bb]earer\s+[A-Za-z0-9._~+/-]{20,}=*/g, "Bearer «redacted»"],
-    [/\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/g, "Basic «redacted»"],
-    [/:(?<=\b[A-Za-z][A-Za-z0-9+.-]*:)\/\//g, "$1:«redacted»@"],
-    [/\b[Tt]oken\s+[A-Za-z0-9._~+/-]{20,}/g, "token «redacted»"],
+    [/:(?<=[A-Za-z][A-Za-z0-9+.-]*:)\/\//g, "$1:«redacted»@"],
     [/("(?:private_key|client_secret|password|api[_-]?key|secret|token)"\s*:\s*)"[^"\\]*(?:\\.[^"\\]*)*"/gi, '$1"«redacted»"'],
 ];
+const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
 const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
+const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
+const KEYWORD_SECRET_PATTERNS = [
+    [keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*"), "Bearer «redacted»"],
+    [keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}"), "Basic «redacted»"],
+    [keywordArm("[Tt]oken", "[A-Za-z0-9._~+/-]", 20), "token «redacted»"],
+];
+const SHAPE_SECRET_PATTERNS = [
+    [valueArm("sk-", "[A-Za-z0-9_-]", 16), "«redacted:key»"],
+    [/(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g, "«redacted:gh-token»"],
+    [/github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59,}/g, "«redacted:gh-token»"],
+    [/xox[baprs]-[A-Za-z0-9-]{10,}/g, "«redacted:slack-token»"],
+    [/AKIA[0-9A-Z]{16,}/g, "«redacted:aws-key»"],
+    [/ASIA[0-9A-Z]{16,}/g, "«redacted:aws-key»"],
+    [/(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test|org)_[A-Za-z0-9]{24,}|pypi-[A-Za-z0-9_-]{85,})/g, "«redacted:key»"],
+    [/xapp-\d-[A-Za-z0-9-]{20,}/g, "«redacted:slack-token»"],
+];
+const FULL_SECRET_PATTERNS = [...STRUCTURAL_SECRET_PATTERNS, ...KEYWORD_SECRET_PATTERNS, ...SHAPE_SECRET_PATTERNS];
 const PEM_HEADER_IDENT_MAX = 31;
 const PEM_BODY_RUN_MIN = 8;
 const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_SPACE = 32, CH_PLUS = 43, CH_MINUS = 45, CH_SLASH = 47;
@@ -216,7 +233,7 @@ function isPemFenceLabel(s, from, to) {
     let end = to;
     if (end - from > PEM_BLOCK_WORD.length && s.startsWith(PEM_BLOCK_WORD, end - PEM_BLOCK_WORD.length))
         end -= PEM_BLOCK_WORD.length;
-    if (end - from <= PEM_KEY_WORD.length)
+    if (end - from < PEM_KEY_WORD.length)
         return false;
     if (!s.startsWith(PEM_KEY_WORD, end - PEM_KEY_WORD.length))
         return false;
@@ -289,7 +306,6 @@ function redactPemFragments(text, json) {
     out.push(text.slice(copied));
     return { out: out.join(""), count };
 }
-const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
 function redactHeadlessPpkBodies(text) {
     const KEY = "Private-Lines: ";
     const n = text.length;
@@ -399,14 +415,14 @@ function redactHeadlessPpkBodies(text) {
     return count === 0 ? { out: text, count: 0 } : { out: out + text.slice(last), count };
 }
 const FRAGMENT_SECRET_PATTERNS = [
-    [/\bsk-(?=[A-Za-z0-9_-]{6,})(?=[A-Za-z0-9_-]{0,64}[0-9A-Z][A-Za-z0-9_-]{0,64}[0-9A-Z])[A-Za-z0-9_-]{6,}/g, "«redacted:key:partial»"],
-    [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{8,}/g, "«redacted:gh-token:partial»"],
-    [/\bgithub_pat_[A-Za-z0-9_]{8,}/g, "«redacted:gh-token:partial»"],
-    [/\bxox[baprs]-[A-Za-z0-9-]{4,}/g, "«redacted:slack-token:partial»"],
-    [/\bAKIA[0-9A-Z]{6,}/g, "«redacted:aws-key:partial»"],
-    [/\bASIA[0-9A-Z]{8,}/g, "«redacted:aws-key:partial»"],
-    [/\b(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-|(?:sk|rk)_(?:live|test|org)_|pypi-)(?=[A-Za-z0-9_-]{8,})(?=[A-Za-z0-9_-]{0,90}[0-9A-Z][A-Za-z0-9_-]{0,90}[0-9A-Z])[A-Za-z0-9_-]{8,}/g, "«redacted:key:partial»"],
-    [/\bxapp-\d-[A-Za-z0-9-]{4,}/g, "«redacted:slack-token:partial»"],
+    [valueArm("sk-", "[A-Za-z0-9_-]", 6), "«redacted:key:partial»"],
+    [valueArm("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8), "«redacted:gh-token:partial»"],
+    [/github_pat_[A-Za-z0-9_]{8,}/g, "«redacted:gh-token:partial»"],
+    [valueArm("xox[baprs]-", "[A-Za-z0-9-]", 4), "«redacted:slack-token:partial»"],
+    [valueArm("AKIA", "[0-9A-Z]", 6), "«redacted:aws-key:partial»"],
+    [valueArm("ASIA", "[0-9A-Z]", 8), "«redacted:aws-key:partial»"],
+    [valueArm("(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-|(?:sk|rk)_(?:live|test|org)_|pypi-)", "[A-Za-z0-9_-]", 8), "«redacted:key:partial»"],
+    [valueArm("xapp-\\d-", "[A-Za-z0-9-]", 4), "«redacted:slack-token:partial»"],
 ];
 const REDACTION_TOKENS = new Set([...FULL_SECRET_PATTERNS.map(([, r]) => r), PEM_FRAGMENT_TOKEN, ...FRAGMENT_SECRET_PATTERNS.map(([, r]) => r)]
     .flatMap((r) => r.match(/«[^»]*»/g) ?? []));
@@ -490,21 +506,33 @@ function scanUserinfoRun(text, i, kind, json) {
     return i;
 }
 const REDACTION_TOKEN_Y = new RegExp(REDACTION_TOKEN_RE_SRC, "y");
+const SHAPE_STICKY = SHAPE_SECRET_PATTERNS.map(([re]) => new RegExp(re.source, "y"));
+function keyShapedUsernameEnd(text, p) {
+    for (const re of SHAPE_STICKY) {
+        re.lastIndex = p;
+        if (re.test(text))
+            return re.lastIndex;
+    }
+    return -1;
+}
 function matchUserinfoAt(text, p, json) {
     REDACTION_TOKEN_Y.lastIndex = p;
     const tokEnd = REDACTION_TOKEN_Y.test(text) ? REDACTION_TOKEN_Y.lastIndex : -1;
     const afterTok = tokEnd >= 0 ? text.charCodeAt(tokEnd) : -1;
     if (afterTok === 64)
         return null;
-    let userEnd;
+    let userEnd = -1;
     let kind;
     if (afterTok === 58) {
         userEnd = tokEnd;
         kind = "token-pass";
     }
+    else if ((() => { const e = keyShapedUsernameEnd(text, p); return e >= 0 && text.charCodeAt(e) === 58 ? ((userEnd = e), true) : false; })()) {
+        kind = "token-pass";
+    }
     else {
         userEnd = scanUserinfoRun(text, p, "plain-user", json);
-        if (userEnd === p || text.charCodeAt(userEnd) !== 58)
+        if (text.charCodeAt(userEnd) !== 58)
             return null;
         kind = "plain-pass";
     }
@@ -534,7 +562,6 @@ function redactUrlUserinfo(text, json) {
     }
     return count === 0 ? { out: text, count: 0 } : { out: out + text.slice(last), count };
 }
-const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
 let ppkScanBytes = 0;
 export function ppkScanBytesForTest() { return ppkScanBytes; }
 export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
@@ -634,10 +661,12 @@ function redactPpkContainers(text, record) {
 }
 const urlUserinfoRule = { label: labelOfToken(URL_USERINFO_PATTERN[1]), arm: "full", run: (text, _wantCount, json) => redactUrlUserinfo(text, json) };
 const SECRET_RULES = [
-    ...FULL_SECRET_PATTERNS.map((p) => (p === URL_USERINFO_PATTERN ? urlUserinfoRule : regexRule(p, "full"))),
+    ...STRUCTURAL_SECRET_PATTERNS.map((p) => (p === URL_USERINFO_PATTERN ? urlUserinfoRule : regexRule(p, "full"))),
     { label: labelOfToken(PPK_CONTAINER_TOKEN), arm: "full", run: (text, _wantCount, _json, observe) => redactPpkContainers(text, observe) },
     { label: labelOfToken(PEM_FRAGMENT_TOKEN), arm: "fragment", run: (text, _wantCount, json) => redactPemFragments(text, json) },
     { label: labelOfToken(HEADLESS_PPK_TOKEN), arm: "fragment", run: (text) => redactHeadlessPpkBodies(text) },
+    ...KEYWORD_SECRET_PATTERNS.map((p) => regexRule(p, "full")),
+    ...SHAPE_SECRET_PATTERNS.map((p) => regexRule(p, "full")),
     ...FRAGMENT_SECRET_PATTERNS.map((p) => regexRule(p, "fragment")),
     { ...urlUserinfoRule, run: (text, _wantCount, json) => ({ out: redactUrlUserinfo(text, json).out, count: 0 }) },
 ];
@@ -670,8 +699,7 @@ function reportHits(hits) {
 export function redactSecrets(text, opts) {
     const wantCount = onRedaction !== undefined && (opts?.observe ?? true);
     const json = opts?.json === true;
-    let r;
-    r = applyRules(text, wantCount, json, opts?.observe ?? true);
+    const r = applyRules(normalizeForRedaction(text), wantCount, json, opts?.observe ?? true);
     reportHits(r.hits);
     return r.out;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sema-agent/server",
-  "version": "7.69.0",
+  "version": "7.71.0",
   "description": "Sema Server — the server/API implementation layer for Sema, wiring core, registry, model providers, and cloud agent execution. Built on @sema-agent/core.",
   "type": "module",
   "license": "BUSL-1.1",
@@ -54,7 +54,7 @@
     "build:binary:run-local:darwin-arm64": "bun build --compile --target=bun-darwin-arm64 src/run-local.ts --outfile dist/run-local-darwin-arm64"
   },
   "dependencies": {
-    "@sema-agent/core": "7.11.2",
+    "@sema-agent/core": "7.13.0",
     "@sema-agent/settings-schema": "1.10.0",
     "e2b": "^2.28.0",
     "libsodium-wrappers": "^0.8.4",
@@ -69,7 +69,7 @@
     "sharp": "^0.35.3"
   },
   "devDependencies": {
-    "@sema-agent/sdk": "8.8.0",
+    "@sema-agent/sdk": "9.1.0",
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/node": "22.10.2",
     "@types/pg": "^8.20.0",