@sema-agent/server 7.74.0 → 7.75.1

package/dist/trace/redact.js

@@ -2,18 +2,292 @@ import { stripFormatCharacters } from "@sema-agent/core";
 import { recordFailOpen } from "../observability/fail-open.js";
 export const normalizeForRedaction = stripFormatCharacters;
 const REDACTION_TOKEN_RE_SRC = String.raw `«redacted(?::[a-z-]{1,16}){0,2}»`;
+const atLeast = (cls, min) => `${cls}{${min}}${cls}*`;
 const ENTROPY_SCAN_MAX = 96;
 const twoStrongAhead = (valueClass) => String.raw `(?=${valueClass}{0,${ENTROPY_SCAN_MAX}}[0-9A-Z]${valueClass}{0,${ENTROPY_SCAN_MAX}}[0-9A-Z])`;
 const IDENT_CONNECTORS = "[_./-]";
 const noConnectorAhead = (valueClass) => String.raw `(?!${valueClass}{0,${ENTROPY_SCAN_MAX}}${IDENT_CONNECTORS}${valueClass})`;
 const WORD_INTERIOR_CLASS = "[A-Za-z0-9]";
-const ANSI_CSI = String.raw `(?:\x1b\[|\\u001[bB]\[|\x9b|\\u009[bB])[0-?]*[ -/]*[@-~]`;
-const ESCAPED_SEPARATOR_TAIL = String.raw `(?<=\\(?:[bfnrt]|u[0-9a-fA-F]{4})|${ANSI_CSI})`;
-const VALUE_ARM_LEFT_BOUNDARY = String.raw `(?:(?<!${WORD_INTERIOR_CLASS})|${ESCAPED_SEPARATOR_TAIL})`;
-const valueArm = (prefixSrc, valueClass, min) => new RegExp(`${VALUE_ARM_LEFT_BOUNDARY}${prefixSrc}${twoStrongAhead(valueClass)}${valueClass}{${min},}`, "g");
+const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_ESC = 27, CH_SPACE = 32, CH_PLUS = 43, CH_MINUS = 45, CH_SLASH = 47;
+const CH_0 = 48, CH_9 = 57, CH_COLON = 58, CH_EQ = 61, CH_AT = 64, CH_A = 65, CH_Z = 90;
+const CH_LBRACKET = 91, CH_BACKSLASH = 92, CH_UNDERSCORE = 95;
+const CH_a = 97, CH_b = 98, CH_f = 102, CH_n = 110, CH_r = 114, CH_t = 116, CH_u = 117, CH_z = 122, CH_TILDE = 126;
+const CH_QUOTE = 34;
+const CH_CSI8 = 155;
+const CH_LAQUO = 171;
+const ESC_CHAR = String.fromCharCode(CH_ESC);
+const CSI8_CHAR = String.fromCharCode(CH_CSI8);
+const WORD_INTERIOR_BYTES = (() => {
+    const one = new RegExp(`^${WORD_INTERIOR_CLASS}$`);
+    const table = new Uint8Array(256);
+    for (let c = 0; c < table.length; c++)
+        if (one.test(String.fromCharCode(c)))
+            table[c] = 1;
+    return table;
+})();
+function hexDigitValue(c) {
+    if (c >= CH_0 && c <= CH_9)
+        return c - CH_0;
+    if (c >= CH_A && c <= CH_A + 5)
+        return c - CH_A + 10;
+    if (c >= CH_a && c <= CH_a + 5)
+        return c - CH_a + 10;
+    return -1;
+}
+function c1LeadLen(s, i, code) {
+    if (s.charCodeAt(i) === code)
+        return 1;
+    if (s.charCodeAt(i) !== CH_BACKSLASH || s.charCodeAt(i + 1) !== CH_u)
+        return 0;
+    let v = 0;
+    for (let k = i + 2; k < i + 6; k++) {
+        const d = hexDigitValue(s.charCodeAt(k));
+        if (d < 0)
+            return 0;
+        v = v * 16 + d;
+    }
+    return v === code ? 6 : 0;
+}
+const CSI_PARAM_RUN = /[0-?]*/y;
+const INTERMEDIATE_RUN = /[ -/]*/y;
+function skipClassRun(re, s, p) {
+    if (p >= s.length)
+        return p;
+    re.lastIndex = p;
+    return re.exec(s) === null ? p : re.lastIndex;
+}
+function scanIntermediateChain(s, p, termLo, termHi, mark) {
+    const n = s.length;
+    for (;;) {
+        p = skipClassRun(INTERMEDIATE_RUN, s, p);
+        if (p >= n)
+            return p;
+        const c = s.charCodeAt(p);
+        if (c >= termLo && c <= termHi)
+            mark(p + 1);
+        if (c !== CH_BACKSLASH)
+            return p;
+        let k = p;
+        while (k < n && s.charCodeAt(k) === CH_BACKSLASH)
+            k += 1;
+        if (k >= n || s.charCodeAt(k) !== CH_QUOTE)
+            return p;
+        p = k + 1;
+    }
+}
+const ESCAPED_SEPARATOR_UNITS = [
+    {
+        leads: ["\\b", "\\f", "\\n", "\\r", "\\t", "\\u"],
+        scan: (s, i, mark) => {
+            if (s.charCodeAt(i) !== CH_BACKSLASH)
+                return i;
+            const c = s.charCodeAt(i + 1);
+            if (c === CH_b || c === CH_f || c === CH_n || c === CH_r || c === CH_t)
+                mark(i + 2);
+            else if (c === CH_u && hexDigitValue(s.charCodeAt(i + 2)) >= 0 && hexDigitValue(s.charCodeAt(i + 3)) >= 0
+                && hexDigitValue(s.charCodeAt(i + 4)) >= 0 && hexDigitValue(s.charCodeAt(i + 5)) >= 0)
+                mark(i + 6);
+            return i;
+        },
+    },
+    {
+        leads: [ESC_CHAR, CSI8_CHAR, "\\u"],
+        scan: (s, i, mark) => {
+            const esc = c1LeadLen(s, i, CH_ESC);
+            let p;
+            if (esc > 0) {
+                if (s.charCodeAt(i + esc) !== CH_LBRACKET)
+                    return i;
+                p = i + esc + 1;
+            }
+            else {
+                const csi8 = c1LeadLen(s, i, CH_CSI8);
+                if (csi8 === 0)
+                    return i;
+                p = i + csi8;
+            }
+            return scanIntermediateChain(s, skipClassRun(CSI_PARAM_RUN, s, p), CH_AT, CH_TILDE, mark);
+        },
+    },
+    {
+        leads: [ESC_CHAR, "\\u"],
+        scan: (s, i, mark) => {
+            const esc = c1LeadLen(s, i, CH_ESC);
+            return esc === 0 ? i : scanIntermediateChain(s, i + esc, CH_0, CH_TILDE, mark);
+        },
+    },
+];
+const ESCAPED_SEPARATOR_NEEDLES = [...new Set(ESCAPED_SEPARATOR_UNITS.flatMap((u) => u.leads))];
+const NEEDLE_FIRST_CHARS = [...new Set(ESCAPED_SEPARATOR_NEEDLES.map((n) => n.slice(0, 1)))];
+const RUN_SKIPPABLE_BYTES = (() => {
+    const table = new Uint8Array(256);
+    for (const first of NEEDLE_FIRST_CHARS) {
+        const code = first.charCodeAt(0);
+        if (code >= table.length)
+            continue;
+        const single = ESCAPED_SEPARATOR_NEEDLES.some((n) => n.length === 1 && n.charCodeAt(0) === code);
+        const selfPair = ESCAPED_SEPARATOR_NEEDLES.some((n) => n.length > 1 && n.charCodeAt(0) === code && n.charCodeAt(1) === code);
+        if (!single && !selfPair)
+            table[code] = 1;
+    }
+    return table;
+})();
+function markEscapedSeparatorTails(s) {
+    const any = anyValueArmRe();
+    let ends;
+    let ascending = true;
+    const mark = (end) => {
+        if (end > s.length)
+            return;
+        const tail = s.charCodeAt(end - 1);
+        if (tail < WORD_INTERIOR_BYTES.length && WORD_INTERIOR_BYTES[tail] !== 1)
+            return;
+        any.lastIndex = end;
+        if (any.exec(s) === null)
+            return;
+        if (ends === undefined)
+            ends = [end];
+        else {
+            if (end < ends[ends.length - 1])
+                ascending = false;
+            ends.push(end);
+        }
+    };
+    const nextAt = new Array(NEEDLE_FIRST_CHARS.length).fill(-1);
+    const nextLead = (from) => {
+        let best = Number.POSITIVE_INFINITY;
+        for (let k = 0; k < NEEDLE_FIRST_CHARS.length; k++) {
+            if (nextAt[k] < from) {
+                const hit = s.indexOf(NEEDLE_FIRST_CHARS[k], from);
+                nextAt[k] = hit === -1 ? Number.POSITIVE_INFINITY : hit;
+            }
+            if (nextAt[k] < best)
+                best = nextAt[k];
+        }
+        return best === Number.POSITIVE_INFINITY ? -1 : best;
+    };
+    const n = s.length;
+    for (let i = nextLead(0); i >= 0;) {
+        let resume = i;
+        for (const unit of ESCAPED_SEPARATOR_UNITS) {
+            const consumed = unit.scan(s, i, mark);
+            if (consumed > resume)
+                resume = consumed;
+        }
+        if (resume === i) {
+            const code = s.charCodeAt(i);
+            if (code < RUN_SKIPPABLE_BYTES.length && RUN_SKIPPABLE_BYTES[code] === 1) {
+                let j = i + 1;
+                while (j < n && s.charCodeAt(j) === code)
+                    j += 1;
+                if (j > i + 1)
+                    resume = j - 1;
+            }
+        }
+        i = nextLead(resume > i ? resume : i + 1);
+    }
+    if (ends === undefined)
+        return undefined;
+    if (!ascending)
+        ends.sort((x, y) => x - y);
+    return ends;
+}
+let escapedTailsText;
+let escapedTailsValue;
+function escapedSeparatorTails(s) {
+    if (escapedTailsText !== s) {
+        escapedTailsValue = markEscapedSeparatorTails(s);
+        escapedTailsText = s;
+    }
+    return escapedTailsValue;
+}
+function forgetEscapedSeparatorTails() {
+    escapedTailsText = undefined;
+    escapedTailsValue = undefined;
+}
+const VALUE_ARM_STICKY = new Map();
+const VALUE_ARM_BODIES = [];
+const valueArm = (prefixSrc, valueClass, min) => {
+    const body = `${prefixSrc}${twoStrongAhead(valueClass)}${atLeast(valueClass, min)}`;
+    const g = new RegExp(`(?<!${WORD_INTERIOR_CLASS})${body}`, "g");
+    VALUE_ARM_STICKY.set(g, new RegExp(body, "y"));
+    VALUE_ARM_BODIES.push(body);
+    return g;
+};
+let anyValueArm;
+function anyValueArmRe() {
+    return (anyValueArm ??= new RegExp(`(?:${VALUE_ARM_BODIES.join("|")})`, "y"));
+}
+function runValueArm(g, y, repl, text, wantCount) {
+    const tails = escapedSeparatorTails(text);
+    if (tails === undefined) {
+        if (!wantCount)
+            return { out: text.replace(g, repl), count: 0 };
+        let hits = 0;
+        const replaced = text.replace(g, () => {
+            hits += 1;
+            return repl;
+        });
+        return { out: replaced, count: hits };
+    }
+    let out;
+    let copied = 0;
+    let count = 0;
+    let pos = 0;
+    let ti = 0;
+    let bIdx = -1;
+    let bEnd = -1;
+    const nextB = (from) => {
+        while (ti < tails.length) {
+            const p = tails[ti];
+            ti += 1;
+            if (p < from)
+                continue;
+            y.lastIndex = p;
+            const m = y.exec(text);
+            if (m !== null) {
+                bIdx = p;
+                bEnd = p + m[0].length;
+                return;
+            }
+        }
+        bIdx = -1;
+    };
+    nextB(0);
+    g.lastIndex = 0;
+    let a = g.exec(text);
+    for (;;) {
+        if (a !== null && a.index < pos) {
+            g.lastIndex = pos;
+            a = g.exec(text);
+        }
+        if (bIdx >= 0 && bIdx < pos)
+            nextB(pos);
+        let idx;
+        let end;
+        if (a !== null && (bIdx < 0 || a.index <= bIdx)) {
+            idx = a.index;
+            end = a.index + a[0].length;
+        }
+        else if (bIdx >= 0) {
+            idx = bIdx;
+            end = bEnd;
+        }
+        else
+            break;
+        (out ??= []).push(text.slice(copied, idx), repl);
+        copied = end;
+        count += 1;
+        pos = end;
+    }
+    if (out === undefined)
+        return { out: text, count: 0 };
+    out.push(text.slice(copied));
+    return { out: out.join(""), count };
+}
 const WORD_SEGMENT_CLASS = "[A-Za-z0-9._~+/-]";
-const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => new RegExp(`(?:${keywordSrc}\\s+${valueClass}{${min},}${tailSrc}` +
-    `|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${valueClass}{${min},}${tailSrc})`, "g");
+const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => new RegExp(`(?:${keywordSrc}\\s+${atLeast(valueClass, min)}${tailSrc}` +
+    `|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${atLeast(valueClass, min)}${tailSrc})`, "g");
 const STRUCTURAL_SECRET_PATTERNS = [
     [/-----BEGIN(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
     [/---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]{0,8192}?---- END SSH2 ENCRYPTED PRIVATE KEY ----/g, "«redacted:private-key»"],
@@ -30,22 +304,17 @@ const KEYWORD_SECRET_PATTERNS = [
 ];
 const SHAPE_SECRET_PATTERNS = [
     [valueArm("sk-", "[A-Za-z0-9_-]", 16), "«redacted:key»"],
-    [/(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g, "«redacted:gh-token»"],
-    [/github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59,}/g, "«redacted:gh-token»"],
-    [/xox[baprs]-[A-Za-z0-9-]{10,}/g, "«redacted:slack-token»"],
-    [/AKIA[0-9A-Z]{16,}/g, "«redacted:aws-key»"],
-    [/ASIA[0-9A-Z]{16,}/g, "«redacted:aws-key»"],
-    [/(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test|org)_[A-Za-z0-9]{24,}|pypi-[A-Za-z0-9_-]{85,})/g, "«redacted:key»"],
-    [/xapp-\d-[A-Za-z0-9-]{20,}/g, "«redacted:slack-token»"],
+    [new RegExp(String.raw `(?:ghp|gho|ghu|ghs|ghr)_${atLeast("[A-Za-z0-9]", 20)}`, "g"), "«redacted:gh-token»"],
+    [new RegExp(String.raw `github_pat_[A-Za-z0-9]{22}_${atLeast("[A-Za-z0-9]", 59)}`, "g"), "«redacted:gh-token»"],
+    [new RegExp(String.raw `xox[baprs]-${atLeast("[A-Za-z0-9-]", 10)}`, "g"), "«redacted:slack-token»"],
+    [new RegExp(String.raw `AKIA${atLeast("[0-9A-Z]", 16)}`, "g"), "«redacted:aws-key»"],
+    [new RegExp(String.raw `ASIA${atLeast("[0-9A-Z]", 16)}`, "g"), "«redacted:aws-key»"],
+    [new RegExp(String.raw `(?:gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-${atLeast("[A-Za-z0-9_-]", 20)}|(?:sk|rk)_(?:live|test|org)_${atLeast("[A-Za-z0-9]", 24)}|pypi-${atLeast("[A-Za-z0-9_-]", 85)})`, "g"), "«redacted:key»"],
+    [new RegExp(String.raw `xapp-\d-${atLeast("[A-Za-z0-9-]", 20)}`, "g"), "«redacted:slack-token»"],
 ];
 const FULL_SECRET_PATTERNS = [...STRUCTURAL_SECRET_PATTERNS, ...KEYWORD_SECRET_PATTERNS, ...SHAPE_SECRET_PATTERNS];
 const PEM_HEADER_IDENT_MAX = 31;
 const PEM_BODY_RUN_MIN = 8;
-const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_SPACE = 32, CH_PLUS = 43, CH_MINUS = 45, CH_SLASH = 47;
-const CH_0 = 48, CH_9 = 57, CH_COLON = 58, CH_EQ = 61, CH_A = 65, CH_Z = 90, CH_BACKSLASH = 92, CH_UNDERSCORE = 95;
-const CH_a = 97, CH_n = 110, CH_r = 114, CH_t = 116, CH_z = 122;
-const CH_QUOTE = 34;
-const CH_LAQUO = 171;
 const isSepCode = (c) => c === CH_SPACE || c === CH_TAB || c === CH_CR || c === CH_LF;
 const isBase64Code = (c) => (c >= CH_A && c <= CH_Z) || (c >= CH_a && c <= CH_z) || (c >= CH_0 && c <= CH_9) || c === CH_PLUS || c === CH_SLASH || c === CH_EQ;
 const isIdentStartCode = (c) => (c >= CH_A && c <= CH_Z) || (c >= CH_a && c <= CH_z);
@@ -421,7 +690,7 @@ function redactHeadlessPpkBodies(text) {
 const FRAGMENT_SECRET_PATTERNS = [
     [valueArm("sk-", "[A-Za-z0-9_-]", 6), "«redacted:key:partial»"],
     [valueArm("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8), "«redacted:gh-token:partial»"],
-    [/github_pat_[A-Za-z0-9_]{8,}/g, "«redacted:gh-token:partial»"],
+    [new RegExp(String.raw `github_pat_${atLeast("[A-Za-z0-9_]", 8)}`, "g"), "«redacted:gh-token:partial»"],
     [valueArm("xox[baprs]-", "[A-Za-z0-9-]", 4), "«redacted:slack-token:partial»"],
     [valueArm("AKIA", "[0-9A-Z]", 6), "«redacted:aws-key:partial»"],
     [valueArm("ASIA", "[0-9A-Z]", 8), "«redacted:aws-key:partial»"],
@@ -454,10 +723,15 @@ const regexRule = ([re, repl], arm) => {
             throw new Error(`redact: replacement ${JSON.stringify(repl)} uses unsupported ref $${d}`);
     }
     const expand = refs.length === 0 ? undefined : (args) => repl.replace(/\$([1-9])/g, (_, d) => args[Number(d)] ?? "");
+    const sticky = VALUE_ARM_STICKY.get(re);
+    if (sticky !== undefined && refs.length > 0)
+        throw new Error(`redact: value arm replacement ${JSON.stringify(repl)} must not use $N refs`);
     return {
         label: labelOfToken(repl),
         arm,
         run: (text, wantCount, json) => {
+            if (sticky !== undefined)
+                return runValueArm(re, sticky, repl, text, wantCount);
             const rx = re;
             if (!wantCount)
                 return { out: text.replace(rx, repl), count: 0 };
@@ -704,6 +978,7 @@ export function redactSecrets(text, opts) {
     const wantCount = onRedaction !== undefined && (opts?.observe ?? true);
     const json = opts?.json === true;
     const r = applyRules(normalizeForRedaction(text), wantCount, json, opts?.observe ?? true);
+    forgetEscapedSeparatorTails();
     reportHits(r.hits);
     return r.out;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sema-agent/server",
-  "version": "7.74.0",
+  "version": "7.75.1",
   "description": "Sema Server — the server/API implementation layer for Sema, wiring core, registry, model providers, and cloud agent execution. Built on @sema-agent/core.",
   "type": "module",
   "license": "BUSL-1.1",
@@ -54,7 +54,7 @@
     "build:binary:run-local:darwin-arm64": "bun build --compile --target=bun-darwin-arm64 src/run-local.ts --outfile dist/run-local-darwin-arm64"
   },
   "dependencies": {
-    "@sema-agent/core": "7.17.1",
+    "@sema-agent/core": "7.17.4",
     "@sema-agent/settings-schema": "1.10.0",
     "e2b": "^2.28.0",
     "libsodium-wrappers": "^0.8.4",
@@ -69,7 +69,7 @@
     "sharp": "^0.35.3"
   },
   "devDependencies": {
-    "@sema-agent/sdk": "9.1.0",
+    "@sema-agent/sdk": "9.3.0",
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/node": "22.10.2",
     "@types/pg": "^8.20.0",