@forwardimpact/libeval 0.1.31 → 0.1.32

package/README.md

@@ -12,3 +12,23 @@ reproducible evidence.
 ```js
 import { createTraceCollector, createTraceQuery, createAgentRunner } from '@forwardimpact/libeval';
 ```
+
+## Trace redaction
+
+`fit-eval run`, `fit-eval supervise`, and `fit-eval facilitate` redact
+secrets in trace artifacts before they reach disk. Two layers compose:
+
+- **Env-var allowlist**, defaulting to `ANTHROPIC_API_KEY`, `GH_TOKEN`,
+  `GITHUB_TOKEN`. The runtime values of these vars are replaced with
+  `[REDACTED:env:NAME]` wherever they appear in tool inputs, tool
+  outputs, assistant text, or orchestrator summaries. Override the list
+  with `LIBEVAL_REDACTION_ENV_VARS=NAME1,NAME2,…` (replaces, not extends).
+- **Credential-shape patterns**, covering Anthropic API keys (`sk-ant-`),
+  GitHub PATs (`ghp_`), installation tokens (`ghs_`), OAuth tokens
+  (`gho_`), and fine-grained PATs (`github_pat_`). Pattern hits become
+  `[REDACTED:pattern:KIND]`.
+
+Redaction is on by default. To disable, set `LIBEVAL_REDACTION_DISABLED=1`
+— a stderr warning fires once per run. Never set this in CI on a public
+repository: workflow artifacts there are downloadable through the
+retention window.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.31",
+  "version": "0.1.32",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/agent-runner.js

@@ -54,7 +54,9 @@ export class AgentRunner {
     if (!deps.cwd) throw new Error("cwd is required");
     if (!deps.query) throw new Error("query is required");
     if (!deps.output) throw new Error("output is required");
+    if (!deps.redactor) throw new Error("redactor is required");
     Object.assign(this, applyDefaults(deps));
+    this.redactor = deps.redactor;
     this.sessionId = null;
     this.buffer = [];
     /** @type {AbortController|null} */
@@ -203,12 +205,16 @@ export class AgentRunner {
    * @param {{pendingBatch: string[], assistantTextCount: number}} state
    */
   #recordLine(message, state) {
-    const line = JSON.stringify(message);
+    const redacted = this.redactor.redactValue(message);
+    const line = JSON.stringify(redacted);
     this.output.write(line + "\n");
     this.buffer.push(line);
     if (this.onLine) this.onLine(line);
     if (this.onBatch) state.pendingBatch.push(line);
 
+    // Session-id / text-block tracking reads the ORIGINAL message —
+    // these fields are not secret carriers, and the trackers rely on
+    // shape, not string contents.
     if (message.type === "system" && message.subtype === "init") {
       this.sessionId = message.session_id;
     }

package/src/commands/facilitate.js

@@ -1,6 +1,7 @@
 import { readFileSync, createWriteStream } from "node:fs";
 import { resolve } from "node:path";
 import { createFacilitator } from "../facilitator.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 
 /**
@@ -62,6 +63,11 @@ function parseFacilitateOptions(values) {
 export async function runFacilitateCommand(values, _args) {
   const opts = parseFacilitateOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   const fileStream = opts.outputPath
     ? createWriteStream(opts.outputPath)
     : null;
@@ -87,6 +93,7 @@ export async function runFacilitateCommand(values, _args) {
     maxTurns: opts.maxTurns,
     facilitatorProfile: opts.facilitatorProfile,
     taskAmend: opts.taskAmend,
+    redactor,
   });
 
   const result = await facilitator.run(opts.taskContent);

package/src/commands/run.js

@@ -3,6 +3,7 @@ import { Writable } from "node:stream";
 import { resolve } from "node:path";
 import { createAgentRunner } from "../agent-runner.js";
 import { composeProfilePrompt } from "../profile-prompt.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 import { SequenceCounter } from "../sequence-counter.js";
 import { createServiceConfig } from "@forwardimpact/libconfig";
@@ -61,6 +62,11 @@ export async function runRunCommand(values, _args) {
     mcpServer,
   } = parseRunOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).
   const fileStream = outputPath ? createWriteStream(outputPath) : null;
@@ -76,9 +82,8 @@ export async function runRunCommand(values, _args) {
   });
   const onLine = (line) => {
     const event = JSON.parse(line);
-    output.write(
-      JSON.stringify({ source: "agent", seq: counter.next(), event }) + "\n",
-    );
+    const tagged = { source: "agent", seq: counter.next(), event };
+    output.write(JSON.stringify(redactor.redactValue(tagged)) + "\n");
   };
 
   let mcpServers = null;
@@ -117,6 +122,7 @@ export async function runRunCommand(values, _args) {
     systemPrompt,
     taskAmend,
     mcpServers,
+    redactor,
   });
 
   const result = await runner.run(taskContent);

package/src/commands/supervise.js

@@ -2,6 +2,7 @@ import { readFileSync, createWriteStream, mkdtempSync } from "node:fs";
 import { resolve, join } from "node:path";
 import { tmpdir } from "node:os";
 import { createSupervisor } from "../supervisor.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 import { createServiceConfig } from "@forwardimpact/libconfig";
 
@@ -60,6 +61,11 @@ function parseSuperviseOptions(values) {
 export async function runSuperviseCommand(values, _args) {
   const opts = parseSuperviseOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).
   const fileStream = opts.outputPath
@@ -104,6 +110,7 @@ export async function runSuperviseCommand(values, _args) {
     agentProfile: opts.agentProfile,
     taskAmend: opts.taskAmend,
     agentMcpServers,
+    redactor,
   });
 
   const result = await supervisor.run(opts.taskContent);

package/src/facilitator.js

@@ -59,7 +59,10 @@ export class Facilitator {
     ctx,
     eventQueue,
     taskAmend,
+    redactor,
   }) {
+    if (!redactor) throw new Error("redactor is required");
+    this.redactor = redactor;
     this.facilitatorRunner = facilitatorRunner;
     this.agents = agents;
     this.messageBus = messageBus;
@@ -327,11 +330,13 @@ export class Facilitator {
   emitLine(source, line) {
     const event = JSON.parse(line);
     this.output.write(
-      JSON.stringify({
-        source,
-        seq: this.counter.next(),
-        event,
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source,
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
     );
   }
 
@@ -340,11 +345,13 @@ export class Facilitator {
    */
   emitOrchestratorEvent(event) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event,
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
     );
   }
 
@@ -353,17 +360,19 @@ export class Facilitator {
    */
   emitSummary(result) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event: {
-          type: "summary",
-          success: result.success,
-          ...(result.verdict && { verdict: result.verdict }),
-          turns: result.turns,
-          ...(result.summary && { summary: result.summary }),
-        },
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
     );
   }
 }
@@ -398,7 +407,9 @@ export function createFacilitator({
   facilitatorProfile,
   profilesDir,
   taskAmend,
+  redactor,
 }) {
+  if (!redactor) throw new Error("redactor is required");
   const resolvedProfilesDir =
     profilesDir ?? resolve(facilitatorCwd, ".claude/agents");
   const systemPromptFor = (profile, trailer) => {
@@ -446,6 +457,7 @@ export function createFacilitator({
       mcpServers: { orchestration: agentServer },
       settingSources: ["project"],
       systemPrompt: systemPromptFor(config.agentProfile, agentTrailer),
+      redactor,
     });
 
     return { name: config.name, role: config.role, runner };
@@ -464,6 +476,7 @@ export function createFacilitator({
       facilitatorProfile,
       FACILITATOR_SYSTEM_PROMPT,
     ),
+    redactor,
   });
 
   facilitator = new Facilitator({
@@ -475,6 +488,7 @@ export function createFacilitator({
     ctx,
     eventQueue,
     taskAmend,
+    redactor,
   });
   return facilitator;
 }

package/src/index.js

@@ -31,3 +31,10 @@ export {
   FACILITATOR_SYSTEM_PROMPT,
   FACILITATED_AGENT_SYSTEM_PROMPT,
 } from "./facilitator.js";
+export {
+  Redactor,
+  createRedactor,
+  createNoopRedactor,
+  DEFAULT_ENV_ALLOWLIST,
+  DEFAULT_PATTERNS,
+} from "./redaction.js";

package/src/redaction.js

@@ -0,0 +1,163 @@
+/**
+ * Redactor — replaces secrets in JSON-serialisable values before they reach
+ * the trace artifact. Composes two layers: an env-var value allowlist and a
+ * set of credential-shape regexes. Both run on every primitive string.
+ *
+ * Stateless after construction: `env` is captured once so in-process
+ * `process.env` writes (e.g. agent-runner.js LIBEVAL_SKILL, commands/run.js
+ * LIBEVAL_AGENT_PROFILE) cannot smuggle a value past the redactor.
+ */
+
+export const DEFAULT_ENV_ALLOWLIST = Object.freeze([
+  "ANTHROPIC_API_KEY",
+  "GH_TOKEN",
+  "GITHUB_TOKEN",
+]);
+
+// Anchored prefixes per
+// https://github.blog/security/application-security/behind-githubs-new-authentication-token-formats/
+// Anthropic prefix is heuristic — the env-allowlist layer is the primary
+// defence for Anthropic keys.
+export const DEFAULT_PATTERNS = Object.freeze([
+  { kind: "anthropic", regex: /sk-ant-[A-Za-z0-9_-]{80,}/g },
+  { kind: "gh-pat", regex: /\bghp_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-installation", regex: /\bghs_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-oauth", regex: /\bgho_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-fine-grained", regex: /\bgithub_pat_[A-Za-z0-9_]{82}\b/g },
+]);
+
+const ENV_PLACEHOLDER = (name) => `[REDACTED:env:${name}]`;
+const PATTERN_PLACEHOLDER = (kind) => `[REDACTED:pattern:${kind}]`;
+
+/**
+ * Build a frozen { name → value } snapshot of the requested env vars.
+ * Empty strings are skipped — a leaked empty env var would otherwise
+ * cause every empty string in the trace to be replaced.
+ */
+function snapshotEnv(env, allowlist) {
+  const snap = {};
+  for (const name of allowlist) {
+    const v = env[name];
+    if (typeof v === "string" && v.length > 0) snap[name] = v;
+  }
+  return Object.freeze(snap);
+}
+
+/** Recursively walk and redact a JSON-serialisable value in place-free style. */
+function walk(value, redactString) {
+  if (typeof value === "string") return redactString(value);
+  if (Array.isArray(value)) return value.map((v) => walk(v, redactString));
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const k of Object.keys(value)) out[k] = walk(value[k], redactString);
+    return out;
+  }
+  return value;
+}
+
+/** Stateless secret redactor — composes env-allowlist and pattern layers. */
+export class Redactor {
+  /**
+   * @param {object} deps
+   * @param {Readonly<Record<string, string>>} deps.envSnapshot - Frozen { name → secret } map captured at construction time.
+   * @param {ReadonlyArray<{kind: string, regex: RegExp}>} deps.patterns - Credential-shape regexes; each match becomes `[REDACTED:pattern:KIND]`.
+   * @param {boolean} deps.enabled - When false, `redactValue` returns its input by reference.
+   */
+  constructor({ envSnapshot, patterns, enabled }) {
+    this.envSnapshot = envSnapshot;
+    this.patterns = patterns;
+    this.enabled = enabled;
+  }
+
+  /**
+   * Redact any JSON-serialisable value by deep-walking and replacing secrets
+   * in every primitive string. Identity on the input when disabled.
+   * @param {unknown} value
+   * @returns {unknown}
+   */
+  redactValue(value) {
+    if (!this.enabled) return value;
+    return walk(value, (s) => this.#redactString(s));
+  }
+
+  /**
+   * Apply the env-allowlist and pattern layers to a single string.
+   * @param {string} s
+   * @returns {string}
+   */
+  #redactString(s) {
+    let out = s;
+    for (const [name, secret] of Object.entries(this.envSnapshot)) {
+      if (out.includes(secret)) {
+        out = out.split(secret).join(ENV_PLACEHOLDER(name));
+      }
+    }
+    for (const { kind, regex } of this.patterns) {
+      out = out.replace(regex, PATTERN_PLACEHOLDER(kind));
+    }
+    return out;
+  }
+}
+
+/**
+ * Build a redactor. Reads `LIBEVAL_REDACTION_DISABLED` and
+ * `LIBEVAL_REDACTION_ENV_VARS` from the supplied env (defaults to
+ * `process.env`). Fires a one-shot stderr warning when constructed
+ * disabled — bypass via `createNoopRedactor()` for silent fixtures.
+ * @param {object} [opts]
+ * @param {Record<string, string|undefined>} [opts.env] - Environment to snapshot. Defaults to `process.env`.
+ * @param {string[]} [opts.allowlist] - Override the env-var name list. Defaults to `DEFAULT_ENV_ALLOWLIST` or the parsed `LIBEVAL_REDACTION_ENV_VARS` value.
+ * @param {ReadonlyArray<{kind: string, regex: RegExp}>} [opts.patterns] - Credential-shape regexes. Defaults to `DEFAULT_PATTERNS`.
+ * @param {boolean} [opts.enabled] - Force enabled/disabled; bypasses `LIBEVAL_REDACTION_DISABLED`.
+ * @returns {Redactor}
+ */
+export function createRedactor({
+  env = process.env,
+  allowlist,
+  patterns = DEFAULT_PATTERNS,
+  enabled,
+} = {}) {
+  const envDisabled = env.LIBEVAL_REDACTION_DISABLED === "1";
+  const resolvedEnabled = enabled ?? !envDisabled;
+  const resolvedAllowlist = allowlist ?? resolveAllowlistFromEnv(env);
+  const envSnapshot = resolvedEnabled
+    ? snapshotEnv(env, resolvedAllowlist)
+    : Object.freeze({});
+  if (!resolvedEnabled) {
+    process.stderr.write(
+      "libeval: trace redaction DISABLED via LIBEVAL_REDACTION_DISABLED — secrets may appear in trace artifact\n",
+    );
+  }
+  return new Redactor({ envSnapshot, patterns, enabled: resolvedEnabled });
+}
+
+/**
+ * Parse `LIBEVAL_REDACTION_ENV_VARS` into a trimmed, non-empty name list.
+ * Falls back to `DEFAULT_ENV_ALLOWLIST` when unset or empty.
+ * @param {Record<string, string|undefined>} env
+ * @returns {string[]}
+ */
+function resolveAllowlistFromEnv(env) {
+  const override = env.LIBEVAL_REDACTION_ENV_VARS;
+  if (typeof override !== "string" || override.length === 0) {
+    return DEFAULT_ENV_ALLOWLIST;
+  }
+  return override
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+/**
+ * Build a disabled redactor whose `redactValue` is the identity function.
+ * Test-fixture form — bypasses `createRedactor` so no stderr warning
+ * fires regardless of env state.
+ * @returns {Redactor}
+ */
+export function createNoopRedactor() {
+  return new Redactor({
+    envSnapshot: Object.freeze({}),
+    patterns: [],
+    enabled: false,
+  });
+}

package/src/supervisor.js

@@ -74,10 +74,13 @@ export class Supervisor {
     ctx,
     messageBus,
     taskAmend,
+    redactor,
   }) {
     if (!agentRunner) throw new Error("agentRunner is required");
     if (!supervisorRunner) throw new Error("supervisorRunner is required");
     if (!output) throw new Error("output is required");
+    if (!redactor) throw new Error("redactor is required");
+    this.redactor = redactor;
     this.agentRunner = agentRunner;
     this.supervisorRunner = supervisorRunner;
     this.output = output;
@@ -406,7 +409,7 @@ export class Supervisor {
       seq: this.counter.next(),
       event,
     };
-    this.output.write(JSON.stringify(tagged) + "\n");
+    this.output.write(JSON.stringify(this.redactor.redactValue(tagged)) + "\n");
   }
 
   /**
@@ -429,11 +432,13 @@ export class Supervisor {
    */
   emitOrchestratorEvent(event) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event,
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
     );
   }
 
@@ -443,17 +448,19 @@ export class Supervisor {
    */
   emitSummary(result) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event: {
-          type: "summary",
-          success: result.success,
-          ...(result.verdict && { verdict: result.verdict }),
-          turns: result.turns,
-          ...(result.summary && { summary: result.summary }),
-        },
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
     );
   }
 }
@@ -498,7 +505,9 @@ export function createSupervisor({
   profilesDir,
   taskAmend,
   agentMcpServers,
+  redactor,
 }) {
+  if (!redactor) throw new Error("redactor is required");
   const resolvedProfilesDir =
     profilesDir ?? resolve(supervisorCwd, ".claude/agents");
   const systemPromptFor = (profile, trailer) => {
@@ -538,6 +547,7 @@ export function createSupervisor({
     settingSources: ["project"],
     systemPrompt: systemPromptFor(agentProfile, AGENT_SYSTEM_PROMPT),
     mcpServers: { orchestration: agentServer, ...agentMcpServers },
+    redactor,
   });
 
   const defaultDisallowed = ["Agent", "Task", "TaskOutput", "TaskStop"];
@@ -564,6 +574,7 @@ export function createSupervisor({
     settingSources: ["project"],
     systemPrompt: systemPromptFor(supervisorProfile, SUPERVISOR_SYSTEM_PROMPT),
     mcpServers: { orchestration: supervisorServer },
+    redactor,
   });
 
   supervisor = new Supervisor({
@@ -574,6 +585,7 @@ export function createSupervisor({
     ctx,
     messageBus,
     taskAmend,
+    redactor,
   });
   return supervisor;
 }