@forwardimpact/libeval 0.1.31 → 0.1.33

package/src/commands/run.js

@@ -3,6 +3,7 @@ import { Writable } from "node:stream";
 import { resolve } from "node:path";
 import { createAgentRunner } from "../agent-runner.js";
 import { composeProfilePrompt } from "../profile-prompt.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 import { SequenceCounter } from "../sequence-counter.js";
 import { createServiceConfig } from "@forwardimpact/libconfig";
@@ -61,6 +62,11 @@ export async function runRunCommand(values, _args) {
     mcpServer,
   } = parseRunOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).
   const fileStream = outputPath ? createWriteStream(outputPath) : null;
@@ -76,9 +82,8 @@ export async function runRunCommand(values, _args) {
   });
   const onLine = (line) => {
     const event = JSON.parse(line);
-    output.write(
-      JSON.stringify({ source: "agent", seq: counter.next(), event }) + "\n",
-    );
+    const tagged = { source: "agent", seq: counter.next(), event };
+    output.write(JSON.stringify(redactor.redactValue(tagged)) + "\n");
   };
 
   let mcpServers = null;
@@ -117,6 +122,7 @@ export async function runRunCommand(values, _args) {
     systemPrompt,
     taskAmend,
     mcpServers,
+    redactor,
   });
 
   const result = await runner.run(taskContent);

package/src/commands/supervise.js

@@ -2,6 +2,7 @@ import { readFileSync, createWriteStream, mkdtempSync } from "node:fs";
 import { resolve, join } from "node:path";
 import { tmpdir } from "node:os";
 import { createSupervisor } from "../supervisor.js";
+import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
 import { createServiceConfig } from "@forwardimpact/libconfig";
 
@@ -60,6 +61,11 @@ function parseSuperviseOptions(values) {
 export async function runSuperviseCommand(values, _args) {
   const opts = parseSuperviseOptions(values);
 
+  // Build the redactor as the first observable side-effect after option
+  // parsing — the env snapshot must freeze BEFORE any in-process
+  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor();
+
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).
   const fileStream = opts.outputPath
@@ -104,6 +110,7 @@ export async function runSuperviseCommand(values, _args) {
     agentProfile: opts.agentProfile,
     taskAmend: opts.taskAmend,
     agentMcpServers,
+    redactor,
   });
 
   const result = await supervisor.run(opts.taskContent);

package/src/facilitator.js

@@ -59,7 +59,10 @@ export class Facilitator {
     ctx,
     eventQueue,
     taskAmend,
+    redactor,
   }) {
+    if (!redactor) throw new Error("redactor is required");
+    this.redactor = redactor;
     this.facilitatorRunner = facilitatorRunner;
     this.agents = agents;
     this.messageBus = messageBus;
@@ -327,11 +330,13 @@ export class Facilitator {
   emitLine(source, line) {
     const event = JSON.parse(line);
     this.output.write(
-      JSON.stringify({
-        source,
-        seq: this.counter.next(),
-        event,
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source,
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
     );
   }
 
@@ -340,11 +345,13 @@ export class Facilitator {
    */
   emitOrchestratorEvent(event) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event,
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
     );
   }
 
@@ -353,17 +360,19 @@ export class Facilitator {
    */
   emitSummary(result) {
     this.output.write(
-      JSON.stringify({
-        source: "orchestrator",
-        seq: this.counter.next(),
-        event: {
-          type: "summary",
-          success: result.success,
-          ...(result.verdict && { verdict: result.verdict }),
-          turns: result.turns,
-          ...(result.summary && { summary: result.summary }),
-        },
-      }) + "\n",
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
     );
   }
 }
@@ -398,7 +407,9 @@ export function createFacilitator({
   facilitatorProfile,
   profilesDir,
   taskAmend,
+  redactor,
 }) {
+  if (!redactor) throw new Error("redactor is required");
   const resolvedProfilesDir =
     profilesDir ?? resolve(facilitatorCwd, ".claude/agents");
   const systemPromptFor = (profile, trailer) => {
@@ -446,6 +457,7 @@ export function createFacilitator({
       mcpServers: { orchestration: agentServer },
       settingSources: ["project"],
       systemPrompt: systemPromptFor(config.agentProfile, agentTrailer),
+      redactor,
     });
 
     return { name: config.name, role: config.role, runner };
@@ -464,6 +476,7 @@ export function createFacilitator({
       facilitatorProfile,
       FACILITATOR_SYSTEM_PROMPT,
     ),
+    redactor,
   });
 
   facilitator = new Facilitator({
@@ -475,6 +488,7 @@ export function createFacilitator({
     ctx,
     eventQueue,
     taskAmend,
+    redactor,
   });
   return facilitator;
 }

package/src/index.js

@@ -23,6 +23,7 @@ export {
   createSupervisedAgentToolServer,
   createFacilitatorToolServer,
   createFacilitatedAgentToolServer,
+  createJudgeToolServer,
 } from "./orchestration-toolkit.js";
 export { MessageBus, createMessageBus } from "./message-bus.js";
 export {
@@ -31,3 +32,11 @@ export {
   FACILITATOR_SYSTEM_PROMPT,
   FACILITATED_AGENT_SYSTEM_PROMPT,
 } from "./facilitator.js";
+export { Judge, createJudge, JUDGE_SYSTEM_PROMPT } from "./judge.js";
+export {
+  Redactor,
+  createRedactor,
+  createNoopRedactor,
+  DEFAULT_ENV_ALLOWLIST,
+  DEFAULT_PATTERNS,
+} from "./redaction.js";

package/src/judge.js

@@ -0,0 +1,211 @@
+/**
+ * Judge — one agent session that inspects a completed agent's work and emits
+ * a verdict via the orchestration `Conclude` tool. Parallel concept to
+ * `Supervisor` and `Facilitator`, but post-hoc and solo: no peer agents,
+ * no message bus, no relay loop. The judge reads the task, optionally
+ * inspects the working directory and trace via read-only tools, and calls
+ * Conclude exactly once.
+ *
+ * Trace lines are tagged `source: "judge"` so consumers can distinguish
+ * judge sessions from supervisor or facilitator sessions in a unified
+ * NDJSON envelope.
+ *
+ * Follows OO+DI: constructor injection, factory function, tests bypass factory.
+ */
+
+import { resolve } from "node:path";
+import { Writable } from "node:stream";
+
+import { createAgentRunner } from "./agent-runner.js";
+import { composeProfilePrompt } from "./profile-prompt.js";
+import { SequenceCounter } from "./sequence-counter.js";
+import {
+  createJudgeToolServer,
+  createOrchestrationContext,
+} from "./orchestration-toolkit.js";
+
+/**
+ * System-prompt trailer appended to the judge's main thread. Always applied,
+ * even when a `judgeProfile` is supplied — the profile layers on top of the
+ * trailer, the same way `SUPERVISOR_SYSTEM_PROMPT` and
+ * `FACILITATOR_SYSTEM_PROMPT` work for their respective roles.
+ */
+export const JUDGE_SYSTEM_PROMPT =
+  "You are a post-hoc judge for an agent task benchmark. " +
+  "The agent has already completed its work and an objective scoring step has already run; your role is to confirm or override the verdict by inspecting the agent's working directory and trace. " +
+  "You have read-only inspection tools — Read, Glob, Grep, Bash — to investigate; do not modify the working directory. " +
+  "Conclude ends the session with a verdict ('success' or 'failure') and a one-paragraph summary; verdict='success' iff the agent's work meets the criteria stated in the task. " +
+  "Call Conclude as your final action — do not deliberate across multiple turns.";
+
+const DEFAULT_JUDGE_ALLOWED_TOOLS = ["Read", "Glob", "Grep", "Bash"];
+
+const devNull = new Writable({
+  write(_chunk, _enc, cb) {
+    cb();
+  },
+});
+
+/** Run a single post-hoc judge session and emit a verdict via Conclude. */
+export class Judge {
+  /**
+   * @param {object} deps
+   * @param {import("./agent-runner.js").AgentRunner} deps.runner - The judge's AgentRunner.
+   * @param {import("stream").Writable} deps.output - Stream to emit tagged NDJSON to.
+   * @param {object} deps.ctx - Orchestration context (the Conclude handler writes to it).
+   * @param {import("./redaction.js").Redactor} deps.redactor
+   * @param {string} [deps.taskAmend] - Opaque addendum appended to the task before delivery.
+   */
+  constructor({ runner, output, ctx, redactor, taskAmend }) {
+    if (!runner) throw new Error("runner is required");
+    if (!output) throw new Error("output is required");
+    if (!ctx) throw new Error("ctx is required");
+    if (!redactor) throw new Error("redactor is required");
+    this.runner = runner;
+    this.output = output;
+    this.ctx = ctx;
+    this.redactor = redactor;
+    this.taskAmend = taskAmend ?? null;
+    this.counter = new SequenceCounter();
+  }
+
+  /**
+   * Run the judge session.
+   * @param {string} task - The judge prompt (with placeholders already substituted).
+   * @returns {Promise<{success: boolean, verdict: string|null, summary: string|null, turns: number}>}
+   */
+  async run(task) {
+    const fullTask = this.taskAmend ? `${task}\n\n${this.taskAmend}` : task;
+    const result = await this.runner.run(fullTask);
+
+    if (this.ctx.concluded) {
+      const success = this.ctx.verdict === "success";
+      const outcome = {
+        success,
+        verdict: this.ctx.verdict,
+        summary: this.ctx.summary ?? null,
+        turns: 1,
+      };
+      this.emitSummary(outcome);
+      return outcome;
+    }
+
+    // The judge ended without calling Conclude. Surface that explicitly so
+    // callers can distinguish "judge said fail" from "judge never voted."
+    const outcome = {
+      success: false,
+      verdict: null,
+      summary: null,
+      turns: result.success ? 1 : 0,
+    };
+    this.emitSummary(outcome);
+    return outcome;
+  }
+
+  /**
+   * Tag a single NDJSON line with `source: "judge"` and emit it to the
+   * judge's output stream. Wired into the underlying AgentRunner via the
+   * `onLine` callback so the judge's stream is the single source of truth
+   * for the session's trace.
+   * @param {string} line
+   */
+  emitLine(line) {
+    const event = JSON.parse(line);
+    const tagged = { source: "judge", seq: this.counter.next(), event };
+    this.output.write(JSON.stringify(this.redactor.redactValue(tagged)) + "\n");
+  }
+
+  /**
+   * Emit a final orchestrator summary line, wrapped in the universal envelope.
+   * @param {{success: boolean, verdict?: string|null, summary?: string|null, turns: number}} result
+   */
+  emitSummary(result) {
+    this.output.write(
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
+    );
+  }
+}
+
+/**
+ * Factory function — wires the AgentRunner with the judge orchestration server
+ * and the JUDGE_SYSTEM_PROMPT trailer. A `judgeProfile` (when supplied) layers
+ * on top of the trailer via `composeProfilePrompt`, matching the
+ * supervisor/facilitator pattern.
+ *
+ * @param {object} deps
+ * @param {string} deps.cwd - Judge working directory. Defaults to the directory whose `.claude/agents` holds `judgeProfile`.
+ * @param {function} deps.query - SDK query function (injected for testing).
+ * @param {import("stream").Writable} deps.output - Trace output stream.
+ * @param {import("./redaction.js").Redactor} deps.redactor
+ * @param {string} [deps.model]
+ * @param {number} [deps.maxTurns] - Default 5 (the judge is expected to act in turn 1; 5 leaves headroom for tool inspection).
+ * @param {string[]} [deps.allowedTools] - Default `["Read","Glob","Grep","Bash"]` — read-only inspection.
+ * @param {string} [deps.judgeProfile] - Profile name; resolved into the system prompt via `composeProfilePrompt`.
+ * @param {string} [deps.profilesDir] - Defaults to `<cwd>/.claude/agents`.
+ * @param {string} [deps.taskAmend]
+ * @returns {Judge}
+ */
+export function createJudge({
+  cwd,
+  query,
+  output,
+  redactor,
+  model,
+  maxTurns,
+  allowedTools,
+  judgeProfile,
+  profilesDir,
+  taskAmend,
+}) {
+  if (!cwd) throw new Error("cwd is required");
+  if (!query) throw new Error("query is required");
+  if (!output) throw new Error("output is required");
+  if (!redactor) throw new Error("redactor is required");
+
+  const resolvedProfilesDir = profilesDir ?? resolve(cwd, ".claude/agents");
+  const systemPrompt = judgeProfile
+    ? composeProfilePrompt(judgeProfile, {
+        profilesDir: resolvedProfilesDir,
+        trailer: JUDGE_SYSTEM_PROMPT,
+      })
+    : {
+        type: "preset",
+        preset: "claude_code",
+        append: JUDGE_SYSTEM_PROMPT,
+      };
+
+  const ctx = createOrchestrationContext();
+  ctx.participants = [{ name: "judge", role: "judge" }];
+  const judgeServer = createJudgeToolServer(ctx);
+
+  let judge;
+  const onLine = (line) => judge.emitLine(line);
+
+  const runner = createAgentRunner({
+    cwd,
+    query,
+    output: devNull,
+    model,
+    maxTurns: maxTurns ?? 5,
+    allowedTools: allowedTools ?? DEFAULT_JUDGE_ALLOWED_TOOLS,
+    onLine,
+    settingSources: ["project"],
+    systemPrompt,
+    mcpServers: { orchestration: judgeServer },
+    redactor,
+  });
+
+  judge = new Judge({ runner, output, ctx, redactor, taskAmend });
+  return judge;
+}

package/src/orchestration-toolkit.js

@@ -279,6 +279,31 @@ export function createSupervisedAgentToolServer(ctx) {
   });
 }
 
+/**
+ * Judge tools: Conclude only.
+ *
+ * The judge runs a single post-hoc session with no peer participants —
+ * Ask/Answer/Announce/Redirect/RollCall are all moot. The judge inspects
+ * the agent's working directory and trace via the host's read-only tools
+ * and emits its verdict via Conclude.
+ *
+ * @param {object} ctx - Orchestration context
+ * @returns {object} MCP server config (type: "sdk")
+ */
+export function createJudgeToolServer(ctx) {
+  return createSdkMcpServer({
+    name: "orchestration",
+    tools: [
+      tool(
+        "Conclude",
+        "End the session with a verdict and a summary. verdict='success' if the agent's work meets the criteria stated in the task; 'failure' otherwise.",
+        { verdict: z.enum(["success", "failure"]), summary: z.string() },
+        createConcludeHandler(ctx),
+      ),
+    ],
+  });
+}
+
 /**
  * Facilitator tools: Ask + Announce + Conclude + RollCall.
  *

package/src/redaction.js

@@ -0,0 +1,163 @@
+/**
+ * Redactor — replaces secrets in JSON-serialisable values before they reach
+ * the trace artifact. Composes two layers: an env-var value allowlist and a
+ * set of credential-shape regexes. Both run on every primitive string.
+ *
+ * Stateless after construction: `env` is captured once so in-process
+ * `process.env` writes (e.g. agent-runner.js LIBEVAL_SKILL, commands/run.js
+ * LIBEVAL_AGENT_PROFILE) cannot smuggle a value past the redactor.
+ */
+
+export const DEFAULT_ENV_ALLOWLIST = Object.freeze([
+  "ANTHROPIC_API_KEY",
+  "GH_TOKEN",
+  "GITHUB_TOKEN",
+]);
+
+// Anchored prefixes per
+// https://github.blog/security/application-security/behind-githubs-new-authentication-token-formats/
+// Anthropic prefix is heuristic — the env-allowlist layer is the primary
+// defence for Anthropic keys.
+export const DEFAULT_PATTERNS = Object.freeze([
+  { kind: "anthropic", regex: /sk-ant-[A-Za-z0-9_-]{80,}/g },
+  { kind: "gh-pat", regex: /\bghp_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-installation", regex: /\bghs_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-oauth", regex: /\bgho_[A-Za-z0-9]{36}\b/g },
+  { kind: "gh-fine-grained", regex: /\bgithub_pat_[A-Za-z0-9_]{82}\b/g },
+]);
+
+const ENV_PLACEHOLDER = (name) => `[REDACTED:env:${name}]`;
+const PATTERN_PLACEHOLDER = (kind) => `[REDACTED:pattern:${kind}]`;
+
+/**
+ * Build a frozen { name → value } snapshot of the requested env vars.
+ * Empty strings are skipped — a leaked empty env var would otherwise
+ * cause every empty string in the trace to be replaced.
+ */
+function snapshotEnv(env, allowlist) {
+  const snap = {};
+  for (const name of allowlist) {
+    const v = env[name];
+    if (typeof v === "string" && v.length > 0) snap[name] = v;
+  }
+  return Object.freeze(snap);
+}
+
+/** Recursively walk and redact a JSON-serialisable value in place-free style. */
+function walk(value, redactString) {
+  if (typeof value === "string") return redactString(value);
+  if (Array.isArray(value)) return value.map((v) => walk(v, redactString));
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const k of Object.keys(value)) out[k] = walk(value[k], redactString);
+    return out;
+  }
+  return value;
+}
+
+/** Stateless secret redactor — composes env-allowlist and pattern layers. */
+export class Redactor {
+  /**
+   * @param {object} deps
+   * @param {Readonly<Record<string, string>>} deps.envSnapshot - Frozen { name → secret } map captured at construction time.
+   * @param {ReadonlyArray<{kind: string, regex: RegExp}>} deps.patterns - Credential-shape regexes; each match becomes `[REDACTED:pattern:KIND]`.
+   * @param {boolean} deps.enabled - When false, `redactValue` returns its input by reference.
+   */
+  constructor({ envSnapshot, patterns, enabled }) {
+    this.envSnapshot = envSnapshot;
+    this.patterns = patterns;
+    this.enabled = enabled;
+  }
+
+  /**
+   * Redact any JSON-serialisable value by deep-walking and replacing secrets
+   * in every primitive string. Identity on the input when disabled.
+   * @param {unknown} value
+   * @returns {unknown}
+   */
+  redactValue(value) {
+    if (!this.enabled) return value;
+    return walk(value, (s) => this.#redactString(s));
+  }
+
+  /**
+   * Apply the env-allowlist and pattern layers to a single string.
+   * @param {string} s
+   * @returns {string}
+   */
+  #redactString(s) {
+    let out = s;
+    for (const [name, secret] of Object.entries(this.envSnapshot)) {
+      if (out.includes(secret)) {
+        out = out.split(secret).join(ENV_PLACEHOLDER(name));
+      }
+    }
+    for (const { kind, regex } of this.patterns) {
+      out = out.replace(regex, PATTERN_PLACEHOLDER(kind));
+    }
+    return out;
+  }
+}
+
+/**
+ * Build a redactor. Reads `LIBEVAL_REDACTION_DISABLED` and
+ * `LIBEVAL_REDACTION_ENV_VARS` from the supplied env (defaults to
+ * `process.env`). Fires a one-shot stderr warning when constructed
+ * disabled — bypass via `createNoopRedactor()` for silent fixtures.
+ * @param {object} [opts]
+ * @param {Record<string, string|undefined>} [opts.env] - Environment to snapshot. Defaults to `process.env`.
+ * @param {string[]} [opts.allowlist] - Override the env-var name list. Defaults to `DEFAULT_ENV_ALLOWLIST` or the parsed `LIBEVAL_REDACTION_ENV_VARS` value.
+ * @param {ReadonlyArray<{kind: string, regex: RegExp}>} [opts.patterns] - Credential-shape regexes. Defaults to `DEFAULT_PATTERNS`.
+ * @param {boolean} [opts.enabled] - Force enabled/disabled; bypasses `LIBEVAL_REDACTION_DISABLED`.
+ * @returns {Redactor}
+ */
+export function createRedactor({
+  env = process.env,
+  allowlist,
+  patterns = DEFAULT_PATTERNS,
+  enabled,
+} = {}) {
+  const envDisabled = env.LIBEVAL_REDACTION_DISABLED === "1";
+  const resolvedEnabled = enabled ?? !envDisabled;
+  const resolvedAllowlist = allowlist ?? resolveAllowlistFromEnv(env);
+  const envSnapshot = resolvedEnabled
+    ? snapshotEnv(env, resolvedAllowlist)
+    : Object.freeze({});
+  if (!resolvedEnabled) {
+    process.stderr.write(
+      "libeval: trace redaction DISABLED via LIBEVAL_REDACTION_DISABLED — secrets may appear in trace artifact\n",
+    );
+  }
+  return new Redactor({ envSnapshot, patterns, enabled: resolvedEnabled });
+}
+
+/**
+ * Parse `LIBEVAL_REDACTION_ENV_VARS` into a trimmed, non-empty name list.
+ * Falls back to `DEFAULT_ENV_ALLOWLIST` when unset or empty.
+ * @param {Record<string, string|undefined>} env
+ * @returns {string[]}
+ */
+function resolveAllowlistFromEnv(env) {
+  const override = env.LIBEVAL_REDACTION_ENV_VARS;
+  if (typeof override !== "string" || override.length === 0) {
+    return DEFAULT_ENV_ALLOWLIST;
+  }
+  return override
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+/**
+ * Build a disabled redactor whose `redactValue` is the identity function.
+ * Test-fixture form — bypasses `createRedactor` so no stderr warning
+ * fires regardless of env state.
+ * @returns {Redactor}
+ */
+export function createNoopRedactor() {
+  return new Redactor({
+    envSnapshot: Object.freeze({}),
+    patterns: [],
+    enabled: false,
+  });
+}