@forwardimpact/libeval 0.1.49 → 0.1.51

package/src/benchmark/runner.js

@@ -4,7 +4,7 @@
  * Phases per (task, runIndex):
  *   1. WorkdirManager.start → seed CWD + run pre-flight probe
  *   2. Supervisor session (agent + supervisor) → produce traces + submission
- *   3. Scorer.runScoring → exit-code-driven verdict via fd-3 NDJSON
+ *   3. Invariants.runInvariants → exit-code-driven verdict via fd-3 NDJSON
  *   4. Judge.runJudge → Conclude-driven verdict mapped to pass/fail
  *   5. WorkdirManager.teardown → process-group cleanup
  *
@@ -22,9 +22,10 @@ import { join, resolve as resolvePath } from "node:path";
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
 import { createSupervisor } from "../supervisor.js";
 import { installApm as defaultInstallApm } from "./apm-installer.js";
+import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
 import { validateResultRecord } from "./result.js";
-import { runScoring } from "./scorer.js";
+import { runInvariants } from "./invariants.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
 import { createWorkdirManager } from "./workdir.js";
 
@@ -59,15 +60,17 @@ export class BenchmarkRunner {
    *   write a valid NDJSON trace to `workdir.agentTracePath`. Default uses
    *   `createAgentRunner` with the harness `BASE_TOOLS` allowlist. Internal
    *   testing only — not part of the public API.
-   * @param {Function} [opts.runScoring] - Test seam: replaces `runScoring`.
-   *   Same contract as `runScoring(task, ctx)`. Internal testing only.
+   * @param {Function} [opts.runInvariants] - Test seam: replaces `runInvariants`.
+   *   Same contract as `runInvariants(task, ctx)`. Internal testing only.
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
-   *   contract as `runJudge(task, workdir, scoring, deps)`. Internal testing
+   *   contract as `runJudge(task, workdir, invariants, deps)`. Internal testing
    *   only.
    * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
    *   Same contract as `installApm(family, outputDir)`. Lets tests inject a
    *   fake `apm` spawn (or skip the install entirely) so the suite never
    *   shells out to a real `apm` binary. Internal testing only.
+   * @param {Function} [opts.installNpm] - Test seam: replaces `installNpm`.
+   *   Same contract as `installNpm(family, stagingDir)`. Internal testing only.
    */
   constructor({
     family,
@@ -83,9 +86,10 @@ export class BenchmarkRunner {
     termGraceMs,
     // Test seams — default to the real implementations.
     runAgent,
-    runScoring: runScoringHook,
+    runInvariants: runInvariantsHook,
     runJudge: runJudgeHook,
     installApm: installApmHook,
+    installNpm: installNpmHook,
   }) {
     if (!family) throw new Error("family is required");
     if (!Number.isInteger(runs) || runs < 1)
@@ -108,9 +112,10 @@ export class BenchmarkRunner {
     this.maxTurns = maxTurns;
     this.termGraceMs = termGraceMs;
     this._runAgentHook = runAgent ?? null;
-    this._runScoringHook = runScoringHook ?? runScoring;
+    this._runInvariantsHook = runInvariantsHook ?? runInvariants;
     this._runJudgeHook = runJudgeHook ?? runJudge;
     this._installApmHook = installApmHook ?? defaultInstallApm;
+    this._installNpmHook = installNpmHook ?? defaultInstallNpm;
   }
 
   /**
@@ -126,6 +131,7 @@ export class BenchmarkRunner {
     await mkdir(this.output, { recursive: true });
     const { stagingDir, skillSetHash, judgeProfilesDir } =
       await this._installApmHook(family, this.output);
+    await this._installNpmHook(family, stagingDir);
 
     const tasks = family.tasks();
     if (this.profiles.judge) {
@@ -185,7 +191,7 @@ export class BenchmarkRunner {
       }
       const { costUsd, turns, submission, agentError } =
         await this.#runAgentSafe(task, workdir);
-      const scoring = await this._runScoringHook(task, {
+      const invariants = await this._runInvariantsHook(task, {
         cwd: workdir.cwd,
         port: workdir.port,
         runDir: workdir.runDir,
@@ -200,7 +206,7 @@ export class BenchmarkRunner {
         judgeVerdict = await this._runJudgeHook(
           task,
           workdir,
-          scoring,
+          invariants,
           {
             query: this.query,
             model: this.judgeModel,
@@ -211,7 +217,7 @@ export class BenchmarkRunner {
         );
       }
       const verdict =
-        scoring.verdict === "pass" &&
+        invariants.verdict === "pass" &&
         (judgeVerdict === null || judgeVerdict.verdict === "pass")
           ? "pass"
           : "fail";
@@ -219,7 +225,7 @@ export class BenchmarkRunner {
         taskId: task.id,
         runIndex,
         verdict,
-        scoring,
+        invariants,
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd,

package/src/benchmark/task-family.js

@@ -9,7 +9,7 @@
  *       judge.task.md
  *       hooks/                # harness-only; never copied to agent CWD
  *         preflight.sh
- *         score.sh
+ *         invariants.sh
  *       specs/                # copied into agent CWD
  *       workdir/              # copied into agent CWD
  *
@@ -104,7 +104,7 @@ async function discoverTasks(rootPath) {
     const supervisorPath = join(taskDir, "supervisor.task.md");
     const judgePath = join(taskDir, "judge.task.md");
     const preflightPath = join(taskDir, "hooks", "preflight.sh");
-    const scorePath = join(taskDir, "hooks", "score.sh");
+    const invariantsPath = join(taskDir, "hooks", "invariants.sh");
     tasks.push({
       id: entry.name,
       paths: {
@@ -114,7 +114,9 @@ async function discoverTasks(rootPath) {
         judge: (await fileExists(judgePath)) ? judgePath : null,
         hooks: join(taskDir, "hooks"),
         preflight: (await fileExecutable(preflightPath)) ? preflightPath : null,
-        score: (await fileExecutable(scorePath)) ? scorePath : null,
+        invariants: (await fileExecutable(invariantsPath))
+          ? invariantsPath
+          : null,
         specs: join(taskDir, "specs"),
         workdir: join(taskDir, "workdir"),
       },
@@ -236,7 +238,7 @@ function run(cmd, args) {
 /**
  * @typedef {object} Task
  * @property {string} id - Task name (directory name under tasks/)
- * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, score: string|null, specs: string, workdir: string}} paths
+ * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, specs: string, workdir: string}} paths
  */
 
 /**

package/src/benchmark/workdir.js

@@ -4,7 +4,7 @@
  * the pre-flight smoke probe, and tear down the process group at end of run.
  *
  * The Workdir handle threads `cwd`, `port`, `pgid`, and trace paths through
- * runAgent → score → judge → teardown.
+ * runAgent → invariants → judge → teardown.
  */
 
 import { spawn } from "node:child_process";
@@ -13,6 +13,8 @@ import { createServer } from "node:net";
 import { connect } from "node:net";
 import { join } from "node:path";
 
+import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
+
 import { loadEnv } from "./env-loader.js";
 
 const DEFAULT_TERM_GRACE_MS = 5_000;
@@ -38,13 +40,23 @@ export class WorkdirManager {
    * @param {string} deps.stagingDir - Output of `installApm(...)`.
    * @param {string} deps.runOutputDir - Root run-output directory (parent of `runs/`).
    */
-  constructor({ stagingDir, runOutputDir, termGraceMs, familyRootPath }) {
+  constructor({
+    stagingDir,
+    runOutputDir,
+    termGraceMs,
+    familyRootPath,
+    runtime,
+  }) {
     if (!stagingDir) throw new Error("stagingDir is required");
     if (!runOutputDir) throw new Error("runOutputDir is required");
     this.stagingDir = stagingDir;
     this.runOutputDir = runOutputDir;
     this.termGraceMs = termGraceMs ?? DEFAULT_TERM_GRACE_MS;
     this.familyRootPath = familyRootPath ?? null;
+    // `loadEnv` is the only collaborator routed through the runtime today; the
+    // rest of this manager still uses raw streaming/net/process-group APIs the
+    // runtime surface does not yet cover.
+    this.runtime = runtime ?? null;
   }
 
   /**
@@ -70,12 +82,20 @@ export class WorkdirManager {
     await cp(join(this.stagingDir, ".claude"), join(cwd, ".claude"), {
       recursive: true,
     });
+    await cp(join(this.stagingDir, "node_modules"), join(cwd, "node_modules"), {
+      recursive: true,
+    }).catch((e) => {
+      if (e.code !== "ENOENT") throw e;
+    });
 
     const envDirs = [
       ...(this.familyRootPath ? [this.familyRootPath] : []),
       ...(task.paths.taskDir ? [task.paths.taskDir] : []),
     ];
-    const envNames = envDirs.length > 0 ? await loadEnv(envDirs, cwd) : [];
+    const envNames =
+      envDirs.length > 0
+        ? await loadEnv(envDirs, cwd, this.runtime ?? createDefaultRuntime())
+        : [];
 
     const port = await allocatePort();
     const agentTracePath = join(runDir, "agent.ndjson");

package/src/commands/assert.js

@@ -1,4 +1,3 @@
-import { existsSync, readFileSync } from "node:fs";
 import { basename } from "node:path";
 import jmespath from "jmespath";
 
@@ -6,10 +5,11 @@ import jmespath from "jmespath";
  * Evaluate an assertion and return the structured result.
  * @param {object} values - { grep?: string, query?: string, exists?: boolean, not?: boolean, message?: string }
  * @param {string[]} args - [testName, file]
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`): `existsSync`, `readFileSync`.
  * @returns {{ test: string, pass: boolean, message?: string }}
  */
 // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: assertion dispatch by type
-export function evaluateAssertion(values, args) {
+export function evaluateAssertion(values, args, fsSync) {
   const testName = args[0];
   if (!testName) throw new Error("assert: missing test name");
 
@@ -34,16 +34,16 @@ export function evaluateAssertion(values, args) {
   let result;
   if (values.exists) {
     if (!file) throw new Error("assert: missing file argument");
-    result = assertExists(file);
+    result = assertExists(file, fsSync);
   } else if (values.grep) {
     if (!file) throw new Error("assert: missing file argument for --grep");
-    result = assertGrep(values.grep, file);
+    result = assertGrep(values.grep, file, fsSync);
   } else if (values["cites-job"]) {
     if (!file) throw new Error("assert: missing file argument for --cites-job");
-    result = assertCitesJob(values["cites-job"], file);
+    result = assertCitesJob(values["cites-job"], file, fsSync);
   } else {
     if (!file) throw new Error("assert: missing file argument for --query");
-    result = assertQuery(values.query, file);
+    result = assertQuery(values.query, file, fsSync);
   }
 
   if (values.not) {
@@ -66,23 +66,31 @@ export function evaluateAssertion(values, args) {
 }
 
 /**
- * Run an assertion, write JSON to stdout, and set process.exitCode on failure.
- * @param {object} values
- * @param {string[]} args
+ * Run an assertion, write JSON to stdout, and return a failure envelope when
+ * the assertion does not pass.
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runAssertCommand(values, args) {
-  const result = evaluateAssertion(values, args);
-  process.stdout.write(JSON.stringify(result) + "\n");
-  if (!result.pass) process.exitCode = 1;
+export async function runAssertCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const args = [ctx.args["test-name"], ctx.args.file];
+  let result;
+  try {
+    result = evaluateAssertion(ctx.options, args, runtime.fsSync);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  runtime.proc.stdout.write(JSON.stringify(result) + "\n");
+  return result.pass ? { ok: true } : { ok: false, code: 1, error: "" };
 }
 
-function assertExists(file) {
-  if (existsSync(file)) return { pass: true };
+function assertExists(file, fsSync) {
+  if (fsSync.existsSync(file)) return { pass: true };
   return { pass: false, message: `${file} not found` };
 }
 
-function assertGrep(pattern, file) {
-  const content = readFileSync(file, "utf8");
+function assertGrep(pattern, file, fsSync) {
+  const content = fsSync.readFileSync(file, "utf8");
   const re = new RegExp(pattern, "im");
   if (re.test(content)) return { pass: true };
   return {
@@ -91,8 +99,8 @@ function assertGrep(pattern, file) {
   };
 }
 
-function assertQuery(expression, file) {
-  const content = readFileSync(file, "utf8");
+function assertQuery(expression, file, fsSync) {
+  const content = fsSync.readFileSync(file, "utf8");
   const data = parseJsonOrNdjson(content);
   const result = jmespath.search(data, expression);
   const truthy =
@@ -109,8 +117,8 @@ function assertQuery(expression, file) {
 
 const JOB_TAG_RE = /<job\s+user="([^"]*)"\s+goal="([^"]*)">/;
 
-function assertCitesJob(jobFile, file) {
-  const jobContent = readFileSync(jobFile, "utf8");
+function assertCitesJob(jobFile, file, fsSync) {
+  const jobContent = fsSync.readFileSync(jobFile, "utf8");
   const match = JOB_TAG_RE.exec(jobContent);
   if (!match) {
     return {
@@ -119,7 +127,7 @@ function assertCitesJob(jobFile, file) {
     };
   }
   const citation = `${match[1]}: ${match[2]}`;
-  const content = readFileSync(file, "utf8");
+  const content = fsSync.readFileSync(file, "utf8");
   if (content.includes(citation)) return { pass: true };
   return { pass: false, message: `missing "${citation}"` };
 }

package/src/commands/benchmark-invariants.js

@@ -0,0 +1,74 @@
+/**
+ * `fit-benchmark invariants` — check a single task's invariants against a
+ * post-run workdir directory without invoking an agent (P6/P7). Useful for
+ * re-checking an agent's output against revised grading material.
+ */
+
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+
+import { validateInvariantsRecord } from "../benchmark/result.js";
+import { runInvariants } from "../benchmark/invariants.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkInvariantsCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const familyInput = values.family;
+  if (!familyInput)
+    return { ok: false, code: 1, error: "--family is required" };
+  const taskId = values.task;
+  if (!taskId) return { ok: false, code: 1, error: "--task is required" };
+  const workdirArg = values.workdir;
+  if (!workdirArg)
+    return { ok: false, code: 1, error: "--workdir is required" };
+
+  const family = await loadTaskFamily(familyInput);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task)
+    return { ok: false, code: 1, error: `task not found in family: ${taskId}` };
+
+  const runDir = resolve(workdirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await allocatePort();
+
+  const invariants = await runInvariants(task, { cwd, port, runDir });
+  const record = {
+    taskId: task.id,
+    invariants,
+    exitCode: invariants.exitCode,
+  };
+  validateInvariantsRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    runtime.fsSync.writeFileSync(resolve(values.output), line);
+  } else {
+    runtime.proc.stdout.write(line);
+  }
+  return invariants.verdict === "pass"
+    ? { ok: true }
+    : { ok: false, code: 1, error: "" };
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}

package/src/commands/benchmark-report.js

@@ -9,24 +9,31 @@ import { resolve } from "node:path";
 import { aggregate, renderTextReport } from "../benchmark/report.js";
 
 /**
- * @param {object} values
- * @param {string[]} _args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runBenchmarkReportCommand(values, _args) {
+export async function runBenchmarkReportCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
   const inputDir = values.input ?? "benchmark-runs";
   const kRaw = values.k ?? "1,3,5";
-  const kValues = kRaw.split(",").map((t) => {
-    const n = Number.parseInt(t.trim(), 10);
-    if (!Number.isFinite(n) || n < 1) {
-      throw new Error(
-        "--k must be a comma-separated list of positive integers",
-      );
-    }
-    return n;
-  });
+  let kValues;
+  try {
+    kValues = kRaw.split(",").map((t) => {
+      const n = Number.parseInt(t.trim(), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(
+          "--k must be a comma-separated list of positive integers",
+        );
+      }
+      return n;
+    });
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
   const format = values.format ?? "json";
   if (format !== "json" && format !== "text") {
-    throw new Error("--format must be 'json' or 'text'");
+    return { ok: false, code: 1, error: "--format must be 'json' or 'text'" };
   }
 
   const report = await aggregate({
@@ -35,8 +42,9 @@ export async function runBenchmarkReportCommand(values, _args) {
     includeRuns: format === "text",
   });
   if (format === "text") {
-    process.stdout.write(renderTextReport(report, kValues) + "\n");
+    runtime.proc.stdout.write(renderTextReport(report, kValues) + "\n");
   } else {
-    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
+    runtime.proc.stdout.write(JSON.stringify(report, null, 2) + "\n");
   }
+  return { ok: true };
 }

package/src/commands/benchmark-run.js

@@ -10,22 +10,37 @@ import { createConfig } from "@forwardimpact/libconfig";
 import { createBenchmarkRunner } from "../benchmark/runner.js";
 
 /**
- * @param {object} values
- * @param {string[]} _args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runBenchmarkRunCommand(values, _args) {
-  const opts = parseRunOptions(values);
+export async function runBenchmarkRunCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  let opts;
+  try {
+    opts = parseRunOptions(values);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
   const config = await createConfig("script", "benchmark");
-  process.env.ANTHROPIC_API_KEY = await config.anthropicToken();
+  runtime.proc.env.ANTHROPIC_API_KEY = await config.anthropicToken();
+
+  // The Claude Agent SDK spawns a `claude` subprocess that inherits
+  // process.env. NODE_EXTRA_CA_CERTS causes undici (the HTTP client
+  // inside that subprocess) to fail with UND_ERR_INVALID_ARG on
+  // Node 22+, aborting every API call after 10 retries. Strip it
+  // before the SDK loads so the subprocess gets a clean environment.
+  delete runtime.proc.env.NODE_EXTRA_CA_CERTS;
+
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const runner = createBenchmarkRunner({ ...opts, query });
 
   let anyFail = false;
   for await (const record of runner.run()) {
-    process.stdout.write(JSON.stringify(record) + "\n");
+    runtime.proc.stdout.write(JSON.stringify(record) + "\n");
     if (record.verdict !== "pass") anyFail = true;
   }
-  process.exit(anyFail ? 1 : 0);
+  return anyFail ? { ok: false, code: 1, error: "" } : { ok: true };
 }
 
 function parseRunOptions(values) {

package/src/commands/by-discussion.js

@@ -1,9 +1,16 @@
-import { readdirSync, statSync, openSync, readSync, closeSync } from "node:fs";
+import { closeSync, openSync, readSync } from "node:fs";
 import { join } from "node:path";
 
+const FIRST_LINE_CAP = 64 * 1024;
+
 /**
- * Read the first newline-terminated line of a file. Bounded to 64 KiB
- * which is well above any orchestrator envelope.
+ * Read the first newline-terminated line of a file, bounded to the first
+ * {@link FIRST_LINE_CAP} bytes. Trace `.ndjson` files can be many MB; the
+ * Step 2.6 meta header is always small, so a bounded `readSync` avoids
+ * loading whole files into memory just to inspect the header. This uses
+ * `node:fs` directly because the `runtime.fsSync` surface exposes no
+ * positional `openSync`/`readSync` — the file is grandfathered for
+ * `import:fs` in `check-ambient-deps.deny.yml` until that seam exists.
  *
  * @param {string} path
  * @returns {string}
@@ -11,11 +18,11 @@ import { join } from "node:path";
 function readFirstLine(path) {
   const fd = openSync(path, "r");
   try {
-    const buf = Buffer.alloc(65536);
+    const buf = Buffer.alloc(FIRST_LINE_CAP);
     const bytes = readSync(fd, buf, 0, buf.length, 0);
-    const slice = buf.slice(0, bytes).toString("utf8");
-    const nl = slice.indexOf("\n");
-    return nl === -1 ? slice : slice.slice(0, nl);
+    const text = buf.toString("utf8", 0, bytes);
+    const nl = text.indexOf("\n");
+    return nl === -1 ? text : text.slice(0, nl);
   } finally {
     closeSync(fd);
   }
@@ -30,13 +37,14 @@ function readFirstLine(path) {
  *
  * @param {string} dir
  * @param {string} discussionId
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
  * @returns {Array<{path: string, mtimeMs: number}>}
  */
-export function findTracesByDiscussion(dir, discussionId) {
+export function findTracesByDiscussion(dir, discussionId, fsSync) {
   const matches = [];
   let entries;
   try {
-    entries = readdirSync(dir);
+    entries = fsSync.readdirSync(dir);
   } catch {
     return [];
   }
@@ -58,7 +66,7 @@ export function findTracesByDiscussion(dir, discussionId) {
     const event = parsed.event ?? parsed;
     if (event?.type !== "meta") continue;
     if (event.discussion_id !== discussionId) continue;
-    matches.push({ path, mtimeMs: statSync(path).mtimeMs });
+    matches.push({ path, mtimeMs: fsSync.statSync(path).mtimeMs });
   }
   matches.sort((a, b) => a.mtimeMs - b.mtimeMs);
   return matches;
@@ -70,15 +78,18 @@ export function findTracesByDiscussion(dir, discussionId) {
  * line, ordered by first-event timestamp (file mtime ascending). The
  * result is usable with `xargs cat` for a chronological merge.
  *
- * @param {object} values
- * @param {string[]} args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runByDiscussionCommand(values, args) {
-  const [discussionId, traceDirArg] = args;
-  if (!discussionId) throw new Error("<discussion-id> is required");
-  const dir = traceDirArg ?? values["trace-dir"] ?? "traces";
-  const matches = findTracesByDiscussion(dir, discussionId);
+export async function runByDiscussionCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const discussionId = ctx.args["discussion-id"];
+  if (!discussionId)
+    return { ok: false, code: 1, error: "<discussion-id> is required" };
+  const dir = ctx.args["trace-dir"] ?? ctx.options["trace-dir"] ?? "traces";
+  const matches = findTracesByDiscussion(dir, discussionId, runtime.fsSync);
   for (const { path } of matches) {
-    process.stdout.write(`${path}\n`);
+    runtime.proc.stdout.write(`${path}\n`);
   }
+  return { ok: true };
 }

package/src/commands/callback.js

@@ -1,5 +1,3 @@
-import { readFileSync } from "node:fs";
-
 /**
  * Scan an NDJSON trace and return the last orchestrator summary event,
  * the first `meta` event's `discussion_id`, and any structured replies
@@ -11,13 +9,14 @@ import { readFileSync } from "node:fs";
  * its channel semantics.
  *
  * @param {string} traceFile
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
  * @returns {{verdict: string, summary: string, replies: object[], trigger?: object, discussionId?: string} | null}
  */
 // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: NDJSON scan with malformed-line tolerance + meta/summary dual extraction
-function readTraceSummary(traceFile) {
+function readTraceSummary(traceFile, fsSync) {
   let summary = null;
   let metaDiscussionId = null;
-  for (const line of readFileSync(traceFile, "utf8").split("\n")) {
+  for (const line of fsSync.readFileSync(traceFile, "utf8").split("\n")) {
     if (!line.trim()) continue;
     let record;
     try {
@@ -40,6 +39,9 @@ function readTraceSummary(traceFile) {
         ...(record.event.discussion_id && {
           discussionId: record.event.discussion_id,
         }),
+        ...(typeof record.event.lastActedSeq === "number" && {
+          lastActedSeq: record.event.lastActedSeq,
+        }),
       };
     }
   }
@@ -64,20 +66,24 @@ function readTraceSummary(traceFile) {
  * }
  * ```
  *
- * @param {object} values - Parsed option values from cli.parse()
- * @param {string[]} _args - Positional arguments
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runCallbackCommand(values, _args) {
+export async function runCallbackCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
   const traceFile = values["trace-file"];
   const callbackUrl = values["callback-url"];
   const correlationId = values["correlation-id"];
   const runUrl = values["run-url"] ?? "";
   const discussionIdOverride = values["discussion-id"] ?? null;
 
-  if (!traceFile) throw new Error("--trace-file is required");
-  if (!callbackUrl) throw new Error("--callback-url is required");
+  if (!traceFile)
+    return { ok: false, code: 1, error: "--trace-file is required" };
+  if (!callbackUrl)
+    return { ok: false, code: 1, error: "--callback-url is required" };
 
-  const found = readTraceSummary(traceFile) ?? {
+  const found = readTraceSummary(traceFile, runtime.fsSync) ?? {
     verdict: "failed",
     summary: "Run ended without producing a summary.",
     replies: [],
@@ -86,10 +92,12 @@ export async function runCallbackCommand(values, _args) {
   const discussionId = found.discussionId ?? discussionIdOverride ?? null;
   const payload = {
     correlation_id: correlationId,
+    kind: "terminal",
     verdict: found.verdict,
     summary: found.summary,
     run_url: runUrl,
     replies: found.replies,
+    last_acted_seq: found.lastActedSeq ?? -1,
     ...(discussionId && { discussion_id: discussionId }),
     ...(found.trigger && { trigger: found.trigger }),
   };
@@ -99,6 +107,7 @@ export async function runCallbackCommand(values, _args) {
     body: JSON.stringify(payload),
   });
   if (!res.ok) {
-    throw new Error(`Callback POST failed: ${res.status}`);
+    return { ok: false, code: 1, error: `Callback POST failed: ${res.status}` };
   }
+  return { ok: true };
 }