@forwardimpact/libeval 0.1.50 → 0.1.52

package/src/commands/benchmark-invariants.js

@@ -0,0 +1,74 @@
+/**
+ * `fit-benchmark invariants` — check a single task's invariants against a
+ * post-run workdir directory without invoking an agent (P6/P7). Useful for
+ * re-checking an agent's output against revised grading material.
+ */
+
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+
+import { validateInvariantsRecord } from "../benchmark/result.js";
+import { runInvariants } from "../benchmark/invariants.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkInvariantsCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const familyInput = values.family;
+  if (!familyInput)
+    return { ok: false, code: 1, error: "--family is required" };
+  const taskId = values.task;
+  if (!taskId) return { ok: false, code: 1, error: "--task is required" };
+  const workdirArg = values.workdir;
+  if (!workdirArg)
+    return { ok: false, code: 1, error: "--workdir is required" };
+
+  const family = await loadTaskFamily(familyInput, runtime);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task)
+    return { ok: false, code: 1, error: `task not found in family: ${taskId}` };
+
+  const runDir = resolve(workdirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await allocatePort();
+
+  const invariants = await runInvariants(task, { cwd, port, runDir }, runtime);
+  const record = {
+    taskId: task.id,
+    invariants,
+    exitCode: invariants.exitCode,
+  };
+  validateInvariantsRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    runtime.fsSync.writeFileSync(resolve(values.output), line);
+  } else {
+    runtime.proc.stdout.write(line);
+  }
+  return invariants.verdict === "pass"
+    ? { ok: true }
+    : { ok: false, code: 1, error: "" };
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}

package/src/commands/benchmark-report.js

@@ -9,34 +9,43 @@ import { resolve } from "node:path";
 import { aggregate, renderTextReport } from "../benchmark/report.js";
 
 /**
- * @param {object} values
- * @param {string[]} _args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runBenchmarkReportCommand(values, _args) {
+export async function runBenchmarkReportCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
   const inputDir = values.input ?? "benchmark-runs";
   const kRaw = values.k ?? "1,3,5";
-  const kValues = kRaw.split(",").map((t) => {
-    const n = Number.parseInt(t.trim(), 10);
-    if (!Number.isFinite(n) || n < 1) {
-      throw new Error(
-        "--k must be a comma-separated list of positive integers",
-      );
-    }
-    return n;
-  });
+  let kValues;
+  try {
+    kValues = kRaw.split(",").map((t) => {
+      const n = Number.parseInt(t.trim(), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(
+          "--k must be a comma-separated list of positive integers",
+        );
+      }
+      return n;
+    });
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
   const format = values.format ?? "json";
   if (format !== "json" && format !== "text") {
-    throw new Error("--format must be 'json' or 'text'");
+    return { ok: false, code: 1, error: "--format must be 'json' or 'text'" };
   }
 
   const report = await aggregate({
     inputDir: resolve(inputDir),
     kValues,
     includeRuns: format === "text",
+    runtime,
   });
   if (format === "text") {
-    process.stdout.write(renderTextReport(report, kValues) + "\n");
+    runtime.proc.stdout.write(renderTextReport(report, kValues) + "\n");
   } else {
-    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
+    runtime.proc.stdout.write(JSON.stringify(report, null, 2) + "\n");
   }
+  return { ok: true };
 }

package/src/commands/benchmark-run.js

@@ -10,30 +10,37 @@ import { createConfig } from "@forwardimpact/libconfig";
 import { createBenchmarkRunner } from "../benchmark/runner.js";
 
 /**
- * @param {object} values
- * @param {string[]} _args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runBenchmarkRunCommand(values, _args) {
-  const opts = parseRunOptions(values);
+export async function runBenchmarkRunCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  let opts;
+  try {
+    opts = parseRunOptions(values);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
   const config = await createConfig("script", "benchmark");
-  process.env.ANTHROPIC_API_KEY = await config.anthropicToken();
+  runtime.proc.env.ANTHROPIC_API_KEY = await config.anthropicToken();
 
   // The Claude Agent SDK spawns a `claude` subprocess that inherits
   // process.env. NODE_EXTRA_CA_CERTS causes undici (the HTTP client
   // inside that subprocess) to fail with UND_ERR_INVALID_ARG on
   // Node 22+, aborting every API call after 10 retries. Strip it
   // before the SDK loads so the subprocess gets a clean environment.
-  delete process.env.NODE_EXTRA_CA_CERTS;
+  delete runtime.proc.env.NODE_EXTRA_CA_CERTS;
 
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
-  const runner = createBenchmarkRunner({ ...opts, query });
+  const runner = createBenchmarkRunner({ ...opts, query, runtime });
 
   let anyFail = false;
   for await (const record of runner.run()) {
-    process.stdout.write(JSON.stringify(record) + "\n");
+    runtime.proc.stdout.write(JSON.stringify(record) + "\n");
     if (record.verdict !== "pass") anyFail = true;
   }
-  process.exit(anyFail ? 1 : 0);
+  return anyFail ? { ok: false, code: 1, error: "" } : { ok: true };
 }
 
 function parseRunOptions(values) {

package/src/commands/by-discussion.js

@@ -1,23 +1,29 @@
-import { readdirSync, statSync, openSync, readSync, closeSync } from "node:fs";
 import { join } from "node:path";
 
+const FIRST_LINE_CAP = 64 * 1024;
+
 /**
- * Read the first newline-terminated line of a file. Bounded to 64 KiB
- * which is well above any orchestrator envelope.
+ * Read the first newline-terminated line of a file, bounded to the first
+ * {@link FIRST_LINE_CAP} bytes. Trace `.ndjson` files can be many MB; the
+ * Step 2.6 meta header is always small, so a bounded positional read avoids
+ * loading whole files into memory just to inspect the header. The positional
+ * `openSync`/`readSync`/`closeSync` trio is read off the injected
+ * `runtime.fsSync` surface.
  *
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
  * @param {string} path
  * @returns {string}
  */
-function readFirstLine(path) {
-  const fd = openSync(path, "r");
+function readFirstLine(fsSync, path) {
+  const fd = fsSync.openSync(path, "r");
   try {
-    const buf = Buffer.alloc(65536);
-    const bytes = readSync(fd, buf, 0, buf.length, 0);
-    const slice = buf.slice(0, bytes).toString("utf8");
-    const nl = slice.indexOf("\n");
-    return nl === -1 ? slice : slice.slice(0, nl);
+    const buf = Buffer.alloc(FIRST_LINE_CAP);
+    const bytes = fsSync.readSync(fd, buf, 0, buf.length, 0);
+    const text = buf.toString("utf8", 0, bytes);
+    const nl = text.indexOf("\n");
+    return nl === -1 ? text : text.slice(0, nl);
   } finally {
-    closeSync(fd);
+    fsSync.closeSync(fd);
   }
 }
 
@@ -30,13 +36,14 @@ function readFirstLine(path) {
  *
  * @param {string} dir
  * @param {string} discussionId
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
  * @returns {Array<{path: string, mtimeMs: number}>}
  */
-export function findTracesByDiscussion(dir, discussionId) {
+export function findTracesByDiscussion(dir, discussionId, fsSync) {
   const matches = [];
   let entries;
   try {
-    entries = readdirSync(dir);
+    entries = fsSync.readdirSync(dir);
   } catch {
     return [];
   }
@@ -45,7 +52,7 @@ export function findTracesByDiscussion(dir, discussionId) {
     const path = join(dir, entry);
     let firstLine;
     try {
-      firstLine = readFirstLine(path);
+      firstLine = readFirstLine(fsSync, path);
     } catch {
       continue;
     }
@@ -58,7 +65,7 @@ export function findTracesByDiscussion(dir, discussionId) {
     const event = parsed.event ?? parsed;
     if (event?.type !== "meta") continue;
     if (event.discussion_id !== discussionId) continue;
-    matches.push({ path, mtimeMs: statSync(path).mtimeMs });
+    matches.push({ path, mtimeMs: fsSync.statSync(path).mtimeMs });
   }
   matches.sort((a, b) => a.mtimeMs - b.mtimeMs);
   return matches;
@@ -70,15 +77,18 @@ export function findTracesByDiscussion(dir, discussionId) {
  * line, ordered by first-event timestamp (file mtime ascending). The
  * result is usable with `xargs cat` for a chronological merge.
  *
- * @param {object} values
- * @param {string[]} args
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runByDiscussionCommand(values, args) {
-  const [discussionId, traceDirArg] = args;
-  if (!discussionId) throw new Error("<discussion-id> is required");
-  const dir = traceDirArg ?? values["trace-dir"] ?? "traces";
-  const matches = findTracesByDiscussion(dir, discussionId);
+export async function runByDiscussionCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const discussionId = ctx.args["discussion-id"];
+  if (!discussionId)
+    return { ok: false, code: 1, error: "<discussion-id> is required" };
+  const dir = ctx.args["trace-dir"] ?? ctx.options["trace-dir"] ?? "traces";
+  const matches = findTracesByDiscussion(dir, discussionId, runtime.fsSync);
   for (const { path } of matches) {
-    process.stdout.write(`${path}\n`);
+    runtime.proc.stdout.write(`${path}\n`);
   }
+  return { ok: true };
 }

package/src/commands/callback.js

@@ -1,5 +1,3 @@
-import { readFileSync } from "node:fs";
-
 /**
  * Scan an NDJSON trace and return the last orchestrator summary event,
  * the first `meta` event's `discussion_id`, and any structured replies
@@ -11,13 +9,14 @@ import { readFileSync } from "node:fs";
  * its channel semantics.
  *
  * @param {string} traceFile
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
  * @returns {{verdict: string, summary: string, replies: object[], trigger?: object, discussionId?: string} | null}
  */
 // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: NDJSON scan with malformed-line tolerance + meta/summary dual extraction
-function readTraceSummary(traceFile) {
+function readTraceSummary(traceFile, fsSync) {
   let summary = null;
   let metaDiscussionId = null;
-  for (const line of readFileSync(traceFile, "utf8").split("\n")) {
+  for (const line of fsSync.readFileSync(traceFile, "utf8").split("\n")) {
     if (!line.trim()) continue;
     let record;
     try {
@@ -40,6 +39,9 @@ function readTraceSummary(traceFile) {
         ...(record.event.discussion_id && {
           discussionId: record.event.discussion_id,
         }),
+        ...(typeof record.event.lastActedSeq === "number" && {
+          lastActedSeq: record.event.lastActedSeq,
+        }),
       };
     }
   }
@@ -64,20 +66,24 @@ function readTraceSummary(traceFile) {
  * }
  * ```
  *
- * @param {object} values - Parsed option values from cli.parse()
- * @param {string[]} _args - Positional arguments
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
-export async function runCallbackCommand(values, _args) {
+export async function runCallbackCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
   const traceFile = values["trace-file"];
   const callbackUrl = values["callback-url"];
   const correlationId = values["correlation-id"];
   const runUrl = values["run-url"] ?? "";
   const discussionIdOverride = values["discussion-id"] ?? null;
 
-  if (!traceFile) throw new Error("--trace-file is required");
-  if (!callbackUrl) throw new Error("--callback-url is required");
+  if (!traceFile)
+    return { ok: false, code: 1, error: "--trace-file is required" };
+  if (!callbackUrl)
+    return { ok: false, code: 1, error: "--callback-url is required" };
 
-  const found = readTraceSummary(traceFile) ?? {
+  const found = readTraceSummary(traceFile, runtime.fsSync) ?? {
     verdict: "failed",
     summary: "Run ended without producing a summary.",
     replies: [],
@@ -86,10 +92,12 @@ export async function runCallbackCommand(values, _args) {
   const discussionId = found.discussionId ?? discussionIdOverride ?? null;
   const payload = {
     correlation_id: correlationId,
+    kind: "terminal",
     verdict: found.verdict,
     summary: found.summary,
     run_url: runUrl,
     replies: found.replies,
+    last_acted_seq: found.lastActedSeq ?? -1,
     ...(discussionId && { discussion_id: discussionId }),
     ...(found.trigger && { trigger: found.trigger }),
   };
@@ -99,6 +107,7 @@ export async function runCallbackCommand(values, _args) {
     body: JSON.stringify(payload),
   });
   if (!res.ok) {
-    throw new Error(`Callback POST failed: ${res.status}`);
+    return { ok: false, code: 1, error: `Callback POST failed: ${res.status}` };
   }
+  return { ok: true };
 }

package/src/commands/discuss.js

@@ -1,5 +1,5 @@
-import { createWriteStream } from "node:fs";
 import { resolve } from "node:path";
+import { isoTimestamp } from "@forwardimpact/libutil";
 import { createDiscusser } from "../discusser.js";
 import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
@@ -17,10 +17,14 @@ function parseAgentProfiles(raw, cwd, maxTurns) {
  * Parse and validate discuss command options. Exported so tests can verify
  * defaults and the legacy-flag clean break.
  * @param {object} values - Parsed option values
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {object}
  */
-export function parseDiscussOptions(values) {
-  const { task: taskContent, amend: taskAmend } = resolveTaskContent(values);
+export function parseDiscussOptions(values, runtime) {
+  const { task: taskContent, amend: taskAmend } = resolveTaskContent(
+    values,
+    runtime,
+  );
 
   const profilesRaw = values["agent-profiles"];
   const agentCwd = resolve(values["agent-cwd"] ?? ".");
@@ -40,6 +44,9 @@ export function parseDiscussOptions(values) {
     }
   }
 
+  const maxLeadTurnsRaw = values["max-lead-turns"] ?? "200";
+  const maxLeadTurns = parseInt(maxLeadTurnsRaw, 10);
+
   return {
     taskContent,
     taskAmend,
@@ -48,9 +55,13 @@ export function parseDiscussOptions(values) {
     leadModel: values["lead-model"] ?? "claude-opus-4-7[1m]",
     agentModel: values["agent-model"] ?? "claude-opus-4-7[1m]",
     maxTurns,
+    maxLeadTurns,
     outputPath: values.output,
     discussionId: values["discussion-id"] ?? null,
     resumeContext,
+    callbackUrl: runtime.proc.env.CALLBACK_URL ?? null,
+    inboxUrl: runtime.proc.env.INBOX_URL ?? null,
+    correlationId: runtime.proc.env.CORRELATION_ID ?? null,
   };
 }
 
@@ -59,27 +70,29 @@ export function parseDiscussOptions(values) {
  * semantics, threading `discussion_id` through the trace so multi-run
  * conversations are queryable as one.
  *
- * @param {object} values - Parsed option values
- * @param {string[]} _args - Positional arguments
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: boolean, code?: number, error?: string}>}
  */
-export async function runDiscussCommand(values, _args) {
-  const opts = parseDiscussOptions(values);
+export async function runDiscussCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const opts = parseDiscussOptions(ctx.options, runtime);
 
-  const redactor = createRedactor();
+  const redactor = createRedactor({ runtime });
 
   const fileStream = opts.outputPath
-    ? createWriteStream(opts.outputPath)
+    ? runtime.fs.createWriteStream(opts.outputPath)
     : null;
   const output = fileStream
     ? createTeeWriter({
         fileStream,
-        textStream: process.stdout,
+        textStream: runtime.proc.stdout,
         mode: "supervised",
+        now: () => isoTimestamp(runtime.clock.now()),
       })
-    : process.stdout;
+    : runtime.proc.stdout;
 
   if (opts.leadProfile) {
-    process.env.LIBEVAL_AGENT_PROFILE = opts.leadProfile;
+    runtime.proc.env.LIBEVAL_AGENT_PROFILE = opts.leadProfile;
   }
 
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
@@ -93,8 +106,13 @@ export async function runDiscussCommand(values, _args) {
     query,
     output,
     maxTurns: opts.maxTurns,
+    maxLeadTurns: opts.maxLeadTurns,
     taskAmend: opts.taskAmend,
     redactor,
+    callbackUrl: opts.callbackUrl,
+    inboxUrl: opts.inboxUrl,
+    correlationId: opts.correlationId,
+    runtime,
   });
 
   const result = await discusser.run(opts.taskContent);
@@ -104,5 +122,5 @@ export async function runDiscussCommand(values, _args) {
     await new Promise((r) => fileStream.end(r));
   }
 
-  process.exit(result.success ? 0 : 1);
+  return result.success ? { ok: true } : { ok: false, code: 1, error: "" };
 }

package/src/commands/facilitate.js

@@ -1,5 +1,5 @@
-import { createWriteStream } from "node:fs";
 import { resolve } from "node:path";
+import { isoTimestamp } from "@forwardimpact/libutil";
 import { createFacilitator } from "../facilitator.js";
 import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
@@ -23,10 +23,14 @@ function parseAgentProfiles(raw, cwd, maxTurns) {
  * coverage of the `--max-turns` → per-agent threading contract; not part
  * of the package's public API.
  * @param {object} values - Parsed option values
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @returns {object} Parsed options
  */
-export function parseFacilitateOptions(values) {
-  const { task: taskContent, amend: taskAmend } = resolveTaskContent(values);
+export function parseFacilitateOptions(values, runtime) {
+  const { task: taskContent, amend: taskAmend } = resolveTaskContent(
+    values,
+    runtime,
+  );
 
   const profilesRaw = values["agent-profiles"];
   if (!profilesRaw) throw new Error("--agent-profiles is required");
@@ -59,30 +63,32 @@ export function parseFacilitateOptions(values) {
  *
  * Usage: fit-eval facilitate [options]
  *
- * @param {object} values - Parsed option values from cli.parse()
- * @param {string[]} _args - Positional arguments
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: boolean, code?: number, error?: string}>}
  */
-export async function runFacilitateCommand(values, _args) {
-  const opts = parseFacilitateOptions(values);
+export async function runFacilitateCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const opts = parseFacilitateOptions(ctx.options, runtime);
 
   // Build the redactor as the first observable side-effect after option
   // parsing — the env snapshot must freeze BEFORE any in-process
-  // process.env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
-  const redactor = createRedactor();
+  // env writes the command performs (e.g. LIBEVAL_AGENT_PROFILE).
+  const redactor = createRedactor({ runtime });
 
   const fileStream = opts.outputPath
-    ? createWriteStream(opts.outputPath)
+    ? runtime.fs.createWriteStream(opts.outputPath)
     : null;
   const output = fileStream
     ? createTeeWriter({
         fileStream,
-        textStream: process.stdout,
+        textStream: runtime.proc.stdout,
         mode: "supervised",
+        now: () => isoTimestamp(runtime.clock.now()),
       })
-    : process.stdout;
+    : runtime.proc.stdout;
 
   if (opts.facilitatorProfile) {
-    process.env.LIBEVAL_AGENT_PROFILE = opts.facilitatorProfile;
+    runtime.proc.env.LIBEVAL_AGENT_PROFILE = opts.facilitatorProfile;
   }
 
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
@@ -97,6 +103,7 @@ export async function runFacilitateCommand(values, _args) {
     facilitatorProfile: opts.facilitatorProfile,
     taskAmend: opts.taskAmend,
     redactor,
+    runtime,
   });
 
   const result = await facilitator.run(opts.taskContent);
@@ -106,5 +113,5 @@ export async function runFacilitateCommand(values, _args) {
     await new Promise((r) => fileStream.end(r));
   }
 
-  process.exit(result.success ? 0 : 1);
+  return result.success ? { ok: true } : { ok: false, code: 1, error: "" };
 }

package/src/commands/output.js

@@ -1,3 +1,4 @@
+import { isoTimestamp } from "@forwardimpact/libutil";
 import { createTraceCollector } from "@forwardimpact/libeval";
 
 /**
@@ -6,29 +7,30 @@ import { createTraceCollector } from "@forwardimpact/libeval";
  *
  * Usage: fit-eval output [--format=json|text] < trace.ndjson
  *
- * @param {object} values - Parsed option values from cli.parse()
- * @param {string[]} args - Positional arguments
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true}>}
  */
-export async function runOutputCommand(values, _args) {
+export async function runOutputCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
   const format =
     values.format === "text" || values.format === "json"
       ? values.format
       : "json";
-  const collector = createTraceCollector();
+  const collector = createTraceCollector({
+    now: () => isoTimestamp(runtime.clock.now()),
+  });
 
-  const chunks = [];
-  for await (const chunk of process.stdin) {
-    chunks.push(chunk);
-  }
-  const input = Buffer.concat(chunks).toString("utf8");
-
-  for (const line of input.split("\n")) {
+  // `runtime.proc.stdin` is an AsyncIterable of UTF-8 lines (newline-split by
+  // the runtime), so each yielded value is exactly one NDJSON record.
+  for await (const line of runtime.proc.stdin) {
     collector.addLine(line);
   }
 
   if (format === "text") {
-    process.stdout.write(collector.toText() + "\n");
+    runtime.proc.stdout.write(collector.toText() + "\n");
   } else {
-    process.stdout.write(JSON.stringify(collector.toJSON()) + "\n");
+    runtime.proc.stdout.write(JSON.stringify(collector.toJSON()) + "\n");
   }
+  return { ok: true };
 }