@forwardimpact/libeval 0.1.49 → 0.1.51

package/README.md

@@ -71,11 +71,12 @@ while participants work in parallel — nothing blocks the LLM thread.
 
 ### Discuss-mode replies
 
-In discussion mode, Answer calls routed to the lead are captured as
-thread replies delivered via the bridge callback. The lead delegates work
-via Ask; each agent's Answer becomes a separate reply posted to the
-discussion thread. No explicit reply tool is needed on the lead surface —
-the message bus intercepts answers and appends them to `ctx.replies[]`.
+In discussion mode, Answer calls routed to the lead are streamed to
+the discussion thread as they are produced — each agent's Answer becomes
+a separate reply posted immediately, not batched at session end. The
+lead and agents can also call `Acknowledge` to post brief messages
+directly to the thread (status updates, human follow-up responses).
+The message bus intercepts answers and appends them to `ctx.replies[]`.
 
 `RequestForComment` is a separate coordination tool available on agent
 roles (facilitated agents and discuss agents). It queues an intent to
@@ -104,8 +105,8 @@ only feeds the summary's `success`/`verdict`.
 | Fac. agent   | ✓   | ✓      | ✓        | ✓        |          | `RequestForComment`            |
 | Supervisor   | ✓   | ✓      | ✓        | ✓        | ✓        |                                |
 | Sup. agent   | ✓   | ✓      | ✓        | ✓        |          |                                |
-| Discuss lead | ✓   | ✓      | ✓        | ✓        |          | `Recess`, `Adjourn`            |
-| Discuss agt  | ✓   | ✓      | ✓        | ✓        |          | `RequestForComment`            |
+| Discuss lead | ✓   | ✓      | ✓        | ✓        |          | `Recess`, `Adjourn`, `Acknowledge` |
+| Discuss agt  | ✓   | ✓      | ✓        | ✓        |          | `RequestForComment`, `Acknowledge` |
 | Judge        |     |        |          |          | ✓        |                                |
 
 Ask's `to` accepts a participant name on multi-participant roles
@@ -169,7 +170,9 @@ downloadable through retention.
 | `orchestration-toolkit.js`                                  | Shared Ask/Answer/Announce/Conclude/RollCall/RequestForComment handlers + builders. |
 | `orchestration-loop.js`                                     | Unified lead+participant loop; reminder/violation handling.          |
 | `facilitator.js` / `supervisor.js` / `discusser.js` / `judge.js` | Per-mode class + factory + system prompt.                       |
-| `discuss-tools.js`                                          | Discuss-only `Recess`/`Adjourn`.                                     |
+| `discuss-tools.js`                                          | Discuss-only `Recess`/`Adjourn`/`Acknowledge`.                       |
+| `reply-emitter.js`                                          | Fire-and-forget POST of reply/ack events to the callback URL.        |
+| `inbox-poller.js`                                           | Long-poll the bridge inbox for injected human messages.              |
 | `trace-collector.js` / `trace-query.js` / `trace-github.js` | Trace ingestion / querying / GitHub-attachment helpers.              |
 | `redaction.js`                                              | Env-var allowlist + credential-shape pattern redaction.              |
 

package/bin/fit-benchmark.js

@@ -4,10 +4,11 @@ import "@forwardimpact/libpreflight/node22";
 
 import { readFileSync, realpathSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
+import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
 
 import { runBenchmarkRunCommand } from "../src/commands/benchmark-run.js";
-import { runBenchmarkScoreCommand } from "../src/commands/benchmark-score.js";
+import { runBenchmarkInvariantsCommand } from "../src/commands/benchmark-invariants.js";
 import { runBenchmarkReportCommand } from "../src/commands/benchmark-report.js";
 
 // `bun build --compile` injects FIT_BENCHMARK_VERSION via --define, eliminating
@@ -26,7 +27,8 @@ export const definition = {
   commands: [
     {
       name: "run",
-      args: "",
+      args: [],
+      handler: runBenchmarkRunCommand,
       description:
         "Run every task in a family for N runs and emit one result record per (task, runIndex).",
       options: {
@@ -78,10 +80,11 @@ export const definition = {
       },
     },
     {
-      name: "score",
-      args: "",
+      name: "invariants",
+      args: [],
+      handler: runBenchmarkInvariantsCommand,
       description:
-        "Score a single task against a post-run workdir without invoking an agent.",
+        "Check a single task's invariants against a post-run workdir without invoking an agent.",
       options: {
         family: {
           type: "string",
@@ -94,7 +97,7 @@ export const definition = {
         workdir: {
           type: "string",
           description:
-            "Post-run directory; <workdir>/cwd/ is the agent CWD scoring runs against",
+            "Post-run directory; <workdir>/cwd/ is the agent CWD invariants run against",
         },
         output: {
           type: "string",
@@ -104,7 +107,8 @@ export const definition = {
     },
     {
       name: "report",
-      args: "",
+      args: [],
+      handler: runBenchmarkReportCommand,
       description:
         "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
       options: {
@@ -132,7 +136,7 @@ export const definition = {
   examples: [
     "fit-benchmark run --family=./families/coding",
     "fit-benchmark run --family=./families/coding --runs=10 --agent-model=claude-sonnet-4-6",
-    "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark invariants --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
     "fit-benchmark report --format=text",
     "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
   ],
@@ -152,35 +156,30 @@ export const definition = {
   ],
 };
 
-const cli = createCli(definition);
 const logger = createLogger("benchmark");
 
-const COMMANDS = {
-  run: runBenchmarkRunCommand,
-  score: runBenchmarkScoreCommand,
-  report: runBenchmarkReportCommand,
-};
-
 async function main() {
-  const parsed = cli.parse(process.argv.slice(2));
-  if (!parsed) process.exit(0);
-
-  const { values, positionals } = parsed;
+  const runtime = createDefaultRuntime();
+  const cli = createCli(definition, { runtime });
+  const parsed = cli.parse(runtime.proc.argv.slice(2));
+  if (!parsed) return runtime.proc.exit(0);
 
+  const { positionals } = parsed;
   if (positionals.length === 0) {
     cli.usageError("no command specified");
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  const [command, ...args] = positionals;
-  const handler = COMMANDS[command];
-
-  if (!handler) {
+  const command = positionals[0];
+  if (!definition.commands.some((c) => c.name === command)) {
     cli.usageError(`unknown command "${command}"`);
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  await handler(values, args);
+  const result = await cli.dispatch(parsed, { deps: { runtime } });
+  const envelope = result ?? { ok: true };
+  if (!envelope.ok && envelope.error) cli.error(envelope.error);
+  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
 }
 
 // Run main only when invoked as a CLI. Importing for tests (e.g. parity)
@@ -188,7 +187,7 @@ async function main() {
 if (import.meta.url === `file://${realpathSync(process.argv[1])}`) {
   main().catch((error) => {
     logger.exception("main", error);
-    cli.error(error.message);
+    createCli(definition).error(error.message);
     process.exit(1);
   });
 }

package/bin/fit-eval.js

@@ -4,6 +4,7 @@ import "@forwardimpact/libpreflight/node22";
 
 import { readFileSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
+import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
 
 import { runOutputCommand } from "../src/commands/output.js";
@@ -14,6 +15,19 @@ import { runFacilitateCommand } from "../src/commands/facilitate.js";
 import { runDiscussCommand } from "../src/commands/discuss.js";
 import { runCallbackCommand } from "../src/commands/callback.js";
 
+// `tee` streams stdin→stdout via Node's `pipeline`, which needs real stream
+// objects the runtime surface does not expose; it keeps the legacy
+// `(values, args)` signature and this adapter bridges it into dispatch.
+async function teeHandler(ctx) {
+  const out = ctx.args.output;
+  try {
+    await runTeeCommand(ctx.options, out ? [out] : []);
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, code: 1, error: error.message };
+  }
+}
+
 // `bun build --compile` injects FIT_EVAL_VERSION via --define, eliminating
 // the readFileSync branch in the compiled binary (which would ENOENT against
 // the bunfs virtual mount). Source execution falls through to package.json.
@@ -34,6 +48,29 @@ const LEAD_OPTIONS = {
   },
 };
 
+// Shared task-input flags: --task-file (path), --task-text (inline), and
+// --task-event (path to native GitHub event JSON composed into a task via
+// libeval/src/events/github.js). Exactly one of the three is required.
+const TASK_INPUT_OPTIONS = {
+  "task-file": {
+    type: "string",
+    description: "Path to a markdown task file",
+  },
+  "task-text": {
+    type: "string",
+    description: "Inline task text (alternative to --task-file)",
+  },
+  "task-event": {
+    type: "string",
+    description:
+      "Path to a native GitHub event payload JSON, composed into the task via libeval/src/events/github.js (reads $GITHUB_EVENT_NAME)",
+  },
+  "task-amend": {
+    type: "string",
+    description: "Additional text appended to the task",
+  },
+};
+
 const definition = {
   name: "fit-eval",
   version: VERSION,
@@ -42,21 +79,12 @@ const definition = {
   commands: [
     {
       name: "run",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runRunCommand,
       description: "Run a single agent autonomously on a defined task",
       options: {
-        "task-file": {
-          type: "string",
-          description: "Path to a markdown task file",
-        },
-        "task-text": {
-          type: "string",
-          description: "Inline task text (alternative to --task-file)",
-        },
-        "task-amend": {
-          type: "string",
-          description: "Additional text appended to the task",
-        },
+        ...TASK_INPUT_OPTIONS,
         "agent-model": {
           type: "string",
           description:
@@ -88,22 +116,13 @@ const definition = {
     },
     {
       name: "supervise",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runSuperviseCommand,
       description:
         "Run a supervisor–agent relay — typical shape for agent-as-judge evaluations",
       options: {
-        "task-file": {
-          type: "string",
-          description: "Path to a markdown task file",
-        },
-        "task-text": {
-          type: "string",
-          description: "Inline task text (alternative to --task-file)",
-        },
-        "task-amend": {
-          type: "string",
-          description: "Additional text appended to the task",
-        },
+        ...TASK_INPUT_OPTIONS,
         "agent-model": {
           type: "string",
           description:
@@ -142,22 +161,13 @@ const definition = {
     },
     {
       name: "facilitate",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runFacilitateCommand,
       description:
         "Run a facilitator with N participants — typical shape for multi-agent collaboration",
       options: {
-        "task-file": {
-          type: "string",
-          description: "Path to a markdown task file",
-        },
-        "task-text": {
-          type: "string",
-          description: "Inline task text (alternative to --task-file)",
-        },
-        "task-amend": {
-          type: "string",
-          description: "Additional text appended to the task",
-        },
+        ...TASK_INPUT_OPTIONS,
         "agent-model": {
           type: "string",
           description: "Claude model for agents (default: claude-opus-4-7[1m])",
@@ -188,22 +198,13 @@ const definition = {
     },
     {
       name: "discuss",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runDiscussCommand,
       description:
         "Run an async, suspendable discussion — Chair + N participants + bridge callback",
       options: {
-        "task-file": {
-          type: "string",
-          description: "Path to a markdown task file",
-        },
-        "task-text": {
-          type: "string",
-          description: "Inline task text (alternative to --task-file)",
-        },
-        "task-amend": {
-          type: "string",
-          description: "Additional text appended to the task",
-        },
+        ...TASK_INPUT_OPTIONS,
         "agent-model": {
           type: "string",
           description: "Claude model for agents (default: claude-opus-4-7[1m])",
@@ -238,19 +239,25 @@ const definition = {
     },
     {
       name: "output",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runOutputCommand,
       description:
         "Read NDJSON from stdin and emit a structured or readable form",
     },
     {
       name: "tee",
-      args: "[output.ndjson]",
+      args: ["output"],
+      argsUsage: "[output.ndjson]",
+      handler: teeHandler,
       description:
         "Stream readable text to stdout while saving raw NDJSON to a file",
     },
     {
       name: "callback",
-      args: "",
+      args: [],
+      argsUsage: "",
+      handler: runCallbackCommand,
       description:
         "Extract the terminal summary from an NDJSON trace and POST it to a callback URL",
       options: {
@@ -319,43 +326,34 @@ const definition = {
   ],
 };
 
-const cli = createCli(definition);
 const logger = createLogger("eval");
 
-const COMMANDS = {
-  output: runOutputCommand,
-  tee: runTeeCommand,
-  run: runRunCommand,
-  supervise: runSuperviseCommand,
-  facilitate: runFacilitateCommand,
-  discuss: runDiscussCommand,
-  callback: runCallbackCommand,
-};
-
 async function main() {
-  const parsed = cli.parse(process.argv.slice(2));
-  if (!parsed) process.exit(0);
-
-  const { values, positionals } = parsed;
+  const runtime = createDefaultRuntime();
+  const cli = createCli(definition, { runtime });
+  const parsed = cli.parse(runtime.proc.argv.slice(2));
+  if (!parsed) return runtime.proc.exit(0);
 
+  const { positionals } = parsed;
   if (positionals.length === 0) {
     cli.usageError("no command specified");
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  const [command, ...args] = positionals;
-  const handler = COMMANDS[command];
-
-  if (!handler) {
+  const command = positionals[0];
+  if (!definition.commands.some((c) => c.name === command)) {
     cli.usageError(`unknown command "${command}"`);
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  await handler(values, args);
+  const result = await cli.dispatch(parsed, { deps: { runtime } });
+  const envelope = result ?? { ok: true };
+  if (!envelope.ok && envelope.error) cli.error(envelope.error);
+  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
 }
 
 main().catch((error) => {
   logger.exception("main", error);
-  cli.error(error.message);
+  createCli(definition).error(error.message);
   process.exit(1);
 });

package/bin/fit-trace.js

@@ -4,6 +4,7 @@ import "@forwardimpact/libpreflight/node22";
 
 import { readFileSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
+import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createScriptConfig } from "@forwardimpact/libconfig";
 import { createLogger } from "@forwardimpact/libtelemetry";
 
@@ -46,7 +47,9 @@ const definition = {
   commands: [
     {
       name: "runs",
-      args: "[pattern]",
+      args: ["pattern"],
+      argsUsage: "[pattern]",
+      handler: runRunsCommand,
       description:
         "List recent GitHub Actions workflow runs (default pattern: agent)",
       options: {
@@ -63,7 +66,9 @@ const definition = {
     },
     {
       name: "download",
-      args: "<run-id>",
+      args: ["run-id"],
+      argsUsage: "<run-id>",
+      handler: runDownloadCommand,
       description: "Download trace artifact and convert to structured JSON",
       options: {
         dir: { type: "string", description: "Output directory" },
@@ -77,32 +82,44 @@ const definition = {
     },
     {
       name: "overview",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runOverviewCommand,
       description: "Metadata, summary, turn count, tool frequency",
     },
     {
       name: "count",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runCountCommand,
       description: "Number of turns",
     },
     {
       name: "batch",
-      args: "<file> <from> <to>",
+      args: ["file", "from", "to"],
+      argsUsage: "<file> <from> <to>",
+      handler: runBatchCommand,
       description: "Turns in range [from, to) (zero-indexed)",
     },
     {
       name: "head",
-      args: "<file> [N]",
+      args: ["file", "n"],
+      argsUsage: "<file> [N]",
+      handler: runHeadCommand,
       description: "First N turns (default 10)",
     },
     {
       name: "tail",
-      args: "<file> [N]",
+      args: ["file", "n"],
+      argsUsage: "<file> [N]",
+      handler: runTailCommand,
       description: "Last N turns (default 10)",
     },
     {
       name: "search",
-      args: "<file> <pattern>",
+      args: ["file", "pattern"],
+      argsUsage: "<file> <pattern>",
+      handler: runSearchCommand,
       description: "Search all content for regex pattern",
       options: {
         limit: {
@@ -121,22 +138,30 @@ const definition = {
     },
     {
       name: "tools",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runToolsCommand,
       description: "Tool usage frequency (descending)",
     },
     {
       name: "tool",
-      args: "<file> <name>",
+      args: ["file", "name"],
+      argsUsage: "<file> <name>",
+      handler: runToolCommand,
       description: "All turns involving a specific tool",
     },
     {
       name: "errors",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runErrorsCommand,
       description: "Tool results with isError=true",
     },
     {
       name: "reasoning",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runReasoningCommand,
       description: "Agent reasoning text only",
       options: {
         from: { type: "string", description: "Start at turn index" },
@@ -145,27 +170,37 @@ const definition = {
     },
     {
       name: "timeline",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runTimelineCommand,
       description: "Compact one-line-per-turn overview",
     },
     {
       name: "stats",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runStatsCommand,
       description: "Token usage and cost breakdown",
     },
     {
       name: "init",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runInitCommand,
       description: "Full system/init event",
     },
     {
       name: "turn",
-      args: "<file> <index>",
+      args: ["file", "index"],
+      argsUsage: "<file> <index>",
+      handler: runTurnCommand,
       description: "Single turn by index",
     },
     {
       name: "by-discussion",
-      args: "<discussion-id> [trace-dir]",
+      args: ["discussion-id", "trace-dir"],
+      argsUsage: "<discussion-id> [trace-dir]",
+      handler: runByDiscussionCommand,
       description:
         "List trace files whose meta header carries the given discussion_id, ordered by first-event timestamp",
       options: {
@@ -177,7 +212,9 @@ const definition = {
     },
     {
       name: "filter",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runFilterCommand,
       description: "Filter turns by role, tool, or error status",
       options: {
         role: {
@@ -196,7 +233,9 @@ const definition = {
     },
     {
       name: "split",
-      args: "<file>",
+      args: ["file"],
+      argsUsage: "<file>",
+      handler: runSplitCommand,
       description:
         "Split a combined trace into per-source files following the `trace--<case>--<participant>.<role>.ndjson` convention",
       options: {
@@ -217,9 +256,11 @@ const definition = {
     },
     {
       name: "assert",
-      args: "<test-name> <file>",
+      args: ["test-name", "file"],
+      argsUsage: "<test-name> <file>",
+      handler: runAssertCommand,
       description:
-        "Shell-friendly assertion — outputs structured JSON for scoring hooks",
+        "Shell-friendly assertion — outputs structured JSON for invariant hooks",
       options: {
         grep: {
           type: "string",
@@ -299,57 +340,42 @@ const definition = {
   ],
 };
 
-const cli = createCli(definition);
 const logger = createLogger("trace");
 
-const COMMANDS = {
-  runs: runRunsCommand,
-  download: runDownloadCommand,
-  overview: runOverviewCommand,
-  count: runCountCommand,
-  batch: runBatchCommand,
-  head: runHeadCommand,
-  tail: runTailCommand,
-  search: runSearchCommand,
-  tools: runToolsCommand,
-  tool: runToolCommand,
-  errors: runErrorsCommand,
-  reasoning: runReasoningCommand,
-  timeline: runTimelineCommand,
-  stats: runStatsCommand,
-  init: runInitCommand,
-  turn: runTurnCommand,
-  filter: runFilterCommand,
-  split: runSplitCommand,
-  assert: runAssertCommand,
-  "by-discussion": runByDiscussionCommand,
-};
+// Commands that talk to the GitHub API need a config-backed token resolver;
+// the rest only read local trace files through the runtime.
+const NEEDS_CONFIG = new Set(["runs", "download"]);
 
 async function main() {
-  const parsed = cli.parse(process.argv.slice(2));
-  if (!parsed) process.exit(0);
-
-  const { values, positionals } = parsed;
+  const runtime = createDefaultRuntime();
+  const cli = createCli(definition, { runtime });
+  const parsed = cli.parse(runtime.proc.argv.slice(2));
+  if (!parsed) return runtime.proc.exit(0);
 
+  const { positionals } = parsed;
   if (positionals.length === 0) {
     cli.usageError("no command specified");
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  const [command, ...args] = positionals;
-  const handler = COMMANDS[command];
-
-  if (!handler) {
+  const command = positionals[0];
+  if (!definition.commands.some((c) => c.name === command)) {
     cli.usageError(`unknown command "${command}"`);
-    process.exit(2);
+    return runtime.proc.exit(2);
   }
 
-  const config = await createScriptConfig("eval");
-  await handler(values, args, { config });
+  const config = NEEDS_CONFIG.has(command)
+    ? await createScriptConfig("eval")
+    : undefined;
+
+  const result = await cli.dispatch(parsed, { deps: { runtime, config } });
+  const envelope = result ?? { ok: true };
+  if (!envelope.ok && envelope.error) cli.error(envelope.error);
+  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
 }
 
 main().catch((error) => {
   logger.exception("main", error);
-  cli.error(error.message);
+  createCli(definition).error(error.message);
   process.exit(1);
 });