@forwardimpact/libharness 1.1.0 → 1.2.1

package/README.md

@@ -2,8 +2,9 @@
 
 <!-- BEGIN:description — Do not edit. Generated from package.json. -->
 
-Agent evaluation framework — prove whether agent changes improved outcomes with
-reproducible evidence.
+Autonomous agent team harness — coordinate a lead and participant agents in one
+async session, with eval, benchmark, and trace tooling to prove the changes
+worked.
 
 <!-- END:description -->
 
@@ -53,7 +54,8 @@ Every Ask returns immediately and registers a pending entry keyed by an
 lead. Answer's `askId` is optional — the handler is forgiving:
 
 - **Provided + matches an ask owed by the caller** → routes to that asker.
-- **Provided but unknown or wrong addressee** → `isError` with a pointed message.
+- **Provided but unknown or wrong addressee** → `isError` with a pointed
+  message.
 - **Omitted + exactly one ask owed to the caller** → auto-picks it.
 - **Omitted + 0 or many asks owed** → broadcasts as Announce.
 
@@ -209,6 +211,17 @@ lists the `Edit()` rules that were tried.
 
 ## Documentation
 
-- [Agent Evaluations Guide](https://www.forwardimpact.team/docs/libraries/agent-evaluations/index.md) — how to run an eval and read its trace.
-- [Agent Collaboration Guide](https://www.forwardimpact.team/docs/libraries/agent-collaboration/index.md) — supervise / facilitate / discuss in depth.
-- [Trace Analysis Guide](https://www.forwardimpact.team/docs/libraries/trace-analysis/index.md) — analysing NDJSON traces with `fit-trace`.
+- [Coordinate an Agent Team](https://www.forwardimpact.team/docs/libraries/coordinate-team/index.md)
+  — run a lead and N participant agents in one async session (supervise /
+  facilitate / discuss) with Ask/Answer/Announce and a single NDJSON trace.
+- [Run an Eval](https://www.forwardimpact.team/docs/libraries/prove-changes/run-eval/index.md)
+  — author a judge profile, run an eval locally, wire it into CI, and inspect
+  the resulting trace.
+- [Prove Agent Changes](https://www.forwardimpact.team/docs/libraries/prove-changes/index.md)
+  — end-to-end workflow from dataset generation through evaluation to trace
+  analysis, including multi-agent collaboration sessions.
+- [Analyze Traces](https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md)
+  — read the NDJSON traces produced by `fit-harness` with `fit-trace`.
+- [Agent Teams](https://www.forwardimpact.team/docs/products/agent-teams/index.md)
+  — author the profiles consumed by `--agent-profile`, `--lead-profile`, and
+  `--agent-profiles`.

package/bin/fit-harness.js

@@ -295,6 +295,12 @@ const definition = {
     "fit-harness output --format=text < trace.ndjson",
   ],
   documentation: [
+    {
+      title: "Coordinate an Agent Team",
+      url: "https://www.forwardimpact.team/docs/libraries/coordinate-team/index.md",
+      description:
+        "Run a lead and N participant agents in one async session — supervise, facilitate, or discuss — with Ask/Answer/Announce and a single NDJSON trace.",
+    },
     {
       title: "Run an Eval",
       url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-eval/index.md",

package/bin/fit-trace.js

@@ -350,7 +350,7 @@ const definition = {
       options: {
         mode: {
           type: "string",
-          description: "Execution mode: run, supervise, or facilitate",
+          description: "Execution mode: run, supervise, facilitate, or discuss",
         },
         case: {
           type: "string",

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "1.1.0",
-  "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
+  "version": "1.2.1",
+  "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
+    "orchestration",
+    "supervisor",
+    "facilitator",
     "eval",
-    "agent",
     "trace",
-    "claude-code",
-    "supervisor"
+    "agent"
   ],
   "homepage": "https://www.forwardimpact.team",
   "repository": {
@@ -18,12 +19,20 @@
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",
   "jobs": [
+    {
+      "user": "Platform Builders",
+      "goal": "Coordinate an Agent Team",
+      "trigger": "Coordinating several agents in one session means hand-rolling message passing, turn-taking, and termination, and every orchestration script re-solves it.",
+      "bigHire": "coordinate a lead and participant agents over async messages in one session.",
+      "littleHire": "run a supervised pair, a facilitated meeting, or a multi-agent discussion without writing the message bus and turn loop.",
+      "competesWith": "manual orchestration scripts; hand-rolled message passing and turn-taking; sequential single-agent calls; running one agent at a time"
+    },
     {
       "user": "Platform Builders",
       "goal": "Prove Agent Changes",
       "trigger": "An eval passes locally but fails in CI and the only output is 'assertion failed.'",
       "bigHire": "prove whether agent changes improved outcomes with reproducible evidence.",
-      "littleHire": "run an eval and get a trace that shows exactly what the agent did.",
+      "littleHire": "run an eval or benchmark and get a trace that shows exactly what the agent did.",
       "competesWith": "manual before/after comparison; trusting gut feeling over evidence; skipping evaluation entirely"
     }
   ],

package/src/agent-runner.js

@@ -7,6 +7,7 @@
  */
 
 import { AGENT_MODEL } from "@forwardimpact/libutil/models";
+import { resolveClaudeCodeExecutable } from "./claude-code-executable.js";
 
 const DEFAULT_ALLOWED_TOOLS = ["Bash", "Read", "Glob", "Grep", "Write", "Edit"];
 
@@ -56,6 +57,10 @@ export class AgentRunner {
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
    * @param {string[]} [deps.disallowedTools] - Tools to explicitly remove from the model's context
    * @param {Record<string, object>} [deps.mcpServers] - MCP server configs to pass to the SDK query
+   * @param {string} [deps.pathToClaudeCodeExecutable] - Absolute path to the
+   *   native `claude` CLI the SDK should spawn. Set for compiled fit-* binaries,
+   *   which can't self-resolve the SDK's platform optional dependency; omitted
+   *   from source runs so the SDK resolves its own version-matched binary.
    * @param {object} deps.redactor
    * @param {import("@forwardimpact/libutil/runtime").Runtime} [deps.runtime] -
    *   Ambient collaborators. Only `proc.env` is read (to record Skill
@@ -79,6 +84,9 @@ export class AgentRunner {
     this.systemPrompt = deps.systemPrompt ?? null;
     this.disallowedTools = deps.disallowedTools ?? [];
     this.mcpServers = deps.mcpServers ?? null;
+    // Optional; read only through a truthy guard in #callOptions, so an absent
+    // value stays undefined rather than needing a `?? null` default.
+    this.pathToClaudeCodeExecutable = deps.pathToClaudeCodeExecutable;
     this.taskAmend = deps.taskAmend ?? null;
     this.sessionId = null;
     /** @type {AbortController|null} */
@@ -158,6 +166,9 @@ export class AgentRunner {
       }),
       ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
       ...(this.mcpServers && { mcpServers: this.mcpServers }),
+      ...(this.pathToClaudeCodeExecutable && {
+        pathToClaudeCodeExecutable: this.pathToClaudeCodeExecutable,
+      }),
     };
   }
 
@@ -250,7 +261,14 @@ export class AgentRunner {
   }
 }
 
-/** Factory function — wires real dependencies. */
+/**
+ * Factory function — wires real dependencies. Resolves the native `claude`
+ * executable for compiled fit-* binaries so the SDK doesn't fail to find its
+ * own platform optional dependency; an explicit `deps` value overrides it.
+ */
 export function createAgentRunner(deps) {
-  return new AgentRunner(deps);
+  return new AgentRunner({
+    pathToClaudeCodeExecutable: resolveClaudeCodeExecutable(),
+    ...deps,
+  });
 }

package/src/claude-code-executable.js

@@ -0,0 +1,44 @@
+/**
+ * Resolve the Claude Code CLI the Agent SDK should spawn.
+ *
+ * `query()` spawns a native `claude` binary that the SDK resolves from its own
+ * platform-specific optional dependency (`@anthropic-ai/claude-agent-sdk-<platform>`).
+ * `bun build --compile` bundles the SDK's JavaScript but not that separate
+ * native package — it is not part of the import graph — so a compiled fit-*
+ * binary can't self-resolve it and `query()` throws "Native CLI binary for
+ * <platform> not found".
+ *
+ * In a compiled binary we point the SDK at the standalone `claude` on PATH,
+ * installed beside fit-harness by the bootstrap action's `fit-install.sh`.
+ * Running from source keeps `node_modules`, where the SDK resolves its own
+ * version-matched binary, so there we return undefined and defer to the SDK.
+ */
+import { LIBCLI_IS_COMPILED } from "@forwardimpact/libcli";
+
+/**
+ * @param {object} [deps]
+ * @param {(cmd: string) => string | null | undefined} [deps.which] -
+ *   PATH resolver (injected for testing).
+ * @param {boolean} [deps.isCompiled] -
+ *   Whether this is a `bun --compile` binary (injected for testing).
+ * @returns {string | undefined} absolute path to `claude`, or undefined to
+ *   defer resolution to the SDK.
+ */
+export function resolveClaudeCodeExecutable({
+  which = defaultWhich,
+  isCompiled = LIBCLI_IS_COMPILED,
+} = {}) {
+  if (!isCompiled) return undefined;
+  return which("claude") ?? undefined;
+}
+
+/**
+ * A compiled fit-* binary runs under the Bun runtime, which exposes a
+ * synchronous PATH resolver. The `typeof` guard keeps this safe under Node too.
+ */
+function defaultWhich(cmd) {
+  if (typeof Bun !== "undefined" && typeof Bun.which === "function") {
+    return Bun.which(cmd);
+  }
+  return null;
+}

package/src/commands/benchmark-definition.js

@@ -169,7 +169,7 @@ export const definition = {
       title: "Automate with GitHub Actions",
       url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
       description:
-        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+        "Run benchmarks in CI with the forwardimpact/benchmark action.",
     },
   ],
 };

package/src/commands/discuss.js

@@ -29,9 +29,11 @@ export function parseDiscussOptions(values, runtime) {
   );
 
   const profilesRaw = values["agent-profiles"];
-  const agentCwd = resolve(values["agent-cwd"] ?? ".");
+  // `||` (not `??`) so an empty-string flag from a CI forwarder falls back to
+  // the default rather than overriding it with "".
+  const agentCwd = resolve(values["agent-cwd"] || ".");
 
-  const maxTurnsRaw = values["max-turns"] ?? "40";
+  const maxTurnsRaw = values["max-turns"] || "40";
   const maxTurns = maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10);
 
   const agentConfigs = parseAgentProfiles(profilesRaw, agentCwd, maxTurns);
@@ -46,21 +48,21 @@ export function parseDiscussOptions(values, runtime) {
     }
   }
 
-  const maxLeadTurnsRaw = values["max-lead-turns"] ?? "200";
+  const maxLeadTurnsRaw = values["max-lead-turns"] || "200";
   const maxLeadTurns = parseInt(maxLeadTurnsRaw, 10);
 
   return {
     taskContent,
     taskAmend,
     agentConfigs,
-    leadProfile: values["lead-profile"] ?? undefined,
+    leadProfile: values["lead-profile"] || undefined,
     leadModel: values["lead-model"] || LEAD_MODEL,
     agentModel: values["agent-model"] || AGENT_MODEL,
     maxTurns,
     maxLeadTurns,
     outputPath: values.output,
     workTracker: resolveWorkTracker(values, runtime?.proc?.env),
-    discussionId: values["discussion-id"] ?? null,
+    discussionId: values["discussion-id"] || null,
     resumeContext,
     callbackUrl: runtime.proc.env.CALLBACK_URL ?? null,
     inboxUrl: runtime.proc.env.INBOX_URL ?? null,

package/src/commands/facilitate.js

@@ -36,9 +36,11 @@ export function parseFacilitateOptions(values, runtime) {
 
   const profilesRaw = values["agent-profiles"];
   if (!profilesRaw) throw new Error("--agent-profiles is required");
-  const agentCwd = resolve(values["agent-cwd"] ?? ".");
+  // `||` (not `??`) so an empty-string flag from a CI forwarder falls back to
+  // the default rather than overriding it with "".
+  const agentCwd = resolve(values["agent-cwd"] || ".");
 
-  const maxTurnsRaw = values["max-turns"] ?? "20";
+  const maxTurnsRaw = values["max-turns"] || "20";
   const maxTurns = maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10);
 
   // Thread --max-turns into each participant: without this, every facilitated
@@ -51,12 +53,12 @@ export function parseFacilitateOptions(values, runtime) {
     taskContent,
     taskAmend,
     agentConfigs,
-    facilitatorCwd: resolve(values["facilitator-cwd"] ?? "."),
+    facilitatorCwd: resolve(values["facilitator-cwd"] || "."),
     agentModel: values["agent-model"] || AGENT_MODEL,
     facilitatorModel: values["lead-model"] || LEAD_MODEL,
     maxTurns,
     outputPath: values.output,
-    facilitatorProfile: values["lead-profile"] ?? undefined,
+    facilitatorProfile: values["lead-profile"] || undefined,
     workTracker: resolveWorkTracker(values, runtime?.proc?.env),
   };
 }

package/src/commands/run.js

@@ -22,22 +22,24 @@ export function parseRunOptions(values, runtime) {
     values,
     runtime,
   );
-  const maxTurnsRaw = values["max-turns"] ?? "50";
+  // `||` (not `??`) so an empty-string flag from a CI forwarder falls back to
+  // the default, rather than overriding it with "".
+  const maxTurnsRaw = values["max-turns"] || "50";
 
   return {
     taskContent,
     taskAmend,
-    cwd: resolve(values.cwd ?? "."),
+    cwd: resolve(values.cwd || "."),
     agentModel: values["agent-model"] || AGENT_MODEL,
     maxTurns: maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10),
     outputPath: values.output,
-    agentProfile: values["agent-profile"] ?? undefined,
+    agentProfile: values["agent-profile"] || undefined,
     workTracker: resolveWorkTracker(values, runtime?.proc?.env),
     allowedTools: (
-      values["allowed-tools"] ??
+      values["allowed-tools"] ||
       "Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
     ).split(","),
-    mcpServer: values["mcp-server"] ?? undefined,
+    mcpServer: values["mcp-server"] || undefined,
   };
 }
 

package/src/commands/supervise.js

@@ -21,35 +21,37 @@ export async function parseSuperviseOptions(values, runtime) {
   );
   const supervisorAllowedToolsRaw = values["supervisor-allowed-tools"];
 
+  // `||` (not `??`) throughout so an empty-string flag from a CI forwarder
+  // falls back to the default rather than overriding it with "".
   const tmpRoot = runtime.proc.env.TMPDIR ?? "/tmp";
   const agentCwd = resolve(
-    values["agent-cwd"] ??
+    values["agent-cwd"] ||
       (await runtime.fs.mkdtemp(join(tmpRoot, "fit-harness-agent-"))),
   );
 
   return {
     taskContent,
     taskAmend,
-    supervisorCwd: resolve(values["supervisor-cwd"] ?? "."),
+    supervisorCwd: resolve(values["supervisor-cwd"] || "."),
     agentCwd,
     agentModel: values["agent-model"] || AGENT_MODEL,
     supervisorModel: values["lead-model"] || LEAD_MODEL,
     maxTurns: (() => {
-      const raw = values["max-turns"] ?? "200";
+      const raw = values["max-turns"] || "200";
       return raw === "0" ? 0 : parseInt(raw, 10);
     })(),
     outputPath: values.output,
-    supervisorProfile: values["lead-profile"] ?? undefined,
-    agentProfile: values["agent-profile"] ?? undefined,
+    supervisorProfile: values["lead-profile"] || undefined,
+    agentProfile: values["agent-profile"] || undefined,
     workTracker: resolveWorkTracker(values, runtime?.proc?.env),
     allowedTools: (
-      values["allowed-tools"] ??
+      values["allowed-tools"] ||
       "Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
     ).split(","),
     supervisorAllowedTools: supervisorAllowedToolsRaw
       ? supervisorAllowedToolsRaw.split(",")
       : undefined,
-    mcpServer: values["mcp-server"] ?? undefined,
+    mcpServer: values["mcp-server"] || undefined,
   };
 }
 

package/src/commands/trace.js

@@ -367,9 +367,12 @@ export async function runStatsCommand(ctx) {
  */
 export async function runCostCommand(ctx) {
   const { runtime } = ctx.deps;
-  const cost = computeTraceCost(
-    runtime.fsSync.readFileSync(ctx.args.file, "utf8"),
-  );
+  // Tolerate a missing/empty trace: a CI step reports cost under `always()`,
+  // so the trace may not exist (the run failed before producing one). Print
+  // nothing and exit 0 rather than throwing — the caller needs no `if [ -f ]`.
+  const file = ctx.args.file;
+  if (!file || !runtime.fsSync.existsSync(file)) return { ok: true };
+  const cost = computeTraceCost(runtime.fsSync.readFileSync(file, "utf8"));
   if (ctx.options.markdown) {
     runtime.proc.stdout.write(renderCostMarkdown(cost));
   } else {
@@ -512,9 +515,12 @@ export async function runSplitCommand(ctx) {
   const file = ctx.args.file;
   if (!file) return { ok: false, code: 1, error: "split: missing input file" };
 
+  // `discuss` has the same lead + N-participants shape as `facilitate`, and the
+  // splitter buckets purely by envelope `source` (mode-independent), so it is
+  // accepted alongside the structural modes — the CLI owns this, not callers.
   const mode = ctx.options.mode;
   if (!mode) return { ok: false, code: 1, error: "split: --mode is required" };
-  if (!["run", "supervise", "facilitate"].includes(mode)) {
+  if (!["run", "supervise", "facilitate", "discuss"].includes(mode)) {
     return { ok: false, code: 1, error: `split: invalid --mode "${mode}"` };
   }
 

package/src/index.js

@@ -11,6 +11,7 @@ export {
   pickTraceArtifact,
 } from "./trace-github.js";
 export { AgentRunner, createAgentRunner } from "./agent-runner.js";
+export { resolveClaudeCodeExecutable } from "./claude-code-executable.js";
 export {
   composeProfilePrompt,
   composeLeadPrompt,