@forwardimpact/libeval 0.1.13 → 0.1.15

package/bin/fit-eval.js

@@ -8,6 +8,7 @@ import { runOutputCommand } from "../src/commands/output.js";
 import { runTeeCommand } from "../src/commands/tee.js";
 import { runRunCommand } from "../src/commands/run.js";
 import { runSuperviseCommand } from "../src/commands/supervise.js";
+import { runFacilitateCommand } from "../src/commands/facilitate.js";
 
 const { version: VERSION } = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8"),
@@ -20,7 +21,7 @@ const definition = {
   commands: [
     {
       name: "output",
-      args: "[--format=FORMAT]",
+      args: "",
       description: "Process trace and output formatted result",
     },
     {
@@ -30,65 +31,111 @@ const definition = {
     },
     {
       name: "run",
-      args: "[options]",
+      args: "",
       description: "Run a single agent via the Claude Agent SDK",
+      options: {
+        "task-file": { type: "string", description: "Path to task file" },
+        "task-text": { type: "string", description: "Inline task text" },
+        "task-amend": {
+          type: "string",
+          description: "Additional text appended to task",
+        },
+        model: { type: "string", description: "Claude model (default: opus)" },
+        "max-turns": {
+          type: "string",
+          description: "Max agentic turns (default: 50)",
+        },
+        output: { type: "string", description: "Write NDJSON trace to file" },
+        cwd: { type: "string", description: "Working directory" },
+        "agent-profile": { type: "string", description: "Agent profile name" },
+        "allowed-tools": {
+          type: "string",
+          description: "Comma-separated tool list",
+        },
+      },
     },
     {
       name: "supervise",
-      args: "[options]",
+      args: "",
       description: "Run a supervised agent-supervisor relay loop",
+      options: {
+        "task-file": { type: "string", description: "Path to task file" },
+        "task-text": { type: "string", description: "Inline task text" },
+        "task-amend": {
+          type: "string",
+          description: "Additional text appended to task",
+        },
+        model: { type: "string", description: "Claude model (default: opus)" },
+        "max-turns": {
+          type: "string",
+          description: "Max agentic turns (default: 50)",
+        },
+        output: { type: "string", description: "Write NDJSON trace to file" },
+        cwd: { type: "string", description: "Working directory" },
+        "agent-profile": { type: "string", description: "Agent profile name" },
+        "allowed-tools": {
+          type: "string",
+          description: "Comma-separated tool list",
+        },
+        "supervisor-cwd": {
+          type: "string",
+          description: "Supervisor working directory",
+        },
+        "agent-cwd": { type: "string", description: "Agent working directory" },
+        "supervisor-profile": {
+          type: "string",
+          description: "Supervisor profile name",
+        },
+        "supervisor-allowed-tools": {
+          type: "string",
+          description: "Supervisor tool list",
+        },
+      },
+    },
+    {
+      name: "facilitate",
+      args: "",
+      description: "Run a facilitated multi-agent session",
+      options: {
+        "task-file": { type: "string", description: "Path to task file" },
+        "task-text": { type: "string", description: "Inline task text" },
+        "task-amend": {
+          type: "string",
+          description: "Additional text appended to task",
+        },
+        model: { type: "string", description: "Claude model (default: opus)" },
+        "max-turns": {
+          type: "string",
+          description: "Max facilitator LLM turns (default: 20)",
+        },
+        output: { type: "string", description: "Write NDJSON trace to file" },
+        "facilitator-cwd": {
+          type: "string",
+          description: "Facilitator working directory",
+        },
+        "facilitator-profile": {
+          type: "string",
+          description: "Facilitator profile name",
+        },
+        agents: {
+          type: "string",
+          description:
+            "Agent configs: name1:cwd=/tmp/a:role=explorer,name2:cwd=/tmp/b:role=tester",
+        },
+      },
     },
   ],
-  options: {
+  globalOptions: {
     format: { type: "string", description: "Output format (json|text)" },
     help: { type: "boolean", short: "h", description: "Show this help" },
     version: { type: "boolean", description: "Show version" },
     json: { type: "boolean", description: "Output help as JSON" },
-    "task-file": { type: "string", description: "Path to task file" },
-    "task-text": { type: "string", description: "Inline task text" },
-    "task-amend": {
-      type: "string",
-      description: "Additional text appended to task",
-    },
-    model: {
-      type: "string",
-      description: "Claude model (default: opus)",
-    },
-    "max-turns": {
-      type: "string",
-      description: "Max agentic turns (default: 50)",
-    },
-    output: { type: "string", description: "Write NDJSON trace to file" },
-    cwd: { type: "string", description: "Working directory" },
-    "agent-profile": {
-      type: "string",
-      description: "Agent profile name",
-    },
-    "allowed-tools": {
-      type: "string",
-      description: "Comma-separated tool list",
-    },
-    "supervisor-cwd": {
-      type: "string",
-      description: "Supervisor working directory",
-    },
-    "agent-cwd": {
-      type: "string",
-      description: "Agent working directory",
-    },
-    "supervisor-profile": {
-      type: "string",
-      description: "Supervisor profile name",
-    },
-    "supervisor-allowed-tools": {
-      type: "string",
-      description: "Supervisor tool list",
-    },
   },
   examples: [
     "fit-eval output --format=text < trace.ndjson",
     "fit-eval run --task-file=task.md --model=opus",
     "fit-eval supervise --task-file=task.md --supervisor-cwd=.",
+    'fit-eval facilitate --task-file=task.md --agents "explorer:cwd=/tmp/a,tester:cwd=/tmp/b"',
   ],
 };
 
@@ -100,6 +147,7 @@ const COMMANDS = {
   tee: runTeeCommand,
   run: runRunCommand,
   supervise: runSuperviseCommand,
+  facilitate: runFacilitateCommand,
 };
 
 async function main() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "Process Claude Code stream-json output into structured traces",
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",
@@ -28,7 +28,8 @@
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.98",
     "@forwardimpact/libcli": "^0.1.0",
-    "@forwardimpact/libtelemetry": "^0.1.22"
+    "@forwardimpact/libtelemetry": "^0.1.22",
+    "zod": "^3.23.0"
   },
   "publishConfig": {
     "access": "public"

package/src/agent-runner.js

@@ -6,6 +6,28 @@
  * Follows OO+DI: constructor injection, factory function, tests bypass factory.
  */
 
+const DEFAULT_ALLOWED_TOOLS = ["Bash", "Read", "Glob", "Grep", "Write", "Edit"];
+
+function applyDefaults(deps) {
+  return {
+    cwd: deps.cwd,
+    query: deps.query,
+    output: deps.output,
+    model: deps.model ?? "opus",
+    maxTurns: deps.maxTurns ?? 50,
+    allowedTools: deps.allowedTools ?? DEFAULT_ALLOWED_TOOLS,
+    permissionMode: deps.permissionMode ?? "bypassPermissions",
+    onLine: deps.onLine ?? null,
+    onBatch: deps.onBatch ?? null,
+    batchSize: deps.batchSize ?? 3,
+    settingSources: deps.settingSources ?? [],
+    agentProfile: deps.agentProfile ?? null,
+    systemPrompt: deps.systemPrompt ?? null,
+    disallowedTools: deps.disallowedTools ?? [],
+    mcpServers: deps.mcpServers ?? null,
+  };
+}
+
 export class AgentRunner {
   /**
    * @param {object} deps
@@ -23,47 +45,13 @@ export class AgentRunner {
    * @param {string} [deps.agentProfile] - Agent profile name to pass as --agent to the Claude CLI
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
    * @param {string[]} [deps.disallowedTools] - Tools to explicitly remove from the model's context
+   * @param {Record<string, object>} [deps.mcpServers] - MCP server configs to pass to the SDK query
    */
-  constructor({
-    cwd,
-    query,
-    output,
-    model,
-    maxTurns,
-    allowedTools,
-    permissionMode,
-    onLine,
-    onBatch,
-    batchSize,
-    settingSources,
-    agentProfile,
-    systemPrompt,
-    disallowedTools,
-  }) {
-    if (!cwd) throw new Error("cwd is required");
-    if (!query) throw new Error("query is required");
-    if (!output) throw new Error("output is required");
-    this.cwd = cwd;
-    this.query = query;
-    this.output = output;
-    this.model = model ?? "opus";
-    this.maxTurns = maxTurns ?? 50; // 0 means unlimited (omit from SDK)
-    this.allowedTools = allowedTools ?? [
-      "Bash",
-      "Read",
-      "Glob",
-      "Grep",
-      "Write",
-      "Edit",
-    ];
-    this.permissionMode = permissionMode ?? "bypassPermissions";
-    this.onLine = onLine ?? null;
-    this.onBatch = onBatch ?? null;
-    this.batchSize = batchSize ?? 3;
-    this.settingSources = settingSources ?? [];
-    this.agentProfile = agentProfile ?? null;
-    this.systemPrompt = systemPrompt ?? null;
-    this.disallowedTools = disallowedTools ?? [];
+  constructor(deps) {
+    if (!deps.cwd) throw new Error("cwd is required");
+    if (!deps.query) throw new Error("query is required");
+    if (!deps.output) throw new Error("output is required");
+    Object.assign(this, applyDefaults(deps));
     this.sessionId = null;
     this.buffer = [];
     /** @type {AbortController|null} */
@@ -95,6 +83,7 @@ export class AgentRunner {
           }),
           ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
           ...(this.agentProfile && { extraArgs: { agent: this.agentProfile } }),
+          ...(this.mcpServers && { mcpServers: this.mcpServers }),
         },
       });
       return await this.#consumeQuery(iterator);

package/src/commands/facilitate.js

@@ -0,0 +1,109 @@
+import { readFileSync, createWriteStream, mkdtempSync } from "node:fs";
+import { resolve, join } from "node:path";
+import { tmpdir } from "node:os";
+import { createFacilitator } from "../facilitator.js";
+import { createTeeWriter } from "../tee-writer.js";
+
+/**
+ * Parse agent config string into structured configs.
+ * Format: "name1:key=val:key=val,name2:key=val"
+ * @param {string} raw
+ * @returns {Array<{name: string, role: string, cwd: string, maxTurns?: number}>}
+ */
+function parseAgentConfigs(raw) {
+  return raw.split(",").map((spec) => {
+    const parts = spec.split(":");
+    const name = parts[0];
+    const config = { name, role: name };
+    for (let i = 1; i < parts.length; i++) {
+      const [key, val] = parts[i].split("=");
+      if (key === "cwd") config.cwd = resolve(val);
+      else if (key === "role") config.role = val;
+      else if (key === "maxTurns") config.maxTurns = parseInt(val, 10);
+    }
+    if (!config.cwd) {
+      config.cwd = mkdtempSync(join(tmpdir(), `fit-eval-${name}-`));
+    }
+    return config;
+  });
+}
+
+/**
+ * Parse and validate facilitate command options.
+ * @param {object} values - Parsed option values
+ * @returns {object} Parsed options
+ */
+function parseFacilitateOptions(values) {
+  const taskFile = values["task-file"];
+  const taskText = values["task-text"];
+  if (taskFile && taskText)
+    throw new Error("--task-file and --task-text are mutually exclusive");
+  if (!taskFile && !taskText)
+    throw new Error("--task-file or --task-text is required");
+
+  const taskAmend = values["task-amend"] ?? undefined;
+  let taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  if (taskAmend) taskContent += `\n\n${taskAmend}`;
+
+  const agentsRaw = values.agents;
+  if (!agentsRaw) throw new Error("--agents is required");
+
+  const agentConfigs = parseAgentConfigs(agentsRaw);
+  if (agentConfigs.length < 1)
+    throw new Error("--agents must specify at least one agent");
+
+  const maxTurnsRaw = values["max-turns"] ?? "20";
+
+  return {
+    taskContent,
+    agentConfigs,
+    facilitatorCwd: resolve(values["facilitator-cwd"] ?? "."),
+    model: values.model ?? "opus",
+    maxTurns: maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10),
+    outputPath: values.output,
+    facilitatorProfile: values["facilitator-profile"] ?? undefined,
+  };
+}
+
+/**
+ * Facilitate command — run a facilitated multi-agent session.
+ *
+ * Usage: fit-eval facilitate [options]
+ *
+ * @param {object} values - Parsed option values from cli.parse()
+ * @param {string[]} _args - Positional arguments
+ */
+export async function runFacilitateCommand(values, _args) {
+  const opts = parseFacilitateOptions(values);
+
+  const fileStream = opts.outputPath
+    ? createWriteStream(opts.outputPath)
+    : null;
+  const output = fileStream
+    ? createTeeWriter({
+        fileStream,
+        textStream: process.stdout,
+        mode: "supervised",
+      })
+    : process.stdout;
+
+  const { query } = await import("@anthropic-ai/claude-agent-sdk");
+  const facilitator = createFacilitator({
+    facilitatorCwd: opts.facilitatorCwd,
+    agentConfigs: opts.agentConfigs,
+    query,
+    output,
+    model: opts.model,
+    maxTurns: opts.maxTurns,
+    facilitatorProfile: opts.facilitatorProfile,
+  });
+
+  const result = await facilitator.run(opts.taskContent);
+
+  if (fileStream) {
+    await new Promise((r) => output.end(r));
+    await new Promise((r) => fileStream.end(r));
+  }
+
+  process.exit(result.success ? 0 : 1);
+}

package/src/commands/run.js

@@ -1,7 +1,9 @@
 import { readFileSync, createWriteStream } from "node:fs";
+import { Writable } from "node:stream";
 import { resolve } from "node:path";
 import { createAgentRunner } from "../agent-runner.js";
 import { createTeeWriter } from "../tee-writer.js";
+import { SequenceCounter } from "../sequence-counter.js";
 
 /**
  * Parse and validate run command options from parsed values.
@@ -61,14 +63,28 @@ export async function runRunCommand(values, _args) {
     ? createTeeWriter({ fileStream, textStream: process.stdout, mode: "raw" })
     : process.stdout;
 
+  const counter = new SequenceCounter();
+  const devNull = new Writable({
+    write(_chunk, _enc, cb) {
+      cb();
+    },
+  });
+  const onLine = (line) => {
+    const event = JSON.parse(line);
+    output.write(
+      JSON.stringify({ source: "agent", seq: counter.next(), event }) + "\n",
+    );
+  };
+
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const runner = createAgentRunner({
     cwd,
     query,
-    output,
+    output: devNull,
     model,
     maxTurns,
     allowedTools,
+    onLine,
     settingSources: ["project"],
     agentProfile,
   });