@forwardimpact/libeval 0.1.14 → 0.1.16

package/bin/fit-eval.js

@@ -8,6 +8,7 @@ import { runOutputCommand } from "../src/commands/output.js";
 import { runTeeCommand } from "../src/commands/tee.js";
 import { runRunCommand } from "../src/commands/run.js";
 import { runSuperviseCommand } from "../src/commands/supervise.js";
+import { runFacilitateCommand } from "../src/commands/facilitate.js";
 
 const { version: VERSION } = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8"),
@@ -91,6 +92,41 @@ const definition = {
         },
       },
     },
+    {
+      name: "facilitate",
+      args: "",
+      description: "Run a facilitated multi-agent session",
+      options: {
+        "task-file": { type: "string", description: "Path to task file" },
+        "task-text": { type: "string", description: "Inline task text" },
+        "task-amend": {
+          type: "string",
+          description: "Additional text appended to task",
+        },
+        model: { type: "string", description: "Claude model (default: opus)" },
+        "max-turns": {
+          type: "string",
+          description: "Max facilitator LLM turns (default: 20)",
+        },
+        output: { type: "string", description: "Write NDJSON trace to file" },
+        "facilitator-cwd": {
+          type: "string",
+          description: "Facilitator working directory",
+        },
+        "facilitator-profile": {
+          type: "string",
+          description: "Facilitator profile name",
+        },
+        "agent-profiles": {
+          type: "string",
+          description: "Comma-separated agent profile names",
+        },
+        "agent-cwd": {
+          type: "string",
+          description: "Agent working directory (default: .)",
+        },
+      },
+    },
   ],
   globalOptions: {
     format: { type: "string", description: "Output format (json|text)" },
@@ -102,6 +138,7 @@ const definition = {
     "fit-eval output --format=text < trace.ndjson",
     "fit-eval run --task-file=task.md --model=opus",
     "fit-eval supervise --task-file=task.md --supervisor-cwd=.",
+    'fit-eval facilitate --task-file=task.md --agent-profiles "security-engineer,technical-writer"',
   ],
 };
 
@@ -113,6 +150,7 @@ const COMMANDS = {
   tee: runTeeCommand,
   run: runRunCommand,
   supervise: runSuperviseCommand,
+  facilitate: runFacilitateCommand,
 };
 
 async function main() {

package/bin/fit-trace.js

@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+
+import { readFileSync } from "node:fs";
+import { createCli } from "@forwardimpact/libcli";
+import { createLogger } from "@forwardimpact/libtelemetry";
+
+import {
+  runRunsCommand,
+  runDownloadCommand,
+  runOverviewCommand,
+  runCountCommand,
+  runBatchCommand,
+  runHeadCommand,
+  runTailCommand,
+  runSearchCommand,
+  runToolsCommand,
+  runToolCommand,
+  runErrorsCommand,
+  runReasoningCommand,
+  runTimelineCommand,
+  runStatsCommand,
+} from "../src/commands/trace.js";
+
+const { version: VERSION } = JSON.parse(
+  readFileSync(new URL("../package.json", import.meta.url), "utf8"),
+);
+
+const definition = {
+  name: "fit-trace",
+  version: VERSION,
+  description: "Download, query, and search agent execution traces",
+  commands: [
+    {
+      name: "runs",
+      args: "[pattern]",
+      description: "List recent workflow runs (default pattern: agent)",
+      options: {
+        lookback: {
+          type: "string",
+          description: "How far back to search (default: 7d)",
+        },
+        repo: {
+          type: "string",
+          description: "GitHub repo override (default: git remote)",
+        },
+      },
+    },
+    {
+      name: "download",
+      args: "<run-id>",
+      description: "Download trace artifact and convert to structured JSON",
+      options: {
+        dir: { type: "string", description: "Output directory" },
+        artifact: { type: "string", description: "Artifact name override" },
+        repo: {
+          type: "string",
+          description: "GitHub repo override (default: git remote)",
+        },
+      },
+    },
+    {
+      name: "overview",
+      args: "<file>",
+      description: "Metadata, summary, turn count, tool frequency",
+    },
+    {
+      name: "count",
+      args: "<file>",
+      description: "Number of turns",
+    },
+    {
+      name: "batch",
+      args: "<file> <from> <to>",
+      description: "Turns in range [from, to) (zero-indexed)",
+    },
+    {
+      name: "head",
+      args: "<file> [N]",
+      description: "First N turns (default 10)",
+    },
+    {
+      name: "tail",
+      args: "<file> [N]",
+      description: "Last N turns (default 10)",
+    },
+    {
+      name: "search",
+      args: "<file> <pattern>",
+      description: "Search all content for regex pattern",
+      options: {
+        limit: {
+          type: "string",
+          description: "Max results (default: 50)",
+        },
+        context: {
+          type: "string",
+          description: "Surrounding turns per hit (default: 0)",
+        },
+      },
+    },
+    {
+      name: "tools",
+      args: "<file>",
+      description: "Tool usage frequency (descending)",
+    },
+    {
+      name: "tool",
+      args: "<file> <name>",
+      description: "All turns involving a specific tool",
+    },
+    {
+      name: "errors",
+      args: "<file>",
+      description: "Tool results with isError=true",
+    },
+    {
+      name: "reasoning",
+      args: "<file>",
+      description: "Agent reasoning text only",
+      options: {
+        from: { type: "string", description: "Start at turn index" },
+        to: { type: "string", description: "Stop before turn index" },
+      },
+    },
+    {
+      name: "timeline",
+      args: "<file>",
+      description: "Compact one-line-per-turn overview",
+    },
+    {
+      name: "stats",
+      args: "<file>",
+      description: "Token usage and cost breakdown",
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-trace runs --lookback 7d",
+    "fit-trace download 24497273755",
+    "fit-trace overview structured.json",
+    "fit-trace timeline structured.json",
+    "fit-trace search structured.json 'error|fail' --context 1",
+    "fit-trace tool structured.json Bash",
+    "fit-trace batch structured.json 0 20",
+  ],
+};
+
+const cli = createCli(definition);
+const logger = createLogger("trace");
+
+const COMMANDS = {
+  runs: runRunsCommand,
+  download: runDownloadCommand,
+  overview: runOverviewCommand,
+  count: runCountCommand,
+  batch: runBatchCommand,
+  head: runHeadCommand,
+  tail: runTailCommand,
+  search: runSearchCommand,
+  tools: runToolsCommand,
+  tool: runToolCommand,
+  errors: runErrorsCommand,
+  reasoning: runReasoningCommand,
+  timeline: runTimelineCommand,
+  stats: runStatsCommand,
+};
+
+async function main() {
+  const parsed = cli.parse(process.argv.slice(2));
+  if (!parsed) process.exit(0);
+
+  const { values, positionals } = parsed;
+
+  if (positionals.length === 0) {
+    cli.usageError("no command specified");
+    process.exit(2);
+  }
+
+  const [command, ...args] = positionals;
+  const handler = COMMANDS[command];
+
+  if (!handler) {
+    cli.usageError(`unknown command "${command}"`);
+    process.exit(2);
+  }
+
+  await handler(values, args);
+}
+
+main().catch((error) => {
+  logger.exception("main", error);
+  cli.error(error.message);
+  process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Process Claude Code stream-json output into structured traces",
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",
@@ -8,10 +8,12 @@
   "main": "./src/index.js",
   "exports": {
     ".": "./src/index.js",
-    "./bin/fit-eval.js": "./bin/fit-eval.js"
+    "./bin/fit-eval.js": "./bin/fit-eval.js",
+    "./bin/fit-trace.js": "./bin/fit-trace.js"
   },
   "bin": {
-    "fit-eval": "./bin/fit-eval.js"
+    "fit-eval": "./bin/fit-eval.js",
+    "fit-trace": "./bin/fit-trace.js"
   },
   "files": [
     "src/**/*.js",
@@ -26,9 +28,11 @@
     "test": "bun run node --test test/*.test.js"
   },
   "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.2.98",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.112",
     "@forwardimpact/libcli": "^0.1.0",
-    "@forwardimpact/libtelemetry": "^0.1.22"
+    "@forwardimpact/libconfig": "^0.1.0",
+    "@forwardimpact/libtelemetry": "^0.1.22",
+    "zod": "^3.23.0"
   },
   "publishConfig": {
     "access": "public"

package/src/agent-runner.js

@@ -6,6 +6,28 @@
  * Follows OO+DI: constructor injection, factory function, tests bypass factory.
  */
 
+const DEFAULT_ALLOWED_TOOLS = ["Bash", "Read", "Glob", "Grep", "Write", "Edit"];
+
+function applyDefaults(deps) {
+  return {
+    cwd: deps.cwd,
+    query: deps.query,
+    output: deps.output,
+    model: deps.model ?? "opus",
+    maxTurns: deps.maxTurns ?? 50,
+    allowedTools: deps.allowedTools ?? DEFAULT_ALLOWED_TOOLS,
+    permissionMode: deps.permissionMode ?? "bypassPermissions",
+    onLine: deps.onLine ?? null,
+    onBatch: deps.onBatch ?? null,
+    batchSize: deps.batchSize ?? 3,
+    settingSources: deps.settingSources ?? [],
+    agentProfile: deps.agentProfile ?? null,
+    systemPrompt: deps.systemPrompt ?? null,
+    disallowedTools: deps.disallowedTools ?? [],
+    mcpServers: deps.mcpServers ?? null,
+  };
+}
+
 export class AgentRunner {
   /**
    * @param {object} deps
@@ -23,47 +45,13 @@ export class AgentRunner {
    * @param {string} [deps.agentProfile] - Agent profile name to pass as --agent to the Claude CLI
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
    * @param {string[]} [deps.disallowedTools] - Tools to explicitly remove from the model's context
+   * @param {Record<string, object>} [deps.mcpServers] - MCP server configs to pass to the SDK query
    */
-  constructor({
-    cwd,
-    query,
-    output,
-    model,
-    maxTurns,
-    allowedTools,
-    permissionMode,
-    onLine,
-    onBatch,
-    batchSize,
-    settingSources,
-    agentProfile,
-    systemPrompt,
-    disallowedTools,
-  }) {
-    if (!cwd) throw new Error("cwd is required");
-    if (!query) throw new Error("query is required");
-    if (!output) throw new Error("output is required");
-    this.cwd = cwd;
-    this.query = query;
-    this.output = output;
-    this.model = model ?? "opus";
-    this.maxTurns = maxTurns ?? 50; // 0 means unlimited (omit from SDK)
-    this.allowedTools = allowedTools ?? [
-      "Bash",
-      "Read",
-      "Glob",
-      "Grep",
-      "Write",
-      "Edit",
-    ];
-    this.permissionMode = permissionMode ?? "bypassPermissions";
-    this.onLine = onLine ?? null;
-    this.onBatch = onBatch ?? null;
-    this.batchSize = batchSize ?? 3;
-    this.settingSources = settingSources ?? [];
-    this.agentProfile = agentProfile ?? null;
-    this.systemPrompt = systemPrompt ?? null;
-    this.disallowedTools = disallowedTools ?? [];
+  constructor(deps) {
+    if (!deps.cwd) throw new Error("cwd is required");
+    if (!deps.query) throw new Error("query is required");
+    if (!deps.output) throw new Error("output is required");
+    Object.assign(this, applyDefaults(deps));
     this.sessionId = null;
     this.buffer = [];
     /** @type {AbortController|null} */
@@ -95,6 +83,7 @@ export class AgentRunner {
           }),
           ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
           ...(this.agentProfile && { extraArgs: { agent: this.agentProfile } }),
+          ...(this.mcpServers && { mcpServers: this.mcpServers }),
         },
       });
       return await this.#consumeQuery(iterator);
@@ -119,6 +108,7 @@ export class AgentRunner {
           permissionMode: this.permissionMode,
           allowDangerouslySkipPermissions: true,
           abortController,
+          ...(this.mcpServers && { mcpServers: this.mcpServers }),
         },
       });
       return await this.#consumeQuery(iterator);

package/src/commands/facilitate.js

@@ -0,0 +1,95 @@
+import { readFileSync, createWriteStream } from "node:fs";
+import { resolve } from "node:path";
+import { createFacilitator } from "../facilitator.js";
+import { createTeeWriter } from "../tee-writer.js";
+
+/**
+ * Parse comma-separated agent profile names into structured configs.
+ * @param {string} raw - Comma-separated profile names
+ * @param {string} cwd - Shared working directory for all agents
+ * @returns {Array<{name: string, role: string, cwd: string, agentProfile: string}>}
+ */
+function parseAgentProfiles(raw, cwd) {
+  return raw.split(",").map((entry) => {
+    const name = entry.trim();
+    return { name, role: name, cwd, agentProfile: name };
+  });
+}
+
+/**
+ * Parse and validate facilitate command options.
+ * @param {object} values - Parsed option values
+ * @returns {object} Parsed options
+ */
+function parseFacilitateOptions(values) {
+  const taskFile = values["task-file"];
+  const taskText = values["task-text"];
+  if (taskFile && taskText)
+    throw new Error("--task-file and --task-text are mutually exclusive");
+  if (!taskFile && !taskText)
+    throw new Error("--task-file or --task-text is required");
+
+  const taskAmend = values["task-amend"] ?? undefined;
+  let taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  if (taskAmend) taskContent += `\n\n${taskAmend}`;
+
+  const profilesRaw = values["agent-profiles"];
+  if (!profilesRaw) throw new Error("--agent-profiles is required");
+  const agentCwd = resolve(values["agent-cwd"] ?? ".");
+  const agentConfigs = parseAgentProfiles(profilesRaw, agentCwd);
+
+  const maxTurnsRaw = values["max-turns"] ?? "20";
+
+  return {
+    taskContent,
+    agentConfigs,
+    facilitatorCwd: resolve(values["facilitator-cwd"] ?? "."),
+    model: values.model ?? "opus",
+    maxTurns: maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10),
+    outputPath: values.output,
+    facilitatorProfile: values["facilitator-profile"] ?? undefined,
+  };
+}
+
+/**
+ * Facilitate command — run a facilitated multi-agent session.
+ *
+ * Usage: fit-eval facilitate [options]
+ *
+ * @param {object} values - Parsed option values from cli.parse()
+ * @param {string[]} _args - Positional arguments
+ */
+export async function runFacilitateCommand(values, _args) {
+  const opts = parseFacilitateOptions(values);
+
+  const fileStream = opts.outputPath
+    ? createWriteStream(opts.outputPath)
+    : null;
+  const output = fileStream
+    ? createTeeWriter({
+        fileStream,
+        textStream: process.stdout,
+        mode: "supervised",
+      })
+    : process.stdout;
+
+  const { query } = await import("@anthropic-ai/claude-agent-sdk");
+  const facilitator = createFacilitator({
+    facilitatorCwd: opts.facilitatorCwd,
+    agentConfigs: opts.agentConfigs,
+    query,
+    output,
+    model: opts.model,
+    maxTurns: opts.maxTurns,
+    facilitatorProfile: opts.facilitatorProfile,
+  });
+
+  const result = await facilitator.run(opts.taskContent);
+
+  if (fileStream) {
+    await new Promise((r) => output.end(r));
+    await new Promise((r) => fileStream.end(r));
+  }
+
+  process.exit(result.success ? 0 : 1);
+}

package/src/commands/run.js

@@ -1,7 +1,9 @@
 import { readFileSync, createWriteStream } from "node:fs";
+import { Writable } from "node:stream";
 import { resolve } from "node:path";
 import { createAgentRunner } from "../agent-runner.js";
 import { createTeeWriter } from "../tee-writer.js";
+import { SequenceCounter } from "../sequence-counter.js";
 
 /**
  * Parse and validate run command options from parsed values.
@@ -61,14 +63,28 @@ export async function runRunCommand(values, _args) {
     ? createTeeWriter({ fileStream, textStream: process.stdout, mode: "raw" })
     : process.stdout;
 
+  const counter = new SequenceCounter();
+  const devNull = new Writable({
+    write(_chunk, _enc, cb) {
+      cb();
+    },
+  });
+  const onLine = (line) => {
+    const event = JSON.parse(line);
+    output.write(
+      JSON.stringify({ source: "agent", seq: counter.next(), event }) + "\n",
+    );
+  };
+
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const runner = createAgentRunner({
     cwd,
     query,
-    output,
+    output: devNull,
     model,
     maxTurns,
     allowedTools,
+    onLine,
     settingSources: ["project"],
     agentProfile,
   });

package/src/commands/trace.js

@@ -0,0 +1,149 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createTraceCollector } from "@forwardimpact/libeval";
+import { createTraceQuery } from "../trace-query.js";
+import { createTraceGitHub } from "../trace-github.js";
+
+// --- GitHub commands ---
+
+/**
+ * List recent workflow runs matching a pattern.
+ * @param {object} values - Parsed option values
+ * @param {string[]} args - [pattern?]
+ */
+export async function runRunsCommand(values, args) {
+  const gh = await createTraceGitHub({ repo: values.repo });
+  const pattern = args[0] ?? "agent";
+  const lookback = values.lookback ?? "7d";
+  const runs = await gh.listRuns({ pattern, lookback });
+  writeJSON(runs);
+}
+
+/**
+ * Download a trace artifact and auto-convert to structured JSON.
+ * @param {object} values - Parsed option values
+ * @param {string[]} args - [run-id]
+ */
+export async function runDownloadCommand(values, args) {
+  const gh = await createTraceGitHub({ repo: values.repo });
+  const result = await gh.downloadTrace(args[0], {
+    dir: values.dir,
+    name: values.artifact,
+  });
+
+  const ndjsonFile = result.files.find((f) => f.endsWith(".ndjson"));
+  if (ndjsonFile) {
+    const ndjsonPath = join(result.dir, ndjsonFile);
+    const collector = createTraceCollector();
+    for (const line of readFileSync(ndjsonPath, "utf8").split("\n")) {
+      collector.addLine(line);
+    }
+    const structuredPath = join(result.dir, "structured.json");
+    writeFileSync(structuredPath, JSON.stringify(collector.toJSON()) + "\n");
+    result.files.push("structured.json");
+  }
+
+  writeJSON(result);
+}
+
+// --- Query commands ---
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runOverviewCommand(values, args) {
+  writeJSON(loadTrace(args[0]).overview());
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runCountCommand(values, args) {
+  process.stdout.write(String(loadTrace(args[0]).count()) + "\n");
+}
+
+/** @param {object} values @param {string[]} args - [file, from, to] */
+export async function runBatchCommand(values, args) {
+  writeJSON(
+    loadTrace(args[0]).batch(parseInt(args[1], 10), parseInt(args[2], 10)),
+  );
+}
+
+/** @param {object} values @param {string[]} args - [file, N?] */
+export async function runHeadCommand(values, args) {
+  const n = args[1] ? parseInt(args[1], 10) : 10;
+  writeJSON(loadTrace(args[0]).head(n));
+}
+
+/** @param {object} values @param {string[]} args - [file, N?] */
+export async function runTailCommand(values, args) {
+  const n = args[1] ? parseInt(args[1], 10) : 10;
+  writeJSON(loadTrace(args[0]).tail(n));
+}
+
+/** @param {object} values @param {string[]} args - [file, pattern] */
+export async function runSearchCommand(values, args) {
+  const limit = values.limit ? parseInt(values.limit, 10) : 50;
+  const context = values.context ? parseInt(values.context, 10) : 0;
+  writeJSON(loadTrace(args[0]).search(args[1], { limit, context }));
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runToolsCommand(values, args) {
+  writeJSON(loadTrace(args[0]).toolFrequency());
+}
+
+/** @param {object} values @param {string[]} args - [file, name] */
+export async function runToolCommand(values, args) {
+  writeJSON(loadTrace(args[0]).tool(args[1]));
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runErrorsCommand(values, args) {
+  writeJSON(loadTrace(args[0]).errors());
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runReasoningCommand(values, args) {
+  const from = values.from ? parseInt(values.from, 10) : undefined;
+  const to = values.to ? parseInt(values.to, 10) : undefined;
+  writeJSON(loadTrace(args[0]).reasoning({ from, to }));
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runTimelineCommand(values, args) {
+  const lines = loadTrace(args[0]).timeline();
+  process.stdout.write(lines.join("\n") + "\n");
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runStatsCommand(values, args) {
+  writeJSON(loadTrace(args[0]).stats());
+}
+
+// --- Shared helpers ---
+
+/**
+ * Load a trace file. Supports structured JSON and raw NDJSON.
+ * @param {string} file
+ * @returns {import("../trace-query.js").TraceQuery}
+ */
+function loadTrace(file) {
+  const content = readFileSync(file, "utf8");
+
+  try {
+    const parsed = JSON.parse(content);
+    if (parsed.turns) {
+      return createTraceQuery(parsed);
+    }
+  } catch {
+    // Not valid JSON — fall through to NDJSON.
+  }
+
+  const collector = createTraceCollector();
+  for (const line of content.split("\n")) {
+    collector.addLine(line);
+  }
+  return createTraceQuery(collector.toJSON());
+}
+
+/** @param {object} data */
+function writeJSON(data) {
+  process.stdout.write(JSON.stringify(data, null, 2) + "\n");
+}