@forwardimpact/libeval 0.1.6 → 0.1.9

package/bin/fit-eval.js

@@ -29,7 +29,7 @@ Run options:
   --task-text=STRING   Inline task text (mutually exclusive with --task-file)
   --cwd=DIR            Agent working directory (default: .)
   --model=MODEL        Claude model to use (default: opus)
-  --max-turns=N        Maximum agentic turns (default: 50)
+  --max-turns=N        Maximum agentic turns (default: 50, 0 = unlimited)
   --output=PATH        Write NDJSON trace to file (default: stdout)
   --allowed-tools=LIST Comma-separated tools (default: Bash,Read,Glob,Grep,Write,Edit)
   --agent-profile=NAME Agent profile name (passed as --agent to Claude CLI)
@@ -40,7 +40,7 @@ Supervise options:
   --supervisor-cwd=DIR      Supervisor working directory (default: .)
   --agent-cwd=DIR           Agent working directory (default: temp directory)
   --model=MODEL             Claude model to use (default: opus)
-  --max-turns=N             Maximum supervisor ↔ agent exchanges (default: 20)
+  --max-turns=N             Maximum supervisor ↔ agent exchanges (default: 20, 0 = unlimited)
   --output=PATH             Write NDJSON trace to file (default: stdout)
   --allowed-tools=LIST      Comma-separated tools for agent (default: Bash,Read,Glob,Grep,Write,Edit)
   --supervisor-allowed-tools=LIST

package/index.js

@@ -5,5 +5,7 @@ export {
   createSupervisor,
   SUPERVISOR_SYSTEM_PROMPT,
   AGENT_SYSTEM_PROMPT,
+  isComplete,
+  isIntervention,
 } from "./src/supervisor.js";
 export { TeeWriter, createTeeWriter } from "./src/tee-writer.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.6",
+  "version": "0.1.9",
   "description": "Process Claude Code stream-json output into structured traces",
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",

package/src/agent-runner.js

@@ -17,6 +17,8 @@ export class AgentRunner {
    * @param {string[]} [deps.allowedTools] - Tools the agent may use
    * @param {string} [deps.permissionMode] - SDK permission mode
    * @param {function} [deps.onLine] - Callback invoked with each NDJSON line as it's produced
+   * @param {function} [deps.onBatch] - Async callback invoked with a batch of NDJSON lines at flush boundaries: every `batchSize` assistant text blocks, the terminal `result` message, and — on iterator crash/abort — once more in a final flush carrying any lines that never reached a boundary. Receives `(lines, { abort })` where calling `abort()` stops the in-flight SDK session via the AbortController. Optional; assignable at runtime so the Supervisor can swap it per turn.
+   * @param {number} [deps.batchSize] - Assistant text-block messages to accumulate before firing onBatch. Tool-only assistant messages ride along without counting. Default 3: the supervisor reviews the agent every three text turns instead of every turn. The terminal `result` always flushes regardless of count.
    * @param {string[]} [deps.settingSources] - SDK setting sources (e.g. ['project'] to load CLAUDE.md)
    * @param {string} [deps.agentProfile] - Agent profile name to pass as --agent to the Claude CLI
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
@@ -31,6 +33,8 @@ export class AgentRunner {
     allowedTools,
     permissionMode,
     onLine,
+    onBatch,
+    batchSize,
     settingSources,
     agentProfile,
     systemPrompt,
@@ -43,7 +47,7 @@ export class AgentRunner {
     this.query = query;
     this.output = output;
     this.model = model ?? "opus";
-    this.maxTurns = maxTurns ?? 50;
+    this.maxTurns = maxTurns ?? 50; // 0 means unlimited (omit from SDK)
     this.allowedTools = allowedTools ?? [
       "Bash",
       "Read",
@@ -54,101 +58,214 @@ export class AgentRunner {
     ];
     this.permissionMode = permissionMode ?? "bypassPermissions";
     this.onLine = onLine ?? null;
+    this.onBatch = onBatch ?? null;
+    this.batchSize = batchSize ?? 3;
     this.settingSources = settingSources ?? [];
     this.agentProfile = agentProfile ?? null;
     this.systemPrompt = systemPrompt ?? null;
     this.disallowedTools = disallowedTools ?? [];
     this.sessionId = null;
     this.buffer = [];
+    /** @type {AbortController|null} */
+    this.currentAbortController = null;
   }
 
   /**
    * Run a new agent session with the given task.
    * @param {string} task - The task prompt
-   * @returns {Promise<{success: boolean, text: string, sessionId: string|null}>}
+   * @returns {Promise<{success: boolean, text: string, sessionId: string|null, error: Error|null, aborted: boolean}>}
    */
   async run(task) {
-    let text = "";
-    let stopReason = null;
-    let error = null;
-
+    const abortController = new AbortController();
+    this.currentAbortController = abortController;
     try {
-      for await (const message of this.query({
+      const iterator = this.query({
         prompt: task,
         options: {
           cwd: this.cwd,
           allowedTools: this.allowedTools,
-          maxTurns: this.maxTurns,
+          ...(this.maxTurns > 0 && { maxTurns: this.maxTurns }),
           model: this.model,
           permissionMode: this.permissionMode,
           allowDangerouslySkipPermissions: true,
           settingSources: this.settingSources,
+          abortController,
           ...(this.disallowedTools.length > 0 && {
             disallowedTools: this.disallowedTools,
           }),
           ...(this.systemPrompt && { systemPrompt: this.systemPrompt }),
           ...(this.agentProfile && { extraArgs: { agent: this.agentProfile } }),
         },
-      })) {
-        const line = JSON.stringify(message);
-        this.output.write(line + "\n");
-        this.buffer.push(line);
-        if (this.onLine) this.onLine(line);
-
-        if (message.type === "system" && message.subtype === "init") {
-          this.sessionId = message.session_id;
-        }
-        if (message.type === "result") {
-          text = message.result ?? "";
-          stopReason = message.subtype;
-        }
-      }
-    } catch (err) {
-      error = err;
+      });
+      return await this.#consumeQuery(iterator);
+    } finally {
+      this.currentAbortController = null;
     }
-
-    // If the SDK already emitted a successful result, honour it even when the
-    // stream throws afterwards (e.g. "Credit balance is too low" during
-    // cleanup). Only treat errors as fatal when no result was received yet.
-    const success = stopReason === "success";
-    return { success, text, sessionId: this.sessionId, error };
   }
 
   /**
    * Resume an existing session with a follow-up prompt.
    * @param {string} prompt - The follow-up prompt
-   * @returns {Promise<{success: boolean, text: string}>}
+   * @returns {Promise<{success: boolean, text: string, sessionId: string|null, error: Error|null, aborted: boolean}>}
    */
   async resume(prompt) {
-    let text = "";
-    let stopReason = null;
-    let error = null;
-
+    const abortController = new AbortController();
+    this.currentAbortController = abortController;
     try {
-      for await (const message of this.query({
+      const iterator = this.query({
         prompt,
         options: {
           resume: this.sessionId,
           permissionMode: this.permissionMode,
           allowDangerouslySkipPermissions: true,
+          abortController,
         },
-      })) {
-        const line = JSON.stringify(message);
-        this.output.write(line + "\n");
-        this.buffer.push(line);
-        if (this.onLine) this.onLine(line);
+      });
+      return await this.#consumeQuery(iterator);
+    } finally {
+      this.currentAbortController = null;
+    }
+  }
+
+  /**
+   * Shared consumer for both `run()` and `resume()`. Iterates the SDK query
+   * iterator, mirroring every line to the output stream / buffer / onLine
+   * callback, and — when `onBatch` is set — flushes accumulated lines to it
+   * at coarse boundaries: every `batchSize` assistant text-block messages,
+   * and the terminal `result` message. Tool-only assistant messages still
+   * accumulate in the pending batch and ride along in the next flush, so
+   * the supervisor always sees the tool calls that led up to each text
+   * block. Raising `batchSize` above 1 is the knob that makes the mid-turn
+   * supervisor review less chatty — with the default of 3, the supervisor
+   * sees the agent in chunks of three text turns instead of every turn.
+   *
+   * Corollary: a turn that is *entirely* tool_use with no text blocks and
+   * then hits `result` produces exactly one flush at `result` regardless
+   * of how many tools ran. That is deliberate — the supervisor only needs
+   * to weigh in when the agent surfaces something text-like to react to.
+   *
+   * INVARIANT: the `await this.onBatch(...)` call below is the ONLY
+   * suspension point in this loop. While it is pending, no further lines
+   * are pulled from the SDK generator. The Supervisor relies on this — its
+   * onBatch callback flips `currentSource` to "supervisor" for the duration
+   * of its mid-turn LLM call, and the invariant guarantees no agent line
+   * can arrive concurrently and be mis-tagged.
+   *
+   * If the supervisor calls `abort()` from inside the callback, the next
+   * iteration of the for-await loop will throw. We catch the throw, check
+   * `currentAbortController.signal.aborted` (avoiding fragility around
+   * AbortError vs DOMException shapes), and report `aborted: true` so the
+   * caller can distinguish "supervisor asked us to stop" from a real error.
+   *
+   * If the iterator throws before a flush boundary, any lines still in the
+   * pending batch would otherwise vanish without the supervisor seeing
+   * them. The `finally` block emits a terminal batch so the supervisor can
+   * observe the partial state (e.g. note a crash or react to an external
+   * abort). A throw from that final flush becomes the returned `error`
+   * only if no earlier error was captured — the original failure wins.
+   * @param {AsyncIterable<object>} iterator
+   * @returns {Promise<{success: boolean, text: string, sessionId: string|null, error: Error|null, aborted: boolean}>}
+   */
+  async #consumeQuery(iterator) {
+    let text = "";
+    let stopReason = null;
+    let error = null;
+    let aborted = false;
+    const state = { pendingBatch: [], assistantTextCount: 0 };
 
+    try {
+      for await (const message of iterator) {
+        this.#recordLine(message, state);
         if (message.type === "result") {
           text = message.result ?? "";
           stopReason = message.subtype;
         }
+        await this.#maybeFlushBatch(message, state);
       }
     } catch (err) {
-      error = err;
+      if (this.currentAbortController?.signal.aborted) {
+        aborted = true;
+      } else {
+        error = err;
+      }
     }
 
+    const flushErr = await this.#terminalFlush(state, { error, aborted });
+    if (flushErr && !error) error = flushErr;
+
     const success = stopReason === "success";
-    return { success, text, error };
+    return { success, text, sessionId: this.sessionId, error, aborted };
+  }
+
+  /**
+   * Mirror a single SDK message to the output stream, buffer, onLine
+   * callback, and (when set) the pending-batch state. Also handles
+   * session id capture and text-block counting so `#consumeQuery` can
+   * stay within the complexity budget.
+   * @param {object} message
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   */
+  #recordLine(message, state) {
+    const line = JSON.stringify(message);
+    this.output.write(line + "\n");
+    this.buffer.push(line);
+    if (this.onLine) this.onLine(line);
+    if (this.onBatch) state.pendingBatch.push(line);
+
+    if (message.type === "system" && message.subtype === "init") {
+      this.sessionId = message.session_id;
+    }
+    if (message.type === "assistant" && hasTextBlock(message)) {
+      state.assistantTextCount++;
+    }
+  }
+
+  /**
+   * Terminal flush — only fires on the abnormal-end paths (iterator
+   * threw or was aborted mid-stream). Delivers any pending lines so the
+   * supervisor sees the partial state instead of losing the tail of
+   * the run. A natural-end iterator that simply ran out of messages
+   * without a `result` marker is treated as an incomplete stub (the
+   * real SDK always terminates with `result`) and its pending batch is
+   * not re-flushed. Returns an error thrown by the flush callback, or
+   * `null` if the flush succeeded or did not fire.
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   * @param {{error: Error|null, aborted: boolean}} outcome
+   * @returns {Promise<Error|null>}
+   */
+  async #terminalFlush(state, { error, aborted }) {
+    const loopEndedAbnormally = Boolean(error || aborted);
+    if (!loopEndedAbnormally) return null;
+    if (!this.onBatch || state.pendingBatch.length === 0) return null;
+    try {
+      const batchLines = state.pendingBatch.splice(0);
+      await this.onBatch(batchLines, {
+        abort: () => this.currentAbortController?.abort(),
+      });
+      return null;
+    } catch (flushErr) {
+      return flushErr;
+    }
+  }
+
+  /**
+   * Flush the pending batch to `onBatch` if either the batchSize threshold
+   * has been reached or the current message is the terminal `result`.
+   * Extracted so that `#consumeQuery` stays within the project's complexity
+   * budget — the flush is one cohesive unit of logic in its own right.
+   * @param {object} message
+   * @param {{pendingBatch: string[], assistantTextCount: number}} state
+   */
+  async #maybeFlushBatch(message, state) {
+    if (!this.onBatch) return;
+    const shouldFlush =
+      message.type === "result" || state.assistantTextCount >= this.batchSize;
+    if (!shouldFlush) return;
+    state.assistantTextCount = 0;
+    const batchLines = state.pendingBatch.splice(0);
+    await this.onBatch(batchLines, {
+      abort: () => this.currentAbortController?.abort(),
+    });
   }
 
   /**
@@ -162,6 +279,24 @@ export class AgentRunner {
   }
 }
 
+/**
+ * Whether an SDK assistant message contains at least one text block.
+ * Only text-block messages count toward the `batchSize` threshold — tool-only
+ * assistant messages accumulate silently into the pending batch and ride along
+ * in the next flush, keeping supervisor LLM cost bounded. Exported so the mock
+ * runner can mirror the real flush predicate without duplicating the logic.
+ * @param {object} message
+ * @returns {boolean}
+ */
+export function hasTextBlock(message) {
+  const content = message.message?.content ?? message.content;
+  if (!Array.isArray(content)) return false;
+  for (const block of content) {
+    if (block.type === "text" && block.text) return true;
+  }
+  return false;
+}
+
 /**
  * Factory function — wires real dependencies.
  * @param {object} deps - Same as AgentRunner constructor

package/src/commands/run.js

@@ -18,6 +18,38 @@ function parseFlag(args, name) {
   return undefined;
 }
 
+/**
+ * Parse and validate run command options from args.
+ * @param {string[]} args
+ * @returns {{ taskContent: string, cwd: string, model: string, maxTurns: number, outputPath: string|undefined, agentProfile: string|undefined, allowedTools: string[] }}
+ */
+function parseRunOptions(args) {
+  const taskFile = parseFlag(args, "task-file");
+  const taskText = parseFlag(args, "task-text");
+  if (taskFile && taskText)
+    throw new Error("--task-file and --task-text are mutually exclusive");
+  if (!taskFile && !taskText)
+    throw new Error("--task-file or --task-text is required");
+
+  const maxTurnsRaw = parseFlag(args, "max-turns") ?? "50";
+  const taskAmend = parseFlag(args, "task-amend") ?? undefined;
+  let taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  if (taskAmend) taskContent += `\n\n${taskAmend}`;
+
+  return {
+    taskContent,
+    cwd: resolve(parseFlag(args, "cwd") ?? "."),
+    model: parseFlag(args, "model") ?? "opus",
+    maxTurns: maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10),
+    outputPath: parseFlag(args, "output"),
+    agentProfile: parseFlag(args, "agent-profile") ?? undefined,
+    allowedTools: (
+      parseFlag(args, "allowed-tools") ??
+      "Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
+    ).split(","),
+  };
+}
+
 /**
  * Run command — execute a single agent via the Claude Agent SDK.
  *
@@ -28,31 +60,24 @@ function parseFlag(args, name) {
  *   --task-text=STRING   Inline task text (mutually exclusive with --task-file)
  *   --cwd=DIR            Agent working directory (default: .)
  *   --model=MODEL        Claude model to use (default: opus)
- *   --max-turns=N        Maximum agentic turns (default: 50)
+ *   --max-turns=N        Maximum agentic turns (default: 50, 0 = unlimited)
  *   --output=PATH        Write NDJSON trace to file (default: stdout)
  *   --allowed-tools=LIST Comma-separated tools (default: Bash,Read,Glob,Grep,Write,Edit)
  *   --agent-profile=NAME Agent profile name (passed as --agent to Claude CLI)
+ *   --task-amend=TEXT     Additional text appended to the task prompt
  *
  * @param {string[]} args - Command arguments
  */
 export async function runRunCommand(args) {
-  const taskFile = parseFlag(args, "task-file");
-  const taskText = parseFlag(args, "task-text");
-  if (taskFile && taskText)
-    throw new Error("--task-file and --task-text are mutually exclusive");
-  if (!taskFile && !taskText)
-    throw new Error("--task-file or --task-text is required");
-
-  const cwd = resolve(parseFlag(args, "cwd") ?? ".");
-  const model = parseFlag(args, "model") ?? "opus";
-  const maxTurns = parseInt(parseFlag(args, "max-turns") ?? "50", 10);
-  const outputPath = parseFlag(args, "output");
-  const agentProfile = parseFlag(args, "agent-profile") ?? undefined;
-  const allowedTools = (
-    parseFlag(args, "allowed-tools") ?? "Bash,Read,Glob,Grep,Write,Edit"
-  ).split(",");
-
-  const taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  const {
+    taskContent,
+    cwd,
+    model,
+    maxTurns,
+    outputPath,
+    agentProfile,
+    allowedTools,
+  } = parseRunOptions(args);
 
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).

package/src/commands/supervise.js

@@ -19,6 +19,50 @@ function parseFlag(args, name) {
   return undefined;
 }
 
+/**
+ * Parse all supervise flags from args into an options object.
+ * @param {string[]} args
+ * @returns {object}
+ */
+function parseSuperviseOptions(args) {
+  const taskFile = parseFlag(args, "task-file");
+  const taskText = parseFlag(args, "task-text");
+  if (taskFile && taskText)
+    throw new Error("--task-file and --task-text are mutually exclusive");
+  if (!taskFile && !taskText)
+    throw new Error("--task-file or --task-text is required");
+
+  const supervisorAllowedToolsRaw = parseFlag(args, "supervisor-allowed-tools");
+
+  const taskAmend = parseFlag(args, "task-amend") ?? undefined;
+  let taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  if (taskAmend) taskContent += `\n\n${taskAmend}`;
+
+  return {
+    taskContent,
+    supervisorCwd: resolve(parseFlag(args, "supervisor-cwd") ?? "."),
+    agentCwd: resolve(
+      parseFlag(args, "agent-cwd") ??
+        mkdtempSync(join(tmpdir(), "fit-eval-agent-")),
+    ),
+    model: parseFlag(args, "model") ?? "opus",
+    maxTurns: (() => {
+      const raw = parseFlag(args, "max-turns") ?? "20";
+      return raw === "0" ? 0 : parseInt(raw, 10);
+    })(),
+    outputPath: parseFlag(args, "output"),
+    supervisorProfile: parseFlag(args, "supervisor-profile") ?? undefined,
+    agentProfile: parseFlag(args, "agent-profile") ?? undefined,
+    allowedTools: (
+      parseFlag(args, "allowed-tools") ??
+      "Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
+    ).split(","),
+    supervisorAllowedTools: supervisorAllowedToolsRaw
+      ? supervisorAllowedToolsRaw.split(",")
+      : undefined,
+  };
+}
+
 /**
  * Supervise command — run two agents in a relay loop via the Claude Agent SDK.
  *
@@ -30,45 +74,23 @@ function parseFlag(args, name) {
  *   --supervisor-cwd=DIR      Supervisor working directory (default: .)
  *   --agent-cwd=DIR           Agent working directory (default: temp directory)
  *   --model=MODEL             Claude model to use (default: opus)
- *   --max-turns=N             Maximum supervisor ↔ agent exchanges (default: 20)
+ *   --max-turns=N             Maximum supervisor / agent exchanges (default: 20, 0 = unlimited)
  *   --output=PATH             Write NDJSON trace to file (default: stdout)
  *   --allowed-tools=LIST      Comma-separated tools for the agent (default: Bash,Read,Glob,Grep,Write,Edit)
  *   --supervisor-profile=NAME Supervisor agent profile name (passed as --agent to Claude CLI)
  *   --agent-profile=NAME      Agent profile name (passed as --agent to Claude CLI)
+ *   --task-amend=TEXT          Additional text appended to the task prompt
  *
  * @param {string[]} args - Command arguments
  */
 export async function runSuperviseCommand(args) {
-  const taskFile = parseFlag(args, "task-file");
-  const taskText = parseFlag(args, "task-text");
-  if (taskFile && taskText)
-    throw new Error("--task-file and --task-text are mutually exclusive");
-  if (!taskFile && !taskText)
-    throw new Error("--task-file or --task-text is required");
-
-  const supervisorCwd = resolve(parseFlag(args, "supervisor-cwd") ?? ".");
-  const agentCwd = resolve(
-    parseFlag(args, "agent-cwd") ??
-      mkdtempSync(join(tmpdir(), "fit-eval-agent-")),
-  );
-  const model = parseFlag(args, "model") ?? "opus";
-  const maxTurns = parseInt(parseFlag(args, "max-turns") ?? "20", 10);
-  const outputPath = parseFlag(args, "output");
-  const supervisorProfile = parseFlag(args, "supervisor-profile") ?? undefined;
-  const agentProfile = parseFlag(args, "agent-profile") ?? undefined;
-  const allowedTools = (
-    parseFlag(args, "allowed-tools") ?? "Bash,Read,Glob,Grep,Write,Edit"
-  ).split(",");
-  const supervisorAllowedToolsRaw = parseFlag(args, "supervisor-allowed-tools");
-  const supervisorAllowedTools = supervisorAllowedToolsRaw
-    ? supervisorAllowedToolsRaw.split(",")
-    : undefined;
-
-  const taskContent = taskFile ? readFileSync(taskFile, "utf8") : taskText;
+  const opts = parseSuperviseOptions(args);
 
   // When --output is specified, stream text to stdout while writing NDJSON to file.
   // Otherwise, write NDJSON directly to stdout (backwards-compatible).
-  const fileStream = outputPath ? createWriteStream(outputPath) : null;
+  const fileStream = opts.outputPath
+    ? createWriteStream(opts.outputPath)
+    : null;
   const output = fileStream
     ? createTeeWriter({
         fileStream,
@@ -79,19 +101,19 @@ export async function runSuperviseCommand(args) {
 
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const supervisor = createSupervisor({
-    supervisorCwd,
-    agentCwd,
+    supervisorCwd: opts.supervisorCwd,
+    agentCwd: opts.agentCwd,
     query,
     output,
-    model,
-    maxTurns,
-    allowedTools,
-    supervisorAllowedTools,
-    supervisorProfile,
-    agentProfile,
+    model: opts.model,
+    maxTurns: opts.maxTurns,
+    allowedTools: opts.allowedTools,
+    supervisorAllowedTools: opts.supervisorAllowedTools,
+    supervisorProfile: opts.supervisorProfile,
+    agentProfile: opts.agentProfile,
   });
 
-  const result = await supervisor.run(taskContent);
+  const result = await supervisor.run(opts.taskContent);
 
   if (fileStream) {
     await new Promise((r) => output.end(r));