@forwardimpact/libeval 0.1.25 → 0.1.27

package/bin/fit-eval.js

@@ -178,25 +178,25 @@ const definition = {
   documentation: [
     {
       title: "Agent Evaluations",
-      url: "https://www.forwardimpact.team/docs/guides/agent-evaluations/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/agent-evaluations/index.md",
       description:
         "Author a judge profile, run an eval locally, wire it into CI, and inspect the resulting trace.",
     },
     {
       title: "Agent Collaboration",
-      url: "https://www.forwardimpact.team/docs/guides/agent-collaboration/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/agent-collaboration/index.md",
       description:
         "Author a facilitator and participant profiles, run a multi-agent session, and read the message flow.",
     },
     {
       title: "Trace Analysis",
-      url: "https://www.forwardimpact.team/docs/guides/trace-analysis/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/trace-analysis/index.md",
       description:
         "Read the NDJSON traces produced by `fit-eval` with `fit-trace` — grounded-theory method and worked examples.",
     },
     {
       title: "Agent Teams",
-      url: "https://www.forwardimpact.team/docs/guides/agent-teams/index.md",
+      url: "https://www.forwardimpact.team/docs/products/agent-teams/index.md",
       description:
         "How to author the agent, supervisor, and facilitator profiles consumed by --agent-profile, --supervisor-profile, --facilitator-profile, and --agent-profiles.",
     },

package/bin/fit-trace.js

@@ -215,19 +215,19 @@ const definition = {
   documentation: [
     {
       title: "Trace Analysis",
-      url: "https://www.forwardimpact.team/docs/guides/trace-analysis/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/trace-analysis/index.md",
       description:
         "The full method walkthrough with worked examples (an eval that failed, a multi-agent session that stalled).",
     },
     {
       title: "Agent Evaluations",
-      url: "https://www.forwardimpact.team/docs/guides/agent-evaluations/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/agent-evaluations/index.md",
       description:
         "How `fit-eval supervise` produces the traces this skill analyzes.",
     },
     {
       title: "Agent Collaboration",
-      url: "https://www.forwardimpact.team/docs/guides/agent-collaboration/index.md",
+      url: "https://www.forwardimpact.team/docs/libraries/agent-collaboration/index.md",
       description:
         "How `fit-eval facilitate` produces multi-agent traces; `split` is the bridge into per-source files.",
     },

package/package.json

@@ -1,9 +1,30 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.25",
-  "description": "Process Claude Code stream-json output into structured traces",
+  "version": "0.1.27",
+  "description": "Agent evaluation: collect Claude Code traces, run agent loops, supervise multi-step workflows.",
+  "keywords": [
+    "eval",
+    "agent",
+    "trace",
+    "claude-code",
+    "supervisor"
+  ],
+  "homepage": "https://www.forwardimpact.team",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/forwardimpact/monorepo.git",
+    "directory": "libraries/libeval"
+  },
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",
+  "forwardimpact": {
+    "capability": "agent-self-improvement",
+    "needs": [
+      "Parse and query Claude Code trace NDJSON files",
+      "Drive an LLM agent through a scripted run and capture its trace",
+      "Supervise a multi-step or multi-agent workflow"
+    ]
+  },
   "type": "module",
   "main": "./src/index.js",
   "exports": {
@@ -20,10 +41,6 @@
     "bin/**/*.js",
     "README.md"
   ],
-  "engines": {
-    "bun": ">=1.2.0",
-    "node": ">=18.0.0"
-  },
   "scripts": {
     "test": "bun test test/*.test.js"
   },
@@ -37,6 +54,10 @@
   "devDependencies": {
     "@forwardimpact/libharness": "^0.1.14"
   },
+  "engines": {
+    "bun": ">=1.2.0",
+    "node": ">=18.0.0"
+  },
   "publishConfig": {
     "access": "public"
   }

package/src/agent-runner.js

@@ -211,8 +211,9 @@ export class AgentRunner {
     if (message.type === "system" && message.subtype === "init") {
       this.sessionId = message.session_id;
     }
-    if (message.type === "assistant" && hasTextBlock(message)) {
-      state.assistantTextCount++;
+    if (message.type === "assistant") {
+      if (hasTextBlock(message)) state.assistantTextCount++;
+      trackSkillInvocation(message);
     }
   }
 
@@ -293,6 +294,20 @@ export function hasTextBlock(message) {
   return false;
 }
 
+function trackSkillInvocation(message) {
+  const content = message.message?.content ?? message.content;
+  if (!Array.isArray(content)) return;
+  for (const block of content) {
+    if (
+      block.type === "tool_use" &&
+      block.name === "Skill" &&
+      block.input?.skill
+    ) {
+      process.env.LIBEVAL_SKILL = block.input.skill;
+    }
+  }
+}
+
 /**
  * Factory function — wires real dependencies.
  * @param {object} deps - Same as AgentRunner constructor

package/src/commands/facilitate.js

@@ -73,6 +73,10 @@ export async function runFacilitateCommand(values, _args) {
       })
     : process.stdout;
 
+  if (opts.facilitatorProfile) {
+    process.env.LIBEVAL_AGENT_PROFILE = opts.facilitatorProfile;
+  }
+
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const facilitator = createFacilitator({
     facilitatorCwd: opts.facilitatorCwd,

package/src/commands/run.js

@@ -78,6 +78,10 @@ export async function runRunCommand(values, _args) {
     );
   };
 
+  if (agentProfile) {
+    process.env.LIBEVAL_AGENT_PROFILE = agentProfile;
+  }
+
   const systemPrompt = agentProfile
     ? composeProfilePrompt(agentProfile, {
         profilesDir: resolve(cwd, ".claude/agents"),

package/src/commands/supervise.js

@@ -71,6 +71,10 @@ export async function runSuperviseCommand(values, _args) {
       })
     : process.stdout;
 
+  if (opts.agentProfile) {
+    process.env.LIBEVAL_AGENT_PROFILE = opts.agentProfile;
+  }
+
   const { query } = await import("@anthropic-ai/claude-agent-sdk");
   const supervisor = createSupervisor({
     supervisorCwd: opts.supervisorCwd,

package/src/facilitator.js

@@ -180,42 +180,48 @@ export class Facilitator {
     let messages = this.messageBus.drain(agent.name);
     if (messages.length === 0) return;
 
-    this.emitOrchestratorEvent({
-      type: "agent_start",
-      agent: agent.name,
-    });
+    this.emitOrchestratorEvent({ type: "agent_start", agent: agent.name });
     await agent.runner.run(formatMessages(messages));
-    if (this.ctx.concluded) return;
+    if (await this.#settleAgentTurn(agent)) return;
+
+    // Loop: check for new messages, resume if any
+    while (!this.ctx.concluded) {
+      messages = await this.#awaitAgentMessages(agent.name);
+      if (messages.length === 0) break;
+      await agent.runner.resume(formatMessages(messages));
+      if (await this.#settleAgentTurn(agent)) break;
+    }
+  }
+
+  /**
+   * Enforce pending-ask and emit turn_complete. Returns true when the
+   * session has concluded and the caller should stop.
+   */
+  async #settleAgentTurn(agent) {
+    if (this.ctx.concluded) return true;
     await this.#enforcePendingAsk(agent);
-    if (this.ctx.concluded) return;
+    if (this.ctx.concluded) return true;
     this.eventQueue.enqueue({
       type: "lifecycle",
       agent: agent.name,
       status: "turn_complete",
     });
+    return false;
+  }
 
-    // Loop: check for new messages, resume if any
-    while (!this.ctx.concluded) {
-      messages = this.messageBus.drain(agent.name);
-      if (messages.length === 0) {
-        await Promise.race([
-          this.messageBus.waitForMessages(agent.name),
-          this.concludePromise,
-        ]);
-        if (this.ctx.concluded) break;
-        messages = this.messageBus.drain(agent.name);
-        if (messages.length === 0) break;
-      }
-      await agent.runner.resume(formatMessages(messages));
-      if (this.ctx.concluded) break;
-      await this.#enforcePendingAsk(agent);
-      if (this.ctx.concluded) break;
-      this.eventQueue.enqueue({
-        type: "lifecycle",
-        agent: agent.name,
-        status: "turn_complete",
-      });
-    }
+  /**
+   * Wait for messages addressed to `name`, returning an empty array when
+   * the session concludes first.
+   */
+  async #awaitAgentMessages(name) {
+    const messages = this.messageBus.drain(name);
+    if (messages.length > 0) return messages;
+    await Promise.race([
+      this.messageBus.waitForMessages(name),
+      this.concludePromise,
+    ]);
+    if (this.ctx.concluded) return [];
+    return this.messageBus.drain(name);
   }
 
   /**

package/src/render/turn-renderer.js

@@ -0,0 +1,92 @@
+/**
+ * Turn renderer — maps a structured turn into formatted text lines.
+ *
+ * Shared by `TeeWriter.flushTurns()` (live stream) and
+ * `TraceCollector.toText()` (offline replay) so both emit identical output
+ * (spec 540).
+ */
+
+import {
+  renderTextLine,
+  renderToolCallLine,
+  renderToolResultLine,
+} from "./line-renderer.js";
+import {
+  hintForCall,
+  previewForResult,
+  simplifyToolName,
+} from "./tool-hints.js";
+
+/**
+ * Render a single turn to formatted text lines.
+ *
+ * @param {object} turn - Structured turn object
+ * @param {boolean} withPrefix - Whether to include source labels
+ * @returns {string[]} Array of rendered line strings
+ */
+export function renderTurnLines(turn, withPrefix) {
+  if (turn.role === "assistant") return renderAssistantTurn(turn, withPrefix);
+  if (turn.role === "tool_result")
+    return renderToolResultTurn(turn, withPrefix);
+  if (turn.role === "system") return renderSystemTurn(turn, withPrefix);
+  if (turn.role === "user") return renderUserTurn(turn, withPrefix);
+  return [];
+}
+
+/** @param {object} turn @param {boolean} withPrefix @returns {string[]} */
+function renderAssistantTurn(turn, withPrefix) {
+  const lines = [];
+  for (const block of turn.content) {
+    if (block.type === "text") {
+      lines.push(
+        renderTextLine({ source: turn.source, text: block.text, withPrefix }),
+      );
+    } else if (block.type === "tool_use") {
+      lines.push(
+        renderToolCallLine({
+          source: turn.source,
+          toolName: simplifyToolName(block.name),
+          hint: hintForCall(block.name, block.input),
+          withPrefix,
+        }),
+      );
+    }
+  }
+  return lines;
+}
+
+/** @param {object} turn @param {boolean} withPrefix @returns {string[]} */
+function renderToolResultTurn(turn, withPrefix) {
+  return [
+    renderToolResultLine({
+      source: turn.source,
+      preview: previewForResult(turn.content, turn.isError),
+      withPrefix,
+    }),
+  ];
+}
+
+/** @param {object} turn @param {boolean} withPrefix @returns {string[]} */
+function renderSystemTurn(turn, withPrefix) {
+  const label = turn.subtype ?? "system";
+  return [
+    renderTextLine({ source: turn.source, text: `[${label}]`, withPrefix }),
+  ];
+}
+
+/** @param {object} turn @param {boolean} withPrefix @returns {string[]} */
+function renderUserTurn(turn, withPrefix) {
+  const lines = [];
+  for (const block of turn.content) {
+    if (block.type === "text") {
+      lines.push(
+        renderTextLine({
+          source: turn.source,
+          text: `[user] ${block.text}`,
+          withPrefix,
+        }),
+      );
+    }
+  }
+  return lines;
+}

package/src/supervisor.js

@@ -172,39 +172,26 @@ export class Supervisor {
           : await this.agentRunner.run(relay);
         agentCalled = true;
 
-        if (agentResult.error && !agentResult.aborted) {
-          this.emitSummary({ success: false, turns: turn });
-          return { exit: { success: false, turns: turn } };
-        }
+        const outcome = this.#classifyAgentOutcome(
+          agentResult,
+          turn,
+          interventions,
+        );
 
-        if (this.ctx.concluded) {
-          this.emitSummary({
-            success: true,
-            turns: turn,
-            summary: this.ctx.summary,
-          });
-          return { exit: { success: true, turns: turn } };
-        }
+        if (outcome.type === "exit") return { exit: outcome.exit };
+        if (outcome.type === "intervention_limit") return { exit: null };
 
-        if (agentResult.aborted && this.ctx.redirect) {
+        if (outcome.type === "redirect") {
           interventions++;
-          const redirect = this.ctx.redirect;
-          this.ctx.redirect = null;
-          if (interventions >= MAX_INTERVENTIONS_PER_TURN) {
-            this.emitOrchestratorEvent({ type: "intervention_limit", turn });
-            return { exit: null };
-          }
-          relay = redirect.message;
+          relay = outcome.relay;
           this.emitOrchestratorEvent({ type: "intervention_relayed", turn });
           continue;
         }
 
-        if (this.#checkAsk("agent") === "recheck" && !this.ctx.concluded) {
-          const reminders = this.messageBus.drain("agent");
-          if (reminders.length > 0) {
-            relay = formatMessages(reminders);
-            continue;
-          }
+        const askRelay = this.#drainAgentAskRelay();
+        if (askRelay) {
+          relay = askRelay;
+          continue;
         }
 
         return { exit: null };
@@ -214,6 +201,50 @@ export class Supervisor {
     }
   }
 
+  /**
+   * Classify the outcome of a single agent execution within #runAgentTurn.
+   * @returns {{type: string, exit?: object|null, relay?: string}}
+   */
+  #classifyAgentOutcome(agentResult, turn, interventions) {
+    if (agentResult.error && !agentResult.aborted) {
+      this.emitSummary({ success: false, turns: turn });
+      return { type: "exit", exit: { success: false, turns: turn } };
+    }
+
+    if (this.ctx.concluded) {
+      this.emitSummary({
+        success: true,
+        turns: turn,
+        summary: this.ctx.summary,
+      });
+      return { type: "exit", exit: { success: true, turns: turn } };
+    }
+
+    if (agentResult.aborted && this.ctx.redirect) {
+      const redirect = this.ctx.redirect;
+      this.ctx.redirect = null;
+      if (interventions + 1 >= MAX_INTERVENTIONS_PER_TURN) {
+        this.emitOrchestratorEvent({ type: "intervention_limit", turn });
+        return { type: "intervention_limit" };
+      }
+      return { type: "redirect", relay: redirect.message };
+    }
+
+    return { type: "continue" };
+  }
+
+  /**
+   * If the agent has an unanswered ask, drain reminders and return a
+   * formatted relay string. Returns null when no relay is needed.
+   * @returns {string|null}
+   */
+  #drainAgentAskRelay() {
+    if (this.#checkAsk("agent") !== "recheck" || this.ctx.concluded)
+      return null;
+    const reminders = this.messageBus.drain("agent");
+    return reminders.length > 0 ? formatMessages(reminders) : null;
+  }
+
   /**
    * Mid-turn supervisor review fired from inside the agent's onBatch hook.
    * Runs the supervisor's LLM against the batch and aborts the agent if

package/src/tee-writer.js

@@ -17,16 +17,7 @@
 
 import { Writable } from "node:stream";
 import { TraceCollector } from "./trace-collector.js";
-import {
-  renderTextLine,
-  renderToolCallLine,
-  renderToolResultLine,
-} from "./render/line-renderer.js";
-import {
-  hintForCall,
-  previewForResult,
-  simplifyToolName,
-} from "./render/tool-hints.js";
+import { renderTurnLines } from "./render/turn-renderer.js";
 import { isSuppressedOrchestratorEvent } from "./render/orchestrator-filter.js";
 
 export class TeeWriter extends Writable {
@@ -134,56 +125,8 @@ export class TeeWriter extends Writable {
     const withPrefix = this.mode !== "raw";
     while (this.turnsEmitted < turns.length) {
       const turn = turns[this.turnsEmitted++];
-      if (turn.role === "assistant") {
-        for (const block of turn.content) {
-          if (block.type === "text") {
-            this.textStream.write(
-              renderTextLine({
-                source: turn.source,
-                text: block.text,
-                withPrefix,
-              }),
-            );
-          } else if (block.type === "tool_use") {
-            this.textStream.write(
-              renderToolCallLine({
-                source: turn.source,
-                toolName: simplifyToolName(block.name),
-                hint: hintForCall(block.name, block.input),
-                withPrefix,
-              }),
-            );
-          }
-        }
-      } else if (turn.role === "tool_result") {
-        this.textStream.write(
-          renderToolResultLine({
-            source: turn.source,
-            preview: previewForResult(turn.content, turn.isError),
-            withPrefix,
-          }),
-        );
-      } else if (turn.role === "system") {
-        const label = turn.subtype ?? "system";
-        this.textStream.write(
-          renderTextLine({
-            source: turn.source,
-            text: `[${label}]`,
-            withPrefix,
-          }),
-        );
-      } else if (turn.role === "user") {
-        for (const block of turn.content) {
-          if (block.type === "text") {
-            this.textStream.write(
-              renderTextLine({
-                source: turn.source,
-                text: `[user] ${block.text}`,
-                withPrefix,
-              }),
-            );
-          }
-        }
+      for (const line of renderTurnLines(turn, withPrefix)) {
+        this.textStream.write(line);
       }
     }
   }

package/src/trace-collector.js

@@ -9,16 +9,7 @@
  * one formatting path (spec 540).
  */
 
-import {
-  renderTextLine,
-  renderToolCallLine,
-  renderToolResultLine,
-} from "./render/line-renderer.js";
-import {
-  hintForCall,
-  previewForResult,
-  simplifyToolName,
-} from "./render/tool-hints.js";
+import { renderTurnLines } from "./render/turn-renderer.js";
 import { isSuppressedOrchestratorEvent } from "./render/orchestrator-filter.js";
 
 export class TraceCollector {
@@ -270,68 +261,10 @@ export class TraceCollector {
     const out = [];
 
     for (const turn of this.turns) {
-      if (turn.role === "assistant") {
-        for (const block of turn.content) {
-          if (block.type === "text") {
-            out.push(
-              renderTextLine({
-                source: turn.source,
-                text: block.text,
-                withPrefix,
-              }),
-            );
-          } else if (block.type === "tool_use") {
-            out.push(
-              renderToolCallLine({
-                source: turn.source,
-                toolName: simplifyToolName(block.name),
-                hint: hintForCall(block.name, block.input),
-                withPrefix,
-              }),
-            );
-          }
-        }
-      } else if (turn.role === "tool_result") {
-        out.push(
-          renderToolResultLine({
-            source: turn.source,
-            preview: previewForResult(turn.content, turn.isError),
-            withPrefix,
-          }),
-        );
-      } else if (turn.role === "system") {
-        const label = turn.subtype ?? "system";
-        out.push(
-          renderTextLine({
-            source: turn.source,
-            text: `[${label}]`,
-            withPrefix,
-          }),
-        );
-      } else if (turn.role === "user") {
-        for (const block of turn.content) {
-          if (block.type === "text") {
-            out.push(
-              renderTextLine({
-                source: turn.source,
-                text: `[user] ${block.text}`,
-                withPrefix,
-              }),
-            );
-          }
-        }
-      }
+      out.push(...renderTurnLines(turn, withPrefix));
     }
 
-    // Trailing result block — the one summary line humans want (spec 540).
-    let tail = "";
-    if (this.result) {
-      const duration = formatDuration(this.result.durationMs);
-      const cost = Number(this.result.totalCostUsd).toFixed(4);
-      tail =
-        "\n" +
-        `--- Result: ${this.result.result} | Turns: ${this.result.numTurns} | Cost: $${cost} | Duration: ${duration} ---`;
-    }
+    const tail = this.#formatResultTail();
 
     // Each rendered line already ends with `\n`; concatenate, drop the
     // trailing newline, then append the tail so the output shape stays
@@ -341,6 +274,20 @@ export class TraceCollector {
     const body = out.join("").replace(/\n$/, "");
     return body + tail;
   }
+
+  /**
+   * Format the trailing result summary line (spec 540).
+   * @returns {string}
+   */
+  #formatResultTail() {
+    if (!this.result) return "";
+    const duration = formatDuration(this.result.durationMs);
+    const cost = Number(this.result.totalCostUsd).toFixed(4);
+    return (
+      "\n" +
+      `--- Result: ${this.result.result} | Turns: ${this.result.numTurns} | Cost: $${cost} | Duration: ${duration} ---`
+    );
+  }
 }
 
 /**

package/src/trace-github.js

@@ -48,7 +48,6 @@ export class TraceGitHub {
     const data = await this.#get(url);
     const runs = data.workflow_runs ?? [];
 
-    // eslint-disable-next-line security/detect-non-literal-regexp -- pattern is caller-controlled, not untrusted input
     const re = new RegExp(pattern, "i");
     return runs
       .filter((r) => re.test(r.name))

package/src/trace-query.js

@@ -81,24 +81,12 @@ export class TraceQuery {
    */
   filter(opts = {}) {
     const { role, toolName, isError } = opts;
-    return this.turns.filter((turn) => {
-      if (role !== undefined && turn.role !== role) return false;
-      if (isError !== undefined) {
-        if (turn.role !== "tool_result") return false;
-        if (turn.isError !== isError) return false;
-      }
-      if (toolName !== undefined) {
-        if (turn.role === "assistant") {
-          const has = turn.content.some(
-            (b) => b.type === "tool_use" && b.name === toolName,
-          );
-          if (!has) return false;
-        } else {
-          return false;
-        }
-      }
-      return true;
-    });
+    return this.turns.filter(
+      (turn) =>
+        matchesRole(turn, role) &&
+        matchesError(turn, isError) &&
+        matchesToolName(turn, toolName),
+    );
   }
 
   /** @returns {number} */
@@ -151,7 +139,6 @@ export class TraceQuery {
    */
   search(pattern, opts = {}) {
     const { context = 0, limit = 50, full = false } = opts;
-    // eslint-disable-next-line security/detect-non-literal-regexp -- pattern is caller-controlled, not untrusted input
     const re = new RegExp(pattern, "gi");
     const hits = [];
 
@@ -200,30 +187,18 @@ export class TraceQuery {
    * @returns {object[]}
    */
   tool(name) {
-    const toolUseIds = new Set();
-    const results = [];
-
-    for (const turn of this.turns) {
-      if (turn.role === "assistant") {
-        const hasTool = turn.content.some(
-          (b) => b.type === "tool_use" && b.name === name,
-        );
-        if (hasTool) {
-          results.push(turn);
-          for (const b of turn.content) {
-            if (b.type === "tool_use" && b.name === name && b.toolUseId) {
-              toolUseIds.add(b.toolUseId);
-            }
-          }
-        }
-      } else if (
-        turn.role === "tool_result" &&
-        toolUseIds.has(turn.toolUseId)
-      ) {
-        results.push(turn);
-      }
-    }
-    return results;
+    const toolUseIds = collectToolUseIds(this.turns, name);
+    const assistantTurns = this.turns.filter(
+      (t) =>
+        t.role === "assistant" &&
+        t.content.some((b) => b.type === "tool_use" && b.name === name),
+    );
+    const resultTurns = this.turns.filter(
+      (t) => t.role === "tool_result" && toolUseIds.has(t.toolUseId),
+    );
+    return [...assistantTurns, ...resultTurns].sort(
+      (a, b) => a.index - b.index,
+    );
   }
 
   /**
@@ -343,6 +318,57 @@ export class TraceQuery {
   }
 }
 
+/**
+ * @param {object} turn
+ * @param {string|undefined} role
+ * @returns {boolean}
+ */
+function matchesRole(turn, role) {
+  return role === undefined || turn.role === role;
+}
+
+/**
+ * @param {object} turn
+ * @param {boolean|undefined} isError
+ * @returns {boolean}
+ */
+function matchesError(turn, isError) {
+  if (isError === undefined) return true;
+  return turn.role === "tool_result" && turn.isError === isError;
+}
+
+/**
+ * @param {object} turn
+ * @param {string|undefined} toolName
+ * @returns {boolean}
+ */
+function matchesToolName(turn, toolName) {
+  if (toolName === undefined) return true;
+  return (
+    turn.role === "assistant" &&
+    turn.content.some((b) => b.type === "tool_use" && b.name === toolName)
+  );
+}
+
+/**
+ * Collect all toolUseIds for a given tool name from assistant turns.
+ * @param {object[]} turns
+ * @param {string} name
+ * @returns {Set<string>}
+ */
+function collectToolUseIds(turns, name) {
+  const ids = new Set();
+  for (const turn of turns) {
+    if (turn.role !== "assistant") continue;
+    for (const b of turn.content) {
+      if (b.type === "tool_use" && b.name === name && b.toolUseId) {
+        ids.add(b.toolUseId);
+      }
+    }
+  }
+  return ids;
+}
+
 /**
  * Search a single turn for regex matches. Returns array of match descriptions.
  * @param {object} turn