@forwardimpact/libharness 3.0.2 → 3.1.0

package/README.md

@@ -214,17 +214,39 @@ also lists the `Edit()` rules it tried.
 
 ## Documentation
 
-- [Coordinate an Agent Team](https://www.forwardimpact.team/docs/libraries/coordinate-team/index.md)
+- [Coordinate an Agent Team](https://www.gemba.team/docs/coordinate-team/index.md)
   — run a lead and N participant agents in one async session (supervise /
   facilitate / discuss) with Ask/Answer/Announce and a single NDJSON trace.
-- [Run an Eval](https://www.forwardimpact.team/docs/libraries/prove-changes/run-eval/index.md)
+- [Run an Eval](https://www.gemba.team/docs/prove-changes/run-eval/index.md)
   — author a judge profile, run an eval locally, wire it into CI, and inspect
   the trace it produces.
-- [Prove Agent Changes](https://www.forwardimpact.team/docs/libraries/prove-changes/index.md)
+- [Prove Agent Changes](https://www.gemba.team/docs/prove-changes/index.md)
   — the end-to-end workflow from dataset generation through evaluation to
   trace analysis, with multi-agent collaboration sessions.
-- [Analyze Traces](https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md)
+- [Analyze Traces](https://www.gemba.team/docs/prove-changes/trace-analysis/index.md)
   — read the NDJSON traces produced by `gemba-harness` with `gemba-trace`.
 - [Agent Teams](https://www.forwardimpact.team/docs/products/agent-teams/index.md)
   — author the profiles consumed by `--agent-profile`, `--lead-profile`, and
   `--agent-profiles`.
+
+## Documentation home
+
+libharness is an import-only library. It declares no `bin`. The
+`gemba-harness`, `gemba-trace`, `gemba-benchmark`, and `gemba-selfedit`
+commands ship with the Gemba product, which imports these modules. Run them
+with `npx gemba-harness`, `npx gemba-trace`, or `npx gemba-benchmark`, or use
+the installed `gemba-*` binaries. `gemba-selfedit` publishes no bare launcher.
+Install `@forwardimpact/gemba` to get it.
+
+The package publishes as `@forwardimpact/libharness` on the Forward Impact npm
+scope. Install it with `npm install @forwardimpact/libharness`. Its task guides
+live on the Gemba site at <https://www.gemba.team/>. The Forward Impact library
+guide tree at <https://www.forwardimpact.team/docs/libraries/index.md> is not
+this library's guide home.
+
+**Decision (2026-08-26):** the split package scope and guide host are
+deliberate. libharness stays a Gear npm package, so `package.json .homepage`
+keeps <https://www.forwardimpact.team>. The agent-runtime guides moved to
+gemba.team with the rest of the Gemba product. The `## Documentation` list
+above carries the current URLs. Old `forwardimpact.team/docs/libraries/`
+addresses forward to gemba.team.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",

package/src/benchmark/judge.js

@@ -9,17 +9,19 @@
  *
  *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
  *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
- *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
+ *   {{AGENT_TRACE_PATH}}    — absolute path to the cell's agent lane,
+ *                             trace--<case>--agent.agent.ndjson (materialized
+ *                             before any session runs)
  *   {{GRADE_RESULT}}        — JSON grade object plus the merged check rows
  *   {{SKILL_SET_HASH}}      — SHA-256 from apm.lock.yaml
  *   {{TASK_ID}}             — task name (directory under tasks/)
  *   {{TASK_DIR}}            — path to the agent working directory
  *
- * The adapter reads the judge verdict directly from the orchestration
- * context's `concluded` flag. It does not parse the trace on the happy path.
- * `parseConcludeFromTrace` stays for offline analysis. It is also a fallback
- * when the runtime ctx is not available, for example when you re-grade a
- * historical run from its judge.ndjson file.
+ * The judge verdict is captured from the orchestration context's
+ * `concluded` flag directly — no trace parsing on the happy path.
+ * `parseConcludeFromTrace` is preserved for offline analysis and as a
+ * fallback when the runtime ctx isn't available (e.g. re-grading a
+ * historical run from its preserved judge lane file).
  */
 
 import { createJudge } from "../judge.js";

package/src/benchmark/raw-summary.js

@@ -0,0 +1,79 @@
+/**
+ * Raw-trace summary for the benchmark runner: one post-session read of the
+ * preserved raw combined trace derives cost, turns, and submission. Named as
+ * its own module so summarization never re-entangles with splitting — the
+ * coupling that caused the original split-policy divergence.
+ */
+
+import { sumTraceCost } from "../cost.js";
+import { parseEnvelopeLine } from "../trace-split.js";
+
+/**
+ * One read of the preserved raw combined trace:
+ *   cost       — `sumTraceCost` over the lines (the one cost path),
+ *   turns      — last orchestrator-source `summary` event's `turns`,
+ *   submission — last agent-source assistant text block.
+ *
+ * An empty (materialized-stub) raw file yields zeros and an empty
+ * submission; malformed and blank lines are tolerated.
+ *
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} rawTracePath
+ * @returns {Promise<{costUsd: number,
+ *   costBreakdown: {agent: number, supervisor: number},
+ *   turns: number, submission: string}>}
+ */
+export async function summarizeRawTrace(runtime, rawTracePath) {
+  const content = await runtime.fs.readFile(rawTracePath, "utf8");
+  const lines = content.split("\n");
+  const { totalCostUsd, bySource } = sumTraceCost(lines);
+  const { turns, submission } = deriveTurnsAndSubmission(lines);
+
+  return {
+    costUsd: totalCostUsd,
+    costBreakdown: {
+      agent: bySource.agent ?? 0,
+      supervisor: bySource.supervisor ?? 0,
+    },
+    turns,
+    submission,
+  };
+}
+
+/**
+ * One walk of the parsed envelope lines: the last orchestrator `summary`
+ * event's `turns` and the last agent assistant text block.
+ * @param {string[]} lines
+ * @returns {{turns: number, submission: string}}
+ */
+function deriveTurnsAndSubmission(lines) {
+  let turns = 0;
+  let submission = "";
+  for (const line of lines) {
+    const envelope = parseEnvelopeLine(line);
+    if (!envelope) continue;
+    const inner = envelope.event;
+    if (envelope.source === "agent" && inner.type === "assistant") {
+      const text = extractText(inner);
+      if (text) submission = text;
+    }
+    if (envelope.source === "orchestrator" && inner.type === "summary") {
+      turns = inner.turns ?? 0;
+    }
+  }
+  return { turns, submission };
+}
+
+/**
+ * Last text block of an assistant event's content, or null when none exists.
+ * @param {object} inner - Unwrapped event.
+ * @returns {string|null}
+ */
+function extractText(inner) {
+  const content = inner.message?.content ?? inner.content;
+  if (!Array.isArray(content)) return null;
+  for (let i = content.length - 1; i >= 0; i--) {
+    if (content[i].type === "text" && content[i].text) return content[i].text;
+  }
+  return null;
+}

package/src/benchmark/result.js

@@ -104,9 +104,15 @@ const HAPPY_RECORD = z.object({
   score: z.number().min(0).max(1).optional(),
   submission: z.string(),
   judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
+  // Trace paths are relative to the run output directory — valid on the
+  // runner and inside a downloaded trace artifact alike. Raw and
+  // agent/supervisor lanes are materialized at workdir allocation, so they
+  // are present on every executed cell; the judge lane exists only on
+  // judged cells.
+  rawTracePath: z.string(),
   agentTracePath: z.string(),
   supervisorTracePath: z.string(),
-  judgeTracePath: z.string(),
+  judgeTracePath: z.string().optional(),
   agentError: AGENT_ERROR_SHAPE.optional(),
   preflightError: z.undefined().optional(),
 });
@@ -115,12 +121,12 @@ const PREFLIGHT_RECORD = z.object({
   ...COMMON_FIELDS,
   costUsd: z.literal(0),
   preflightError: PREFLIGHT_ERROR_SHAPE,
-  // The runner allocates the trace paths in WorkdirManager.start, even on
-  // preflight failure. The record then stays uniform across branches, and
-  // downstream consumers can reference the paths without conditional fields.
-  agentTracePath: z.string(),
-  supervisorTracePath: z.string(),
-  judgeTracePath: z.string(),
+  // No trace-path fields: a preflight-failure record references only traces
+  // a session produced, even though the materialized stubs exist on disk.
+  rawTracePath: z.undefined().optional(),
+  agentTracePath: z.undefined().optional(),
+  supervisorTracePath: z.undefined().optional(),
+  judgeTracePath: z.undefined().optional(),
   invariants: z.undefined().optional(),
   grade: z.undefined().optional(),
   hiddenTests: z.undefined().optional(),

package/src/benchmark/runner.js

@@ -19,11 +19,11 @@
  * ledger. The iterator mirrors the same stream to CLI stdout.
  */
 
-import { join, resolve as resolvePath } from "node:path";
+import { join, relative, resolve as resolvePath } from "node:path";
 
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
-import { sumTraceCost } from "../cost.js";
 import { createSupervisor } from "../supervisor.js";
+import { splitTrace } from "../trace-split.js";
 import { installApm as defaultInstallApm } from "./apm-installer.js";
 import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
@@ -31,8 +31,8 @@ import { validateResultRecord } from "./result.js";
 import { runInvariants } from "./invariants.js";
 import { runHiddenTests } from "./hidden-tests.js";
 import { runProducersAndGrade } from "./grade.js";
+import { summarizeRawTrace } from "./raw-summary.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
-import { splitAndSummarize } from "./trace-split.js";
 import { createWorkdirManager } from "./workdir.js";
 import { CellScheduler } from "./scheduler.js";
 
@@ -77,9 +77,10 @@ export class BenchmarkRunner {
    *   to `AGENT_WATCHDOG_MS`. A test injects its own to force a stall in-test.
    * @param {number} [opts.termGraceMs] - SIGTERM→SIGKILL grace (ms) for the per-task process group.
    * @param {Function} [opts.runAgent] - Test seam: replaces the agent-under-test
-   *   session. Must return `{costUsd, turns, submission, agentError?}` and
-   *   write a valid NDJSON trace to `workdir.agentTracePath`. Default uses
-   *   `createAgentRunner` with the harness `BASE_TOOLS` allowlist. Internal
+   *   session. Must run the session, stream `{source, seq, event}` envelopes
+   *   to `workdir.rawTracePath`, and return `{agentError?}` — cost, turns,
+   *   and submission are always derived from the raw file by the shared
+   *   split/summary pipeline, so the seam exercises the real path. Internal
    *   testing only. It is not part of the public API.
    * @param {import("@forwardimpact/libutil/runtime").Runtime} opts.runtime -
    *   The host injects these ambient collaborators (`fs`, `subprocess`,
@@ -256,12 +257,12 @@ export class BenchmarkRunner {
         t0,
       });
     } catch (e) {
-      // `wm.start()` (port acquire + workdir/env seed) is the one throw site
-      // that `#executeCell` does not catch. Turn it into the runner's own
-      // fallback record so `#runOne` never rejects. The scheduler's
-      // one-record-per-cell contract depends on that. `report` skips the
-      // fallback because it fails the schema, the same as any other
-      // runner-side schema failure.
+      // Catches the throw sites `#executeCell` does not: `wm.start()` (port
+      // acquire + workdir/env seed) and the shared split/summary pipeline.
+      // Turn either into the runner's own fallback record so `#runOne` never
+      // rejects. The scheduler's one-record-per-cell contract depends on
+      // that. `report` skips the fallback because it fails the schema, the
+      // same as any other runner-side schema failure.
       return {
         taskId: task.id,
         runIndex,
@@ -301,8 +302,8 @@ export class BenchmarkRunner {
     }
     {
       const agentRun = await this.#runAgentSafe(task, workdir);
-      const { costUsd, turns, submission, agentError } = agentRun;
-      const breakdown = agentRun.costBreakdown ?? { agent: 0, supervisor: 0 };
+      const { costUsd, costBreakdown, turns, submission, agentError } =
+        agentRun;
       const graded = await this.#gradeCell(family, task, workdir);
       const { invariants, hiddenRows, engineError, rows, grade } = graded;
       const { judgeVerdict, judgeCost } = await this.#judgeCell({
@@ -319,6 +320,7 @@ export class BenchmarkRunner {
       // or a judge that fails zeroes the effective score. Full marks does not
       // zero it. A fractional score with verdict fail is the point.
       const scoreValid = graded.healthy && grade.gatesPass && judgePass;
+      const tracePaths = await this.#traceRecordPaths(workdir);
       const record = {
         taskId: task.id,
         runIndex,
@@ -337,15 +339,9 @@ export class BenchmarkRunner {
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd: costUsd + judgeCost,
-        costBreakdown: {
-          agent: breakdown.agent ?? 0,
-          supervisor: breakdown.supervisor ?? 0,
-          judge: judgeCost,
-        },
+        costBreakdown: { ...costBreakdown, judge: judgeCost },
         turns,
-        agentTracePath: workdir.agentTracePath,
-        supervisorTracePath: workdir.supervisorTracePath,
-        judgeTracePath: workdir.judgeTracePath,
+        ...tracePaths,
         profiles: {
           agent: this.profiles.agent,
           supervisor: null,
@@ -424,39 +420,42 @@ export class BenchmarkRunner {
   }
 
   /**
-   * Dispatch to either the injected hook or the default `#runAgent`. Either
-   * path can throw. Catch the error here so it becomes an `agentError` on the
-   * record (spec criterion 1: records on agent failure). The iterator then
-   * does not abort.
+   * Dispatch to either the injected hook or the default `#runAgent`, then run
+   * the shared pipeline once: split the preserved raw trace into lanes and
+   * derive cost/turns/submission from the same file. Either session path can
+   * throw; catch here so a session error becomes an `agentError` on the
+   * record rather than aborting the whole iterator — the pipeline still runs,
+   * so failed cells keep coherent (possibly empty) lanes and zeroed totals.
+   * A pipeline throw itself (the raw file is materialized at allocation, so
+   * only an fs-level fault) propagates to `#runOne`'s fallback record.
    */
   async #runAgentSafe(task, workdir) {
+    let agentError = null;
     try {
-      if (this._runAgentHook) {
-        const r = await this._runAgentHook(task, workdir, this);
-        return { agentError: null, ...r };
-      }
-      return await this.#runAgent(task, workdir);
+      const r = this._runAgentHook
+        ? await this._runAgentHook(task, workdir, this)
+        : await this.#runAgent(task, workdir);
+      agentError = r?.agentError ?? null;
     } catch (e) {
-      return {
-        costUsd: 0,
-        costBreakdown: { agent: 0, supervisor: 0 },
-        turns: 0,
-        submission: "",
-        agentError: { message: e.message ?? String(e), aborted: false },
-      };
+      agentError = { message: e.message ?? String(e), aborted: false };
     }
+    await splitTrace(this.runtime, workdir.rawTracePath, {
+      caseId: workdir.caseId,
+      outputDir: workdir.runDir,
+    });
+    const summary = await summarizeRawTrace(this.runtime, workdir.rawTracePath);
+    return { ...summary, agentError };
   }
 
   /**
-   * Run the agent-under-test under a Supervisor. The supervisor writes
-   * a combined tagged NDJSON trace. After the session, this method splits
-   * the trace into agent.ndjson and supervisor.ndjson. It also extracts
-   * cost/turns/submission.
+   * Run the agent-under-test under a Supervisor. The supervisor streams the
+   * combined tagged NDJSON envelope trace to `workdir.rawTracePath`, which is
+   * preserved for the life of the run output; `#runAgentSafe` splits it into
+   * the convention-named lanes and summarizes it afterwards.
    */
   async #runAgent(task, workdir) {
     const fs = this.runtime.fs;
-    const combinedPath = join(workdir.runDir, ".combined.ndjson");
-    const combinedStream = fs.createWriteStream(combinedPath);
+    const combinedStream = fs.createWriteStream(workdir.rawTracePath);
     const supervisorInstructions = task.paths.supervisor
       ? await fs.readFile(task.paths.supervisor, "utf8").catch(() => null)
       : null;
@@ -510,27 +509,31 @@ export class BenchmarkRunner {
       this.runtime.clock.clearTimeout(watchdog);
       await new Promise((r) => combinedStream.end(r));
     }
-    const summary = await splitAndSummarize(
-      this.runtime,
-      combinedPath,
-      workdir.agentTracePath,
-      workdir.supervisorTracePath,
-    );
-    // `sumTraceCost` sums the cost across every participant's result events
-    // from the one combined trace, and attributes it per source. Read the
-    // trace before you unlink it.
-    const combined = await fs.readFile(combinedPath, "utf8");
-    const { totalCostUsd, bySource } = sumTraceCost(combined.split("\n"));
-    await fs.unlink(combinedPath).catch(() => {});
-    return {
-      ...summary,
-      costUsd: totalCostUsd,
-      costBreakdown: {
-        agent: bySource.agent ?? 0,
-        supervisor: bySource.supervisor ?? 0,
-      },
-      agentError,
+    return { agentError };
+  }
+
+  /**
+   * Run-output-relative trace-path record fields, each present only when its
+   * file exists: raw and agent/supervisor lanes are materialized at workdir
+   * allocation (present on every executed cell); the judge lane exists only
+   * on judged cells. Relative paths stay valid inside a downloaded artifact.
+   */
+  async #traceRecordPaths(workdir) {
+    const fields = {
+      rawTracePath: workdir.rawTracePath,
+      agentTracePath: workdir.agentTracePath,
+      supervisorTracePath: workdir.supervisorTracePath,
+      judgeTracePath: workdir.judgeTracePath,
     };
+    const out = {};
+    for (const [field, absPath] of Object.entries(fields)) {
+      const exists = await this.runtime.fs
+        .access(absPath)
+        .then(() => true)
+        .catch(() => false);
+      if (exists) out[field] = relative(this.output, absPath);
+    }
+    return out;
   }
 
   async #buildJudgeContext(task, workdir, skillSetHash) {
@@ -579,9 +582,9 @@ export class BenchmarkRunner {
       skillSetHash,
       familyRevision,
       durationMs,
-      agentTracePath: workdir.agentTracePath,
-      supervisorTracePath: workdir.supervisorTracePath,
-      judgeTracePath: workdir.judgeTracePath,
+      // No trace-path fields: even though the materialized stubs exist on
+      // disk, a preflight-failure record references only traces a session
+      // produced.
     };
   }
 

package/src/benchmark/task-family.js

@@ -35,6 +35,8 @@
 import { createHash } from "node:crypto";
 import { join, posix, relative, resolve, sep } from "node:path";
 
+import { isValidTaskId } from "../trace-identity.js";
+
 const GIT_URL_RE = /^(git@|https?:\/\/|ssh:\/\/|git:\/\/)/;
 const SKIP_DIRS = new Set([".git", "node_modules"]);
 // POSIX `X_OK` (execute permission). Node's fs honours the numeric mode, so we
@@ -129,6 +131,13 @@ async function discoverTasks(runtime, rootPath) {
 }
 
 async function loadTask(fs, taskDir, id) {
+  // The rule itself lives in the identity module; the loader only invokes
+  // the predicate so an unbuildable case id fails before any agent spend.
+  if (!isValidTaskId(id)) {
+    throw new Error(
+      `invalid task id '${id}': task directory names must not contain "--" or start/end with "-"`,
+    );
+  }
   const supervisorPath = join(taskDir, "supervisor.task.md");
   const judgePath = join(taskDir, "judge.task.md");
   const preflightPath = join(taskDir, "hooks", "preflight.sh");

package/src/benchmark/workdir.js

@@ -16,6 +16,11 @@ import { createServer } from "node:net";
 import { connect } from "node:net";
 import { join } from "node:path";
 
+import {
+  buildCaseId,
+  laneFilename,
+  rawTraceFilename,
+} from "../trace-identity.js";
 import { loadEnv } from "./env-loader.js";
 import { buildHookEnv } from "./hook-env.js";
 
@@ -28,6 +33,8 @@ const DEFAULT_TERM_GRACE_MS = 5_000;
  * @property {number} port - Allocated TCP port for the agent.
  * @property {number} pgid - Process-group id captured from the preflight child.
  * @property {*} scaffold - Reserved per design § Components. v1 sets null.
+ * @property {string} caseId - Grid-unique case identity `<taskId>-r<runIndex>`.
+ * @property {string} rawTracePath - Combined raw envelope trace (kept).
  * @property {string} agentTracePath
  * @property {string} supervisorTracePath
  * @property {string} judgeTracePath
@@ -72,8 +79,7 @@ export class WorkdirManager {
    */
   async start(task, runIndex) {
     const fs = this.runtime.fs;
-    const slug = task.id.replace("/", "__");
-    const runDir = join(this.runOutputDir, "runs", slug, String(runIndex));
+    const runDir = join(this.runOutputDir, "runs", task.id, String(runIndex));
     const cwd = join(runDir, "cwd");
     await fs.mkdir(cwd, { recursive: true });
 
@@ -124,11 +130,25 @@ export class WorkdirManager {
     const envNames =
       envDirs.length > 0 ? await loadEnv(envDirs, cwd, this.runtime) : [];
 
-    const port = await this.ports.acquire();
-    const agentTracePath = join(runDir, "agent.ndjson");
-    const supervisorTracePath = join(runDir, "supervisor.ndjson");
-    const judgeTracePath = join(runDir, "judge.ndjson");
+    // Allocate identity and trace paths before acquiring the port so an
+    // invalid task id cannot leak a port reservation.
+    const caseId = buildCaseId(task.id, runIndex);
+    const rawTracePath = join(runDir, rawTraceFilename(caseId));
+    const agentTracePath = join(runDir, laneFilename(caseId, "agent", "agent"));
+    const supervisorTracePath = join(
+      runDir,
+      laneFilename(caseId, "supervisor", "supervisor"),
+    );
+    const judgeTracePath = join(runDir, laneFilename(caseId, "judge", "judge"));
+    // Materialize the raw and agent/supervisor lanes empty at allocation so
+    // every path that reaches the judge — including a pre-session agent
+    // failure — finds them on disk. The judge lane is written by the judge
+    // session itself and exists only on judged cells.
+    for (const p of [rawTracePath, agentTracePath, supervisorTracePath]) {
+      await fs.writeFile(p, "");
+    }
 
+    const port = await this.ports.acquire();
     const preflight = task.paths.preflight
       ? await runPreflight(this.runtime, task.paths.preflight, cwd, port, {
           taskId: task.id,
@@ -144,6 +164,8 @@ export class WorkdirManager {
       port,
       pgid: preflight.pgid,
       scaffold: null,
+      caseId,
+      rawTracePath,
       agentTracePath,
       supervisorTracePath,
       judgeTracePath,

package/src/commands/benchmark-definition.js

@@ -166,13 +166,13 @@ export const definition = {
   documentation: [
     {
       title: "Run a Benchmark",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      url: "https://www.gemba.team/docs/prove-changes/run-benchmark/index.md",
       description:
         "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
     },
     {
       title: "Automate with GitHub Actions",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      url: "https://www.gemba.team/docs/prove-changes/run-benchmark/ci-workflow/index.md",
       description:
         "Run benchmarks in CI with the forwardimpact/gemba-benchmark action.",
     },

package/src/commands/trace.js

@@ -3,6 +3,7 @@ import { isoTimestamp } from "@forwardimpact/libutil";
 import { createTraceCollector, sumTraceCost } from "@forwardimpact/libharness";
 import { createTraceQuery } from "../trace-query.js";
 import { createTraceGitHub } from "../trace-github.js";
+import { splitTrace } from "../trace-split.js";
 import { stripSignatures } from "../signature-filter.js";
 import { runOver, aggregate, compareTwo } from "../trace-multi.js";
 import {
@@ -100,7 +101,9 @@ export async function runRunsCommand(ctx) {
 }
 
 /**
- * Resolve a participant's lane trace for a known run id in one keyed lookup.
+ * Resolve a trace lane for a known run id in one keyed lookup. The key may
+ * be an exact member filename, a case id, or a participant name; ambiguous
+ * keys error with the matching candidates.
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  */
 export async function runFindCommand(ctx) {
@@ -110,7 +113,7 @@ export async function runFindCommand(ctx) {
     repo: ctx.options.repo,
     runtime,
   });
-  const result = await gh.findByKey(ctx.args["run-id"], ctx.args.participant, {
+  const result = await gh.findByKey(ctx.args["run-id"], ctx.args.key, {
     dir: ctx.options.dir,
   });
   writeJSON(runtime, result, ctx.options);
@@ -118,7 +121,22 @@ export async function runFindCommand(ctx) {
 }
 
 /**
- * Download a trace artifact and auto-convert to structured JSON.
+ * The single `.ndjson` member to auto-convert to structured JSON, or null
+ * when the artifact carries zero or several. Multi-member bundles (kata
+ * dispatch, harness matrix, eval shards) get no `structured.json` — the
+ * prior first-member conversion picked an arbitrary lane, which was actively
+ * misleading; the analysis verbs read the `.ndjson` members directly.
+ * @param {string[]} files - Extracted member paths, relative to the artifact dir.
+ * @returns {string|null}
+ */
+export function structuredConvertTarget(files) {
+  const ndjson = files.filter((f) => f.endsWith(".ndjson"));
+  return ndjson.length === 1 ? ndjson[0] : null;
+}
+
+/**
+ * Download a trace artifact; auto-convert to structured JSON only when the
+ * artifact carries exactly one `.ndjson` member.
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  */
 export async function runDownloadCommand(ctx) {
@@ -133,7 +151,7 @@ export async function runDownloadCommand(ctx) {
     name: ctx.options.artifact,
   });
 
-  const ndjsonFile = result.files.find((f) => f.endsWith(".ndjson"));
+  const ndjsonFile = structuredConvertTarget(result.files);
   if (ndjsonFile) {
     const ndjsonPath = join(result.dir, ndjsonFile);
     const collector = createTraceCollector({
@@ -496,27 +514,13 @@ export async function runCompareCommand(ctx) {
 
 // --- Split command ---
 
-/**
- * A valid source name starts with a lowercase letter. The rest uses lowercase
- * alphanumeric characters or hyphens.
- */
-const VALID_SOURCE_NAME = /^[a-z][a-z0-9-]*$/;
-
-/**
- * Sources whose name is itself a structural role. The splitter classifies
- * each one into the role it represents.
- */
-const STRUCTURAL_ROLES = new Set(["agent", "supervisor", "facilitator"]);
-
 /**
  * Split a combined NDJSON trace into per-source files. The output names
  * follow the `trace--<case>--<participant>.<role>.ndjson` convention.
  *
- * Each valid envelope source becomes one output file. Structural sources
- * (`agent`, `supervisor`, `facilitator`) classify into the matching role.
- * They use their own name as participant. Profile-named sources (e.g.
- * `staff-engineer`) classify as agents with the profile in the participant
- * slot. The command drops orchestrator events and invalid source names.
+ * The command owns the CLI concerns only: input and `--mode` validation,
+ * defaults, and output-dir creation. The shared `splitTrace` implementation
+ * owns the split itself, including source-to-role classification.
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  */
@@ -525,10 +529,11 @@ export async function runSplitCommand(ctx) {
   const file = ctx.args.file;
   if (!file) return { ok: false, code: 1, error: "split: missing input file" };
 
-  // `discuss` has the same lead + N-participants shape as `facilitate`. The
-  // splitter buckets purely by envelope `source`, which is mode-independent.
-  // So the CLI accepts `discuss` alongside the structural modes. The CLI owns
-  // this rule. Callers do not.
+  // `discuss` has the same lead + N-participants shape as `facilitate`, and the
+  // splitter buckets purely by envelope `source` (mode-independent), so it is
+  // accepted alongside the structural modes. The CLI owns this, not callers.
+  // `--mode` stays required-but-inert: the harness action passes it and that
+  // surface is out of scope for the shared-split extraction.
   const mode = ctx.options.mode;
   if (!mode) return { ok: false, code: 1, error: "split: --mode is required" };
   if (!["run", "supervise", "facilitate", "discuss"].includes(mode)) {
@@ -539,52 +544,10 @@ export async function runSplitCommand(ctx) {
   const outputDir = ctx.options["output-dir"] || dirname(file);
   runtime.fsSync.mkdirSync(outputDir, { recursive: true });
 
-  const buckets = parseBuckets(runtime.fsSync.readFileSync(file, "utf8"));
-
-  for (const [source, lines] of buckets.entries()) {
-    if (!VALID_SOURCE_NAME.test(source)) continue;
-    const role = STRUCTURAL_ROLES.has(source) ? source : "agent";
-    const outPath = join(
-      outputDir,
-      `trace--${caseId}--${source}.${role}.ndjson`,
-    );
-    runtime.fsSync.writeFileSync(outPath, lines.join("\n") + "\n");
-  }
+  await splitTrace(runtime, file, { caseId, outputDir });
   return { ok: true };
 }
 
-/**
- * Parse NDJSON content into per-source buckets of unwrapped event lines.
- * Skips empty lines, malformed JSON, non-envelope lines, and orchestrator events.
- * @param {string} content - Raw NDJSON file content
- * @returns {Map<string, string[]>} source name -> array of unwrapped JSON lines
- */
-function parseBuckets(content) {
-  const buckets = new Map();
-
-  for (const raw of content.split("\n")) {
-    const trimmed = raw.trim();
-    if (!trimmed) continue;
-
-    let envelope;
-    try {
-      envelope = JSON.parse(trimmed);
-    } catch {
-      continue;
-    }
-
-    if (!envelope.event || typeof envelope.source !== "string") continue;
-    if (envelope.source === "orchestrator") continue;
-
-    if (!buckets.has(envelope.source)) {
-      buckets.set(envelope.source, []);
-    }
-    buckets.get(envelope.source).push(JSON.stringify(envelope.event));
-  }
-
-  return buckets;
-}
-
 // --- Shared helpers ---
 
 /**

package/src/index.js

@@ -7,9 +7,9 @@ export {
   createTraceGitHub,
   detectRepoSlug,
   parseGitRemote,
-  participantInNames,
   pickTraceArtifact,
 } from "./trace-github.js";
+export { participantInNames } from "./trace-identity.js";
 export { AgentRunner, createAgentRunner } from "./agent-runner.js";
 export { resolveClaudeCodeExecutable } from "./claude-code-executable.js";
 export {

package/src/trace-github.js

@@ -4,6 +4,8 @@ import { Readable } from "node:stream";
 
 import { isoTimestamp } from "@forwardimpact/libutil";
 
+import { nameMatchesKey, participantInNames } from "./trace-identity.js";
+
 const API = "https://api.github.com";
 
 /**
@@ -48,14 +50,18 @@ export class TraceGitHub {
    * trace content.
    *
    * @param {object} [opts]
-   * @param {string} [opts.pattern] - Case-insensitive regex to match workflow name (default: "kata|agent", which covers `Kata: Shift`, `Kata: Dispatch`, and any `agent`-named workflow)
+   * @param {string} [opts.pattern] - Case-insensitive regex to match workflow name (default: "kata|agent|eval|benchmark" — covers `Kata: Shift`, `Kata: Dispatch`, benchmark-driven eval workflows, and any `agent`-named workflow)
    * @param {number} [opts.limit=50] - Max runs to return from GitHub API
    * @param {string} [opts.lookback="7d"] - How far back to search (e.g. "7d", "24h", "2w")
    * @param {string} [opts.participant] - Participant name. When set, the method filters and annotates runs by trace lane
    * @returns {Promise<object[]>} Array of {workflow, runId, status, conclusion, createdAt, branch, url[, match]}
    */
   async listRuns(opts = {}) {
-    const { pattern = "kata|agent", limit = 50, lookback = "7d" } = opts;
+    const {
+      pattern = "kata|agent|eval|benchmark",
+      limit = 50,
+      lookback = "7d",
+    } = opts;
     const cutoff = parseLookback(lookback, this.runtime.clock.now());
 
     const params = new URLSearchParams({
@@ -140,33 +146,41 @@ export class TraceGitHub {
     }
 
     // Dispatch host: one shared artifact whose members name the participant.
-    // Download and list member filenames (names only).
+    // Download and list member filenames (names only). Members are nested
+    // relative paths (`runs/<taskId>/<idx>/trace--*` on eval artifacts), so
+    // match on basenames — the `trace--` prefix check never matches a nested
+    // path directly.
     for (const artifact of traceArtifacts) {
       const { files } = await this.downloadTrace(runId, {
         name: artifact.name,
       });
-      if (participantInNames(files, participant)) return "confirmed";
+      const basenames = files.map((f) => path.basename(f));
+      if (participantInNames(basenames, participant)) return "confirmed";
     }
     return "omit";
   }
 
   /**
-   * Resolve a participant's lane trace path for a known run in one keyed
-   * lookup. The method does not enumerate runs. It does not inspect trace
-   * content.
+   * Resolve a trace lane path for a known run in one keyed lookup. The method
+   * does not enumerate runs. It does not inspect trace content. The key is an
+   * exact member filename, a case id, or a participant name.
    *
-   * Matrix host: the artifact name carries the participant (no download).
-   * Dispatch host: download the shared `trace--*` artifact. Return the
-   * extracted member file whose name carries the participant.
+   * Matrix host: the artifact name carries the key, so no download happens.
+   * Dispatch host: download every `trace--*` artifact and match member
+   * basenames against the key. Exactly one match resolves. Several matches
+   * throw an error that lists the candidates, so the caller narrows the key.
+   * This replaces a silent first-match, which returned an arbitrary cell's
+   * lane on eval runs, because every cell emits the same participants.
    *
    * @param {number|string} runId
-   * @param {string} participant
+   * @param {string} key - Exact member filename, case id, or participant name.
    * @param {object} [opts]
    * @param {string} [opts.dir] - Output directory for a downloaded dispatch artifact
-   * @returns {Promise<{runId: (number|string), participant: string, host: "matrix"|"dispatch", artifact: string, path: string}>}
-   * @throws {Error} when the run has no trace artifacts, or none carries the participant's lane.
+   * @returns {Promise<{runId: (number|string), key: string, host: "matrix"|"dispatch", artifact: string, path: string}>}
+   * @throws {Error} when the run has no trace artifacts, no member matches
+   *   the key, or several members match.
    */
-  async findByKey(runId, participant, opts = {}) {
+  async findByKey(runId, key, opts = {}) {
     const url = `${API}/repos/${this.owner}/${this.repo}/actions/runs/${runId}/artifacts`;
     const data = await this.#get(url);
     const artifacts = data.artifacts ?? [];
@@ -177,41 +191,57 @@ export class TraceGitHub {
       throw new Error(`No trace artifacts for run ${runId}`);
     }
 
-    // Matrix host: the artifact name carries the participant. No download.
+    // Matrix host: the artifact name carries the key. No download.
     const matrix = traceArtifacts.find((a) =>
-      participantInNames([a.name], participant),
+      participantInNames([a.name], key),
     );
     if (matrix) {
       return {
         runId,
-        participant,
+        key,
         host: "matrix",
         artifact: matrix.name,
         path: matrix.name,
       };
     }
 
-    // Dispatch host: download the shared artifact and match a member filename.
+    // Dispatch host: download every shared artifact and collect the members
+    // whose basename matches the key (members are nested relative paths).
+    // Each artifact extracts into its own subdirectory — a shared extract
+    // dir would re-list earlier artifacts' members on every iteration, so a
+    // uniquely-matching key on a multi-artifact (sharded) run would throw a
+    // spurious ambiguity error.
+    const baseDir = opts.dir ?? `/tmp/trace-${runId}`;
+    const matches = [];
     for (const artifact of traceArtifacts) {
       const { dir, files } = await this.downloadTrace(runId, {
         name: artifact.name,
-        dir: opts.dir,
+        dir: path.join(baseDir, artifact.name),
       });
-      const member = files.find((f) => participantInNames([f], participant));
-      if (member) {
-        return {
-          runId,
-          participant,
-          host: "dispatch",
-          artifact: artifact.name,
-          path: path.join(dir, member),
-        };
+      for (const member of files) {
+        if (nameMatchesKey(path.basename(member), key)) {
+          matches.push({ artifact: artifact.name, dir, member });
+        }
       }
     }
 
-    throw new Error(
-      `No trace lane for participant "${participant}" in run ${runId}`,
-    );
+    if (matches.length === 1) {
+      const m = matches[0];
+      return {
+        runId,
+        key,
+        host: "dispatch",
+        artifact: m.artifact,
+        path: path.join(m.dir, m.member),
+      };
+    }
+    if (matches.length > 1) {
+      const names = matches.map((m) => m.member).join(", ");
+      throw new Error(
+        `Ambiguous key "${key}" for run ${runId}: matches ${names}. Narrow the key to a case id or exact filename.`,
+      );
+    }
+    throw new Error(`No trace lane for key "${key}" in run ${runId}`);
   }
 
   /**
@@ -271,9 +301,9 @@ export class TraceGitHub {
       );
     }
 
-    // List extracted files.
-    const entries = await fs.readdir(dir);
-    const files = entries.filter((f) => !f.endsWith(".zip"));
+    // List extracted files — recursively, since eval artifacts carry nested
+    // members (`runs/<taskId>/<idx>/trace--*`).
+    const files = await listExtractedFiles(this.runtime, dir);
 
     return { dir, artifact: artifact.name, files };
   }
@@ -301,34 +331,22 @@ export class TraceGitHub {
 }
 
 /**
- * Test whether a participant's trace lane is present in a list of names.
- *
- * This function matches the two trace-name shapes by *name* only (never by
- * content):
- *   - matrix artifact name: `trace--<participant>`
- *   - dispatch member filename: `trace--<case>--<participant>.<role>.ndjson`
- *
- * The `--` separator delimits the participant segment. The segment ends at
- * the next `--`, at a `.`, or at the end of the string. So a substring like
- * `release` does not match `release-engineer` and vice versa.
- *
- * @param {string[]} names - Artifact names or extracted member filenames.
- * @param {string} participant - Participant name to look for.
- * @returns {boolean}
+ * List every regular file under `dir` recursively, as paths relative to
+ * `dir`, excluding `*.zip` (the downloaded archive itself). Sorted for a
+ * deterministic member order.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} dir
+ * @returns {Promise<string[]>}
  */
-export function participantInNames(names, participant) {
-  return names.some((name) => {
-    if (!name.startsWith("trace--")) return false;
-    const rest = name.slice("trace--".length);
-    // Matrix: `<participant>` is the whole remainder (artifact name).
-    if (rest === participant) return true;
-    // Dispatch: `<case>--<participant>.<role>.ndjson`.
-    const sep = rest.indexOf("--");
-    if (sep === -1) return false;
-    const afterCase = rest.slice(sep + 2);
-    const participantSegment = afterCase.split(".")[0];
-    return participantSegment === participant;
+export async function listExtractedFiles(runtime, dir) {
+  const entries = await runtime.fs.readdir(dir, {
+    recursive: true,
+    withFileTypes: true,
   });
+  return entries
+    .filter((e) => e.isFile() && !e.name.endsWith(".zip"))
+    .map((e) => path.relative(dir, path.join(e.parentPath ?? e.path, e.name)))
+    .sort();
 }
 
 /**

package/src/trace-identity.js

@@ -0,0 +1,132 @@
+/**
+ * Trace identity grammar — the single owner of case ids and lane filenames.
+ *
+ * Builds case ids (`<taskId>-r<runIndex>`), builds raw/lane filenames under
+ * the shared `trace--` convention, validates task ids, and parses names back
+ * into identity. Workdir allocation, task-family loading, the shared split
+ * module, and GitHub discovery all invoke this module — files agreeing by
+ * convention is the drifted-copies pattern this module retires.
+ */
+
+import { basename } from "node:path";
+
+/**
+ * Task ids must not contain "--" or start/end with "-": the "--" delimiter
+ * and the terminal "-r<digits>" suffix then parse unambiguously.
+ * @param {string} id
+ * @returns {boolean}
+ */
+export function isValidTaskId(id) {
+  if (typeof id !== "string" || id.length === 0) return false;
+  if (id.includes("--")) return false;
+  if (id.startsWith("-") || id.endsWith("-")) return false;
+  return true;
+}
+
+/**
+ * Build the grid-unique case id `<taskId>-r<runIndex>`. Shards partition one
+ * grid, so (task, runIndex) is already grid- and shard-unique.
+ * @param {string} taskId
+ * @param {number} runIndex
+ * @returns {string}
+ * @throws {Error} when `isValidTaskId(taskId)` is false or `runIndex` is not
+ *   a non-negative integer.
+ */
+export function buildCaseId(taskId, runIndex) {
+  if (!isValidTaskId(taskId)) {
+    throw new Error(
+      `invalid task id '${taskId}': task ids must not contain "--" or start/end with "-"`,
+    );
+  }
+  if (!Number.isInteger(runIndex) || runIndex < 0) {
+    throw new Error(`invalid run index '${runIndex}': must be an integer ≥ 0`);
+  }
+  return `${taskId}-r${runIndex}`;
+}
+
+/**
+ * Filename of the combined raw envelope trace: `trace--<caseId>.raw.ndjson`.
+ * @param {string} caseId
+ * @returns {string}
+ */
+export function rawTraceFilename(caseId) {
+  return `trace--${caseId}.raw.ndjson`;
+}
+
+/**
+ * Filename of a per-participant lane:
+ * `trace--<caseId>--<participant>.<role>.ndjson`.
+ * @param {string} caseId
+ * @param {string} participant
+ * @param {string} role
+ * @returns {string}
+ */
+export function laneFilename(caseId, participant, role) {
+  return `trace--${caseId}--${participant}.${role}.ndjson`;
+}
+
+/**
+ * Parse `trace--<case>--<participant>.<role>.ndjson` into `{caseName,
+ * participant}`. On no match, `caseName` is the basename minus its final
+ * `.ndjson` extension only and `participant` is null.
+ * @param {string} file
+ * @returns {{caseName: string, participant: string|null}}
+ */
+export function parseIdentity(file) {
+  const name = basename(file);
+  const match = name.match(/^trace--(.+?)--(.+?)\.[^.]+\.ndjson$/);
+  if (match) {
+    return { caseName: match[1], participant: match[2] };
+  }
+  return { caseName: name.replace(/\.ndjson$/, ""), participant: null };
+}
+
+/**
+ * Test whether a participant's trace lane is present in a list of names.
+ *
+ * Matches the two trace-naming shapes by *name* only (never by content):
+ *   - matrix artifact name: `trace--<participant>`
+ *   - dispatch member filename: `trace--<case>--<participant>.<role>.ndjson`
+ *
+ * The participant segment is delimited by `--` and ends at the next `--`, `.`,
+ * or end-of-string, so a substring like `release` does not match
+ * `release-engineer` and vice versa.
+ *
+ * Kept as a distinct shape from {@link parseIdentity} deliberately: this
+ * matcher also accepts bare artifact names with no extension, which the
+ * filename regex cannot.
+ *
+ * @param {string[]} names - Artifact names or extracted member filenames.
+ * @param {string} participant - Participant name to look for.
+ * @returns {boolean}
+ */
+export function participantInNames(names, participant) {
+  return names.some((name) => {
+    if (!name.startsWith("trace--")) return false;
+    const rest = name.slice("trace--".length);
+    // Matrix: `<participant>` is the whole remainder (artifact name).
+    if (rest === participant) return true;
+    // Dispatch: `<case>--<participant>.<role>.ndjson`.
+    const sep = rest.indexOf("--");
+    if (sep === -1) return false;
+    const afterCase = rest.slice(sep + 2);
+    const participantSegment = afterCase.split(".")[0];
+    return participantSegment === participant;
+  });
+}
+
+/**
+ * Keyed-lookup rule for one name: true when `key` equals the exact basename,
+ * the parsed case segment, or the parsed participant segment. Derives case
+ * and participant via {@link parseIdentity} and reuses the
+ * {@link participantInNames} single-name check — no second grammar.
+ * @param {string} name - A member basename or artifact name.
+ * @param {string} key - Exact filename, case id, or participant name.
+ * @returns {boolean}
+ */
+export function nameMatchesKey(name, key) {
+  if (name === key) return true;
+  const identity = parseIdentity(name);
+  if (identity.participant !== null && identity.caseName === key) return true;
+  return participantInNames([name], key);
+}

package/src/trace-multi.js

@@ -12,6 +12,8 @@
  */
 import { basename } from "node:path";
 
+import { parseIdentity } from "./trace-identity.js";
+
 /**
  * Load each file → `TraceQuery`. Run `query(tq)`. Tag each emitted record
  * with `source: <basename>` only when the caller supplies more than one file.
@@ -84,20 +86,3 @@ export function compareTwo(a, b, load) {
     bIdentity: parseIdentity(b),
   });
 }
-
-/**
- * Parse `trace--<case>--<participant>.<role>.ndjson` into `{caseName,
- * participant}`. On no match, `caseName` is the basename without its final
- * `.ndjson` extension. The function removes that one extension only.
- * `participant` is then null.
- * @param {string} file
- * @returns {{caseName: string, participant: string|null}}
- */
-export function parseIdentity(file) {
-  const name = basename(file);
-  const match = name.match(/^trace--(.+?)--(.+?)\.[^.]+\.ndjson$/);
-  if (match) {
-    return { caseName: match[1], participant: match[2] };
-  }
-  return { caseName: name.replace(/\.ndjson$/, ""), participant: null };
-}

package/src/trace-split.js

@@ -0,0 +1,103 @@
+/**
+ * Shared trace-split implementation — the single owner of source-to-role
+ * classification. Both the `gemba-trace split` command and the benchmark
+ * runner drive this module, so exactly one split policy exists (the same
+ * treatment the one-cost-path rule gives `sumTraceCost`).
+ */
+
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+
+import { laneFilename } from "./trace-identity.js";
+
+/** Valid source name pattern: lowercase letter, then lowercase alphanumeric or hyphen. */
+const VALID_SOURCE_NAME = /^[a-z][a-z0-9-]*$/;
+
+/**
+ * Sources whose name is itself a structural role; classified into the role
+ * they represent. `judge` is structural so the judge lane classifies under
+ * one rule — no current producer feeds judge-source envelopes through split
+ * (the judge is its own session), so kata split output is unchanged.
+ */
+const STRUCTURAL_ROLES = new Set([
+  "agent",
+  "supervisor",
+  "facilitator",
+  "judge",
+]);
+
+/**
+ * Split a combined `{source, seq, event}` NDJSON trace into per-source lane
+ * files named by `laneFilename(caseId, source, role)`.
+ *
+ * Classification: sources in the structural-role set ("agent", "supervisor",
+ * "facilitator", "judge") take their own name as role; any other valid source
+ * name classifies as role "agent" with the source as participant. Skips
+ * empty/malformed/non-envelope lines and orchestrator events; drops sources
+ * failing `/^[a-z][a-z0-9-]*$/`. Lane files carry unwrapped event JSON, one
+ * per line.
+ *
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime -
+ *   Ambient collaborators; streams via `runtime.fs`.
+ * @param {string} inputPath - Combined NDJSON trace to split.
+ * @param {object} opts
+ * @param {string} opts.caseId - Case identity embedded in lane filenames.
+ * @param {string} opts.outputDir - Directory the lane files are written to.
+ * @returns {Promise<string[]>} Paths written, resolved against `outputDir`
+ *   (absolute iff `outputDir` is absolute).
+ */
+export async function splitTrace(runtime, inputPath, { caseId, outputDir }) {
+  const fs = runtime.fs;
+  const rl = createInterface({
+    input: fs.createReadStream(inputPath),
+    crlfDelay: Infinity,
+  });
+  const streams = new Map();
+  const paths = [];
+  for await (const line of rl) {
+    const envelope = parseEnvelopeLine(line);
+    if (!envelope) continue;
+    if (envelope.source === "orchestrator") continue;
+    if (!VALID_SOURCE_NAME.test(envelope.source)) continue;
+
+    let stream = streams.get(envelope.source);
+    if (!stream) {
+      const role = STRUCTURAL_ROLES.has(envelope.source)
+        ? envelope.source
+        : "agent";
+      const outPath = join(
+        outputDir,
+        laneFilename(caseId, envelope.source, role),
+      );
+      stream = fs.createWriteStream(outPath);
+      streams.set(envelope.source, stream);
+      paths.push(outPath);
+    }
+    stream.write(JSON.stringify(envelope.event) + "\n");
+  }
+  await Promise.all(
+    [...streams.values()].map((s) => new Promise((r) => s.end(r))),
+  );
+  return paths;
+}
+
+/**
+ * Parse one NDJSON line into a `{source, seq, event}` envelope, or null when
+ * the line is blank, malformed, or not an envelope. Shared with the
+ * raw-summary reader so exactly one envelope-line parser exists; callers add
+ * their own source filters.
+ * @param {string} line
+ * @returns {{source: string, event: object}|null}
+ */
+export function parseEnvelopeLine(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+  let envelope;
+  try {
+    envelope = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+  if (!envelope.event || typeof envelope.source !== "string") return null;
+  return envelope;
+}

package/src/benchmark/trace-split.js

@@ -1,74 +0,0 @@
-/**
- * Split the combined supervisor trace for the benchmark runner. One pass
- * over the tagged NDJSON envelope stream separates agent events from
- * supervisor and orchestrator events. The same pass extracts the run summary.
- */
-
-import { createInterface } from "node:readline";
-
-/**
- * Split the combined supervisor trace into agent and supervisor files in a
- * single pass. The same pass extracts the turn count and the submission.
- * Agent-source events go to `agentPath`. Supervisor and orchestrator events
- * go to `supervisorPath`.
- *
- * This function deliberately does not sum cost. The caller derives it from
- * the same combined trace with `sumTraceCost`. One cost path then serves the
- * benchmark, callback, and `gemba-trace cost` consumers.
- * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
- * @param {string} combinedPath
- * @param {string} agentPath
- * @param {string} supervisorPath
- * @returns {Promise<{turns: number, submission: string}>}
- */
-// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
-export async function splitAndSummarize(
-  runtime,
-  combinedPath,
-  agentPath,
-  supervisorPath,
-) {
-  const fs = runtime.fs;
-  const agentStream = fs.createWriteStream(agentPath);
-  const supStream = fs.createWriteStream(supervisorPath);
-  const rl = createInterface({
-    input: fs.createReadStream(combinedPath),
-    crlfDelay: Infinity,
-  });
-  let turns = 0;
-  let submission = "";
-  for await (const line of rl) {
-    if (!line.trim()) continue;
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    const target = event.source === "agent" ? agentStream : supStream;
-    target.write(line + "\n");
-    const inner = event.event;
-    if (!inner) continue;
-    if (event.source === "agent" && inner.type === "assistant") {
-      const text = extractText(inner);
-      if (text) submission = text;
-    }
-    if (event.source === "orchestrator" && inner.type === "summary") {
-      turns = inner.turns ?? 0;
-    }
-  }
-  await Promise.all([
-    new Promise((r) => agentStream.end(r)),
-    new Promise((r) => supStream.end(r)),
-  ]);
-  return { turns, submission };
-}
-
-function extractText(inner) {
-  const content = inner.message?.content ?? inner.content;
-  if (!Array.isArray(content)) return null;
-  for (let i = content.length - 1; i >= 0; i--) {
-    if (content[i].type === "text" && content[i].text) return content[i].text;
-  }
-  return null;
-}