@forwardimpact/libharness 3.0.1 → 3.1.0

package/README.md

@@ -214,17 +214,39 @@ also lists the `Edit()` rules it tried.
 
 ## Documentation
 
-- [Coordinate an Agent Team](https://www.forwardimpact.team/docs/libraries/coordinate-team/index.md)
+- [Coordinate an Agent Team](https://www.gemba.team/docs/coordinate-team/index.md)
   — run a lead and N participant agents in one async session (supervise /
   facilitate / discuss) with Ask/Answer/Announce and a single NDJSON trace.
-- [Run an Eval](https://www.forwardimpact.team/docs/libraries/prove-changes/run-eval/index.md)
+- [Run an Eval](https://www.gemba.team/docs/prove-changes/run-eval/index.md)
   — author a judge profile, run an eval locally, wire it into CI, and inspect
   the trace it produces.
-- [Prove Agent Changes](https://www.forwardimpact.team/docs/libraries/prove-changes/index.md)
+- [Prove Agent Changes](https://www.gemba.team/docs/prove-changes/index.md)
   — the end-to-end workflow from dataset generation through evaluation to
   trace analysis, with multi-agent collaboration sessions.
-- [Analyze Traces](https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md)
+- [Analyze Traces](https://www.gemba.team/docs/prove-changes/trace-analysis/index.md)
   — read the NDJSON traces produced by `gemba-harness` with `gemba-trace`.
 - [Agent Teams](https://www.forwardimpact.team/docs/products/agent-teams/index.md)
   — author the profiles consumed by `--agent-profile`, `--lead-profile`, and
   `--agent-profiles`.
+
+## Documentation home
+
+libharness is an import-only library. It declares no `bin`. The
+`gemba-harness`, `gemba-trace`, `gemba-benchmark`, and `gemba-selfedit`
+commands ship with the Gemba product, which imports these modules. Run them
+with `npx gemba-harness`, `npx gemba-trace`, or `npx gemba-benchmark`, or use
+the installed `gemba-*` binaries. `gemba-selfedit` publishes no bare launcher.
+Install `@forwardimpact/gemba` to get it.
+
+The package publishes as `@forwardimpact/libharness` on the Forward Impact npm
+scope. Install it with `npm install @forwardimpact/libharness`. Its task guides
+live on the Gemba site at <https://www.gemba.team/>. The Forward Impact library
+guide tree at <https://www.forwardimpact.team/docs/libraries/index.md> is not
+this library's guide home.
+
+**Decision (2026-08-26):** the split package scope and guide host are
+deliberate. libharness stays a Gear npm package, so `package.json .homepage`
+keeps <https://www.forwardimpact.team>. The agent-runtime guides moved to
+gemba.team with the rest of the Gemba product. The `## Documentation` list
+above carries the current URLs. Old `forwardimpact.team/docs/libraries/`
+addresses forward to gemba.team.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",

package/src/benchmark/hidden-tests.js

@@ -1,7 +1,7 @@
 /**
  * Hidden-test engine — runs a task's `tests/` overlay against the post-run
  * agent CWD. The engine stages each file at its mirrored path. It runs each
- * check with `node --test`. It converts the exit status into one check row.
+ * check with `bun test`. It converts the exit status into one check row.
  * It restores the tree, so the judge sees the workdir exactly as the agent
  * left it.
  *
@@ -69,7 +69,25 @@ async function runOneCheck(task, ctx, runtime, timeoutMs, check) {
 }
 
 /**
- * Spawn `node --test <staged path>` from the agent CWD under the hook env
+ * Reads a child pipe to a string. Returns what it read when the stream tears
+ * down mid-read, because a pipe that closes as the child exits is a race, not
+ * a check failure. The caller reads the exit code for the verdict.
+ *
+ * @param {import("node:stream").Readable} stream - Child stdout or stderr.
+ * @returns {Promise<string>} Everything read before the stream ended.
+ */
+async function drainQuietly(stream) {
+  let out = "";
+  try {
+    for await (const chunk of stream) out += chunk.toString();
+  } catch {
+    // The stream closed under us. Keep what arrived.
+  }
+  return out;
+}
+
+/**
+ * Spawn `bun test <staged path>` from the agent CWD under the hook env
  * and map the exit status onto one row. The clock timer SIGKILLs a child
  * that outlives the per-check budget. The row then fails with a timeout
  * message.
@@ -83,27 +101,40 @@ async function spawnCheck(task, ctx, runtime, timeoutMs, check) {
     hooksDir: task.paths.hooks,
     familyDir: ctx.familyDir,
   });
-  // An inherited test-runner context makes the child `node --test` report
-  // exit 0 even when its tests fail. A check that fails would then mint a row
-  // that passes whenever the harness itself runs under `node --test`.
-  delete env.NODE_TEST_CONTEXT;
-  const child = runtime.subprocess.spawn("node", ["--test", check.stagePath], {
-    cwd: ctx.cwd,
-    env,
-    stdio: ["ignore", "pipe", "pipe"],
-  });
+  // `bun test` sets no test-context variable that a nested run inherits, so
+  // a child reports its own exit status even when the harness itself runs
+  // under a test runner. That removes the `NODE_TEST_CONTEXT` scrub the
+  // `node --test` engine needed to stop a failing check minting a passing
+  // row. The "fractional score" grade test is the standing guard: it fails
+  // if a failing check ever reports success again.
+  //
+  // Pass the ABSOLUTE staged path. `bun test` reads its argument as a
+  // substring filter over discovered paths, not as one file, so the relative
+  // `app/test/x.test.js` also matches an agent-authored `sub/app/test/
+  // x.test.js` and folds that file's result into this check's row. An
+  // absolute path cannot be a substring of a deeper path, so it selects
+  // exactly the staged file. One `*.test.js` stays one check.
+  const child = runtime.subprocess.spawn(
+    "bun",
+    ["test", join(ctx.cwd, check.stagePath)],
+    {
+      cwd: ctx.cwd,
+      env,
+      stdio: ["ignore", "pipe", "pipe"],
+    },
+  );
   let timedOut = false;
   const timer = runtime.clock.setTimeout(() => {
     timedOut = true;
     child.kill("SIGKILL");
   }, timeoutMs);
-  const drainStdout = (async () => {
-    for await (const _chunk of child.stdout) {
-      // discard
-    }
-  })();
-  let stderr = "";
-  for await (const chunk of child.stderr) stderr += chunk.toString();
+  // Both pipes drain only so a chatty child never blocks on a full pipe.
+  // A child that exits while its pipe is still open makes the async iterator
+  // reject with ERR_STREAM_PREMATURE_CLOSE on some runtimes. That teardown
+  // race is not a check result, so it must not throw out of the check. The
+  // exit code below is the verdict.
+  const drainStdout = drainQuietly(child.stdout);
+  const stderr = await drainQuietly(child.stderr);
   await drainStdout;
   const exit = await child.exitCode;
   runtime.clock.clearTimeout(timer);

package/src/benchmark/judge.js

@@ -9,17 +9,19 @@
  *
  *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
  *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
- *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
+ *   {{AGENT_TRACE_PATH}}    — absolute path to the cell's agent lane,
+ *                             trace--<case>--agent.agent.ndjson (materialized
+ *                             before any session runs)
  *   {{GRADE_RESULT}}        — JSON grade object plus the merged check rows
  *   {{SKILL_SET_HASH}}      — SHA-256 from apm.lock.yaml
  *   {{TASK_ID}}             — task name (directory under tasks/)
  *   {{TASK_DIR}}            — path to the agent working directory
  *
- * The adapter reads the judge verdict directly from the orchestration
- * context's `concluded` flag. It does not parse the trace on the happy path.
- * `parseConcludeFromTrace` stays for offline analysis. It is also a fallback
- * when the runtime ctx is not available, for example when you re-grade a
- * historical run from its judge.ndjson file.
+ * The judge verdict is captured from the orchestration context's
+ * `concluded` flag directly — no trace parsing on the happy path.
+ * `parseConcludeFromTrace` is preserved for offline analysis and as a
+ * fallback when the runtime ctx isn't available (e.g. re-grading a
+ * historical run from its preserved judge lane file).
  */
 
 import { createJudge } from "../judge.js";

package/src/benchmark/raw-summary.js

@@ -0,0 +1,79 @@
+/**
+ * Raw-trace summary for the benchmark runner: one post-session read of the
+ * preserved raw combined trace derives cost, turns, and submission. Named as
+ * its own module so summarization never re-entangles with splitting — the
+ * coupling that caused the original split-policy divergence.
+ */
+
+import { sumTraceCost } from "../cost.js";
+import { parseEnvelopeLine } from "../trace-split.js";
+
+/**
+ * One read of the preserved raw combined trace:
+ *   cost       — `sumTraceCost` over the lines (the one cost path),
+ *   turns      — last orchestrator-source `summary` event's `turns`,
+ *   submission — last agent-source assistant text block.
+ *
+ * An empty (materialized-stub) raw file yields zeros and an empty
+ * submission; malformed and blank lines are tolerated.
+ *
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} rawTracePath
+ * @returns {Promise<{costUsd: number,
+ *   costBreakdown: {agent: number, supervisor: number},
+ *   turns: number, submission: string}>}
+ */
+export async function summarizeRawTrace(runtime, rawTracePath) {
+  const content = await runtime.fs.readFile(rawTracePath, "utf8");
+  const lines = content.split("\n");
+  const { totalCostUsd, bySource } = sumTraceCost(lines);
+  const { turns, submission } = deriveTurnsAndSubmission(lines);
+
+  return {
+    costUsd: totalCostUsd,
+    costBreakdown: {
+      agent: bySource.agent ?? 0,
+      supervisor: bySource.supervisor ?? 0,
+    },
+    turns,
+    submission,
+  };
+}
+
+/**
+ * One walk of the parsed envelope lines: the last orchestrator `summary`
+ * event's `turns` and the last agent assistant text block.
+ * @param {string[]} lines
+ * @returns {{turns: number, submission: string}}
+ */
+function deriveTurnsAndSubmission(lines) {
+  let turns = 0;
+  let submission = "";
+  for (const line of lines) {
+    const envelope = parseEnvelopeLine(line);
+    if (!envelope) continue;
+    const inner = envelope.event;
+    if (envelope.source === "agent" && inner.type === "assistant") {
+      const text = extractText(inner);
+      if (text) submission = text;
+    }
+    if (envelope.source === "orchestrator" && inner.type === "summary") {
+      turns = inner.turns ?? 0;
+    }
+  }
+  return { turns, submission };
+}
+
+/**
+ * Last text block of an assistant event's content, or null when none exists.
+ * @param {object} inner - Unwrapped event.
+ * @returns {string|null}
+ */
+function extractText(inner) {
+  const content = inner.message?.content ?? inner.content;
+  if (!Array.isArray(content)) return null;
+  for (let i = content.length - 1; i >= 0; i--) {
+    if (content[i].type === "text" && content[i].text) return content[i].text;
+  }
+  return null;
+}

package/src/benchmark/result.js

@@ -104,9 +104,15 @@ const HAPPY_RECORD = z.object({
   score: z.number().min(0).max(1).optional(),
   submission: z.string(),
   judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
+  // Trace paths are relative to the run output directory — valid on the
+  // runner and inside a downloaded trace artifact alike. Raw and
+  // agent/supervisor lanes are materialized at workdir allocation, so they
+  // are present on every executed cell; the judge lane exists only on
+  // judged cells.
+  rawTracePath: z.string(),
   agentTracePath: z.string(),
   supervisorTracePath: z.string(),
-  judgeTracePath: z.string(),
+  judgeTracePath: z.string().optional(),
   agentError: AGENT_ERROR_SHAPE.optional(),
   preflightError: z.undefined().optional(),
 });
@@ -115,12 +121,12 @@ const PREFLIGHT_RECORD = z.object({
   ...COMMON_FIELDS,
   costUsd: z.literal(0),
   preflightError: PREFLIGHT_ERROR_SHAPE,
-  // The runner allocates the trace paths in WorkdirManager.start, even on
-  // preflight failure. The record then stays uniform across branches, and
-  // downstream consumers can reference the paths without conditional fields.
-  agentTracePath: z.string(),
-  supervisorTracePath: z.string(),
-  judgeTracePath: z.string(),
+  // No trace-path fields: a preflight-failure record references only traces
+  // a session produced, even though the materialized stubs exist on disk.
+  rawTracePath: z.undefined().optional(),
+  agentTracePath: z.undefined().optional(),
+  supervisorTracePath: z.undefined().optional(),
+  judgeTracePath: z.undefined().optional(),
   invariants: z.undefined().optional(),
   grade: z.undefined().optional(),
   hiddenTests: z.undefined().optional(),

package/src/benchmark/runner.js

@@ -19,11 +19,11 @@
  * ledger. The iterator mirrors the same stream to CLI stdout.
  */
 
-import { join, resolve as resolvePath } from "node:path";
+import { join, relative, resolve as resolvePath } from "node:path";
 
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
-import { sumTraceCost } from "../cost.js";
 import { createSupervisor } from "../supervisor.js";
+import { splitTrace } from "../trace-split.js";
 import { installApm as defaultInstallApm } from "./apm-installer.js";
 import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
@@ -31,8 +31,8 @@ import { validateResultRecord } from "./result.js";
 import { runInvariants } from "./invariants.js";
 import { runHiddenTests } from "./hidden-tests.js";
 import { runProducersAndGrade } from "./grade.js";
+import { summarizeRawTrace } from "./raw-summary.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
-import { splitAndSummarize } from "./trace-split.js";
 import { createWorkdirManager } from "./workdir.js";
 import { CellScheduler } from "./scheduler.js";
 
@@ -77,9 +77,10 @@ export class BenchmarkRunner {
    *   to `AGENT_WATCHDOG_MS`. A test injects its own to force a stall in-test.
    * @param {number} [opts.termGraceMs] - SIGTERM→SIGKILL grace (ms) for the per-task process group.
    * @param {Function} [opts.runAgent] - Test seam: replaces the agent-under-test
-   *   session. Must return `{costUsd, turns, submission, agentError?}` and
-   *   write a valid NDJSON trace to `workdir.agentTracePath`. Default uses
-   *   `createAgentRunner` with the harness `BASE_TOOLS` allowlist. Internal
+   *   session. Must run the session, stream `{source, seq, event}` envelopes
+   *   to `workdir.rawTracePath`, and return `{agentError?}` — cost, turns,
+   *   and submission are always derived from the raw file by the shared
+   *   split/summary pipeline, so the seam exercises the real path. Internal
    *   testing only. It is not part of the public API.
    * @param {import("@forwardimpact/libutil/runtime").Runtime} opts.runtime -
    *   The host injects these ambient collaborators (`fs`, `subprocess`,
@@ -256,12 +257,12 @@ export class BenchmarkRunner {
         t0,
       });
     } catch (e) {
-      // `wm.start()` (port acquire + workdir/env seed) is the one throw site
-      // that `#executeCell` does not catch. Turn it into the runner's own
-      // fallback record so `#runOne` never rejects. The scheduler's
-      // one-record-per-cell contract depends on that. `report` skips the
-      // fallback because it fails the schema, the same as any other
-      // runner-side schema failure.
+      // Catches the throw sites `#executeCell` does not: `wm.start()` (port
+      // acquire + workdir/env seed) and the shared split/summary pipeline.
+      // Turn either into the runner's own fallback record so `#runOne` never
+      // rejects. The scheduler's one-record-per-cell contract depends on
+      // that. `report` skips the fallback because it fails the schema, the
+      // same as any other runner-side schema failure.
       return {
         taskId: task.id,
         runIndex,
@@ -301,8 +302,8 @@ export class BenchmarkRunner {
     }
     {
       const agentRun = await this.#runAgentSafe(task, workdir);
-      const { costUsd, turns, submission, agentError } = agentRun;
-      const breakdown = agentRun.costBreakdown ?? { agent: 0, supervisor: 0 };
+      const { costUsd, costBreakdown, turns, submission, agentError } =
+        agentRun;
       const graded = await this.#gradeCell(family, task, workdir);
       const { invariants, hiddenRows, engineError, rows, grade } = graded;
       const { judgeVerdict, judgeCost } = await this.#judgeCell({
@@ -319,6 +320,7 @@ export class BenchmarkRunner {
       // or a judge that fails zeroes the effective score. Full marks does not
       // zero it. A fractional score with verdict fail is the point.
       const scoreValid = graded.healthy && grade.gatesPass && judgePass;
+      const tracePaths = await this.#traceRecordPaths(workdir);
       const record = {
         taskId: task.id,
         runIndex,
@@ -337,15 +339,9 @@ export class BenchmarkRunner {
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd: costUsd + judgeCost,
-        costBreakdown: {
-          agent: breakdown.agent ?? 0,
-          supervisor: breakdown.supervisor ?? 0,
-          judge: judgeCost,
-        },
+        costBreakdown: { ...costBreakdown, judge: judgeCost },
         turns,
-        agentTracePath: workdir.agentTracePath,
-        supervisorTracePath: workdir.supervisorTracePath,
-        judgeTracePath: workdir.judgeTracePath,
+        ...tracePaths,
         profiles: {
           agent: this.profiles.agent,
           supervisor: null,
@@ -424,39 +420,42 @@ export class BenchmarkRunner {
   }
 
   /**
-   * Dispatch to either the injected hook or the default `#runAgent`. Either
-   * path can throw. Catch the error here so it becomes an `agentError` on the
-   * record (spec criterion 1: records on agent failure). The iterator then
-   * does not abort.
+   * Dispatch to either the injected hook or the default `#runAgent`, then run
+   * the shared pipeline once: split the preserved raw trace into lanes and
+   * derive cost/turns/submission from the same file. Either session path can
+   * throw; catch here so a session error becomes an `agentError` on the
+   * record rather than aborting the whole iterator — the pipeline still runs,
+   * so failed cells keep coherent (possibly empty) lanes and zeroed totals.
+   * A pipeline throw itself (the raw file is materialized at allocation, so
+   * only an fs-level fault) propagates to `#runOne`'s fallback record.
    */
   async #runAgentSafe(task, workdir) {
+    let agentError = null;
     try {
-      if (this._runAgentHook) {
-        const r = await this._runAgentHook(task, workdir, this);
-        return { agentError: null, ...r };
-      }
-      return await this.#runAgent(task, workdir);
+      const r = this._runAgentHook
+        ? await this._runAgentHook(task, workdir, this)
+        : await this.#runAgent(task, workdir);
+      agentError = r?.agentError ?? null;
     } catch (e) {
-      return {
-        costUsd: 0,
-        costBreakdown: { agent: 0, supervisor: 0 },
-        turns: 0,
-        submission: "",
-        agentError: { message: e.message ?? String(e), aborted: false },
-      };
+      agentError = { message: e.message ?? String(e), aborted: false };
     }
+    await splitTrace(this.runtime, workdir.rawTracePath, {
+      caseId: workdir.caseId,
+      outputDir: workdir.runDir,
+    });
+    const summary = await summarizeRawTrace(this.runtime, workdir.rawTracePath);
+    return { ...summary, agentError };
   }
 
   /**
-   * Run the agent-under-test under a Supervisor. The supervisor writes
-   * a combined tagged NDJSON trace. After the session, this method splits
-   * the trace into agent.ndjson and supervisor.ndjson. It also extracts
-   * cost/turns/submission.
+   * Run the agent-under-test under a Supervisor. The supervisor streams the
+   * combined tagged NDJSON envelope trace to `workdir.rawTracePath`, which is
+   * preserved for the life of the run output; `#runAgentSafe` splits it into
+   * the convention-named lanes and summarizes it afterwards.
    */
   async #runAgent(task, workdir) {
     const fs = this.runtime.fs;
-    const combinedPath = join(workdir.runDir, ".combined.ndjson");
-    const combinedStream = fs.createWriteStream(combinedPath);
+    const combinedStream = fs.createWriteStream(workdir.rawTracePath);
     const supervisorInstructions = task.paths.supervisor
       ? await fs.readFile(task.paths.supervisor, "utf8").catch(() => null)
       : null;
@@ -510,27 +509,31 @@ export class BenchmarkRunner {
       this.runtime.clock.clearTimeout(watchdog);
       await new Promise((r) => combinedStream.end(r));
     }
-    const summary = await splitAndSummarize(
-      this.runtime,
-      combinedPath,
-      workdir.agentTracePath,
-      workdir.supervisorTracePath,
-    );
-    // `sumTraceCost` sums the cost across every participant's result events
-    // from the one combined trace, and attributes it per source. Read the
-    // trace before you unlink it.
-    const combined = await fs.readFile(combinedPath, "utf8");
-    const { totalCostUsd, bySource } = sumTraceCost(combined.split("\n"));
-    await fs.unlink(combinedPath).catch(() => {});
-    return {
-      ...summary,
-      costUsd: totalCostUsd,
-      costBreakdown: {
-        agent: bySource.agent ?? 0,
-        supervisor: bySource.supervisor ?? 0,
-      },
-      agentError,
+    return { agentError };
+  }
+
+  /**
+   * Run-output-relative trace-path record fields, each present only when its
+   * file exists: raw and agent/supervisor lanes are materialized at workdir
+   * allocation (present on every executed cell); the judge lane exists only
+   * on judged cells. Relative paths stay valid inside a downloaded artifact.
+   */
+  async #traceRecordPaths(workdir) {
+    const fields = {
+      rawTracePath: workdir.rawTracePath,
+      agentTracePath: workdir.agentTracePath,
+      supervisorTracePath: workdir.supervisorTracePath,
+      judgeTracePath: workdir.judgeTracePath,
     };
+    const out = {};
+    for (const [field, absPath] of Object.entries(fields)) {
+      const exists = await this.runtime.fs
+        .access(absPath)
+        .then(() => true)
+        .catch(() => false);
+      if (exists) out[field] = relative(this.output, absPath);
+    }
+    return out;
   }
 
   async #buildJudgeContext(task, workdir, skillSetHash) {
@@ -579,9 +582,9 @@ export class BenchmarkRunner {
       skillSetHash,
       familyRevision,
       durationMs,
-      agentTracePath: workdir.agentTracePath,
-      supervisorTracePath: workdir.supervisorTracePath,
-      judgeTracePath: workdir.judgeTracePath,
+      // No trace-path fields: even though the materialized stubs exist on
+      // disk, a preflight-failure record references only traces a session
+      // produced.
     };
   }
 

package/src/benchmark/task-family.js

@@ -35,6 +35,8 @@
 import { createHash } from "node:crypto";
 import { join, posix, relative, resolve, sep } from "node:path";
 
+import { isValidTaskId } from "../trace-identity.js";
+
 const GIT_URL_RE = /^(git@|https?:\/\/|ssh:\/\/|git:\/\/)/;
 const SKIP_DIRS = new Set([".git", "node_modules"]);
 // POSIX `X_OK` (execute permission). Node's fs honours the numeric mode, so we
@@ -129,6 +131,13 @@ async function discoverTasks(runtime, rootPath) {
 }
 
 async function loadTask(fs, taskDir, id) {
+  // The rule itself lives in the identity module; the loader only invokes
+  // the predicate so an unbuildable case id fails before any agent spend.
+  if (!isValidTaskId(id)) {
+    throw new Error(
+      `invalid task id '${id}': task directory names must not contain "--" or start/end with "-"`,
+    );
+  }
   const supervisorPath = join(taskDir, "supervisor.task.md");
   const judgePath = join(taskDir, "judge.task.md");
   const preflightPath = join(taskDir, "hooks", "preflight.sh");

package/src/benchmark/workdir.js

@@ -16,6 +16,11 @@ import { createServer } from "node:net";
 import { connect } from "node:net";
 import { join } from "node:path";
 
+import {
+  buildCaseId,
+  laneFilename,
+  rawTraceFilename,
+} from "../trace-identity.js";
 import { loadEnv } from "./env-loader.js";
 import { buildHookEnv } from "./hook-env.js";
 
@@ -28,6 +33,8 @@ const DEFAULT_TERM_GRACE_MS = 5_000;
  * @property {number} port - Allocated TCP port for the agent.
  * @property {number} pgid - Process-group id captured from the preflight child.
  * @property {*} scaffold - Reserved per design § Components. v1 sets null.
+ * @property {string} caseId - Grid-unique case identity `<taskId>-r<runIndex>`.
+ * @property {string} rawTracePath - Combined raw envelope trace (kept).
  * @property {string} agentTracePath
  * @property {string} supervisorTracePath
  * @property {string} judgeTracePath
@@ -72,8 +79,7 @@ export class WorkdirManager {
    */
   async start(task, runIndex) {
     const fs = this.runtime.fs;
-    const slug = task.id.replace("/", "__");
-    const runDir = join(this.runOutputDir, "runs", slug, String(runIndex));
+    const runDir = join(this.runOutputDir, "runs", task.id, String(runIndex));
     const cwd = join(runDir, "cwd");
     await fs.mkdir(cwd, { recursive: true });
 
@@ -124,11 +130,25 @@ export class WorkdirManager {
     const envNames =
       envDirs.length > 0 ? await loadEnv(envDirs, cwd, this.runtime) : [];
 
-    const port = await this.ports.acquire();
-    const agentTracePath = join(runDir, "agent.ndjson");
-    const supervisorTracePath = join(runDir, "supervisor.ndjson");
-    const judgeTracePath = join(runDir, "judge.ndjson");
+    // Allocate identity and trace paths before acquiring the port so an
+    // invalid task id cannot leak a port reservation.
+    const caseId = buildCaseId(task.id, runIndex);
+    const rawTracePath = join(runDir, rawTraceFilename(caseId));
+    const agentTracePath = join(runDir, laneFilename(caseId, "agent", "agent"));
+    const supervisorTracePath = join(
+      runDir,
+      laneFilename(caseId, "supervisor", "supervisor"),
+    );
+    const judgeTracePath = join(runDir, laneFilename(caseId, "judge", "judge"));
+    // Materialize the raw and agent/supervisor lanes empty at allocation so
+    // every path that reaches the judge — including a pre-session agent
+    // failure — finds them on disk. The judge lane is written by the judge
+    // session itself and exists only on judged cells.
+    for (const p of [rawTracePath, agentTracePath, supervisorTracePath]) {
+      await fs.writeFile(p, "");
+    }
 
+    const port = await this.ports.acquire();
     const preflight = task.paths.preflight
       ? await runPreflight(this.runtime, task.paths.preflight, cwd, port, {
           taskId: task.id,
@@ -144,6 +164,8 @@ export class WorkdirManager {
       port,
       pgid: preflight.pgid,
       scaffold: null,
+      caseId,
+      rawTracePath,
       agentTracePath,
       supervisorTracePath,
       judgeTracePath,

package/src/commands/benchmark-definition.js

@@ -166,13 +166,13 @@ export const definition = {
   documentation: [
     {
       title: "Run a Benchmark",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      url: "https://www.gemba.team/docs/prove-changes/run-benchmark/index.md",
       description:
         "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
     },
     {
       title: "Automate with GitHub Actions",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      url: "https://www.gemba.team/docs/prove-changes/run-benchmark/ci-workflow/index.md",
       description:
         "Run benchmarks in CI with the forwardimpact/gemba-benchmark action.",
     },