@forwardimpact/libharness 2.0.0 → 3.0.0

package/README.md

@@ -17,12 +17,12 @@ traces they produce, and edits skill files under controlled conditions.
 
 | CLI             | Purpose                                                                |
 | --------------- | ---------------------------------------------------------------------- |
-| `fit-harness`      | Run agents in `run`/`supervise`/`facilitate`/`discuss` subcommands.    |
-| `fit-trace`     | Download, query, and analyze NDJSON traces produced by `fit-harness`.     |
-| `fit-benchmark` | Run task families for N runs each and aggregate pass@k.                |
-| `fit-selfedit`  | Write stdin to `.claude/**` paths, gated by settings.json + branch.    |
+| `gemba-harness`      | Run agents in `run`/`supervise`/`facilitate`/`discuss` subcommands.    |
+| `gemba-trace`     | Download, query, and analyze NDJSON traces produced by `gemba-harness`.     |
+| `gemba-benchmark` | Run task families for N runs each and aggregate pass@k.                |
+| `gemba-selfedit`  | Write stdin to `.claude/**` paths, gated by settings.json + branch.    |
 
-`fit-harness`'s subcommands share one orchestration loop and one async tool
+`gemba-harness`'s subcommands share one orchestration loop and one async tool
 surface, below. The `judge` role is a profile passed to `supervise`.
 
 ## Modes
@@ -147,9 +147,9 @@ Each line is `{ "source": "<participant|orchestrator>", "seq": N, "event":
 {…} }`. `seq` is monotonic across the whole trace; `orchestrator` emits
 `session_start`, `agent_start`, `protocol_violation`, `lead_turn_limit`,
 and `summary`. `event` is the SDK event verbatim or the orchestrator
-payload. `fit-trace` consumes this format.
+payload. `gemba-trace` consumes this format.
 
-Redaction is on by default for `fit-harness run`/`supervise`/`facilitate`
+Redaction is on by default for `gemba-harness run`/`supervise`/`facilitate`
 and composes two layers:
 
 - **Env-var allowlist** — `ANTHROPIC_API_KEY`, `GH_TOKEN`, `GITHUB_TOKEN`
@@ -178,7 +178,7 @@ downloadable through retention.
 | `trace-collector.js` / `trace-query.js` / `trace-github.js` | Trace ingestion / querying / GitHub-attachment helpers.              |
 | `redaction.js`                                              | Env-var allowlist + credential-shape pattern redaction.              |
 
-## fit-selfedit
+## gemba-selfedit
 
 A narrow, audited bypass for sessions where `Edit`/`Write` (and bash
 writes) are blocked against paths the project's own allowlist permits.
@@ -186,7 +186,7 @@ Reads stdin, writes the target, exits 0 / 2 (safeguard violation) / 1
 (I/O error).
 
 ```sh
-echo "<content>" | bunx fit-selfedit <path>
+echo "<content>" | bunx gemba-selfedit <path>
 ```
 
 Two safeguards, checked in order:
@@ -221,7 +221,7 @@ lists the `Edit()` rules that were tried.
   — end-to-end workflow from dataset generation through evaluation to trace
   analysis, including multi-agent collaboration sessions.
 - [Analyze Traces](https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md)
-  — read the NDJSON traces produced by `fit-harness` with `fit-trace`.
+  — read the NDJSON traces produced by `gemba-harness` with `gemba-trace`.
 - [Agent Teams](https://www.forwardimpact.team/docs/products/agent-teams/index.md)
   — author the profiles consumed by `--agent-profile`, `--lead-profile`, and
   `--agent-profiles`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "2.0.0",
+  "version": "3.0.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",
@@ -40,20 +40,22 @@
   "main": "./src/index.js",
   "exports": {
     ".": "./src/index.js",
-    "./bin/fit-harness.js": "./bin/fit-harness.js",
-    "./bin/fit-trace.js": "./bin/fit-trace.js",
-    "./bin/fit-benchmark.js": "./bin/fit-benchmark.js",
-    "./bin/fit-selfedit.js": "./bin/fit-selfedit.js"
-  },
-  "bin": {
-    "fit-harness": "./bin/fit-harness.js",
-    "fit-trace": "./bin/fit-trace.js",
-    "fit-benchmark": "./bin/fit-benchmark.js",
-    "fit-selfedit": "./bin/fit-selfedit.js"
+    "./commands/output.js": "./src/commands/output.js",
+    "./commands/tee.js": "./src/commands/tee.js",
+    "./commands/run.js": "./src/commands/run.js",
+    "./commands/supervise.js": "./src/commands/supervise.js",
+    "./commands/facilitate.js": "./src/commands/facilitate.js",
+    "./commands/discuss.js": "./src/commands/discuss.js",
+    "./commands/callback.js": "./src/commands/callback.js",
+    "./commands/scan-logs.js": "./src/commands/scan-logs.js",
+    "./commands/trace.js": "./src/commands/trace.js",
+    "./commands/assert.js": "./src/commands/assert.js",
+    "./commands/by-discussion.js": "./src/commands/by-discussion.js",
+    "./commands/benchmark-definition.js": "./src/commands/benchmark-definition.js",
+    "./commands/selfedit.js": "./src/commands/selfedit.js"
   },
   "files": [
     "src/**/*.js",
-    "bin/**/*.js",
     "README.md"
   ],
   "scripts": {

package/src/agent-runner.js

@@ -1,7 +1,7 @@
 /**
  * AgentRunner — runs a single Claude Agent SDK session and emits raw
- * NDJSON events to an output stream. Building block for `fit-harness run`,
- * `fit-harness supervise`, `fit-harness facilitate`, and `fit-harness discuss`.
+ * NDJSON events to an output stream. Building block for `gemba-harness run`,
+ * `gemba-harness supervise`, `gemba-harness facilitate`, and `gemba-harness discuss`.
  *
  * Follows OO+DI: constructor injection, factory function, tests bypass factory.
  */
@@ -37,7 +37,7 @@ function modelDidWork(result) {
   return tokens > 0 || (cost ?? 0) > 0;
 }
 
-// fit-harness and kata-action run headless in CI/CD with no human to answer
+// gemba-harness and kata-action run headless in CI/CD with no human to answer
 // permission prompts. The SDK is always launched in bypass mode — not
 // overridable — so a future caller can't accidentally reduce permissions.
 const PERMISSION_MODE = "bypassPermissions";

package/src/benchmark/task-family.js

@@ -62,7 +62,7 @@ export async function loadTaskFamily(rootPathOrGitUrl, runtime) {
   let familyRevision;
   if (isGit) {
     const dir = await runtime.fs.mkdtemp(
-      join(tmpdir(runtime), "fit-benchmark-family-"),
+      join(tmpdir(runtime), "gemba-benchmark-family-"),
     );
     await gitClone(runtime, rootPathOrGitUrl, dir);
     rootPath = dir;

package/src/benchmark/trace-split.js

@@ -13,7 +13,7 @@ import { createInterface } from "node:readline";
  *
  * Cost is deliberately not summed here — the caller derives it from the same
  * combined trace via `sumTraceCost`, so there is one cost path across the
- * benchmark, callback, and `fit-trace cost` consumers.
+ * benchmark, callback, and `gemba-trace cost` consumers.
  * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
  * @param {string} combinedPath
  * @param {string} agentPath

package/src/claude-code-executable.js

@@ -9,7 +9,7 @@
  * <platform> not found".
  *
  * In a compiled binary we point the SDK at the standalone `claude` on PATH,
- * installed beside fit-harness by the bootstrap action's `fit-install.sh`.
+ * installed beside gemba-harness by the bootstrap action's `fit-install.sh`.
  * Running from source keeps `node_modules`, where the SDK resolves its own
  * version-matched binary, so there we return undefined and defer to the SDK.
  */

package/src/commands/benchmark-definition.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * `gemba-benchmark` CLI definition. Lives in `src/` so the bin stays an
  * execute-on-import entry point — launcher packages import the bin to run
  * it — while tests import the definition without running the CLI.
  */
@@ -13,7 +13,7 @@ import {
 } from "@forwardimpact/libutil/models";
 
 export const definition = {
-  name: "fit-benchmark",
+  name: "gemba-benchmark",
   description:
     "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
   commands: [
@@ -154,14 +154,14 @@ export const definition = {
     json: { type: "boolean", description: "Output help as JSON" },
   },
   examples: [
-    "fit-benchmark run --family=./families/coding",
-    "fit-benchmark run --family=./families/coding --task=todo-api --runs=1",
-    "fit-benchmark run --family=./families/coding --work-tracker=filesystem",
-    "fit-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
-    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
-    "fit-benchmark grade --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
-    "fit-benchmark report --format=text",
-    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+    "gemba-benchmark run --family=./families/coding",
+    "gemba-benchmark run --family=./families/coding --task=todo-api --runs=1",
+    "gemba-benchmark run --family=./families/coding --work-tracker=filesystem",
+    "gemba-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
+    `gemba-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "gemba-benchmark grade --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
+    "gemba-benchmark report --format=text",
+    "gemba-benchmark report --input=./runs/today --k=1,3,5 --format=text",
   ],
   documentation: [
     {

package/src/commands/benchmark-grade.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark grade` — run both check-row producers (the hidden test
+ * `gemba-benchmark grade` — run both check-row producers (the hidden test
  * suite and the invariants script) against a post-run workdir directory and
  * grade the merged rows with the same derivation the benchmark runner uses.
  * No agent and no judge run, so authors validate a task's grading material

package/src/commands/benchmark-report.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
+ * `gemba-benchmark report` — aggregate `results.jsonl` into pass@k via the
  * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
  * to render a markdown table. --detail=compact drops the per-task detail
  * sections so a sharded run's per-shard summary stays short (the merge job

package/src/commands/benchmark-run.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark run` — run every task in a family for N runs, stream each
+ * `gemba-benchmark run` — run every task in a family for N runs, stream each
  * ResultRecord to stdout (one JSON line per record), and append to the
  * canonical `<output>/results.jsonl` for the report subcommand.
  */

package/src/commands/by-discussion.js

@@ -72,7 +72,7 @@ export function findTracesByDiscussion(dir, discussionId, fsSync) {
 }
 
 /**
- * `fit-trace by-discussion <discussion-id> [trace-dir]` — list trace
+ * `gemba-trace by-discussion <discussion-id> [trace-dir]` — list trace
  * files whose meta header carries the given discussion_id, one per
  * line, ordered by first-event timestamp (file mtime ascending). The
  * result is usable with `xargs cat` for a chronological merge.

package/src/commands/facilitate.js

@@ -68,7 +68,7 @@ export function parseFacilitateOptions(values, runtime) {
 /**
  * Facilitate command — run a facilitated multi-agent session.
  *
- * Usage: fit-harness facilitate [options]
+ * Usage: gemba-harness facilitate [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/output.js

@@ -5,7 +5,7 @@ import { createTraceCollector } from "@forwardimpact/libharness";
  * Output command — process a complete NDJSON trace from stdin and write
  * formatted output to stdout.
  *
- * Usage: fit-harness output [--format=json|text] < trace.ndjson
+ * Usage: gemba-harness output [--format=json|text] < trace.ndjson
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: true}>}

package/src/commands/run.js

@@ -197,7 +197,7 @@ export async function wireRunSession({
 /**
  * Run command — execute a single agent via the Claude Agent SDK.
  *
- * Usage: fit-harness run [options]
+ * Usage: gemba-harness run [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/scan-logs.js

@@ -1,9 +1,9 @@
 /**
- * `fit-harness scan-logs` — scan a run's log archive for secret literals and
+ * `gemba-harness scan-logs` — scan a run's log archive for secret literals and
  * fail closed.
  *
  * A run-lifecycle concern (not an NDJSON trace, so it lives here rather than
- * in `fit-trace`): after a CI run that handled secrets, download or accept the
+ * in `gemba-trace`): after a CI run that handled secrets, download or accept the
  * run's own log archive and assert none of a supplied set of literals leaked
  * into it. Any hit exits non-zero; any download/extract failure also exits
  * non-zero — the gate must never silently disarm.

package/src/commands/selfedit.js

@@ -0,0 +1,124 @@
+/**
+ * Safeguard-and-write logic behind the `gemba-selfedit` bin: write content
+ * to a path that .claude/settings.json permits Edit on, while on a non-main
+ * git branch. See libraries/libharness/README.md § gemba-selfedit for the
+ * full rationale.
+ */
+
+import { resolve, relative, dirname } from "node:path";
+
+import { minimatch } from "minimatch";
+
+/** A safeguard violation — callers map it to exit code 2. */
+export class SelfeditError extends Error {
+  /** @param {string} message failure description */
+  constructor(message) {
+    super(message);
+    this.name = "SelfeditError";
+  }
+}
+
+/**
+ * Check every safeguard for a selfedit write, then perform it.
+ *
+ * Safeguards (checked in order):
+ * 1. The nearest .claude/settings.json must contain an Edit(<glob>) rule in
+ *    permissions.allow[] that resolves to the target path.
+ * 2. HEAD must not be detached and the current branch must not be 'main'.
+ * 3. The target's parent directory must exist.
+ *
+ * @param {string} targetArg target path as given on the command line
+ * @param {Buffer} content bytes to write
+ * @param {{ runtime: object }} deps runtime bag (fsSync, proc, subprocess,
+ *   finder); targetArg resolves against `runtime.proc.cwd()`
+ * @returns {{ bytes: number, relativeTarget: string, matchedPattern: string,
+ *   branch: string }} what was written and which rule allowed it
+ * @throws {SelfeditError} on any safeguard violation
+ */
+export function runSelfeditCommand(targetArg, content, { runtime }) {
+  const { fsSync, proc, subprocess, finder } = runtime;
+  const cwd = proc.cwd();
+  const absoluteTarget = resolve(cwd, targetArg);
+
+  // Safeguard 1: settings.json must grant Edit() on this path. Resolve the
+  // finder off the runtime bag rather than constructing a Finder here.
+  const settingsPath = finder.findUpward(
+    dirname(absoluteTarget),
+    ".claude/settings.json",
+    20,
+  );
+  if (!settingsPath) {
+    throw new SelfeditError(
+      `no .claude/settings.json found walking upward from ${dirname(absoluteTarget)}`,
+    );
+  }
+
+  const projectRoot = dirname(dirname(settingsPath));
+  const relativeTarget = relative(projectRoot, absoluteTarget);
+
+  let settings;
+  try {
+    settings = JSON.parse(fsSync.readFileSync(settingsPath, "utf8"));
+  } catch (err) {
+    throw new SelfeditError(`failed to parse ${settingsPath}: ${err.message}`);
+  }
+
+  const allowRules = settings?.permissions?.allow;
+  if (!Array.isArray(allowRules)) {
+    throw new SelfeditError(`${settingsPath} has no permissions.allow[] array`);
+  }
+
+  const editPatterns = allowRules
+    .filter((rule) => typeof rule === "string")
+    .map((rule) => rule.match(/^Edit\((.+)\)$/)?.[1])
+    .filter(Boolean);
+
+  if (editPatterns.length === 0) {
+    throw new SelfeditError(
+      `${settingsPath} has no Edit() rules in permissions.allow[]`,
+    );
+  }
+
+  const matchedPattern = editPatterns.find((pattern) =>
+    minimatch(relativeTarget, pattern, { dot: true }),
+  );
+  if (!matchedPattern) {
+    throw new SelfeditError(
+      `no Edit() rule in ${relative(projectRoot, settingsPath)} matches '${relativeTarget}' ` +
+        `(tried: ${editPatterns.map((p) => `Edit(${p})`).join(", ")})`,
+    );
+  }
+
+  // Safeguard 2: branch must not be main and HEAD must not be detached.
+  const git = subprocess.runSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
+    cwd,
+  });
+  if (git.exitCode !== 0) {
+    throw new SelfeditError(
+      "failed to read current git branch (not inside a git repository?)",
+    );
+  }
+  const branch = git.stdout.trim();
+
+  if (branch === "HEAD") {
+    throw new SelfeditError(
+      "HEAD is detached — refusing (check out a non-main branch first)",
+    );
+  }
+  if (branch === "main") {
+    throw new SelfeditError(
+      "refusing to write while on branch 'main' — switch to a feature branch",
+    );
+  }
+
+  const parent = dirname(absoluteTarget);
+  if (!fsSync.existsSync(parent)) {
+    throw new SelfeditError(
+      `parent directory '${relative(projectRoot, parent)}' does not exist`,
+    );
+  }
+
+  fsSync.writeFileSync(absoluteTarget, content);
+
+  return { bytes: content.length, relativeTarget, matchedPattern, branch };
+}

package/src/commands/supervise.js

@@ -27,7 +27,7 @@ export async function parseSuperviseOptions(values, runtime) {
   const tmpRoot = runtime.proc.env.TMPDIR ?? "/tmp";
   const agentCwd = resolve(
     values["agent-cwd"] ||
-      (await runtime.fs.mkdtemp(join(tmpRoot, "fit-harness-agent-"))),
+      (await runtime.fs.mkdtemp(join(tmpRoot, "gemba-harness-agent-"))),
   );
 
   return {
@@ -62,7 +62,7 @@ export async function parseSuperviseOptions(values, runtime) {
  * orchestration loop. The supervisor delegates work through Ask, sees
  * each reply on its next turn, and ends with Conclude.
  *
- * Usage: fit-harness supervise [options]
+ * Usage: gemba-harness supervise [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/tee.js

@@ -9,7 +9,7 @@ import { createTeeWriter } from "../tee-writer.js";
  * re-delimits each record with a newline so the TeeWriter's line splitter sees
  * the same framing the raw byte stream produced.
  *
- * Usage: fit-harness tee [output.ndjson] < trace.ndjson
+ * Usage: gemba-harness tee [output.ndjson] < trace.ndjson
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/trace.js

@@ -578,7 +578,7 @@ function parseBuckets(content) {
 
 /**
  * Compute total + per-source cost from raw file content. A structured JSON
- * trace (from `fit-trace download`) carries its total in `summary.totalCostUsd`
+ * trace (from `gemba-trace download`) carries its total in `summary.totalCostUsd`
  * but no per-source split; raw NDJSON is summed via `sumTraceCost`.
  * @param {string} content - Raw file content (structured JSON or NDJSON).
  * @returns {{totalCostUsd: number, bySource: Record<string, number>}}

package/src/cost.js

@@ -13,7 +13,7 @@
  *
  * This mirrors `TraceCollector.handleResult`, which accumulates the same
  * figure for its summary footer — kept as a standalone pure helper so the
- * benchmark runner, the callback command, and `fit-trace cost` share one
+ * benchmark runner, the callback command, and `gemba-trace cost` share one
  * implementation rather than each re-deriving it (and drifting).
  */
 

package/src/trace-collector.js

@@ -271,7 +271,7 @@ export class TraceCollector {
 
   /**
    * Render the accumulated turns as human-readable text — the same path the
-   * live `TeeWriter` stream uses, so `fit-harness output --format=text` over a
+   * live `TeeWriter` stream uses, so `gemba-harness output --format=text` over a
    * captured trace reproduces what the live workflow log showed.
    *
    * Source prefixes are emitted whenever at least one turn has a non-null

package/src/trace-multi.js

@@ -1,5 +1,5 @@
 /**
- * Multi-file orchestrator for cross-trace `fit-trace` verbs.
+ * Multi-file orchestrator for cross-trace `gemba-trace` verbs.
  *
  * Two functions centralise the load-tag-concat (`runOver`) and
  * aggregate-and-sort (`aggregate`) policies so every cross-trace verb shares

package/src/trace-render.js

@@ -1,5 +1,5 @@
 /**
- * Text renderers for `fit-trace` query output.
+ * Text renderers for `gemba-trace` query output.
  *
  * One named export per renderable verb. Each renderer accepts the query result
  * plus `{multi, signatures}` and returns a string. `multi` controls

package/bin/fit-benchmark.js

@@ -1,44 +0,0 @@
-#!/usr/bin/env node
-
-import "@forwardimpact/libpreflight/node22";
-
-import { createCli } from "@forwardimpact/libcli";
-import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
-import { createLogger } from "@forwardimpact/libtelemetry";
-
-import { definition } from "../src/commands/benchmark-definition.js";
-
-const runtime = createDefaultRuntime();
-const logger = createLogger("benchmark", runtime);
-
-async function main() {
-  const cli = createCli(definition, {
-    runtime,
-    packageJsonUrl: new URL("../package.json", import.meta.url),
-  });
-  const parsed = cli.parse(runtime.proc.argv.slice(2));
-  if (!parsed) return runtime.proc.exit(0);
-
-  const { positionals } = parsed;
-  if (positionals.length === 0) {
-    cli.usageError("no command specified");
-    return runtime.proc.exit(2);
-  }
-
-  const command = positionals[0];
-  if (!definition.commands.some((c) => c.name === command)) {
-    cli.usageError(`unknown command "${command}"`);
-    return runtime.proc.exit(2);
-  }
-
-  const result = await cli.dispatch(parsed, { deps: { runtime } });
-  const envelope = result ?? { ok: true };
-  if (!envelope.ok && envelope.error) cli.error(envelope.error);
-  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
-}
-
-main().catch((error) => {
-  logger.exception("main", error);
-  createCli(definition, { runtime }).error(error.message);
-  process.exit(1);
-});