@forwardimpact/libharness 1.4.0 → 3.0.0

package/src/commands/benchmark-definition.js

@@ -1,11 +1,11 @@
 /**
- * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * `gemba-benchmark` CLI definition. Lives in `src/` so the bin stays an
  * execute-on-import entry point — launcher packages import the bin to run
  * it — while tests import the definition without running the CLI.
  */
 
 import { runBenchmarkRunCommand } from "./benchmark-run.js";
-import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkGradeCommand } from "./benchmark-grade.js";
 import { runBenchmarkReportCommand } from "./benchmark-report.js";
 import {
   BENCHMARK_AGENT_MODEL,
@@ -13,7 +13,7 @@ import {
 } from "@forwardimpact/libutil/models";
 
 export const definition = {
-  name: "fit-benchmark",
+  name: "gemba-benchmark",
   description:
     "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
   commands: [
@@ -95,11 +95,11 @@ export const definition = {
       },
     },
     {
-      name: "invariants",
+      name: "grade",
       args: [],
-      handler: runBenchmarkInvariantsCommand,
+      handler: runBenchmarkGradeCommand,
       description:
-        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+        "Grade a single task against a post-run workdir without invoking an agent: run the hidden test suite and the invariants script, then derive the verdict from the check rows (the exit mirrors it).",
       options: {
         family: {
           type: "string",
@@ -112,7 +112,7 @@ export const definition = {
         "run-dir": {
           type: "string",
           description:
-            "Post-run directory whose cwd/ subdir is the agent CWD; invariants run against that cwd — the path hooks receive as $AGENT_CWD",
+            "Post-run directory whose cwd/ subdir is the agent CWD; both producers run against that cwd — the path hooks receive as $AGENT_CWD",
         },
         output: {
           type: "string",
@@ -154,14 +154,14 @@ export const definition = {
     json: { type: "boolean", description: "Output help as JSON" },
   },
   examples: [
-    "fit-benchmark run --family=./families/coding",
-    "fit-benchmark run --family=./families/coding --task=todo-api --runs=1",
-    "fit-benchmark run --family=./families/coding --work-tracker=filesystem",
-    "fit-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
-    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
-    "fit-benchmark invariants --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
-    "fit-benchmark report --format=text",
-    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+    "gemba-benchmark run --family=./families/coding",
+    "gemba-benchmark run --family=./families/coding --task=todo-api --runs=1",
+    "gemba-benchmark run --family=./families/coding --work-tracker=filesystem",
+    "gemba-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
+    `gemba-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "gemba-benchmark grade --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
+    "gemba-benchmark report --format=text",
+    "gemba-benchmark report --input=./runs/today --k=1,3,5 --format=text",
   ],
   documentation: [
     {

package/src/commands/benchmark-grade.js

@@ -0,0 +1,82 @@
+/**
+ * `gemba-benchmark grade` — run both check-row producers (the hidden test
+ * suite and the invariants script) against a post-run workdir directory and
+ * grade the merged rows with the same derivation the benchmark runner uses.
+ * No agent and no judge run, so authors validate a task's grading material
+ * against fixtures without paying for agent sessions; the process exit
+ * mirrors the graded verdict.
+ */
+
+import { join, resolve } from "node:path";
+
+import { validateGradeRecord } from "../benchmark/result.js";
+import { runInvariants } from "../benchmark/invariants.js";
+import { runHiddenTests } from "../benchmark/hidden-tests.js";
+import { runProducersAndGrade } from "../benchmark/grade.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+import { probeFreePort } from "../benchmark/workdir.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkGradeCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const familyInput = values.family;
+  if (!familyInput)
+    return { ok: false, code: 1, error: "--family is required" };
+  const taskId = values.task;
+  if (!taskId) return { ok: false, code: 1, error: "--task is required" };
+  const runDirArg = values["run-dir"];
+  if (!runDirArg) return { ok: false, code: 1, error: "--run-dir is required" };
+
+  const family = await loadTaskFamily(familyInput, runtime);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task)
+    return { ok: false, code: 1, error: `task not found in family: ${taskId}` };
+
+  const runDir = resolve(runDirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await probeFreePort();
+  const cellCtx = { cwd, port, runDir, familyDir: family.rootPath };
+
+  const { invariants, hiddenRows, engineError, healthy, grade } =
+    await runProducersAndGrade(task, cellCtx, runtime, {
+      runInvariants,
+      runHiddenTests,
+    });
+  // Same effective-score rule as the runner, minus the judge (none runs
+  // here): an unhealthy grader or a failing gate zeroes the score, so a
+  // crashed hook can never mint marks from the rows it emitted before dying.
+  // Unlike a runner record — where `grade.score` stays the raw fraction and
+  // the zeroing lands on the top-level `score` — this record has no second
+  // field, so `grade.score` carries the effective value here.
+  if (grade.score !== undefined && !(healthy && grade.gatesPass)) {
+    grade.score = 0;
+  }
+  const record = {
+    taskId: task.id,
+    grade,
+    invariants,
+    ...(task.tests && {
+      hiddenTests: {
+        details: hiddenRows,
+        ...(engineError && { error: engineError.message }),
+      },
+    }),
+    // Mirrors the script for diagnosis; the graded verdict drives the exit.
+    exitCode: invariants.exitCode,
+  };
+  validateGradeRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    runtime.fsSync.writeFileSync(resolve(values.output), line);
+  } else {
+    runtime.proc.stdout.write(line);
+  }
+  return grade.verdict === "pass"
+    ? { ok: true }
+    : { ok: false, code: 1, error: "" };
+}

package/src/commands/benchmark-report.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
+ * `gemba-benchmark report` — aggregate `results.jsonl` into pass@k via the
  * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
  * to render a markdown table. --detail=compact drops the per-task detail
  * sections so a sharded run's per-shard summary stays short (the merge job

package/src/commands/benchmark-run.js

@@ -1,5 +1,5 @@
 /**
- * `fit-benchmark run` — run every task in a family for N runs, stream each
+ * `gemba-benchmark run` — run every task in a family for N runs, stream each
  * ResultRecord to stdout (one JSON line per record), and append to the
  * canonical `<output>/results.jsonl` for the report subcommand.
  */

package/src/commands/by-discussion.js

@@ -72,7 +72,7 @@ export function findTracesByDiscussion(dir, discussionId, fsSync) {
 }
 
 /**
- * `fit-trace by-discussion <discussion-id> [trace-dir]` — list trace
+ * `gemba-trace by-discussion <discussion-id> [trace-dir]` — list trace
  * files whose meta header carries the given discussion_id, one per
  * line, ordered by first-event timestamp (file mtime ascending). The
  * result is usable with `xargs cat` for a chronological merge.

package/src/commands/facilitate.js

@@ -68,7 +68,7 @@ export function parseFacilitateOptions(values, runtime) {
 /**
  * Facilitate command — run a facilitated multi-agent session.
  *
- * Usage: fit-harness facilitate [options]
+ * Usage: gemba-harness facilitate [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/output.js

@@ -5,7 +5,7 @@ import { createTraceCollector } from "@forwardimpact/libharness";
  * Output command — process a complete NDJSON trace from stdin and write
  * formatted output to stdout.
  *
- * Usage: fit-harness output [--format=json|text] < trace.ndjson
+ * Usage: gemba-harness output [--format=json|text] < trace.ndjson
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: true}>}

package/src/commands/run.js

@@ -197,7 +197,7 @@ export async function wireRunSession({
 /**
  * Run command — execute a single agent via the Claude Agent SDK.
  *
- * Usage: fit-harness run [options]
+ * Usage: gemba-harness run [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/scan-logs.js

@@ -1,9 +1,9 @@
 /**
- * `fit-harness scan-logs` — scan a run's log archive for secret literals and
+ * `gemba-harness scan-logs` — scan a run's log archive for secret literals and
  * fail closed.
  *
  * A run-lifecycle concern (not an NDJSON trace, so it lives here rather than
- * in `fit-trace`): after a CI run that handled secrets, download or accept the
+ * in `gemba-trace`): after a CI run that handled secrets, download or accept the
  * run's own log archive and assert none of a supplied set of literals leaked
  * into it. Any hit exits non-zero; any download/extract failure also exits
  * non-zero — the gate must never silently disarm.

package/src/commands/selfedit.js

@@ -0,0 +1,124 @@
+/**
+ * Safeguard-and-write logic behind the `gemba-selfedit` bin: write content
+ * to a path that .claude/settings.json permits Edit on, while on a non-main
+ * git branch. See libraries/libharness/README.md § gemba-selfedit for the
+ * full rationale.
+ */
+
+import { resolve, relative, dirname } from "node:path";
+
+import { minimatch } from "minimatch";
+
+/** A safeguard violation — callers map it to exit code 2. */
+export class SelfeditError extends Error {
+  /** @param {string} message failure description */
+  constructor(message) {
+    super(message);
+    this.name = "SelfeditError";
+  }
+}
+
+/**
+ * Check every safeguard for a selfedit write, then perform it.
+ *
+ * Safeguards (checked in order):
+ * 1. The nearest .claude/settings.json must contain an Edit(<glob>) rule in
+ *    permissions.allow[] that resolves to the target path.
+ * 2. HEAD must not be detached and the current branch must not be 'main'.
+ * 3. The target's parent directory must exist.
+ *
+ * @param {string} targetArg target path as given on the command line
+ * @param {Buffer} content bytes to write
+ * @param {{ runtime: object }} deps runtime bag (fsSync, proc, subprocess,
+ *   finder); targetArg resolves against `runtime.proc.cwd()`
+ * @returns {{ bytes: number, relativeTarget: string, matchedPattern: string,
+ *   branch: string }} what was written and which rule allowed it
+ * @throws {SelfeditError} on any safeguard violation
+ */
+export function runSelfeditCommand(targetArg, content, { runtime }) {
+  const { fsSync, proc, subprocess, finder } = runtime;
+  const cwd = proc.cwd();
+  const absoluteTarget = resolve(cwd, targetArg);
+
+  // Safeguard 1: settings.json must grant Edit() on this path. Resolve the
+  // finder off the runtime bag rather than constructing a Finder here.
+  const settingsPath = finder.findUpward(
+    dirname(absoluteTarget),
+    ".claude/settings.json",
+    20,
+  );
+  if (!settingsPath) {
+    throw new SelfeditError(
+      `no .claude/settings.json found walking upward from ${dirname(absoluteTarget)}`,
+    );
+  }
+
+  const projectRoot = dirname(dirname(settingsPath));
+  const relativeTarget = relative(projectRoot, absoluteTarget);
+
+  let settings;
+  try {
+    settings = JSON.parse(fsSync.readFileSync(settingsPath, "utf8"));
+  } catch (err) {
+    throw new SelfeditError(`failed to parse ${settingsPath}: ${err.message}`);
+  }
+
+  const allowRules = settings?.permissions?.allow;
+  if (!Array.isArray(allowRules)) {
+    throw new SelfeditError(`${settingsPath} has no permissions.allow[] array`);
+  }
+
+  const editPatterns = allowRules
+    .filter((rule) => typeof rule === "string")
+    .map((rule) => rule.match(/^Edit\((.+)\)$/)?.[1])
+    .filter(Boolean);
+
+  if (editPatterns.length === 0) {
+    throw new SelfeditError(
+      `${settingsPath} has no Edit() rules in permissions.allow[]`,
+    );
+  }
+
+  const matchedPattern = editPatterns.find((pattern) =>
+    minimatch(relativeTarget, pattern, { dot: true }),
+  );
+  if (!matchedPattern) {
+    throw new SelfeditError(
+      `no Edit() rule in ${relative(projectRoot, settingsPath)} matches '${relativeTarget}' ` +
+        `(tried: ${editPatterns.map((p) => `Edit(${p})`).join(", ")})`,
+    );
+  }
+
+  // Safeguard 2: branch must not be main and HEAD must not be detached.
+  const git = subprocess.runSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
+    cwd,
+  });
+  if (git.exitCode !== 0) {
+    throw new SelfeditError(
+      "failed to read current git branch (not inside a git repository?)",
+    );
+  }
+  const branch = git.stdout.trim();
+
+  if (branch === "HEAD") {
+    throw new SelfeditError(
+      "HEAD is detached — refusing (check out a non-main branch first)",
+    );
+  }
+  if (branch === "main") {
+    throw new SelfeditError(
+      "refusing to write while on branch 'main' — switch to a feature branch",
+    );
+  }
+
+  const parent = dirname(absoluteTarget);
+  if (!fsSync.existsSync(parent)) {
+    throw new SelfeditError(
+      `parent directory '${relative(projectRoot, parent)}' does not exist`,
+    );
+  }
+
+  fsSync.writeFileSync(absoluteTarget, content);
+
+  return { bytes: content.length, relativeTarget, matchedPattern, branch };
+}

package/src/commands/supervise.js

@@ -27,7 +27,7 @@ export async function parseSuperviseOptions(values, runtime) {
   const tmpRoot = runtime.proc.env.TMPDIR ?? "/tmp";
   const agentCwd = resolve(
     values["agent-cwd"] ||
-      (await runtime.fs.mkdtemp(join(tmpRoot, "fit-harness-agent-"))),
+      (await runtime.fs.mkdtemp(join(tmpRoot, "gemba-harness-agent-"))),
   );
 
   return {
@@ -62,7 +62,7 @@ export async function parseSuperviseOptions(values, runtime) {
  * orchestration loop. The supervisor delegates work through Ask, sees
  * each reply on its next turn, and ends with Conclude.
  *
- * Usage: fit-harness supervise [options]
+ * Usage: gemba-harness supervise [options]
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/tee.js

@@ -9,7 +9,7 @@ import { createTeeWriter } from "../tee-writer.js";
  * re-delimits each record with a newline so the TeeWriter's line splitter sees
  * the same framing the raw byte stream produced.
  *
- * Usage: fit-harness tee [output.ndjson] < trace.ndjson
+ * Usage: gemba-harness tee [output.ndjson] < trace.ndjson
  *
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: boolean, code?: number, error?: string}>}

package/src/commands/trace.js

@@ -578,7 +578,7 @@ function parseBuckets(content) {
 
 /**
  * Compute total + per-source cost from raw file content. A structured JSON
- * trace (from `fit-trace download`) carries its total in `summary.totalCostUsd`
+ * trace (from `gemba-trace download`) carries its total in `summary.totalCostUsd`
  * but no per-source split; raw NDJSON is summed via `sumTraceCost`.
  * @param {string} content - Raw file content (structured JSON or NDJSON).
  * @returns {{totalCostUsd: number, bySource: Record<string, number>}}

package/src/cost.js

@@ -13,7 +13,7 @@
  *
  * This mirrors `TraceCollector.handleResult`, which accumulates the same
  * figure for its summary footer — kept as a standalone pure helper so the
- * benchmark runner, the callback command, and `fit-trace cost` share one
+ * benchmark runner, the callback command, and `gemba-trace cost` share one
  * implementation rather than each re-deriving it (and drifting).
  */
 

package/src/trace-collector.js

@@ -271,7 +271,7 @@ export class TraceCollector {
 
   /**
    * Render the accumulated turns as human-readable text — the same path the
-   * live `TeeWriter` stream uses, so `fit-harness output --format=text` over a
+   * live `TeeWriter` stream uses, so `gemba-harness output --format=text` over a
    * captured trace reproduces what the live workflow log showed.
    *
    * Source prefixes are emitted whenever at least one turn has a non-null

package/src/trace-multi.js

@@ -1,5 +1,5 @@
 /**
- * Multi-file orchestrator for cross-trace `fit-trace` verbs.
+ * Multi-file orchestrator for cross-trace `gemba-trace` verbs.
  *
  * Two functions centralise the load-tag-concat (`runOver`) and
  * aggregate-and-sort (`aggregate`) policies so every cross-trace verb shares

package/src/trace-render.js

@@ -1,5 +1,5 @@
 /**
- * Text renderers for `fit-trace` query output.
+ * Text renderers for `gemba-trace` query output.
  *
  * One named export per renderable verb. Each renderer accepts the query result
  * plus `{multi, signatures}` and returns a string. `multi` controls

package/bin/fit-benchmark.js

@@ -1,44 +0,0 @@
-#!/usr/bin/env node
-
-import "@forwardimpact/libpreflight/node22";
-
-import { createCli } from "@forwardimpact/libcli";
-import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
-import { createLogger } from "@forwardimpact/libtelemetry";
-
-import { definition } from "../src/commands/benchmark-definition.js";
-
-const runtime = createDefaultRuntime();
-const logger = createLogger("benchmark", runtime);
-
-async function main() {
-  const cli = createCli(definition, {
-    runtime,
-    packageJsonUrl: new URL("../package.json", import.meta.url),
-  });
-  const parsed = cli.parse(runtime.proc.argv.slice(2));
-  if (!parsed) return runtime.proc.exit(0);
-
-  const { positionals } = parsed;
-  if (positionals.length === 0) {
-    cli.usageError("no command specified");
-    return runtime.proc.exit(2);
-  }
-
-  const command = positionals[0];
-  if (!definition.commands.some((c) => c.name === command)) {
-    cli.usageError(`unknown command "${command}"`);
-    return runtime.proc.exit(2);
-  }
-
-  const result = await cli.dispatch(parsed, { deps: { runtime } });
-  const envelope = result ?? { ok: true };
-  if (!envelope.ok && envelope.error) cli.error(envelope.error);
-  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
-}
-
-main().catch((error) => {
-  logger.exception("main", error);
-  createCli(definition, { runtime }).error(error.message);
-  process.exit(1);
-});