@forwardimpact/libharness 2.0.0 → 3.0.0

package/bin/fit-harness.js

@@ -1,412 +0,0 @@
-#!/usr/bin/env node
-
-import "@forwardimpact/libpreflight/node22";
-
-import { createCli } from "@forwardimpact/libcli";
-import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
-import { createLogger } from "@forwardimpact/libtelemetry";
-
-import { runOutputCommand } from "../src/commands/output.js";
-import { runTeeCommand } from "../src/commands/tee.js";
-import { runRunCommand } from "../src/commands/run.js";
-import { runSuperviseCommand } from "../src/commands/supervise.js";
-import { runFacilitateCommand } from "../src/commands/facilitate.js";
-import { runDiscussCommand } from "../src/commands/discuss.js";
-import { runCallbackCommand } from "../src/commands/callback.js";
-import { runScanLogsCommand } from "../src/commands/scan-logs.js";
-import { AGENT_MODEL, LEAD_MODEL } from "@forwardimpact/libutil/models";
-
-const LEAD_OPTIONS = {
-  "lead-profile": {
-    type: "string",
-    description: "Lead role profile name (supervisor / facilitator / chair)",
-  },
-  "lead-model": {
-    type: "string",
-    description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
-  },
-};
-
-// Advisor consult flags, shared by all four session modes.
-const ADVISOR_OPTIONS = {
-  "advisor-model": {
-    type: "string",
-    description:
-      "Claude model for advisor consults; omitting the flag disables the Advisor tool (default: off)",
-  },
-  "advisor-max-uses": {
-    type: "string",
-    description:
-      "Session-wide consult budget shared by all participants (default: 3; requires --advisor-model)",
-  },
-};
-
-// Shared task-input flags: --task-file (path), --task-text (inline), and
-// --task-event (path to native GitHub event JSON composed into a task via
-// libharness/src/events/github.js). Exactly one of the three is required.
-const TASK_INPUT_OPTIONS = {
-  "task-file": {
-    type: "string",
-    description: "Path to a markdown task file",
-  },
-  "task-text": {
-    type: "string",
-    description: "Inline task text (alternative to --task-file)",
-  },
-  "task-event": {
-    type: "string",
-    description:
-      "Path to a native GitHub event payload JSON, composed into the task via libharness/src/events/github.js (reads $GITHUB_EVENT_NAME)",
-  },
-  "task-amend": {
-    type: "string",
-    description: "Additional text appended to the task",
-  },
-};
-
-const definition = {
-  name: "fit-harness",
-  description:
-    "Run agents and capture NDJSON traces — for agent evaluations or multi-agent collaboration",
-  commands: [
-    {
-      name: "run",
-      args: [],
-      argsUsage: "",
-      handler: runRunCommand,
-      description: "Run a single agent autonomously on a defined task",
-      options: {
-        ...TASK_INPUT_OPTIONS,
-        "agent-model": {
-          type: "string",
-          description: `Claude model for the agent (default: ${AGENT_MODEL})`,
-        },
-        "max-turns": {
-          type: "string",
-          description: "Max agentic turns (default: 50, 0 = unlimited)",
-        },
-        output: {
-          type: "string",
-          description: "Write the NDJSON trace to a file",
-        },
-        cwd: { type: "string", description: "Working directory for the agent" },
-        "agent-profile": {
-          type: "string",
-          description: "Agent profile name to load",
-        },
-        "work-tracker": {
-          type: "string",
-          description:
-            "Active work-item tracker (github|filesystem, default: github)",
-        },
-        "allowed-tools": {
-          type: "string",
-          description: "Comma-separated tool allowlist",
-        },
-        "mcp-server": {
-          type: "string",
-          description:
-            "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
-        },
-        ...ADVISOR_OPTIONS,
-      },
-    },
-    {
-      name: "supervise",
-      args: [],
-      argsUsage: "",
-      handler: runSuperviseCommand,
-      description:
-        "Run a supervisor–agent relay — typical shape for agent-as-judge evaluations",
-      options: {
-        ...TASK_INPUT_OPTIONS,
-        "agent-model": {
-          type: "string",
-          description: `Claude model for the agent (default: ${AGENT_MODEL})`,
-        },
-        ...LEAD_OPTIONS,
-        "max-turns": {
-          type: "string",
-          description:
-            "Max agentic turns per runner invocation (default: 200, 0 = unlimited)",
-        },
-        output: {
-          type: "string",
-          description: "Write the NDJSON trace to a file",
-        },
-        "agent-profile": { type: "string", description: "Agent profile name" },
-        "work-tracker": {
-          type: "string",
-          description:
-            "Active work-item tracker (github|filesystem, default: github)",
-        },
-        "allowed-tools": {
-          type: "string",
-          description: "Agent tool allowlist",
-        },
-        "supervisor-cwd": {
-          type: "string",
-          description: "Supervisor working directory",
-        },
-        "agent-cwd": { type: "string", description: "Agent working directory" },
-        "supervisor-allowed-tools": {
-          type: "string",
-          description: "Supervisor tool allowlist",
-        },
-        "mcp-server": {
-          type: "string",
-          description:
-            "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
-        },
-        ...ADVISOR_OPTIONS,
-      },
-    },
-    {
-      name: "facilitate",
-      args: [],
-      argsUsage: "",
-      handler: runFacilitateCommand,
-      description:
-        "Run a facilitator with N participants — typical shape for multi-agent collaboration",
-      options: {
-        ...TASK_INPUT_OPTIONS,
-        "agent-model": {
-          type: "string",
-          description: `Claude model for agents (default: ${AGENT_MODEL})`,
-        },
-        ...LEAD_OPTIONS,
-        "max-turns": {
-          type: "string",
-          description: "Max agentic turns (default: 20, 0 = unlimited)",
-        },
-        output: {
-          type: "string",
-          description: "Write the NDJSON trace to a file",
-        },
-        "facilitator-cwd": {
-          type: "string",
-          description: "Facilitator working directory",
-        },
-        "agent-profiles": {
-          type: "string",
-          description:
-            "Comma-separated list of participant profile names (required)",
-        },
-        "agent-cwd": {
-          type: "string",
-          description: "Working directory shared by participants (default: .)",
-        },
-        "work-tracker": {
-          type: "string",
-          description:
-            "Active work-item tracker (github|filesystem, default: github)",
-        },
-        ...ADVISOR_OPTIONS,
-      },
-    },
-    {
-      name: "discuss",
-      args: [],
-      argsUsage: "",
-      handler: runDiscussCommand,
-      description:
-        "Run an async, suspendable discussion — Chair + N participants + bridge callback",
-      options: {
-        ...TASK_INPUT_OPTIONS,
-        "agent-model": {
-          type: "string",
-          description: `Claude model for agents (default: ${AGENT_MODEL})`,
-        },
-        ...LEAD_OPTIONS,
-        "max-turns": {
-          type: "string",
-          description: "Max agentic turns (default: 40, 0 = unlimited)",
-        },
-        output: {
-          type: "string",
-          description: "Write the NDJSON trace to a file",
-        },
-        "agent-profiles": {
-          type: "string",
-          description: "Comma-separated participant profile names (optional)",
-        },
-        "agent-cwd": {
-          type: "string",
-          description: "Working directory shared by participants (default: .)",
-        },
-        "discussion-id": {
-          type: "string",
-          description:
-            "Stable id for the threaded conversation; carried through traces for linking",
-        },
-        "resume-context": {
-          type: "string",
-          description: "JSON-serialized prior state for a resumed run",
-        },
-        "work-tracker": {
-          type: "string",
-          description:
-            "Active work-item tracker (github|filesystem, default: github)",
-        },
-        ...ADVISOR_OPTIONS,
-      },
-    },
-    {
-      name: "output",
-      args: [],
-      argsUsage: "",
-      handler: runOutputCommand,
-      description:
-        "Read NDJSON from stdin and emit a structured or readable form",
-    },
-    {
-      name: "tee",
-      args: ["output"],
-      argsUsage: "[output.ndjson]",
-      handler: runTeeCommand,
-      description:
-        "Stream readable text to stdout while saving raw NDJSON to a file",
-    },
-    {
-      name: "callback",
-      args: [],
-      argsUsage: "",
-      handler: runCallbackCommand,
-      description:
-        "Extract the terminal summary from an NDJSON trace and POST it to a callback URL",
-      options: {
-        "trace-file": {
-          type: "string",
-          description: "Path to the NDJSON trace file",
-        },
-        "callback-url": {
-          type: "string",
-          description: "URL to POST the summary to",
-        },
-        "correlation-id": {
-          type: "string",
-          description: "Correlation ID to include in the payload",
-        },
-        "run-url": {
-          type: "string",
-          description: "GitHub Actions run URL (optional)",
-        },
-        "discussion-id": {
-          type: "string",
-          description:
-            "Discussion id (fallback when the trace lacks a meta event)",
-        },
-      },
-    },
-    {
-      name: "scan-logs",
-      args: [],
-      argsUsage: "",
-      handler: runScanLogsCommand,
-      description:
-        "Scan a run's log archive for secret literals; exit non-zero on any hit, fail closed on an unreadable archive",
-      options: {
-        archive: {
-          type: "string",
-          description: "Path to an already-resolved log archive (.zip)",
-        },
-        "run-id": {
-          type: "string",
-          description:
-            "GitHub Actions run id to download the log archive for (with --repo)",
-        },
-        repo: {
-          type: "string",
-          description: "owner/repo for the --run-id download",
-        },
-        secret: {
-          type: "string",
-          multiple: true,
-          description:
-            "Repeatable label=literal; the literal (everything after the first =) is searched for in the logs",
-        },
-      },
-    },
-  ],
-  globalOptions: {
-    format: { type: "string", description: "Output format (json|text)" },
-    help: { type: "boolean", short: "h", description: "Show this help" },
-    version: { type: "boolean", description: "Show version" },
-    json: { type: "boolean", description: "Output help as JSON" },
-  },
-  examples: [
-    "fit-harness run --task-file=task.md --output=trace.ndjson",
-    "fit-harness run --task-file=task.md --work-tracker=filesystem --output=trace.ndjson",
-    "fit-harness supervise --task-file=task.md --lead-profile=judge --agent-profile=coder --output=trace.ndjson",
-    'fit-harness facilitate --task-file=task.md --lead-profile=lead --agent-profiles="security-engineer,technical-writer" --output=trace.ndjson',
-    'fit-harness discuss --task-file=task.md --lead-profile=release-engineer --agent-profiles="staff-engineer,security-engineer" --discussion-id=GD_kw...',
-    "fit-harness output --format=text < trace.ndjson",
-  ],
-  documentation: [
-    {
-      title: "Coordinate an Agent Team",
-      url: "https://www.forwardimpact.team/docs/libraries/coordinate-team/index.md",
-      description:
-        "Run a lead and N participant agents in one async session — supervise, facilitate, or discuss — with Ask/Answer/Announce and a single NDJSON trace.",
-    },
-    {
-      title: "Run an Eval",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-eval/index.md",
-      description:
-        "Author a judge profile, run an eval locally, wire it into CI, and inspect the resulting trace.",
-    },
-    {
-      title: "Prove Agent Changes",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/index.md",
-      description:
-        "End-to-end workflow from dataset generation through evaluation to trace analysis, including multi-agent collaboration sessions.",
-    },
-    {
-      title: "Analyze Traces",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md",
-      description:
-        "Read the NDJSON traces produced by `fit-harness` with `fit-trace` — grounded-theory method and worked examples.",
-    },
-    {
-      title: "Agent Teams",
-      url: "https://www.forwardimpact.team/docs/products/agent-teams/index.md",
-      description:
-        "How to author the profiles consumed by --agent-profile, --lead-profile, and --agent-profiles.",
-    },
-  ],
-};
-
-const runtime = createDefaultRuntime();
-const logger = createLogger("eval", runtime);
-
-async function main() {
-  const cli = createCli(definition, {
-    runtime,
-    packageJsonUrl: new URL("../package.json", import.meta.url),
-  });
-  const parsed = cli.parse(runtime.proc.argv.slice(2));
-  if (!parsed) return runtime.proc.exit(0);
-
-  const { positionals } = parsed;
-  if (positionals.length === 0) {
-    cli.usageError("no command specified");
-    return runtime.proc.exit(2);
-  }
-
-  const command = positionals[0];
-  if (!definition.commands.some((c) => c.name === command)) {
-    cli.usageError(`unknown command "${command}"`);
-    return runtime.proc.exit(2);
-  }
-
-  const result = await cli.dispatch(parsed, { deps: { runtime } });
-  const envelope = result ?? { ok: true };
-  if (!envelope.ok && envelope.error) cli.error(envelope.error);
-  runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
-}
-
-main().catch((error) => {
-  logger.exception("main", error);
-  createCli(definition, { runtime }).error(error.message);
-  process.exit(1);
-});

package/bin/fit-selfedit.js

@@ -1,165 +0,0 @@
-#!/usr/bin/env node
-/**
- * fit-selfedit — write stdin to a path that .claude/settings.json
- * permits Edit on, while on a non-main git branch. See
- * libraries/libharness/README.md § fit-selfedit for the full rationale.
- */
-
-import "@forwardimpact/libpreflight/node22";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { parseArgs } from "node:util";
-import { resolve, relative, dirname } from "node:path";
-import { execFileSync } from "node:child_process";
-
-import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
-import { minimatch } from "minimatch";
-
-const HELP = `fit-selfedit — write stdin to a settings.json-allowed path on a non-main branch.
-
-Usage:
-  echo content | fit-selfedit <path>
-  fit-selfedit <path> < input.txt
-
-Safeguards (checked in order):
-  1. The nearest .claude/settings.json must contain an Edit(<glob>) rule
-     in permissions.allow[] that resolves to the target path.
-  2. HEAD must not be detached and the current branch must not be 'main'.
-
-Exit codes:
-  0  wrote the file
-  2  safeguard violation (no settings.json, no matching Edit rule, on
-     main, detached HEAD, missing parent directory, TTY stdin)
-  1  unexpected I/O error
-
-Why this exists:
-  Some session harnesses block Edit/Write (and interactive bash writes)
-  on .claude/skills/**, even when the project allowlist permits them.
-  This CLI is a narrow, audited bypass: a subprocess write that still
-  has to clear the project allowlist and the normal merge gates.
-`;
-
-function fail(message) {
-  process.stderr.write(`fit-selfedit: ${message}\n`);
-  process.exit(2);
-}
-
-const { values, positionals } = parseArgs({
-  options: {
-    help: { type: "boolean", short: "h" },
-    version: { type: "boolean" },
-  },
-  allowPositionals: true,
-});
-
-if (values.help) {
-  process.stdout.write(HELP);
-  process.exit(0);
-}
-
-if (values.version) {
-  const pkg = JSON.parse(
-    readFileSync(new URL("../package.json", import.meta.url), "utf8"),
-  );
-  process.stdout.write(`${pkg.version}\n`);
-  process.exit(0);
-}
-
-const [targetArg, ...extra] = positionals;
-if (!targetArg) fail("missing <path> (try --help)");
-if (extra.length > 0) fail(`unexpected extra arguments: ${extra.join(" ")}`);
-
-const absoluteTarget = resolve(process.cwd(), targetArg);
-
-// Safeguard 1: settings.json must grant Edit() on this path. The bin is the
-// sole construction site for the runtime; resolve the finder off the bag
-// rather than constructing a Finder here (Success Criterion 9).
-const runtime = createDefaultRuntime();
-const settingsPath = runtime.finder.findUpward(
-  dirname(absoluteTarget),
-  ".claude/settings.json",
-  20,
-);
-if (!settingsPath) {
-  fail(
-    `no .claude/settings.json found walking upward from ${dirname(absoluteTarget)}`,
-  );
-}
-
-const projectRoot = dirname(dirname(settingsPath));
-const relativeTarget = relative(projectRoot, absoluteTarget);
-
-let settings;
-try {
-  settings = JSON.parse(readFileSync(settingsPath, "utf8"));
-} catch (err) {
-  fail(`failed to parse ${settingsPath}: ${err.message}`);
-}
-
-const allowRules = settings?.permissions?.allow;
-if (!Array.isArray(allowRules)) {
-  fail(`${settingsPath} has no permissions.allow[] array`);
-}
-
-const editPatterns = allowRules
-  .filter((rule) => typeof rule === "string")
-  .map((rule) => rule.match(/^Edit\((.+)\)$/)?.[1])
-  .filter(Boolean);
-
-if (editPatterns.length === 0) {
-  fail(`${settingsPath} has no Edit() rules in permissions.allow[]`);
-}
-
-const matchedPattern = editPatterns.find((pattern) =>
-  minimatch(relativeTarget, pattern, { dot: true }),
-);
-if (!matchedPattern) {
-  fail(
-    `no Edit() rule in ${relative(projectRoot, settingsPath)} matches '${relativeTarget}' ` +
-      `(tried: ${editPatterns.map((p) => `Edit(${p})`).join(", ")})`,
-  );
-}
-
-// Safeguard 2: branch must not be main and HEAD must not be detached.
-let branch;
-try {
-  branch = execFileSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
-    stdio: ["ignore", "pipe", "pipe"],
-    encoding: "utf8",
-  }).trim();
-} catch {
-  fail("failed to read current git branch (not inside a git repository?)");
-}
-
-if (branch === "HEAD") {
-  fail("HEAD is detached — refusing (check out a non-main branch first)");
-}
-if (branch === "main") {
-  fail("refusing to write while on branch 'main' — switch to a feature branch");
-}
-
-const parent = dirname(absoluteTarget);
-if (!existsSync(parent)) {
-  fail(`parent directory '${relative(projectRoot, parent)}' does not exist`);
-}
-
-if (process.stdin.isTTY) {
-  fail(
-    "stdin is a TTY — pipe content in (e.g. `echo … | fit-selfedit <path>`)",
-  );
-}
-
-const chunks = [];
-for await (const chunk of process.stdin) chunks.push(chunk);
-const content = Buffer.concat(chunks);
-
-try {
-  writeFileSync(absoluteTarget, content);
-} catch (err) {
-  process.stderr.write(`fit-selfedit: write failed: ${err.message}\n`);
-  process.exit(1);
-}
-
-process.stderr.write(
-  `fit-selfedit: wrote ${content.length} byte${content.length === 1 ? "" : "s"} to ${relativeTarget} ` +
-    `(matched Edit(${matchedPattern}), branch ${branch})\n`,
-);