faberun 0.3.0

package/src/engine/live-preflight.mjs

@@ -0,0 +1,299 @@
+/**
+ * Spending one trivial token on each routed runtime before spending the run's.
+ *
+ * A static probe proves a binary exists and reports a version. It does not
+ * prove the credential works, the quota is not spent, or the model answers --
+ * and each of those fails a run several minutes in, after a worktree and a
+ * campaign event already exist. So this sends a real prompt, in a throwaway git
+ * repository, with every runtime clamped to its read-only mode by
+ * `safeLiveRuntime`.
+ *
+ * Every provider string that comes back is redacted against the environment
+ * before it reaches a log: a preflight failure is exactly where a token tends
+ * to appear in an error message.
+ */
+import { emptyUsage } from "../run/usage.mjs";
+import { errorMessage } from "../util.mjs";
+import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { normalizeProviderResult, probeRuntime, providerCommand } from "../harnesses/index.mjs";
+import { reachableRuntimes } from "../host/preflight.mjs";
+import { spawn } from "node:child_process";
+import { boundedGitSync } from "../repo/worktree.mjs";
+import { tmpdir } from "node:os";
+import { validateContract } from "../contract/index.mjs";
+import { validateNodeSnapshot } from "../contract/snapshot.mjs";
+import { compactCost as formatCost } from "../util.mjs";
+
+/** @typedef {import("../harnesses/index.mjs").ProbeResult} ProbeResult */
+/** @typedef {import("../harnesses/index.mjs").ProviderEnvelope} ProviderEnvelope */
+/** @typedef {import("../contract/index.mjs").RuntimeSnapshot} RuntimeSnapshot */
+/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
+
+const LIVE_PREFLIGHT_PROMPT = "Respond with exactly FABERUN_PREFLIGHT_OK and do not use tools.";
+const LIVE_PREFLIGHT_OUTPUT_LIMIT_BYTES = 512 * 1024;
+/**
+ * @param {string} contractPath
+ * @param {{static?: boolean, liveTimeoutSec?: number}} [options]
+ * @returns {Promise<ProbeResult[]>}
+ */
+export async function preflightContract(contractPath, options = {}) {
+  const absoluteContractPath = resolve(contractPath);
+  const contract = validateContract(JSON.parse(readFileSync(absoluteContractPath, "utf8")), absoluteContractPath);
+  const runtimes = reachableRuntimes(contract);
+  const staticChecks = await Promise.all([...runtimes.values()].map(({ runtime, requiredCapabilitySets }) =>
+    probeRuntime(runtime, { cwd: contract.cwd, requiredCapabilitySets }),
+  ));
+  if (options.static === true) return staticChecks;
+
+  const timeoutSec = livePreflightTimeout(options.liveTimeoutSec);
+  let liveRepo;
+  try {
+    liveRepo = createLivePreflightRepo();
+  } catch (error) {
+    return staticChecks.map((check) => ({
+      ...check,
+      ok: false,
+      live: true,
+      liveStatus: "failed",
+      detail: `${check.detail ?? "static probe failed"} · live preflight repository failed: ${redactProviderText(errorMessage(error))}`,
+    }));
+  }
+  try {
+    return await Promise.all(staticChecks.map(async (check, index) => {
+      const runtime = [...runtimes.values()][index].runtime;
+      const live = await livePreflight(runtime, liveRepo, timeoutSec);
+      const liveDetail = live.status === "done"
+        ? `live done · usage ${formatUsage(live.usage)} · cost ${formatCost(live.costUsd)}`
+        : `live ${live.status} · ${live.error?.code ?? "provider_error"}: ${redactProviderText(live.error?.message ?? "generation failed")} · usage ${formatUsage(live.usage)} · cost ${formatCost(live.costUsd)}`;
+      return {
+        ...check,
+        ok: check.ok && live.status === "done",
+        live: true,
+        liveStatus: live.status,
+        usage: live.usage,
+        costUsd: live.costUsd,
+        detail: `${check.detail ?? "static probe failed"} · ${liveDetail}`,
+      };
+    }));
+  } finally {
+    rmSync(liveRepo, { recursive: true, force: true });
+  }
+}
+/** @param {number|undefined} configured */
+function livePreflightTimeout(configured) {
+  const raw = configured ?? (process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC === undefined
+    ? 15
+    : Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC));
+  if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
+    throw new TypeError("preflight live timeout must be a positive number of seconds");
+  }
+  return raw;
+}
+/** @returns {string} */
+function createLivePreflightRepo() {
+  const directory = mkdtempSync(join(tmpdir(), "faberun-preflight-"));
+  const result = boundedGitSync(["init", "-q", directory], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+  if (result.status !== 0 || result.error) {
+    rmSync(directory, { recursive: true, force: true });
+    const reason = result.stderr ? redactProviderText(String(result.stderr)) : result.error?.message;
+    throw new Error(`git init failed${reason ? `: ${reason}` : ""}`);
+  }
+  return directory;
+}
+/**
+ * @param {RuntimeSnapshot} runtime
+ * @returns {RuntimeSnapshot}
+ */
+function safeLiveRuntime(runtime) {
+  if (runtime.harness === "codex") return { ...runtime, sandbox: "read-only" };
+  if (runtime.harness === "dsh") return { ...runtime, sandbox: "read-only" };
+  if (runtime.harness === "claude") return { ...runtime, permissionMode: "plan" };
+  // `plan` is the ZCode mode that reads without writing; the adapter's own
+  // default is `yolo`, which a preflight prompt must never reach.
+  if (runtime.harness === "zcode") return { ...runtime, permissionMode: "plan" };
+  return { ...runtime };
+}
+/**
+ * @param {RuntimeSnapshot} runtime
+ * @param {string} cwd
+ * @param {number} timeoutSec
+ * @returns {Promise<ProviderEnvelope>}
+ */
+function livePreflight(runtime, cwd, timeoutSec) {
+  const safeRuntime = safeLiveRuntime(runtime);
+  let command;
+  try {
+    command = providerCommand(safeRuntime, LIVE_PREFLIGHT_PROMPT);
+  } catch (error) {
+    return Promise.resolve({
+      status: "failed",
+      result: null,
+      continuationId: null,
+      usage: emptyUsage(),
+      costUsd: null,
+      error: { code: "command_invalid", message: errorMessage(error) },
+    });
+  }
+  return new Promise((settle) => {
+    /** @type {import("node:child_process").ChildProcessWithoutNullStreams} */
+    let child;
+    try {
+      const env = { ...process.env };
+      for (const [key, value] of Object.entries(command.env ?? {})) {
+        if (value === null) delete env[key];
+        else env[key] = value;
+      }
+      delete env.FABERUN_NOTIFY_BIN;
+      child = /** @type {import("node:child_process").ChildProcessWithoutNullStreams} */ (spawn(command.executable, command.args, {
+        cwd,
+        env,
+        detached: process.platform !== "win32",
+        stdio: [command.promptTransport === "stdin" ? "pipe" : "ignore", "pipe", "pipe"],
+      }));
+    } catch (error) {
+      settle({
+        status: "failed",
+        result: null,
+        continuationId: null,
+        usage: emptyUsage(),
+        costUsd: null,
+        error: { code: "spawn_error", message: errorMessage(error) },
+      });
+      return;
+    }
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    let timedOut = false;
+    /** @type {ReturnType<typeof setTimeout>|null} */
+    let timer = null;
+    /** @type {ReturnType<typeof setTimeout>|null} */
+    let killTimer = null;
+    /** @param {ProviderEnvelope} envelope */
+    const finish = (envelope) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      if (killTimer) clearTimeout(killTimer);
+      settle(envelope);
+    };
+    /** @param {NodeJS.Signals} name */
+    const signal = (name) => {
+      try {
+        if (process.platform === "win32") child.kill(name);
+        else process.kill(-/** @type {number} */ (child.pid), name);
+      } catch {
+        // ESRCH: the child is already gone, so there is no process to signal.
+      }
+    };
+    child.stdout.on("data", (chunk) => { stdout = appendBounded(stdout, chunk, LIVE_PREFLIGHT_OUTPUT_LIMIT_BYTES); });
+    child.stderr.on("data", (chunk) => { stderr = appendBounded(stderr, chunk, LIVE_PREFLIGHT_OUTPUT_LIMIT_BYTES); });
+    child.once("error", (error) => finish({
+      status: "failed",
+      result: null,
+      continuationId: null,
+      usage: emptyUsage(),
+      costUsd: null,
+      error: { code: "spawn_error", message: redactProviderText(errorMessage(error)) },
+    }));
+    child.once("close", (exitCode, signalName) => {
+      if (timedOut) {
+        finish({
+          status: "failed",
+          result: null,
+          continuationId: null,
+          usage: emptyUsage(),
+          costUsd: null,
+          error: { code: "preflight_timeout", message: `live generation timed out after ${timeoutSec}s` },
+        });
+        return;
+      }
+      /** @type {ProviderEnvelope} */
+      let envelope;
+      try {
+        envelope = normalizeProviderResult(safeRuntime, stdout, exitCode, signalName, { stderr });
+      } catch (error) {
+        envelope = {
+          status: "failed",
+          result: null,
+          continuationId: null,
+          usage: emptyUsage(),
+          costUsd: null,
+          error: { code: "invalid_output", message: redactProviderText(errorMessage(error)) },
+        };
+      }
+      if (envelope.error) envelope.error = { ...envelope.error, message: redactProviderText(envelope.error.message) };
+      finish(envelope);
+    });
+    timer = setTimeout(() => {
+      timedOut = true;
+      signal("SIGTERM");
+      killTimer = setTimeout(() => signal("SIGKILL"), 100);
+    }, timeoutSec * 1_000);
+    if (command.promptTransport === "stdin") child.stdin.end(command.input);
+  });
+}
+/** @param {string} current @param {Uint8Array|string} chunk @param {number} limit */
+function appendBounded(current, chunk, limit) {
+  const combined = Buffer.concat([Buffer.from(current), Buffer.from(chunk)]);
+  return (combined.length > limit ? combined.subarray(combined.length - limit) : combined).toString("utf8");
+}
+/** @param {{inputTokens: number|null, outputTokens: number|null, cacheReadInputTokens: number|null}|undefined} usage */
+function formatUsage(usage) {
+  if (!usage) return "in - out - cache -";
+  return `in ${compactMetric(usage.inputTokens)} out ${compactMetric(usage.outputTokens)} cache ${compactMetric(usage.cacheReadInputTokens)}`;
+}
+/** @param {number|null|undefined} value */
+function compactMetric(value) {
+  return typeof value === "number" && Number.isFinite(value) ? String(value) : "-";
+}
+/** @param {string} value */
+function redactProviderText(value) {
+  let result = String(value);
+  for (const secret of Object.values(process.env)) {
+    if (typeof secret === "string" && secret.length >= 4) result = result.split(secret).join("[REDACTED]");
+  }
+  return result;
+}
+/**
+ * @param {ValidatedContract} contract
+ * @returns {string[]}
+ */
+export function reusedDoneWarnings(contract) {
+  /** @type {string[]} */
+  const warnings = [];
+  const runsDir = join(contract.cwd, ".runs");
+  if (!existsSync(runsDir)) return warnings;
+  const ownRunDir = join(runsDir, contract.id);
+  for (const name of readdirSync(runsDir)) {
+    const otherRunDir = join(runsDir, name);
+    const nodeDir = join(otherRunDir, "nodes");
+    if (otherRunDir === ownRunDir || !existsSync(nodeDir)) continue;
+    const relevantNodes = contract.nodes.filter((node) => existsSync(join(nodeDir, `${node.id}.json`)));
+    if (!relevantNodes.length) continue;
+    const otherContractPath = join(otherRunDir, "contract.json");
+    if (!existsSync(otherContractPath)) throw new TypeError(`missing persisted contract ${otherContractPath}`);
+    // A historical contract may reference paths that no longer exist (e.g. a
+    // rename); reusing its done nodes is best-effort and must not block a new run.
+    let otherContract;
+    try {
+      otherContract = validateContract(JSON.parse(readFileSync(otherContractPath, "utf8")), otherContractPath, { persisted: true });
+    } catch {
+      continue;
+    }
+    for (const node of relevantNodes) {
+      const statePath = join(nodeDir, `${node.id}.json`);
+      const otherNode = otherContract.nodes.find((candidate) => candidate.id === node.id);
+      if (!otherNode) continue;
+      try {
+        if (validateNodeSnapshot(JSON.parse(readFileSync(statePath, "utf8")), otherNode).status === "done") warnings.push(`node ${node.id} is already done in run ${name}`);
+      } catch {
+        // Historical snapshots are advisory only. A snapshot written by an
+        // older protocol revision must not prevent a new run from starting;
+        // the new run still validates its own contract and snapshots strictly.
+      }
+    }
+  }
+  return warnings;
+}

package/src/engine/mutation.mjs

@@ -0,0 +1,146 @@
+/**
+ * Mutation verification: prove a suite asserts by breaking the code on purpose.
+ *
+ * A test that asserts nothing still exits zero, so re-running a command cannot
+ * tell it apart from a real one. This runner rewrites one comparison or logical
+ * operator in the files the node declared, re-runs the same argv, and restores
+ * the file. A mutant the suite fails on is killed; one it survives is a test
+ * that did not assert. The operator set is deliberately six swaps that rarely
+ * break syntax and the sample is a deterministic eight, because a gate that
+ * fails at random, or on a mutant that does not compile, is worse than no gate.
+ *
+ * It is a separate module from `run-command.mjs` so that the "doing" of a
+ * verification run and the "which file to break" policy do not grow together;
+ * the runner receives the argv executor as a callback rather than importing it,
+ * which is also what keeps the import graph acyclic.
+ */
+import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+/** At most this many mutants run for one verification entry. */
+export const MUTATION_BUDGET = 8;
+
+/**
+ * The six operators, and only these: comparison and logical swaps. Each swap
+ * keeps the expression well formed, so a surviving mutant is signal about the
+ * suite rather than about the parser.
+ *
+ * @type {Readonly<Record<string, string>>}
+ */
+const MUTATION_SUBSTITUTIONS = Object.freeze({
+  "===": "!==",
+  "!==": "===",
+  "<=": "<",
+  ">=": ">",
+  "&&": "||",
+  "||": "&&",
+});
+
+/** `!==` first so the `===`/`!==` alternatives cannot shadow each other. */
+const MUTATION_PATTERN = /!==|===|<=|>=|&&|\|\|/gu;
+
+/** @typedef {import("../contract/verification.mjs").VerificationAttemptResult} VerificationAttemptResult */
+/** @typedef {import("../contract/verification.mjs").VerificationCommand} VerificationCommand */
+
+/**
+ * One operator occurrence that can be swapped.
+ *
+ * @typedef {{path: string, absolute: string, offset: number, from: string, to: string}} MutationCandidate
+ */
+
+/**
+ * @typedef {{writeFiles?: string[], run: (attempt: number) => Promise<VerificationAttemptResult>}} MutationRunOptions
+ */
+
+/**
+ * @typedef {{passed: boolean, killed: number, total: number, threshold: number, attempts: VerificationAttemptResult[]}} MutationResult
+ */
+
+/**
+ * Every swap in the declared files, ordered by (path, offset) so the sample is
+ * a function of the tree and not of the filesystem or the clock.
+ *
+ * @param {string} baseCwd
+ * @param {string[]} writeFiles
+ * @returns {MutationCandidate[]}
+ */
+function mutationCandidates(baseCwd, writeFiles) {
+  /** @type {MutationCandidate[]} */
+  const candidates = [];
+  for (const path of [...new Set(writeFiles)].sort()) {
+    const absolute = resolve(baseCwd, path);
+    if (!existsSync(absolute) || !statSync(absolute).isFile()) continue;
+    const text = readFileSync(absolute, "utf8");
+    for (const match of text.matchAll(MUTATION_PATTERN)) {
+      candidates.push({ path, absolute, offset: match.index, from: match[0], to: MUTATION_SUBSTITUTIONS[match[0]] });
+    }
+  }
+  candidates.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : left.offset - right.offset));
+  return candidates;
+}
+
+/**
+ * Pick at most `budget` candidates spread evenly across the ordered list. The
+ * first and last are always kept, so a large file cannot lose a whole region
+ * from the sample silently.
+ *
+ * @param {MutationCandidate[]} candidates
+ * @param {number} budget
+ * @returns {MutationCandidate[]}
+ */
+function evenlySpaced(candidates, budget) {
+  if (candidates.length <= budget) return [...candidates];
+  const last = candidates.length - 1;
+  /** @type {MutationCandidate[]} */
+  const selected = [];
+  for (let index = 0; index < budget; index += 1) selected.push(candidates[Math.round((index * last) / (budget - 1))]);
+  return selected;
+}
+
+/**
+ * Run the entry once untouched and then once per sampled mutant. The original
+ * content is held in memory and written back in `finally`, so a mutant that
+ * times out, throws or kills the process still leaves the tree as it was.
+ *
+ * @param {VerificationCommand} command
+ * @param {string} baseCwd
+ * @param {MutationRunOptions} options
+ * @returns {Promise<MutationResult>}
+ */
+export async function runMutation(command, baseCwd, options) {
+  const mutation = command.mutation;
+  if (!mutation) throw new TypeError("mutation runner requires a command.mutation threshold");
+  const threshold = mutation.threshold;
+  const selected = evenlySpaced(mutationCandidates(baseCwd, options.writeFiles ?? []), MUTATION_BUDGET);
+  /** @type {VerificationAttemptResult[]} */
+  const attempts = [];
+  const baseline = await options.run(1);
+  attempts.push(baseline);
+  if (!baseline.passed) return { passed: false, killed: 0, total: selected.length, threshold, attempts };
+  /** @type {Map<string, string>} */
+  const originals = new Map();
+  let killed = 0;
+  for (const [index, candidate] of selected.entries()) {
+    let original = originals.get(candidate.absolute);
+    if (original === undefined) {
+      original = readFileSync(candidate.absolute, "utf8");
+      originals.set(candidate.absolute, original);
+    }
+    const mutated = `${original.slice(0, candidate.offset)}${candidate.to}${original.slice(candidate.offset + candidate.from.length)}`;
+    /** @type {VerificationAttemptResult} */
+    let result;
+    try {
+      writeFileSync(candidate.absolute, mutated);
+      result = await options.run(index + 2);
+    } finally {
+      writeFileSync(candidate.absolute, original);
+    }
+    attempts.push(result);
+    if (!result.passed) killed += 1;
+  }
+  const total = selected.length;
+  // No operators to break is a vacuous pass: there is no mutant the suite could
+  // have failed to kill. The caller declared the entry, so an empty target is a
+  // measurement of nothing, not a suite that proved nothing.
+  return { passed: total === 0 || killed / total >= threshold, killed, total, threshold, attempts };
+}