faberun 0.3.0

package/src/engine/failover.mjs

@@ -0,0 +1,192 @@
+/**
+ * Worker and judge failover: one declared hop per runtime.
+ *
+ * Each runtime may declare `fallback`, naming at most one other runtime. A
+ * role that exhausts its current runtime gets exactly that one hop — never a
+ * chain, never a synthesized ordering over every other runtime in the
+ * contract. Reachability is therefore always one enumeration away: the
+ * runtime a role started on, and (if declared) the runtime its `fallback`
+ * names. contract.mjs validates the field is never a self-loop; because a
+ * hop is bounded at one, a multi-runtime cycle is structurally impossible.
+ */
+import { harnessCapabilities } from "../harnesses/index.mjs";
+import { nextSameTierRuntime } from "./runtime-discovery.mjs";
+import { routeRuntime } from "../contract/runtime.mjs";
+
+/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
+/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
+
+/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
+/** @typedef {import("../contract/index.mjs").RuntimeSnapshot} RuntimeSnapshot */
+/** @typedef {{id: string, type: string, runtime?: string, gate: {runtime?: string}}} EdgeNode */
+
+/**
+ * Every runtime id in cost order: declared costRank ascending, unranked last,
+ * declaration order breaking ties so reporting stays deterministic.
+ *
+ * "Unranked last" is a separate sort key rather than a sentinel rank, because
+ * costRank only has to be a finite non-negative number — a contract may
+ * legitimately declare Number.MAX_SAFE_INTEGER or more, and any sentinel drawn
+ * from the same number line would tie with it or lose to it. Ranked runtimes
+ * therefore sort ahead of unranked ones by construction, whatever they cost.
+ *
+ * @param {ValidatedContract} contract
+ * @returns {string[]}
+ */
+function rankedRuntimeIds(contract) {
+  return Object.entries(contract.runtimes)
+    .map(([id, runtime], order) => ({ id, order, rank: runtime.costRank, unranked: runtime.costRank === undefined ? 1 : 0 }))
+    .sort((left, right) => left.unranked - right.unranked
+      || (left.unranked ? 0 : /** @type {number} */ (left.rank) - /** @type {number} */ (right.rank))
+      || left.order - right.order)
+    .map((entry) => entry.id);
+}
+
+/**
+ * The one-hop chain out of a runtime: its declared `fallback`, or nothing.
+ *
+ * @param {ValidatedContract} contract
+ * @param {"worker"|"judge"} role unused; kept for signature stability across callers
+ * @param {string} currentRuntime
+ * @returns {string[]}
+ */
+export function synthesizedChain(contract, role, currentRuntime) {
+  const fallback = contract.runtimes[currentRuntime]?.fallback;
+  return fallback ? [fallback] : [];
+}
+
+/**
+ * The next hop out of `currentRuntime`, or null when its one declared
+ * fallback has already been attempted this revision.
+ *
+ * @param {ValidatedContract} contract
+ * @param {"worker"|"judge"} role
+ * @param {string} currentRuntime
+ * @param {Iterable<string|null>} [attempted]
+ * @returns {string|null}
+ */
+export function nextSynthesizedRuntime(contract, role, currentRuntime, attempted = []) {
+  const spent = new Set(attempted);
+  return synthesizedChain(contract, role, currentRuntime).find((id) => !spent.has(id)) ?? null;
+}
+
+/**
+ * The hop number to record for this exhaustion, in the role's current revision.
+ *
+ * Hop is the failover budget: bounded at one, so a run cannot walk its
+ * runtimes forever. Only an actual edge spends from it. A quota-reset retry
+ * stays on the runtime the node already warmed, so it costs no runtime and
+ * must cost no hop either — charging it would let one wait consume the
+ * budget the later real edge needs.
+ *
+ * @param {{routing?: {currentOverride?: {role?: string, revision?: number, hop?: number}|null, history?: {role?: string, revision?: number, hop?: number}[]}|null}} state
+ * @param {"worker"|"judge"} role
+ * @param {number} revision
+ * @param {{kind: "reset"|"failover"}} schedule
+ * @returns {number}
+ */
+export function nextHop(state, role, revision, schedule) {
+  const override = state.routing?.currentOverride;
+  const previousHop = override?.role === role && (override.revision ?? revision) === revision
+    ? override.hop ?? 0
+    : Math.max(0, ...(state.routing?.history ?? [])
+      .filter((entry) => entry.role === role && (entry.revision ?? revision) === revision)
+      .map((entry) => entry.hop ?? 0));
+  return schedule.kind === "reset" ? previousHop : previousHop + 1;
+}
+
+/**
+ * The one-hop failover target out of `current`, if its runtime declares one.
+ * Preflight walks this so the runtime a run could actually spend on is
+ * capability-checked before the first provider starts — and only this one
+ * hop, never the fallback's own fallback, since a node can take only one hop.
+ *
+ * @param {ValidatedContract} contract
+ * @param {{node: EdgeNode, role: "worker"|"judge", runtimeId: string}} current
+ * @param {{routing?: {assignments?: {worker?: string, judge?: string}, availability?: Record<string, {available: boolean, exhaustedUntil: string|null, reason: string}>}}|null} [state]
+ * @returns {RuntimeSnapshot[]}
+ */
+export function failoverTargets(contract, current, state = null) {
+  const declared = synthesizedChain(contract, current.role, current.runtimeId);
+  if (declared.length) return declared.map((id) => runtimeSnapshot(contract, id));
+  const attempted = new Set([current.runtimeId]);
+  const composed = state ? nextSameTierRuntime(contract, state.routing ?? {}, current.role, current.runtimeId, attempted) : null;
+  return composed ? [runtimeSnapshot(contract, composed)] : [];
+}
+
+/**
+ * Every declared fallback edge, for reporting and documentation.
+ *
+ * @param {ValidatedContract} contract
+ * @returns {{from: string, to: string, source: "declared"}[]}
+ */
+export function failoverEdges(contract) {
+  return rankedRuntimeIds(contract)
+    .filter((id) => contract.runtimes[id].fallback)
+    .map((id) => ({ from: id, to: /** @type {string} */ (contract.runtimes[id].fallback), source: /** @type {"declared"} */ ("declared") }));
+}
+
+/**
+ * Merge one runtime's capability requirements into the reachable-runtime map.
+ * The first snapshot wins; every later requirement set is accumulated, so a
+ * runtime reached twice is still checked against both callers' demands.
+ *
+ * @param {Map<string, {runtime: RuntimeSnapshot, requiredCapabilitySets: import("../harnesses/index.mjs").CapabilityRequirements[]}>} runtimes
+ * @param {RuntimeSnapshot} runtime
+ * @param {import("../harnesses/index.mjs").CapabilityRequirements[]} requiredCapabilitySets
+ */
+export function addRuntimeRequirement(runtimes, runtime, requiredCapabilitySets) {
+  const incoming = requiredCapabilitySets.filter((requirements) => requirements && Object.keys(requirements).length);
+  const current = runtimes.get(runtime.id);
+  if (!current) runtimes.set(runtime.id, { runtime, requiredCapabilitySets: incoming });
+  else runtimes.set(runtime.id, { runtime: current.runtime, requiredCapabilitySets: [...current.requiredCapabilitySets, ...incoming] });
+}
+
+/**
+ * Resolve one declared runtime id into the snapshot shape every routing
+ * decision hands on, capabilities and vendor included.
+ *
+ * @param {ValidatedContract} contract
+ * @param {string} id
+ * @returns {RuntimeSnapshot}
+ */
+export function runtimeSnapshot(contract, id) {
+  const runtime = contract.runtimes[id];
+  if (!runtime) throw new Error(`unknown persisted runtime: ${id}`);
+  return { id, ...runtime, capabilities: harnessCapabilities(runtime) };
+}
+
+/**
+ * A node parked on a failover or quota-reset backoff is not schedulable yet.
+ * The window belongs to the role that opened it, so a worker backoff never
+ * holds back a judge on the same node.
+ *
+ * @param {{routing?: {currentOverride?: {role?: string, backoffUntil?: string}|null}|null}} state
+ * @param {string} phase
+ * @returns {boolean}
+ */
+export function routingBackoffActive(state, phase) {
+  const override = state.routing?.currentOverride;
+  return Boolean(override?.role === phase && override.backoffUntil && Date.parse(override.backoffUntil) > Date.now());
+}
+
+/**
+ * @param {ValidatedContract} contract
+ * @param {ValidatedNode} node
+ * @param {NodeSnapshot} state
+ * @param {"worker"|"judge"} role
+ * @returns {RuntimeSnapshot}
+ */
+export function routeRuntimeForState(contract, node, state, role) {
+  const override = state.routing?.currentOverride;
+  if (override?.role === role && contract.runtimes[override.runtime]) {
+    const runtime = contract.runtimes[override.runtime];
+    return { id: override.runtime, ...runtime, capabilities: harnessCapabilities(runtime) };
+  }
+  const assigned = state.routing?.assignments?.[role];
+  if (assigned && contract.runtimes[assigned]) {
+    const runtime = contract.runtimes[assigned];
+    return { id: assigned, ...runtime, capabilities: harnessCapabilities(runtime) };
+  }
+  return /** @type {RuntimeSnapshot} */ (routeRuntime(contract, node, role));
+}

package/src/engine/gate.mjs

@@ -0,0 +1,183 @@
+/**
+ * The provider gate: a standalone program the controller spawns in the worker's
+ * place, which holds the provider process until the controller writes the
+ * release file. It exists so the controller can record an operation intent
+ * before a single token is spent -- the gate is already the running process and
+ * its pid is already persisted, so a crash between spawn and release leaves a
+ * recoverable record rather than an unknown effect.
+ *
+ * It is spawned by path (`spawn(process.execPath, [gate.mjs])`), not imported,
+ * and it talks to its parent only through the environment:
+ *   FABERUN_GATE_CONFIG        the invocation to run, as JSON
+ *   FABERUN_GATE_RELEASE       the file whose appearance releases it
+ *   FABERUN_GATE_PARENT_PID    the controller it must not outlive
+ *   FABERUN_GATE_PARENT_TOKEN  that pid's start token, so a reused pid
+ *                                     is not mistaken for a live controller
+ *
+ * Until 2026-09-11 this was a `String.raw` template inside node.mjs, spawned
+ * with `node -e`. As a real file it is covered by `npm run check` and by
+ * `tsc` -- which found eleven type errors in it on the first run, none of them
+ * reachable while the code was a string -- and it shows up in a stack trace.
+ *
+ * Nothing here is exported for a caller: this file is a program. Every name
+ * below is module-local.
+ */
+import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
+import { spawn } from "node:child_process";
+
+/** @typedef {{executable: string, args: string[], cwd: string, promptTransport: "stdin"|"argv", harness: string, env: Record<string, string|null>|null, stdoutPath: string, stderrPath: string}} GateConfig */
+
+/**
+ * @param {string} name
+ * @returns {string}
+ */
+function requiredEnv(name) {
+  const value = process.env[name];
+  if (!value) throw new Error(`${name} is required`);
+  return value;
+}
+
+/** @type {GateConfig} */
+const config = JSON.parse(readFileSync(requiredEnv("FABERUN_GATE_CONFIG"), "utf8"));
+const releasePath = requiredEnv("FABERUN_GATE_RELEASE");
+const parentPid = Number(process.env.FABERUN_GATE_PARENT_PID);
+const parentToken = process.env.FABERUN_GATE_PARENT_TOKEN || null;
+const maxLogBytes = 512 * 1024;
+
+/**
+ * @param {number} pid
+ * @returns {string|null}
+ */
+function startToken(pid) {
+  if (process.platform !== "linux" || !pid) return null;
+  try {
+    const stat = readFileSync("/proc/" + pid + "/stat", "utf8").trim();
+    return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19] ?? null;
+  } catch { return null; }
+}
+
+/** @returns {boolean} */
+function parentAlive() {
+  try {
+    process.kill(parentPid, 0);
+  } catch (error) {
+    return error !== null && typeof error === "object" && "code" in error
+      && /** @type {{code: unknown}} */ (error).code === "EPERM";
+  }
+  return !parentToken || process.platform !== "linux" || startToken(parentPid) === parentToken;
+}
+
+/** @type {import("node:child_process").ChildProcess|null} */
+let provider = null;
+let inputEnded = config.promptTransport !== "stdin";
+/** @type {Buffer[]} */
+const pendingInput = [];
+if (config.promptTransport === "stdin") {
+  process.stdin.on("data", (chunk) => {
+    if (provider) provider.stdin?.write(chunk);
+    else pendingInput.push(chunk);
+  });
+  process.stdin.on("end", () => {
+    inputEnded = true;
+    if (provider) provider.stdin?.end();
+  });
+}
+
+/** @param {NodeJS.Signals} signal */
+function killGroup(signal) {
+  try { process.kill(-process.pid, signal); } catch {
+    // ESRCH: the process group is already gone, so there is nothing to signal.
+  }
+}
+
+function stopProvider() {
+  try { provider?.kill("SIGTERM"); } catch {
+    // No provider yet, or it already exited: a failed SIGTERM needs no action.
+  }
+  setTimeout(() => killGroup("SIGKILL"), 100).unref();
+}
+
+/**
+ * Providers write directly into the log files: a provider with non-blocking
+ * stdout (EAGAIN on a full pipe) must never die because the controller's event
+ * loop is briefly busy. Cap the files to the last maxLogBytes afterwards.
+ *
+ * @param {string} path
+ * @param {boolean} [preservePrefix]
+ */
+function capLog(path, preservePrefix = false) {
+  try {
+    const size = statSync(path).size;
+    if (size <= maxLogBytes) return;
+    if (preservePrefix) {
+      const prefixLimit = Math.min(64 * 1024, maxLogBytes - 1);
+      const prefix = Buffer.alloc(prefixLimit);
+      const prefixFd = openSync(path, "r");
+      readSync(prefixFd, prefix, 0, prefixLimit, 0);
+      closeSync(prefixFd);
+      const prefixEnd = prefix.lastIndexOf(10);
+      if (prefixEnd >= 0) {
+        const tailLimit = maxLogBytes - prefixEnd - 1;
+        const tail = Buffer.alloc(tailLimit);
+        const tailFd = openSync(path, "r");
+        readSync(tailFd, tail, 0, tailLimit, size - tailLimit);
+        closeSync(tailFd);
+        const tailStart = tail.indexOf(10);
+        const suffix = tailStart >= 0 ? tail.subarray(tailStart + 1) : Buffer.alloc(0);
+        const out = openSync(path, "w");
+        writeSync(out, Buffer.concat([prefix.subarray(0, prefixEnd + 1), suffix]));
+        closeSync(out);
+        return;
+      }
+    }
+    const fd = openSync(path, "r");
+    const buffer = Buffer.alloc(maxLogBytes);
+    readSync(fd, buffer, 0, maxLogBytes, size - maxLogBytes);
+    closeSync(fd);
+    const out = openSync(path, "w");
+    writeSync(out, buffer);
+    closeSync(out);
+  } catch {
+    // Best-effort cap: any filesystem error leaves the log uncapped, which is safe.
+  }
+}
+
+/** @returns {Record<string, string|undefined>} */
+function childEnv() {
+  const merged = { ...process.env };
+  for (const [key, value] of Object.entries(config.env ?? {})) {
+    if (value === null) delete merged[key];
+    else merged[key] = value;
+  }
+  // Worker providers are not a notification surface: strip the controller-only
+  // transport after the harness overlay so no harness can reintroduce it.
+  delete merged.FABERUN_NOTIFY_BIN;
+  return merged;
+}
+
+process.on("SIGTERM", () => stopProvider());
+process.on("SIGINT", () => stopProvider());
+
+const timer = setInterval(() => {
+  if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
+  if (!existsSync(releasePath)) return;
+  clearInterval(timer);
+  const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
+  const stderrFd = openSync(config.stderrPath, "wx", 0o600);
+  provider = spawn(config.executable, config.args, {
+    cwd: config.cwd,
+    env: childEnv(),
+    stdio: [config.promptTransport === "stdin" ? "pipe" : "ignore", stdoutFd, stderrFd],
+  });
+  if (config.promptTransport === "stdin") {
+    for (const chunk of pendingInput) provider.stdin?.write(chunk);
+    pendingInput.length = 0;
+    if (inputEnded) provider.stdin?.end();
+  }
+  provider.once("error", () => process.exitCode = 127);
+  provider.once("close", (code) => {
+    capLog(config.stdoutPath, config.harness === "codex");
+    capLog(config.stderrPath);
+    process.exit(code ?? 1);
+  });
+}, 10);