catherd-cli 0.2.1 → 1.0.0

package/src/infra/dispatch-dir.ts

@@ -0,0 +1,278 @@
+import {
+  closeSync,
+  existsSync,
+  linkSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+  writeSync,
+} from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { EXIT_REASONS, type ExitInfo } from "../domain/record.ts";
+import { lockHeld, withFileLock } from "./filelock.ts";
+import { isAlive, selfIdentity } from "./proc.ts";
+import { PRIVATE_FILE, readVersioned } from "./store.ts";
+
+export function dispatchPaths(dir: string) {
+  return {
+    brief: join(dir, "brief.md"),
+    spec: join(dir, "spec.json"),
+    proc: join(dir, "proc.json"),
+    events: join(dir, "events.jsonl"),
+    stderr: join(dir, "stderr"),
+    reply: join(dir, "reply.md"),
+    exit: join(dir, "exit.json"),
+    claim: join(dir, "claim"),
+    cancel: join(dir, "cancel"),
+    /** held by the one supervisor of this dispatch for its lifetime (filelock, dead holders reclaimed) */
+    supervisorLock: join(dir, "supervisor.lock"),
+    /** exists from launch until a `wait` hands the dispatch's record to the orchestrator */
+    collect: join(dir, "collect"),
+    /** while a `wait` hands the record back: names that wait's process, so a crash leaves it reclaimable */
+    lease: join(dir, "collect.lease"),
+    supervisorLog: join(dir, "supervisor.log"),
+  };
+}
+
+/**
+ * Exclusive create: the first caller finalizes; every later caller reads the record it wrote. The claim
+ * names its claimant (pid and start time), so a later caller can tell a dead claimant from a slow one.
+ */
+export function tryClaim(dir: string): boolean {
+  let fd: number;
+  try {
+    fd = openSync(dispatchPaths(dir).claim, "wx", PRIVATE_FILE);
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+    return false;
+  }
+  try {
+    writeSync(fd, JSON.stringify(selfIdentity()));
+  } catch (e) {
+    // a claim that names no one would hold every other finalizer off until it is stale: drop it
+    closeSync(fd);
+    try {
+      unlinkSync(dispatchPaths(dir).claim);
+    } catch {
+      // already gone
+    }
+    throw e;
+  }
+  closeSync(fd);
+  return true;
+}
+
+/** Who holds the claim, or null when it names no one (an older build's empty claim, or one being written). */
+export function readClaimant(dir: string): { pid: number; startTime: string | null } | null {
+  try {
+    const c = JSON.parse(readFileSync(dispatchPaths(dir).claim, "utf8")) as {
+      pid?: unknown;
+      startTime?: unknown;
+    };
+    if (typeof c.pid !== "number") return null;
+    return { pid: c.pid, startTime: typeof c.startTime === "string" ? c.startTime : null };
+  } catch {
+    return null;
+  }
+}
+
+/** Marks a dispatch as one a `wait` is to hand back; written before its launch. */
+export function markForCollect(dir: string): void {
+  writeFileSync(dispatchPaths(dir).collect, "", { mode: PRIVATE_FILE });
+}
+
+const errno = (e: unknown): string | undefined => (e as NodeJS.ErrnoException).code;
+
+/** Test seams: `beforeTakeover` runs after a collector judged a lease dead, before it takes the lock. */
+export const collectSeams = { beforeTakeover: async (_dir: string): Promise<void> => {} };
+/** The hard link a lease is made with; tests replace it to stand for a filesystem without hard links. */
+export const leaseFs = { link: linkSync };
+
+/** An ownerless lease this old was left by a crash mid-write (only the no-hard-link fallback can leave one). */
+const OWNERLESS_STALE_MS = 5_000;
+/** A lease temp file this old, or whose pid is dead, was left by a crash. */
+const TEMP_STALE_MS = 60_000;
+
+const self = selfIdentity;
+
+/** Who holds a collection lease, or null when none can be read. */
+function leaseOwner(file: string): { pid: number; startTime: string | null } | null {
+  try {
+    const c = JSON.parse(readFileSync(file, "utf8")) as { pid?: unknown; startTime?: unknown };
+    if (typeof c.pid !== "number") return null;
+    return { pid: c.pid, startTime: typeof c.startTime === "string" ? c.startTime : null };
+  } catch {
+    return null;
+  }
+}
+
+function olderThan(file: string, ms: number): boolean {
+  try {
+    return statSync(file).mtimeMs < Date.now() - ms;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * A lease no live process holds: it exists, and its owner is dead, or it names none and is too old to be
+ * one still being written.
+ */
+function leaseDead(file: string): boolean {
+  if (!existsSync(file)) return false;
+  const who = leaseOwner(file);
+  return who ? !isAlive(who.pid, who.startTime) : olderThan(file, OWNERLESS_STALE_MS);
+}
+
+/**
+ * Whether a `wait` still has this dispatch's record to hand back: it has the mark, or a lease whose
+ * collector died before handing the record back.
+ */
+export function awaitsCollect(dir: string): boolean {
+  const p = dispatchPaths(dir);
+  return existsSync(p.collect) || leaseDead(p.lease);
+}
+
+/** Creates a lease naming this process, exclusively: true when it is now this process's. */
+function createLease(dir: string): boolean {
+  const p = dispatchPaths(dir);
+  const content = JSON.stringify(self());
+  const tmp = `${p.lease}.${process.pid}.${Math.random().toString(36).slice(2)}`;
+  writeFileSync(tmp, content, { mode: PRIVATE_FILE });
+  try {
+    // linked in whole: no reader ever sees a lease without its owner
+    leaseFs.link(tmp, p.lease);
+    return true;
+  } catch (e) {
+    if (errno(e) === "EEXIST") return false;
+    if (!["EPERM", "ENOTSUP", "EXDEV", "EOPNOTSUPP"].includes(errno(e) ?? "")) throw e;
+  } finally {
+    rmSync(tmp, { force: true });
+  }
+  // no hard links here: an exclusive create, its owner written at once (an ownerless lease is young)
+  let fd: number;
+  try {
+    fd = openSync(p.lease, "wx", PRIVATE_FILE);
+  } catch (e) {
+    if (errno(e) === "EEXIST") return false;
+    throw e;
+  }
+  try {
+    writeSync(fd, content);
+  } finally {
+    closeSync(fd);
+  }
+  return true;
+}
+
+/**
+ * Turns a dead collector's lease back into the mark, serialized by a lock on the lease, and only after
+ * reading it again under the lock: the mark is written before the lease goes, so the record is never
+ * without both, and a lease that is not dead (gone, or a live collector's) is left alone. A moved lease
+ * is never linked back.
+ */
+async function reviveDeadLease(dir: string): Promise<void> {
+  const p = dispatchPaths(dir);
+  await withFileLock(p.lease, () => {
+    if (!leaseDead(p.lease)) return;
+    markForCollect(dir);
+    rmSync(p.lease, { force: true });
+    sweepLeaseTemps(dir);
+  });
+}
+
+/**
+ * Collects the dispatch as a lease naming this process: true for the one caller that got it, so each
+ * record reaches one `wait` only. The common path takes no lock: the lease is created exclusively, and
+ * the record is this caller's only if the mark still exists then; the mark goes only once the lease
+ * exists. A dead collector's lease is first turned back into the mark (`reviveDeadLease`), then
+ * collected the same way; a live collector's is never taken. The caller ends the lease with `endCollect`
+ * once the record has gone out, or turns it back into the mark with `putBackCollect`.
+ */
+export async function tryCollect(dir: string): Promise<boolean> {
+  const p = dispatchPaths(dir);
+  if (!existsSync(p.collect)) {
+    if (!leaseDead(p.lease)) return false;
+    await collectSeams.beforeTakeover(dir);
+    await reviveDeadLease(dir);
+  }
+  if (!createLease(dir)) {
+    // a lease is there: collect it only if it is dead, and only by way of the mark
+    if (!leaseDead(p.lease)) return false;
+    await collectSeams.beforeTakeover(dir);
+    await reviveDeadLease(dir);
+    if (!createLease(dir)) return false;
+  }
+  // no lease was there when this one was made: the record is ours only if the mark still is
+  if (!existsSync(p.collect)) {
+    endCollect(dir);
+    return false;
+  }
+  rmSync(p.collect, { force: true });
+  return true;
+}
+
+/** Ends this process's lease (never another's): the record has gone out. Sweeps leftover temp files. */
+export function endCollect(dir: string): void {
+  const p = dispatchPaths(dir);
+  const who = leaseOwner(p.lease);
+  // the pid alone: while this process lives no other can hold its pid, so a lease naming it is its own
+  if (who?.pid === process.pid) rmSync(p.lease, { force: true });
+  sweepLeaseTemps(dir);
+}
+
+/** Turns this process's lease back into the mark: the record did not go out. */
+export function putBackCollect(dir: string): void {
+  markForCollect(dir);
+  endCollect(dir);
+}
+
+const LEASE_TEMP = /^collect\.lease\.(?:dead\.)?(\d+)\.[^.]+$/;
+
+/** Removes lease temp files a crash left behind: a dead pid's, or any older than a minute. */
+function sweepLeaseTemps(dir: string): void {
+  let names: string[];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return;
+  }
+  for (const n of names) {
+    const m = LEASE_TEMP.exec(n);
+    if (!m) continue;
+    const f = join(dir, n);
+    if (!isAlive(Number(m[1]), null) || olderThan(f, TEMP_STALE_MS)) rmSync(f, { force: true });
+  }
+}
+
+const ExitFileSchema = z.looseObject({
+  schema: z.literal(1),
+  code: z.number().nullable(),
+  signal: z.string().nullable(),
+  reason: z.enum(EXIT_REASONS),
+  endedAt: z.string(),
+});
+
+export function readExit(dir: string): ExitInfo | null {
+  try {
+    const e = readVersioned(dispatchPaths(dir).exit, ExitFileSchema, 1);
+    return { code: e.code, signal: e.signal, reason: e.reason, endedAt: e.endedAt };
+  } catch {
+    return null;
+  }
+}
+
+export function requestCancel(dir: string): void {
+  writeFileSync(dispatchPaths(dir).cancel, new Date().toISOString(), { mode: PRIVATE_FILE });
+}
+
+/** The target of the dispatch's supervisor lock (`supervisor.lock`, as `dispatchPaths` names it). */
+export const supervisorLockTarget = (dir: string): string => join(dir, "supervisor");
+
+/** Whether a live supervisor holds the dispatch: launch evidence even before launch.json or proc.json. */
+export const supervisorAlive = (dir: string): boolean => lockHeld(supervisorLockTarget(dir));

package/src/infra/env.ts

@@ -0,0 +1,43 @@
+/** catherd's own secrets; the user's backend credentials (OPENAI_API_KEY, …) stay, workers need them. */
+const SECRET_ENV = new Set(["TYPESAFE_API_KEY"]);
+
+/** `base` without catherd's own secrets and without unset keys; for every process catherd starts. */
+export function scrubSecrets(base: Record<string, string | undefined>): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const [k, v] of Object.entries(base)) if (v !== undefined && !SECRET_ENV.has(k)) env[k] = v;
+  return env;
+}
+
+export function workerEnv(
+  base: Record<string, string | undefined>,
+  overrides: Record<string, string>,
+  cwd: string,
+): Record<string, string> {
+  return { ...scrubSecrets(base), ...overrides, PWD: cwd };
+}
+
+/** What a preflight check may see; everything else, credentials included, stays out (spec §4.7). */
+const CHECK_ENV = new Set([
+  "PATH",
+  "HOME",
+  "USER",
+  "LOGNAME",
+  "SHELL",
+  "LANG",
+  "TZ",
+  "TERM",
+  "TMPDIR",
+  "CI",
+  "NODE_ENV",
+  "CATHERD_HOME",
+  "CATHERD_LOCK_SLOTS",
+]);
+const CHECK_ENV_PATTERN = /^(LC_[A-Z_]+|XDG_[A-Z_]+_(HOME|DIR|DIRS))$/;
+
+/** A strict allowlist of `base` plus PWD, for lane-authored shell text such as preflight checks. */
+export function checkEnv(base: Record<string, string | undefined>, cwd: string): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const [k, v] of Object.entries(base))
+    if (v !== undefined && (CHECK_ENV.has(k) || CHECK_ENV_PATTERN.test(k))) env[k] = v;
+  return { ...env, PWD: cwd };
+}

package/src/infra/filelock.ts

@@ -0,0 +1,159 @@
+import { closeSync, openSync, readFileSync, rmSync, statSync, writeSync } from "node:fs";
+import { CatherdError } from "../domain/errors.ts";
+import { isAlive, isValidPid, selfIdentity } from "./proc.ts";
+import { PRIVATE_FILE } from "./store.ts";
+
+interface Holder {
+  pid: number;
+  startTime: string | null;
+}
+
+/** A lock or reclaim marker this old whose owner cannot be identified was left by a crash. */
+const STALE_MS = 5_000;
+
+const me = (): Holder => selfIdentity();
+
+/** The holder recorded in `lock`, or null when it is missing, empty, unparsable or malformed. */
+function readHolder(lock: string): Holder | null {
+  let v: unknown;
+  try {
+    v = JSON.parse(readFileSync(lock, "utf8"));
+  } catch {
+    return null;
+  }
+  const h = v as Partial<Holder> | null;
+  if (typeof h !== "object" || h === null || !isValidPid(h.pid)) return null;
+  if (h.startTime !== null && typeof h.startTime !== "string") return null;
+  return { pid: h.pid, startTime: h.startTime };
+}
+
+function olderThan(file: string, ms: number): boolean {
+  try {
+    return statSync(file).mtimeMs < Date.now() - ms;
+  } catch {
+    return false;
+  }
+}
+
+/** Creates `file` exclusively with `content`; false when it already exists. */
+function tryCreate(file: string, content: string): boolean {
+  try {
+    const fd = openSync(file, "wx", PRIVATE_FILE);
+    try {
+      writeSync(fd, content);
+    } finally {
+      closeSync(fd);
+    }
+    return true;
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+    return false;
+  }
+}
+
+const tryTake = (lock: string, self: Holder): boolean => tryCreate(lock, JSON.stringify(self));
+
+/** Whether `lock` is abandoned: its holder is dead, or it names no valid holder and is older than STALE_MS. */
+function abandoned(lock: string, h: Holder | null): boolean {
+  return h ? !isAlive(h.pid, h.startTime) : olderThan(lock, STALE_MS);
+}
+
+const sameHolder = (a: Holder | null, b: Holder | null): boolean =>
+  a === b || (a !== null && b !== null && a.pid === b.pid && a.startTime === b.startTime);
+
+/**
+ * Removes an abandoned lock. Reclaims are serialised by `${lock}.reclaim`, and the reclaimer
+ * re-reads the lock while holding it and removes it only if it still names the same abandoned
+ * holder, so a waiter can never delete a lock that a live process took in the meantime.
+ */
+function reclaimIfDead(lock: string): void {
+  const seen = readHolder(lock);
+  if (!abandoned(lock, seen)) return;
+  const marker = `${lock}.reclaim`;
+  if (!tryCreate(marker, String(process.pid))) {
+    // A reclaim takes milliseconds; a marker this old was left by a reclaimer that crashed.
+    if (olderThan(marker, STALE_MS)) rmSync(marker, { force: true });
+    return;
+  }
+  try {
+    const now = readHolder(lock);
+    if (sameHolder(now, seen) && abandoned(lock, now)) rmSync(lock, { force: true });
+  } finally {
+    rmSync(marker, { force: true });
+  }
+}
+
+const timedOut = (target: string, lock: string) =>
+  new CatherdError("E_IO_LOCK", `timed out waiting for the lock on ${target}`, {
+    fix: `if no catherd process is running, delete ${lock}`,
+  });
+
+function release(lock: string, self: Holder): void {
+  const h = readHolder(lock);
+  if (h?.pid === self.pid && h.startTime === self.startTime) rmSync(lock, { force: true });
+}
+
+export async function withFileLock<T>(
+  target: string,
+  fn: () => T | Promise<T>,
+  o: { timeoutMs?: number; pollMs?: number } = {},
+): Promise<T> {
+  const lock = `${target}.lock`;
+  const self = me();
+  const deadline = Date.now() + (o.timeoutMs ?? 10_000);
+  while (!tryTake(lock, self)) {
+    reclaimIfDead(lock);
+    if (Date.now() > deadline) throw timedOut(target, lock);
+    await Bun.sleep(o.pollMs ?? 25);
+  }
+  try {
+    return await fn();
+  } finally {
+    release(lock, self);
+  }
+}
+
+/**
+ * `withFileLock` for a synchronous critical section, waiting with a blocking sleep. For writers whose
+ * callers are synchronous (`withProfilesLock` in the profile service); the lock is held for milliseconds.
+ * Not reentrant: a section must not take the same lock again.
+ */
+export function withFileLockSync<T>(
+  target: string,
+  fn: () => T,
+  o: { timeoutMs?: number; pollMs?: number } = {},
+): T {
+  const lock = `${target}.lock`;
+  const self = me();
+  const deadline = Date.now() + (o.timeoutMs ?? 10_000);
+  while (!tryTake(lock, self)) {
+    reclaimIfDead(lock);
+    if (Date.now() > deadline) throw timedOut(target, lock);
+    Bun.sleepSync(o.pollMs ?? 25);
+  }
+  try {
+    return fn();
+  } finally {
+    release(lock, self);
+  }
+}
+
+/**
+ * One attempt at the lock on `target`, reclaiming it first from a dead holder. Returns its release,
+ * or null while a live holder keeps it. For locks held for a long time, like the heavy-command slots.
+ */
+export function tryLock(target: string): (() => void) | null {
+  const lock = `${target}.lock`;
+  const self = me();
+  if (!tryTake(lock, self)) {
+    reclaimIfDead(lock);
+    if (!tryTake(lock, self)) return null;
+  }
+  return () => release(lock, self);
+}
+
+/** Whether a live process holds the lock on `target` (an unreadable holder counts as none). */
+export function lockHeld(target: string): boolean {
+  const h = readHolder(`${target}.lock`);
+  return h !== null && isAlive(h.pid, h.startTime);
+}

package/src/infra/git.ts

@@ -0,0 +1,91 @@
+import { statSync } from "node:fs";
+import { join } from "node:path";
+import { parsePorcelainZ, type Snapshot } from "../domain/changes.ts";
+import { CatherdError } from "../domain/errors.ts";
+import { scrubSecrets } from "./env.ts";
+
+/** How one git command ended: its stdout, a failure (exit null when git could not start), or a timeout. */
+export type GitResult =
+  | { kind: "ok"; out: string }
+  | { kind: "failed"; exit: number | null }
+  | { kind: "timed-out" };
+
+/** How long one git command may run by default; finalize's claim window counts it (tests shorten it). */
+export const gitLimits = { timeoutMs: 15_000 };
+
+/**
+ * `git -C repo <args>` with a timeout. GIT_OPTIONAL_LOCKS=0 keeps `status` from taking index.lock,
+ * which would fail a worker's own git command running at the same moment. On a timeout git is killed
+ * and its stdout is not awaited, since a grandchild may still hold the pipe open.
+ */
+export async function git(repo: string, args: string[], timeoutMs = gitLimits.timeoutMs): Promise<GitResult> {
+  let p: Bun.Subprocess<"ignore", "pipe", "ignore">;
+  try {
+    p = Bun.spawn(["git", "-C", repo, ...args], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "ignore",
+      env: { ...scrubSecrets(process.env), GIT_OPTIONAL_LOCKS: "0", LC_ALL: "C" },
+    });
+  } catch {
+    return { kind: "failed", exit: null };
+  }
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    p.kill("SIGKILL");
+  }, timeoutMs);
+  const out = new Response(p.stdout).text().catch(() => "");
+  const code = await p.exited;
+  clearTimeout(timer);
+  if (timedOut) return { kind: "timed-out" };
+  return code === 0 ? { kind: "ok", out: await out } : { kind: "failed", exit: code };
+}
+
+const gitBroken = (repo: string, args: string[], r: GitResult) =>
+  new CatherdError(
+    "E_IO_UNEXPECTED",
+    `git ${args[0]} ${r.kind === "timed-out" ? "timed out" : "failed"} in ${repo}`,
+    { fix: `check that git works in ${repo}` },
+  );
+
+export async function gitToplevel(dir: string): Promise<string | null> {
+  const r = await git(dir, ["rev-parse", "--show-toplevel"]);
+  return r.kind === "ok" ? r.out.trim() || null : null;
+}
+
+/** The short HEAD, or null when git cannot say (no commit, not a repo, failure, timeout). */
+export async function gitHead(repo: string, timeoutMs?: number): Promise<string | null> {
+  const r = await git(repo, ["rev-parse", "--short", "HEAD"], timeoutMs);
+  return (r.kind === "ok" && r.out.trim()) || null;
+}
+
+/** False for a missing commit; throws E_IO_UNEXPECTED when git times out, since that proves nothing. */
+export async function commitExists(repo: string, commit: string, timeoutMs?: number): Promise<boolean> {
+  const args = ["cat-file", "-e", `${commit}^{commit}`];
+  const r = await git(repo, args, timeoutMs);
+  if (r.kind === "timed-out") throw gitBroken(repo, args, r);
+  return r.kind === "ok";
+}
+
+function fingerprint(repo: string, xy: string, path: string): string {
+  try {
+    const s = statSync(join(repo, path));
+    return `${xy} ${s.size} ${s.mtimeMs}`;
+  } catch {
+    return `${xy} gone`;
+  }
+}
+
+/**
+ * The repo's dirty files, each with a fingerprint. Throws E_IO_UNEXPECTED when git fails or times out
+ * (including outside a repository), so a broken git never reads as a clean tree.
+ */
+export async function statusSnapshot(repo: string, timeoutMs?: number): Promise<Snapshot> {
+  const args = ["status", "--porcelain=v1", "-z", "--untracked-files=all"];
+  const r = await git(repo, args, timeoutMs);
+  if (r.kind !== "ok") throw gitBroken(repo, args, r);
+  const snap: Snapshot = {};
+  for (const { xy, path } of parsePorcelainZ(r.out)) snap[path] = fingerprint(repo, xy, path);
+  return snap;
+}

package/src/infra/heavy-lock.ts

@@ -0,0 +1,36 @@
+import { availableParallelism } from "node:os";
+import { join } from "node:path";
+import { tryLock } from "./filelock.ts";
+import { locksDir } from "./paths.ts";
+import { ensurePrivateDir } from "./store.ts";
+
+/** A profile's `lock.heavy` as a slot count: a number, or half the cores. */
+export function heavySlots(setting: number | "cpus/2" | undefined): number {
+  if (typeof setting === "number" && Number.isFinite(setting)) return Math.max(1, Math.floor(setting));
+  return Math.max(1, Math.floor(availableParallelism() / 2));
+}
+
+/**
+ * Runs `fn` holding one of `slots` machine-wide heavy-command slots, waiting for a free one. A slot is
+ * a file lock, so a slot whose holder died, or died before writing its name, is reclaimed.
+ */
+export async function withHeavySlot<T>(
+  slots: number,
+  fn: (slot: number) => T | Promise<T>,
+  o: { pollMs?: number } = {},
+): Promise<T> {
+  const dir = locksDir();
+  ensurePrivateDir(dir);
+  for (;;) {
+    for (let i = 0; i < slots; i++) {
+      const release = tryLock(join(dir, `slot-${i}`));
+      if (!release) continue;
+      try {
+        return await fn(i);
+      } finally {
+        release();
+      }
+    }
+    await Bun.sleep(o.pollMs ?? 500);
+  }
+}