catherd-cli 1.4.0 → 2.0.0-beta.20261022113309

package/src/services/failover-rungs.ts

@@ -0,0 +1,109 @@
+import {
+  catalogPairs,
+  claudeBilled,
+  downgrade,
+  intelligenceOf,
+  quotaOf,
+  rankStandIns,
+} from "../domain/failover.ts";
+import { pairOf } from "../domain/catalog.ts";
+import { tryParseRung } from "../domain/ids.ts";
+import type { Role } from "../domain/roles.ts";
+import { effortAllowed } from "../domain/select.ts";
+import { standInFor } from "./backends.ts";
+import { loadCatalog } from "./catalog-service.ts";
+import type { Deps, ProfileView } from "./ports.ts";
+import { listProfiles, readProfileDoc } from "./profile-store.ts";
+import { runProfile } from "./run-pin.ts";
+import type { Run } from "./run-store.ts";
+
+/** The headless twin of a native Claude rung is what the daemon runs: one backend name for both. */
+const backendOf = (rung: string): string | null => {
+  const b = tryParseRung(rung)?.backend ?? null;
+  return b === "claude" ? "claude-code" : b;
+};
+
+/**
+ * Every backend the user configured (spec 2.0 decision 6): those of every rung in every saved profile, and of
+ * every stand-in a profile's `failover` names, not only the profiles a run uses.
+ */
+export function configuredBackends(): string[] {
+  const out = new Set<string>();
+  for (const name of listProfiles()) {
+    let doc: ReturnType<typeof readProfileDoc>;
+    try {
+      doc = readProfileDoc(name);
+    } catch {
+      continue;
+    }
+    const rungs = [
+      ...Object.values(doc.roles ?? {}).flatMap((r) => r?.rungs ?? []),
+      ...Object.values(doc.failover ?? {}),
+    ];
+    for (const r of rungs) {
+      const b = backendOf(r);
+      if (b) out.add(b);
+    }
+  }
+  return [...out].sort();
+}
+
+/** The backends a limit may move work to: every configured one, and those of the run's own profile. */
+export function failoverBackends(profile: ProfileView): Set<string> {
+  const backends = new Set(configuredBackends());
+  for (const r of [
+    ...Object.values(profile.roles).flatMap((x) => x?.rungs ?? []),
+    ...Object.values(profile.failover),
+  ]) {
+    const b = backendOf(r);
+    if (b) backends.add(b);
+  }
+  return backends;
+}
+
+const pairsOf = (c: ReturnType<typeof loadCatalog>): { rung: string; intelligence: number }[] =>
+  catalogPairs(c).map((rung) => ({
+    rung,
+    intelligence: (pairOf(c, rung) as { intelligence: number }).intelligence,
+  }));
+
+/** Plan 29's index through one adapter, so plan 33 may rank on something else: every indexed rung and its index. */
+export function indexedPairs(
+  _deps: Deps,
+  o: { repo?: string; runDir?: string } = {},
+): { rung: string; intelligence: number }[] {
+  return pairsOf(loadCatalog(o));
+}
+
+/**
+ * Where a unit goes when `rung`'s quota hit a limit (spec 2.0 decision 6): the profile's stand-in when its quota
+ * is free and it is not weaker; else the nearest indexed rung of any configured backend on a free quota that is
+ * not weaker, a Claude rung last. Never a step down: a rung with no index fails over only to its stand-in, and
+ * null means every candidate is held or weaker, so the caller waits for the hold. Plan 33: a stand-in above
+ * its model's `maxEffort`, or below medium for a coding worker, is never taken, whatever its quota or cost.
+ */
+export function failoverRung(deps: Deps, run: Run, role: Role, rung: string, held: string[]): string | null {
+  const from = tryParseRung(rung);
+  if (!from) return null;
+  const profile = runProfile(deps, run);
+  const c = loadCatalog({ repo: run.meta.repo, runDir: run.dir });
+  const free = (r: string): boolean => {
+    const p = tryParseRung(r);
+    return (
+      p !== null &&
+      quotaOf(p) !== quotaOf(from) &&
+      !held.includes(quotaOf(p)) &&
+      effortAllowed(profile.maxEffort ?? {}, role, r)
+    );
+  };
+  const stand = standInFor(profile.failover, rung, run.meta.repo);
+  const explicit = stand !== null && free(stand) && !downgrade(c, rung, stand) ? stand : null;
+  if (explicit && !claudeBilled(explicit)) return explicit;
+  if (intelligenceOf(c, rung) === null) return explicit;
+  const backends = failoverBackends(profile);
+  const pool = pairsOf(c)
+    .map((p) => p.rung)
+    .filter((r) => backends.has(backendOf(r) ?? ""));
+  const ranked = rankStandIns(c, profile.billing, rung, pool).filter(free);
+  return ranked.find((r) => !claudeBilled(r)) ?? explicit ?? ranked[0] ?? null;
+}

package/src/services/finalize.ts

@@ -8,6 +8,11 @@ import { isCatherdError } from "../domain/errors.ts";
 import { parseRung } from "../domain/ids.ts";
 import {
   type ExitInfo,
+  parseReplyDesign,
+  parseReplyEnvironment,
+  parseReplyExtras,
+  parseReplyHead,
+  parseReplyRejections,
   parseReplyStatus,
   type RunRecord,
   THREAD_HEAVY_INPUT,
@@ -25,6 +30,7 @@ import {
   writeTextAtomic,
 } from "../infra/store.ts";
 import { type Dispatch, dispatchState, listDispatches, readProc } from "./dispatches.ts";
+import { currentOwns } from "./lane-edit.ts";
 import { tail } from "./run-debug.ts";
 import { appendRecord, readRecords, recordsOnThread, type Run, runPaths } from "./run-store.ts";
 
@@ -75,7 +81,7 @@ function othersOwns(run: Run, self: Dispatch, start: number, end: number): strin
       const to = ended.get(d.admit.dispatchId) ?? (Date.parse(readExit(d.dir)?.endedAt ?? "") || Date.now());
       return from <= end && to >= start;
     })
-    .flatMap((d) => d.admit.owns);
+    .flatMap((d) => [...d.admit.owns, ...(d.admit.ownsImplicit ?? [])]);
 }
 
 /** The tree after the run, or null when git cannot say: the record is still written, its changes unknown. */
@@ -186,13 +192,24 @@ async function compute(run: Run, d: Dispatch): Promise<RunRecord> {
   const { changedOwned, violations } = after
     ? splitChanges(
         changedPaths(a.before, after),
-        a.owns,
+        // spec 1.5 owns_add: paths the lane gained while it ran are its own, not violations; a laneless
+        // writer is held to its implicit docs lane (plan 21)
+        a.lane !== null
+          ? [...new Set([...a.owns, ...currentOwns(run, a.lane)])]
+          : a.owns.length
+            ? a.owns
+            : (a.ownsImplicit ?? []),
         // from admission, when the before-snapshot was taken, not from the worker's start
         othersOwns(run, d, Date.parse(a.admittedAt), end),
         a.lane !== null || a.access === "read-only",
       )
     : { changedOwned: [], violations: [] };
   const reported = parseReplyStatus(replyText);
+  const environment = parseReplyEnvironment(replyText);
+  const head = parseReplyHead(replyText);
+  const extras = parseReplyExtras(replyText);
+  const rejected = parseReplyRejections(replyText);
+  const design = parseReplyDesign(replyText);
   return {
     schema: 1,
     runId: run.id,
@@ -219,6 +236,11 @@ async function compute(run: Run, d: Dispatch): Promise<RunRecord> {
     ...(after ? {} : { gitUnavailable: true }),
     replyStatus: reported.status,
     replyWhy: reported.why,
+    ...(environment ? { environment } : {}),
+    ...(head ? { head } : {}),
+    ...(extras.length ? { extras } : {}),
+    ...(rejected.length ? { rejected } : {}),
+    ...(design.length ? { design } : {}),
     threadHeavy: o.tokens.input >= THREAD_HEAVY_INPUT,
     access: a.access,
     isolated: a.isolated,

package/src/services/gate-env.ts

@@ -0,0 +1,217 @@
+import { existsSync, renameSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { z } from "zod";
+import { CatherdError } from "../domain/errors.ts";
+import { ROLE_ENV } from "../domain/role-scope.ts";
+import { isCatherdSecret } from "../infra/env.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { gitToplevel } from "../infra/git.ts";
+import { DISPATCH_ID_ENV } from "../infra/lock-activity.ts";
+import { repoDir } from "../infra/paths.ts";
+import { ensurePrivateDir, readVersioned, writeJsonAtomic } from "../infra/store.ts";
+import { knowledgeFor } from "./run-store.ts";
+
+// Plan 23: the gate environment a repo's verifier and preflight need (DOCKER_HOST, HTTPS_PROXY, a
+// TESTCONTAINERS_* setting), kept beside the repo's knowledge.md, so no verifier sets it up by hand. A value is
+// stored as is; a secret is stored by reference only: the name of the env var that holds it, read when a
+// role or a check starts, never written to disk.
+
+const GATE_ENV_SCHEMA = 1;
+
+const EntrySchema = z.union([z.strictObject({ value: z.string() }), z.strictObject({ from: z.string() })]);
+export type GateEnvEntry = z.infer<typeof EntrySchema>;
+
+const GateEnvSchema = z.looseObject({
+  schema: z.literal(GATE_ENV_SCHEMA),
+  vars: z.record(z.string(), EntrySchema),
+});
+type GateEnvFile = z.infer<typeof GateEnvSchema>;
+
+/** `<data>/repos/<repo key>/gate-env.json`: where a toplevel keyed it (before 1.5, and with no origin). */
+export const gateEnvFile = (toplevel: string): string => join(repoDir(toplevel), "gate-env.json");
+
+/**
+ * Spec 1.5 "Knowledge keyed by git origin": the gate-env.json beside the repo's knowledge.md, which every
+ * worktree and clone of one repository shares. A toplevel-keyed file this worktree left before is migrated on
+ * read: its variables join the shared file (the shared file's own win), and it is renamed `.migrated`.
+ */
+export async function gateEnvFor(toplevel: string): Promise<string> {
+  const file = join(dirname(await knowledgeFor(toplevel)), "gate-env.json");
+  const legacy = gateEnvFile(toplevel);
+  if (file === legacy || !existsSync(legacy)) return file;
+  ensurePrivateDir(dirname(file));
+  await withFileLock(file, () => {
+    if (!existsSync(legacy)) return;
+    const vars = { ...readVarsAt(legacy), ...readVarsAt(file) };
+    writeJsonAtomic(file, { schema: GATE_ENV_SCHEMA, vars } satisfies GateEnvFile);
+    renameSync(legacy, `${legacy}.migrated`);
+  });
+  return file;
+}
+
+function readVarsAt(file: string): Record<string, GateEnvEntry> {
+  if (!existsSync(file)) return {};
+  return readVersioned<GateEnvFile>(file, GateEnvSchema, GATE_ENV_SCHEMA).vars;
+}
+
+const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** A name a secret goes by: such a variable is set by reference (--from), never by value. */
+const SECRET_NAME = /KEY|SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIAL|AUTH/i;
+
+/**
+ * The names catherd sets in a dispatch itself: the role's identity, dispatch id, testcontainers session, scratch
+ * and working dir, the process basics, catherd's own dirs, and the homes an isolated CLI is moved to (codex,
+ * cursor, opencode, claude). A gate env entry under one would take the role's identity or isolation.
+ */
+const RESERVED = new Set([
+  ROLE_ENV,
+  DISPATCH_ID_ENV,
+  "TESTCONTAINERS_SESSION_ID",
+  "TMPDIR",
+  "PWD",
+  "PATH",
+  "HOME",
+  "CATHERD_HOME",
+  "CATHERD_CONFIG_DIR",
+  "CATHERD_DATA_DIR",
+  "CODEX_HOME",
+  "CLAUDE_CONFIG_DIR",
+  "CURSOR_CONFIG_DIR",
+  "CURSOR_DATA_DIR",
+  "XDG_CONFIG_HOME",
+  "XDG_DATA_HOME",
+]);
+
+/** The repo's gate environment, by name; empty when none is set. */
+export async function readGateEnv(toplevel: string): Promise<Record<string, GateEnvEntry>> {
+  return readVarsAt(await gateEnvFor(toplevel));
+}
+
+async function top(repo: string): Promise<string> {
+  const t = await gitToplevel(repo);
+  if (!t)
+    throw new CatherdError("E_IO_PATH", `${repo} is not inside a git repository`, {
+      fix: "pass the path of the repository",
+    });
+  return t;
+}
+
+function assertName(name: string): void {
+  if (!NAME.test(name))
+    throw new CatherdError("E_INPUT_INVALID", `"${name}" is not an env var name`, {
+      fix: "use letters, digits and _, not starting with a digit, e.g. DOCKER_HOST",
+    });
+}
+
+async function update(
+  repo: string,
+  change: (vars: Record<string, GateEnvEntry>) => void,
+): Promise<{ repo: string; vars: Record<string, GateEnvEntry> }> {
+  const t = await top(repo);
+  const file = await gateEnvFor(t);
+  ensurePrivateDir(dirname(file));
+  return withFileLock(file, () => {
+    const vars = { ...readVarsAt(file) };
+    change(vars);
+    writeJsonAtomic(file, { schema: GATE_ENV_SCHEMA, vars } satisfies GateEnvFile);
+    return { repo: t, vars };
+  });
+}
+
+/**
+ * Sets one variable of the repo's gate environment: a value, or `from`, the name of the env var that holds a
+ * secret. A secret-looking name (…KEY, …TOKEN, …PASSWORD) takes `from` only, so no secret lands on disk.
+ */
+export async function setGateEnv(
+  repo: string,
+  name: string,
+  entry: GateEnvEntry,
+): Promise<{ repo: string; vars: Record<string, GateEnvEntry> }> {
+  assertName(name);
+  if (RESERVED.has(name))
+    throw new CatherdError("E_INPUT_INVALID", `${name} is set by catherd for every role`, {
+      fix: "the gate environment holds what a gate needs (DOCKER_HOST, a proxy, a TESTCONTAINERS_* setting), not a role's identity, scratch or home",
+    });
+  if ("from" in entry) {
+    assertName(entry.from);
+    if (isCatherdSecret(entry.from))
+      throw new CatherdError("E_INPUT_INVALID", `${entry.from} is catherd's own secret: no role gets it`, {
+        fix: "reference the env var that holds the gate's own secret",
+      });
+  } else if (SECRET_NAME.test(name))
+    throw new CatherdError(
+      "E_INPUT_INVALID",
+      `${name} looks like a secret: catherd stores it by reference only`,
+      {
+        fix: `export it in your shell, then: catherd knowledge env set ${name} --from <the env var that holds it>`,
+      },
+    );
+  return update(repo, (vars) => {
+    vars[name] = entry;
+  });
+}
+
+/** Removes one variable; refused when it is not set. */
+export function removeGateEnv(
+  repo: string,
+  name: string,
+): Promise<{ repo: string; vars: Record<string, GateEnvEntry> }> {
+  return update(repo, (vars) => {
+    if (!(name in vars))
+      throw new CatherdError("E_INPUT_INVALID", `${name} is not in the gate environment`, {
+        fix: "catherd knowledge env list shows what is set",
+      });
+    delete vars[name];
+  });
+}
+
+/** The variables as briefs show them: `NAME=value`, a reference as `NAME=$FROM`. */
+export const gateEnvLines = (vars: Record<string, GateEnvEntry>): string[] =>
+  Object.entries(vars)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([k, e]) => ("from" in e ? `${k}=$${e.from}` : `${k}=${e.value}`));
+
+/** The literal values, for a spec written to disk; references stay names (`refs`), resolved at spawn. */
+export function gateEnvParts(vars: Record<string, GateEnvEntry>): {
+  values: Record<string, string>;
+  refs: Record<string, string>;
+} {
+  const values: Record<string, string> = {};
+  const refs: Record<string, string> = {};
+  for (const [k, e] of Object.entries(vars)) {
+    if ("from" in e) refs[k] = e.from;
+    else values[k] = e.value;
+  }
+  return { values, refs };
+}
+
+/**
+ * The env the gate environment adds to `base`: its values, and each reference read from `base`, else from
+ * `fallback` (the login env); a reference neither holds is left out, and named in `missing`.
+ */
+export function resolveGateEnv(
+  vars: Record<string, GateEnvEntry>,
+  base: Record<string, string | undefined>,
+  fallback: () => Record<string, string | undefined> = () => ({}),
+): { env: Record<string, string>; missing: string[] } {
+  const { values, refs } = gateEnvParts(vars);
+  return resolveRefs(values, refs, base, fallback);
+}
+
+/** `values` plus each of `refs` (name → the env var holding it) read from `base`, else `fallback`. */
+export function resolveRefs(
+  values: Record<string, string>,
+  refs: Record<string, string>,
+  base: Record<string, string | undefined>,
+  fallback: () => Record<string, string | undefined> = () => ({}),
+): { env: Record<string, string>; missing: string[] } {
+  const env = { ...values };
+  const missing: string[] = [];
+  let login: Record<string, string | undefined> | null = null;
+  for (const [k, from] of Object.entries(refs)) {
+    const v = base[from] ?? (login ??= fallback())[from];
+    if (v === undefined) missing.push(k);
+    else env[k] = v;
+  }
+  return { env, missing };
+}