catherd-cli 1.3.0 → 1.5.0

package/src/infra/paths.ts

@@ -34,6 +34,41 @@ export function repoKey(toplevel: string): string {
 }
 
 export const repoDir = (toplevel: string): string => join(dataDir(), "repos", repoKey(toplevel));
+
+/**
+ * One repository however it is cloned or checked out (spec 1.5 "Knowledge keyed by git origin"):
+ * `git@github.com:a/b.git`, `https://user@github.com/a/b` and `ssh://git@github.com/a/b.git/` are all
+ * `github.com/a/b`. The host is lower-cased; the path keeps its case.
+ */
+export function normalizeOrigin(url: string): string {
+  let s = url
+    .trim()
+    .replace(/\/+$/, "")
+    .replace(/\.git$/, "");
+  const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(s)?.[1]?.toLowerCase() ?? null;
+  const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(s);
+  if (scp && !scheme) s = `${scp[1]}/${scp[2]}`;
+  else s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "").replace(/^[^@/]+@/, "");
+  const [host = "", ...path] = s.split("/");
+  // only the scheme's own default port goes: two servers on one host on other ports are two repos (PR #50)
+  const port = /:(\d+)$/.exec(host)?.[1];
+  const bare = port !== undefined && scheme !== null && DEFAULT_PORTS[scheme] === port;
+  return [(bare ? host.slice(0, -port.length - 1) : host).toLowerCase(), ...path].join("/");
+}
+
+/** Each git URL scheme's default port, which a URL may name or leave out for the same server. */
+const DEFAULT_PORTS: Record<string, string> = {
+  ssh: "22",
+  "git+ssh": "22",
+  "ssh+git": "22",
+  https: "443",
+  http: "80",
+  git: "9418",
+};
+
+/** Where what runs of one repository learned lives, keyed by its origin, whatever worktree it runs in. */
+export const originDir = (url: string): string =>
+  join(dataDir(), "repos", `origin-${repoKey(normalizeOrigin(url))}`);
 export const runsDir = (toplevel: string): string => join(repoDir(toplevel), "runs");
 export const logsDir = (): string => join(dataDir(), "logs");
 export const discoveryDir = (): string => join(dataDir(), "discovery");

package/src/infra/peer-inbox.ts

@@ -51,10 +51,12 @@ export function sendToInbox(
   const socketPath = target.socketPath;
   return new Promise<SendResult>((resolve) => {
     let settled = false;
+    // once the frame is being written a failure may still have delivered it: such a result carries its msg_id
+    let writing = false;
     const done = (r: SendResult) => {
       if (settled) return;
       settled = true;
-      resolve(r);
+      resolve(writing && r.outcome === "error" ? { ...r, msgId } : r);
     };
     let s: ReturnType<typeof createConnection>;
     try {
@@ -77,6 +79,7 @@ export function sendToInbox(
     });
     s.on("connect", () => {
       clearTimeout(timer);
+      writing = true;
       s.write(frameText(content, priority, target.token, msgId), (err) => {
         if (err) {
           s.destroy();

package/src/infra/role-mcp.ts

@@ -0,0 +1,69 @@
+import { fileURLToPath } from "node:url";
+import { type RoleMcpContext, ROLE_MCP_SERVER, roleMcpTools } from "../domain/role-tools.ts";
+import { scrubSecrets } from "./env.ts";
+import { configDir, dataDir } from "./paths.ts";
+
+export const ROLE_MCP_ENTRY = fileURLToPath(new URL("../entry/mcp/role-bin.ts", import.meta.url));
+
+/** How long Codex waits for the role server to start (spec 1.5 plan 21, #42 finding 8). */
+export const ROLE_MCP_STARTUP_SEC = 30;
+
+export type RoleServerStart = { ok: true; ms: number } | { ok: false; error: string };
+
+/**
+ * Doctor's probe: a cold start of the role server's entry, which loads the server's whole module graph and
+ * exits (`--probe`), timed against the startup timeout. Never a model turn.
+ */
+export async function probeRoleServer(timeoutMs = ROLE_MCP_STARTUP_SEC * 1000): Promise<RoleServerStart> {
+  const started = performance.now();
+  const p = Bun.spawn([process.execPath, ROLE_MCP_ENTRY, "--probe"], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...scrubSecrets(process.env), CATHERD_NO_SYNC: "1" },
+  });
+  const timer = setTimeout(() => p.kill("SIGKILL"), timeoutMs);
+  try {
+    const code = await p.exited;
+    const ms = performance.now() - started;
+    const out = await new Response(p.stdout).text();
+    if (code === 0 && out.trim() === "ready") return { ok: true, ms };
+    const err = (await new Response(p.stderr).text()).trim().split("\n").at(-1);
+    return {
+      ok: false,
+      error: p.signalCode === "SIGKILL" ? `no answer in ${timeoutMs} ms` : err || `exit ${code}`,
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Launch the same installed package as the dispatching process, without resolving another release. */
+export function roleMcpConfig(context: RoleMcpContext): {
+  command: string;
+  args: string[];
+  env: Record<string, string>;
+} {
+  return {
+    command: process.execPath,
+    args: [ROLE_MCP_ENTRY, context.role, context.run],
+    env: { CATHERD_CONFIG_DIR: configDir(), CATHERD_DATA_DIR: dataDir() },
+  };
+}
+
+export function codexRoleMcpArgs(context: RoleMcpContext): string[] {
+  const config = roleMcpConfig(context);
+  const prefix = `mcp_servers.${ROLE_MCP_SERVER}`;
+  const settings = [
+    `${prefix}.command=${JSON.stringify(config.command)}`,
+    `${prefix}.args=${JSON.stringify(config.args)}`,
+    ...Object.entries(config.env).map(([key, value]) => `${prefix}.env.${key}=${JSON.stringify(value)}`),
+    `${prefix}.enabled=true`,
+    // required: a role without its run tools cannot work; doctor's probe shows it starts well inside the timeout
+    `${prefix}.required=true`,
+    `${prefix}.startup_timeout_sec=${ROLE_MCP_STARTUP_SEC}`,
+    `${prefix}.enabled_tools=${JSON.stringify(roleMcpTools(context.role))}`,
+    ...roleMcpTools(context.role).map((tool) => `${prefix}.tools.${tool}.approval_mode="approve"`),
+  ];
+  return settings.flatMap((setting) => ["-c", setting]);
+}

package/src/infra/sources/arena.ts

@@ -1,5 +1,5 @@
 import { effortWord } from "../../domain/sources.ts";
-import { type SourceTransport, sourceGet } from "./http.ts";
+import { SourceError, type SourceTransport, sourceGet } from "./http.ts";
 import { isoDay, num, type SourceRow } from "./rows.ts";
 
 /** Spec 1.2 §3.1: the Agent Arena boards and WebDev (CC-BY-4.0). */
@@ -18,10 +18,18 @@ export const ARENA_PAGE = "https://huggingface.co/datasets/lmarena-ai/leaderboar
 export const arenaUrl = (config: ArenaConfig): string =>
   `https://datasets-server.huggingface.co/rows?dataset=lmarena-ai/leaderboard-dataset&config=${config}&split=latest&length=100`;
 
-/** Every config, in parallel; one failure fails the source, which keeps its last answer. */
+/**
+ * Every config, in parallel; one failure fails the source, which keeps its last answer. A config that answers
+ * no rows is a failure too (1.2 minor): an answer of the wrong shape never replaces a good one.
+ */
 export async function fetchArena(t: SourceTransport = {}): Promise<Record<ArenaConfig, unknown>> {
   const got = await Promise.all(
-    ARENA_CONFIGS.map(async (c) => [c, (await sourceGet(arenaUrl(c), t)).json()]),
+    ARENA_CONFIGS.map(async (c) => {
+      const body = (await sourceGet(arenaUrl(c), t)).json() as { rows?: unknown } | null;
+      if (!Array.isArray(body?.rows) || body.rows.length === 0)
+        throw new SourceError(`Arena ${c} answered no rows`);
+      return [c, body];
+    }),
   );
   return Object.fromEntries(got) as Record<ArenaConfig, unknown>;
 }

package/src/infra/sources/artificial-analysis.ts

@@ -26,7 +26,11 @@ export async function fetchArtificialAnalysis(key: string, t: SourceTransport =
   const o = { ...t, headers: { "x-api-key": key } };
   try {
     const r = await sourceGet(AA_MODELS_URL, o);
-    return { path: "models", pages: [r.json()], rateLimitRemaining: rateLimitRemaining(r.headers) };
+    const body = r.json() as { data?: unknown } | null;
+    // (1.2 minor) an answer of the wrong shape is a failure: the last good one stays
+    if (!Array.isArray(body?.data) || body.data.length === 0)
+      throw new SourceError("Artificial Analysis answered no models", r.status, r.headers);
+    return { path: "models", pages: [body], rateLimitRemaining: rateLimitRemaining(r.headers) };
   } catch (e) {
     if (!(e instanceof SourceError) || (e.status !== 403 && e.status !== 404)) throw e;
   }
@@ -41,6 +45,7 @@ export async function fetchArtificialAnalysis(key: string, t: SourceTransport =
     const total = num(body.pagination?.total_pages);
     if (total !== null && page >= total) break;
   }
+  if (pages.length === 0) throw new SourceError("Artificial Analysis answered an empty first page");
   return { path: "free", pages, rateLimitRemaining: remaining };
 }
 
@@ -53,10 +58,12 @@ export async function testAaKey(
   t: SourceTransport = {},
 ): Promise<{ result: "ok" | "refused" | "unchecked"; error?: string; rateLimitRemaining: number | null }> {
   try {
+    // spec 1.2 §9: one request, never retried (a 429 or 5xx leaves the key unchecked)
     const r = await sourceGet(aaFreeUrl(1), {
       attemptMs: 10_000,
       deadlineMs: 20_000,
       ...t,
+      retries: 0,
       headers: { "x-api-key": key },
     });
     return { result: "ok", rateLimitRemaining: rateLimitRemaining(r.headers) };

package/src/infra/sources/cache.ts

@@ -59,6 +59,8 @@ const StateSchema = z.looseObject({
         error: z.string().nullable(),
         /** Artificial Analysis: `x-ratelimit-remaining` of its last answer */
         rateLimitRemaining: z.number().nullable().default(null),
+        /** when that count was read (1.2 minor); absent in a state written before 1.5 */
+        rateLimitAt: z.iso.datetime().nullable().optional(),
       }),
     )
     .default({}),

package/src/infra/supervisor.ts

@@ -1,10 +1,13 @@
+import { scrubSecrets } from "./env.ts";
 import type { Subprocess } from "bun";
 import { appendFileSync, closeSync, existsSync, openSync, readSync, statSync } from "node:fs";
+import { basename } from "node:path";
 import { z } from "zod";
 import { errorMessage } from "../domain/errors.ts";
 import type { ExitInfo, ExitReason } from "../domain/record.ts";
 import { dispatchPaths, supervisorLockTarget } from "./dispatch-dir.ts";
 import { tryLock } from "./filelock.ts";
+import { lastLockOutput } from "./lock-activity.ts";
 import { log } from "./log.ts";
 import { killGroup, processStartTime } from "./proc.ts";
 import { PRIVATE_FILE, writeJsonAtomic } from "./store.ts";
@@ -16,6 +19,8 @@ export const SuperviseSpecSchema = z.looseObject({
   cmd: z.string(),
   args: z.array(z.string()),
   env: z.record(z.string(), z.string()),
+  /** plan 23: env vars set from another one at spawn (a gate env secret by reference): name → source name */
+  envFrom: z.record(z.string(), z.string()).optional(),
   cwd: z.string(),
   stdinPath: z.string().nullable(),
   idleMs: z.number().positive(),
@@ -23,15 +28,28 @@ export const SuperviseSpecSchema = z.looseObject({
   killGraceMs: z.number().nonnegative(),
   graceAfterFinalMs: z.number().nonnegative().nullable(),
   pollMs: z.number().positive(),
+  /** plan 23: when provider retries end an attempt; PROVIDER_RETRY when absent (tests shorten it) */
+  providerRetry: z
+    .object({ attempts: z.number().positive(), ms: z.number().positive(), pollMs: z.number().positive() })
+    .partial()
+    .optional(),
 });
 export type SuperviseSpec = z.infer<typeof SuperviseSpecSchema>;
 
+/**
+ * Plan 23: an attempt that has only retried its provider (3 retries, or 3 min of nothing else) ends as
+ * provider-unavailable; a quiet session's own retry field is read every pollMs.
+ */
+export const PROVIDER_RETRY = { attempts: 3, ms: 180_000, pollMs: 30_000 };
+
 /** What one stream line tells the supervisor: the terminal event, the thread, a tool call opening or closing. */
 interface LineInfo {
   final?: boolean;
   thread?: string;
   /** a tool call the CLI started (`open`) or finished: while one is open the run is busy, however quiet */
   item?: { id: string; open: boolean };
+  /** plan 23: a model step started; one right after another, with nothing between, is a provider retry */
+  step?: boolean;
 }
 
 export interface SuperviseHooks {
@@ -39,8 +57,23 @@ export interface SuperviseHooks {
   /** `sinceMs`: when this run started, so a backend can ignore what an earlier run on the thread left */
   isBusy?(thread: string | null, sinceMs: number): Promise<boolean>;
   interrupt?(thread: string | null): Promise<void>;
+  /** plan 23: the provider retry attempt the session waits out (its own retry field), or null */
+  providerRetry?(thread: string | null, sinceMs: number): Promise<number | null>;
+  /**
+   * The poll loop's time: what it reads as now and how it waits a poll. The real clock when absent; a test
+   * passes its own so the wall, idle and retry limits are decided by the time it drives, never by how fast a
+   * loaded runner happens to schedule timers.
+   */
+  clock?: SuperviseClock;
+}
+
+export interface SuperviseClock {
+  now(): number;
+  sleep(ms: number): Promise<void>;
 }
 
+const REAL_CLOCK: SuperviseClock = { now: () => Date.now(), sleep: (ms) => Bun.sleep(ms) };
+
 /** Reads whatever the child appended to `file` since `offset`, as complete lines. */
 function readNew(file: string, state: { offset: number; rest: string; decoder: TextDecoder }): string[] {
   if (!existsSync(file)) return [];
@@ -78,7 +111,7 @@ async function bounded<T>(call: () => Promise<T> | undefined, ms: number, fallba
   }
 }
 
-function groupAlive(pgid: number): boolean {
+export function groupAlive(pgid: number): boolean {
   try {
     process.kill(-pgid, 0);
     return true;
@@ -88,7 +121,7 @@ function groupAlive(pgid: number): boolean {
 }
 
 /** SIGTERM the group, give it `graceMs` to leave, then SIGKILL whatever is left of it. */
-async function stopGroup(pgid: number, graceMs: number, pollMs: number): Promise<void> {
+export async function stopGroup(pgid: number, graceMs: number, pollMs: number): Promise<void> {
   killGroup(pgid, "SIGTERM");
   const end = Date.now() + graceMs;
   while (Date.now() < end && groupAlive(pgid))
@@ -139,7 +172,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
     fds.push(stderr);
     child = Bun.spawn([spec.cmd, ...spec.args], {
       cwd: spec.cwd,
-      env: spec.env,
+      env: scrubSecrets(spec.env),
       stdin,
       stdout,
       stderr,
@@ -149,7 +182,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
       backend: spec.backend,
       dispatch: spec.dispatchDir,
       argv: [spec.cmd, ...spec.args],
-      env: spec.env,
+      env: scrubSecrets(spec.env),
       cwd: spec.cwd,
       pid: child.pid,
     });
@@ -167,6 +200,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
   }
 
   const hookMs = Math.min(10_000, spec.idleMs);
+  const clock = hooks.clock ?? REAL_CLOCK;
   let reason: ExitReason | null = null;
   let thread: string | null = null;
   let failure: { error: unknown } | null = null;
@@ -186,8 +220,12 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
       startedAt: new Date().toISOString(),
     });
 
-    const started = Date.now();
+    const started = clock.now();
+    const dispatchId = basename(spec.dispatchDir);
     let lastActivity = started;
+    // plan 23: the last output seen from a `catherd lock` of the role, kept after the lock exits (its file goes),
+    // so the wall counts from that output, never from the role's start again
+    let lockAt = started;
     let finalAt: number | null = null;
     // spec §3.6: a quiet stretch of half the idle timeout, not busy, is a stall, reported once per dispatch
     let stalled = false;
@@ -195,34 +233,88 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
     const open = new Set<string>();
     const stream = { offset: 0, rest: "", decoder: new TextDecoder("utf-8") };
 
+    // plan 23: a step that starts right after another with nothing between is a provider retry; retries are
+    // not progress, and enough of them (or a long enough stretch) end the attempt as provider-unavailable
+    let lastWasStep = false;
+    let retries = 0;
+    let retrySince: number | null = null;
+    let lastRetryPoll = started;
+    const retry = { ...PROVIDER_RETRY, ...spec.providerRetry };
+
     while (!done && reason === null) {
-      await Bun.sleep(spec.pollMs);
+      await clock.sleep(spec.pollMs);
       const lines = readNew(p.events, stream);
-      if (lines.length) {
-        lastActivity = Date.now();
-        stallChecked = false;
-      }
+      let progress = false;
       for (const line of lines) {
         let d: LineInfo | undefined;
         try {
           d = hooks.onLine?.(line);
         } catch {
-          continue; // one line the hook cannot read must not end supervision
+          // one line the hook cannot read must not end supervision; it is output all the same
+          progress = true;
+          lastWasStep = false;
+          continue;
         }
-        if (d?.thread) thread = d.thread;
-        if (d?.final) finalAt ??= Date.now();
+        if (d?.step && lastWasStep) {
+          retries++;
+          retrySince ??= clock.now();
+        } else {
+          progress = true;
+          if (!d?.step) {
+            retries = 0;
+            retrySince = null;
+          }
+        }
+        lastWasStep = d?.step === true;
+        if (d?.thread && d.thread !== thread) {
+          thread = d.thread;
+          // spec 1.5 plan 21: a live role's thread is on disk, so no delivery ever targets it
+          try {
+            writeJsonAtomic(p.thread, { schema: 1, thread, at: new Date().toISOString() });
+          } catch (e) {
+            log("warn", "supervise", { dispatch: spec.dispatchDir, error: errorMessage(e) });
+          }
+        }
+        if (d?.final) finalAt ??= clock.now();
         if (d?.item) {
           if (d.item.open) open.add(d.item.id);
           else open.delete(d.item.id);
         }
       }
+      if (progress) {
+        lastActivity = clock.now();
+        stallChecked = false;
+      }
+      // plan 23: a session quiet for a while may be waiting out a provider retry the stream does not show
+      if (
+        hooks.providerRetry &&
+        clock.now() - lastActivity >= retry.pollMs &&
+        clock.now() - lastRetryPoll >= retry.pollMs
+      ) {
+        lastRetryPoll = clock.now();
+        const attempt = await bounded(() => hooks.providerRetry?.(thread, started), hookMs, null);
+        if (attempt !== null) {
+          retrySince ??= clock.now();
+          retries = Math.max(retries, attempt);
+        } else {
+          // the session says it is not retrying: the steps seen were a retry that worked, and the quiet since
+          // is work the stream does not show yet (a long tool call), never a provider outage
+          retries = 0;
+          retrySince = null;
+        }
+      }
       // a worker that ended on its own is recorded as it ended, even if a cancel or a limit arrived meanwhile
       if (done) break;
-      const now = Date.now();
+      const now = clock.now();
+      lockAt = Math.max(lockAt, lastLockOutput(dispatchId) ?? lockAt);
       if (existsSync(p.cancel)) reason = "cancelled";
-      else if (now - started >= spec.wallMs) reason = "wall-timeout";
+      else if (now - lockAt >= spec.wallMs)
+        // plan 23: once a `catherd lock` of the role has written, the wall counts from its last output
+        reason = "wall-timeout";
       else if (finalAt !== null && spec.graceAfterFinalMs !== null && now - finalAt >= spec.graceAfterFinalMs)
         reason = "after-final";
+      else if (retries >= retry.attempts || (retrySince !== null && now - retrySince >= retry.ms))
+        reason = "provider-unavailable";
       else if (now - lastActivity >= spec.idleMs) {
         const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
         // the busy check can take up to hookMs: a worker that ended meanwhile is recorded as it ended, read
@@ -230,7 +322,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
         if (done || child.exitCode !== null || child.signalCode !== null) break;
         if (busy) {
           // a new quiet stretch starts: it gets its own stall check
-          lastActivity = Date.now();
+          lastActivity = clock.now();
           stallChecked = false;
         } else reason = "idle-timeout";
       } else if (!stalled && !stallChecked && now - lastActivity >= spec.idleMs / 2) {
@@ -239,7 +331,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
         stallChecked = true;
         const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
         if (busy) {
-          lastActivity = Date.now();
+          lastActivity = clock.now();
           stallChecked = false;
         } else if (!done) {
           stalled = true;
@@ -248,7 +340,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
             writeJsonAtomic(p.stall, {
               schema: 1,
               at: new Date().toISOString(),
-              quietMs: Date.now() - lastActivity,
+              quietMs: clock.now() - lastActivity,
             });
           } catch (e) {
             log("warn", "stall", { dispatch: spec.dispatchDir, error: errorMessage(e) });