catherd-cli 1.4.0 → 2.0.0-beta.20261022113309

package/src/infra/launch.ts

@@ -1,25 +1,81 @@
 import { closeSync, openSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { ROLE_ENV } from "../domain/role-scope.ts";
 import { scrubSecrets } from "./env.ts";
 import { log as logRow } from "./log.ts";
 import { PRIVATE_FILE } from "./store.ts";
 
-/** A thin executable that supervises one spec; unlike src/cli.ts it never loads the TUI. */
+/** A thin executable that supervises one spec; it never loads src/cli.ts. */
 export const SUPERVISE_ENTRY = fileURLToPath(new URL("../entry/supervise-bin.ts", import.meta.url));
 
-/** Starts `supervise-bin.ts <spec>` in its own session and returns at once; it outlives this process. */
-export function launchSupervisor(specPath: string): number {
-  const log = openSync(join(dirname(specPath), "supervisor.log"), "a", PRIVATE_FILE);
-  const p = Bun.spawn([process.execPath, SUPERVISE_ENTRY, specPath], {
+/** The thin executable that drives one run; like the supervisor's, it never loads src/cli.ts. */
+export const DAEMON_ENTRY = fileURLToPath(new URL("../entry/daemon-bin.ts", import.meta.url));
+
+/** The thin executable that runs one milestone check (plan 30b); like the others it never loads src/cli.ts. */
+export const CHECK_ENTRY = fileURLToPath(new URL("../entry/check-bin.ts", import.meta.url));
+
+/**
+ * Starts `entry args` in its own session, its output appended to `logFile`, and returns at once: the process
+ * outlives this one.
+ */
+export function launchDetached(
+  entry: string,
+  args: string[],
+  logFile: string,
+  env: Record<string, string>,
+): number {
+  const log = openSync(logFile, "a", PRIVATE_FILE);
+  const p = Bun.spawn([process.execPath, entry, ...args], {
     stdin: "ignore",
     stdout: log,
     stderr: log,
     detached: true,
-    env: scrubSecrets(process.env),
+    env,
   });
   closeSync(log);
   p.unref();
-  logRow("info", "launch", { spec: specPath, pid: p.pid });
   return p.pid;
 }
+
+/** Starts `supervise-bin.ts <spec>` in its own session and returns at once; it outlives this process. */
+export function launchSupervisor(specPath: string): number {
+  const pid = launchDetached(
+    SUPERVISE_ENTRY,
+    [specPath],
+    join(dirname(specPath), "supervisor.log"),
+    scrubSecrets(process.env),
+  );
+  logRow("info", "launch", { spec: specPath, pid });
+  return pid;
+}
+
+/**
+ * The daemon's env (spec 2.0, plan 30 ruling 4): the launching process's own, so Jev's key reaches its routing,
+ * without the session identity of whoever launched it, and never a role's scope: it is nobody's role.
+ */
+export function daemonEnv(base: Record<string, string | undefined> = process.env): Record<string, string> {
+  const env = scrubSecrets(base);
+  if (base.TYPESAFE_API_KEY !== undefined) env.TYPESAFE_API_KEY = base.TYPESAFE_API_KEY;
+  delete env[ROLE_ENV];
+  return env;
+}
+
+/** Starts `daemon-bin.ts <run id>` for a run; its log is `<run dir>/daemon.log`. */
+export function launchDaemon(runId: string, runDir: string): number {
+  const pid = launchDetached(DAEMON_ENTRY, [runId], join(runDir, "daemon.log"), daemonEnv());
+  logRow("info", "launch", { daemon: runId, pid });
+  return pid;
+}
+
+/** Starts `check-bin.ts <flight file>` detached, so a tool timeout that kills its caller does not lose the check. */
+export function launchCheckWorker(flight: string): number {
+  const pid = launchDetached(
+    CHECK_ENTRY,
+    [flight],
+    join(dirname(flight), "worker.log"),
+    scrubSecrets(process.env),
+  );
+  logRow("info", "launch", { check: flight, pid });
+  return pid;
+}

package/src/infra/lock-activity.ts

@@ -0,0 +1,88 @@
+import { existsSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { locksDir } from "./paths.ts";
+import { isAlive, processStartTime } from "./proc.ts";
+import { ensurePrivateDir, writeJsonAtomic } from "./store.ts";
+
+// Plan 23: a role's long gate runs behind `catherd lock`. While such a lock child is alive and writing, the
+// role's wall timeout counts from its last output, not from the role's start. The lock writes here, in the
+// locks dir every role may write; the supervisor reads it.
+
+/** The env var the supervisor's spec carries, so a `catherd lock` inside the role knows its dispatch. */
+export const DISPATCH_ID_ENV = "CATHERD_DISPATCH_ID";
+
+/** A lock child reports at most this often, so a chatty command does not write a file per line. */
+export const ACTIVITY_EVERY_MS = 1_000;
+
+const SAFE_ID = /^[A-Za-z0-9][\w.-]{0,127}$/;
+
+/** `<locks>/activity/<dispatch id>`: one file per live `catherd lock` of that dispatch. */
+export const activityDir = (dispatchId: string): string => join(locksDir(), "activity", dispatchId);
+
+interface Activity {
+  schema: 1;
+  pid: number;
+  startTime: string | null;
+  /** the last time the lock's command wrote output (its start, before the first line) */
+  at: string;
+}
+
+/**
+ * A reporter for this `catherd lock` process, or null outside a dispatch (no id, or one that is not a plain
+ * id: it names a path). `tick` records output, at most once per ACTIVITY_EVERY_MS; `done` removes the file.
+ */
+export function activityReporter(
+  dispatchId: string | undefined,
+  now: () => number = Date.now,
+  everyMs = ACTIVITY_EVERY_MS,
+): { tick: () => void; done: () => void } | null {
+  if (!dispatchId || !SAFE_ID.test(dispatchId) || dispatchId.includes("..")) return null;
+  const dir = activityDir(dispatchId);
+  const file = join(dir, `${process.pid}.json`);
+  const startTime = processStartTime(process.pid);
+  let last = 0;
+  const write = (t: number): void => {
+    try {
+      ensurePrivateDir(dir);
+      writeJsonAtomic(file, {
+        schema: 1,
+        pid: process.pid,
+        startTime,
+        at: new Date(t).toISOString(),
+      } satisfies Activity);
+      last = t;
+    } catch {
+      // advisory: a lock that cannot report still runs its command
+    }
+  };
+  write(now());
+  return {
+    tick: () => {
+      const t = now();
+      if (t - last >= everyMs) write(t);
+    },
+    done: () => rmSync(file, { force: true }),
+  };
+}
+
+/**
+ * When a live `catherd lock` of the dispatch last wrote output, in ms; null when none is alive. A file whose
+ * process is gone is left alone (its lock may still be finishing its own cleanup) and never counts.
+ */
+export function lastLockOutput(dispatchId: string): number | null {
+  const dir = activityDir(dispatchId);
+  if (!existsSync(dir)) return null;
+  let latest: number | null = null;
+  for (const name of readdirSync(dir)) {
+    if (!name.endsWith(".json")) continue;
+    try {
+      const a = JSON.parse(readFileSync(join(dir, name), "utf8")) as Activity;
+      if (a.schema !== 1 || !isAlive(a.pid, a.startTime)) continue;
+      const at = Date.parse(a.at);
+      if (!Number.isNaN(at)) latest = Math.max(latest ?? at, at);
+    } catch {
+      // a file being written, or damaged: skipped this poll
+    }
+  }
+  return latest;
+}

package/src/infra/login-env.ts

@@ -0,0 +1,48 @@
+import { scrubSecrets } from "./env.ts";
+
+// Plan 23: what a login shell of the user's sees (DOCKER_HOST for OrbStack or rootless Docker, a proxy, a
+// toolchain's PATH), which a server started by a desktop app or a plugin launcher may not inherit.
+// `$SHELL -lc env` is run once per process and kept.
+
+/** How long the login shell may take; one that hangs (a prompt in a profile) counts as an empty env. */
+export const loginShell = { timeoutMs: 5_000 };
+
+let captured: Record<string, string> | null = null;
+
+/** Parses `env -0` output, or plain `env` lines from a shell whose env has no -0. */
+export function parseEnv(out: string): Record<string, string> {
+  const env: Record<string, string> = {};
+  const rows = out.includes("\0") ? out.split("\0") : out.split("\n");
+  for (const row of rows) {
+    const eq = row.indexOf("=");
+    if (eq > 0) env[row.slice(0, eq)] = row.slice(eq + 1);
+  }
+  return env;
+}
+
+function capture(): Record<string, string> {
+  const shell = process.env.SHELL || "/bin/sh";
+  try {
+    const r = Bun.spawnSync([shell, "-lc", "env -0 2>/dev/null || env"], {
+      env: scrubSecrets(process.env),
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "ignore",
+      timeout: loginShell.timeoutMs,
+    });
+    return r.exitCode === 0 ? scrubSecrets(parseEnv(r.stdout.toString())) : {};
+  } catch {
+    return {};
+  }
+}
+
+/** The user's login environment, without catherd's secrets; empty when the shell cannot say. Captured once. */
+export function loginEnv(): Record<string, string> {
+  captured ??= capture();
+  return captured;
+}
+
+/** Forgets the captured env (tests, which change SHELL). */
+export function resetLoginEnv(): void {
+  captured = null;
+}

package/src/infra/memory.ts

@@ -0,0 +1,45 @@
+import { readFileSync } from "node:fs";
+import { freemem, totalmem } from "node:os";
+
+const MB = 2 ** 20;
+
+const sysctlLevel = (): string | null => {
+  try {
+    const r = Bun.spawnSync(["sysctl", "-n", "kern.memorystatus_level"]);
+    return r.exitCode === 0 ? r.stdout.toString() : null;
+  } catch {
+    return null;
+  }
+};
+
+const procMeminfo = (): string | null => {
+  try {
+    return readFileSync("/proc/meminfo", "utf8");
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * The memory the machine could give a new process, in MB. macOS reports almost nothing through `os.freemem()`
+ * (it counts cache as used), so it is the kernel's free percentage of the total; Linux has `MemAvailable`;
+ * anything unreadable falls back to `os.freemem()`. Never throws.
+ */
+export function availableMemoryMb(
+  o: {
+    platform?: NodeJS.Platform;
+    sysctl?: () => string | null;
+    meminfo?: () => string | null;
+  } = {},
+): number {
+  const platform = o.platform ?? process.platform;
+  if (platform === "darwin") {
+    const level = Number.parseInt((o.sysctl ?? sysctlLevel)() ?? "", 10);
+    if (Number.isInteger(level) && level >= 0 && level <= 100)
+      return Math.floor(((level / 100) * totalmem()) / MB);
+  } else if (platform === "linux") {
+    const kb = /^MemAvailable:\s+(\d+) kB$/m.exec((o.meminfo ?? procMeminfo)() ?? "")?.[1];
+    if (kb !== undefined) return Math.floor(Number(kb) / 1024);
+  }
+  return Math.floor(freemem() / MB);
+}

package/src/infra/ntfy.ts

@@ -0,0 +1,40 @@
+import type { NotifyMessage } from "../domain/notify.ts";
+import { log } from "./log.ts";
+
+/** A header value as ntfy reads it: plain ASCII as it is, anything else as an RFC 2047 encoded word. */
+const headerValue = (s: string): string =>
+  /^[\x20-\x7e]*$/.test(s) ? s : `=?UTF-8?B?${Buffer.from(s).toString("base64")}?=`;
+
+/**
+ * Posts one message to an ntfy topic. True once the server accepted it; any error, a non-2xx answer or a
+ * server slower than `timeoutMs` is false, never a throw, and the log never carries the topic (the url is a
+ * secret of the user's).
+ */
+export async function postNotify(
+  url: string,
+  m: NotifyMessage,
+  o: { fetch?: typeof fetch; timeoutMs?: number } = {},
+): Promise<boolean> {
+  const timeoutMs = o.timeoutMs ?? 10_000;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    const sent = (o.fetch ?? fetch)(url, {
+      method: "POST",
+      body: m.message,
+      headers: { Title: headerValue(m.title), Tags: m.tags.join(",") },
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    // a fetch that ignores its signal still gives up
+    const gaveUp = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`no answer in ${timeoutMs} ms`)), timeoutMs);
+    });
+    const res = await Promise.race([sent, gaveUp]);
+    if (!res.ok) log("warn", "notify", { ntfy: res.status });
+    return res.ok;
+  } catch (e) {
+    log("warn", "notify", { ntfy: e instanceof Error ? e.name : "error" });
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/infra/open-url.ts

@@ -0,0 +1,42 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+/**
+ * A page that sends the browser on to `url`, so the opener's argv (readable by every local user, and kept in
+ * /proc/<pid>/cmdline) carries a path instead of the token. The dir is 0700 and the file 0600; `remove` deletes both.
+ */
+export function writeRedirectPage(url: string): {
+  path: string;
+  remove(): void;
+} {
+  const dir = mkdtempSync(join(tmpdir(), "catherd-panel-"));
+  const path = join(dir, "sign-in.html");
+  writeFileSync(
+    path,
+    `<!doctype html><meta charset="utf-8"><title>catherd panel</title><script>location.replace(${JSON.stringify(url)})</script>\n`,
+    { mode: 0o600 },
+  );
+  return { path, remove: () => rmSync(dir, { recursive: true, force: true }) };
+}
+
+/** The command that opens `url` in the user's browser on `platform`. */
+export function openerFor(platform: NodeJS.Platform, url: string): string[] {
+  if (platform === "darwin") return ["open", url];
+  if (platform === "win32") return ["cmd", "/c", "start", "", url];
+  return ["xdg-open", url];
+}
+
+/** Opens `url` in the default browser, detached; a machine with no opener simply opens nothing. */
+export function openUrl(url: string): void {
+  try {
+    Bun.spawn(openerFor(process.platform, url), {
+      stdin: "ignore",
+      stdout: "ignore",
+      stderr: "ignore",
+      detached: true,
+    }).unref();
+  } catch {
+    // no opener on PATH: the printed link still works
+  }
+}

package/src/infra/paths.ts

@@ -34,9 +34,44 @@ export function repoKey(toplevel: string): string {
 }
 
 export const repoDir = (toplevel: string): string => join(dataDir(), "repos", repoKey(toplevel));
+
+/**
+ * One repository however it is cloned or checked out (spec 1.5 "Knowledge keyed by git origin"):
+ * `git@github.com:a/b.git`, `https://user@github.com/a/b` and `ssh://git@github.com/a/b.git/` are all
+ * `github.com/a/b`. The host is lower-cased; the path keeps its case.
+ */
+export function normalizeOrigin(url: string): string {
+  let s = url
+    .trim()
+    .replace(/\/+$/, "")
+    .replace(/\.git$/, "");
+  const scheme = /^([a-z][a-z0-9+.-]*):\/\//i.exec(s)?.[1]?.toLowerCase() ?? null;
+  const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(s);
+  if (scp && !scheme) s = `${scp[1]}/${scp[2]}`;
+  else s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "").replace(/^[^@/]+@/, "");
+  const [host = "", ...path] = s.split("/");
+  // only the scheme's own default port goes: two servers on one host on other ports are two repos (PR #50)
+  const port = /:(\d+)$/.exec(host)?.[1];
+  const bare = port !== undefined && scheme !== null && DEFAULT_PORTS[scheme] === port;
+  return [(bare ? host.slice(0, -port.length - 1) : host).toLowerCase(), ...path].join("/");
+}
+
+/** Each git URL scheme's default port, which a URL may name or leave out for the same server. */
+const DEFAULT_PORTS: Record<string, string> = {
+  ssh: "22",
+  "git+ssh": "22",
+  "ssh+git": "22",
+  https: "443",
+  http: "80",
+  git: "9418",
+};
+
+/** Where what runs of one repository learned lives, keyed by its origin, whatever worktree it runs in. */
+export const originDir = (url: string): string =>
+  join(dataDir(), "repos", `origin-${repoKey(normalizeOrigin(url))}`);
 export const runsDir = (toplevel: string): string => join(repoDir(toplevel), "runs");
 export const logsDir = (): string => join(dataDir(), "logs");
 export const discoveryDir = (): string => join(dataDir(), "discovery");
 export const locksDir = (): string => join(dataDir(), "locks");
-/** Spec 1.2 §3.3: each source's last good answer, the sync state and lock, and what the sync derived. */
-export const sourcesDir = (): string => join(dataDir(), "sources");
+/** The Slopalytics index: the last good answer of the proxy, one intelligence per model and effort. */
+export const indexPath = (): string => join(dataDir(), "slopalytics.json");

package/src/infra/role-mcp.ts

@@ -0,0 +1,69 @@
+import { fileURLToPath } from "node:url";
+import { type RoleMcpContext, ROLE_MCP_SERVER, roleMcpTools } from "../domain/role-tools.ts";
+import { scrubSecrets } from "./env.ts";
+import { configDir, dataDir } from "./paths.ts";
+
+export const ROLE_MCP_ENTRY = fileURLToPath(new URL("../entry/mcp/role-bin.ts", import.meta.url));
+
+/** How long Codex waits for the role server to start (spec 1.5 plan 21, #42 finding 8). */
+export const ROLE_MCP_STARTUP_SEC = 30;
+
+export type RoleServerStart = { ok: true; ms: number } | { ok: false; error: string };
+
+/**
+ * Doctor's probe: a cold start of the role server's entry, which loads the server's whole module graph and
+ * exits (`--probe`), timed against the startup timeout. Never a model turn.
+ */
+export async function probeRoleServer(timeoutMs = ROLE_MCP_STARTUP_SEC * 1000): Promise<RoleServerStart> {
+  const started = performance.now();
+  const p = Bun.spawn([process.execPath, ROLE_MCP_ENTRY, "--probe"], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...scrubSecrets(process.env), CATHERD_NO_SYNC: "1" },
+  });
+  const timer = setTimeout(() => p.kill("SIGKILL"), timeoutMs);
+  try {
+    const code = await p.exited;
+    const ms = performance.now() - started;
+    const out = await new Response(p.stdout).text();
+    if (code === 0 && out.trim() === "ready") return { ok: true, ms };
+    const err = (await new Response(p.stderr).text()).trim().split("\n").at(-1);
+    return {
+      ok: false,
+      error: p.signalCode === "SIGKILL" ? `no answer in ${timeoutMs} ms` : err || `exit ${code}`,
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Launch the same installed package as the dispatching process, without resolving another release. */
+export function roleMcpConfig(context: RoleMcpContext): {
+  command: string;
+  args: string[];
+  env: Record<string, string>;
+} {
+  return {
+    command: process.execPath,
+    args: [ROLE_MCP_ENTRY, context.role, context.run],
+    env: { CATHERD_CONFIG_DIR: configDir(), CATHERD_DATA_DIR: dataDir() },
+  };
+}
+
+export function codexRoleMcpArgs(context: RoleMcpContext): string[] {
+  const config = roleMcpConfig(context);
+  const prefix = `mcp_servers.${ROLE_MCP_SERVER}`;
+  const settings = [
+    `${prefix}.command=${JSON.stringify(config.command)}`,
+    `${prefix}.args=${JSON.stringify(config.args)}`,
+    ...Object.entries(config.env).map(([key, value]) => `${prefix}.env.${key}=${JSON.stringify(value)}`),
+    `${prefix}.enabled=true`,
+    // required: a role without its run tools cannot work; doctor's probe shows it starts well inside the timeout
+    `${prefix}.required=true`,
+    `${prefix}.startup_timeout_sec=${ROLE_MCP_STARTUP_SEC}`,
+    `${prefix}.enabled_tools=${JSON.stringify(roleMcpTools(context.role))}`,
+    ...roleMcpTools(context.role).map((tool) => `${prefix}.tools.${tool}.approval_mode="approve"`),
+  ];
+  return settings.flatMap((setting) => ["-c", setting]);
+}

package/src/infra/slopalytics.ts

@@ -0,0 +1,30 @@
+import { existsSync } from "node:fs";
+import { parseProxy, type IndexFile, IndexFileSchema, type SlopRecord } from "../domain/slopalytics.ts";
+import { readVersioned, writeJsonAtomic } from "./store.ts";
+
+export const SLOPALYTICS_URL = "https://slopalyticsproxy.vercel.app";
+
+/** The proxy's answer, parsed. Throws on a transport error, a non-2xx, or an answer with no indexed pair. */
+export async function fetchProxy(
+  o: { fetch?: typeof fetch; timeoutMs?: number } = {},
+): Promise<{ fetchedAt: string | null; models: SlopRecord[] }> {
+  const res = await (o.fetch ?? fetch)(SLOPALYTICS_URL, {
+    signal: AbortSignal.timeout(o.timeoutMs ?? 15_000),
+  });
+  if (!res.ok) throw new Error(`http ${res.status}`);
+  const parsed = parseProxy(await res.json());
+  if (parsed.models.length === 0) throw new Error("no indexed pairs");
+  return parsed;
+}
+
+/** The cached index, or null when there is none or it cannot be read (the next refresh rewrites it). */
+export function readIndexFile(file: string): IndexFile | null {
+  if (!existsSync(file)) return null;
+  try {
+    return readVersioned(file, IndexFileSchema, 1);
+  } catch {
+    return null;
+  }
+}
+
+export const writeIndexFile = (file: string, f: IndexFile): void => writeJsonAtomic(file, f, { mode: 0o600 });

package/src/infra/store.ts

@@ -150,9 +150,14 @@ function endsMidLine(file: string): boolean {
  * tail left by an earlier crash is ended first, in the same write, so the new row stays whole.
  */
 export function appendJsonl(file: string, value: unknown): void {
+  appendJsonlBatch(file, [value]);
+}
+
+/** Several rows in one write, so a crash leaves all of them or at worst a torn last one. */
+export function appendJsonlBatch(file: string, values: unknown[]): void {
   ensurePrivateDir(dirname(file));
   const prefix = endsMidLine(file) ? "\n" : "";
-  appendPrivate(file, `${prefix}${JSON.stringify(value)}\n`);
+  appendPrivate(file, prefix + values.map((v) => `${JSON.stringify(v)}\n`).join(""));
 }
 
 const madePrivate = new Set<string>();