@staix/agent-hub 0.12.4 → 0.12.5

package/src/hub/child-process.ts

@@ -11,13 +11,108 @@ export function childEnv(source: NodeJS.ProcessEnv = process.env): NodeJS.Proces
   return env;
 }
 
+/** One process: its identity is the pid with its start time (`lstart`, which exec keeps); group and command are evidence. */
+export interface ProcRow {
+  pid: number;
+  ppid: number;
+  pgid: number;
+  started: string;
+  command: string;
+}
+
+/**
+ * `ps -axo pid=,ppid=,pgid=,stat=,lstart=,command=` output (unlimited width when not on a terminal), read in the C
+ * locale. A zombie is left out: it has exited, and only its parent's wait is missing.
+ */
+export function parseProcessTable(text: string): ProcRow[] {
+  return text.split("\n").flatMap((line) => {
+    const m = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\w{3} \w{3} [ \d]\d \d\d:\d\d:\d\d \d{4})\s+(.*)$/.exec(line);
+    return m && !m[4]!.startsWith("Z") ? [{ pid: Number(m[1]), ppid: Number(m[2]), pgid: Number(m[3]), started: m[5]!, command: m[6]! }] : [];
+  });
+}
+
+const PS = ["ps", "-axo", "pid=,ppid=,pgid=,stat=,lstart=,command="];
+// `lstart` is local time: one zone for every reader, or a reader in another zone sees every identity as changed.
+const PS_OPTIONS = { env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, detached: true } as const;
+/** The rows of one read, or undefined when they do not show the reader: a parse that found nothing is not a table. */
+const own = (text: string) => { const rows = parseProcessTable(text); return rows.some((row) => row.pid === process.pid) ? rows : undefined; };
+
+/**
+ * The process table, or undefined when it cannot be read or does not show this process: a parse that found nothing
+ * (a localized `lstart`, a changed `ps`) is not an empty table. Read with LC_ALL=C for that reason (and TZ=UTC, so
+ * start times compare across readers), by a `ps` in a
+ * process group of its own (a Ctrl-C to the caller's group would kill it), bounded, and tried twice.
+ */
+export function processTable(): ProcRow[] | undefined {
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const r = Bun.spawnSync(PS, { ...PS_OPTIONS, stdout: "pipe", stderr: "pipe", timeout: 10_000 });
+      const rows = r.exitCode === 0 ? own(r.stdout.toString()) : undefined;
+      if (rows) return rows;
+    } catch {
+      // tried again below
+    }
+  }
+  return undefined;
+}
+
+/** `processTable`, without blocking the event loop: a hub stop reads the table many times while it keeps serving. */
+export async function readProcessTable(): Promise<ProcRow[] | undefined> {
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const p = Bun.spawn(PS, { ...PS_OPTIONS, stdout: "pipe", stderr: "ignore" });
+      const timer = setTimeout(() => p.kill("SIGKILL"), 10_000);
+      const [text, code] = await Promise.all([new Response(p.stdout).text(), p.exited]).finally(() => clearTimeout(timer));
+      const rows = code === 0 ? own(text) : undefined;
+      if (rows) return rows;
+    } catch {
+      // tried again below
+    }
+  }
+  return undefined;
+}
+
+/** Every descendant of `pid` in `rows`, by parent links. */
+export function descendantsOf(rows: ProcRow[], pid: number): ProcRow[] {
+  const out: ProcRow[] = [];
+  const parents = new Set([pid]);
+  for (let grew = true; grew; ) {
+    grew = false;
+    for (const row of rows) {
+      if (parents.has(row.ppid) && !parents.has(row.pid)) {
+        parents.add(row.pid);
+        out.push(row);
+        grew = true;
+      }
+    }
+  }
+  return out;
+}
+
 /**
  * Stop a child owned by this hub and confirm that it exited. A timeout is an
  * incomplete shutdown, even after SIGKILL, because callers must not reuse a
  * port or claim ownership while the old process may still be alive.
+ *
+ * `group`: the child was spawned `detached` and leads its own process group; see `stopGroup`. `table` replaces the
+ * process table read (tests).
  */
-export async function stopOwnedProcess(proc: ChildProcess, termMs = 1_000, killMs = 2_000): Promise<void> {
-  if (proc.exitCode !== null || proc.signalCode !== null || proc.pid === undefined) return;
+export async function stopOwnedProcess(proc: ChildProcess, { termMs = 1_000, killMs = 2_000, group = false, table = readProcessTable }: { termMs?: number; killMs?: number; group?: boolean; table?: () => ProcRow[] | undefined | Promise<ProcRow[] | undefined> } = {}): Promise<void> {
+  if (proc.exitCode !== null || proc.signalCode !== null || proc.pid === undefined) {
+    // ponytail: a leader that exited before the stop (its launcher killed from outside) is not swept, as its pid may be
+    // reused once its group empties; its pipes are dropped so a survivor cannot keep the hub alive, and a group that
+    // still has members fails the stop instead of reading as done. Record the leader's start time at spawn to sweep it.
+    if (!group || proc.pid === undefined) return;
+    dropPipes(proc);
+    if (emptied.has(proc)) return; // its group was seen gone since: the id may be someone else's now
+    // A pid is not given out while a group with that id exists: a live process with the leader's pid means the group was
+    // emptied and the id is someone else's now. Members without it are what the leader left (zombies are not listed).
+    const rows = await table();
+    const members = rows ? (rows.some((r) => r.pid === proc.pid) ? [] : rows.filter((r) => r.pgid === proc.pid)) : undefined;
+    if (members ? members.length : !groupGone(proc.pid)) throw new Error(`owned child ${proc.pid} exited before the stop and its process group still has members${members ? ` (${members.map((r) => r.pid).join(", ")})` : ""}: not signalled; stop them to restart it`);
+    return;
+  }
+  if (group) return stopGroup(proc, proc.pid, termMs, killMs, table);
 
   const waitForExit = (timeoutMs: number): Promise<boolean> =>
     new Promise((resolve) => {
@@ -53,3 +148,129 @@ export async function stopOwnedProcess(proc: ChildProcess, termMs = 1_000, killM
   if (proc.exitCode !== null || proc.signalCode !== null || (await waitForExit(killMs))) return;
   throw new Error(`owned child ${proc.pid} shutdown incomplete after SIGKILL`);
 }
+
+const dropPipes = (proc: ChildProcess) => {
+  proc.stdout?.destroy();
+  proc.stderr?.destroy();
+};
+
+const emptied = new WeakSet<ChildProcess>();
+/**
+ * Follows the group of a child spawned `detached` after the child exits, until the group is gone: a later stop then
+ * never inspects the group id, which can be reused once the group is empty (issue #113).
+ */
+export function trackGroup(proc: ChildProcess): void {
+  proc.once("exit", () => {
+    const pid = proc.pid;
+    if (pid === undefined) return;
+    const check = () => { if (groupGone(pid)) emptied.add(proc); else setTimeout(check, 1_000).unref(); };
+    check();
+  });
+}
+
+/** Whether no process is left in group `pgid` (macOS answers EPERM for a group of zombies). */
+function groupGone(pgid: number): boolean {
+  try { process.kill(-pgid, 0); return false; } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    return code === "ESRCH" || (code === "EPERM" && process.platform === "darwin");
+  }
+}
+
+/**
+ * Stops a child that leads its own process group, with everything it started (issue #113). A launcher that forwards
+ * signals to a native child (Codex's `codex.js`) otherwise dies alone at SIGKILL while its child, still at work, is
+ * re-parented to init with the launcher's pipes and keeps the hub alive; and the native child starts processes that
+ * lead groups of their own (Codex's MCP servers and tool commands), which no group signal reaches.
+ *
+ * SIGTERM to the group, a grace period in which what the tree starts is recorded while its parent still runs, then
+ * freeze, enumerate, kill: what is left is stopped (SIGSTOP) before the table is read again, because a stopped process
+ * starts nothing, so that read sees all of it; then SIGKILL. A snapshot taken while the tree runs misses what it starts
+ * next (three review rounds found such a gap). Done means the table shows none of it; without any table, the group's
+ * own answer (ESRCH, or EPERM on macOS for a group of zombies) is used, which cannot see groups below it.
+ */
+async function stopGroup(proc: ChildProcess, pid: number, termMs: number, killMs: number, table: () => ProcRow[] | undefined | Promise<ProcRow[] | undefined>): Promise<void> {
+  const key = (r: { pid: number; started: string }) => `${r.pid}@${r.started}`;
+  const found = new Map<string, ProcRow>();
+  const exited = () => proc.exitCode !== null || proc.signalCode !== null;
+  let leader: ProcRow | undefined;
+  const live = (rows: ProcRow[], f: ProcRow) => rows.some((r) => r.pid === f.pid && r.started === f.started);
+  /** Reads the table and records what runs below the leader or anything recorded that still runs. */
+  const look = async (): Promise<ProcRow[] | undefined> => {
+    const rows = await table();
+    if (!rows) return undefined;
+    if (!leader && !exited()) leader = rows.find((r) => r.pid === pid); // never a pid taken over after the leader exited
+    const roots = [...(leader && live(rows, leader) ? [leader] : []), ...[...found.values()].filter((f) => live(rows, f))];
+    for (const root of roots) for (const r of descendantsOf(rows, root.pid)) found.set(key(r), r);
+    // A group is followed while it is known to be the same one: its leader or a recorded member still in it.
+    for (const id of [pid, ...[...found.values()].filter((f) => f.pgid === f.pid).map((f) => f.pid)]) {
+      // Current rows, never a recorded pgid: a member may have left the group since it was recorded.
+      const known = (id === pid && leader && live(rows, leader)) || rows.some((r) => r.pgid === id && r.pid !== id && found.has(key(r))) || rows.some((r) => r.pid === id && r.pgid === id && found.has(key(r)));
+      if (known) for (const r of rows) if (r.pgid === id && r.pid !== pid) found.set(key(r), r);
+    }
+    return rows;
+  };
+  const left = (rows: ProcRow[]) => [...found.values()].filter((f) => live(rows, f));
+  /**
+   * Running in the leader's group, or the group of a recorded process, without being recorded: after an empty moment the
+   * group id can be someone else's, so it is never signalled, but the stop is not done while it runs.
+   */
+  const unproven = (rows: ProcRow[]) => {
+    const ids = new Set([pid, ...[...found.values()].filter((f) => f.pgid === f.pid).map((f) => f.pid)]);
+    return rows.filter((r) => ids.has(r.pgid) && r.pid !== process.pid && !found.has(key(r)) && !(leader && r.pid === leader.pid && r.started === leader.started));
+  };
+  // Errors are not results here: the table read back decides. A sent signal says the target existed at that moment.
+  const signal = (target: number, sig: NodeJS.Signals) => { try { process.kill(target, sig); return true; } catch { return false; } };
+  const each = (rows: ProcRow[], sig: NodeJS.Signals) => rows.filter((r) => signal(r.pgid === r.pid ? -r.pid : r.pid, sig));
+  /** A read that takes longer than `ms` is given up: a frozen tree must not wait on a slow `ps` for its SIGKILL. */
+  const within = (ms: number) => Promise.race([look(), Bun.sleep(ms).then(() => undefined)]);
+  // What leads a group of its own (an MCP server, a tool command) gets a SIGTERM of its own and the same grace period:
+  // a git process stopped by SIGKILL leaves its index lock behind.
+  const termed = new Set<string>();
+  const term = (rows: ProcRow[] | undefined) => {
+    for (const r of rows ? left(rows) : []) if (r.pgid === r.pid && !termed.has(key(r))) { termed.add(key(r)); signal(-r.pid, "SIGTERM"); }
+  };
+  try {
+    const first = await look();
+    if (!exited()) signal(-pid, "SIGTERM"); // a reaped leader's group id is not proven any more
+    term(first);
+    // The grace period, for the leader and for what it started: what they start meanwhile is recorded while its parent
+    // still runs.
+    for (const end = Date.now() + termMs; Date.now() < end; ) {
+      await Bun.sleep(100);
+      const rows = await look();
+      term(rows);
+      if (exited() && rows && !left(rows).length) break;
+    }
+    // Freeze, enumerate, kill: a stopped process starts nothing, so a read after the freeze sees all of it.
+    for (const end = Date.now() + killMs; ; ) {
+      const rows = await look();
+      const rest = rows ? left(rows) : undefined;
+      const strangers = rows ? unproven(rows) : [];
+      if (exited() && !strangers.length && (rest ? !rest.length : !found.size && groupGone(pid))) return;
+      if (Date.now() >= end) {
+        throw new Error(`owned child ${pid}: ${rest ? `${rest.length + strangers.length} process(es) of its group or below it still running${strangers.length ? `, ${strangers.length} not proven its own and left alone` : ""}` : "the process table cannot be read to confirm what it started is gone"}`);
+      }
+      // What is stopped is killed or continued, never left frozen: the group by whether its STOP was sent (its id stays
+      // reserved while a member lives), each process by whether the read after the freeze still shows it. What a read
+      // after the freeze shows for the first time (started between the read and the STOPs, in a group of its own) is
+      // frozen too, and read again, before anything is killed.
+      const groupStopped = !exited() && signal(-pid, "SIGSTOP");
+      const stopped = new Map<string, ProcRow>();
+      let frozen: ProcRow[] | undefined;
+      for (let todo = rest ?? [], round = 0; ; round++) {
+        for (const r of each(todo, "SIGSTOP")) stopped.set(key(r), r);
+        frozen = await within(1_000);
+        todo = frozen ? left(frozen).filter((r) => !stopped.has(key(r))) : [];
+        if (!todo.length || round >= 4) break;
+      }
+      // Without a read after the freeze, only what the STOP reached is killed: a stopped process keeps its pid.
+      const kill = frozen ? left(frozen) : [...stopped.values()];
+      each(kill, "SIGKILL");
+      if (groupStopped) signal(-pid, "SIGKILL");
+      for (const r of stopped.values()) if (!kill.some((k) => k.pid === r.pid && k.started === r.started)) signal(r.pgid === r.pid ? -r.pid : r.pid, "SIGCONT");
+      await Bun.sleep(50);
+    }
+  } finally {
+    dropPipes(proc); // a process that left the tree between two reads may still hold them: it must not keep the hub alive
+  }
+}

package/src/hub/daemon.ts

@@ -1,5 +1,5 @@
 import { createHash, randomBytes, randomUUID } from "node:crypto";
-import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { appendFileSync, chmodSync, existsSync, fstatSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { homedir } from "node:os";
 import { spawn } from "node:child_process";
@@ -28,7 +28,7 @@ import { Board, CLASSES, type Task, type TaskClass } from "./board.ts";
 import { Budget, claudeWindows, codexWindows, DEFAULT_BUDGET, type BudgetConfig } from "./budget.ts";
 import { HUB } from "./envelope.ts";
 import { trimToTokens } from "../memory/recall.ts";
-import { closeSync, openSync, readSync, statSync } from "node:fs";
+import { closeSync, constants as fsConstants, openSync, readSync, statSync } from "node:fs";
 import { basename, isAbsolute } from "node:path";
 import { realPath } from "./project.ts";
 import type { BusEvent } from "./bus.ts";
@@ -539,15 +539,15 @@ export async function startDaemon(opts: DaemonOptions) {
       if (!factsOn()) return undefined;
       let offered: ReturnType<Facts["due"]>;
       try { offered = facts.due(peer, undefined, true); } catch (error) { log(`integration facts for ${peer}: ${(error as Error).message}`); return undefined; }
-      if (offered) event({ type: "fact", peer, id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, bytes: offered.bytes, via: "done" });
+      if (offered) event({ type: "fact", peer, id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, named: offered.named, bytes: offered.bytes, via: "done" });
       return offered ? { id: offered.id, text: offered.text } : undefined;
     },
     ackFacts: (peer, id) => acked(peer, id, "done"),
     factsCurrent: (peer) => !factsOn() || facts.current(peer),
-    since: hubStartedAt,
+    splitProfile: (peer) => splitProfile(peer),
     recordCohort: (c) => event({ type: "cohort", ...c }),
     // The trace holds peer names and numbers only (never task text): it is the inputs a later check of the prediction needs.
-    recordSplit: (task, p) => event({ type: "split", task, verdict: p.verdict, ...(p.single ? { single: p.single, splitS: p.splitS, singleS: p.singleS } : {}), ...(p.verdict === "unknown" ? { reason: p.trace.at(-1)!.replace(/^ {2}unknown: /, "").slice(0, 200) } : {}), trace: p.trace.slice(1).map((l) => l.trim()).slice(0, 10) }),
+    recordSplit: (task, p, where) => event({ type: "split", task, where, verdict: p.verdict, ...(p.single ? { single: p.single, splitS: p.splitS, singleS: p.singleS } : {}), ...(p.verdict === "unknown" ? { reason: p.trace.at(-1)!.replace(/^ {2}unknown: /, "").slice(0, 200) } : {}), trace: p.trace.slice(1).map((l) => l.trim()).slice(0, 10) }),
     failing: () => bus.failingPeers(),
     held: () => Object.fromEntries(bus.knownPeers().flatMap((peer) => { const hold = queueHold(peer); return hold ? [[peer, hold]] : []; })),
   });
@@ -1018,6 +1018,46 @@ export async function startDaemon(opts: DaemonOptions) {
       };
     } catch { return {}; }
   };
+  /**
+   * Claude Code's version, from the last row of its transcript that names one (its rows carry it): the last MiB is read,
+   * opened without blocking and only as a regular file, and the last version found is kept for a tail of big rows.
+   */
+  const claudeVersions = new Map<string, string>();
+  const claudeVersion = (): string | undefined => {
+    const path = claudeSession().transcriptPath;
+    if (!path) return undefined;
+    let fd: number | undefined;
+    try {
+      fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NONBLOCK);
+      const st = fstatSync(fd);
+      if (!st.isFile()) return claudeVersions.get(path);
+      const buf = Buffer.alloc(Math.min(st.size, 1024 * 1024));
+      readSync(fd, buf, 0, buf.length, st.size - buf.length);
+      for (const line of buf.toString("utf8").split("\n").reverse()) {
+        try {
+          const version = JSON.parse(line)?.version;
+          const found = typeof version === "string" ? /^\d+\.\d+\.\d+(?:-[\w.]+)?/.exec(version)?.[0] : undefined;
+          if (found) {
+            claudeVersions.set(path, found);
+            return found;
+          }
+        } catch { /* the cut first line, or not JSON */ }
+      }
+    } catch { /* no transcript yet */ } finally { if (fd !== undefined) closeSync(fd); }
+    return claudeVersions.get(path);
+  };
+  /**
+   * A peer's split profile (issue #109): the hub's version, its agent's, and the hook profile, which is the hub's own (a
+   * turn-free project runs the facts hooks in Claude and steers facts into Codex). Undefined while a version is unknown.
+   * ponytail: the user's and plugins' hooks are not seen here, and only Claude and Codex report a version; read hooks
+   * from the native records, as the benchmark ledger does, and add other agents' versions when a pair needs them.
+   */
+  function splitProfile(peer: PeerId): string | undefined {
+    try {
+      const version = peer === "codex" ? (bus.peers.get("codex") as { version?: string } | undefined)?.version : peer === "claude" ? claudeVersion() : undefined;
+      return version ? `hub ${VERSION}; ${peer} ${version}; ${turnFree() ? "turn-free" : "advisory"}` : undefined; // the regime in force: a PII task suspends turn-free
+    } catch { return undefined; } // asked before the daemon finished starting
+  }
   /** Claude persists projects/<slug>/<sessionId>.jsonl only with the first turn. Prefer the
    * path Claude itself reported through the status line; fall back to the slug computation. */
   const claudeTranscriptPersisted = (sessionId: string): boolean => {
@@ -1552,7 +1592,7 @@ export async function startDaemon(opts: DaemonOptions) {
             // Refused, it never went in: not an unread offer, and the next boundary offers it again. Unanswered, it may
             // have: its readback can still come.
             if (outcome === "refused") facts.drop("codex", offered.id);
-            event({ type: "fact", peer: "codex", id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, bytes: offered.bytes, via: "steer", ms, ...(outcome === "accepted" ? { rttMs: Math.round(performance.now() - sent) } : {}), accepted: outcome === "accepted", ...(outcome === "unanswered" ? { unanswered: true } : {}), ...(offered.probe ? { probe: true } : {}), ...(offered.coverage ? { coverage: true } : {}) });
+            event({ type: "fact", peer: "codex", id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, named: offered.named, bytes: offered.bytes, via: "steer", ms, ...(outcome === "accepted" ? { rttMs: Math.round(performance.now() - sent) } : {}), accepted: outcome === "accepted", ...(outcome === "unanswered" ? { unanswered: true } : {}), ...(offered.probe ? { probe: true } : {}), ...(offered.coverage ? { coverage: true } : {}) });
           });
         },
         appPort: opts.codexAppPort,
@@ -2143,7 +2183,7 @@ export async function startDaemon(opts: DaemonOptions) {
             return void reply({ t: "facts", ok: true });
           }
           const offered = offerFor(peer, toolUseId);
-          if (offered) event({ type: "fact", peer, id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, bytes: offered.bytes, via: "hook", ms: Math.round(performance.now() - started), ...(typeof msg.startedMs === "number" && Number.isFinite(msg.startedMs) ? { hookMs: Math.round(msg.startedMs) } : {}), ...(offered.probe ? { probe: true } : {}), ...(offered.coverage ? { coverage: true } : {}) });
+          if (offered) event({ type: "fact", peer, id: offered.id, files: offered.files, plans: offered.plans, unknown: offered.unknown, named: offered.named, bytes: offered.bytes, via: "hook", ms: Math.round(performance.now() - started), ...(typeof msg.startedMs === "number" && Number.isFinite(msg.startedMs) ? { hookMs: Math.round(msg.startedMs) } : {}), ...(offered.probe ? { probe: true } : {}), ...(offered.coverage ? { coverage: true } : {}) });
           return void reply({ t: "facts", ok: true, ...(offered ? { text: offered.text, id: offered.id } : {}) });
         } catch (error) {
           log(`facts for ${c.peer}: ${(error as Error).message}`);

package/src/hub/events.ts

@@ -12,10 +12,10 @@ export type HubEvent =
   | { type: "overflow" | "undeliverable"; id: string; from: string; peer: string }
   | { type: "stale"; id: string; from: string; peer: string; task?: string }
   | { type: "quiet"; id: string; from: string; peers: string[] }
-  | { type: "fact"; peer: string; id: string; files: number; plans: number; unknown: number; bytes: number; via: "hook" | "steer" | "done"; ms?: number; hookMs?: number; rttMs?: number; accepted?: boolean; unanswered?: boolean; probe?: boolean; coverage?: boolean }
+  | { type: "fact"; peer: string; id: string; files: number; plans: number; unknown: number; named: number; bytes: number; via: "hook" | "steer" | "done"; ms?: number; hookMs?: number; rttMs?: number; accepted?: boolean; unanswered?: boolean; probe?: boolean; coverage?: boolean }
   | { type: "fact_ack"; peer: string; id: string; via: string; ms: number }
   | { type: "capability"; peer: string; state: "verified" | "lost"; via?: string }
-  | { type: "split"; task: number; verdict: "split" | "single" | "unknown"; single?: string; splitS?: number; singleS?: number; reason?: string; trace?: string[] }
+  | { type: "split"; task: number; where?: "routing" | "cohort"; verdict: "split" | "single" | "unknown"; single?: string; splitS?: number; singleS?: number; reason?: string; trace?: string[] }
   | { type: "state"; peer: string; state: string }
   | { type: "turn_start"; peer: string; turn: string }
   | { type: "turn_end"; peer: string; turn: string; ms: number; tokens?: number; files?: number; snapshotMs?: number }