@staix/agent-hub 0.4.0 → 0.5.0

package/src/hub/lifecycle.ts

@@ -7,6 +7,7 @@ import { ControlClient, PROTOCOL, readControl } from "./control-client.ts";
 import { startDaemon } from "./daemon.ts";
 import { Registry, type Project } from "./registry.ts";
 import { CODEX_APP, CODEX_PROXY, CONTROL, SWITCHYARD } from "./ports.ts";
+import { assertLifecycleAvailable, recoveryLock } from "./recovery-store.ts";
 
 export type ProjectInspection = {
   state: "running" | "stopped" | "stopping" | "unavailable" | "incompatible" | "missing" | "starting";
@@ -85,6 +86,7 @@ async function available(base: number): Promise<boolean> {
 
 /** Called only in the detached daemon process. A competing starter exits without touching state. */
 export async function runProjectDaemon(project: Project, unattended = false): Promise<void> {
+  assertLifecycleAvailable();
   const registry = new Registry();
   const instanceId = randomUUID();
   let claimed = false;
@@ -122,6 +124,7 @@ export async function runProjectDaemon(project: Project, unattended = false): Pr
 }
 
 export async function startProject(project: Project, options: { unattended?: boolean; env?: NodeJS.ProcessEnv } = {}): Promise<any> {
+  assertLifecycleAvailable();
   const registry = new Registry();
   try {
     const current = registry.get(project.id);
@@ -138,10 +141,14 @@ export async function startProject(project: Project, options: { unattended?: boo
       renameSync(temporary, marker);
       const log = openSync(join(project.stateDir, "hub.log"), "a");
       try {
+        const launchEnv = { ...(options.env ?? process.env) };
+        // Native sessions can outlive the operation which launched them. An expired
+        // operation ID must not make a later ordinary startup load a retired snapshot.
+        if (launchEnv.AGENTHUB_RECOVERY_OPERATION !== recoveryLock()) delete launchEnv.AGENTHUB_RECOVERY_OPERATION;
         const child = spawn(process.execPath, [join(import.meta.dir, "../cli/main.ts"), "--project", project.root, "daemon",
           ...(options.unattended ? ["--unattended"] : [])], {
           cwd: project.root, detached: true, stdio: ["ignore", log, log],
-          env: { ...(options.env ?? process.env), AGENTHUB_STATE_DIR: project.stateDir, AGENTHUB_PROJECT_DIR: project.root,
+          env: { ...launchEnv, AGENTHUB_STATE_DIR: project.stateDir, AGENTHUB_PROJECT_DIR: project.root,
             AGENTHUB_UNATTENDED: options.unattended ? "1" : "0" },
         });
         child.on("error", () => {}); // bounded readiness below reports failure with the project's log path
@@ -160,6 +167,7 @@ export async function startProject(project: Project, options: { unattended?: boo
 }
 
 export async function stopProject(project: Project, expectedInstance?: string): Promise<void> {
+  assertLifecycleAvailable();
   const inspection = await inspectProject(project);
   if (inspection.state === "stopped") return;
   if (inspection.state !== "running" && inspection.state !== "stopping") throw new Error(inspection.error ?? "hub ownership is not verified; not stopping it");

package/src/hub/peers.ts

@@ -15,6 +15,8 @@ export interface PeerAdapter {
   onState?: (state: PeerState) => void;
   /** Set by the bus. A delivery that had resolved turned out not to reach the agent: put it back. */
   onFailed?: (envs: Envelope[]) => void;
+  /** Safe restart metadata only: ids and launch parameters, never prompt/message text. */
+  recoveryMetadata?(): Record<string, unknown>;
 }
 
 export const DEFAULT_WATCHDOG_MS = 300_000;

package/src/hub/recovery-store.ts

@@ -0,0 +1,89 @@
+import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { Database } from "bun:sqlite";
+import { hubHome } from "./project.ts";
+
+export const operationIdPattern = /^[a-f0-9-]{36}$/;
+export function operationPath(id: string, home = hubHome()): string {
+  if (!operationIdPattern.test(id)) throw new Error("invalid recovery operation ID");
+  return join(home, "recovery", `${id}.json`);
+}
+
+/** Private state only. Never put tokens or transcript/message bodies in an operation receipt. */
+export function atomicPrivateJSON(file: string, value: unknown): void {
+  const temporary = `${file}.${randomUUID()}.tmp`;
+  try {
+    writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: "wx" });
+    chmodSync(temporary, 0o600);
+    renameSync(temporary, file);
+  } finally { rmSync(temporary, { force: true }); }
+}
+
+export function readOperation<T>(id: string, home = hubHome()): T {
+  return JSON.parse(readFileSync(operationPath(id, home), "utf8")) as T;
+}
+
+export function writeOperation(id: string, value: unknown, home = hubHome()): void {
+  mkdirSync(join(home, "recovery"), { recursive: true, mode: 0o700 });
+  atomicPrivateJSON(operationPath(id, home), value);
+}
+
+export function recoveryLock(home = hubHome()): string | undefined {
+  const file = join(home, "recovery.lock");
+  try {
+    const data = JSON.parse(readFileSync(file, "utf8"));
+    if (!operationIdPattern.test(data.operationId)) throw new Error("invalid recovery lock");
+    return data.operationId;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+    throw new Error("recovery lock is unreadable; inspect it before changing runtimes", { cause: error });
+  }
+}
+
+export function acquireRecoveryLock(id: string, home = hubHome()): void {
+  operationPath(id, home);
+  mkdirSync(home, { recursive: true, mode: 0o700 });
+  const existing = recoveryLock(home);
+  if (existing === id) return;
+  try { writeFileSync(join(home, "recovery.lock"), JSON.stringify({ operationId: id }), { mode: 0o600, flag: "wx" }); }
+  catch { throw new Error(`another recovery operation owns this machine: ${recoveryLock(home) ?? "unknown"}`); }
+}
+
+export function releaseRecoveryLock(id: string, home = hubHome()): void {
+  if (recoveryLock(home) !== id) throw new Error("recovery lock ownership changed");
+  rmSync(join(home, "recovery.lock"));
+}
+
+/** Used by CLI and manager lifecycle paths. A detached operation carries its own ID. */
+export function assertLifecycleAvailable(): void {
+  const owner = recoveryLock();
+  if (owner && process.env.AGENTHUB_RECOVERY_OPERATION !== owner) {
+    throw new Error(`recovery operation ${owner} is active; use ahub recovery status|resume ${owner}`);
+  }
+}
+
+/** An exclusive runner claim. Never steal a live or uncertain owner on resume. */
+export function claimRunner(id: string, home = hubHome()): () => void {
+  const path = `${operationPath(id, home)}.runner.db`;
+  const db = new Database(path, { create: true });
+  chmodSync(path, 0o600);
+  const nonce = randomUUID();
+  try {
+    db.run("PRAGMA busy_timeout = 3000");
+    db.run("CREATE TABLE IF NOT EXISTS runner (slot INTEGER PRIMARY KEY, pid INTEGER NOT NULL, nonce TEXT NOT NULL)");
+    db.transaction(() => {
+      const row = db.query("SELECT pid FROM runner WHERE slot = 1").get() as { pid: number } | null;
+      if (row) {
+        if (!Number.isSafeInteger(row.pid) || row.pid < 1) throw new Error("invalid recovery runner ownership");
+        try { process.kill(row.pid, 0); throw new Error(`recovery runner ${row.pid} is still alive`); }
+        catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; }
+      }
+      db.query("INSERT OR REPLACE INTO runner (slot, pid, nonce) VALUES (1, ?, ?)").run(process.pid, nonce);
+    }).immediate();
+  } catch (error) { db.close(); throw error; }
+  return () => {
+    db.query("DELETE FROM runner WHERE slot = 1 AND nonce = ?").run(nonce);
+    db.close();
+  };
+}

package/src/hub/restart.ts

@@ -0,0 +1,97 @@
+import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import type { BusSnapshot } from "./bus.ts";
+import type { PeerId, PeerState } from "./envelope.ts";
+import { MAX_HOP } from "./envelope.ts";
+
+export const RESTART_SCHEMA_VERSION = 1;
+
+export type RecoveryPhase = "preparing" | "prepared" | "restored" | "released";
+
+export interface RestartPeerSnapshot {
+  id: PeerId;
+  state: PeerState;
+  queueIds: string[];
+  launch?: Record<string, unknown>;
+  /** Safe native correlation metadata. Never include prompt/message text here. */
+  threadId?: string;
+  sessionId?: string;
+}
+
+export interface RestartSnapshot {
+  schemaVersion: 1;
+  projectRoot: string;
+  projectId: string;
+  sourceInstanceId: string;
+  operationId: string;
+  committedAt: number;
+  bus: BusSnapshot;
+  manualPaused: PeerId[];
+  peers: RestartPeerSnapshot[];
+  integrity?: {
+    queues: Record<string, string[]>;
+    manualPaused: PeerId[];
+    boardDigest: string;
+    budgetDigest: string;
+  };
+}
+
+export interface RestartReadExpectation {
+  projectRoot: string;
+  projectId: string;
+  operationId?: string;
+}
+
+export function restartPath(stateDir: string): string {
+  return join(stateDir, "restart.json");
+}
+
+export function releasedRestartPath(stateDir: string, operationId: string): string {
+  const safe = createHash("sha256").update(operationId).digest("hex");
+  return join(stateDir, `restart.released.${safe}.json`);
+}
+
+/** Read only a matching controlled-restart snapshot. Unknown or malformed state is ignored and must block recovery upstream. */
+export function readRestartSnapshot(stateDir: string, expected: RestartReadExpectation): RestartSnapshot | undefined {
+  let parsed: unknown;
+  try { parsed = JSON.parse(readFileSync(restartPath(stateDir), "utf8")); } catch { return undefined; }
+  if (!parsed || typeof parsed !== "object") return undefined;
+  const s = parsed as Partial<RestartSnapshot>;
+  if (s.schemaVersion !== RESTART_SCHEMA_VERSION || s.projectRoot !== expected.projectRoot || s.projectId !== expected.projectId || typeof s.operationId !== "string" || !s.operationId || typeof s.sourceInstanceId !== "string" || !s.sourceInstanceId || !s.bus) return undefined;
+  if (expected.operationId !== undefined && s.operationId !== expected.operationId) return undefined;
+  const peerId = (value: unknown): value is string => typeof value === "string" && /^(?:hub|user|[a-z][a-z0-9-]{0,31})$/.test(value);
+  const isEnvelope = (value: unknown, seen = false): boolean => {
+    if (!value || typeof value !== "object") return false;
+    const e = value as Record<string, unknown>;
+    return typeof e.id === "string" && e.id.length > 0 && e.id.length <= 256 && typeof e.trace === "string" && e.trace.length > 0 && e.trace.length <= 256 && peerId(e.from) && typeof e.body === "string" && Number.isSafeInteger(e.hop) && (e.hop as number) >= 0 && (seen || (e.hop as number) <= MAX_HOP) && Number.isFinite(e.ts) && ["chat", "task", "review", "status", "budget", "presence"].includes(e.kind as string) && ["important", "status", "fyi"].includes(e.priority as string) && (e.to === undefined || (Array.isArray(e.to) && e.to.every(peerId)));
+  };
+  const bus = s.bus as Partial<BusSnapshot>;
+  if (bus.schemaVersion !== 1 || !bus.queues || typeof bus.queues !== "object" || !bus.prefaces || typeof bus.prefaces !== "object" || !Array.isArray(bus.seen) || !bus.attempts || typeof bus.attempts !== "object" || !Array.isArray(bus.withdrawn)) return undefined;
+  if (!Object.keys(bus.queues).every(peerId) || !Object.keys(bus.prefaces).every(peerId) || !Object.values(bus.queues).every((queue) => Array.isArray(queue) && queue.every((e) => isEnvelope(e))) || !Object.values(bus.prefaces).every((e) => isEnvelope(e)) || !bus.seen.every((e) => isEnvelope(e, true)) || !bus.withdrawn.every((id) => typeof id === "string" && id.length > 0) || !Object.values(bus.attempts).every((n) => Number.isSafeInteger(n) && n > 0)) return undefined;
+  if (!Array.isArray(s.manualPaused) || !s.manualPaused.every((id) => typeof id === "string") || !Array.isArray(s.peers) || !s.peers.every((peer) => peer && typeof peer === "object" && typeof peer.id === "string" && ["idle", "busy", "paused", "offline"].includes(peer.state as string) && Array.isArray(peer.queueIds) && peer.queueIds.every((id) => typeof id === "string"))) return undefined;
+  if (s.integrity !== undefined && (!s.integrity || typeof s.integrity !== "object" || typeof s.integrity.boardDigest !== "string" || typeof s.integrity.budgetDigest !== "string" || !Array.isArray(s.integrity.manualPaused) || !s.integrity.manualPaused.every((id) => typeof id === "string") || !s.integrity.queues || typeof s.integrity.queues !== "object" || !Object.values(s.integrity.queues).every((ids) => Array.isArray(ids) && ids.every((id) => typeof id === "string")))) return undefined;
+  return s as RestartSnapshot;
+}
+
+/** Atomic, owner-readable recovery state. The file never carries delivery bodies outside this 0600 path. */
+export function writeRestartSnapshot(stateDir: string, snapshot: RestartSnapshot): void {
+  mkdirSync(stateDir, { recursive: true });
+  const file = restartPath(stateDir);
+  const tmp = `${file}.${snapshot.sourceInstanceId}.tmp`;
+  writeFileSync(tmp, `${JSON.stringify(snapshot)}\n`, { mode: 0o600 });
+  chmodSync(tmp, 0o600);
+  renameSync(tmp, file);
+  chmodSync(file, 0o600);
+}
+
+export function removeRestartSnapshot(stateDir: string): void {
+  try { unlinkSync(restartPath(stateDir)); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+}
+
+/** Preserve an uncertain release outcome for manual reconciliation without making it authoritative on startup. */
+export function archiveRestartSnapshot(stateDir: string, operationId: string): void {
+  renameSync(restartPath(stateDir), releasedRestartPath(stateDir, operationId));
+  chmodSync(releasedRestartPath(stateDir, operationId), 0o600);
+}