@arnilo/prism-tools 0.0.0-stage → 0.13.1

package/coding/src/agent/process/sessions-host.ts

@@ -0,0 +1,222 @@
+/**
+ * Shared mutable registry for managed process sessions.
+ * Lifted out of the createProcessSessions closure so phase modules take it explicitly.
+ */
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import { createHash } from "node:crypto";
+import { isAbsolute, relative, resolve } from "node:path";
+import { DEFAULT_MAX_TERMINAL_COLUMNS, DEFAULT_MAX_TERMINAL_ROWS } from "../limits.js";
+import type { OutputAccumulator } from "../output-accumulator.js";
+import { type ProcessRecoveryBackend, ProcessRecoveryError, resolveProcessRecoveryLimits } from "./recovery.js";
+import {
+  type CodingProcessEvent,
+  type CreateProcessSessionsOptions,
+  type ProcessExitResult,
+  type ProcessPtyHandle,
+  type ProcessSandboxHandle,
+  type ProcessSession,
+  ProcessSessionError,
+  type ProcessSessionState,
+  type ProcessTerminalRequest,
+  type ResolvedProcessSessionLimits,
+  resolveProcessSessionLimits,
+} from "./types.js";
+
+/** Default TERM for PTY sessions (validated <= maxTerminalTermBytes). */
+export const DEFAULT_TERM = "xterm-256color";
+
+export function ownershipKey(
+  ownership: CreateProcessSessionsOptions["ownership"],
+  identity: CreateProcessSessionsOptions["identity"],
+): string {
+  if (ownership) {
+    return `${ownership.tenantId ?? ""}:${ownership.accountId ?? ""}:${ownership.userId ?? ""}`;
+  }
+  if (identity) {
+    return `${identity.tenantId}:${identity.accountId ?? ""}:${identity.userId ?? ""}`;
+  }
+  return "default";
+}
+
+export function commandFingerprint(command: string, args: readonly string[]): string {
+  return createHash("sha256")
+    .update(JSON.stringify([command, ...args]))
+    .digest("hex");
+}
+
+export function isInsideRoot(root: string, target: string): boolean {
+  const from = resolve(root);
+  const to = resolve(target);
+  if (to === from) return true;
+  const rel = relative(from, to);
+  return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel);
+}
+
+export function nowIso(): string {
+  return new Date().toISOString();
+}
+
+export interface SessionRecord {
+  id: string;
+  owner: string;
+  workspace: string;
+  command: string;
+  args: readonly string[];
+  commandFingerprint: string;
+  policyDecision: string;
+  startedAt: string;
+  exitedAt?: string;
+  releaseOnCancel: boolean;
+  expiresAt: number;
+  state: ProcessSessionState;
+  exitCode: number | null;
+  child?: ChildProcessWithoutNullStreams;
+  backend?: ProcessSandboxHandle;
+  pty?: ProcessPtyHandle;
+  ptyTerminal?: { columns: number; rows: number; term: string };
+  ptyResizeAt: number[];
+  pid?: number;
+  accumulator: OutputAccumulator;
+  waiters: Array<(result: ProcessExitResult) => void>;
+  /** Readers waiting for the next publication (`ProcessSession.onOutput`). */
+  outputListeners: Set<() => void>;
+  stdinClosed: boolean;
+  handle: ProcessSession;
+  /** Durable recovery bookkeeping (plan 026 Task 5); absent when durability is not configured. */
+  backendRef?: string;
+  recoveryFencingToken: number;
+  recoveryVersion: number;
+  /** Opaque lease claim token for renewal/release; never serialized, memory only. */
+  recoveryLeaseToken?: string;
+  /** Serializes durable transition writes per record so CAS order never inverts on slow stores. */
+  recoveryWriteChain?: Promise<void>;
+}
+
+export interface SessionsHost {
+  readonly workspace: string;
+  readonly limits: ResolvedProcessSessionLimits;
+  readonly defaultOwner: string;
+  readonly policy: CreateProcessSessionsOptions["policy"];
+  readonly onEvent: CreateProcessSessionsOptions["onEvent"];
+  readonly sandbox: CreateProcessSessionsOptions["sandbox"];
+  readonly ptyBackend: CreateProcessSessionsOptions["ptyBackend"];
+  readonly ptyResizeCapable: boolean;
+  readonly sessions: Map<string, SessionRecord>;
+  disposed: boolean;
+  sandboxLost: boolean;
+  readonly checkpoints: CreateProcessSessionsOptions["checkpoints"];
+  readonly leases: CreateProcessSessionsOptions["leases"];
+  readonly ownerId: CreateProcessSessionsOptions["ownerId"];
+  readonly recoveryBackend: ProcessRecoveryBackend | undefined;
+  readonly recoveryLimits: ReturnType<typeof resolveProcessRecoveryLimits>;
+  readonly durable: boolean;
+  readonly ownership: CreateProcessSessionsOptions["ownership"];
+}
+
+export function createSessionsHost(options: CreateProcessSessionsOptions): SessionsHost {
+  const checkpoints = options.checkpoints;
+  const leases = options.leases;
+  const ownerId = options.ownerId;
+  const recoveryBackend =
+    options.recoveryBackend ??
+    (options.sandbox && typeof options.sandbox.attachProcess === "function"
+      ? { attach: (ref: string) => options.sandbox!.attachProcess!(ref) }
+      : undefined);
+  const durable = checkpoints !== undefined || leases !== undefined || ownerId !== undefined || recoveryBackend !== undefined;
+  if (durable && !(checkpoints && leases && ownerId)) {
+    throw new ProcessRecoveryError(
+      "ERR_PRISM_RECOVERY_UNSUPPORTED",
+      "durable process recovery requires checkpoints, leases, and ownerId together",
+    );
+  }
+  return {
+    workspace: resolve(options.cwd),
+    limits: resolveProcessSessionLimits(options.limits),
+    defaultOwner: ownershipKey(options.ownership, options.identity),
+    policy: options.policy,
+    onEvent: options.onEvent,
+    sandbox: options.sandbox,
+    ptyBackend: options.ptyBackend,
+    ptyResizeCapable: options.ptyBackend?.capabilities?.resize === true,
+    sessions: new Map(),
+    disposed: false,
+    sandboxLost: false,
+    checkpoints,
+    leases,
+    ownerId,
+    recoveryBackend,
+    recoveryLimits: resolveProcessRecoveryLimits(options.recoveryLimits),
+    durable,
+    ownership: options.ownership,
+  };
+}
+
+/**
+ * Durable recovery seams. `createSessionsHost` already fails closed when a durable host
+ * is missing any of the three, so callers under a `host.durable` guard can take them here
+ * without non-null assertions — and a host that somehow lost one fails closed again.
+ */
+export function durableSeams(host: SessionsHost): {
+  readonly checkpoints: NonNullable<SessionsHost["checkpoints"]>;
+  readonly leases: NonNullable<SessionsHost["leases"]>;
+  readonly ownerId: NonNullable<SessionsHost["ownerId"]>;
+} {
+  const { checkpoints, leases, ownerId } = host;
+  if (!checkpoints || !leases || !ownerId) {
+    throw new ProcessRecoveryError(
+      "ERR_PRISM_RECOVERY_UNSUPPORTED",
+      "durable process recovery requires checkpoints, leases, and ownerId together",
+    );
+  }
+  return { checkpoints, leases, ownerId };
+}
+
+export function emit(host: SessionsHost, event: CodingProcessEvent): void {
+  host.onEvent?.(event);
+}
+
+export function assertNotDisposed(host: SessionsHost): void {
+  if (host.disposed) throw new ProcessSessionError("ERR_PRISM_PROCESS_STATE", "process sessions disposed");
+}
+
+export function isTerminalState(state: ProcessSessionState): boolean {
+  return state === "exited" || state === "killed" || state === "released" || state === "expired" || state === "unknown";
+}
+
+/** Wakes every reader waiting on this record: a chunk landed, or the session went terminal. */
+export function notifyOutput(record: SessionRecord): void {
+  for (const listener of [...record.outputListeners]) {
+    try {
+      listener();
+    } catch {
+      // swallowed on purpose: a reader must never fail the process that produced the output
+    }
+  }
+}
+
+export function settleWaiters(record: SessionRecord): void {
+  const result: ProcessExitResult = { exitCode: record.exitCode, state: record.state };
+  const waiters = record.waiters;
+  record.waiters = [];
+  for (const resolveWait of waiters) resolveWait(result);
+  notifyOutput(record);
+}
+
+export function resolveTerminal(
+  host: SessionsHost,
+  terminal: ProcessTerminalRequest | undefined,
+): { columns: number; rows: number; term: string } {
+  const columns = terminal?.columns ?? Math.min(DEFAULT_MAX_TERMINAL_COLUMNS, host.limits.maxTerminalColumns);
+  const rows = terminal?.rows ?? Math.min(DEFAULT_MAX_TERMINAL_ROWS, host.limits.maxTerminalRows);
+  const term = terminal?.term ?? DEFAULT_TERM;
+  if (!Number.isSafeInteger(columns) || columns < 1 || columns > host.limits.maxTerminalColumns) {
+    throw new ProcessSessionError("ERR_PRISM_PROCESS_PTY_LIMIT", `columns must be 1..${host.limits.maxTerminalColumns}`);
+  }
+  if (!Number.isSafeInteger(rows) || rows < 1 || rows > host.limits.maxTerminalRows) {
+    throw new ProcessSessionError("ERR_PRISM_PROCESS_PTY_LIMIT", `rows must be 1..${host.limits.maxTerminalRows}`);
+  }
+  if (Buffer.byteLength(term, "utf8") > host.limits.maxTerminalTermBytes) {
+    throw new ProcessSessionError("ERR_PRISM_PROCESS_PTY_LIMIT", `term exceeds maxTerminalTermBytes (${host.limits.maxTerminalTermBytes})`);
+  }
+  return { columns, rows, term };
+}

package/coding/src/agent/process/sessions-monitor.ts

@@ -0,0 +1,141 @@
+/**
+ * Durable persist / CAS transitions for managed process sessions.
+ */
+
+import type { ProcessRecoveryRecord } from "./recovery.js";
+import {
+  buildProcessRecoveryRecord,
+  deleteProcessRecoveryRecord,
+  loadProcessRecoveryRecords,
+  PROCESS_RECOVERY_LEASE_NAMESPACE,
+  releaseRecordLease,
+  saveProcessRecoveryRecord,
+} from "./recovery.js";
+import { durableSeams, isTerminalState, nowIso, type SessionRecord, type SessionsHost } from "./sessions-host.js";
+
+// Durable transition write: fire-and-forget CAS (fence/version conflicts mean
+// another replica moved the record first — the newer state wins). The crash
+// window between a terminal transition and its durable write converges on
+// recovery to attach/terminal/unknown, never a duplicate spawn.
+export function persistTransition(host: SessionsHost, record: SessionRecord): void {
+  if (!host.durable || record.recoveryVersion === 0) return;
+  const seams = durableSeams(host);
+  const next = buildProcessRecoveryRecord({
+    id: record.id,
+    owner: record.owner,
+    workspace: record.workspace,
+    command: record.command,
+    args: record.args,
+    commandFingerprint: record.commandFingerprint,
+    policyDecision: record.policyDecision,
+    startedAt: record.startedAt,
+    state: record.state,
+    exitCode: record.exitCode,
+    releaseOnCancel: record.releaseOnCancel,
+    expiresAt: record.expiresAt,
+    ...(record.backendRef !== undefined ? { backendRef: record.backendRef } : {}),
+    ...(record.ptyTerminal !== undefined ? { pty: record.ptyTerminal } : {}),
+    fencingToken: record.recoveryFencingToken,
+  });
+  const expectedVersion = record.recoveryVersion;
+  record.recoveryVersion += 1;
+  const write = async (): Promise<void> => {
+    try {
+      await saveProcessRecoveryRecord({
+        checkpoints: seams.checkpoints,
+        record: next,
+        expectedVersion,
+        version: record.recoveryVersion,
+        ownership: host.ownership,
+      });
+    } catch {
+      // stale fence or store failure: the durable record keeps its last state
+      return;
+    }
+    void evictRecoveryOverflow(host);
+    // Terminal transition: release the record lease (clean shutdown makes
+    // recovery immediate). Live running/starting transitions renew it so a
+    // crashed replica's lease lapses within TTL while a live one stays
+    // fenced. Best effort on both.
+    if (isTerminalState(record.state)) {
+      if (record.recoveryLeaseToken) {
+        void releaseRecordLease({
+          leases: seams.leases,
+          id: record.id,
+          ownerId: seams.ownerId,
+          token: record.recoveryLeaseToken,
+          ownership: host.ownership,
+        });
+        record.recoveryLeaseToken = undefined;
+      }
+    } else if (record.recoveryLeaseToken) {
+      void seams.leases
+        .renewLease({
+          namespace: PROCESS_RECOVERY_LEASE_NAMESPACE,
+          key: `recover:${record.id}`,
+          ownerId: seams.ownerId,
+          token: record.recoveryLeaseToken,
+          ttlMs: host.recoveryLimits.leaseTtlMs,
+          ...host.ownership,
+        })
+        .catch(() => {
+          // lease lapsed or fenced elsewhere; recovery is attestation-gated
+        });
+    }
+  };
+  // Per-record chain: transition CAS writes must never invert order on slow
+  // stores (a later terminal write landing before the running write would
+  // fail its CAS and leave the record running forever).
+  record.recoveryWriteChain = (record.recoveryWriteChain ?? Promise.resolve()).then(write, write);
+}
+
+// Bound durable record growth: after a terminal transition, drop the oldest
+// terminal records until at most maxRecords remain (running/starting records
+// are never evicted). Best effort, bounded work.
+export async function evictRecoveryOverflow(host: SessionsHost): Promise<void> {
+  if (!host.durable) return;
+  const seams = durableSeams(host);
+  try {
+    const page = await loadProcessRecoveryRecords({
+      checkpoints: seams.checkpoints,
+      limits: { ...host.recoveryLimits, maxRecords: host.recoveryLimits.maxRecords + 1 },
+      ownership: host.ownership,
+    });
+    if (page.records.length <= host.recoveryLimits.maxRecords) return;
+    const overflow = page.records.length - host.recoveryLimits.maxRecords;
+    let evicted = 0;
+    for (const { record } of [...page.records].reverse()) {
+      if (evicted >= overflow) break;
+      if (record.state === "running" || record.state === "starting") continue;
+      await deleteProcessRecoveryRecord({ checkpoints: seams.checkpoints, id: record.id, ownership: host.ownership });
+      evicted += 1;
+    }
+  } catch {
+    // best effort: caps are enforced again on the next transition
+  }
+}
+
+// Atomic starting|running -> unknown (never an exit code). Returns false when
+// a fence/version conflict means another replica moved the record first.
+export async function persistRecoveryUnknown(
+  host: SessionsHost,
+  current: ProcessRecoveryRecord,
+  version: number,
+  signal?: AbortSignal,
+): Promise<boolean> {
+  const next = buildProcessRecoveryRecord({ ...current, state: "unknown", exitCode: null, updatedAt: nowIso() });
+  const seams = durableSeams(host);
+  try {
+    await saveProcessRecoveryRecord({
+      checkpoints: seams.checkpoints,
+      record: next,
+      expectedVersion: version,
+      version: version + 1,
+      ownership: host.ownership,
+      signal,
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}

package/coding/src/agent/process/sessions-recovery.ts

@@ -0,0 +1,241 @@
+/**
+ * Durable recover / reattach phase for managed process sessions.
+ */
+import { OutputAccumulator } from "../output-accumulator.js";
+import {
+  acquireRecordLease,
+  attachWithTimeout,
+  buildProcessRecoveryRecord,
+  loadProcessRecoveryRecord,
+  loadProcessRecoveryRecords,
+  ProcessRecoveryError,
+  type ProcessRecoveryRecord,
+  type ProcessRecoveryRecordReport,
+  type ProcessRecoveryReport,
+  releaseRecordLease,
+  saveProcessRecoveryRecord,
+} from "./recovery.js";
+import { assertNotDisposed, durableSeams, isTerminalState, nowIso, type SessionRecord, type SessionsHost } from "./sessions-host.js";
+import { persistRecoveryUnknown } from "./sessions-monitor.js";
+import { makeHandle } from "./sessions-spawn.js";
+import { sweepExpired, terminateRecord } from "./sessions-teardown.js";
+import type { ProcessPtyHandle, ProcessSandboxHandle, ProcessSession } from "./types.js";
+
+// Reattach an attested handle into the live registry. Recovered sessions
+// expose control (input/signal/kill/release/resize/wait) through the handle;
+// output streaming is not re-established after a restart — the host backend
+// owns any buffered output behind its opaque ref.
+export function attachRecoveredHandle(
+  host: SessionsHost,
+  current: ProcessRecoveryRecord,
+  handle: ProcessPtyHandle | ProcessSandboxHandle,
+  version: number,
+): void {
+  const accumulator = new OutputAccumulator({
+    maxBytes: host.limits.maxOutputChunkBytes,
+    maxLines: 100_000,
+    maxTotalOutputBytes: host.limits.maxTotalOutputBytes,
+    tempFilePrefix: "prism-proc",
+  });
+  const record: SessionRecord = {
+    id: current.id,
+    owner: current.owner,
+    workspace: current.workspace,
+    command: current.command,
+    args: current.args,
+    commandFingerprint: current.commandFingerprint,
+    policyDecision: current.policyDecision,
+    startedAt: current.startedAt,
+    releaseOnCancel: current.releaseOnCancel,
+    expiresAt: current.expiresAt,
+    state: "running",
+    exitCode: null,
+    ptyTerminal: current.pty,
+    ptyResizeAt: [],
+    accumulator,
+    waiters: [],
+    outputListeners: new Set(),
+    stdinClosed: false,
+    handle: null as unknown as ProcessSession,
+    backendRef: current.backendRef,
+    recoveryFencingToken: current.fencingToken,
+    recoveryVersion: version,
+  };
+  if (current.pty !== undefined) {
+    record.pty = handle as ProcessPtyHandle;
+  } else {
+    record.backend = handle as ProcessSandboxHandle;
+  }
+  record.handle = makeHandle(host, record);
+  host.sessions.set(record.id, record);
+  void handle
+    .wait()
+    .then((result) => {
+      if (record.state !== "running" && record.state !== "starting") return;
+      terminateRecord(host, record, "exited", result.exitCode);
+    })
+    .catch(() => {
+      if (record.state !== "running" && record.state !== "starting") return;
+      terminateRecord(host, record, "unknown", null);
+    });
+}
+
+export async function recoverSessions(host: SessionsHost, recoverOptions?: { signal?: AbortSignal }): Promise<ProcessRecoveryReport> {
+  assertNotDisposed(host);
+  if (!host.durable) {
+    throw new ProcessRecoveryError("ERR_PRISM_RECOVERY_UNSUPPORTED", "durable process recovery is not configured on this host");
+  }
+  const seams = durableSeams(host);
+  sweepExpired(host);
+  const { records } = await loadProcessRecoveryRecords({
+    checkpoints: seams.checkpoints,
+    limits: host.recoveryLimits,
+    ownership: host.ownership,
+    signal: recoverOptions?.signal,
+  });
+  const report: ProcessRecoveryRecordReport[] = [];
+  let attached = 0;
+  let terminal = 0;
+  let unknown = 0;
+  for (const { record } of records) {
+    const live = host.sessions.get(record.id);
+    if (live) {
+      // Already live in this registry: nothing to do, report the truth.
+      report.push({ id: record.id, outcome: "attached", state: live.state, exitCode: live.exitCode });
+      if (live.state === "running" || live.state === "starting") attached += 1;
+      else terminal += 1;
+      continue;
+    }
+    if (
+      record.state === "exited" ||
+      record.state === "killed" ||
+      record.state === "released" ||
+      record.state === "expired" ||
+      record.state === "unknown"
+    ) {
+      report.push({ id: record.id, outcome: "terminal", state: record.state, exitCode: record.exitCode });
+      terminal += 1;
+      continue;
+    }
+    // starting | running durable record without a live handle: attach-if-
+    // attested, else atomic unknown. Never fabricate an exit code.
+    const lease = await acquireRecordLease({
+      leases: seams.leases,
+      id: record.id,
+      ownerId: seams.ownerId,
+      ttlMs: host.recoveryLimits.leaseTtlMs,
+      ownership: host.ownership,
+      signal: recoverOptions?.signal,
+    });
+    if (!lease) {
+      // Another replica owns or is recovering this record.
+      report.push({ id: record.id, outcome: "unknown", state: record.state, exitCode: null });
+      unknown += 1;
+      continue;
+    }
+    try {
+      // Fresh CAS read under the lease: another replica may have moved the record.
+      const fresh = await loadProcessRecoveryRecord({
+        checkpoints: seams.checkpoints,
+        id: record.id,
+        limits: host.recoveryLimits,
+        ownership: host.ownership,
+        signal: recoverOptions?.signal,
+      });
+      if (!fresh) {
+        report.push({ id: record.id, outcome: "terminal", state: "unknown", exitCode: null, error: "ERR_PRISM_RECOVERY_UNKNOWN" });
+        terminal += 1;
+        continue;
+      }
+      const current = fresh.record;
+      if (
+        current.state === "exited" ||
+        current.state === "killed" ||
+        current.state === "released" ||
+        current.state === "expired" ||
+        current.state === "unknown"
+      ) {
+        report.push({ id: current.id, outcome: "terminal", state: current.state, exitCode: current.exitCode });
+        terminal += 1;
+        continue;
+      }
+      if (Date.now() >= current.expiresAt) {
+        // Lifetime lapsed while unrecovered: persist the expiry.
+        const expired = buildProcessRecoveryRecord({
+          ...current,
+          state: "expired",
+          exitCode: null,
+          updatedAt: nowIso(),
+        });
+        await saveProcessRecoveryRecord({
+          checkpoints: seams.checkpoints,
+          record: expired,
+          expectedVersion: fresh.version,
+          version: fresh.version + 1,
+          ownership: host.ownership,
+          signal: recoverOptions?.signal,
+        });
+        report.push({ id: current.id, outcome: "terminal", state: "expired", exitCode: null });
+        terminal += 1;
+        continue;
+      }
+      const active = [...host.sessions.values()].filter((s) => s.state === "running" || s.state === "starting").length;
+      if (active >= host.limits.maxSessions) {
+        // Cannot admit another live session: record unknown (fails closed).
+        await persistRecoveryUnknown(host, current, fresh.version, recoverOptions?.signal);
+        report.push({ id: current.id, outcome: "unknown", state: "unknown", exitCode: null });
+        unknown += 1;
+        continue;
+      }
+      let attachErrorCode: ProcessRecoveryRecordReport["error"];
+      if (current.backendRef !== undefined && host.recoveryBackend) {
+        try {
+          const handle = await attachWithTimeout(host.recoveryBackend, current.backendRef, host.recoveryLimits.attachTimeoutMs);
+          if (handle) {
+            attachRecoveredHandle(host, current, handle, fresh.version);
+            attached += 1;
+            report.push({ id: current.id, outcome: "attached", state: "running", exitCode: null });
+            continue;
+          }
+        } catch (attachError) {
+          attachErrorCode = attachError instanceof ProcessRecoveryError ? attachError.code : ("ERR_PRISM_RECOVERY_UNKNOWN" as const);
+        }
+      }
+      // No ref, no backend, unattested attach, or attach failure: atomic unknown.
+      const saved = await persistRecoveryUnknown(host, current, fresh.version, recoverOptions?.signal);
+      if (!saved) {
+        // CAS/fence conflict: another replica moved the record; re-report its state.
+        const again = await loadProcessRecoveryRecord({
+          checkpoints: seams.checkpoints,
+          id: current.id,
+          limits: host.recoveryLimits,
+          ownership: host.ownership,
+          signal: recoverOptions?.signal,
+        });
+        const reported = again?.record;
+        report.push({
+          id: current.id,
+          outcome: reported && isTerminalState(reported.state) ? "terminal" : "unknown",
+          state: reported?.state ?? "unknown",
+          exitCode: reported?.exitCode ?? null,
+          error: attachErrorCode,
+        });
+        if (reported && isTerminalState(reported.state)) terminal += 1;
+        else unknown += 1;
+        continue;
+      }
+      report.push({ id: current.id, outcome: "unknown", state: "unknown", exitCode: null, error: attachErrorCode });
+      unknown += 1;
+    } finally {
+      await releaseRecordLease({
+        leases: seams.leases,
+        id: record.id,
+        ownerId: seams.ownerId,
+        token: lease.token,
+        ownership: host.ownership,
+        signal: recoverOptions?.signal,
+      });
+    }
+  }
+  return { records: report, attached, terminal, unknown };
+}