chainlesschain 0.162.160 → 0.162.162

package/src/lib/background-agent-supervisor.js

@@ -15,12 +15,169 @@ import { randomBytes } from "node:crypto";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getHomeDir } from "./paths.js";
-
-export const _deps = { spawn, spawnSync };
+import { deriveSessionState } from "./session-lifecycle.js";
 
 export const DEFAULT_HEARTBEAT_INTERVAL_MS = 5000;
 export const DEFAULT_HEARTBEAT_STALE_MS = 120000;
 
+/**
+ * Pid identity tolerance (Gap 1: OS pid reuse). A pid alone does not identify
+ * a process — after the worker dies the OS can hand the same pid to an
+ * unrelated process, making `kill(pid, 0)` lie: the session shows "running"
+ * until the heartbeat goes stale and, far worse, `stopBackgroundAgent` would
+ * `taskkill /T /F` an innocent process tree. The state file records
+ * `startedAt` (launcher clock, written just before the spawn), so we compare
+ * it against the pid's REAL creation time. Reuse is one-sided: a process
+ * that took over the pid can only have been created AFTER the original
+ * worker died — i.e. noticeably later than startedAt. Creation at/before
+ * startedAt (± tolerance for clock skew and spawn latency) is our process.
+ */
+export const PID_IDENTITY_TOLERANCE_MS = 60000;
+
+const START_TIME_CACHE_TTL_MS = 10000;
+const _startTimeCache = new Map();
+
+/** CIM_DATETIME (`20260711120000.500000+480`) → epoch ms, null when unparseable. */
+export function parseCimDateToMs(raw) {
+  const m =
+    /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})\.(\d{6})([+-])(\d{3})$/.exec(
+      String(raw || "").trim(),
+    );
+  if (!m) return null;
+  const localAsUtc = Date.UTC(
+    Number(m[1]),
+    Number(m[2]) - 1,
+    Number(m[3]),
+    Number(m[4]),
+    Number(m[5]),
+    Number(m[6]),
+    Math.floor(Number(m[7]) / 1000),
+  );
+  const offsetMinutes = (m[8] === "-" ? -1 : 1) * Number(m[9]);
+  return localAsUtc - offsetMinutes * 60000;
+}
+
+/**
+ * Read a process's creation time (epoch ms) with ONE synchronous probe, or
+ * null when it cannot be determined (missing tools, permissions, platform
+ * quirks). Callers must treat null as "unknown" and FAIL OPEN to the legacy
+ * kill(pid, 0) answer — a broken probe must never declare a live worker dead
+ * (nor green-light killing a process we could not identify as ours; see the
+ * per-call-site policy).
+ */
+function defaultReadProcessStartTimeMs(pid) {
+  const target = Number(pid);
+  if (!Number.isInteger(target) || target <= 0) return null;
+  if (process.platform === "win32") {
+    // wmic first (fast, ~100-300ms); PowerShell CIM as fallback (wmic is
+    // removed from recent Windows 11 builds).
+    try {
+      const r = spawnSync(
+        "wmic",
+        [
+          "process",
+          "where",
+          `ProcessId=${target}`,
+          "get",
+          "CreationDate",
+          "/value",
+        ],
+        { windowsHide: true, encoding: "utf8", timeout: 5000 },
+      );
+      if (!r.error && r.status === 0) {
+        const m = /CreationDate=([^\r\n]+)/.exec(r.stdout || "");
+        const ms = m ? parseCimDateToMs(m[1]) : null;
+        if (ms !== null) return ms;
+      }
+    } catch {
+      /* fall through to PowerShell */
+    }
+    try {
+      const script =
+        `$p = Get-CimInstance Win32_Process -Filter 'ProcessId=${target}' -ErrorAction SilentlyContinue; ` +
+        "if ($p -and $p.CreationDate) { [DateTimeOffset]::new($p.CreationDate).ToUnixTimeMilliseconds() }";
+      const r = spawnSync(
+        "powershell",
+        ["-NoProfile", "-NonInteractive", "-Command", script],
+        { windowsHide: true, encoding: "utf8", timeout: 10000 },
+      );
+      if (!r.error && r.status === 0) {
+        const n = Number(String(r.stdout || "").trim());
+        if (Number.isFinite(n) && n > 0) return n;
+      }
+    } catch {
+      /* fail open below */
+    }
+    return null;
+  }
+  // POSIX: `ps -o lstart=` gives an absolute start timestamp on Linux + macOS
+  // (procps and BSD ps both support it; busybox ps does not → null → open).
+  try {
+    const r = spawnSync("ps", ["-o", "lstart=", "-p", String(target)], {
+      encoding: "utf8",
+      timeout: 5000,
+    });
+    if (!r.error && r.status === 0) {
+      const t = Date.parse(String(r.stdout || "").trim());
+      if (Number.isFinite(t) && t > 0) return t;
+    }
+  } catch {
+    /* fail open below */
+  }
+  return null;
+}
+
+/**
+ * Kill a whole process tree (Gap 2 orphan reclaim). Windows: `taskkill /T`.
+ * POSIX: negative-pid group signal (the worker spawns the agent child
+ * detached → its own group), falling back to a direct kill.
+ */
+function defaultKillProcessTree(pid, signal = "SIGKILL") {
+  const target = Number(pid);
+  if (!Number.isInteger(target) || target <= 0) return false;
+  try {
+    if (process.platform === "win32") {
+      const r = spawnSync("taskkill", ["/PID", String(target), "/T", "/F"], {
+        windowsHide: true,
+        encoding: "utf8",
+      });
+      return !r.error && r.status === 0;
+    }
+    try {
+      process.kill(-target, signal);
+      return true;
+    } catch {
+      process.kill(target, signal);
+      return true;
+    }
+  } catch {
+    return false;
+  }
+}
+
+export const _deps = {
+  spawn,
+  spawnSync,
+  readProcessStartTimeMs: defaultReadProcessStartTimeMs,
+  killProcessTree: defaultKillProcessTree,
+};
+
+/** Cached probe (default impl only — injected probes run uncached for tests). */
+function processStartTimeMs(pid) {
+  if (_deps.readProcessStartTimeMs !== defaultReadProcessStartTimeMs) {
+    const raw = _deps.readProcessStartTimeMs(pid);
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? n : null;
+  }
+  const key = Number(pid);
+  const at = Date.now();
+  const hit = _startTimeCache.get(key);
+  if (hit && at - hit.at < START_TIME_CACHE_TTL_MS) return hit.value;
+  const value = defaultReadProcessStartTimeMs(key);
+  _startTimeCache.set(key, { at, value });
+  return value;
+}
+
 function nowMs(options = {}) {
   return typeof options.now === "number" ? options.now : Date.now();
 }
@@ -136,12 +293,81 @@ export function isProcessAlive(pid) {
   }
 }
 
+/**
+ * Is the live process at `pid` the SAME process the state file recorded at
+ * `expectedStartedAtMs` — or a pid-reusing stranger? (Gap 1.)
+ *
+ * - dead pid → false (not the same process; it is no process at all)
+ * - no anchor (legacy state without startedAt) → true (legacy semantics)
+ * - probe failed → true (FAIL OPEN: never declare a live worker dead — and
+ *   never kill — on the strength of a broken probe; callers fall back to the
+ *   plain kill(pid,0) behavior this system always had)
+ * - creation time > startedAt + tolerance → false (a reuser can only be
+ *   born after the original died)
+ */
+export function isSameProcess(pid, expectedStartedAtMs, options = {}) {
+  if (!isProcessAlive(pid)) return false;
+  const expected = Number(expectedStartedAtMs);
+  if (!Number.isFinite(expected) || expected <= 0) return true;
+  const actual = processStartTimeMs(pid);
+  if (actual === null) return true;
+  const tolerance = Number.isFinite(Number(options.toleranceMs))
+    ? Number(options.toleranceMs)
+    : PID_IDENTITY_TOLERANCE_MS;
+  return actual <= expected + tolerance;
+}
+
+/**
+ * Reap a lost worker's recorded agent child (Gap 2). The worker persists
+ * `agentPid` + `agentStartedAt` per turn; when the WORKER dies (crash /
+ * pid-reuse detection) nothing used to kill that grandchild — it kept
+ * running unsupervised. Kill it, tree-wide, IF we can prove identity:
+ *
+ * - `agentStartedAt` anchor is REQUIRED — unlike the read-only liveness
+ *   probe, this path kills, so a legacy state without the anchor fails
+ *   CLOSED (no kill) rather than risking a pid-reused stranger;
+ * - with the anchor present, a failed creation-time probe fails open to the
+ *   kill (the orphan leak is the common case, reuse the rare one — and the
+ *   anchor already bounds the window to this session's lifetime).
+ *
+ * @returns {boolean} true when a kill was actually issued
+ */
+export function reclaimOrphanAgentProcess(state) {
+  const agentPid = Number(state?.agentPid);
+  if (!Number.isInteger(agentPid) || agentPid <= 0) return false;
+  if (agentPid === Number(state?.pid)) return false; // never the worker itself
+  const anchor = Number(state?.agentStartedAt);
+  if (!Number.isFinite(anchor) || anchor <= 0) return false; // fail closed
+  if (!isSameProcess(agentPid, anchor)) return false; // reused / already gone
+  return _deps.killProcessTree(agentPid, "SIGKILL") === true;
+}
+
 export function normalizeBackgroundAgentTitle(title) {
   const value = String(title || "").trim();
   if (!value) throw new Error("Background agent title cannot be empty");
   return value.slice(0, 160);
 }
 
+/**
+ * The canonical unified lifecycle state (session-lifecycle.js) for a session —
+ * folds the supervisor `status` + worker `phase` + `pendingApprovals` into one
+ * of the 10 canonical states so every consumer (dashboard, `cc daemon list
+ * --json`, an IDE bridge) reasons about a single vocabulary. Pure; call it at a
+ * DISPLAY boundary only. It is deliberately NOT baked into
+ * effectiveBackgroundAgentState because that function's output is spread back
+ * into writeBackgroundAgentState by the rename/pin/stop read-modify-write
+ * paths — a derived field there would leak into the on-disk schema.
+ */
+export function sessionLifecycleState(state) {
+  return deriveSessionState(state);
+}
+
+/** A display copy of `state` with the derived `lifecycleState` attached. */
+function withLifecycleState(state) {
+  if (!state) return state;
+  return { ...state, lifecycleState: deriveSessionState(state) };
+}
+
 export function effectiveBackgroundAgentState(state, options = {}) {
   if (!state) return null;
   if (state.status !== "running") return state;
@@ -164,25 +390,53 @@ export function effectiveBackgroundAgentState(state, options = {}) {
       endedAt: state.endedAt || t,
       lostReason: "process-exited",
     };
+  } else if (!isSameProcess(state.pid, state.startedAt)) {
+    // Gap 1: the pid is alive but belongs to a process created well after
+    // this session started — the OS reused the worker's pid.
+    next = {
+      ...state,
+      status: "lost",
+      endedAt: state.endedAt || t,
+      lostReason: "pid-reused",
+    };
   }
-  if (next !== state && options.persist !== false) {
-    try {
-      writeBackgroundAgentState(next);
-    } catch {
-      /* best-effort; callers still get the corrected state */
+  if (next !== state) {
+    // Gap 2: the worker is gone (dead pid or pid-reused stranger) — reap the
+    // recorded agent grandchild so a crashed worker never leaves it running
+    // unsupervised. A merely-stale heartbeat with a live, verified worker
+    // still owns its child, so isSameProcess gates the reclaim.
+    if (!isSameProcess(state.pid, state.startedAt)) {
+      try {
+        reclaimOrphanAgentProcess(state);
+      } catch {
+        /* best-effort */
+      }
+    }
+    if (options.persist !== false) {
+      try {
+        writeBackgroundAgentState(next);
+      } catch {
+        /* best-effort; callers still get the corrected state */
+      }
     }
   }
   return next;
 }
 
 export function listBackgroundAgents(options = {}) {
-  return readdirSync(backgroundAgentsDir())
-    .filter((name) => name.endsWith(".json") && !name.includes(".job."))
-    .map((name) => readBackgroundAgentState(name.slice(0, -5)))
-    .filter(Boolean)
-    .map((state) => effectiveBackgroundAgentState(state, options))
-    .filter((state) => options.all || state.status === "running")
-    .sort((a, b) => (b.startedAt || 0) - (a.startedAt || 0));
+  return (
+    readdirSync(backgroundAgentsDir())
+      .filter((name) => name.endsWith(".json") && !name.includes(".job."))
+      .map((name) => readBackgroundAgentState(name.slice(0, -5)))
+      .filter(Boolean)
+      .map((state) => effectiveBackgroundAgentState(state, options))
+      .filter((state) => options.all || state.status === "running")
+      .sort((a, b) => (b.startedAt || 0) - (a.startedAt || 0))
+      // Display-only enrichment: attach the canonical unified lifecycle state.
+      // This is a list feed (never written back to disk), so the derived field
+      // cannot leak into the on-disk schema the way it would in the mutate paths.
+      .map(withLifecycleState)
+  );
 }
 
 function sleepSyncMs(ms) {
@@ -430,6 +684,7 @@ export function launchBackgroundAgent({
     pid: null,
     workerPid: null,
     agentPid: null,
+    agentStartedAt: null,
     status: "running",
     startedAt: Date.now(),
     heartbeatAt: Date.now(),
@@ -496,7 +751,35 @@ export function readBackgroundAgentLog(id, options = {}) {
 export function stopBackgroundAgent(id) {
   const state = effectiveBackgroundAgentState(readBackgroundAgentState(id));
   if (!state) throw new Error(`Background agent not found: ${id}`);
-  if (state.status !== "running") return { ...state, stopped: false };
+  if (state.status !== "running") {
+    // Gap 2: stopping an already-lost session still reaps a leaked agent
+    // child recorded before the worker died (identity-guarded inside).
+    if (state.status === "lost") {
+      try {
+        reclaimOrphanAgentProcess(state);
+      } catch {
+        /* best-effort */
+      }
+    }
+    return { ...state, stopped: false };
+  }
+  // Gap 1: last-instant identity re-check before the kill — never
+  // taskkill/SIGTERM a pid the OS has re-assigned to an unrelated process.
+  if (!isSameProcess(state.pid, state.startedAt)) {
+    try {
+      reclaimOrphanAgentProcess(state);
+    } catch {
+      /* best-effort */
+    }
+    const lost = {
+      ...state,
+      status: "lost",
+      endedAt: Date.now(),
+      lostReason: "pid-reused",
+    };
+    writeBackgroundAgentState(lost);
+    return { ...lost, stopped: false };
+  }
   if (process.platform === "win32") {
     const killed = _deps.spawnSync(
       "taskkill",
@@ -517,6 +800,30 @@ export function stopBackgroundAgent(id) {
     } catch {
       process.kill(Number(state.pid), "SIGTERM");
     }
+    // The worker detaches the agent child into its OWN process group (so a
+    // crashed worker's child can be group-killed) — which also means the
+    // worker-group SIGTERM above no longer reaches it. Kill the agent group
+    // explicitly, identity-guarded via its agentStartedAt anchor.
+    const agentPid = Number(state.agentPid);
+    const anchor = Number(state.agentStartedAt);
+    if (
+      Number.isInteger(agentPid) &&
+      agentPid > 0 &&
+      agentPid !== Number(state.pid) &&
+      Number.isFinite(anchor) &&
+      anchor > 0 &&
+      isSameProcess(agentPid, anchor)
+    ) {
+      try {
+        process.kill(-agentPid, "SIGTERM");
+      } catch {
+        try {
+          process.kill(agentPid, "SIGTERM");
+        } catch {
+          /* already gone */
+        }
+      }
+    }
   }
   const next = {
     ...state,

package/src/lib/backpressure-policy.js

@@ -0,0 +1,105 @@
+/**
+ * Outbound backpressure policy — bounds server→client buffering when a
+ * consumer drains too slowly, so a stuck WS client can't grow the sender's
+ * memory without limit (the classic slow-consumer blowup) and the UI degrades
+ * explicitly instead of freezing.
+ *
+ * Pure + deterministic (no transport, no timers): the caller feeds the
+ * transport's current queued-byte count (e.g. `ws.bufferedAmount`) and a frame
+ * class; the policy decides send-vs-drop with hysteresis and reports the
+ * lag-episode transitions the caller turns into a notice.
+ *
+ * Designed to compose with EventReplayBuffer (event-seq-replay.js): a dropped
+ * *droppable* frame should still be RECORDED (it gets a seq and stays in the
+ * bounded replay buffer), just not sent. The seq gap on the next *sent* frame
+ * then trips the consumer's SeqGapTracker, which requests replaySince — so the
+ * dropped frames are recovered once the consumer catches up (or a truncated
+ * replay tells it to full-resync). Critical frames are never dropped.
+ */
+
+const DEFAULT_HIGH_WATER_BYTES = 4 << 20; // 4 MiB queued → consumer is behind
+const DEFAULT_LOW_WATER_BYTES = 1 << 20; // 1 MiB → considered caught up again
+
+/** A frame that must always be delivered (lifecycle, errors, acks). */
+export const CRITICAL = "critical";
+/** A frame safe to drop under pressure and recover via replay/resync. */
+export const DROPPABLE = "droppable";
+
+export class BackpressurePolicy {
+  constructor({
+    highWaterBytes = DEFAULT_HIGH_WATER_BYTES,
+    lowWaterBytes = DEFAULT_LOW_WATER_BYTES,
+  } = {}) {
+    const high = Math.max(
+      1,
+      Math.floor(highWaterBytes) || DEFAULT_HIGH_WATER_BYTES,
+    );
+    // Low water must sit below high water or hysteresis collapses to a flapping
+    // single threshold; clamp it to at most high-1.
+    const low = Math.min(Math.max(0, Math.floor(lowWaterBytes) || 0), high - 1);
+    this._high = high;
+    this._low = low;
+    this._lagging = false;
+    this._dropped = 0; // dropped since the CURRENT lag episode started
+    this._droppedTotal = 0; // dropped across the whole connection
+  }
+
+  /** True while the consumer is behind (buffered ≥ high, until it drains ≤ low). */
+  get lagging() {
+    return this._lagging;
+  }
+
+  /** Frames dropped across the whole connection. */
+  get droppedTotal() {
+    return this._droppedTotal;
+  }
+
+  /**
+   * Decide what to do with one outbound frame.
+   * @param {number} bufferedBytes transport's currently-queued bytes
+   *   (`ws.bufferedAmount`); non-finite → treated as 0 (never lags).
+   * @param {"critical"|"droppable"} frameClass
+   * @returns {{send:boolean, drop:boolean, lagStarted:boolean, lagEnded:boolean,
+   *   dropped:number}} `dropped` = count dropped in the episode that just
+   *   started/ended (for the caller's notice); `send:false` only ever for a
+   *   droppable frame while lagging.
+   */
+  evaluate(bufferedBytes, frameClass) {
+    const b = Number.isFinite(bufferedBytes) ? bufferedBytes : 0;
+    let lagStarted = false;
+    let lagEnded = false;
+
+    if (!this._lagging && b >= this._high) {
+      this._lagging = true;
+      this._dropped = 0;
+      lagStarted = true;
+    } else if (this._lagging && b <= this._low) {
+      this._lagging = false;
+      lagEnded = true;
+    }
+
+    // While lagging, shed droppable frames to let the buffer drain; critical
+    // frames (and everything while not lagging) always go out.
+    if (this._lagging && frameClass === DROPPABLE) {
+      this._dropped += 1;
+      this._droppedTotal += 1;
+      return {
+        send: false,
+        drop: true,
+        lagStarted,
+        lagEnded,
+        dropped: this._dropped,
+      };
+    }
+
+    // `dropped` carries the episode's running count so a lag-end notice can
+    // report how many droppable frames were shed.
+    return {
+      send: true,
+      drop: false,
+      lagStarted,
+      lagEnded,
+      dropped: this._dropped,
+    };
+  }
+}