@oar-sh/oar 0.9.5 → 0.9.7

package/server/services/session-worker-stop-service.mjs

@@ -20,18 +20,22 @@ function toPidList(rows, extraPid) {
     .filter((pid) => Number.isInteger(pid) && pid > 0);
 }
 
-function enumeratePids({ sdkSessionId, worker, processInspector, platform }) {
+// The process list is read off the relay's thread, fresh (a stop must see
+// the children started a moment ago). Throws when the list could not be
+// read: the caller must not pretend it stopped what it could not see.
+async function enumeratePids({ sdkSessionId, worker, processInspector, platform }) {
   const rows = platform === 'win32'
-    ? (
-      processInspector?.findWindowsProcessTreeForSession?.(sdkSessionId)
-      || processInspector?.findWindowsProcessesForSession?.(sdkSessionId)
-      || processInspector?.findProcessesForSession?.(sdkSessionId)
-      || []
-    )
-    : (processInspector?.findProcessesForSession?.(sdkSessionId) || []);
-  return toPidList(rows, worker?.pid);
+    ? await processInspector?.findWindowsProcessTreeForSessionAsync?.(sdkSessionId, { fresh: true })
+    : await processInspector?.findProcessesForSessionAsync?.(sdkSessionId, { fresh: true });
+  return toPidList(rows || [], worker?.pid);
 }
 
+async function stopWindowsPids(processInspector, pids) {
+  await processInspector?.stopWindowsPidsAsync?.(pids);
+}
+
+export const PROCESS_LIST_UNREADABLE = 'process-list-unreadable';
+
 export async function stopSessionWorkerProcesses({
   sdkSessionId,
   worker = null,
@@ -61,7 +65,14 @@ export async function stopSessionWorkerProcesses({
 
   if (!sid) return done({ ok: false, error: 'Missing session id' });
 
-  const pids = enumeratePids({ sdkSessionId: sid, worker, processInspector, platform });
+  let pids;
+  try {
+    pids = await enumeratePids({ sdkSessionId: sid, worker, processInspector, platform });
+  } catch (error) {
+    // Nothing is stopped blind: the known pid alone would leave the tree
+    // running, and the caller would relaunch next to it.
+    return done({ ok: false, pids: [], remainingPids: [], error: `${PROCESS_LIST_UNREADABLE}: ${error?.message || error}` });
+  }
   if (!pids.length) return done({ pids: [] });
 
   const survivors = (list) => list.filter((pid) => isPidAliveImpl(pid));
@@ -80,7 +91,7 @@ export async function stopSessionWorkerProcesses({
     if (platform === 'win32') {
       // stopWindowsPids already walks the descendant tree leaf-first and uses
       // Stop-Process -Force, so there is no gentler first pass to attempt.
-      processInspector?.stopWindowsPids?.(pids);
+      await stopWindowsPids(processInspector, pids);
     } else {
       for (const pid of pids) {
         try {
@@ -104,9 +115,9 @@ export async function stopSessionWorkerProcesses({
   try {
     if (platform === 'win32') {
       // Re-enumerate: children spawned after the first snapshot are invisible to it.
-      const rediscovered = enumeratePids({ sdkSessionId: sid, worker, processInspector, platform });
+      const rediscovered = await enumeratePids({ sdkSessionId: sid, worker, processInspector, platform });
       escalationTargets = [...new Set([...remainingPids, ...rediscovered.filter((pid) => isPidAliveImpl(pid))])];
-      processInspector?.stopWindowsPids?.(escalationTargets);
+      await stopWindowsPids(processInspector, escalationTargets);
     } else {
       for (const pid of escalationTargets) {
         try {

package/server/services/session-worker-supervisor-service.mjs

@@ -63,6 +63,19 @@ export function createSessionWorkerSupervisor({
   maxRestartRetries = 5,
   restartBackoffBaseMs = 1_000,
   restartBackoffMaxMs = 30_000,
+  // Exhausted is not for ever: what failed the launches can be gone — a
+  // process list the relay could not read for an hour was (2026-10-01) — and
+  // the pending work would otherwise wait for a hand-made launch or a relay
+  // restart. One more launch is allowed this long after the last failure; a
+  // failure exhausts the session again at once. 0 keeps it exhausted.
+  restartExhaustedRetryAfterMs = 60_000,
+  // ...and not for longer than this after the first exhaustion: then the
+  // session stays exhausted until a launch is asked for (the Retry button,
+  // the launch or kill route, a relay restart). 0 retries without end.
+  restartExhaustedRetryWindowMs = 10 * 60_000,
+  // Told about launch episodes, for the note in the conversation:
+  // ({ type: 'exhausted' | 'retry-failed' | 'stopped' | 'launched', sdkSessionId, lifecycle }).
+  onLaunchState = null,
   idleEvictionMs = 0,
   // Copilot-kind workers boot a full CLI (auth, model catalog, workspace scan)
   // before their extension can heartbeat; measured median is ~35s. A 30s
@@ -88,6 +101,8 @@ export function createSessionWorkerSupervisor({
   const retryLimit = Math.max(0, clampInt(maxRestartRetries, 5));
   const backoffBaseMs = Math.max(1, clampInt(restartBackoffBaseMs, 1_000));
   const backoffMaxMs = Math.max(backoffBaseMs, clampInt(restartBackoffMaxMs, 30_000));
+  const exhaustedRetryAfterMs = Math.max(0, clampInt(restartExhaustedRetryAfterMs, 60_000));
+  const exhaustedRetryWindowMs = Math.max(0, clampInt(restartExhaustedRetryWindowMs, 10 * 60_000));
   const idleTimeoutMs = clampInt(idleEvictionMs, 0);
   const heartbeatStaleAfterMs = Math.max(1_000, clampInt(heartbeatTimeoutMs, 90_000));
   const recoveryGraceMs = Math.max(0, clampInt(degradedRecoveryGraceMs, 10_000));
@@ -129,6 +144,10 @@ export function createSessionWorkerSupervisor({
       backoffMs: 0,
       nextRestartAtMs: null,
       restartExhausted: false,
+      // The launch episode: when the launches were first exhausted, and
+      // whether the relay has stopped trying again by itself.
+      exhaustedSinceMs: null,
+      retriesStopped: false,
       lastError: null,
       lastFailureAtMs: null,
       lastActivityAtMs: nowAtMs,
@@ -167,6 +186,11 @@ export function createSessionWorkerSupervisor({
       backoffMs: clampInt(lifecycle.backoffMs, 0),
       nextRestartAt: lifecycle.nextRestartAtMs ? new Date(lifecycle.nextRestartAtMs).toISOString() : null,
       restartExhausted: Boolean(lifecycle.restartExhausted),
+      exhaustedSince: lifecycle.exhaustedSinceMs ? new Date(lifecycle.exhaustedSinceMs).toISOString() : null,
+      retriesStopped: Boolean(lifecycle.retriesStopped),
+      retryAgainAt: lifecycle.restartExhausted && !lifecycle.retriesStopped && lifecycle.lastFailureAtMs && exhaustedRetryAfterMs > 0
+        ? new Date(lifecycle.lastFailureAtMs + exhaustedRetryAfterMs).toISOString()
+        : null,
       lastError: lifecycle.lastError || null,
       lastFailureAt: lifecycle.lastFailureAtMs ? new Date(lifecycle.lastFailureAtMs).toISOString() : null,
       lastActivityAt: lifecycle.lastActivityAtMs ? new Date(lifecycle.lastActivityAtMs).toISOString() : null,
@@ -189,6 +213,16 @@ export function createSessionWorkerSupervisor({
     };
   }
 
+  // The caller's listener must not break the supervisor.
+  function notifyLaunchState(type, sessionId) {
+    if (typeof onLaunchState !== 'function') return;
+    try {
+      onLaunchState({ type, sdkSessionId: sessionId, lifecycle: toLifecycleSnapshot(sessionId) });
+    } catch (error) {
+      try { log?.(`launch state listener failed (${type}, session=${String(sessionId).slice(0, 8)}): ${error?.message || error}`); } catch {}
+    }
+  }
+
   function touchActivity(sessionId, atMs = nowMs()) {
     return setLifecycle(sessionId, { lastActivityAtMs: atMs });
   }
@@ -394,16 +428,26 @@ export function createSessionWorkerSupervisor({
     const failedAtMs = nowMs();
 
     if (nextRetryCount > retryLimit) {
+      const firstExhaustion = !lifecycle.restartExhausted;
+      const exhaustedSinceMs = lifecycle.exhaustedSinceMs || failedAtMs;
+      const retriesStopped = exhaustedRetryAfterMs === 0
+        || (exhaustedRetryWindowMs > 0 && failedAtMs - exhaustedSinceMs >= exhaustedRetryWindowMs);
       const exhausted = setLifecycle(sessionId, {
         retryCount: nextRetryCount,
         backoffMs: 0,
         nextRestartAtMs: null,
         restartExhausted: true,
+        exhaustedSinceMs,
+        retriesStopped,
         lastError: message,
         lastFailureAtMs: failedAtMs,
         lastActivityAtMs: failedAtMs,
       });
       registerDegradedState(sessionId, 'restart-exhausted', { atMs: failedAtMs });
+      notifyLaunchState(
+        firstExhaustion ? 'exhausted' : (retriesStopped && !lifecycle.retriesStopped ? 'stopped' : 'retry-failed'),
+        sessionId,
+      );
       return { scheduled: false, exhausted: true, lifecycle: toLifecycleSnapshot(sessionId, exhausted) };
     }
 
@@ -432,6 +476,8 @@ export function createSessionWorkerSupervisor({
       backoffMs: 0,
       nextRestartAtMs: null,
       restartExhausted: false,
+      exhaustedSinceMs: null,
+      retriesStopped: false,
       lastError: null,
       lastFailureAtMs: null,
       lastActivityAtMs: nowAtMs,
@@ -454,7 +500,13 @@ export function createSessionWorkerSupervisor({
     const lifecycle = lifecycleBySession.get(sessionId);
     if (!lifecycle) return { allowed: true, reason: null, lifecycle: null };
     if (lifecycle.restartExhausted) {
-      return { allowed: false, reason: 'restart-exhausted', lifecycle: toLifecycleSnapshot(sessionId, lifecycle) };
+      const failedAtMs = Number(lifecycle.lastFailureAtMs || 0);
+      const dueAgain = !lifecycle.retriesStopped
+        && exhaustedRetryAfterMs > 0 && failedAtMs > 0 && nowMs() - failedAtMs >= exhaustedRetryAfterMs;
+      if (!dueAgain) {
+        return { allowed: false, reason: 'restart-exhausted', lifecycle: toLifecycleSnapshot(sessionId, lifecycle) };
+      }
+      return { allowed: true, reason: null, lifecycle: toLifecycleSnapshot(sessionId, lifecycle) };
     }
     if (lifecycle.nextRestartAtMs && nowMs() < lifecycle.nextRestartAtMs) {
       return { allowed: false, reason: 'restart-delayed', lifecycle: toLifecycleSnapshot(sessionId, lifecycle) };
@@ -545,6 +597,7 @@ export function createSessionWorkerSupervisor({
       // backoff/exhausted lifecycle state was left behind by earlier failures.
       clearRestartSchedule(sessionId, { resetDegradedState: false });
       assessSessionHealth(sessionId, reusedWorker, { nowAtMs, questionPending: false });
+      notifyLaunchState('launched', sessionId);
       return { ok: true, reused: true, worker: reusedWorker, lifecycle: toLifecycleSnapshot(sessionId) };
     }
 
@@ -609,6 +662,7 @@ export function createSessionWorkerSupervisor({
           });
           clearRestartSchedule(sessionId);
           assessSessionHealth(sessionId, readyState, { questionPending: false });
+          notifyLaunchState('launched', sessionId);
           return { ok: true, reused: false, worker: readyState, lifecycle: toLifecycleSnapshot(sessionId) };
         }
         const simulatedReady = setWorkerState(sessionId, {
@@ -625,6 +679,7 @@ export function createSessionWorkerSupervisor({
         });
         clearRestartSchedule(sessionId);
         assessSessionHealth(sessionId, simulatedReady, { questionPending: false });
+        notifyLaunchState('launched', sessionId);
         return { ok: true, reused: false, worker: simulatedReady, lifecycle: toLifecycleSnapshot(sessionId) };
       } catch (error) {
         const scheduled = scheduleRestart(sessionId, error, { incrementRetry: true });
@@ -778,9 +833,41 @@ export function createSessionWorkerSupervisor({
     return evicted;
   }
 
+  /**
+   * An idle worker whose process is gone (killed from outside, crashed
+   * between turns) leaves the registry, so it is not listed as ready with a
+   * pid that no longer exists; the next message starts a new one either way.
+   * A worker that is processing, or that a question card waits on, is left
+   * to the stale-pid handling, which knows about the work that is open. A
+   * heartbeat inside the stale window outranks the pid: the pid on record
+   * can be a launcher's. A worker that was stopped from outside therefore
+   * stays listed until that window has passed (90 s by default); one that
+   * the relay's own kill route stopped leaves the list at once.
+   */
+  function retireDeadIdleWorkers({ keep = new Set() } = {}) {
+    const retired = [];
+    const heartbeatCutoff = nowMs() - heartbeatStaleAfterMs;
+    for (const worker of registry.listWorkers()) {
+      const sessionId = normalizeSessionId(worker.sdkSessionId);
+      if (!sessionId || worker.status !== 'ready') continue;
+      if (pendingStarts.has(sessionId) || keep.has(sessionId)) continue;
+      const pidProbe = checkPidLiveness(worker);
+      if (!pidProbe.checked || pidProbe.alive) continue;
+      const lifecycle = getOrCreateLifecycle(sessionId);
+      if (lifecycle.awaitingHeartbeat) continue;
+      if (lifecycle.lastHeartbeatAtMs && lifecycle.lastHeartbeatAtMs > heartbeatCutoff) continue;
+      if (!registry.removeWorker(sessionId)) continue;
+      lifecycleBySession.delete(sessionId);
+      emitMonitorLog(`worker ${worker.workerId || sessionId} was idle and its process (pid ${worker.pid}) is gone; no longer listed`);
+      retired.push({ sdkSessionId: sessionId, workerId: worker.workerId || null, pid: worker.pid });
+    }
+    return retired;
+  }
+
   function snapshot({ pendingQuestionSessionIds = null } = {}) {
-    const workers = registry.listWorkers();
     const questionPendingSet = normalizePendingQuestionSessionIds(pendingQuestionSessionIds);
+    retireDeadIdleWorkers({ keep: questionPendingSet });
+    const workers = registry.listWorkers();
     const workerBySession = new Map();
     for (const worker of workers) {
       const sid = normalizeSessionId(worker?.sdkSessionId);