@oar-sh/oar 0.9.5 → 0.9.6

package/server/services/session-worker-process-service.mjs

@@ -1,6 +1,8 @@
 'use strict';
 
-import { execFileSync } from 'child_process';
+import { execFile, execFileSync } from 'child_process';
+
+import { collectProcessDescendants, isChildProcessOf } from '../../shared/worker-runtime/process-tree.mjs';
 
 function normalizeSessionId(value) {
   const text = String(value || '').trim();
@@ -51,9 +53,39 @@ function normalizeWindowsProcess(proc) {
     parentProcessId: Number.isInteger(Number(proc?.parentProcessId)) ? Number(proc.parentProcessId) : null,
     name: String(proc?.name || ''),
     commandLine: String(proc?.commandLine || ''),
+    createdAt: Number(proc?.createdAt) > 0 ? Number(proc.createdAt) : 0,
   };
 }
 
+// The relay's own session workers: `node <kind>-session-worker.mjs --session-id <id>`.
+function isRelaySessionWorker(proc) {
+  return String(proc?.commandLine || '').toLowerCase().includes('-session-worker');
+}
+
+// Every process with its parent and the time it was created.
+const WINDOWS_PROCESS_SNAPSHOT_SCRIPT = [
+  '$list = Get-CimInstance Win32_Process | ForEach-Object {',
+  '  [pscustomobject]@{',
+  '    processId = [int]$_.ProcessId;',
+  '    parentProcessId = [int]$_.ParentProcessId;',
+  '    name = [string]$_.Name;',
+  '    commandLine = [string]$_.CommandLine;',
+  '    createdAt = $(if ($_.CreationDate) { [int64](($_.CreationDate.ToUniversalTime() - [datetime]"1970-01-01").TotalMilliseconds) } else { [int64]0 });',
+  '  }',
+  '};',
+  '$list | ConvertTo-Json -Depth 3 -Compress',
+].join(' ');
+
+/** How long the process list may take when a worker waits for it. */
+const WINDOWS_PROCESS_SNAPSHOT_TIMEOUT_MS = 10_000;
+
+function parseWindowsProcessSnapshot(output) {
+  const text = String(output || '').trim();
+  if (!text) return [];
+  const parsed = JSON.parse(text);
+  return Array.isArray(parsed) ? parsed : [parsed];
+}
+
 function looksLikeCopilotWorkerProcess(proc) {
   const name = String(proc?.name || '').trim().toLowerCase();
   const cmd = String(proc?.commandLine || '').trim().toLowerCase();
@@ -94,6 +126,7 @@ function looksLikeCopilotWorkerProcess(proc) {
 export function createSessionWorkerProcessInspector({
   platform = process.platform,
   execFileSyncImpl = execFileSync,
+  execFileImpl = execFile,
 } = {}) {
   function getPosixProcessSnapshot() {
     if (platform === 'win32') return [];
@@ -119,24 +152,37 @@ export function createSessionWorkerProcessInspector({
 
   function getWindowsProcessSnapshot() {
     if (platform !== 'win32') return [];
-    const script = [
-      '$list = Get-CimInstance Win32_Process | ForEach-Object {',
-      '  [pscustomobject]@{',
-      '    processId = [int]$_.ProcessId;',
-      '    parentProcessId = [int]$_.ParentProcessId;',
-      '    name = [string]$_.Name;',
-      '    commandLine = [string]$_.CommandLine;',
-      '  }',
-      '};',
-      '$list | ConvertTo-Json -Depth 3 -Compress',
-    ].join(' ');
-    const output = execFileSyncImpl('powershell.exe', ['-NoProfile', '-Command', script], {
+    const output = execFileSyncImpl('powershell.exe', ['-NoProfile', '-Command', WINDOWS_PROCESS_SNAPSHOT_SCRIPT], {
       stdio: ['ignore', 'pipe', 'ignore'],
     });
-    const text = String(output || '').trim();
-    if (!text) return [];
-    const parsed = JSON.parse(text);
-    return Array.isArray(parsed) ? parsed : [parsed];
+    return parseWindowsProcessSnapshot(output);
+  }
+
+  /**
+   * The same list without holding the caller's thread for it: a session
+   * worker reads it while it serves a conversation, and PowerShell takes about
+   * a second to start. Rejects when the list could not be read.
+   */
+  function readWindowsProcessSnapshot() {
+    if (platform !== 'win32') return Promise.resolve([]);
+    return new Promise((resolve, reject) => {
+      execFileImpl(
+        'powershell.exe',
+        ['-NoProfile', '-Command', WINDOWS_PROCESS_SNAPSHOT_SCRIPT],
+        { windowsHide: true, timeout: WINDOWS_PROCESS_SNAPSHOT_TIMEOUT_MS, maxBuffer: 64 * 1024 * 1024 },
+        (error, stdout) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          try {
+            resolve(parseWindowsProcessSnapshot(stdout).map(normalizeWindowsProcess).filter((proc) => proc.processId));
+          } catch (parseError) {
+            reject(parseError);
+          }
+        },
+      );
+    });
   }
 
   function isWindowsSessionMatch(proc, target, targetPattern) {
@@ -175,24 +221,20 @@ export function createSessionWorkerProcessInspector({
       .map(normalizeWindowsProcess)
       .filter((proc) => proc.processId);
     const byPid = new Map(snapshot.map((proc) => [proc.processId, proc]));
-    const childrenByParent = new Map();
-    for (const proc of snapshot) {
-      if (!proc.parentProcessId) continue;
-      const children = childrenByParent.get(proc.parentProcessId) || [];
-      children.push(proc);
-      childrenByParent.set(proc.parentProcessId, children);
-    }
 
     const related = new Map();
     const addProcess = (proc) => {
       if (proc?.processId) related.set(proc.processId, proc);
     };
-    const addDescendants = (pid) => {
-      for (const child of childrenByParent.get(pid) || []) {
-        if (related.has(child.processId)) continue;
-        addProcess(child);
-        addDescendants(child.processId);
-      }
+    const addDescendants = (proc) => {
+      const found = collectProcessDescendants(snapshot, proc, {
+        accept: (child) => {
+          if (related.has(child.processId)) return false;
+          // The worker of another session is never part of this one's tree.
+          return !(isRelaySessionWorker(child) && !isWindowsSessionMatch(child, target, targetPattern));
+        },
+      });
+      for (const child of found) addProcess(child);
     };
     const addWrapperAncestors = (proc) => {
       let current = proc;
@@ -201,6 +243,8 @@ export function createSessionWorkerProcessInspector({
         seen.add(current.parentProcessId);
         const parent = byPid.get(current.parentProcessId);
         if (!parent || !isWindowsWrapperProcess(parent)) return;
+        // Windows hands a pid out again: see `isChildProcessOf`.
+        if (!isChildProcessOf(current, parent)) return;
         addProcess(parent);
         current = parent;
       }
@@ -208,7 +252,7 @@ export function createSessionWorkerProcessInspector({
 
     for (const proc of snapshot.filter((candidate) => isWindowsSessionMatch(candidate, target, targetPattern))) {
       addProcess(proc);
-      addDescendants(proc.processId);
+      addDescendants(proc);
       addWrapperAncestors(proc);
     }
 
@@ -259,18 +303,26 @@ export function createSessionWorkerProcessInspector({
       '$ErrorActionPreference = "Continue"',
       '$ids = @(' + ids.join(',') + ')',
       'try {',
-      '  $snapshot = @(Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { [pscustomobject]@{ processId = [int]$_.ProcessId; parentProcessId = [int]$_.ParentProcessId } })',
+      // A child is a process started after its parent: the pid of an ended
+      // process is handed out again, and the worker of another session must
+      // not be taken for a child because of it.
+      '  $snapshot = @(Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { [pscustomobject]@{ processId = [int]$_.ProcessId; parentProcessId = [int]$_.ParentProcessId; createdAt = $(if ($_.CreationDate) { [int64](($_.CreationDate.ToUniversalTime() - [datetime]"1970-01-01").TotalMilliseconds) } else { [int64]0 }); worker = ([string]$_.CommandLine -like "*-session-worker*") } })',
       '} catch {',
       '  Write-Error $_',
       '  exit 2',
       '}',
       '$targets = [System.Collections.Generic.HashSet[int]]::new()',
       'foreach ($id in $ids) { [void]$targets.Add([int]$id) }',
+      '$createdAt = @{}',
+      'foreach ($proc in $snapshot) { $createdAt[[int]$proc.processId] = [int64]$proc.createdAt }',
       '$changed = $true',
       'while ($changed) {',
       '  $changed = $false',
       '  foreach ($proc in $snapshot) {',
       '    if ($targets.Contains([int]$proc.parentProcessId) -and -not $targets.Contains([int]$proc.processId)) {',
+      '      if ($proc.worker) { continue }',
+      '      $parentCreatedAt = [int64]$createdAt[[int]$proc.parentProcessId]',
+      '      if ($parentCreatedAt -gt 0 -and [int64]$proc.createdAt -gt 0 -and [int64]$proc.createdAt -lt $parentCreatedAt) { continue }',
       '      [void]$targets.Add([int]$proc.processId)',
       '      $changed = $true',
       '    }',
@@ -314,6 +366,7 @@ export function createSessionWorkerProcessInspector({
     findWindowsProcessForSession,
     findWindowsProcessTreeForSession,
     getWindowsProcessSnapshot,
+    readWindowsProcessSnapshot,
     stopWindowsPids,
   };
 }

package/server/services/session-worker-registry-service.mjs

@@ -60,7 +60,7 @@ function normalizeStatus(value) {
   return ALLOWED_STATUSES.has(mapped) ? mapped : 'new';
 }
 
-const STEERING_HOLD_REASONS = new Set(['question', 'compaction', 'adoption', 'delivery', 'other']);
+const STEERING_HOLD_REASONS = new Set(['question', 'compaction', 'adoption', 'delivery', 'tool', 'other']);
 const STEERING_MESSAGE_ID_MAX_LENGTH = 64;
 const STEERING_CANCELLABLE_IDS_MAX = 50;
 

package/server/services/session-worker-supervisor-service.mjs

@@ -778,9 +778,41 @@ export function createSessionWorkerSupervisor({
     return evicted;
   }
 
+  /**
+   * An idle worker whose process is gone (killed from outside, crashed
+   * between turns) leaves the registry, so it is not listed as ready with a
+   * pid that no longer exists; the next message starts a new one either way.
+   * A worker that is processing, or that a question card waits on, is left
+   * to the stale-pid handling, which knows about the work that is open. A
+   * heartbeat inside the stale window outranks the pid: the pid on record
+   * can be a launcher's. A worker that was stopped from outside therefore
+   * stays listed until that window has passed (90 s by default); one that
+   * the relay's own kill route stopped leaves the list at once.
+   */
+  function retireDeadIdleWorkers({ keep = new Set() } = {}) {
+    const retired = [];
+    const heartbeatCutoff = nowMs() - heartbeatStaleAfterMs;
+    for (const worker of registry.listWorkers()) {
+      const sessionId = normalizeSessionId(worker.sdkSessionId);
+      if (!sessionId || worker.status !== 'ready') continue;
+      if (pendingStarts.has(sessionId) || keep.has(sessionId)) continue;
+      const pidProbe = checkPidLiveness(worker);
+      if (!pidProbe.checked || pidProbe.alive) continue;
+      const lifecycle = getOrCreateLifecycle(sessionId);
+      if (lifecycle.awaitingHeartbeat) continue;
+      if (lifecycle.lastHeartbeatAtMs && lifecycle.lastHeartbeatAtMs > heartbeatCutoff) continue;
+      if (!registry.removeWorker(sessionId)) continue;
+      lifecycleBySession.delete(sessionId);
+      emitMonitorLog(`worker ${worker.workerId || sessionId} was idle and its process (pid ${worker.pid}) is gone; no longer listed`);
+      retired.push({ sdkSessionId: sessionId, workerId: worker.workerId || null, pid: worker.pid });
+    }
+    return retired;
+  }
+
   function snapshot({ pendingQuestionSessionIds = null } = {}) {
-    const workers = registry.listWorkers();
     const questionPendingSet = normalizePendingQuestionSessionIds(pendingQuestionSessionIds);
+    retireDeadIdleWorkers({ keep: questionPendingSet });
+    const workers = registry.listWorkers();
     const workerBySession = new Map();
     for (const worker of workers) {
       const sid = normalizeSessionId(worker?.sdkSessionId);