mixdog 0.9.147 → 0.9.149

package/src/runtime/agent/orchestrator/tools/lib/native-spawn-client.mjs

@@ -11,14 +11,125 @@ import { ensureSpawnBinary, findCachedSpawnBinary } from '../spawn-binary-fetche
 
 const RESTART_BACKOFF_MS = 30_000;
 const SERVER_READY_TIMEOUT_MS = 5_000;
+const REQUIRED_LIFECYCLE_CAPS = Object.freeze([
+  'trackedForeground',
+  'promoteTask',
+  'cancelOwner',
+]);
 
 let _server = null;
 let _binaryPath = undefined;
 let _lastFailureAt = 0;
 const _tasks = new Map();
-const _pidToRequest = new Map();
 const _taskEvents = new EventEmitter();
 
+// Retention of SETTLED tasks. A completed task stays queryable so a late
+// `task read` still answers, but the spawn server's `task_released` event is
+// the only thing that used to drop it — a server that never sends one (old
+// binary, crash, cancelled owner) grew _tasks without bound for the daemon's
+// whole life. Terminal entries now carry their own TTL plus a hard cap, and
+// releaseNativeTask() gives owners an explicit release.
+const RETAINED_TASK_TTL_MS = 10 * 60_000;
+const RETAINED_TASK_LIMIT = 256;
+const RETAINED_TASK_SWEEP_MS = 60_000;
+const _terminalAtMs = new Map();
+let _retentionSweepTimer = null;
+
+// The spawn server keeps its OWN retained-jobs map, so a client-side drop must
+// tell it to drop the same entry — otherwise the leak just moves one process
+// over. Older servers without the capability simply keep their cap-based
+// pruning.
+function _requestServerTaskRelease(jobId) {
+  const server = _server;
+  if (!server?.ready || server.caps?.releaseTask !== true) return false;
+  try {
+    server.child.stdin.write(`${JSON.stringify({ id: ++server.sequence, releaseTask: jobId })}\n`);
+    return true;
+  } catch { return false; }
+}
+
+function _stopRetentionSweep() {
+  if (!_retentionSweepTimer) return;
+  clearInterval(_retentionSweepTimer);
+  _retentionSweepTimer = null;
+}
+
+// Retention must not depend on new task events arriving: a daemon that goes
+// quiet right after its last command would otherwise hold that entry (and the
+// server's) forever. The sweep runs on its own unref'd cadence and stops
+// itself as soon as nothing is retained.
+function _startRetentionSweep() {
+  if (_retentionSweepTimer || _terminalAtMs.size === 0) return;
+  _retentionSweepTimer = setInterval(() => {
+    _sweepRetainedTasks();
+    if (_terminalAtMs.size === 0) _stopRetentionSweep();
+  }, RETAINED_TASK_SWEEP_MS);
+  _retentionSweepTimer.unref?.();
+}
+
+function _dropTaskEntry(jobId, { notifyServer = true } = {}) {
+  _tasks.delete(jobId);
+  _terminalAtMs.delete(jobId);
+  _startedAtOverrides.delete(jobId);
+  if (notifyServer) _requestServerTaskRelease(jobId);
+  if (_terminalAtMs.size === 0) _stopRetentionSweep();
+}
+
+function _sweepRetainedTasks(now = Date.now()) {
+  for (const [jobId, at] of _terminalAtMs) {
+    if (now - at >= RETAINED_TASK_TTL_MS) _dropTaskEntry(jobId);
+  }
+  // Insertion-ordered: the oldest terminal entries go first.
+  let overflow = _terminalAtMs.size - RETAINED_TASK_LIMIT;
+  if (overflow <= 0) return;
+  for (const jobId of [..._terminalAtMs.keys()]) {
+    if (overflow-- <= 0) break;
+    _dropTaskEntry(jobId);
+  }
+}
+
+function _markTerminalTask(jobId, status) {
+  if (status === 'running') {
+    _terminalAtMs.delete(jobId);
+    return;
+  }
+  if (!_terminalAtMs.has(jobId)) _terminalAtMs.set(jobId, Date.now());
+  _sweepRetainedTasks();
+  _startRetentionSweep();
+}
+
+/** Drop a settled task's retained state. Running tasks are never released
+ *  (cancel them first); returns true when an entry was actually dropped. */
+export function releaseNativeTask(jobId) {
+  const key = String(jobId || '').trim();
+  const task = key ? _tasks.get(key) : null;
+  if (!task || task.status === 'running') return false;
+  _dropTaskEntry(key);
+  return true;
+}
+
+function nativeSpawnCompatibilityError(server) {
+  if (!server?.ready) return null;
+  const missingCaps = REQUIRED_LIFECYCLE_CAPS.filter((cap) => server.caps?.[cap] !== true);
+  if (missingCaps.length === 0) return null;
+  const error = new Error(
+    `native spawn server is incompatible; missing required lifecycle capabilities: ${missingCaps.join(', ')}`,
+  );
+  error.code = 'NATIVE_SPAWN_INCOMPATIBLE';
+  error.missingCaps = missingCaps;
+  return error;
+}
+
+function assertNativeSpawnCompatibility(server) {
+  const error = nativeSpawnCompatibilityError(server);
+  if (error) throw error;
+}
+
+export function _assertNativeSpawnCapabilitiesForTest(caps = {}) {
+  assertNativeSpawnCompatibility({ ready: true, caps });
+  return true;
+}
+
 // Warm-standby correction. The spawn server stamps a task with the moment its
 // PROCESS was created, which for a pre-spawned standby shell is up to the
 // standby TTL (2 min) before the command was ever fed to it — every consumer
@@ -80,6 +191,7 @@ function acceptTask(message) {
   const task = normalizeTask(message?.task);
   if (!task) return null;
   _tasks.set(task.jobId, task);
+  _markTerminalTask(task.jobId, task.status);
   _taskEvents.emit(task.jobId, task);
   return task;
 }
@@ -223,7 +335,7 @@ export function setNativeSpawnRequestIdle(child, idle = true) {
   return true;
 }
 
-function _teardown(error) {
+function _teardown(error, { kill = true } = {}) {
   const server = _server;
   _server = null;
   _lastFailureAt = Date.now();
@@ -242,9 +354,12 @@ function _teardown(error) {
       finishedAt: new Date().toISOString(),
     };
     _tasks.set(jobId, failed);
+    _markTerminalTask(jobId, failed.status);
     _taskEvents.emit(jobId, { ...failed });
   }
-  try { server.child.kill(); } catch {}
+  if (kill) {
+    try { server.child.kill(); } catch {}
+  }
 }
 
 export async function shutdownNativeSpawnServer(reason = 'process-exit', timeoutMs = 1_000) {
@@ -255,8 +370,13 @@ export async function shutdownNativeSpawnServer(reason = 'process-exit', timeout
     child.once('exit', () => resolve(true));
     child.once('error', () => resolve(true));
   });
+  // Graceful stop first: detach the module handle and fail pending requests,
+  // but do NOT kill the server here. Closing stdin is its shutdown signal, and
+  // it needs that window to terminate the process GROUPS it owns (POSIX) —
+  // killing the server first preempted that cleanup and orphaned children.
+  // SIGKILL below is the fallback for a server that misses the window.
+  _teardown(new Error(`native spawn server shutdown (${reason})`), { kill: false });
   try { child.stdin?.end?.(); } catch {}
-  _teardown(new Error(`native spawn server shutdown (${reason})`));
   let timer;
   const stopped = await Promise.race([
     exited,
@@ -305,7 +425,13 @@ function _ensureServer() {
       settleReady(true);
       return;
     }
-    if (String(message?.event || '').startsWith('task_')) acceptTask(message);
+    const event = String(message?.event || '');
+    if (event === 'task_released') {
+      const jobId = String(message?.jobId || '');
+      if (jobId) _dropTaskEntry(jobId, { notifyServer: false });
+    } else if (event.startsWith('task_')) {
+      acceptTask(message);
+    }
     const pending = server.pending.get(Number(message?.id));
     if (!pending) return;
     pending.onMessage(message);
@@ -333,7 +459,10 @@ export async function warmNativeSpawnServer() {
 }
 
 export async function ensureNativeSpawnServer() {
-  if (_server?.ready) return true;
+  if (_server?.ready) {
+    assertNativeSpawnCompatibility(_server);
+    return true;
+  }
   let binary = _resolveBinary();
   if (!binary) {
     binary = await ensureSpawnBinary(getPluginData());
@@ -367,12 +496,14 @@ export async function ensureNativeSpawnServer() {
       });
     }
   }
+  assertNativeSpawnCompatibility(server);
   return true;
 }
 
 export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
   const server = _ensureServer();
   if (!server) return null;
+  assertNativeSpawnCompatibility(server);
   const id = ++server.sequence;
   const fake = new NativeSpawnChild(id, (cancelId) => {
     try { server.child.stdin.write(`${JSON.stringify({ cancel: cancelId })}\n`); } catch {}
@@ -382,6 +513,12 @@ export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
       return true;
     } catch { return false; }
   });
+  // File capture engages only when the RUNNING server advertises it and the
+  // caller supplied both paths. An older server would ignore the unknown
+  // fields and keep piping, so the capability gate decides rather than hope.
+  const _fileCapture = server.caps?.fileCapture === true
+    && Boolean(spawnOptions.stdoutPath)
+    && Boolean(spawnOptions.stderrPath);
   const request = {
     id,
     program: String(shell || ''),
@@ -394,6 +531,10 @@ export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
     outputLimit: Math.max(0, Number(spawnOptions.outputLimit) || 0),
     mergeStderr: spawnOptions.mergeStderr === true,
     rawOutput: spawnOptions.rawOutput === true,
+    ...(_fileCapture ? {
+      stdoutPath: String(spawnOptions.stdoutPath),
+      stderrPath: String(spawnOptions.stderrPath),
+    } : {}),
     ...(spawnOptions.stdinPipe === true ? { stdinPipe: true } : {}),
     command: spawnOptions.command || undefined,
     shellType: spawnOptions.shellType || undefined,
@@ -417,7 +558,6 @@ export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
       const event = String(message?.event || '');
       if (event === 'spawned') {
         fake.pid = Number(message.pid) || undefined;
-        if (fake.pid) _pidToRequest.set(fake.pid, id);
         fake.emit('spawn');
         return;
       }
@@ -440,7 +580,6 @@ export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
       }
       else if (event === 'exit') {
         server.pending.delete(id);
-        if (fake.pid) _pidToRequest.delete(fake.pid);
         fake.exitCode = message.code == null ? null : Number(message.code);
         fake.signalCode = message.signal || null;
         fake._closed = true;
@@ -463,7 +602,8 @@ export function tryNativeSpawn({ shell, argv, spawnOptions = {}, cwd } = {}) {
   }
   return {
     child: fake,
-    adoptErrorHandler(handler) {
+    fileCapture: _fileCapture,
+    attachErrorHandler(handler) {
       fake.on('error', handler);
     },
   };
@@ -474,6 +614,20 @@ export function getNativeTask(jobId) {
   return task ? { ...task } : null;
 }
 
+/** Whether the live server can hand a child its own capture-file fds.
+ *  Callers ask BEFORE opening those files so a server without the capability
+ *  never leaves an empty pair behind.
+ *
+ *  win32 EXCEPTION: there is no process group to interrogate there, so a
+ *  command's surviving descendants are observed by the stdout/stderr pipes
+ *  they INHERITED still being held after the shell itself exited (see
+ *  lib/shell-descendants.mjs). File capture removes exactly that evidence.
+ *  POSIX loses nothing — its survivor check reads the process group. */
+export function nativeSpawnFileCaptureReady() {
+  if (process.platform === 'win32') return false;
+  return _server?.ready === true && _server?.caps?.fileCapture === true;
+}
+
 export function listNativeTasks() {
   return [..._tasks.values()].map((task) => ({ ...task }));
 }
@@ -494,23 +648,81 @@ export function cancelNativeTask(jobId) {
   return task;
 }
 
-export function waitNativeTask(jobId, timeoutMs = 30_000) {
+/** Cancel every running task owned by one session.
+ *
+ *  Reports what the kill could actually prove:
+ *    cancelled    tasks that were running when the cancel was issued
+ *    unconfirmed  jobIds whose cancel reached NOTHING — no live spawn-server
+ *                 link, so the process stays observable but unaddressable from
+ *                 this host (the Windows survivor case)
+ *    confirmed    false while any unconfirmed entry exists
+ *  Delivery to the owning spawn server counts as confirmed (it holds the
+ *  handle and completes the kill); an undelivered signal never does. */
+export function cancelNativeTasks({ ownerSessionId = null } = {}) {
+  const owner = String(ownerSessionId || '').trim();
+  if (!owner) return { cancelled: 0, unconfirmed: [], confirmed: true };
+  const running = [..._tasks.values()].filter((task) =>
+    task.status === 'running' && String(task.ownerSessionId || '') === owner);
+  const server = _server;
+  let ownerCancelDelivered = false;
+  if (server?.caps?.cancelOwner === true) {
+    try {
+      server.child.stdin.write(`${JSON.stringify({
+        id: ++server.sequence,
+        cancelOwnerSession: owner,
+      })}\n`);
+      ownerCancelDelivered = true;
+    } catch { ownerCancelDelivered = false; }
+  }
+  // Also issue direct per-task cancellation for entries already visible in
+  // this client so teardown does not wait on the owner-wide command round trip.
+  // A teardown never BOOTS a spawn server just to cancel: with the link gone
+  // there is nothing left for the signal to travel through, and pretending
+  // otherwise is what turned an unkillable survivor into a false `cancelled`.
+  const unconfirmed = [];
+  for (const task of running) {
+    let delivered = ownerCancelDelivered;
+    if (server) {
+      try { delivered = Boolean(cancelNativeTask(task.jobId)) || delivered; } catch { /* undelivered */ }
+    }
+    if (!delivered) unconfirmed.push(task.jobId);
+  }
+  return { cancelled: running.length, unconfirmed, confirmed: unconfirmed.length === 0 };
+}
+
+// Completion-event wait. Resolves the moment the task leaves `running`, so the
+// timeout is a ceiling and never a fixed delay. An abort signal resolves with
+// the CURRENT state instead of leaving the caller blocked: a cancelled turn
+// must return immediately, not serve out the remaining ceiling.
+export function waitNativeTask(jobId, timeoutMs = 30_000, signal = null) {
   const key = String(jobId || '');
   const current = getNativeTask(key);
   if (!current || current.status !== 'running') return Promise.resolve(current);
+  if (signal?.aborted) return Promise.resolve(current);
   return new Promise((resolve) => {
     let timer = null;
-    const done = (task) => {
-      if (task?.status === 'running') return;
+    let onAbort = null;
+    let settled = false;
+    const finish = (task) => {
+      if (settled) return;
+      settled = true;
       if (timer) clearTimeout(timer);
+      if (onAbort && signal) {
+        try { signal.removeEventListener('abort', onAbort); } catch {}
+      }
       unsubscribe();
       resolve(task ? { ...task } : null);
     };
+    const done = (task) => {
+      if (task?.status === 'running') return;
+      finish(task);
+    };
     const unsubscribe = subscribeNativeTask(key, done);
-    timer = setTimeout(() => {
-      unsubscribe();
-      resolve(getNativeTask(key));
-    }, Math.max(1, Number(timeoutMs) || 30_000));
+    if (signal) {
+      onAbort = () => finish(getNativeTask(key));
+      try { signal.addEventListener('abort', onAbort, { once: true }); } catch {}
+    }
+    timer = setTimeout(() => finish(getNativeTask(key)), Math.max(1, Number(timeoutMs) || 30_000));
     timer.unref?.();
   });
 }
@@ -572,19 +784,10 @@ export async function startNativeTask({
   });
 }
 
-export function adoptNativeTaskByPid({
-  pid,
-  command = '',
-  cwd = '',
-  timeoutMs = 0,
-  shellType = null,
-  ownerSessionId = null,
-  clientHostPid = null,
-  jobId = `job_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-} = {}) {
-  const requestId = _pidToRequest.get(Number(pid));
-  const server = _ensureServer();
-  if (!server || !requestId) return null;
+function requestNativeTaskState(jobId, payload, errorLabel) {
+  const key = String(jobId || '').trim();
+  const server = _server;
+  if (!server || !key) return null;
   const id = ++server.sequence;
   return new Promise((resolve, reject) => {
     let timer = null;
@@ -597,14 +800,14 @@ export function adoptNativeTaskByPid({
     };
     server.pending.set(id, {
       fail(error) {
-        finish(null, error instanceof Error ? error : new Error(String(error || 'native adoption failed')));
+        finish(null, error instanceof Error ? error : new Error(String(error || `${errorLabel} failed`)));
       },
       onMessage(message) {
         const event = String(message?.event || '');
-        if (event === 'task_started') {
-          finish(getNativeTask(jobId));
+        if (event === 'task_started' || event === 'task_status') {
+          finish(getNativeTask(key));
         } else if (event === 'error') {
-          const error = new Error(String(message.message || 'native adoption failed'));
+          const error = new Error(String(message.message || `${errorLabel} failed`));
           if (message.code) error.code = String(message.code);
           finish(null, error);
         }
@@ -612,28 +815,64 @@ export function adoptNativeTaskByPid({
     });
     _setServerReferenced(server, true);
     try {
-      server.child.stdin.write(`${JSON.stringify({
-        id,
-        adopt: requestId,
-        jobId,
-        timeoutMs,
-        command,
-        cwd,
-        shellType,
-        ownerSessionId,
-        clientHostPid,
-      })}\n`);
+      server.child.stdin.write(`${JSON.stringify({ id, ...payload })}\n`);
     } catch (error) {
       finish(null, error);
       return;
     }
     timer = setTimeout(() => {
-      finish(null, Object.assign(new Error('native task adoption timeout'), { code: 'ETIMEDOUT' }));
+      finish(null, Object.assign(new Error(`${errorLabel} timeout`), { code: 'ETIMEDOUT' }));
     }, 5_000);
     timer.unref?.();
   });
 }
 
+export function trackNativeForegroundTask({
+  child,
+  jobId,
+  command = '',
+  cwd = '',
+  shellType = null,
+  ownerSessionId = null,
+  clientHostPid = null,
+} = {}) {
+  const requestId = Number(child?._id) || 0;
+  const server = _server;
+  if (!server || server.caps?.trackedForeground !== true || !requestId) return null;
+  return requestNativeTaskState(jobId, {
+    track: requestId,
+    jobId,
+    command,
+    cwd,
+    shellType,
+    ownerSessionId,
+    clientHostPid,
+  }, 'native task tracking');
+}
+
+export function promoteNativeTask({
+  jobId,
+  command = '',
+  cwd = '',
+  timeoutMs = 0,
+  shellType = null,
+  ownerSessionId = null,
+  clientHostPid = null,
+} = {}) {
+  const key = String(jobId || '').trim();
+  const server = _server;
+  if (server?.caps?.promoteTask !== true || !getNativeTask(key)) return null;
+  return requestNativeTaskState(key, {
+    promoteTask: key,
+    timeoutMs,
+    command,
+    cwd,
+    shellType,
+    ownerSessionId,
+    clientHostPid,
+  }, 'native task promotion');
+}
+
 // Capability handshake: true only after the connected spawn server announced
 // stdinPipe support in its ready line. An older binary silently ignores the
 // unknown stdinPipe field and gives the child a null stdin — a standby taken
@@ -648,10 +887,21 @@ export function _resetNativeSpawnClientForTest() {
   _binaryPath = undefined;
   _lastFailureAt = 0;
   _tasks.clear();
-  _pidToRequest.clear();
+  _terminalAtMs.clear();
+  _stopRetentionSweep();
+  _startedAtOverrides.clear();
   _taskEvents.removeAllListeners();
 }
 
+/** Test-only: seed a task record exactly as a spawn-server report would, so
+ *  cancellation/teardown paths can be exercised without a live server. */
+export function _seedNativeTaskForTest(task) {
+  const normalized = normalizeTask(task);
+  if (!normalized) return null;
+  _tasks.set(normalized.jobId, normalized);
+  return normalized;
+}
+
 export function _setNativeSpawnBinaryForTest(binaryPath) {
   const path = String(binaryPath || '');
   if (!path || !existsSync(path)) throw new Error(`native spawn test binary not found: ${path}`);

package/src/runtime/agent/orchestrator/tools/lib/native-spawn-client.test.mjs

@@ -0,0 +1,104 @@
+import assert from 'node:assert/strict';
+import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import test from 'node:test';
+import { setTimeout as delay } from 'node:timers/promises';
+
+import { execShellCommand } from '../shell-command.mjs';
+import { retireShellJobRecord } from '../builtin/lib/shell-job-records.mjs';
+import {
+  _assertNativeSpawnCapabilitiesForTest,
+  _resetNativeSpawnClientForTest,
+  _setNativeSpawnBinaryForTest,
+  cancelNativeTasks,
+  listNativeTasks,
+  shutdownNativeSpawnServer,
+  waitNativeTask,
+} from './native-spawn-client.mjs';
+
+test('native spawn rejects binaries without the required lifecycle protocol', () => {
+  assert.throws(
+    () => _assertNativeSpawnCapabilitiesForTest({
+      trackedForeground: true,
+      promoteTask: true,
+    }),
+    (error) => error?.code === 'NATIVE_SPAWN_INCOMPATIBLE'
+      && error.missingCaps?.includes('cancelOwner'),
+  );
+  assert.equal(_assertNativeSpawnCapabilitiesForTest({
+    trackedForeground: true,
+    promoteTask: true,
+    cancelOwner: true,
+  }), true);
+});
+
+test('foreground shell has stable owner identity before promotion', { timeout: 15_000 }, async (t) => {
+  const binary = resolve(
+    'native/mixdog-spawn/target/debug',
+    process.platform === 'win32' ? 'mixdog-spawn.exe' : 'mixdog-spawn',
+  );
+  if (!existsSync(binary)) {
+    t.skip(`native spawn test binary not found: ${binary}`);
+    return;
+  }
+  const root = mkdtempSync(join(tmpdir(), 'mixdog-native-owner-'));
+  const previousDataDir = process.env.MIXDOG_DATA_DIR;
+  process.env.MIXDOG_DATA_DIR = root;
+  const ownerSessionId = `sess_native_owner_${process.pid}`;
+  let jobId = null;
+  const admission = {
+    async acquire() {
+      return {
+        signal: null,
+        async detachDependency() {},
+        async release() {},
+      };
+    },
+  };
+  try {
+    _setNativeSpawnBinaryForTest(binary);
+    const running = execShellCommand({
+      shell: process.execPath,
+      shellArg: '',
+      command: 'native stable foreground ownership test',
+      directArgv: ['-e', 'setTimeout(() => {}, 10000)'],
+      env: process.env,
+      cwd: process.cwd(),
+      timeoutMs: 5_000,
+      autoBackgroundMs: 250,
+      backgroundOnTimeout: true,
+      promotedTimeoutMs: 5_000,
+      backgroundDeadlineMs: 5_000,
+      ownerSessionId,
+      clientHostPid: process.pid,
+      admission,
+    });
+
+    let foreground = null;
+    for (let attempt = 0; attempt < 100; attempt += 1) {
+      foreground = listNativeTasks().find((task) =>
+        task.status === 'running' && task.ownerSessionId === ownerSessionId);
+      if (foreground) break;
+      await delay(10);
+    }
+    assert.ok(foreground, 'foreground process was not owner-tracked at spawn');
+    jobId = foreground.jobId;
+
+    const result = await running;
+    assert.equal(result.backgrounded, true);
+    assert.equal(result.jobId, jobId);
+
+    assert.ok(cancelNativeTasks({ ownerSessionId }).cancelled >= 1);
+    const terminal = await waitNativeTask(jobId, 3_000);
+    assert.notEqual(terminal?.status, 'running');
+    assert.equal(terminal?.status, 'cancelled');
+  } finally {
+    if (jobId) retireShellJobRecord(jobId);
+    await shutdownNativeSpawnServer('test-finish').catch(() => {});
+    _resetNativeSpawnClientForTest();
+    if (previousDataDir === undefined) delete process.env.MIXDOG_DATA_DIR;
+    else process.env.MIXDOG_DATA_DIR = previousDataDir;
+    rmSync(root, { recursive: true, force: true });
+  }
+});