mixdog 0.9.50 → 0.9.52

package/src/standalone/agent-tool.mjs

@@ -15,6 +15,7 @@ import { presentErrorText, errorLine } from '../runtime/shared/err-text.mjs';
 import { updateJsonAtomicSync } from '../runtime/shared/atomic-file.mjs';
 import { normalizeAgentPermission } from '../runtime/shared/markdown-frontmatter.mjs';
 import { ensureProcessListenerHeadroom } from '../runtime/shared/process-listener-headroom.mjs';
+import { resourceAdmission } from '../runtime/shared/resource-admission.mjs';
 import { prepareAgentSession } from '../runtime/agent/orchestrator/agent-runtime/session-builder.mjs';
 import {
   abortAgentProgressWatchdog,
@@ -76,20 +77,6 @@ const STANDALONE_SOURCE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)),
 // cap and restore strictly-unbounded prep.
 const DEFAULT_SPAWN_PREP_TIMEOUT_MS = envTimeoutMs('MIXDOG_AGENT_SPAWN_PREP_TIMEOUT_MS', 120_000);
 
-// Global spawn-start stagger: unlimited-N parallel fan-out otherwise fires all
-// first provider calls in the same instant, racing the server-side prompt-
-// cache write/propagation window. Default 0 (off): mirrors the explore fan-out
-// finding — the first spawn's prompt-cache write only lands after its iter1
-// completes (~seconds), so a sub-second stagger yields ~no cross-spawn cache
-// reads while charging every later spawn the full delay, i.e. pure fan-out
-// latency for negligible hit-rate gain. Kept as a knob for tuning: set
-// MIXDOG_SPAWN_STAGGER_MS>0 to re-enable. When >0 it chains (not a fixed lane
-// count) so it scales to any N: each new spawn's start is pushed to at least
-// STAGGER_MS after the previous spawn's start; sequential/non-overlapping
-// spawns pay zero added latency. Applied inside the deferred job body (see
-// startDeferredSpawnJob) so the agent tool call itself still returns task_id
-// immediately.
-const SPAWN_STAGGER_MS = envTimeoutMs('MIXDOG_SPAWN_STAGGER_MS', 0);
 const TAG_TOMBSTONE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
 const MAX_TAG_TOMBSTONES = 500;
 
@@ -156,20 +143,6 @@ export function resolveAgentSpawnPreset(config, args = {}) {
   if (!preset) throw new Error(`agent: preset "${presetName}" not found`);
   return { presetName, preset };
 }
-let lastSpawnStartAt = 0;
-async function waitForSpawnStagger() {
-  if (SPAWN_STAGGER_MS <= 0) return;
-  const now = Date.now();
-  const myStart = Math.max(now, lastSpawnStartAt + SPAWN_STAGGER_MS);
-  lastSpawnStartAt = myStart;
-  const delay = myStart - now;
-  if (delay <= 0) return;
-  await new Promise((resolve) => {
-    const t = setTimeout(resolve, delay);
-    t.unref?.();
-  });
-}
-
 export function createStandaloneAgent({ cfgMod, reg, mgr, dataDir, cwd: defaultCwd, onSubagentEvent }) {
   // Optional bridge to the standard hook bus for SubagentStart / SubagentStop.
   // Best-effort: a hook error must never affect worker spawn/finish.
@@ -1042,6 +1015,7 @@ export function createStandaloneAgent({ cfgMod, reg, mgr, dataDir, cwd: defaultC
       ...(clientHostPid ? { clientHostPid } : {}),
     });
     let task;
+    const admissionController = new AbortController();
     task = startBackgroundTask({
       surface: 'agent',
       operation: type,
@@ -1052,18 +1026,27 @@ export function createStandaloneAgent({ cfgMod, reg, mgr, dataDir, cwd: defaultC
       resultType: 'agent_task_result',
       renderResult: (result) => renderResult(result),
       cancel: () => {
+        try { admissionController.abort(new Error('agent task cancelled before resource admission')); } catch {}
         const currentMeta = task?.meta || jobMeta;
         if (currentMeta?.sessionId) {
           try { mgr.closeSession(currentMeta.sessionId, 'agent-task-cancel'); } catch {}
         }
       },
       run: async () => {
-        // Yield one macrotask before doing agent work. startBackgroundTask uses
-        // a Promise microtask, which otherwise begins CPU-heavy spawn prep
-        // before the TUI receives/render the "running" result.
-        await new Promise((resolve) => setImmediate(resolve));
-        if (task?.status === 'cancelled') return null;
-        return await run(task);
+        const lease = await resourceAdmission.acquire('agent', {
+          signal: admissionController.signal,
+          label: jobMeta?.tag || type,
+        });
+        try {
+          // Yield one macrotask before doing agent work. startBackgroundTask uses
+          // a Promise microtask, which otherwise begins CPU-heavy spawn prep
+          // before the TUI receives/render the "running" result.
+          await new Promise((resolve) => setImmediate(resolve));
+          if (task?.status === 'cancelled') return null;
+          return await resourceAdmission.runWithLease(lease, () => run(task));
+        } finally {
+          await lease.release();
+        }
       },
     });
     return task;
@@ -1071,10 +1054,6 @@ export function createStandaloneAgent({ cfgMod, reg, mgr, dataDir, cwd: defaultC
 
   function startDeferredSpawnJob(args, callerCwd, context, notifyContext, extras = {}) {
     return startJob('spawn', pendingSpawnMeta(args, extras), async (job) => {
-      // Stagger concurrent spawn starts before prep/model-call begins (see
-      // SPAWN_STAGGER_MS above). Runs inside the job body, so the tool call
-      // that queued this job already returned task_id before this awaits.
-      await waitForSpawnStagger();
       if (job?.status === 'cancelled') return null;
       // prepareSpawn (ensureProvider/prepareAgentSession) runs before runSpawn
       // installs its progress watchdog, so guard prep with an internal env-

package/src/standalone/channel-daemon-client.mjs

@@ -6,6 +6,7 @@
 // worker->parent notify path arrives on a persistent SSE stream (GET /events),
 // replacing the old node-IPC `{type:'notify'}` messages.
 import http from 'node:http';
+import { randomUUID } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 
 function parsePid(value) {
@@ -41,7 +42,29 @@ export function readDaemonDiscovery(discoveryPath) {
 function request({ port, method = 'GET', path = '/', token, body = null, timeoutMs = 10_000 }) {
   return new Promise((resolve, reject) => {
     const payload = body == null ? null : JSON.stringify(body);
-    const req = http.request({
+    let req = null;
+    let response = null;
+    let ended = false;
+    let settled = false;
+    const finish = (fn, value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(deadline);
+      fn(value);
+    };
+    const fail = (error) => finish(reject, error instanceof Error ? error : new Error(String(error)));
+    const lifecycleTimeout = () => {
+      const error = new Error(`daemon request timed out: ${method} ${path}`);
+      fail(error);
+      try { response?.destroy?.(error); } catch {}
+      try { req?.destroy?.(error); } catch {}
+    };
+    // http.request's timeout is socket-idle only. This deadline covers headers
+    // and the entire response body so a truncated post-header response cannot
+    // leave reconnect registration pending forever.
+    const deadline = setTimeout(lifecycleTimeout, timeoutMs);
+    deadline.unref?.();
+    req = http.request({
       hostname: '127.0.0.1',
       port,
       path,
@@ -54,23 +77,31 @@ function request({ port, method = 'GET', path = '/', token, body = null, timeout
       },
       timeout: timeoutMs,
     }, (res) => {
+      response = res;
+      if (settled) { try { res.resume?.(); } catch {} return; }
       let data = '';
       res.setEncoding('utf8');
-      res.on('data', (chunk) => { data += chunk; });
-      res.on('end', () => {
+      res.on('data', (chunk) => { if (!settled) data += chunk; });
+      res.once('aborted', () => fail(new Error(`daemon response aborted: ${method} ${path}`)));
+      res.once('error', (error) => fail(error));
+      res.once('end', () => {
+        ended = true;
         let parsed = null;
         try { parsed = data ? JSON.parse(data) : null; } catch {}
         if (res.statusCode && res.statusCode >= 400) {
           const err = new Error(parsed?.error || data || `HTTP ${res.statusCode}`);
           err.statusCode = res.statusCode;
-          reject(err);
+          fail(err);
           return;
         }
-        resolve(parsed ?? {});
+        finish(resolve, parsed ?? {});
+      });
+      res.once('close', () => {
+        if (!ended && !settled) fail(new Error(`daemon response closed before end: ${method} ${path}`));
       });
     });
-    req.on('error', reject);
-    req.on('timeout', () => { req.destroy(); reject(new Error(`daemon request timed out: ${method} ${path}`)); });
+    req.on('error', fail);
+    req.on('timeout', lifecycleTimeout);
     if (payload) req.write(payload);
     req.end();
   });
@@ -96,29 +127,104 @@ export async function attachToDaemon({
   log = () => {},
   onFatal = () => {},
 } = {}) {
-  if (!discovery?.port || !discovery?.token) throw new Error('daemon discovery {port, token} required');
+  const expectedPid = parsePid(discovery?.pid);
+  if (!discovery?.port || !discovery?.token || !expectedPid) throw new Error('daemon discovery {port, pid, token} required');
   const { port, token: serverToken } = discovery;
+  const staleDiscoveryError = (reason) => {
+    const err = new Error(reason);
+    err.daemonDiscoveryStale = true;
+    return err;
+  };
+  const isExpectedDaemon = (health) => Number(health?.pid) === expectedPid;
 
-  const reg = await request({
-    port,
-    token: serverToken,
-    method: 'POST',
-    path: '/client/register',
-    body: { leadPid, cwd },
-    timeoutMs: 3000,
-  });
+  const initialHealth = await probeDaemonHealth({ port, token: serverToken, timeoutMs: 800 });
+  if (!isExpectedDaemon(initialHealth)) throw staleDiscoveryError('daemon discovery pid does not match health');
+
+  let reg;
+  try {
+    reg = await request({
+      port,
+      token: serverToken,
+      method: 'POST',
+      path: '/client/register',
+      body: { leadPid, cwd },
+      timeoutMs: 3000,
+    });
+  } catch (err) {
+    if (err?.statusCode === 401 || err?.statusCode === 403) {
+      const stale = staleDiscoveryError(`daemon register rejected (${err.statusCode})`);
+      stale.daemonAuthRejected = true;
+      throw stale;
+    }
+    throw err;
+  }
   let clientToken = reg?.token;
   if (!clientToken) throw new Error('daemon register returned no client token');
 
   let sseReq = null;
   let closed = false;
+  let fatal = false;
   let reconnectTimer = null;
-  // Bounded SSE reconnect: after repeated failures (daemon dead/restarted) give
-  // up and signal the owner via onFatal so it can invalidate the cached attach
-  // and re-attach fresh (re-read discovery, respawn-if-dead) rather than spin
-  // forever against a dead port.
+  let reconnectProbe = false;
+  let reconnectRegistration = null;
+  let reconnectRegistrationId = null;
+  let reconnectReplaceToken = null;
+  let lifecycle = 0;
+  let stableTimer = null;
+  let closePromise = null;
+  // Bounded reconnect is only for a verified-live daemon's transient SSE loss.
+  // A stale/dead endpoint signals onFatal immediately so the owner re-reads
+  // discovery instead of spinning against the captured port.
   let reconnectAttempts = 0;
   const MAX_RECONNECTS = 5;
+  const STABLE_STREAM_MS = 5_000;
+
+  async function deregister(token, { registrationId = null, replaceToken = null } = {}) {
+    if (!token) return;
+    try {
+      await request({
+        port, token: serverToken, method: 'POST', path: '/client/deregister',
+        body: {
+          token,
+          ...(registrationId ? { registrationId, replaceToken, leadPid, cwd } : {}),
+        },
+        timeoutMs: 1500,
+      });
+    } catch { /* best-effort; daemon sweep reaps us */ }
+  }
+
+  function clearStableTimer() {
+    if (stableTimer) { try { clearTimeout(stableTimer); } catch {} stableTimer = null; }
+  }
+
+  function signalFatal(reason) {
+    if (closed || fatal) return;
+    fatal = true;
+    closed = true;
+    lifecycle++;
+    if (reconnectTimer) { try { clearTimeout(reconnectTimer); } catch {} reconnectTimer = null; }
+    clearStableTimer();
+    try { sseReq?.destroy?.(); } catch {}
+    log(`sse stale endpoint (${reason}); signalling re-attach`);
+    try { onFatal(reason); } catch {}
+  }
+
+  // A stream ending can be a transient connection loss, but must not leave this
+  // client retrying a dead discovery endpoint. Verify that the original daemon
+  // is still alive before spending the bounded reconnect budget on it.
+  function handleStreamLoss(reason) {
+    if (closed || fatal || reconnectTimer || reconnectProbe) return;
+    reconnectProbe = true;
+    void probeDaemonHealth({ port, token: serverToken, timeoutMs: 800 }).then((health) => {
+      reconnectProbe = false;
+      if (closed || fatal) return;
+      if (!isExpectedDaemon(health)) {
+        signalFatal(reason);
+        return;
+      }
+      scheduleReconnect(reason);
+    });
+  }
 
   function openStream() {
     if (closed) return;
@@ -129,15 +235,29 @@ export async function attachToDaemon({
       method: 'GET',
       headers: { Accept: 'text/event-stream', 'X-Mixdog-Daemon-Token': serverToken },
     }, (res) => {
+      if (req !== sseReq || closed) { res.resume(); return; }
       if (res.statusCode !== 200) {
         res.resume();
-        scheduleReconnect('bad sse status ' + res.statusCode);
+        // A token rejection means this port now belongs to a different daemon;
+        // re-read discovery now rather than re-registering against it.
+        if (res.statusCode === 401 || res.statusCode === 403) {
+          signalFatal('bad sse status ' + res.statusCode);
+        } else {
+          handleStreamLoss('bad sse status ' + res.statusCode);
+        }
         return;
       }
       res.setEncoding('utf8');
-      reconnectAttempts = 0; // a live stream resets the failure budget
+      // A bare 200 followed by an immediate end is not a stable stream. Only
+      // reset the bounded reconnect budget after this exact stream stays live.
+      clearStableTimer();
+      stableTimer = setTimeout(() => {
+        if (!closed && !fatal && req === sseReq) reconnectAttempts = 0;
+      }, STABLE_STREAM_MS);
+      stableTimer.unref?.();
       let buf = '';
       res.on('data', (chunk) => {
+        if (req !== sseReq || closed) return;
         buf += chunk;
         let idx;
         while ((idx = buf.indexOf('\n\n')) >= 0) {
@@ -153,35 +273,67 @@ export async function attachToDaemon({
           }
         }
       });
-      res.on('end', () => scheduleReconnect('sse ended'));
-      res.on('error', () => scheduleReconnect('sse error'));
+      res.on('end', () => {
+        if (req === sseReq) {
+          clearStableTimer();
+          handleStreamLoss('sse ended');
+        }
+      });
+      res.on('error', () => {
+        if (req === sseReq) {
+          clearStableTimer();
+          handleStreamLoss('sse error');
+        }
+      });
+    });
+    req.on('error', () => {
+      if (req === sseReq) signalFatal('sse req error');
     });
-    req.on('error', () => scheduleReconnect('sse req error'));
-    req.end();
     sseReq = req;
+    req.end();
   }
 
   function scheduleReconnect(reason) {
     if (closed || reconnectTimer) return;
     if (++reconnectAttempts > MAX_RECONNECTS) {
-      log(`sse giving up after ${reconnectAttempts} attempts (${reason}); signalling re-attach`);
-      closed = true;
-      try { sseReq?.destroy?.(); } catch {}
-      try { onFatal(reason); } catch {}
+      signalFatal(`giving up after ${reconnectAttempts} attempts (${reason})`);
       return;
     }
     log(`sse reconnect scheduled (${reason}, attempt ${reconnectAttempts})`);
     reconnectTimer = setTimeout(() => {
       reconnectTimer = null;
+      if (closed || fatal) return;
       // Re-register (the daemon may have pruned us) then reopen the stream.
       // Re-register mints a fresh client token (the old one was pruned); adopt
       // it so the reopened stream and subsequent calls target the live entry.
       // reattach:true — a reconnect is NOT a new terminal, so it must NOT steal
       // the ownership seat (last-wins is reserved for genuine fresh registers).
-      request({ port, token: serverToken, method: 'POST', path: '/client/register', body: { leadPid, cwd, reattach: true }, timeoutMs: 3000 })
-        .then((r) => { if (r?.token) clientToken = r.token; })
-        .catch(() => {})
-        .finally(() => openStream());
+      const generation = lifecycle;
+      const registrationId = reconnectRegistrationId ||= randomUUID();
+      const replaceToken = reconnectReplaceToken ||= clientToken;
+      const registration = request({
+        port, token: serverToken, method: 'POST', path: '/client/register',
+        body: {
+          leadPid, cwd, reattach: true, replaceToken,
+          registrationId,
+        }, timeoutMs: 3000,
+      }).then(async (r) => {
+        const freshToken = r?.token;
+        if (!freshToken) return false;
+        if (closed || fatal || generation !== lifecycle) {
+          await deregister(freshToken);
+          return false;
+        }
+        clientToken = freshToken;
+        reconnectRegistrationId = null;
+        reconnectReplaceToken = null;
+        return true;
+      }).catch(() => false);
+      reconnectRegistration = registration;
+      void registration.finally(() => {
+        if (reconnectRegistration === registration) reconnectRegistration = null;
+        if (!closed && !fatal && generation === lifecycle) openStream();
+      });
     }, 1000);
     reconnectTimer.unref?.();
   }
@@ -213,13 +365,23 @@ export async function attachToDaemon({
   }
 
   async function close(reason = 'client close') {
+    if (closePromise) return closePromise;
     closed = true;
+    lifecycle++;
     if (reconnectTimer) { try { clearTimeout(reconnectTimer); } catch {} reconnectTimer = null; }
+    clearStableTimer();
     try { sseReq?.destroy?.(); } catch {}
-    try {
-      await request({ port, token: serverToken, method: 'POST', path: '/client/deregister', body: { token: clientToken }, timeoutMs: 1500 });
-    } catch { /* best-effort; daemon sweep reaps us */ }
-    log(`detached (${reason})`);
+    // A re-register already in flight can mint a fresh token after close().
+    // Await it so its stale branch deregisters that token before close resolves.
+    const pendingRegistration = reconnectRegistration;
+    const registrationId = reconnectRegistrationId;
+    const replaceToken = reconnectReplaceToken;
+    closePromise = (async () => {
+      if (pendingRegistration) await pendingRegistration;
+      await deregister(clientToken, { registrationId, replaceToken });
+      log(`detached (${reason})`);
+    })();
+    return closePromise;
   }
 
   return { call, close, clientToken, port };