@yemi33/minions 0.1.2044 → 0.1.2046

package/dashboard.js

@@ -2042,6 +2042,13 @@ function _markStatusCacheBuilt() {
   _statusCacheJson = null;
   _statusCacheGzip = null;
   _statusCacheVersion++;
+  // A4: keep body.version.statusCacheVersion in sync with the ETag we're
+  // about to return. Without this, the field lags by up to 60s because
+  // `version` is built in slow-state, making refresh-diagnostics unable
+  // to distinguish "server cache pinned" from "slow-state TTL not expired."
+  if (_statusCache && _statusCache.version) {
+    _statusCache.version.statusCacheVersion = _statusCacheVersion;
+  }
 }
 
 function getStatus() {
@@ -2330,6 +2337,17 @@ const CC_LOG_ERROR_MAX_LEN = 80; // truncate exception messages in [cc-stream] l
 const CC_STREAM_REATTACH_GRACE_MS = 60000; // keep CC job alive briefly after disconnect so the UI can reattach
 const CC_STREAM_DONE_RETENTION_MS = 30000; // retain final payload briefly so reconnect can still receive it
 const CC_LIVE_STREAM_MAX_AGE_MS = shared.ENGINE_DEFAULTS.ccLiveStreamMaxAgeMs;
+// W-mpmwxni2000c25c7-b — CC/doc-chat turn watchdog. Resolves per-call from
+// CONFIG.engine.ccTurnTimeoutMs (defaults to ENGINE_DEFAULTS.ccTurnTimeoutMs)
+// so an operator can shorten/lengthen the wall-clock cap without a code
+// change. callLLM's own `timeout` opt only kills the spawned child after a
+// long idle stretch; this turn-level cap kills WHICHEVER LLM call is in
+// flight inside ccCall/ccCallStreaming (resume → fresh → final retry).
+function _resolveCcTurnTimeoutMs() {
+  const cfg = CONFIG && CONFIG.engine;
+  const candidate = cfg && Number.isFinite(cfg.ccTurnTimeoutMs) ? cfg.ccTurnTimeoutMs : shared.ENGINE_DEFAULTS.ccTurnTimeoutMs;
+  return Number.isFinite(candidate) && candidate > 0 ? candidate : 0;
+}
 // Doc-chat is interactive — long-doc edits with multi-step Read+Write tool use can run
 // well past 5 min on `canEdit:true` paths. Bumped to 1 hour (matching CC) so legitimate
 // edits aren't killed mid-stream and the backend timeout never beats the user's reading
@@ -3384,6 +3402,22 @@ function _invokeDocChatViaPool({ prompt, model, effort, engineConfig, systemProm
   let timeoutTimer = null;
   let resolveResult;
   const promise = new Promise((resolve) => { resolveResult = resolve; });
+  // W-mpmwxni2000c25c7-c — build a single failure envelope shape from a
+  // typed Error (or a plain Error). Reads `.code` / `.retriable` if the
+  // pool stamped them; falls back to safe defaults otherwise so callers
+  // see a consistent `{ ..., errorCode, errorRetriable }` shape. Sub-item
+  // b's SSE writer consumes these to render a structured error event
+  // instead of grepping the stderr string for a code.
+  const _failureEnvelope = (err, defaultCode) => ({
+    text: accumulated,
+    sessionId: sessionHandle ? sessionHandle.sessionId : null,
+    code: 1,
+    usage: {},
+    raw: accumulated,
+    stderr: String((err && err.message) || err || 'cc-worker-pool failure'),
+    errorCode: (err && err.code) || defaultCode || null,
+    errorRetriable: (err && err.retriable !== undefined) ? err.retriable : true,
+  });
   const finalize = (envelope) => {
     if (settled) return;
     settled = true;
@@ -3408,14 +3442,18 @@ function _invokeDocChatViaPool({ prompt, model, effort, engineConfig, systemProm
     timeoutTimer = setTimeout(() => {
       try { sessionHandle && sessionHandle.cancel(); } catch { /* swallow */ }
       try { ccWorkerPool.closeTab(tabKey); } catch { /* swallow */ }
-      finalize({
-        text: accumulated,
-        sessionId: sessionHandle ? sessionHandle.sessionId : null,
-        code: 1,
-        usage: {},
-        raw: accumulated,
-        stderr: `doc-chat-pool: timeout after ${timeoutMs}ms`,
-      });
+      // W-mpmwxni2000c25c7-c — convert the legacy synthesized
+      // `{ code: 1, stderr: 'doc-chat-pool: timeout after Xms' }` shape into
+      // a typed-error envelope so the SSE writer can render the same
+      // structured error event for timeouts as for spawn/handshake/exit
+      // failures. The error code carries `cc-turn-timeout`; consumers
+      // grep on that instead of parsing the stderr string.
+      const timeoutErr = ccWorkerPool._typedError(
+        `doc-chat-pool: timeout after ${timeoutMs}ms`,
+        ccWorkerPool.ERROR_CODES.CC_TURN_TIMEOUT,
+        true
+      );
+      finalize(_failureEnvelope(timeoutErr, ccWorkerPool.ERROR_CODES.CC_TURN_TIMEOUT));
     }, timeoutMs);
     if (typeof timeoutTimer.unref === 'function') timeoutTimer.unref();
   }
@@ -3429,14 +3467,10 @@ function _invokeDocChatViaPool({ prompt, model, effort, engineConfig, systemProm
         systemPromptHash: _docChatPromptHash,
       });
     } catch (err) {
-      return finalize({
-        text: '',
-        sessionId: null,
-        code: 1,
-        usage: {},
-        raw: '',
-        stderr: String((err && err.message) || err || 'cc-worker-pool spawn failed'),
-      });
+      // Pool stamps `.code` (worker-spawn-failed / acp-handshake-failed) on
+      // every error from getSession; fall back to worker-spawn-failed if
+      // the error is a plain Error from somewhere unexpected.
+      return finalize(_failureEnvelope(err, ccWorkerPool.ERROR_CODES.WORKER_SPAWN_FAILED));
     }
     if (cancelled) {
       try { sessionHandle.cancel(); } catch { /* swallow */ }
@@ -3464,14 +3498,15 @@ function _invokeDocChatViaPool({ prompt, model, effort, engineConfig, systemProm
         finalize({ text: accumulated, sessionId: sessionHandle.sessionId, code: 0, usage: {}, raw: accumulated, stderr: '' });
       },
       onError: (err) => {
-        finalize({
-          text: accumulated,
-          sessionId: sessionHandle.sessionId,
-          code: cancelled ? 0 : 1,
-          usage: {},
-          raw: accumulated,
-          stderr: String((err && err.message) || err || 'cc-worker-pool stream error'),
-        });
+        if (cancelled) {
+          // User-driven cancel — not a real error, treat as a clean exit.
+          finalize({ text: accumulated, sessionId: sessionHandle.sessionId, code: 0, usage: {}, raw: accumulated, stderr: '' });
+          return;
+        }
+        // Pool stamps `.code` (worker-died for mid-stream proc exit).
+        // Fallback default is worker-died because the stream onError is
+        // overwhelmingly fired from the post-handshake exit handler.
+        finalize(_failureEnvelope(err, ccWorkerPool.ERROR_CODES.WORKER_DIED));
       },
     });
   })();
@@ -3968,12 +4003,65 @@ async function _retryDocChatAfterResumeFailure({ result, initialPass, freshSessi
 // Shape the per-failure debug envelope (raw stderr + classification metadata)
 // shared by hard failures and partial recoveries — keeps the wire shape in lockstep.
 function _buildDocChatErrorEnvelope(result) {
+  // W-mpmwxni2000c25c7-b — also surface the typed `error: {message, code,
+  // retriable}` envelope when llm.callLLM* produced one, so doc-chat clients
+  // get the same shape Command Center handlers emit.
+  const typed = result && result.error;
   return {
     code: result.code ?? null,
     stderr: String(result.stderr || '').slice(-2048),
     errorClass: result.errorClass || null,
     errorMessage: result.errorMessage || null,
     runtime: result.runtime || null,
+    ...(typed ? {
+      typedCode: typed.code || null,
+      typedMessage: typed.message || null,
+      retriable: typed.retriable !== false,
+    } : {}),
+  };
+}
+
+// W-mpmwxni2000c25c7-b — race a ccDocCall* promise against a wall-clock turn
+// timer. On expiry, fires `abortFn` (killing the in-flight CLI) and resolves
+// with a doc-chat-shaped failure payload that flows through the existing
+// _docChatFailureResponse / SSE error event paths. timeoutMs <= 0 disables
+// the watchdog (passthrough).
+async function _raceCcDocChatTimeout(callPromise, timeoutMs, abortFn, label) {
+  if (!timeoutMs || timeoutMs <= 0) return callPromise;
+  let timer = null;
+  let timedOut = false;
+  const timeoutPromise = new Promise((resolve) => {
+    timer = setTimeout(() => {
+      timedOut = true;
+      try { if (abortFn) abortFn(); } catch { /* swallow */ }
+      resolve(null);
+    }, timeoutMs);
+    // NOTE: do NOT unref — Node would exit the event loop while awaiting the
+    // call promise (Promises don't keep the loop open; timers/I/O do). Cleared
+    // immediately on the success path below.
+  });
+  const winner = await Promise.race([callPromise, timeoutPromise]);
+  if (!timedOut) {
+    clearTimeout(timer);
+    return winner;
+  }
+  // Drain the in-flight call so its cleanup runs before we hand back the
+  // synthetic envelope.
+  await callPromise.catch(() => null);
+  const message = `${label || 'doc-chat'} turn timed out after ${timeoutMs}ms`;
+  return {
+    answer: 'Document chat request timed out — try again.',
+    toolUses: [],
+    error: {
+      code: 'cc-turn-timeout',
+      stderr: '',
+      errorClass: 'cc-turn-timeout',
+      errorMessage: message,
+      runtime: null,
+      typedCode: 'cc-turn-timeout',
+      typedMessage: message,
+      retriable: true,
+    },
   };
 }
 
@@ -5848,19 +5936,17 @@ const server = http.createServer(async (req, res) => {
   }
 
   async function handleKnowledgeSweep(req, res) {
-    // Source of truth = kb-sweep-state.json + PID liveness. The sweep now runs
-    // as a detached child (engine/kb-sweep-runner.js) so it survives
-    // `minions restart`; the in-memory `global._kbSweep*` flags from the old
-    // in-process implementation are gone.
+    // Source of truth = kb-sweep-state.json + PID liveness. The sweep runs as
+    // a detached child (engine/kb-sweep-runner.js) so it survives
+    // `minions restart`. The actual spawn logic lives in
+    // engine/kb-sweep.js::spawnSweepRunnerDetached — shared with the engine's
+    // auto-sweep tick phase.
     const {
-      readSweepLiveness, staleGuardMs, KB_SWEEP_STATE_PATH, KB_SWEEP_LOG_PATH, KB_SWEEP_RUNNER_PATH,
+      readSweepLiveness, staleGuardMs, spawnSweepRunnerDetached, KB_SWEEP_STATE_PATH,
     } = require('./engine/kb-sweep');
     const entryCount = ((await queries.getKnowledgeBaseEntries()) || []).length;
     const guardMs = staleGuardMs(entryCount);
 
-    // Synchronous pre-claim BEFORE awaiting the body so a concurrent POST
-    // arriving in the same tick sees in-flight state and can't double-spawn.
-    const sweepToken = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
     const liveness = readSweepLiveness({ entryCount });
     if (liveness.inFlight && liveness.stale) {
       const reason = !liveness.alive
@@ -5875,79 +5961,15 @@ const server = http.createServer(async (req, res) => {
       });
     }
 
-    // Claim the slot synchronously by writing a "starting" state. The runner
-    // will overwrite this with status:'in-flight' + its real pid once it boots.
-    // readSweepLiveness grants a 15s boot-grace to "starting" records with no pid.
-    const startedAt = Date.now();
-    try {
-      safeWrite(KB_SWEEP_STATE_PATH, JSON.stringify({
-        status: 'starting', startedAt, startedAtIso: new Date().toISOString(),
-        sweepToken, pid: null,
-      }));
-    } catch (e) {
-      console.error(`[kb-sweep] failed to write starting state: ${e.message}`);
-    }
-
     const body = await readBody(req).catch(() => ({}));
-
-    // Persist body to a temp file so spawn doesn't have to serialize large
-    // pinnedKeys arrays via argv. Skip when body is empty.
-    let bodyFile = null;
-    if (body && (Array.isArray(body.pinnedKeys) || body.dryRun != null)) {
-      bodyFile = path.join(ENGINE_DIR, `tmp-kb-sweep-body-${sweepToken}.json`);
-      try { safeWrite(bodyFile, JSON.stringify(body)); }
-      catch (e) {
-        console.error(`[kb-sweep] failed to write body-file ${bodyFile}: ${e.message}`);
-        bodyFile = null;
-      }
-    }
-
-    const { spawn: cpSpawn } = require('child_process');
-    // Open log fd in append mode so spawn can pipe stdio there. Child inherits
-    // the fd; parent closes its copy after spawn returns successfully.
-    let logFdNum = null;
-    let stdio = ['ignore', 'ignore', 'ignore'];
-    try {
-      logFdNum = fs.openSync(KB_SWEEP_LOG_PATH, 'a');
-      stdio = ['ignore', logFdNum, logFdNum];
-    } catch (e) {
-      console.error(`[kb-sweep] failed to open log ${KB_SWEEP_LOG_PATH}: ${e.message}`);
-    }
-
-    const spawnArgs = ['--sweep-token', sweepToken];
-    if (bodyFile) spawnArgs.push('--body-file', bodyFile);
-
-    let proc;
-    try {
-      proc = cpSpawn(process.execPath, [KB_SWEEP_RUNNER_PATH, ...spawnArgs], {
-        cwd: MINIONS_DIR, stdio, detached: true, windowsHide: true,
-        env: { ...process.env },
-      });
-    } catch (e) {
-      if (logFdNum != null) try { fs.closeSync(logFdNum); } catch { /* ignore */ }
-      if (bodyFile) try { fs.unlinkSync(bodyFile); } catch { /* ignore */ }
-      // Release the "starting" claim on synchronous spawn failure so the user
-      // can retry immediately.
-      try { shared.safeUnlink(KB_SWEEP_STATE_PATH); } catch { /* ignore */ }
-      return jsonReply(res, 500, { error: `spawn failed: ${e.message}` });
+    const result = spawnSweepRunnerDetached({
+      pinnedKeys: Array.isArray(body?.pinnedKeys) ? body.pinnedKeys : undefined,
+      dryRun: body?.dryRun,
+    });
+    if (!result.ok) {
+      return jsonReply(res, 500, { error: result.error || 'spawn failed' });
     }
-    if (logFdNum != null) try { fs.closeSync(logFdNum); } catch { /* ignore */ }
-
-    // Conditional CAS: only update the state file from "starting" → "in-flight"
-    // if our sweepToken still owns it. If the (fast) runner already wrote
-    // "completed"/"failed" or its own "in-flight", leave that newer state alone.
-    try {
-      const current = safeJson(KB_SWEEP_STATE_PATH);
-      if (current && current.status === 'starting' && current.sweepToken === sweepToken) {
-        safeWrite(KB_SWEEP_STATE_PATH, JSON.stringify({
-          status: 'in-flight', startedAt, startedAtIso: new Date().toISOString(),
-          sweepToken, pid: proc.pid,
-        }));
-      }
-    } catch { /* best-effort */ }
-
-    proc.unref();
-    return jsonReply(res, 202, { ok: true, started: true, sweepToken });
+    return jsonReply(res, 202, { ok: true, started: true, sweepToken: result.sweepToken });
   }
 
 
@@ -6788,7 +6810,14 @@ What would you like to discuss or change? When you're happy, say "approve" and I
       const ccTurnId = 'cct-' + shared.uid();
       const turnSystemPrompt = renderDocChatSystemPromptForTurn(ccTurnId);
 
-      let { answer, partial, warning, toolUses, error: ccError } = await ccDocCall({
+      // W-mpmwxni2000c25c7-b — wall-clock turn watchdog. The doc-chat call
+      // can internally spawn resume + fresh + final-retry LLM calls; we want
+      // ONE wall-clock cap that covers the whole turn so a runtime stuck
+      // mid-stream can't outlive ccTurnTimeoutMs. On expiry the watchdog
+      // calls _docAbort (kills the in-flight CLI) and the synthesized payload
+      // below flows through the existing _docChatFailureResponse path.
+      const _docTurnTimeoutMs = _resolveCcTurnTimeoutMs();
+      const _docCallPromise = ccDocCall({
         message: body.message, document: currentContent, title: body.title,
         filePath: body.filePath, selection: body.selection, canEdit, isJson,
         model: body.model || undefined,
@@ -6798,6 +6827,8 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         systemPrompt: turnSystemPrompt,
         turnId: ccTurnId,
       });
+      const _docCallResult = await _raceCcDocChatTimeout(_docCallPromise, _docTurnTimeoutMs, () => _docAbort && _docAbort(), 'doc-chat');
+      let { answer, partial, warning, toolUses, error: ccError } = _docCallResult;
       const finalize = _finalizeDocChatEdit({
         filePath: body.filePath, fullPath, isJson, canEdit,
         originalContent: currentContent, delimiterContent: null,
@@ -6811,6 +6842,25 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         ccError, partial, warning, toolUses, finalize,
       });
       _docDone = true;
+      // W-mpmwxni2000c25c7-b — track every surfaced doc-chat error code so
+      // /api/metrics reflects silent-error regressions. Hard failures (no
+      // partial recovery, no edited file) graduate to 5xx so the client can
+      // render a real error UI instead of treating the polite "Failed to
+      // process request" string as a successful turn.
+      if (ccError) {
+        const errCode = ccError.typedCode || ccError.errorClass || ccError.code || 'unknown';
+        llm.trackEngineError('doc-chat', errCode);
+        const isHardFailure = !partial && !(finalize && finalize.edited);
+        if (isHardFailure) {
+          const status = errCode === shared.FAILURE_CLASS.CONFIG_ERROR ? 503 : 502;
+          return jsonReply(res, status, {
+            ...payload,
+            error: ccError.typedMessage || ccError.errorMessage || 'Document chat failed',
+            code: errCode,
+            retriable: ccError.retriable !== false,
+          });
+        }
+      }
       return jsonReply(res, 200, payload);
       } finally { _docAbort = null; _docDone = true; docChatInFlight.delete(docKey); }
     } catch (e) { return jsonReply(res, e.statusCode || 500, { error: e.message }); }
@@ -6899,7 +6949,12 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         const ccTurnId = 'cct-' + shared.uid();
         const turnSystemPrompt = renderDocChatSystemPromptForTurn(ccTurnId);
 
-        let { answer, partial, warning, toolUses, error: ccError } = await ccDocCallStreaming({
+        // W-mpmwxni2000c25c7-b — wall-clock turn watchdog (mirrors the
+        // non-stream handleDocChat path). On expiry _docAbort kills the
+        // in-flight LLM and the synthesized payload below flows through the
+        // SSE done frame the client already expects with `error` set.
+        const _docTurnTimeoutMs = _resolveCcTurnTimeoutMs();
+        const _docStreamCallPromise = ccDocCallStreaming({
           message: body.message, document: currentContent, title: body.title,
           filePath: body.filePath, selection: body.selection, canEdit, isJson,
           model: body.model || undefined,
@@ -6912,6 +6967,8 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           systemPrompt: turnSystemPrompt,
           turnId: ccTurnId,
         });
+        const _docStreamResult = await _raceCcDocChatTimeout(_docStreamCallPromise, _docTurnTimeoutMs, () => _docAbort && _docAbort(), 'doc-chat-stream');
+        let { answer, partial, warning, toolUses, error: ccError } = _docStreamResult;
         const finalize = _finalizeDocChatEdit({
           filePath: body.filePath, fullPath, isJson, canEdit,
           originalContent: currentContent, delimiterContent: null,
@@ -6924,6 +6981,23 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           actionFeedback: null, actionParseError: null,
           ccError, partial, warning, toolUses, finalize,
         });
+        // W-mpmwxni2000c25c7-b — track surfaced doc-chat error codes for
+        // /api/metrics and emit a named SSE `event: error` frame so the
+        // client can render a typed error instead of treating the polite
+        // fallback string as a normal completion.
+        if (ccError) {
+          const errCode = ccError.typedCode || ccError.errorClass || ccError.code || 'unknown';
+          llm.trackEngineError('doc-chat', errCode);
+          const isHardFailure = !partial && !(finalize && finalize.edited);
+          if (isHardFailure) {
+            const errPayload = {
+              message: ccError.typedMessage || ccError.errorMessage || 'Document chat failed',
+              code: errCode,
+              retriable: ccError.retriable !== false,
+            };
+            try { res.write(`event: error\ndata: ${JSON.stringify(errPayload)}\n\n`); } catch {}
+          }
+        }
         const { answer: finalAnswer, ...donePayload } = payload;
         writeDocEvent({
           type: 'done',
@@ -7520,21 +7594,40 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         // confirmation chips in the assistant reply.
         const ccTurnId = 'cct-' + shared.uid();
         const turnSystemPrompt = renderCcSystemPromptForTurn(ccTurnId);
-        const result = await ccCall(body.message, { store: 'cc', transcript: body.transcript, systemPrompt: turnSystemPrompt, turnId: ccTurnId });
+        // W-mpmwxni2000c25c7-b — wall-clock turn watchdog. On expiry the
+        // in-flight LLM call is aborted and ccCall returns a synthetic
+        // envelope with error.code === 'cc-turn-timeout'.
+        const turnTimeoutMs = _resolveCcTurnTimeoutMs();
+        const result = await llm.withCcTurnTimeout({
+          timeoutMs: turnTimeoutMs, label: 'command-center',
+        }, (registerAbort) => ccCall(body.message, {
+          store: 'cc', transcript: body.transcript, systemPrompt: turnSystemPrompt, turnId: ccTurnId,
+          onAbortReady: registerAbort,
+        }));
 
-        // Non-zero exit with text = max_turns or partial success — still usable
-        if (!result.text) {
+        // W-mpmwxni2000c25c7-b — typed-error envelope path. Any failure that
+        // produced no usable text is surfaced to the client as 5xx JSON
+        // `{ error, code, retriable }` instead of a polite 200 "I had trouble
+        // processing that" string that silently halves CC retry signal.
+        if (!result.text || result.error) {
+          const errEnvelope = result.error || (result.errorMessage
+            ? { message: result.errorMessage, code: result.errorClass || 'unknown', retriable: true }
+            : { message: 'Command Center returned no output', code: 'empty-output', retriable: true });
+          llm.trackEngineError('command-center', errEnvelope.code);
           const debugInfo = result.code !== 0 ? `(exit code ${result.code})` : '(empty response)';
           const stderrTail = (result.stderr || '').trim().split('\n').filter(Boolean).slice(-5).join(' | ');
-          console.error(`[CC] LLM failed after retries ${debugInfo}: ${stderrTail}`);
-          try { shared.log('warn', `CC failed ${debugInfo}: ${stderrTail.slice(0, 300)}`); } catch {}
-          const hasSession = !!ccSession.sessionId;
-          const retryHint = hasSession
-            ? 'Your session is still active — just send your message again to retry.'
-            : 'Try clicking **New Session** and sending your message again.';
-          return jsonReply(res, 200, {
-            text: `I had trouble processing that ${debugInfo}. ${stderrTail ? 'Detail: ' + stderrTail : ''}\n\n${retryHint}`,
-            actions: [], sessionId: ccSession.sessionId
+          console.error(`[CC] LLM failed after retries ${debugInfo} code=${errEnvelope.code}: ${stderrTail}`);
+          try { shared.log('warn', `CC failed ${debugInfo} code=${errEnvelope.code}: ${stderrTail.slice(0, 300)}`); } catch {}
+          // Missing-runtime is a 503 (service config); auth-failure also 503; other classes 502.
+          const status = result.missingRuntime ? 503
+            : errEnvelope.code === 'auth-failure' ? 503
+              : 502;
+          return jsonReply(res, status, {
+            error: errEnvelope.message,
+            code: errEnvelope.code,
+            retriable: !!errEnvelope.retriable,
+            sessionId: ccSession.sessionId || null,
+            ...(stderrTail ? { stderr: stderrTail.slice(0, 500) } : {}),
           });
         }
 
@@ -7555,7 +7648,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
       } finally {
         _releaseCCTab(tabId);
       }
-    } catch (e) { _releaseCCTab(tabId); return jsonReply(res, e.statusCode || 500, { error: e.message }); }
+    } catch (e) { _releaseCCTab(tabId); return jsonReply(res, e.statusCode || 500, { error: e.message, code: 'handler-exception', retriable: false }); }
   }
 
   /** Build a lightweight input object for SSE tool events — keeps only the fields formatToolSummary needs, with truncated string values. */
@@ -7677,6 +7770,11 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         });
       } catch (err) {
         _emitTimingLog(null, null, Date.now(), 'spawn-failed');
+        // W-mpmwxni2000c25c7-c — pipe the pool's typed-error fields
+        // (`code`, `retriable`) onto the envelope so the SSE writer can
+        // render a structured error event instead of grepping the stderr
+        // string. Pool stamps `.code` (worker-spawn-failed or
+        // acp-handshake-failed) on every getSession rejection.
         return resolveResult({
           text: '',
           sessionId: null,
@@ -7684,6 +7782,8 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           usage: {},
           raw: '',
           stderr: String((err && err.message) || err || 'cc-worker-pool spawn failed'),
+          errorCode: (err && err.code) || ccWorkerPool.ERROR_CODES.WORKER_SPAWN_FAILED,
+          errorRetriable: (err && err.retriable !== undefined) ? err.retriable : true,
         });
       }
       const _tSessionReady = Date.now();
@@ -7730,13 +7830,29 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         },
         onError: (err) => {
           _emitTimingLog(_lifecycle, _tSessionReady, Date.now(), cancelled ? 'cancelled' : 'error');
+          if (cancelled) {
+            resolveResult({
+              text: accumulated,
+              sessionId: sessionHandle.sessionId,
+              code: 0,
+              usage: {},
+              raw: accumulated,
+              stderr: '',
+            });
+            return;
+          }
+          // W-mpmwxni2000c25c7-c — pipe the pool's typed-error fields
+          // through. mid-stream worker exits stamp `.code = 'worker-died'`
+          // on the Error before invoking onError.
           resolveResult({
             text: accumulated,
             sessionId: sessionHandle.sessionId,
-            code: cancelled ? 0 : 1,
+            code: 1,
             usage: {},
             raw: accumulated,
             stderr: String((err && err.message) || err || 'cc-worker-pool stream error'),
+            errorCode: (err && err.code) || ccWorkerPool.ERROR_CODES.WORKER_DIED,
+            errorRetriable: (err && err.retriable !== undefined) ? err.retriable : true,
           });
         },
       });
@@ -8091,73 +8207,100 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           : '';
         const prompt = _joinCcPromptParts(preamble, resumeGuard, carryover, turnHeader, projectContextPart, body.message);
 
-        const { trackEngineUsage: trackUsage } = require('./engine/llm');
+        const { trackEngineUsage: trackUsage, trackEngineError: trackErr, withCcTurnTimeout: withTimeout } = require('./engine/llm');
         const streamModel = CONFIG.engine?.ccModel || shared.ENGINE_DEFAULTS.ccModel;
         const streamEffort = CONFIG.engine?.ccEffort || shared.ENGINE_DEFAULTS.ccEffort;
         const ccMaxTurns = CONFIG.engine?.ccMaxTurns || shared.ENGINE_DEFAULTS.ccMaxTurns;
         let toolUses = [];
-        const llmPromise = _invokeCcStream({
-          prompt, sessionId, liveState, toolUses,
-          model: streamModel, effort: streamEffort, maxTurns: ccMaxTurns,
-          engineConfig: CONFIG.engine,
-          systemPrompt: turnSystemPrompt,
-          tabId,
-        });
-        _ccStreamAbort = llmPromise.abort;
-        liveState.abortFn = _ccStreamAbort;
-        ccInFlightAborts.set(tabId, _ccStreamAbort);
-        const result = await llmPromise;
-        trackUsage('command-center', result.usage);
-
-        if (result.missingRuntime) {
-          finishMissingRuntime(result, liveState);
-          return;
-        }
-
-        // Handle failure — non-zero exit with text = max_turns or partial success, still usable
-        if (!result.text && wasResume && result.code !== 0 && !req.destroyed) {
-          // Resume failed (stale/expired session) — auto-retry as fresh session (skip if client already disconnected)
-          console.log(`[CC-stream] Resume failed (code=${result.code}) — retrying fresh`);
-          const freshPreamble = buildCCStatePreamble();
-          const freshCarryover = _buildTranscriptCarryover(body.transcript, { currentMessage: body.message });
-          const freshPrompt = _joinCcPromptParts(freshPreamble, freshCarryover, turnHeader, projectContextPart, body.message);
-          toolUses = []; // discard stale metadata from the failed resume attempt
-          const retryPromise = _invokeCcStream({
-            prompt: freshPrompt, sessionId: undefined, liveState, toolUses,
+        // W-mpmwxni2000c25c7-b — turn-level watchdog. Wraps the initial
+        // _invokeCcStream PLUS the post-resume-fail retry so the wall clock
+        // covers the entire CC turn (not just one underlying LLM call). On
+        // expiry, whichever call is in flight is aborted; the watchdog
+        // resolves with a synthetic `{ error: { code: 'cc-turn-timeout' } }`
+        // envelope so the SSE error path below kicks in.
+        const turnTimeoutMs = _resolveCcTurnTimeoutMs();
+        const result = await withTimeout({
+          timeoutMs: turnTimeoutMs, label: 'command-center-stream',
+        }, async (registerAbort) => {
+          const llmPromise = _invokeCcStream({
+            prompt, sessionId, liveState, toolUses,
             model: streamModel, effort: streamEffort, maxTurns: ccMaxTurns,
             engineConfig: CONFIG.engine,
             systemPrompt: turnSystemPrompt,
             tabId,
           });
-          _ccStreamAbort = retryPromise.abort;
+          _ccStreamAbort = llmPromise.abort;
           liveState.abortFn = _ccStreamAbort;
           ccInFlightAborts.set(tabId, _ccStreamAbort);
-          const retryResult = await retryPromise;
-          trackUsage('command-center', retryResult.usage);
-          if (retryResult.text) {
-            // Fresh session succeeded — use retryResult from here
-            Object.assign(result, retryResult);
+          registerAbort(_ccStreamAbort);
+          const initial = await llmPromise;
+          trackUsage('command-center', initial.usage);
+
+          if (initial.missingRuntime) return initial;
+
+          // Handle failure — non-zero exit with text = max_turns or partial success, still usable
+          if (!initial.text && wasResume && initial.code !== 0 && !req.destroyed) {
+            // Resume failed (stale/expired session) — auto-retry as fresh session (skip if client already disconnected)
+            console.log(`[CC-stream] Resume failed (code=${initial.code}) — retrying fresh`);
+            const freshPreamble = buildCCStatePreamble();
+            const freshCarryover = _buildTranscriptCarryover(body.transcript, { currentMessage: body.message });
+            const freshPrompt = _joinCcPromptParts(freshPreamble, freshCarryover, turnHeader, projectContextPart, body.message);
+            toolUses = []; // discard stale metadata from the failed resume attempt
+            const retryPromise = _invokeCcStream({
+              prompt: freshPrompt, sessionId: undefined, liveState, toolUses,
+              model: streamModel, effort: streamEffort, maxTurns: ccMaxTurns,
+              engineConfig: CONFIG.engine,
+              systemPrompt: turnSystemPrompt,
+              tabId,
+            });
+            _ccStreamAbort = retryPromise.abort;
+            liveState.abortFn = _ccStreamAbort;
+            ccInFlightAborts.set(tabId, _ccStreamAbort);
+            registerAbort(_ccStreamAbort);
+            const retryResult = await retryPromise;
+            trackUsage('command-center', retryResult.usage);
+            if (retryResult.text) {
+              // Fresh session succeeded — use retryResult from here
+              Object.assign(initial, retryResult);
+              // Clear the error envelope inherited from the failed first attempt
+              // so the success path below doesn't misclassify a recovered turn.
+              if (retryResult.text) { initial.error = null; initial.ok = true; }
+            } else if (retryResult.error) {
+              initial.error = retryResult.error;
+            }
           }
-        }
+          return initial;
+        });
         if (result.missingRuntime) {
           finishMissingRuntime(result, liveState);
           return;
         }
-        if (!result.text) {
+        if (!result.text || result.error) {
           if (req.destroyed) {
             _ccStreamEnded = true;
             _logCcStreamEnd(_ccTelemetry, 'llm-empty-client-gone', { code: result.code });
             return;
           }
-          const debugInfo = result.code !== 0 ? `(exit code ${result.code})` : '(empty response)';
+          // W-mpmwxni2000c25c7-b — surface the typed error envelope as a
+          // distinct SSE `event: error` frame so the client renders a real
+          // error UI (with a retry hint derived from `retriable`) instead of
+          // swallowing a polite 200 "I had trouble processing that" string.
+          const envelope = result.error || (result.errorMessage
+            ? { message: result.errorMessage, code: result.errorClass || 'unknown', retriable: true }
+            : { message: 'Command Center returned no output', code: 'empty-output', retriable: true });
+          trackErr('command-center', envelope.code);
           const stderrTail = (result.stderr || '').trim().split('\n').filter(Boolean).slice(-3).join(' | ');
-          console.error(`[CC-stream] Failed: code=${result.code}, stderr=${(result.stderr || '').slice(0, 500)}, stdout_tail=${(result.raw || '').slice(-500)}`);
-          const retryHint = 'Send your message again to retry.';
-          liveState.donePayload = { type: 'done', text: `I had trouble processing that ${debugInfo}. ${stderrTail ? 'Detail: ' + stderrTail : ''}\n\n${retryHint}`, actions: [], sessionId: null };
+          console.error(`[CC-stream] Failed code=${envelope.code} retriable=${envelope.retriable}: ${(result.stderr || '').slice(0, 500)}; stdout_tail=${(result.raw || '').slice(-500)}`);
+          // Emit `event: error` (named SSE frame), then a `done`-style frame
+          // for clients that only handle the default message channel, then
+          // close cleanly so the EventSource exits its read loop without
+          // throwing a connection-reset.
+          try { res.write(`event: error\ndata: ${JSON.stringify({ message: envelope.message, code: envelope.code, retriable: !!envelope.retriable, ...(stderrTail ? { stderr: stderrTail.slice(0, 500) } : {}) })}\n\n`); } catch {}
+          liveState.donePayload = { type: 'error', error: envelope.message, code: envelope.code, retriable: !!envelope.retriable, sessionId: null };
           if (liveState.writer) liveState.writer(liveState.donePayload);
           if (liveState.endResponse) liveState.endResponse();
           _scheduleCcLiveCleanup(tabId);
-          _logCcStreamEnd(_ccTelemetry, 'llm-failed-fallback-sent', { code: result.code });
+          _logCcStreamEnd(_ccTelemetry, 'llm-failed-error-envelope-sent', { code: result.code, errorCode: envelope.code });
           return;
         }
 
@@ -8613,6 +8756,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           versionCheckInterval: [60000],
           prPollStatusEvery: [1], prPollCommentsEvery: [1],
           agentBusyReassignMs: [0],
+          maxRetriesPerAgent: [1, 20],
         };
         for (const [key, [min, max]] of Object.entries(numericFields)) {
           if (e[key] !== undefined) {
@@ -8728,7 +8872,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           if (_isClear(e.defaultModel)) _deleteEngineConfig('defaultModel');
           else {
             const candidate = String(e.defaultModel);
-            const resolvedCli = config.engine.defaultCli || 'claude';
+            const resolvedCli = config.engine.defaultCli || 'copilot';
             const rejection = await _validateFleetModel(candidate, resolvedCli);
             if (rejection) _clamped.push(`engine.defaultModel: "${candidate}" ${rejection} — kept previous value`);
             else _setEngineConfig('defaultModel', candidate);
@@ -8738,7 +8882,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           if (_isClear(e.ccModel)) _deleteEngineConfig('ccModel');
           else {
             const candidate = String(e.ccModel);
-            const resolvedCli = config.engine.ccCli || config.engine.defaultCli || 'claude';
+            const resolvedCli = config.engine.ccCli || config.engine.defaultCli || 'copilot';
             const rejection = await _validateFleetModel(candidate, resolvedCli);
             if (rejection) _clamped.push(`engine.ccModel: "${candidate}" ${rejection} — kept previous value`);
             else _setEngineConfig('ccModel', candidate);
@@ -8856,7 +9000,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
             if (updates.model === '' || updates.model === null) delete config.agents[id].model;
             else {
               const candidate = String(updates.model);
-              const resolvedCli = config.agents[id].cli || config.engine.defaultCli || 'claude';
+              const resolvedCli = config.agents[id].cli || config.engine.defaultCli || 'copilot';
               const runtimeModelStr = _resolveModelForRuntime(candidate, resolvedCli);
               const knownModels = await _modelsFor(resolvedCli);
               // Two validation paths: