mixdog 0.9.13 → 0.9.15

package/src/tui/engine.mjs

@@ -79,6 +79,11 @@ import {
 import { createToolApproval } from './engine/tool-approval.mjs';
 import { createToolCardResults } from './engine/tool-card-results.mjs';
 import { createAgentJobFeed } from './engine/agent-job-feed.mjs';
+import {
+  appendTuiSteeringPersist,
+  dropTuiSteeringPersist,
+  drainTuiSteeringPersist,
+} from './engine/tui-steering-persist.mjs';
 
 // Source tests resolve from src/tui/engine.mjs; the built bundle resolves from
 // src/tui/dist/index.mjs.
@@ -91,6 +96,26 @@ const TOOL_APPROVAL_TIMEOUT_MS = (() => {
   return Number.isFinite(value) && value > 0 ? Math.max(1000, Math.round(value)) : 120_000;
 })();
 
+// Wall-clock cap for a single lead TUI turn. A provider call that never
+// resolves (e.g. a rate-limit retry loop that spins forever) otherwise leaves
+// runTurn holding busy=true with no unwind, so submit() only ever queues and
+// the UI is permanently input-dead. On trip we abort the in-flight run through
+// the SAME interrupt path Esc uses, force busy=false, and drain the queue.
+// Generous default (30min); env-overridable for tuning/tests.
+const LEAD_TURN_TIMEOUT_MS = (() => {
+  const value = Number(process.env.MIXDOG_LEAD_TURN_TIMEOUT_MS);
+  return Number.isFinite(value) && value > 0 ? Math.max(10_000, Math.round(value)) : 30 * 60_000;
+})();
+
+// Opt-in diagnostic trace for the hang chain (runTurn start/end, busy-queue
+// enqueue/drain, watchdog trip). Quiet by default so it can never tear through
+// the alternate-screen render; enable with MIXDOG_TUI_DEBUG=1.
+const TUI_DEBUG = /^(1|true|yes|on)$/i.test(String(process.env.MIXDOG_TUI_DEBUG || ''));
+const tuiDebug = (msg) => {
+  if (!TUI_DEBUG) return;
+  try { process.stderr.write(`[tui] ${msg}\n`); } catch {}
+};
+
 let _idSeq = 0;
 const nextId = () => `it_${++_idSeq}`;
 
@@ -605,6 +630,12 @@ export async function createEngineSession({
   async function runTurn(userText, options = {}) {
     const turnIndex = state.stats.turns || 0;
     const startedAt = Date.now();
+    // Per-turn epoch. Force-release (watchdog grace) bumps the shared counter so
+    // this turn's own eventual `finally` — which may run LONG after force-release
+    // already started a new turn that reuses the per-session mutex — can detect
+    // it is stale and skip all shared-state writes (busy, activePromptRestore,
+    // turndone, drain kick). Neutralizes the stale unwind without touching the mutex.
+    const turnEpoch = ++leadTurnEpoch;
     const inputBaseline = state.stats.inputTokens;
     const outputBaseline = state.stats.outputTokens;
     const submittedIds = Array.isArray(options.submittedIds) ? options.submittedIds : [];
@@ -625,6 +656,48 @@ export async function createEngineSession({
 
     let assistantText = '';
     let currentAssistantId = null;
+
+    tuiDebug(`runTurn start turn=${turnIndex} pending=${pending.length} timeoutMs=${LEAD_TURN_TIMEOUT_MS}`);
+    // ── Wall-clock watchdog ───────────────────────────────────────────────
+    // If this turn never unwinds (provider call stuck), trip after the cap:
+    // abort the in-flight run via the existing interrupt path (runtime.abort,
+    // the same one Esc uses), which rejects the pending runtime.ask() with a
+    // SessionClosedError so the normal cancelled-turn teardown runs. If even
+    // that unwind is starved, forceReleaseTurn() in the timer hard-clears busy
+    // and kicks the drain so input is never permanently dead.
+    let watchdogTripped = false;
+    let watchdogTimer = null;
+    let watchdogGraceTimer = null;
+    const clearWatchdog = () => {
+      if (watchdogTimer) { clearTimeout(watchdogTimer); watchdogTimer = null; }
+      if (watchdogGraceTimer) { clearTimeout(watchdogGraceTimer); watchdogGraceTimer = null; }
+    };
+    watchdogTimer = setTimeout(() => {
+      if (disposed) return;
+      watchdogTripped = true;
+      const elapsed = Date.now() - startedAt;
+      tuiDebug(`runTurn WATCHDOG TRIP turn=${turnIndex} elapsedMs=${elapsed} — aborting stuck turn`);
+      pushNotice(`Turn timed out after ${Math.round(elapsed / 1000)}s — aborting stuck request. Input is available again.`, 'warn', { transcript: true });
+      try { runtime.abort('cli-react-abort-watchdog'); } catch {}
+      // Belt-and-suspenders: if runtime.abort() did not reject runtime.ask()
+      // (unwind starved), hard-release the turn after a short grace so the
+      // React store is never left with busy=true and no drain in flight.
+      watchdogGraceTimer = setTimeout(() => {
+        if (disposed) return;
+        if (!state.busy) return;
+        if (leadTurnEpoch !== turnEpoch) return; // a newer turn already owns the store
+        tuiDebug(`runTurn WATCHDOG FORCE-RELEASE turn=${turnIndex} — abort unwind starved`);
+        // Bump the epoch FIRST so this (still-stuck) turn's later finally becomes
+        // a no-op for shared state and cannot corrupt the turn we hand off to.
+        leadTurnEpoch++;
+        set({ busy: false, spinner: null, thinking: null, lastTurn: null });
+        activePromptRestore = null;
+        if (draining) draining = false;
+        if (pending.length > 0) void drain();
+      }, 5_000);
+      watchdogGraceTimer.unref?.();
+    }, LEAD_TURN_TIMEOUT_MS);
+    watchdogTimer.unref?.();
     let currentAssistantText = '';
     let thinkingText = '';
     let thinkingStartedAt = 0;
@@ -1446,6 +1519,14 @@ export async function createEngineSession({
       }
     } finally {
       denyAllToolApprovals(cancelled ? 'turn cancelled' : 'turn finished');
+      // Turn is unwinding normally (or via abort) — cancel the wall-clock
+      // watchdog + its force-release grace so they never fire on a live turn.
+      clearWatchdog();
+      // If the watchdog force-release already fired, a NEWER turn now owns the
+      // shared store (busy, activePromptRestore, turndone, drain). This stale
+      // unwind must NOT write shared state or it corrupts that turn. It still
+      // runs its own turn-local teardown (approvals, deferred cards) below.
+      const isStaleUnwind = leadTurnEpoch !== turnEpoch;
       // Flush any still-deferred tool cards into the transcript and cancel their
       // pending push timers so nothing fires (or leaks) after the turn ends. The
       // finalize path above already patches results onto visible cards; this just
@@ -1458,7 +1539,7 @@ export async function createEngineSession({
       flushDeferredBeforeImmediatePush = null;
       const producedTranscriptItem = state.items.length > itemsAtTurnStart;
       const reclaimed = cancelled && activePromptRestore?.reclaimed === true;
-      activePromptRestore = null;
+      if (!isStaleUnwind) activePromptRestore = null;
       closeThinkingSegment();
       const elapsedMs = Date.now() - startedAt;
       const thinkingElapsedMs = thinkingStartedAt ? accumulatedThinkingMs : 0;
@@ -1481,6 +1562,9 @@ export async function createEngineSession({
         && !resultContent
         && !assistantOutput
         && !producedTranscriptItem;
+      if (isStaleUnwind) {
+        tuiDebug(`runTurn STALE UNWIND turn=${turnIndex} — force-released; skipping shared-state writes`);
+      } else {
       if (!isNoOpTurn) {
         state.stats.turns = (state.stats.turns || 0) + 1;
       }
@@ -1502,15 +1586,37 @@ export async function createEngineSession({
         ...agentStatusState({ force: true }),
       });
       flushDeferredExecutionPendingResumeKick();
+      }
     }
-    clearActiveToolSummary();
+    // Shared UI state: a stale unwind must not wipe a newer turn's live
+    // tool-summary line (same epoch rule as the shared-state block above).
+    if (leadTurnEpoch === turnEpoch) clearActiveToolSummary();
     _publishedThinkingActive = false; // turn teardown cleared state.thinking
+    tuiDebug(`runTurn end turn=${turnIndex} status=${cancelled ? 'cancelled' : 'done'} elapsedMs=${Date.now() - startedAt}${watchdogTripped ? ' watchdogTripped=1' : ''} pending=${pending.length}`);
     return cancelled ? 'cancelled' : 'done';
   }
 
   const pending = [];
   let draining = false;
   let activePromptRestore = null;
+  // Monotonic per-lead-turn epoch (see runTurn). Bumped on watchdog
+  // force-release so a stale turn's late `finally` can detect it no longer owns
+  // the shared store and skip all shared-state writes.
+  let leadTurnEpoch = 0;
+
+  const leadSessionId = () => runtime.id;
+
+  function shouldMirrorSteeringEntry(entry) {
+    return isQueuedEntryEditable(entry) && !isSlashQueuedEntry(entry);
+  }
+
+  function commitSteeringQueueEntries(entries) {
+    callCommitCallbacks(entries);
+    const mirrored = (Array.isArray(entries) ? entries : []).filter(
+      (entry) => shouldMirrorSteeringEntry(entry) && !entry.steeringPersistRestored,
+    );
+    if (mirrored.length > 0) dropTuiSteeringPersist(leadSessionId(), mirrored);
+  }
 
   function makeQueueEntry(text, options = {}) {
     const mode = options.mode || 'prompt';
@@ -1528,6 +1634,8 @@ export async function createEngineSession({
       key: options.key || null,
       skipSlashCommands: options.skipSlashCommands === true,
       displayText: mode === 'task-notification' ? notificationDisplayText(displayText) : String(displayText || ''),
+      steeringPersistId: options.steeringPersistId || null,
+      steeringPersistRestored: options.steeringPersistRestored === true,
     };
   }
 
@@ -1606,6 +1714,7 @@ export async function createEngineSession({
         const batch = dequeueQueueBatch('later', { limit: firstBatch ? 1 : Infinity });
         firstBatch = false;
         if (batch.length === 0) break;
+        tuiDebug(`busy-queue drain batch=${batch.length} remaining=${pending.length}`);
         const ids = new Set(batch.map((e) => e.id));
         const merged = mergePromptContents(batch);
         for (const entry of batch) {
@@ -1619,7 +1728,7 @@ export async function createEngineSession({
           displayText: batch.map((entry) => entry.text).filter((text) => String(text || '').trim()).join('\n'),
           pastedImages: batchPastedImages,
           pastedTexts: batchPastedTexts,
-          onCommitted: () => callCommitCallbacks(batch),
+          onCommitted: () => commitSteeringQueueEntries(batch),
           submittedIds: [...ids],
           restorable: nonEditable.length === 0,
           requeueOnAbort: nonEditable,
@@ -1661,7 +1770,11 @@ export async function createEngineSession({
       pendingNotificationKeys.add(entry.key);
     }
     pending.push(entry);
+    if (state.busy && shouldMirrorSteeringEntry(entry)) {
+      appendTuiSteeringPersist(leadSessionId(), entry);
+    }
     if (isQueuedEntryVisible(entry)) set({ queued: [...state.queued, entry] });
+    if (state.busy) tuiDebug(`busy-queue enqueue mode=${entry.mode} pending=${pending.length}`);
     void drain();
     return true;
   }
@@ -1687,10 +1800,26 @@ export async function createEngineSession({
         if (Array.isArray(entry?.content)) return entry.content.length > 0;
         return String(entry?.content ?? '').trim().length > 0;
       });
-    callCommitCallbacks(batch);
+    commitSteeringQueueEntries(batch);
     return out;
   }
 
+  function restoreLeadSteeringFromDisk() {
+    const rows = drainTuiSteeringPersist(leadSessionId());
+    if (!rows.length) return;
+    const restored = [];
+    for (const row of rows) {
+      const entry = makeQueueEntry(row.text, {
+        steeringPersistRestored: true,
+        steeringPersistId: row.steeringPersistId || undefined,
+      });
+      pending.push(entry);
+      if (isQueuedEntryVisible(entry)) restored.push(entry);
+    }
+    if (restored.length > 0) set({ queued: [...state.queued, ...restored] });
+    void drain();
+  }
+
   async function autoClearBeforeSubmit() {
     const cfg = autoClearState();
     const now = Date.now();
@@ -1858,6 +1987,8 @@ export async function createEngineSession({
     return state.stats;
   };
 
+  restoreLeadSteeringFromDisk();
+
   return {
     getState: () => state,
     patchItem,
@@ -2826,11 +2957,13 @@ export async function createEngineSession({
           ...routeState(),
           stats: { ...state.stats },
         });
+        restoreLeadSteeringFromDisk();
         return true;
       } finally {
         set({ commandBusy: false, commandStatus: null });
       }
     },
+
     dispose: async (reason = 'cli-react-exit', options = {}) => {
       if (disposed) return;
       disposed = true;

package/src/ui/statusline.mjs

@@ -344,14 +344,12 @@ function renderNativeStatusline({
     addL2(`${spin} ${B}${label}${R}${elapsedSuffix(shellStatus.elapsedLabel)}`);
   }
   // Memory cycle segment — single unified "Memory" wording for all states:
-  // running -> "⠋ Memory · 12s"; backlog warning -> "⠋ Memory · backlog 512"
-  // (yellow). Nothing when idle (no L2 pollution).
+  // running -> "⠋ Memory · 12s". Backlog is intentionally NOT rendered
+  // (owner preference: cycle-health WARN logs cover it); nothing when idle.
   const memStatus = memoryCycleStatus();
   if (memStatus?.kind === 'running') {
     const elapsed = formatElapsed(Date.now() - memStatus.startedAt);
     addL2(`${spin} ${B}Memory${R}${elapsedSuffix(elapsed)}`);
-  } else if (memStatus?.kind === 'backlog') {
-    addL2(`${spin} ${B}Memory${R} ${D}·${R} ${YLW}backlog ${memStatus.pending}${R}`);
   }
   const l1 = l1Parts.join(sep) || 'mixdog';
   const l2 = l2Parts.join(sep);