mixdog 0.9.53 → 0.9.55

package/src/tui/engine.mjs

@@ -11,6 +11,11 @@
  * This file keeps the stateful createEngineSession store + notification plan.
  */
 import { performance } from 'node:perf_hooks';
+import { mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { randomUUID } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Worker } from 'node:worker_threads';
 import {
   aggregateToolCategoryEntry,
   classifyToolCategory,
@@ -98,6 +103,7 @@ import {
 import { createSessionFlow } from './engine/session-flow.mjs';
 import { createRunTurn } from './engine/turn.mjs';
 import { createEngineApi } from './engine/session-api.mjs';
+import { createFrameBatchedStorePublisher } from './engine/frame-batched-store.mjs';
 
 // Source tests resolve from src/tui/engine.mjs; the built bundle resolves from
 // src/tui/dist/index.mjs.
@@ -134,6 +140,367 @@ const tuiDebug = (msg) => {
 let _idSeq = 0;
 const nextId = () => `it_${++_idSeq}`;
 
+export const TRANSCRIPT_LIVE_ITEM_CAP = 512;
+export const TRANSCRIPT_SPILL_CHUNK_ITEMS = 128;
+const TRANSCRIPT_RESTORE_OVERLAP_ITEMS = 64;
+const TRANSCRIPT_SPILL_STALE_MS = 24 * 60 * 60 * 1000;
+const TRANSCRIPT_SPILL_HEARTBEAT_MS = 10_000;
+const TRANSCRIPT_PROCESS_NONCE = randomUUID();
+
+export function cleanupStaleTranscriptSpillDirs({
+  root = tmpdir(),
+  now = Date.now(),
+  staleMs = TRANSCRIPT_SPILL_STALE_MS,
+} = {}) {
+  try {
+    for (const entry of readdirSync(root, { withFileTypes: true })) {
+      if (!entry.isDirectory() || !entry.name.startsWith('mixdog-transcript-')) continue;
+      const path = join(root, entry.name);
+      try {
+        const ownerPid = Number(/^mixdog-transcript-(\d+)-/.exec(entry.name)?.[1]);
+        let pidAlive = false;
+        if (ownerPid > 0) {
+          try { process.kill(ownerPid, 0); pidAlive = true; } catch {}
+        }
+        if (!pidAlive) {
+          rmSync(path, { recursive: true, force: true });
+          continue;
+        }
+        // A fresh heartbeat proves the owning process is running. A stale one
+        // is ambiguous (suspended owner vs PID reuse), so retain it for the
+        // generous staleMs grace period, then reclaim it even if that PID is
+        // currently alive. This avoids both short suspension data loss and
+        // immortal crash leftovers after PID reuse.
+        let heartbeatAge;
+        try {
+          heartbeatAge = now - statSync(join(path, 'heartbeat')).mtimeMs;
+        } catch {
+          heartbeatAge = now - statSync(path).mtimeMs;
+        }
+        if (heartbeatAge <= staleMs) continue;
+        rmSync(path, { recursive: true, force: true });
+      } catch {}
+    }
+  } catch {}
+}
+
+// Serialized pages deliberately release the old item object graph while
+// keeping every byte restorable. Only `items` is render-live and walkable.
+export function createTranscriptSpillBuffer({
+  cap = TRANSCRIPT_LIVE_ITEM_CAP,
+  chunkSize = TRANSCRIPT_SPILL_CHUNK_ITEMS,
+  workerFactory = (source) => new Worker(source, { eval: true }),
+  onWarning = (message) => tuiDebug(message),
+  writeTimeoutMs = 5000,
+} = {}) {
+  // Publish this process instance's nonce BEFORE cleanup. If the OS reused our
+  // PID after a crash, the old directory's owner nonce now differs from the
+  // live registry and cannot be mistaken for this process.
+  try {
+    writeFileSync(
+      join(tmpdir(), `mixdog-transcript-owner-${process.pid}.json`),
+      JSON.stringify({ pid: process.pid, nonce: TRANSCRIPT_PROCESS_NONCE }),
+      'utf8',
+    );
+  } catch {}
+  cleanupStaleTranscriptSpillDirs();
+  const pages = [];
+  let cursor = null;
+  let spillDir = null;
+  let pageSequence = 0;
+  let spillWorker = null;
+  let workerSpawnCount = 0;
+  let activeWrite = null;
+  let activeWriteTimer = null;
+  let warningEmitted = false;
+  let spillDisabled = false;
+  const writeQueue = [];
+  const heartbeatTimers = new Map();
+  const snapshots = new Set();
+  const cleanupRecords = (records, directory) => {
+    for (const record of records) {
+      record.cancelled = true;
+    }
+    if (directory) {
+      const timer = heartbeatTimers.get(directory);
+      if (timer) clearInterval(timer);
+      heartbeatTimers.delete(directory);
+      try { rmSync(directory, { recursive: true, force: true }); } catch {}
+    }
+  };
+  const ensureSpillDir = () => {
+    if (spillDir) return spillDir;
+    const root = tmpdir();
+    writeFileSync(
+      join(root, `mixdog-transcript-owner-${process.pid}.json`),
+      JSON.stringify({ pid: process.pid, nonce: TRANSCRIPT_PROCESS_NONCE }),
+      'utf8',
+    );
+    spillDir = mkdtempSync(join(root, `mixdog-transcript-${process.pid}-${TRANSCRIPT_PROCESS_NONCE}-`));
+    writeFileSync(
+      join(spillDir, 'owner.json'),
+      JSON.stringify({ pid: process.pid, nonce: TRANSCRIPT_PROCESS_NONCE }),
+      'utf8',
+    );
+    const heartbeat = join(spillDir, 'heartbeat');
+    writeFileSync(heartbeat, String(Date.now()), 'utf8');
+    const heartbeatTimer = setInterval(() => {
+      try { writeFileSync(heartbeat, String(Date.now()), 'utf8'); } catch {}
+    }, TRANSCRIPT_SPILL_HEARTBEAT_MS);
+    heartbeatTimer.unref?.();
+    heartbeatTimers.set(spillDir, heartbeatTimer);
+    return spillDir;
+  };
+  const workerSource = `
+    const { parentPort } = require('node:worker_threads');
+    const { renameSync, writeFileSync } = require('node:fs');
+    parentPort.on('message', ({ id, targetPath, tempPath, items }) => {
+      try {
+        writeFileSync(tempPath, JSON.stringify(items), 'utf8');
+        renameSync(tempPath, targetPath);
+        parentPort.postMessage({ id, ok: true });
+      } catch (error) {
+        parentPort.postMessage({ id, ok: false, error: String(error && error.message || error) });
+      }
+    });`;
+  const ensureWorker = () => {
+    if (spillWorker) return spillWorker;
+    try {
+      const worker = workerFactory(workerSource);
+      spillWorker = worker;
+      workerSpawnCount += 1;
+      worker.on('message', (result) => {
+        if (spillWorker !== worker || result?.id !== activeWrite?.id) return;
+        finishWrite(result?.ok === true, result?.error);
+      });
+      const failWorker = (error) => {
+        if (spillWorker !== worker) return;
+        if (activeWriteTimer) clearTimeout(activeWriteTimer);
+        activeWriteTimer = null;
+        const failed = activeWrite;
+        activeWrite = null;
+        spillWorker = null;
+        try { worker.terminate?.(); } catch {}
+        if (failed) retryOrPin(failed, error?.message);
+        pumpWrites();
+      };
+      worker.on('error', failWorker);
+      worker.on('exit', (code) => {
+        failWorker(new Error(`spill worker exited (${code})`));
+      });
+      worker.unref?.();
+    } catch (error) {
+      spillWorker = null;
+      if (activeWrite) {
+        const failed = activeWrite;
+        activeWrite = null;
+        retryOrPin(failed, error?.message);
+      }
+    }
+    return spillWorker;
+  };
+  const retryOrPin = (record, error) => {
+    if (record.cancelled) return;
+    record.attempts += 1;
+    if (record.attempts <= 2) {
+      writeQueue.unshift(record);
+      return;
+    }
+    record.pinned = true;
+    spillDisabled = true;
+    for (const queued of writeQueue.splice(0)) {
+      if (!queued.cancelled) queued.pinned = true;
+    }
+    if (!warningEmitted) {
+      warningEmitted = true;
+      try { onWarning(`transcript spill write failed; history pinned in memory (${error || 'unknown error'})`); } catch {}
+    }
+  };
+  const finishWrite = (ok, error) => {
+    if (activeWriteTimer) clearTimeout(activeWriteTimer);
+    activeWriteTimer = null;
+    const record = activeWrite;
+    activeWrite = null;
+    if (record && !record.cancelled) {
+      if (ok) record.pendingItems = null;
+      else retryOrPin(record, error);
+    }
+    pumpWrites();
+  };
+  const pumpWrites = () => {
+    if (activeWrite) return;
+    while (writeQueue.length && writeQueue[0].cancelled) writeQueue.shift();
+    if (!writeQueue.length) return;
+    activeWrite = writeQueue.shift();
+    const worker = ensureWorker();
+    if (!worker) {
+      if (activeWrite) {
+        const failed = activeWrite;
+        activeWrite = null;
+        retryOrPin(failed, 'worker unavailable');
+      }
+      queueMicrotask(pumpWrites);
+      return;
+    }
+    // Pages are capped at chunkSize (128 by default), so the structured-clone
+    // post cost is bounded. Serialization and filesystem I/O stay in the worker.
+    // Every attempt writes a distinct temporary file; atomic rename is the sole
+    // commit point. A timed-out old worker can therefore expose only a complete
+    // page (the retry payload is identical), never a partial target JSON file.
+    const tempPath = `${activeWrite.path}.attempt-${activeWrite.attempts}-${randomUUID()}.tmp`;
+    worker.postMessage({
+      id: activeWrite.id,
+      targetPath: activeWrite.path,
+      tempPath,
+      items: activeWrite.pendingItems,
+    });
+    activeWriteTimer = setTimeout(() => {
+      if (!activeWrite || spillWorker !== worker) return;
+      const failed = activeWrite;
+      activeWrite = null;
+      activeWriteTimer = null;
+      spillWorker = null;
+      try { worker.terminate?.(); } catch {}
+      retryOrPin(failed, `write timed out after ${writeTimeoutMs}ms`);
+      pumpWrites();
+    }, Math.max(1, Number(writeTimeoutMs) || 5000));
+    activeWriteTimer.unref?.();
+  };
+  const encode = (items) => {
+    const page = join(ensureSpillDir(), `${++pageSequence}.json`);
+    const record = {
+      id: pageSequence,
+      path: page,
+      pendingItems: items,
+      cancelled: false,
+      attempts: 0,
+      pinned: false,
+    };
+    writeQueue.push(record);
+    pumpWrites();
+    return record;
+  };
+  const decode = (record) => record.pendingItems || JSON.parse(readFileSync(record.path, 'utf8'));
+  return {
+    get hasOlder() { return cursor == null ? pages.length > 0 : cursor > 0; },
+    get hasNewer() { return cursor != null; },
+    reset() {
+      const retained = [...snapshots].some((snapshot) => snapshot.spillDir === spillDir);
+      const oldPages = pages.splice(0);
+      const oldDir = spillDir;
+      pages.length = 0;
+      cursor = null;
+      spillDir = null;
+      pageSequence = 0;
+      spillDisabled = false;
+      warningEmitted = false;
+      if (!retained) cleanupRecords(oldPages, oldDir);
+    },
+    snapshot() {
+      const snapshot = {
+        pages: pages.slice(),
+        cursor,
+        spillDir,
+        pageSequence,
+        spillDisabled,
+        warningEmitted,
+      };
+      snapshots.add(snapshot);
+      return snapshot;
+    },
+    restoreSnapshot(snapshot) {
+      if (!snapshot || !snapshots.has(snapshot)) return false;
+      if (snapshot.spillDir === spillDir) {
+        cursor = snapshot.cursor;
+        snapshots.delete(snapshot);
+        return true;
+      }
+      cleanupRecords(pages, spillDir);
+      pages.splice(0, pages.length, ...snapshot.pages);
+      cursor = snapshot.cursor;
+      spillDir = snapshot.spillDir;
+      pageSequence = snapshot.pageSequence;
+      spillDisabled = snapshot.spillDisabled === true;
+      warningEmitted = snapshot.warningEmitted === true;
+      snapshots.delete(snapshot);
+      return true;
+    },
+    releaseSnapshot(snapshot) {
+      if (!snapshot || !snapshots.delete(snapshot)) return false;
+      if (snapshot.spillDir !== spillDir) cleanupRecords(snapshot.pages, snapshot.spillDir);
+      return true;
+    },
+    dispose() {
+      cleanupRecords(pages, spillDir);
+      for (const snapshot of snapshots) {
+        if (snapshot.spillDir !== spillDir) cleanupRecords(snapshot.pages, snapshot.spillDir);
+      }
+      pages.length = 0;
+      snapshots.clear();
+      cursor = null;
+      spillDir = null;
+      for (const timer of heartbeatTimers.values()) clearInterval(timer);
+      heartbeatTimers.clear();
+      writeQueue.length = 0;
+      activeWrite = null;
+      if (activeWriteTimer) clearTimeout(activeWriteTimer);
+      activeWriteTimer = null;
+      try { spillWorker?.terminate(); } catch {}
+      spillWorker = null;
+    },
+    get workerCount() { return workerSpawnCount; },
+    get pendingWriteCount() {
+      return writeQueue.length + (activeWrite ? 1 : 0);
+    },
+    get pinnedPageCount() {
+      return pages.filter((page) => page.pinned).length;
+    },
+    get disabled() { return spillDisabled; },
+    capLive(items) {
+      let live = Array.isArray(items) ? items : [];
+      if (spillDisabled) return live;
+      while (live.length > cap) {
+        pages.push(encode(live.slice(0, chunkSize)));
+        live = live.slice(chunkSize);
+      }
+      return live;
+    },
+    restoreOlder(liveItems) {
+      const nextCursor = cursor == null ? pages.length - 1 : cursor - 1;
+      if (nextCursor < 0) return null;
+      cursor = nextCursor;
+      const restored = decode(pages[cursor]);
+      const following = cursor + 1 < pages.length
+        ? decode(pages[cursor + 1])
+        : (Array.isArray(liveItems) ? liveItems : []);
+      return [...restored, ...following.slice(0, TRANSCRIPT_RESTORE_OVERLAP_ITEMS)];
+    },
+    restoreNewer(liveItems) {
+      if (cursor == null) return null;
+      const nextCursor = cursor + 1;
+      if (nextCursor >= pages.length) {
+        cursor = null;
+        return { items: null, atLive: true };
+      }
+      cursor = nextCursor;
+      const restored = decode(pages[cursor]);
+      const following = cursor + 1 < pages.length
+        ? decode(pages[cursor + 1])
+        : (Array.isArray(liveItems) ? liveItems : []);
+      return [...restored, ...following.slice(0, TRANSCRIPT_RESTORE_OVERLAP_ITEMS)];
+    },
+  };
+}
+
+export function refillTranscriptViewOverlap(viewItems, previousLiveItems, nextLiveItems) {
+  const view = Array.isArray(viewItems) ? viewItems : null;
+  if (!view) return null;
+  const previousIds = new Set((previousLiveItems || []).map((item) => item?.id).filter((id) => id != null));
+  if (!view.some((item) => previousIds.has(item?.id))) return view;
+  const historical = view.filter((item) => !previousIds.has(item?.id));
+  return [...historical, ...(nextLiveItems || []).slice(0, TRANSCRIPT_RESTORE_OVERLAP_ITEMS)];
+}
+
 // Re-export the shared tool-result/notification helpers so importers (and tests)
 // keep resolving them from engine.mjs unchanged.
 export { toolResultText, toolAggregateDetailFallback, toolGroupedDisplayFallback };
@@ -166,7 +533,13 @@ export function replaceEngineItemsState({
 
 // Shared by the live engine and focused transcript tests so revision/tail
 // regressions exercise the exact mutation implementation used in production.
-export function createEngineItemMutators({ getState, set, itemIndexById }) {
+export function createEngineItemMutators({
+  getState,
+  set,
+  itemIndexById,
+  normalizeItems = (items) => items,
+  itemStateExtra = () => ({}),
+}) {
   const patchItem = (id, patch) => {
     const state = getState();
     let index = itemIndexById.get(id);
@@ -209,15 +582,21 @@ export function createEngineItemMutators({ getState, set, itemIndexById }) {
       streaming: false,
     };
     const index = state.items.length;
-    const items = [...state.items, item];
-    itemIndexById.set(id, index);
+    const items = normalizeItems([...state.items, item]);
+    itemIndexById.clear();
+    for (let i = 0; i < items.length; i++) {
+      const itemId = items[i]?.id;
+      if (itemId != null) itemIndexById.set(itemId, i);
+    }
+    const settledIndex = items.findIndex((entry) => entry?.id === id);
     set({
       items,
       structureRevision: (Number(state.structureRevision) || 0) + 1,
       streamingTail: null,
+      ...itemStateExtra(),
       ...extra,
     });
-    return true;
+    return settledIndex >= 0;
   };
 
   return { patchItem, settleStreamingTail };
@@ -271,6 +650,7 @@ export async function createEngineSession({
   const pendingNotificationKeys = new Set();
   const displayedExecutionNotificationKeys = new Set();
   const bag = {};
+  let state;
   // Route/context/agent-status derivations live in ./engine/context-state.mjs.
   // getState()/getPendingSessionReset() are late-bound to the `state` and
   // `pendingSessionReset` closures declared below; the sync helpers mutate
@@ -285,6 +665,7 @@ export async function createEngineSession({
   } = createContextState({
     runtime,
     getState: () => state,
+    updateState: (patch) => { state = { ...state, ...patch }; },
     getPendingSessionReset: () => flags.pendingSessionReset,
   });
 
@@ -293,8 +674,12 @@ export async function createEngineSession({
     agentJobs: [],
     agentScope: null,
   };
-  let state = {
+  state = {
     items: [],
+    transcriptViewItems: null,
+    transcriptViewRevision: 0,
+    transcriptHistoryBefore: false,
+    transcriptHistoryAfter: false,
     structureRevision: 0,
     streamingTail: null,
     toasts: [],
@@ -333,34 +718,44 @@ export async function createEngineSession({
   syncContextStats({ allowEstimated: true });
   bootProfile('engine:context-ready', { ms: (performance.now() - contextStartedAt).toFixed(1) });
   const listeners = new Set();
-  // Coalesce store notifications: a single onToolCall batch / finalize path
-  // fires many set() calls in one synchronous block (aggregate header sync,
-  // spinner, item pushes). Notifying React on every set() painted the
-  // intermediate layouts (jerky, stuttering header/count jitter). Collapsing the
-  // notifications into one microtask means listeners see only the final state
-  // of the current tick — "draw twice, keep the last one". getState() stays
-  // synchronous and always returns the latest snapshot, so useSyncExternalStore
-  // never tears.
-  let emitScheduled = false;
-  const flushEmit = () => {
-    emitScheduled = false;
-    for (const l of listeners) l();
-  };
-  const emit = () => {
-    if (emitScheduled) return;
-    emitScheduled = true;
-    queueMicrotask(flushEmit);
-  };
+  // React/useSyncExternalStore reads only this immutable published snapshot.
+  // `state` remains the engine's synchronous draft until a frame flush swaps
+  // the complete draft (including its single revision bump) into this slot.
+  let publishedState = process.env.NODE_ENV === 'production' ? state : Object.freeze(state);
+  // The mutable engine draft must never be the object exposed to React.
+  state = { ...state, stats: { ...state.stats } };
+  // Mutations stay synchronous, but React publications are frame-coalesced.
+  // structureRevision is committed by the publisher exactly once immediately
+  // before listeners observe the terminal snapshot for that frame.
+  const publisher = createFrameBatchedStorePublisher({
+    getState: () => state,
+    publishState: (next) => {
+      publishedState = process.env.NODE_ENV === 'production' ? next : Object.freeze(next);
+      // Detach the next draft, including the only intentionally mutable nested
+      // record, so legacy/internal draft writes cannot mutate the publication.
+      state = { ...next, stats: { ...next.stats } };
+    },
+    listeners,
+    isDisposed: () => flags.disposed,
+  });
+  const emit = publisher.emit;
+  const flushEmit = publisher.flush;
+  const flushEmitImmediate = publisher.flushImmediate;
   const set = (patch) => {
     if (!patch || typeof patch !== 'object') return false;
+    const requestsStructureChange = Object.prototype.hasOwnProperty.call(patch, 'structureRevision')
+      && !Object.is(patch.structureRevision, state.structureRevision);
+    const effectivePatch = requestsStructureChange
+      ? Object.fromEntries(Object.entries(patch).filter(([key]) => key !== 'structureRevision'))
+      : patch;
     let changed = false;
-    for (const [key, value] of Object.entries(patch)) {
+    for (const [key, value] of Object.entries(effectivePatch)) {
       if (!Object.is(state[key], value)) {
         changed = true;
         break;
       }
     }
-    if (!changed) return false;
+    if (!changed && !requestsStructureChange) return false;
     // Detect commandBusy releasing (true -> false). Submits that arrived while a
     // session command was in flight were queued and drain bailed on commandBusy;
     // re-kick drain here — one central point covers every command releaser
@@ -368,15 +763,66 @@ export async function createEngineSession({
     const commandBusyReleased = state.commandBusy === true
       && Object.prototype.hasOwnProperty.call(patch, 'commandBusy')
       && patch.commandBusy === false;
-    state = { ...state, ...patch };
+    state = { ...state, ...effectivePatch };
+    if (requestsStructureChange) publisher.markStructureChange();
     emit();
+    // Preserve the old microtask-latency behavior for interaction gates and
+    // long command spinners that intentionally yield before doing heavy work.
+    if (effectivePatch.commandStatus || effectivePatch.toolApproval) {
+      flushEmitImmediate();
+    }
     if (commandBusyReleased) queueMicrotask(() => { void bag.drain?.(); });
     return true;
   };
 
   const itemIndexById = new Map();
-  const replaceItems = (items, { preserveStreamingTail = false } = {}) => {
+  const transcriptSpill = createTranscriptSpillBuffer();
+  const reindexLiveItems = (items) => {
+    itemIndexById.clear();
+    for (let i = 0; i < items.length; i++) {
+      const id = items[i]?.id;
+      if (id != null) itemIndexById.set(id, i);
+    }
+  };
+  const transcriptHistoryFlags = () => ({
+    transcriptHistoryBefore: transcriptSpill.hasOlder,
+    transcriptHistoryAfter: transcriptSpill.hasNewer,
+  });
+  const restoreOlderTranscript = () => {
+    const transcriptViewItems = transcriptSpill.restoreOlder(state.items);
+    if (!transcriptViewItems) return false;
+    set({
+      transcriptViewItems,
+      transcriptViewRevision: state.transcriptViewRevision + 1,
+      ...transcriptHistoryFlags(),
+    });
+    flushEmitImmediate();
+    return true;
+  };
+  const restoreNewerTranscript = () => {
+    const restored = transcriptSpill.restoreNewer(state.items);
+    if (!restored) return false;
+    set({
+      transcriptViewItems: restored.atLive ? null : restored,
+      transcriptViewRevision: state.transcriptViewRevision + 1,
+      ...transcriptHistoryFlags(),
+    });
+    flushEmitImmediate();
+    return true;
+  };
+  const replaceItems = (items, {
+    preserveStreamingTail = false,
+    preserveSpill = false,
+    preserveTranscriptView = false,
+  } = {}) => {
     const nextItems = Array.isArray(items) ? items : [];
+    if (!preserveSpill) transcriptSpill.reset();
+    const liveItems = transcriptSpill.capLive(nextItems);
+    const previousTranscriptView = state.transcriptViewItems;
+    const nextTranscriptView = preserveTranscriptView && previousTranscriptView
+      ? refillTranscriptViewOverlap(previousTranscriptView, state.items, liveItems)
+      : null;
+    const transcriptViewChanged = nextTranscriptView !== previousTranscriptView;
     // Bulk item swap (session load / clear / compact). Derive the prompt-history
     // list from the NEW items and stage it onto state here so App never rescans;
     // the callers that invoke replaceItems always follow with a set({items:...,
@@ -384,22 +830,34 @@ export async function createEngineSession({
     // emit (the accompanying set() diffs the full patch). A bulk swap also
     // discards the old transcript, so drop any tracked active tool calls.
     activeToolCalls.clear();
+    const structureRevision = state.structureRevision;
     state = replaceEngineItemsState({
       state,
-      items: nextItems,
+      items: liveItems,
       itemIndexById,
       preserveStreamingTail,
       extra: {
-        promptHistoryList: buildMergedPromptHistory(recomputePromptHistory(nextItems), loadPromptHistory(state.cwd)),
+        promptHistoryList: preserveSpill
+          ? state.promptHistoryList
+          : buildMergedPromptHistory(recomputePromptHistory(nextItems), loadPromptHistory(state.cwd)),
         activeToolSummary: null,
+        transcriptViewItems: nextTranscriptView,
+        transcriptViewRevision: preserveTranscriptView
+          ? state.transcriptViewRevision + (transcriptViewChanged ? 1 : 0)
+          : state.transcriptViewRevision + 1,
+        ...transcriptHistoryFlags(),
       },
     });
+    // replaceEngineItemsState retains its standalone/test contract. In the live
+    // store, defer its revision increment to the frame publication boundary.
+    state = { ...state, structureRevision };
+    publisher.markStructureChange();
     // replaceItems stages the bulk state before its callers compose their
     // accompanying patch. Emit here as well so an items-only replacement
     // (for example removeNotice) cannot be hidden by the outer set seeing the
     // already-installed array identity.
     emit();
-    return nextItems;
+    return liveItems;
   };
   // --- Prompt-history list (newest-first, deduped) maintained incrementally ---
   // App previously rebuilt this from state.items on EVERY transcript change
@@ -458,8 +916,10 @@ export async function createEngineSession({
     if (!flags.pushingFromDeferredEntry && flags.flushDeferredBeforeImmediatePush) {
       flags.flushDeferredBeforeImmediatePush();
     }
-    const index = state.items.length;
-    const items = [...state.items, item];
+    const uncappedItems = [...state.items, item];
+    const items = transcriptSpill.capLive(uncappedItems);
+    if (items !== uncappedItems) reindexLiveItems(items);
+    const index = items.length - 1;
     if (item?.id != null) itemIndexById.set(item.id, index);
     if (item?.kind === 'user') {
       // Rebuild the derived history against the NEW list (not yet in state) and
@@ -467,11 +927,23 @@ export async function createEngineSession({
       // first — set() diffs against the current state, so a pre-assign would make
       // the references identical and skip emit().
       const promptHistoryList = buildMergedPromptHistory(recomputePromptHistory(items), loadPromptHistory(state.cwd));
-      set({ items, structureRevision: state.structureRevision + 1, promptHistoryList });
+      set({ items, structureRevision: state.structureRevision + 1, promptHistoryList, ...transcriptHistoryFlags() });
+      flushEmitImmediate();
     } else {
-      set({ items, structureRevision: state.structureRevision + 1 });
+      set({ items, structureRevision: state.structureRevision + 1, ...transcriptHistoryFlags() });
     }
   };
+  const appendItems = (newItems, extra = {}) => {
+    if (!Array.isArray(newItems) || newItems.length === 0) return set(extra);
+    const items = transcriptSpill.capLive([...state.items, ...newItems]);
+    reindexLiveItems(items);
+    return set({
+      items,
+      structureRevision: state.structureRevision + 1,
+      ...transcriptHistoryFlags(),
+      ...extra,
+    });
+  };
   const updateStreamingTail = (id, patch = {}, extra = {}) => {
     if (id == null) return false;
     const current = state.streamingTail?.id === id
@@ -499,6 +971,8 @@ export async function createEngineSession({
     getState: () => state,
     set,
     itemIndexById,
+    normalizeItems: (items) => transcriptSpill.capLive(items),
+    itemStateExtra: transcriptHistoryFlags,
   });
   const upsertSyntheticToolItem = (text, id = nextId(), parsed = null) => {
     const synthetic = parseSyntheticAgentMessage(text);
@@ -739,12 +1213,17 @@ export async function createEngineSession({
   Object.assign(bag, {
     runtime, nextId, tuiDebug, LEAD_TURN_TIMEOUT_MS,
     flags, lifecycle, pending, pendingNotificationKeys, displayedExecutionNotificationKeys, clearExecutionDedupState, listeners, itemIndexById,
-    getState: () => state, set,
-    pushItem, patchItem, replaceItems, updateStreamingTail, settleStreamingTail, clearStreamingTail,
+    getState: () => state, getPublishedState: () => publishedState,
+    set, flushEmit, flushEmitImmediate, disposeEmit: publisher.dispose,
+    pushItem, appendItems, patchItem, replaceItems, restoreOlderTranscript, restoreNewerTranscript, updateStreamingTail, settleStreamingTail, clearStreamingTail,
     pushToast, pushNotice, removeNotice, setProgressHint,
     pushUserOrSyntheticItem, pushAsyncAgentResponse, upsertSyntheticToolItem,
     markToolCallActive, markToolCallDone, clearActiveToolSummary, clearToastTimers,
     autoClearState, agentStatusState, baseRouteState, routeState, syncContextStats,
+    disposeTranscriptSpill: () => transcriptSpill.dispose(),
+    snapshotTranscriptSpill: () => transcriptSpill.snapshot(),
+    restoreTranscriptSpill: (snapshot) => transcriptSpill.restoreSnapshot(snapshot),
+    releaseTranscriptSpill: (snapshot) => transcriptSpill.releaseSnapshot(snapshot),
     presentNextToolApproval, finishToolApproval, denyAllToolApprovals, requestToolApproval,
     patchToolCardResult, flushToolResults,
     kickExecutionPendingResume, flushDeferredExecutionPendingResumeKick, scheduleExecutionPendingResumeKick, discardExecutionPendingResume, updateAgentJobCard, subscribeRuntimeNotifications,