mixdog 1.4.1 → 1.4.3

package/src/runtime/agent/orchestrator/session/store/transcript-durability.mjs

@@ -0,0 +1,88 @@
+// store/transcript-durability.mjs
+// What a session's canonical record durably holds, as evidence for the turn
+// checkpoints (manager/turn-checkpoint.mjs) — never inferred from in-memory
+// freshness.
+//
+//   transcriptEpoch  persisted. Names one append-only lineage of the
+//                    transcript: a record of epoch E with `length` messages
+//                    agrees with every other record of E on their common
+//                    prefix. A history rewrite (compaction, a prefix edit,
+//                    a recovery splice) mints a new epoch.
+//   turnCoverage     persisted. The token of the last turn whose finalized
+//                    transcript this record carries, so a checkpoint the
+//                    record already covers is deleted, never replayed.
+//
+// A landed save records, per session object, the exact epoch, coverage and
+// message references it serialized (captured when the bytes were built, so a
+// coalesced or later-built write reports what it actually wrote). Only that
+// record proves what is on disk; a settled promise or an idle queue does not.
+import { randomUUID } from 'node:crypto';
+import { basename, dirname } from 'node:path';
+import { sessionPath } from './session-paths.mjs';
+import { _storedSessionFromFile, STORED_SESSION_UNREADABLE } from './serialize.mjs';
+
+const _landed = new WeakMap(); // session → { epoch, coverage, refs, saveEpoch }
+const _snapshotHooks = new WeakMap(); // session → () => void
+
+export function mintTranscriptEpoch() {
+  return randomUUID();
+}
+
+/** Run `hook` before every serialization of `session` (the turn checkpoint
+ *  journals a history rewrite, under a new epoch, before any record carries
+ *  it). Returns the unregister function. */
+export function setTranscriptSnapshotHook(session, hook) {
+  _snapshotHooks.set(session, hook);
+  return () => {
+    if (_snapshotHooks.get(session) === hook) _snapshotHooks.delete(session);
+  };
+}
+
+/** The transcript identity a save of `session` is about to serialize. */
+export function captureTranscriptSnapshot(session) {
+  _snapshotHooks.get(session)?.();
+  return {
+    epoch: typeof session.transcriptEpoch === 'string' ? session.transcriptEpoch : null,
+    coverage: typeof session.turnCoverage === 'string' ? session.turnCoverage : null,
+    refs: Array.isArray(session.messages) ? session.messages.slice() : [],
+  };
+}
+
+/** A save of `snapshot` (issued under `saveEpoch`) landed on disk. */
+export function recordLandedTranscript(session, snapshot, saveEpoch) {
+  if (!session || !snapshot) return;
+  const current = _landed.get(session);
+  const issued = Number.isFinite(saveEpoch) ? saveEpoch : 0;
+  if (current && current.saveEpoch > issued) return;
+  _landed.set(session, { ...snapshot, saveEpoch: issued });
+}
+
+export function landedTranscript(session) {
+  return (session && _landed.get(session)) || null;
+}
+
+/** Whether the last landed save of `session` holds exactly its transcript. */
+export function landedTranscriptMatches(session) {
+  const landed = landedTranscript(session);
+  const messages = Array.isArray(session?.messages) ? session.messages : null;
+  if (!landed || !messages || landed.epoch !== session.transcriptEpoch) return false;
+  if (landed.refs.length !== messages.length) return false;
+  return landed.refs.every((message, index) => message === messages[index]);
+}
+
+/** The durable transcript a turn opening now would be taken against
+ *  (`{ epoch, refs }`), or null when no landed save holds `session`'s
+ *  current transcript exactly. */
+export function durableTranscriptBase(session) {
+  if (!landedTranscriptMatches(session)) return null;
+  const landed = landedTranscript(session);
+  return { epoch: landed.epoch, refs: landed.refs };
+}
+
+/** The canonical record of `id` as it is on disk (never a cached or live
+ *  copy), or null when it is absent, unreadable or not this session's. */
+export function readDurableSessionRecord(id) {
+  const path = sessionPath(id);
+  const record = _storedSessionFromFile(dirname(path), basename(path));
+  return record && record !== STORED_SESSION_UNREADABLE ? record : null;
+}

package/src/runtime/agent/orchestrator/session/store-agent-worker-rows.mjs

@@ -42,6 +42,7 @@ import {
 import { dataDir, storedAgentWorkerIndexPath, storedLeadWorkerIndexPath } from './store-summary-locations.mjs';
 import { cleanValue, isStoredSessionId, positiveNumber } from './store-summary-fields.mjs';
 import { sessionUsageTotals } from '../../../shared/session-usage-totals.mjs';
+import { todoProgress } from '../runtime-core/todo-list.mjs';
 
 const WORKING_AGENT_STATUS =
   /^(?:connecting|requesting|streaming|tool[-_\s]?running|running|queued|pending|starting|cancelling)$/i;
@@ -327,6 +328,7 @@ const WORKER_HEADER_FIELDS = [
   'totalInputTokens',
   'totalOutputTokens',
   'totalToolCalls',
+  'todo',
   ...SESSION_CANCEL_FIELDS,
 ];
 
@@ -584,6 +586,7 @@ function projectChildWorkerRow(row, context) {
   // beside a header that records a cancel-unconfirmed (possibly running)
   // process. A new row may precede its first session save (null header).
   const session = readWorkerSessionHeader(sessionId);
+  const todo = todoProgress(session);
   const declaredStatus = cleanValue(row.status) || 'running';
   const declaredStage = cleanValue(row.stage || row.status) || 'running';
   const declaredWorking = WORKING_AGENT_STATUS.test(workerLifecycle(row) || 'running');
@@ -649,6 +652,8 @@ function projectChildWorkerRow(row, context) {
     clientHostPid: positiveNumber(row.clientHostPid || session?.clientHostPid, 0) || null,
     taskId: cleanValue(row.task_id || row.taskId) || null,
     ...sessionUsageTotals(session),
+    // The agent's own step list as counts only; a closed agent keeps its last.
+    ...(todo ? { todo } : {}),
   };
 }
 

package/src/runtime/agent/orchestrator/session/store-summary-log.mjs

@@ -19,7 +19,7 @@
 import { readFileSync, statSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 
-export const SUMMARY_LOG_FILENAME = 'session-summaries.log';
+const SUMMARY_LOG_FILENAME = 'session-summaries.log';
 
 export function summaryLogPath(indexPath) {
   return join(dirname(indexPath), SUMMARY_LOG_FILENAME);

package/src/runtime/agent/orchestrator/session/store-summary-reader.mjs

@@ -24,21 +24,34 @@ import {
   stampedDerivation,
   storedSessionCatalogRow,
 } from './store-summary-catalog.mjs';
-import { createStoredTranscriptCache, nextProjectionStamp } from './store-transcript-cache.mjs';
+import {
+  createStoredTranscriptCache,
+  historyStampOf,
+  producedSlice,
+  projectionStampEpoch,
+  transcriptTextHash,
+} from './store-transcript-cache.mjs';
 import {
   forgetStoredTranscriptOffThread,
   OFFLOAD_MIN_CHARS,
   projectStoredTranscriptOffThread,
   storedTranscriptWorkerRunning,
 } from './store-transcript-worker.mjs';
-import { projectStoredTranscript } from './store-transcript-projection.mjs';
+import {
+  cursorPageRows,
+  heldRowsDigester,
+  prepareStoredTranscriptBase,
+  projectStoredTranscript,
+  storedHistorySlice,
+  windowStoredTranscript,
+} from './store-transcript-projection.mjs';
 import { dataDir } from './store-summary-locations.mjs';
 import { isStoredSessionId, workspaceSummary } from './store-summary-fields.mjs';
 import { readArchivedAgentResult } from './store-summary-archived-agent.mjs';
 import {
+  compactionArchivePageFromText,
   compactionArchivePath,
   compactionArchiveRange,
-  readCompactionArchivePage,
   visitCompactionArchive,
 } from './compaction-archive.mjs';
 import { attachmentHistoryRows } from './stored-attachment-history.mjs';
@@ -76,15 +89,55 @@ const storedTranscriptCache = createStoredTranscriptCache();
 // lookup can neither fill nor evict.
 const attachmentHistoryCache = createStoredTranscriptCache();
 
+// The display histories of SMALL sessions (record and archive under
+// OFFLOAD_MIN_CHARS, projected on this thread), kept for slicing within a
+// source-character budget; a larger one never stays on this thread (its
+// history lives in the projection worker, or is projected per read as a
+// bounded window).
+const SMALL_HISTORY_MAX_CHARS = 4 * 1024 * 1024;
+const smallHistories = (() => {
+  const entries = new Map();
+  let chars = 0;
+  const drop = (sessionId) => {
+    const entry = entries.get(sessionId);
+    if (!entry) return;
+    entries.delete(sessionId);
+    chars -= entry.chars;
+  };
+  return {
+    get(sessionId, identity) {
+      const entry = entries.get(sessionId);
+      if (!entry || entry.identity !== identity) return null;
+      entries.delete(sessionId);
+      entries.set(sessionId, entry);
+      return entry.history;
+    },
+    retain(sessionId, identity, sourceChars, history) {
+      drop(sessionId);
+      if (sourceChars > SMALL_HISTORY_MAX_CHARS) return;
+      entries.set(sessionId, { identity, chars: sourceChars, history });
+      chars += sourceChars;
+      while (chars > SMALL_HISTORY_MAX_CHARS && entries.size > 1) drop(entries.keys().next().value);
+    },
+    forget: drop,
+    clear() {
+      entries.clear();
+      chars = 0;
+    },
+  };
+})();
+
 /** Test seam: forget every cached projection. */
 export function clearStoredTranscriptCache() {
   storedTranscriptCache.clear();
+  smallHistories.clear();
 }
 
 /** Drop one session's cold projections: it went live, or its last view left. */
 export function forgetStoredSessionTranscript(sessionId) {
   const id = String(sessionId || '').trim();
   storedTranscriptCache.forget(`${id}|`);
+  smallHistories.forget(id);
   if (id) forgetStoredTranscriptOffThread(id);
 }
 
@@ -386,10 +439,16 @@ export function storedSessionTranscriptStamp(id) {
  *  itself). The answer is the cache's `project` contract; a worker failure
  *  rejects and the cache runs the in-process pipeline instead. */
 async function projectStoredTranscriptElsewhere(known, context) {
-  const { sessionId, recordPath, itemLimit, timing } = context;
+  const { sessionId, recordPath, timing, fingerprint, slice } = context;
   timing.worker = storedTranscriptWorkerRunning(sessionId) ? 'warm' : 'cold';
   const workerStartedAt = performance.now();
-  const answer = await projectStoredTranscriptOffThread({ sessionId, recordPath, itemLimit, known });
+  const answer = await projectStoredTranscriptOffThread({
+    sessionId,
+    recordPath,
+    known,
+    fingerprint,
+    slice: { ...slice, epoch: projectionStampEpoch() },
+  });
   timing.workerMs = performance.now() - workerStartedAt;
   Object.assign(timing, answer.timing);
   context.readState = answer.state;
@@ -397,15 +456,66 @@ async function projectStoredTranscriptElsewhere(known, context) {
   if (answer.same) return answer;
   if (answer.unreadable) return { ...answer, value: null };
   if (answer.stamp) readCanonicalLifecycle.rememberStrictVerdict(recordPath, answer.stamp, answer.lifecycle);
-  // Stamps are process-unique per module realm; re-issue in this one.
-  answer.value.projectionStamp = nextProjectionStamp();
+  if (answer.historyBuilt === false) timing.history = 'retained';
   return answer;
 }
 
+/** A projector (projectStoredTranscript's signature) for a historical-position
+ *  read on this thread (a checkpoint sidecar, or the worker unavailable): the
+ *  base once, then tail windows doubling from twice the page until one holds
+ *  the cursor's row with the page beside it, or history ends. Never the whole
+ *  display history: only as deep as the row itself. */
+const CURSOR_WINDOW_MAX_ITEMS = 8_192;
+
+function cursorWindowProjector(cursor, timing) {
+  return async (sessionId, doc, options) => {
+    const base = await prepareStoredTranscriptBase(sessionId, doc, options);
+    const id = cursor.before ?? cursor.after;
+    for (
+      let itemLimit = Math.max(64, cursor.maxRows * 2);
+      ;
+      itemLimit = Math.min(itemLimit * 2, CURSOR_WINDOW_MAX_ITEMS)
+    ) {
+      timing.cursorWindowItems = itemLimit;
+      const window = await windowStoredTranscript(sessionId, base, { itemLimit, includeMessages: false });
+      const at = window.items.findIndex((item) => item?.id === id);
+      if ((at >= 0 && (cursor.before == null || at >= cursor.maxRows)) || !window.transcriptHasOlder) return window;
+      // Never deeper on this thread: a row past the bound answers `reset`
+      // (the slice finds no such row), one at its edge a shorter page.
+      if (itemLimit >= CURSOR_WINDOW_MAX_ITEMS) return at >= 0 ? window : { ...window, items: [] };
+    }
+  };
+}
+
+/** The slice `request` names of `history` (the display history, or a window
+ *  of it), stamped `<history stamp>/<slice>`. */
+async function sliceOf(history, request, historyStamp) {
+  const proven = Boolean(request.held) && request.baseHistory === historyStamp;
+  const value = storedHistorySlice(
+    history,
+    { ...request, proven },
+    { digestOf: request.held && !proven ? await heldRowsDigester() : null }
+  );
+  const { windowStamp } = value;
+  delete value.windowStamp;
+  value.projectionStamp = `${historyStamp}/${windowStamp}`;
+  return value;
+}
+
 /** Projects the strictly parsed record `text` (read at full stamp `stamp`, or
  *  null when not provably stable) into the cached transcript value. */
 async function produceStoredTranscript(text, stamp, context) {
-  const { sessionId, recordPath, itemLimit, includeMessages, mode, checkpointAbsent, timing } = context;
+  const { sessionId, recordPath, itemLimit, slice, fingerprint, includeMessages, mode, checkpointAbsent, timing } =
+    context;
+  const historyStamp =
+    mode === 'items' ? historyStampOf(sessionId, transcriptTextHash(text), text.length, fingerprint) : null;
+  // A small session's retained history answers every other slice of it.
+  const keepHistory = mode === 'items' && checkpointAbsent && context.sourceChars < OFFLOAD_MIN_CHARS;
+  const retained = keepHistory ? smallHistories.get(sessionId, historyStamp) : null;
+  if (retained) {
+    timing.history = 'retained';
+    return producedSlice(await sliceOf(retained, slice, historyStamp), { historyBuilt: false });
+  }
   const parseStartedAt = performance.now();
   const record = readTopLevelLifecycleRecord(text);
   timing.parseMs = performance.now() - parseStartedAt;
@@ -431,22 +541,29 @@ async function produceStoredTranscript(text, stamp, context) {
   // load the hand-off serves, and projects the load cache's session
   // exactly as before; only that projection is detached.
   const projectStartedAt = performance.now();
-  const value = await projectStoredTranscript(
-    sessionId,
-    handedOver && checkpointAbsent ? JSON.parse(text) : record.doc,
-    {
-      itemLimit,
-      includeMessages,
-      checkpointAbsent,
-    }
-  );
+  const project = keepHistory || !slice.cursor ? projectStoredTranscript : cursorWindowProjector(slice.cursor, timing);
+  const projected = await project(sessionId, handedOver && checkpointAbsent ? JSON.parse(text) : record.doc, {
+    itemLimit,
+    // Only a small session's whole display history is built (and kept) on
+    // this thread; any other item read projects just its bounded window
+    // (a tail of that history) and slices it; a historical-position read
+    // projects the tail window reaching its row (cursorWindowProjector).
+    history: keepHistory,
+    includeMessages,
+    checkpointAbsent,
+  });
   timing.projectMs = performance.now() - projectStartedAt;
-  if (!handedOver || checkpointAbsent || !value || mode !== 'items') return value;
-  try {
-    return structuredClone(value);
-  } catch {
-    return value;
+  if (mode !== 'items' || !projected) return projected;
+  if (keepHistory) smallHistories.retain(sessionId, historyStamp, context.sourceChars, projected);
+  let value = await sliceOf(projected, slice, historyStamp);
+  if (handedOver && !checkpointAbsent) {
+    try {
+      value = structuredClone(value);
+    } catch {
+      // Served as projected.
+    }
   }
+  return producedSlice(value);
 }
 
 /** The rows of one session's whole stored conversation that name attachments
@@ -492,6 +609,37 @@ async function readStoredAttachmentHistory(sessionId, recordPath, archivePath) {
   return value ?? null;
 }
 
+/** One page of a session's compaction archive (`{ after, before, skip,
+ *  limit }`, as readCompactionArchivePage pages it), cut from its visible rows
+ *  built once per archive content in the transcript cache: paging a live
+ *  session through a large segment slices instead of re-parsing it; `paging`
+ *  keeps it cached under pressure (the cache's `pin`). */
+async function readCachedCompactionArchivePage(sessionId, archivePath, range, paging) {
+  const after = Math.max(0, Math.floor(Number(range.after) || 0));
+  const before = Number(range.before);
+  const visible = { after, before: Number.isFinite(before) ? before : Number.POSITIVE_INFINITY };
+  const archive = probePath(archivePath);
+  if (archive.state !== PROBE_PRESENT) return { items: [], hasOlder: false };
+  const { value } = await storedTranscriptCache.read({
+    key: `${sessionId}|archive|${visible.after}:${visible.before}`,
+    group: `${sessionId}|archive`,
+    fingerprint: '',
+    fileStat: archive,
+    pin: paging,
+    loadText: () => {
+      const body = readTextFile(archivePath);
+      return body.state === PROBE_PRESENT ? body.text : null;
+    },
+    produce: async (text) => ({
+      items: (await compactionArchivePageFromText(text, { ...visible, limit: Number.POSITIVE_INFINITY })).items,
+    }),
+  });
+  const items = Array.isArray(value?.items) ? value.items : [];
+  const end = Math.max(0, items.length - Math.max(0, Math.floor(Number(range.skip) || 0)));
+  const start = Math.max(0, end - Math.max(0, Math.floor(Number(range.limit) || 0)));
+  return { items: items.slice(start, end), hasOlder: start > 0 };
+}
+
 /** Read exactly one persisted session for a visible desktop pane. This never
  * enumerates siblings. Normal reads stay independent of runtime ownership;
  * interrupted turns conditionally use the same durable reconnect recovery as
@@ -508,7 +656,7 @@ export async function readStoredSessionTranscript(id, options = {}) {
   // One page of the display-only compaction archive (the live head loader
   // pages it once a runtime's in-memory rows are exhausted).
   if (options.compactionArchive && typeof options.compactionArchive === 'object') {
-    return readCompactionArchivePage(join(dataDir(), 'sessions'), sessionId, options.compactionArchive);
+    return readCachedCompactionArchivePage(sessionId, archivePath, options.compactionArchive, options.paging === true);
   }
   // The checkpoint sidecar shapes the projection as much as the record does,
   // so its identity joins the cache fingerprint. Only a PROVABLY absent
@@ -524,6 +672,51 @@ export async function readStoredSessionTranscript(id, options = {}) {
   const archive = probePath(archivePath);
   const requestedLimit = Number(options.transcriptItemLimit);
   const itemLimit = Number.isFinite(requestedLimit) && requestedLimit > 0 ? requestedLimit : Number.POSITIVE_INFINITY;
+  // `transcriptSkip`: the window's newest rows to leave out (a range read
+  // slices only the rows above them, with `skippedFirstId`).
+  const requestedSkip = Number(options.transcriptSkip);
+  const skip =
+    Number.isFinite(itemLimit) && Number.isSafeInteger(requestedSkip) && requestedSkip > 0
+      ? Math.min(requestedSkip, itemLimit)
+      : 0;
+  // `transcriptHeld` (`{ firstId, count, digest }`): the newest rows a paged
+  // caller holds, proven by `transcriptBaseHistory` (the history stamp they
+  // were served under) or their digest, where the history is: only the page
+  // above them (`transcriptPage`: `{ maxRows, byteBudget }`) is sliced,
+  // whatever their count. Unproven, a tail of at most `maxRows` rows.
+  const heldOption = options.transcriptHeld;
+  const held =
+    Number.isFinite(itemLimit) && heldOption && Number.isSafeInteger(heldOption.count) && heldOption.count > 0
+      ? { firstId: heldOption.firstId, count: heldOption.count, digest: String(heldOption.digest ?? '') }
+      : null;
+  const page = options.transcriptPage || {};
+  // `transcriptCursor` (`{ before | after, maxRows, byteBudget }`): a
+  // historical-position read — the rows next to one row, by its id.
+  const cursorOption = options.transcriptCursor;
+  const cursor =
+    cursorOption && (cursorOption.before != null || cursorOption.after != null)
+      ? {
+          ...(cursorOption.before != null ? { before: cursorOption.before } : { after: cursorOption.after }),
+          maxRows: cursorPageRows(cursorOption.maxRows),
+          byteBudget: Number(cursorOption.byteBudget) > 0 ? Number(cursorOption.byteBudget) : null,
+        }
+      : null;
+  const slice = cursor
+    ? { cursor }
+    : held
+      ? {
+          limit: itemLimit,
+          held,
+          baseHistory: typeof options.transcriptBaseHistory === 'string' ? options.transcriptBaseHistory : null,
+          maxRows: Math.max(1, Math.floor(Number(page.maxRows) || itemLimit)),
+          byteBudget: Number(page.byteBudget) > 0 ? Number(page.byteBudget) : null,
+        }
+      : { limit: itemLimit, skip };
+  const sliceKey = cursor
+    ? `cursor:${cursor.before ?? ''}:${cursor.after ?? ''}:${cursor.maxRows}:${cursor.byteBudget}`
+    : held
+      ? `held:${held.count}:${held.firstId}:${held.digest}:${slice.baseHistory}:${slice.maxRows}:${slice.byteBudget}`
+      : skip;
   // Full stamp the loaded text provably came from (unchanged around the
   // read), or null. Only such a parse may serve other readers.
   let readStamp = null;
@@ -533,11 +726,27 @@ export async function readStoredSessionTranscript(id, options = {}) {
   const mode = includeMessages ? 'messages' : 'items';
   const checkpointAbsent = checkpoint.state === PROBE_ABSENT;
   // Large, checkpoint-free item projections run off the daemon event loop.
-  const offloadable = !includeMessages && checkpointAbsent && recordStat.size >= OFFLOAD_MIN_CHARS;
+  // A large compaction archive counts: the display history holds its rows.
+  const sourceChars = recordStat.size + (archive.state === PROBE_PRESENT ? Number(archive.size) || 0 : 0);
+  const offloadable = !includeMessages && checkpointAbsent && sourceChars >= OFFLOAD_MIN_CHARS;
+  const fingerprint = [
+    checkpoint.state,
+    checkpoint.mtimeMs,
+    checkpoint.ctimeMs,
+    checkpoint.size,
+    checkpoint.ino,
+    checkpoint.dev,
+    archive.state,
+    archive.mtimeMs,
+    archive.size,
+  ].join(':');
   const produceContext = {
     sessionId,
     recordPath,
     itemLimit,
+    slice,
+    sourceChars,
+    fingerprint,
     includeMessages,
     mode,
     checkpointAbsent,
@@ -545,21 +754,16 @@ export async function readStoredSessionTranscript(id, options = {}) {
     // How the record read went (the worker's read or loadText below).
     readState: PROBE_PRESENT,
   };
-  const { value, hit, read } = await storedTranscriptCache.read({
-    key: `${sessionId}|${itemLimit}|${mode}`,
+  const { value, hit, read, built } = await storedTranscriptCache.read({
+    // One slice (window or page) per request; the group keeps one per
+    // session: the display history it is cut from stays with the thread
+    // that built it.
+    key: `${sessionId}|${itemLimit}|${sliceKey}|${mode}`,
     // Growing history windows of one session replace each other.
     group: `${sessionId}|${mode}`,
-    fingerprint: [
-      checkpoint.state,
-      checkpoint.mtimeMs,
-      checkpoint.ctimeMs,
-      checkpoint.size,
-      checkpoint.ino,
-      checkpoint.dev,
-      archive.state,
-      archive.mtimeMs,
-      archive.size,
-    ].join(':'),
+    // A page read keeps its slice cached under pressure.
+    pin: options.paging === true,
+    fingerprint,
     fileStat: recordStat,
     project: offloadable ? (known) => projectStoredTranscriptElsewhere(known, produceContext) : null,
     loadText: async () => {
@@ -589,7 +793,8 @@ export async function readStoredSessionTranscript(id, options = {}) {
   if (typeof options.trace === 'function') {
     options.trace({
       sessionId,
-      hit,
+      // A slice of a history already built counts as a hit: nothing was projected.
+      hit: hit || built === false,
       read,
       ms: performance.now() - startedAt,
       chars: recordStat.size,