mixdog 0.9.55 → 0.9.59

package/src/runtime/agent/orchestrator/session/store.mjs

@@ -15,7 +15,7 @@ import { sanitizeContentForStoredHistory } from '../providers/media-normalizatio
 import { scanTopLevelLifecycle } from './lifecycle-scan.mjs';
 import { rotateBoundedLog, PLUGIN_LOG_MAX_BYTES, PLUGIN_LOG_KEEP_BYTES } from '../../../../lib/mixdog-debug.cjs';
 import { resolveAgentTerminalReapMs } from '../../../../session-runtime/config-helpers.mjs';
-import { getStoreDir, sessionPath, publishHeartbeat, deleteHeartbeat } from './store/paths-heartbeat.mjs';
+import { getStoreDir, sessionPath, publishHeartbeat, deleteHeartbeat, deleteSessionPresence } from './store/paths-heartbeat.mjs';
 import {
     guardedSaveOptions as _guardedSaveOptions,
     cancelSessionWrites as _cancelSessionWrites,
@@ -25,6 +25,7 @@ import {
     waitForWriteCommit as _waitForWriteCommit,
 } from './store/write-guards.mjs';
 import {
+    SESSION_SUMMARY_INDEX_VERSION,
     summaryIndexPath,
     _sessionSummary,
     _normalizeSummaryIndex,
@@ -45,10 +46,64 @@ export {
     _upsertSessionSummary,
     _removeSessionSummary,
 } from './store-summary-index.mjs';
-export { publishHeartbeat, deleteHeartbeat } from './store/paths-heartbeat.mjs';
+export {
+    publishHeartbeat,
+    deleteHeartbeat,
+    publishSessionPresence,
+    deleteSessionPresence,
+    readSessionPresenceMtime,
+    isSessionPresenceOwnerDead,
+} from './store/paths-heartbeat.mjs';
 
 const _lastSaveError = new Map(); // id -> { message, at }
 
+// Recent full-session reads are much hotter than writes while a user hops
+// between conversations. Verify the file identity on every access, but reuse
+// the parsed object while the atomic file has not changed.
+const SESSION_LOAD_CACHE_LIMIT = 8;
+const _sessionLoadCache = new Map(); // path → { signature, session }
+let _sessionLoadCacheDataDir = null;
+
+function _readStoredSessionCached(id, path) {
+    const dataDir = getPluginData();
+    if (_sessionLoadCacheDataDir !== dataDir) {
+        _sessionLoadCacheDataDir = dataDir;
+        _sessionLoadCache.clear();
+    }
+    let signature;
+    try {
+        const info = statSync(path, { bigint: true });
+        signature = `${info.dev}:${info.ino}:${info.size}:${info.mtimeNs}`;
+    } catch {
+        _sessionLoadCache.delete(path);
+        return { exists: existsSync(path), session: null };
+    }
+    const cached = _sessionLoadCache.get(path);
+    if (cached?.signature === signature) {
+        _sessionLoadCache.delete(path);
+        _sessionLoadCache.set(path, cached);
+        return { exists: true, session: cached.session };
+    }
+    try {
+        const stored = JSON.parse(readFileSync(path, 'utf-8'));
+        if (stored?.id !== id) {
+            _sessionLoadCache.delete(path);
+            return { exists: true, session: null };
+        }
+        _sessionLoadCache.delete(path);
+        _sessionLoadCache.set(path, { signature, session: stored });
+        while (_sessionLoadCache.size > SESSION_LOAD_CACHE_LIMIT) {
+            const oldest = _sessionLoadCache.keys().next().value;
+            if (oldest === undefined) break;
+            _sessionLoadCache.delete(oldest);
+        }
+        return { exists: true, session: stored };
+    } catch {
+        _sessionLoadCache.delete(path);
+        return { exists: true, session: null };
+    }
+}
+
 // Listing is much hotter than writing, especially while the desktop session
 // browser is open. Keep the compact sidecar in memory after the first read;
 // local durability mutations update this cache synchronously, while an
@@ -65,6 +120,7 @@ function _ensureSummaryCacheDataDir() {
     if (_summaryCacheDataDir === dataDir) return;
     _summaryCacheDataDir = dataDir;
     _summaryRowsCache = null;
+    _summaryScanCache.clear();
     _summaryCacheUpserts.clear();
     _summaryCacheRemovals.clear();
     _summaryCacheVersions.clear();
@@ -212,15 +268,124 @@ const _savePending = new Map();
 /** Same-process authoritative session snapshots (createSession → loadSession / askSession). */
 const _liveSessions = new Map();
 
+// Session ids whose most recent save attempt was DROPPED by the ownership
+// guard (_shouldDrop: disk generation moved past the caller's expected
+// generation) or failed outright. For these ids the local live snapshot is
+// AHEAD of disk — the on-disk transcript froze at the last landed save — so
+// it must not be evicted (data loss) nor shadowed by the stale disk copy.
+const _droppedSaveIds = new Set();
+
 export function setLiveSession(session) {
     if (!session?.id) return;
     _liveSessions.set(session.id, session);
 }
 
+/**
+ * Cheap authoritative lifecycle read straight from disk (no live/pending
+ * cache). Used by askSession's split-brain re-adoption: a new ask on a
+ * non-closed session claims ownership by adopting the on-disk generation.
+ */
+export function readSessionLifecycleFromDisk(id) {
+    if (!id) return null;
+    const target = sessionPath(id);
+    if (!existsSync(target)) return null;
+    try {
+        const raw = readFileSync(target, 'utf-8');
+        let onDisk = scanTopLevelLifecycle(raw);
+        if (onDisk === null) onDisk = JSON.parse(raw);
+        return {
+            generation: typeof onDisk.generation === 'number' ? onDisk.generation : 0,
+            closed: onDisk.closed === true,
+        };
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Freshness of a session's `.hb` heartbeat sidecar (0 when absent). Used by
+ * the fork-on-resume guard: a fresh heartbeat published by another process
+ * means the session is actively being driven there RIGHT NOW.
+ */
+export function readSessionHeartbeatMtime(id) {
+    if (!id) return 0;
+    return _heartbeatMtime(id);
+}
+
 function _clearLiveSession(id) {
     if (id) _liveSessions.delete(id);
 }
 
+/** True while any pending/in-flight persistence still references this id. */
+function _hasPendingPersistence(id) {
+    if (_savePending.has(id) || _saveAsyncInflight.has(id) || _saveAsyncQueued.has(id)) return true;
+    for (const [, pending] of _deferredSessionSaves) {
+        if (pending?.session?.id === id) return true;
+    }
+    return false;
+}
+
+/**
+ * Drop one session's same-process snapshot once its state is durable on disk.
+ * No-op while any write for the id is still pending/in flight.
+ */
+export function evictLiveSession(id) {
+    if (!id || _hasPendingPersistence(id)) return false;
+    return _liveSessions.delete(id);
+}
+
+// Live snapshots that still carry raw media bytes (images are placeholder'd
+// in the persisted JSON) stay resident for this long after their last use so
+// multi-turn image recognition keeps working across an idle gap. Beyond the
+// TTL the memory cost wins and the snapshot is reclaimed like any other.
+const LIVE_MEDIA_RETENTION_MS = 60 * 60 * 1000; // 1h
+
+function _messagesCarryLiveMedia(messages) {
+    if (!Array.isArray(messages)) return false;
+    for (const m of messages) {
+        if (!m || typeof m !== 'object') continue;
+        if (sanitizeContentForStoredHistory(m.content) !== m.content) return true;
+    }
+    return false;
+}
+
+/**
+ * Idle sweep for the same-process snapshot cache. _liveSessions previously
+ * grew without bound — every clear-fork and every touched user session pinned
+ * its FULL message array (image bytes included) for process lifetime, the
+ * observed multi-GB RSS leak. Disk is the source of truth for anything not
+ * actively owned by this process, so an entry is dropped when it (a) has no
+ * live runtime owner, (b) has no pending persistence, and (c) already exists
+ * on disk — loadSession then falls back to the session file. Media-carrying
+ * snapshots get a grace TTL (see LIVE_MEDIA_RETENTION_MS) because eviction is
+ * lossy for them; text-only snapshots evict losslessly right away.
+ */
+export function evictIdleLiveSessions(options = {}) {
+    const isSessionLive = typeof options.isSessionLive === 'function' ? options.isSessionLive : null;
+    const now = Date.now();
+    let evicted = 0;
+    for (const [id, session] of [..._liveSessions.entries()]) {
+        if (isSessionLive && isSessionLive(id)) continue;
+        if (_hasPendingPersistence(id)) continue;
+        if (!existsSync(sessionPath(id))) continue;
+        // A dropped last save means the disk copy is BEHIND this snapshot
+        // (ownership split-brain). Evicting would lose the only complete
+        // transcript; keep it until a save lands again (re-adoption).
+        if (_droppedSaveIds.has(id)) continue;
+        if (_messagesCarryLiveMedia(session?.messages)) {
+            const lastActive = Math.max(session?.updatedAt || 0, session?.lastUsedAt || 0);
+            if (lastActive > 0 && now - lastActive <= LIVE_MEDIA_RETENTION_MS) continue;
+        }
+        _liveSessions.delete(id);
+        // With no pending persistence the rollback-race version counter for
+        // this id is dead weight — reclaim it too (it regrows from 1 on the
+        // next save, which is safe precisely because nothing is in flight).
+        _summaryCacheVersions.delete(id);
+        evicted++;
+    }
+    return evicted;
+}
+
 const _deleteHeartbeat = deleteHeartbeat;
 
 // ── 150 ms debounce window ────────────────────────────────────────────────────
@@ -355,7 +520,20 @@ function _getOrSpawnWorker() {
     _saveWorker = new Worker(new URL('./save-session-worker.mjs', import.meta.url), {
         execArgv: [],
     });
-    _saveWorker.on('message', ({ ok, saved, error, reqId }) => {
+    // Worker logs arrive as `{ __log }` messages, NOT via piped stdio
+    // (stdout:true/stderr:true): once the parent starts reading a worker's
+    // piped stdio, the underlying MessagePort stays ref'd for the worker's
+    // lifetime regardless of worker.unref(), so every process that ever
+    // saved a session could no longer exit (test runners hung after
+    // completion). The worker overrides its own process.stdout/stderr.write
+    // to forward through this channel, which keeps stray prints off the TUI
+    // frame (routed through the parent's guardable stderr) without holding
+    // the event loop.
+    _saveWorker.on('message', ({ __log, ok, saved, error, reqId }) => {
+        if (__log !== undefined) {
+            try { process.stderr.write(String(__log)); } catch { /* best-effort */ }
+            return;
+        }
         const p = _saveWorkerPending.get(reqId);
         if (!p) return;
         _saveWorkerPending.delete(reqId);
@@ -373,8 +551,16 @@ function _getOrSpawnWorker() {
             // A close/delete may have completed while the worker was writing.
             // Do not let this older completion put an open row back in the
             // process-local cache after its tombstone/removal.
-            if (saved) _queueSessionSummaryUpsert(session, summaryVersion);
-            else _rollbackCachedSessionSummary(id, summaryVersion);
+            if (saved) {
+                _queueSessionSummaryUpsert(session, summaryVersion);
+                _droppedSaveIds.delete(id);
+            } else {
+                // The worker's _shouldDrop declined the write: disk ownership
+                // moved past this snapshot — flag the split-brain so eviction
+                // and disk-over-live arbitration keep the richer local copy.
+                _rollbackCachedSessionSummary(id, summaryVersion);
+                _droppedSaveIds.add(id);
+            }
             clearSessionSaveError(id);
             for (const w of waiters) w.resolve();
         }
@@ -570,25 +756,28 @@ export function _saveSessionSync(session, opts) {
 function _doSaveSync(payload) {
     const { session, opts, summaryVersion = null } = payload;
     const id = session.id;
-    if (_shouldDrop(id, opts)) return false;
+    if (_shouldDrop(id, opts)) { _droppedSaveIds.add(id); return false; }
     const target = sessionPath(id);
     const tmp = target + '.' + randomBytes(6).toString('hex') + '.tmp';
     try {
         writeFileSync(tmp, JSON.stringify(_sessionForDisk(session)), 'utf-8');
         if (_shouldDrop(id, opts)) {
             try { unlinkSync(tmp); } catch { /* ignore cleanup failure */ }
+            _droppedSaveIds.add(id);
             return false;
         }
         const commitControl = _acquireWriteCommit(opts);
         if (commitControl === false || _shouldDrop(id, opts)) {
             try { unlinkSync(tmp); } catch { /* ignore cleanup failure */ }
             _releaseWriteCommit(commitControl);
+            _droppedSaveIds.add(id);
             return false;
         }
         try {
             _renameWithRetrySync(tmp, target);
             _queueSessionSummaryUpsert(session, summaryVersion);
             clearSessionSaveError(id);
+            _droppedSaveIds.delete(id);
         } finally {
             _releaseWriteCommit(commitControl);
         }
@@ -736,6 +925,7 @@ async function _doSave(payload) {
     // First check: upfront, before any disk I/O. Cheap short-circuit when a
     // tombstone is already on disk when the caller arrives.
     if (_shouldDrop(id, opts)) {
+        _droppedSaveIds.add(id);
         _drainQueue(id);
         return false;
     }
@@ -749,6 +939,7 @@ async function _doSave(payload) {
         if (_shouldDrop(id, opts)) {
             try { unlinkSync(tmp); } catch { /* ignore cleanup failure */ }
             process.stderr.write(`[session-store] ${id}: dropped stale save (tombstone planted during write)\n`);
+            _droppedSaveIds.add(id);
             _drainQueue(id);
             return false;
         }
@@ -756,6 +947,7 @@ async function _doSave(payload) {
         if (commitControl === false || _shouldDrop(id, opts)) {
             try { unlinkSync(tmp); } catch { /* ignore cleanup failure */ }
             _releaseWriteCommit(commitControl);
+            _droppedSaveIds.add(id);
             _drainQueue(id);
             return false;
         }
@@ -763,6 +955,7 @@ async function _doSave(payload) {
             _renameWithRetrySync(tmp, target);
             _queueSessionSummaryUpsert(session, summaryVersion);
             clearSessionSaveError(id);
+            _droppedSaveIds.delete(id);
         } finally {
             _releaseWriteCommit(commitControl);
         }
@@ -781,10 +974,51 @@ async function _doSave(payload) {
  * Used by closeSession() to plant a tombstone that races against in-flight
  * saveSession() calls.
  */
-export function markSessionClosed(id, reason = 'manual') {
-    // Cancel any pending debounced save so it cannot overwrite the tombstone
-    // that we are about to plant. The _shouldDrop() guard inside _doSave()
-    // provides a second line of defence, but cancelling here is cheaper.
+function _heartbeatMtime(id) {
+    try {
+        const path = join(getStoreDir(), `${id}.hb`);
+        return existsSync(path) ? (statSync(path).mtimeMs || 0) : 0;
+    } catch {
+        return 0;
+    }
+}
+
+function _runtimeLivenessVeto(id, options = {}) {
+    return typeof options.isSessionLive === 'function' && options.isSessionLive(id);
+}
+
+function _heartbeatLivenessVeto(id, options = {}) {
+    const heartbeatMtime = _heartbeatMtime(id);
+    if (!(heartbeatMtime > 0)) return false;
+    const hasHeartbeatSnapshot = Object.prototype.hasOwnProperty.call(options, 'heartbeatSnapshotMtime');
+    const snapshotMtime = Number(options.heartbeatSnapshotMtime) || 0;
+    if (hasHeartbeatSnapshot && heartbeatMtime > snapshotMtime) return true;
+    const freshMs = Number(options.heartbeatFreshMs);
+    return Number.isFinite(freshMs) && freshMs > 0 && Date.now() - heartbeatMtime <= freshMs;
+}
+
+function _deleteHeartbeatUnlessNewer(id, options = {}) {
+    const hasHeartbeatSnapshot = Object.prototype.hasOwnProperty.call(options, 'heartbeatSnapshotMtime');
+    const snapshotMtime = Number(options.heartbeatSnapshotMtime) || 0;
+    if (!hasHeartbeatSnapshot || _heartbeatMtime(id) <= snapshotMtime) {
+        _deleteHeartbeat(id);
+    }
+}
+
+export function markSessionClosed(id, reason = 'manual', options = {}) {
+    // Caller-provided probes may re-enter the store, so evaluate them before
+    // taking the non-reentrant Atomics commit lock. A veto must also precede
+    // pending-save cancellation so debounce durability remains intact.
+    if (_runtimeLivenessVeto(id, options) || _heartbeatLivenessVeto(id, options)) return null;
+    const closeGuard = _guardedSaveOptions(id);
+    const commitControl = _acquireWriteCommit(closeGuard);
+    if (commitControl === false) return null;
+    try {
+    // Cross-process heartbeat revival after full-TTL silence is accepted as a
+    // best-effort race: the tombstone resurrection guard is the authoritative
+    // post-race arbiter. Re-stat here, but never invoke caller code under lock.
+    if (_heartbeatLivenessVeto(id, options)) return null;
+    // Only a committed close may disrupt pending persistence.
     _clearDebounce(id);
     _cancelSessionWrites(id);
     _uncacheSessionSummary(id);
@@ -834,8 +1068,10 @@ export function markSessionClosed(id, reason = 'manual') {
     _savePending.delete(id);
     clearSessionSaveError(id);
     _clearLiveSession(id);
-    _deleteHeartbeat(id);
+    // Preserve a sidecar published strictly after the sweep's scan snapshot.
+    _deleteHeartbeatUnlessNewer(id, options);
     _queueSessionSummaryUpsert(tombstone);
+    _droppedSaveIds.delete(id);
     // Structured close metric. Single emission point because every close
     // path funnels through markSessionClosed. lifeMs = updatedAt-createdAt
     // straddles the tombstone (updatedAt was just set to Date.now()), so
@@ -861,6 +1097,9 @@ export function markSessionClosed(id, reason = 'manual') {
         }
     } catch { /* logger never breaks the close path */ }
     return newGen;
+    } finally {
+        _releaseWriteCommit(commitControl);
+    }
 }
 
 /**
@@ -898,22 +1137,18 @@ export function bumpSessionGeneration(id, reason = 'detach') {
     _clearLiveSession(id);
     _deleteHeartbeat(id);
     _queueSessionSummaryUpsert(detached);
+    _droppedSaveIds.delete(id);
     return newGen;
 }
 
 export function loadSession(id) {
     const path = sessionPath(id);
-    let stored = null;
-    if (existsSync(path)) {
-        try {
-            stored = JSON.parse(readFileSync(path, 'utf-8'));
-            // An existing file owns this identity. Its contents must validate
-            // before fresher in-memory state is allowed to shadow it.
-            if (stored?.id !== id) return null;
-        } catch {
-            return null;
-        }
-    }
+    // An existing file owns this identity. Its contents must validate before
+    // fresher in-memory state is allowed to shadow it. Unchanged atomic files
+    // reuse their recent parsed object instead of reparsing the full transcript.
+    const disk = _readStoredSessionCached(id, path);
+    if (disk.exists && !disk.session) return null;
+    const stored = disk.session;
     // Read-your-writes: if a save is pending (debouncing, scheduled, or queued
     // behind an in-flight write) return that payload instead of stale disk state.
     // The most-recently-queued slot is checked first (queued > payload).
@@ -923,7 +1158,24 @@ export function loadSession(id) {
         if (inMemory?.id === id) return _ensureLifecycleFields(inMemory);
     }
     const live = _liveSessions.get(id);
-    if (live?.id === id) return _ensureLifecycleFields(live);
+    if (live?.id === id) {
+        // Terminal ↔ desktop interop: `generation` only moves on close/detach
+        // (markSessionClosed / bumpSessionGeneration). A disk record with a
+        // HIGHER generation means another process took ownership of this
+        // session after our snapshot was cached — the local copy is stale and
+        // must not shadow the newer on-disk transcript (its late saves would
+        // be dropped by _shouldDrop's ownership rule anyway).
+        const liveGen = typeof live.generation === 'number' ? live.generation : 0;
+        const storedGen = stored && typeof stored.generation === 'number' ? stored.generation : 0;
+        // A flagged dropped save means the generation moved WITHOUT newer
+        // content landing (detach re-persists stale content) — the local
+        // snapshot is the only complete transcript, so it keeps winning.
+        if (stored && storedGen > liveGen && !_droppedSaveIds.has(id)) {
+            _liveSessions.delete(id);
+        } else {
+            return _ensureLifecycleFields(live);
+        }
+    }
     return stored ? _ensureLifecycleFields(stored) : null;
 }
 
@@ -940,10 +1192,19 @@ export function clearSessionSaveError(id) {
 }
 
 export function deleteSession(id, options = {}) {
+    // Keep caller probes and all vetoes ahead of the non-reentrant lock and
+    // ahead of pending-save disruption, matching markSessionClosed().
+    if (_runtimeLivenessVeto(id, options) || _heartbeatLivenessVeto(id, options)) return false;
+    const deleteGuard = _guardedSaveOptions(id);
+    const commitControl = _acquireWriteCommit(deleteGuard);
+    if (commitControl === false) return false;
+    try {
+    // Cross-process revival after full-TTL silence remains best-effort; the
+    // tombstone resurrection guard is authoritative. Only re-stat .hb here.
+    if (_heartbeatLivenessVeto(id, options)) return false;
     _cancelSessionWrites(id);
     _clearDebounce(id);
     _savePending.delete(id);
-    _waitForWriteCommit(id);
     const path = sessionPath(id);
     let removed = false;
     if (existsSync(path)) {
@@ -953,7 +1214,9 @@ export function deleteSession(id, options = {}) {
         }
         catch { /* fall through to .hb cleanup */ }
     }
-    _deleteHeartbeat(id);
+    // Preserve a sidecar published strictly after the sweep's scan snapshot.
+    _deleteHeartbeatUnlessNewer(id, options);
+    deleteSessionPresence(id);
     _clearLiveSession(id);
     if (removed || !existsSync(path)) clearSessionSaveError(id);
     // deferSummaryUpdate: bulk callers (tombstone sweep) remove thousands of
@@ -962,6 +1225,9 @@ export function deleteSession(id, options = {}) {
     if (options.deferSummaryUpdate === true) _uncacheSessionSummary(id);
     else _queueSessionSummaryRemoval(id);
     return removed;
+    } finally {
+        _releaseWriteCommit(commitControl);
+    }
 }
 const DEFAULT_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes idle — aligned with Anthropic 5m messages tier and OpenAI in-memory cache window
 const AGENT_TERMINAL_STATUSES = new Set(['idle', 'done', 'error']);
@@ -982,6 +1248,9 @@ const RUNNING_STALL_MS = 10 * 60 * 1000;
 // (options.isSessionLive) is additionally protected as defense-in-depth.
 const RESUMABLE_OPEN_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
 const RESUMABLE_OPEN_MAX_COUNT = 300;
+// Blank scratch sessions (zero user/assistant conversation) are reaped once
+// idle this long — see the blank-scratch branch in sweepStaleSessions.
+const BLANK_SCRATCH_MAX_AGE_MS = 60 * 60 * 1000; // 1h
 
 function _storedSessionFromFile(dir, filename, ensureLifecycle = true) {
     if (!filename.endsWith('.json')) return null;
@@ -1043,9 +1312,82 @@ function _withUnpersistedSessions(stored, invalidStorageIds = new Set()) {
     return [...sessionsById.values()].sort((a, b) => b.updatedAt - a.updatedAt);
 }
 
+// ── Incremental storage scan ────────────────────────────────────────────────
+// refreshFromStorage used to re-parse EVERY session JSON (full transcripts,
+// multi-MB files) on each desktop sidebar refresh. A summary only changes when
+// its file changes, so key a per-file row cache on (mtimeMs, size): unchanged
+// files reuse the cached row, changed/new files re-parse, vanished files drop
+// out. Storage stays the truth source — the sidecar index is never trusted.
+const _summaryScanCache = new Map(); // filename → { mtimeMs, size, row|null }
+
+function _scanStoredSessionSummaryRows() {
+    const dir = getStoreDir();
+    if (!existsSync(dir)) {
+        const changed = _summaryScanCache.size > 0;
+        _summaryScanCache.clear();
+        return { rows: [], invalidStorageIds: new Set(), changed };
+    }
+    const files = readdirSync(dir).filter(f => f.endsWith('.json'));
+    const present = new Set(files);
+    let changed = false;
+    for (const key of [..._summaryScanCache.keys()]) {
+        if (!present.has(key)) {
+            _summaryScanCache.delete(key);
+            changed = true;
+        }
+    }
+    const rows = [];
+    const invalidStorageIds = new Set();
+    const markInvalid = (filename) => {
+        const storageId = filename.slice(0, -5);
+        if (/^[A-Za-z0-9_-]+$/.test(storageId)) invalidStorageIds.add(storageId);
+    };
+    for (const f of files) {
+        let fileStat = null;
+        try { fileStat = statSync(join(dir, f)); } catch { /* deleted mid-scan */ }
+        if (!fileStat) {
+            if (_summaryScanCache.delete(f)) changed = true;
+            continue;
+        }
+        const cached = _summaryScanCache.get(f);
+        if (cached && cached.mtimeMs === fileStat.mtimeMs && cached.size === fileStat.size) {
+            if (cached.row) rows.push(cached.row);
+            else markInvalid(f);
+            continue;
+        }
+        const session = _storedSessionFromFile(dir, f);
+        const row = session ? _sessionSummary(session) : null;
+        _summaryScanCache.set(f, { mtimeMs: fileStat.mtimeMs, size: fileStat.size, row });
+        changed = true;
+        if (row) rows.push(row);
+        else markInvalid(f);
+    }
+    rows.sort((a, b) => (b.updatedAt || 0) - (a.updatedAt || 0));
+    return { rows, invalidStorageIds, changed };
+}
+
+// Summary-level twin of _withUnpersistedSessions: overlay queued/in-flight
+// saves that have no disk record yet (read-your-writes for brand-new sessions).
+function _overlayUnpersistedSummaryRows(rows, invalidStorageIds = new Set()) {
+    const byId = new Map(rows.map((row) => [row.id, row]));
+    const addIfUnpersisted = (id, session, opts) => {
+        if (!id || byId.has(id) || invalidStorageIds.has(id) || _isCancelledWrite(opts)) return;
+        if (session?.id !== id) return;
+        const row = _sessionSummary(_ensureLifecycleFields(session));
+        if (row) byId.set(id, row);
+    };
+    for (const [id, pending] of _savePending) {
+        const payload = pending.queued || pending.payload;
+        addIfUnpersisted(id, payload?.session, payload?.opts);
+    }
+    for (const [, pending] of _saveWorkerPending) addIfUnpersisted(pending.id, pending.session, pending.opts);
+    for (const [id, pending] of _saveAsyncQueued) addIfUnpersisted(id, pending.session, pending.opts);
+    return [...byId.values()].sort((a, b) => (b.updatedAt || 0) - (a.updatedAt || 0));
+}
+
 export function rebuildSessionSummaryIndex() {
     _ensureSummaryCacheDataDir();
-    const rows = listStoredSessions().map(_sessionSummary).filter(Boolean);
+    const { rows } = _scanStoredSessionSummaryRows();
     return _setSummaryRowsCache(_writeSummaryIndex(rows));
 }
 
@@ -1057,11 +1399,14 @@ export function listStoredSessionSummaries(options = {}) {
     // authorization), so it must not trust either the cache or sidecar.
     if (options.refreshFromStorage === true) {
         try {
-            const invalidStorageIds = new Set();
-            const stored = listStoredSessions({ _invalidStorageIds: invalidStorageIds });
-            const persistedRows = stored.map(_sessionSummary).filter(Boolean);
-            const rows = _withUnpersistedSessions(stored, invalidStorageIds).map(_sessionSummary).filter(Boolean);
-            try { _writeSummaryIndex(persistedRows); } catch { /* sidecar remains best-effort */ }
+            const { rows: persistedRows, invalidStorageIds, changed } = _scanStoredSessionSummaryRows();
+            const rows = _overlayUnpersistedSummaryRows(persistedRows, invalidStorageIds);
+            // Unchanged scans skip the sidecar rewrite — refresh is called on
+            // every sidebar poll/push and must not grind a multi-MB atomic
+            // write when no session actually changed.
+            if (changed) {
+                try { _writeSummaryIndex(persistedRows); } catch { /* sidecar remains best-effort */ }
+            }
             // A direct scan settles deletion state too; retain only active
             // optimistic write overlays, never a stale local removal.
             _summaryCacheRemovals.clear();
@@ -1083,7 +1428,9 @@ export function listStoredSessionSummaries(options = {}) {
         p = summaryIndexPath();
         hasIndex = existsSync(p);
         if (hasIndex) {
-            indexedRows = _normalizeSummaryIndex(JSON.parse(readFileSync(p, 'utf-8'))).rows;
+            const raw = JSON.parse(readFileSync(p, 'utf-8'));
+            hasIndex = Number(raw?.version) === SESSION_SUMMARY_INDEX_VERSION;
+            if (hasIndex) indexedRows = _normalizeSummaryIndex(raw).rows;
         }
     } catch { /* unreadable/malformed sidecar falls through to rebuild */ }
 
@@ -1290,6 +1637,19 @@ export function sweepStaleSessions(ttlMs, options = {}) {
                 : (sweepTombstones ? tombstoneMaxAgeMs : 0);
             const newestKnown = Math.max(row.updatedAt || 0, row.lastHeartbeatAt || 0, row.createdAt || 0, jsonMtime, heartbeatMtime);
             if (freshnessGateMs > 0 && newestKnown > 0 && now - newestKnown <= freshnessGateMs) {
+                // Fresh agent/legacy sessions survive idle close but still
+                // participate in the resumable-open retention cap. The cap
+                // performs its own commit-edge liveness veto before deletion.
+                if (retainOpen && sweepIdle
+                    && (!(typeof gateOwner === 'string' && gateOwner.length > 0)
+                        || isAgentOwner({ owner: gateOwner }))) {
+                    openCandidates.push({
+                        id: row.id,
+                        lastActive: newestKnown,
+                        heartbeatSnapshotMtime: heartbeatMtime,
+                        heartbeatFreshMs: terminalReapMsForGate ?? maxAge,
+                    });
+                }
                 remaining++;
                 continue;
             }
@@ -1314,6 +1674,33 @@ export function sweepStaleSessions(ttlMs, options = {}) {
             // foreground session, which is idle during a gated sweep). Only the
             // ephemeral agent/ownerless sessions below feed the cap.
             if (typeof effOwner === 'string' && effOwner.length > 0 && !isAgentOwner(ownerRef)) {
+                // Blank-scratch exception to user-session permanence: a session
+                // with ZERO user/assistant conversation (engine boot artifact,
+                // force-killed window, crashed host) has nothing to preserve.
+                // Relaunch storms otherwise pile hundreds of "(blank)" rows
+                // that no sweep may touch. Reap once cold; liveness/heartbeat
+                // vetoes inside deleteSession still protect an in-flight boot.
+                const _msgsArr = Array.isArray(actual?.messages) ? actual.messages : null;
+                const _convCount = _msgsArr
+                    ? _msgsArr.filter((m) => m && (m.role === 'user' || m.role === 'assistant')).length
+                    : (Number(row.messageCount) || 0);
+                const _blankLastActive = Math.max(effUpdatedAt, effLastHb, effCreatedAt, heartbeatMtime || 0);
+                if (sweepIdle && _convCount === 0
+                    && now - _blankLastActive > BLANK_SCRATCH_MAX_AGE_MS
+                    && !(isSessionLive && isSessionLive(row.id))) {
+                    try {
+                        if (deleteSession(row.id, {
+                            deferSummaryUpdate: true,
+                            isSessionLive,
+                            heartbeatSnapshotMtime: heartbeatMtime,
+                            heartbeatFreshMs: BLANK_SCRATCH_MAX_AGE_MS,
+                        })) {
+                            openPruned++;
+                            openPrunedDetails.push({ id: row.id, ageSeconds: Math.floor((now - _blankLastActive) / 1000) });
+                            continue;
+                        }
+                    } catch { /* keep the row on failure */ }
+                }
                 remaining++;
                 continue;
             }
@@ -1321,6 +1708,14 @@ export function sweepStaleSessions(ttlMs, options = {}) {
                 remaining++;
                 continue;
             }
+            // The manager may sweep while unrelated sessions are active. Protect
+            // this specific locally-current/in-flight session regardless of stale
+            // on-disk timestamps; its controller/heartbeat owner decides when it
+            // is safe to become an idle-sweep candidate.
+            if (isSessionLive && isSessionLive(row.id)) {
+                remaining++;
+                continue;
+            }
             // Prefer .hb sidecar mtime — updated at tight cadence (≤5s) without
             // serialising the full JSON, so it reflects true liveness more
             // accurately than the JSON timestamp fields.
@@ -1338,11 +1733,38 @@ export function sweepStaleSessions(ttlMs, options = {}) {
             const terminalReapMs = isCompletedAgent ? terminalReapMsForGate : null;
             const sessionMaxAge = terminalReapMs ?? maxAge;
             if (now - lastActive > sessionMaxAge) {
-                try { markSessionClosed(row.id, 'idle-sweep'); }
+                // Close is destructive and the earlier heartbeat stat can race a
+                // different process publishing fresh liveness. Re-check both
+                // local runtime ownership and the sidecar at the commit edge.
+                if (isSessionLive && isSessionLive(row.id)) {
+                    remaining++;
+                    continue;
+                }
+                let preCloseHeartbeatMtime = 0;
+                try {
+                    const hbPath = join(dir, `${row.id}.hb`);
+                    if (existsSync(hbPath)) preCloseHeartbeatMtime = statSync(hbPath).mtimeMs || 0;
+                } catch { /* sidecar unavailable — retain scan-time gates */ }
+                if (preCloseHeartbeatMtime > 0 && now - preCloseHeartbeatMtime <= sessionMaxAge) {
+                    remaining++;
+                    continue;
+                }
+                let closeResult = null;
+                try {
+                    closeResult = markSessionClosed(row.id, 'idle-sweep', {
+                        isSessionLive,
+                        heartbeatSnapshotMtime: heartbeatMtime,
+                        heartbeatFreshMs: sessionMaxAge,
+                    });
+                }
                 catch (err) {
                     process.stderr.write(`[session-store] idle-sweep close failed for ${row.id}: ${err?.message}\n`);
                     continue;
                 }
+                if (closeResult == null) {
+                    remaining++;
+                    continue;
+                }
                 cleaned++;
                 details.push({
                     id: row.id,
@@ -1351,7 +1773,12 @@ export function sweepStaleSessions(ttlMs, options = {}) {
                     bashSessionId: effBashId,
                 });
             } else {
-                if (retainOpen) openCandidates.push({ id: row.id, lastActive });
+                if (retainOpen) openCandidates.push({
+                    id: row.id,
+                    lastActive,
+                    heartbeatSnapshotMtime: heartbeatMtime,
+                    heartbeatFreshMs: sessionMaxAge,
+                });
                 remaining++;
             }
         }
@@ -1370,7 +1797,12 @@ export function sweepStaleSessions(ttlMs, options = {}) {
             const overCount = kept >= openMaxCount;
             if (!tooOld && !overCount) { kept++; continue; }
             try {
-                if (deleteSession(c.id, { deferSummaryUpdate: true })) {
+                if (deleteSession(c.id, {
+                    deferSummaryUpdate: true,
+                    isSessionLive,
+                    heartbeatSnapshotMtime: c.heartbeatSnapshotMtime,
+                    heartbeatFreshMs: c.heartbeatFreshMs,
+                })) {
                     openPruned++;
                     openPrunedDetails.push({ id: c.id, ageSeconds: Math.floor((now - (c.lastActive || 0)) / 1000) });
                     if (remaining > 0) remaining--;
@@ -1380,14 +1812,14 @@ export function sweepStaleSessions(ttlMs, options = {}) {
             } catch { kept++; }
         }
     }
-    // Orphan .hb reap: a heartbeat sidecar whose .json no longer exists is dead
-    // weight once it is also stale (older than maxAge) — the session JSON was
-    // swept/closed but the .hb lingered (a pre-fix orphaned heartbeat). The
-    // staleness gate avoids nuking the .hb of a session mid-create whose .json
-    // write has not landed yet.
+    // Orphan .hb/.own reap: a heartbeat/presence sidecar whose .json no longer
+    // exists is dead weight once it is also stale (older than maxAge) — the
+    // session JSON was swept/closed but the sidecar lingered (crashed owner or
+    // pre-fix orphan). The staleness gate avoids nuking the sidecar of a
+    // session mid-create whose .json write has not landed yet.
     try {
-        for (const h of readdirSync(dir).filter(f => f.endsWith('.hb'))) {
-            if (existsSync(join(dir, h.replace(/\.hb$/, '.json')))) continue;
+        for (const h of readdirSync(dir).filter(f => f.endsWith('.hb') || f.endsWith('.own'))) {
+            if (existsSync(join(dir, h.replace(/\.(hb|own)$/, '.json')))) continue;
             let hbMtime = 0;
             try { hbMtime = statSync(join(dir, h)).mtimeMs; } catch { continue; }
             if (now - hbMtime > maxAge) {