mixdog 0.9.87 → 0.9.89

package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs

@@ -6,7 +6,7 @@ import { createHash, randomBytes } from 'crypto';
 import { resolvePluginData } from '../../../../shared/plugin-paths.mjs';
 import { updateJsonAtomicSync, updateJsonAtomic } from '../../../../shared/atomic-file.mjs';
 import { promptContentText, isInternalRuntimeNotificationText } from './prompt-utils.mjs';
-import { loadSession, saveSessionAsync } from '../store.mjs';
+import { loadSession, readSessionLifecycleStateFromDisk, saveSessionAsync } from '../store.mjs';
 import { isDeliveredCompletion, logDuplicateSkip } from './delivered-completions.mjs';
 
 const _sessionPendingMessages = new Map();
@@ -40,7 +40,128 @@ const _pendingPersistTails = new Map();
 const _inDeliveryPendingIds = new Map();
 const _ackedPendingIds = new Map();
 const _pendingHydrations = new Map();
+// Claim ledger for the in-delivery ids: the exact entries a drain handed to a
+// turn. A drain consumes the not-yet-flushed persist buffer, so without this
+// copy a failed turn could release ids whose payload exists nowhere (memory,
+// spool, replay) any more. Claims are dropped on ack (delivered) or on
+// release (restored below). A release may only restore while the session
+// record still accepts pending state — a tombstoned session refuses it however
+// old the claim is.
+const _claimedPendingMessages = new Map();
 let _pendingPersistImmediate = null;
+// Lifecycle token stamped on the exact entry object a drain/enqueue observed.
+// Symbol-keyed so it never reaches the spool (JSON.stringify ignores symbols)
+// and never changes the entry's public shape.
+const LIFECYCLE_TOKEN = Symbol('pendingLifecycleToken');
+// Deterministic test seams for the check→mutate windows (close/detach racing
+// an enqueue publish, a spool commit, or a multi-step hydrate cleanup).
+let _pendingTestHooks = null;
+function runPendingTestHook(name, payload) {
+    const hook = _pendingTestHooks?.[name];
+    if (typeof hook !== 'function') return;
+    try { hook(payload); } catch { /* test seam is best-effort */ }
+}
+
+// Authoritative pending-state lifecycle gate. The DURABLE record is the only
+// authority: readSessionLifecycleStateFromDisk reports absence / open / closed
+// / unreadable straight from disk, bypassing every live/pending snapshot, so a
+// stale open live copy (e.g. an id under _droppedSaveIds) can never mask a
+// tombstone. Only an explicit create/reopen/new-generation record — never an
+// ordinary enqueue — makes a session writable again.
+//   'absent'     → never saved: open and writable (generation 0)
+//   'open'       → durable record at this generation
+//   'closed'     → tombstone: refuses every write
+//   'unreadable' → IO error / malformed / foreign record: FAIL CLOSED, and a
+//                  live snapshot may only ADD refusal, never open it
+function pendingSessionLifecycle(sessionId) {
+    let disk = null;
+    try { disk = readSessionLifecycleStateFromDisk(sessionId); } catch { disk = null; }
+    const state = disk?.state;
+    if (state === 'open' || state === 'closed') {
+        return { source: state, closed: state === 'closed', generation: Number(disk.generation) || 0 };
+    }
+    if (state === 'absent') return { source: 'absent', closed: false, generation: 0 };
+    // Unreadable/corrupt/foreign (or the read itself threw): never writable.
+    return { source: 'unreadable', closed: true, generation: 0 };
+}
+
+// Epoch token: the exact lifecycle a claim/enqueue/hydration was taken under.
+function pendingLifecycleToken(lifecycle) {
+    return `${lifecycle.source}:${lifecycle.generation}`;
+}
+
+function currentPendingLifecycleToken(sessionId) {
+    return pendingLifecycleToken(pendingSessionLifecycle(sessionId));
+}
+
+function stampLifecycleToken(entry, token) {
+    if (!entry || typeof entry !== 'object' || !token) return entry;
+    try {
+        Object.defineProperty(entry, LIFECYCLE_TOKEN, {
+            value: token, enumerable: false, configurable: true, writable: true,
+        });
+    } catch { /* frozen entry: the claim map still carries the token */ }
+    return entry;
+}
+
+function entryLifecycleToken(entry) {
+    const token = entry && typeof entry === 'object' ? entry[LIFECYCLE_TOKEN] : null;
+    return typeof token === 'string' && token ? token : null;
+}
+
+// Clone/normalize boundary helper. Every normalized copy of an accepted entry
+// MUST inherit the source entry's IMMUTABLE lifecycle token: a copy that loses
+// it is later restamped with whatever epoch happens to be current, which
+// legitimizes an old-generation entry (drain → claim → release/ack against a
+// reopened owner). Symbols are dropped by object spread and JSON, so every
+// constructor of a derived entry routes through here.
+function carryLifecycleToken(target, source) {
+    const token = entryLifecycleToken(source);
+    return token ? stampLifecycleToken(target, token) : target;
+}
+
+// Pure epoch comparison against an already-read lifecycle (no IO), so a caller
+// that judges many entries reads the durable record once.
+function lifecycleTokenStale(now, sinceToken) {
+    if (sinceToken === null || sinceToken === undefined) return false;
+    const separator = String(sinceToken).indexOf(':');
+    const sinceSource = String(sinceToken).slice(0, separator);
+    const sinceGeneration = Number(String(sinceToken).slice(separator + 1)) || 0;
+    // Only close/detach moves the durable generation, so a differing one means
+    // the session was closed/detached/reopened since: refuse.
+    if (now.generation !== sinceGeneration) return true;
+    // A record that was DURABLE at claim/hydration start must still be durable:
+    // an unreadable record must never be mistaken for an open session. The
+    // reverse (never-saved → first durable save at the same generation) is the
+    // ordinary lifecycle of a fresh session and stays valid.
+    if (sinceSource === 'open' && now.source !== 'open') return true;
+    return false;
+}
+
+// True when pending state for this session must NOT be (re)created/published:
+// tombstoned, or the durable lifecycle epoch moved since `sinceToken`.
+function pendingLifecycleInvalidated(sessionId, sinceToken = null) {
+    const now = pendingSessionLifecycle(sessionId);
+    if (now.closed) return true;
+    return lifecycleTokenStale(now, sinceToken);
+}
+
+// Epoch check for DESTRUCTIVE spool/ledger mutations (ack, prune, clear,
+// foreign drain). Deliberately tombstone-TOLERANT: deleting the rows of the
+// epoch that just closed is legitimate (the close path itself does it), while
+// a generation move — the reopened/detached owner taking over — means those
+// rows are no longer ours to touch. Callers re-evaluate this INSIDE the spool
+// transaction, immediately before mutating.
+function pendingLifecycleEpochMoved(sessionId, sinceToken = null) {
+    if (sinceToken === null || sinceToken === undefined) return false;
+    const now = pendingSessionLifecycle(sessionId);
+    const separator = String(sinceToken).indexOf(':');
+    const sinceGeneration = Number(String(sinceToken).slice(separator + 1)) || 0;
+    if (now.generation !== sinceGeneration) return true;
+    // An unreadable/foreign record is never authority for a deletion.
+    if (now.source === 'unreadable') return true;
+    return false;
+}
 
 function pendingIdSet(map, sessionId) {
     let ids = map.get(sessionId);
@@ -228,7 +349,8 @@ function normalizePendingMessageEntry(entry) {
         out = fallback ? { content: fallback, text: text || fallback } : null;
     }
     if (!out) return null;
-    return marker ? { ...out, ...identity, ...marker } : { ...out, ...identity };
+    // The lifecycle token rides along every derived copy (see carryLifecycleToken).
+    return carryLifecycleToken(marker ? { ...out, ...identity, ...marker } : { ...out, ...identity }, entry);
 }
 
 function pendingMessageText(entry) {
@@ -247,7 +369,7 @@ function pendingMessageQueueEntry(entry) {
         ? { notificationKind: COMPLETION_NOTIFICATION_KIND, enqueuedAt: normalized.enqueuedAt }
         : null;
     const base = { ...identity, content: normalized.content, text: normalized.text || promptContentText(normalized.content).trim() };
-    return marker ? { ...base, ...marker } : base;
+    return carryLifecycleToken(marker ? { ...base, ...marker } : base, entry);
 }
 
 // Canonical persisted-queue entry: a plain string for user/steering messages,
@@ -285,8 +407,16 @@ function normalizePersistedEntry(entry, options = {}) {
 
 function persistPendingMessages(sessionId, messages) {
     if (!isValidPendingSessionId(sessionId)) return 0;
-    const persistedMessages = (Array.isArray(messages) ? messages : [messages])
-        .map(normalizePersistedEntry)
+    const sourceMessages = Array.isArray(messages) ? messages : [messages];
+    const persistedMessages = sourceMessages
+        .map((entry) => {
+            const normalized = normalizePersistedEntry(entry);
+            if (!normalized) return null;
+            // Carry the lifecycle token observed when this entry was accepted
+            // so the commit below can drop it if the session moved on.
+            const token = entryLifecycleToken(entry);
+            return token ? stampLifecycleToken(normalized, token) : normalized;
+        })
         .filter(Boolean);
     if (persistedMessages.length === 0) return 0;
     // Async lock wait: this runs on the lead/TUI main process (tool-exec +
@@ -295,9 +425,27 @@ function persistPendingMessages(sessionId, messages) {
     // Best-effort: the returned promise is fire-and-forget; depth is reported
     // optimistically from the buffered batch length.
     const operation = updateJsonAtomic(pendingMessagesPath(), (raw) => {
+        // Durable commit window: re-read the lifecycle INSIDE the spool lock.
+        // A cross-process close/detach between acceptance and this commit must
+        // drop the old-generation input without touching the new owner's rows.
+        runPendingTestHook('persist:beforeCommit', { sessionId });
+        const committable = persistedMessages.filter((entry) => !pendingLifecycleInvalidated(
+            sessionId,
+            entryLifecycleToken(entry),
+        ));
+        if (committable.length === 0) return undefined;
         const next = normalizePendingStore(raw);
         const q = Array.isArray(next.sessions[sessionId]) ? next.sessions[sessionId] : [];
-        q.push(...persistedMessages);
+        // Idempotent by id: a restored claim (releasePendingMessages) may be
+        // durable already, and re-appending it would leave two spool rows for
+        // one queued input (double replay after a restart).
+        const existingIds = new Set(q.map(pendingMessageId).filter(Boolean));
+        const additions = committable.filter((entry) => {
+            const id = pendingMessageId(entry);
+            return !id || !existingIds.has(id);
+        });
+        if (additions.length === 0) return undefined;
+        q.push(...additions);
         next.sessions[sessionId] = q;
         const now = Date.now();
         next.updatedAt = now;
@@ -340,6 +488,8 @@ function schedulePendingMessagePersist(sessionId, message) {
     if (!isValidPendingSessionId(sessionId)) return 0;
     const persistedMessage = normalizePersistedEntry(message);
     if (!persistedMessage) return 0;
+    const token = entryLifecycleToken(message);
+    if (token) stampLifecycleToken(persistedMessage, token);
     const q = _pendingPersistBuffers.get(sessionId) || [];
     q.push(persistedMessage);
     _pendingPersistBuffers.set(sessionId, q);
@@ -360,13 +510,75 @@ function takeBufferedPendingMessages(sessionId) {
     return buffered.slice();
 }
 
-export function acknowledgePendingMessages(sessionId, deliveredEntries) {
-    const ids = new Set((Array.isArray(deliveredEntries) ? deliveredEntries : [])
+function claimPendingEntries(sessionId, entries) {
+    if (!Array.isArray(entries) || entries.length === 0) return;
+    const currentToken = currentPendingLifecycleToken(sessionId);
+    const prior = _claimedPendingMessages.get(sessionId);
+    // Claims are stamped with the lifecycle epoch observed at claim time; a
+    // claim from an older epoch is stale and never merges into the new one.
+    const claims = prior && prior.token === currentToken ? prior : { token: currentToken, entries: new Map() };
+    for (const entry of entries) {
+        const id = pendingMessageId(entry);
+        if (!id) continue;
+        // The token also rides on the delivered entry OBJECT: a later claim or
+        // ack may replace/remove this session's map state, and the old release
+        // must still judge itself by the epoch it was handed.
+        //
+        // NEVER restamp: the token an entry was ACCEPTED under is immutable.
+        // Overwriting it with the epoch current at claim time was exactly how
+        // a generation-0 entry that survived a generation-1 detach got
+        // legitimized (and then restored by a failed turn). Entries reaching
+        // here without a token were never stamped (legacy/foreign paths) and
+        // are claimed under the epoch observed now.
+        if (!entryLifecycleToken(entry)) stampLifecycleToken(entry, currentToken);
+        claims.entries.set(id, entry);
+    }
+    if (claims.entries.size > 0) _claimedPendingMessages.set(sessionId, claims);
+}
+
+function dropPendingClaims(sessionId, ids) {
+    const claims = _claimedPendingMessages.get(sessionId);
+    if (!claims) return;
+    for (const id of ids) claims.entries.delete(id);
+    if (claims.entries.size === 0) _claimedPendingMessages.delete(sessionId);
+}
+
+function pendingIdStillQueued(sessionId, id) {
+    for (const map of [_sessionPendingMessages, _pendingPersistBuffers, _hydratedPendingMessages]) {
+        const q = map.get(sessionId);
+        if (Array.isArray(q) && q.some((entry) => pendingMessageId(entry) === id)) return true;
+    }
+    return false;
+}
+
+export function acknowledgePendingMessages(sessionId, deliveredEntries, options = {}) {
+    const entries = Array.isArray(deliveredEntries) ? deliveredEntries : [];
+    // Expected lifecycle epoch of THIS acknowledgement: the immutable token of
+    // the delivered entries (explicit option for spool-sourced entries that
+    // never carried one), falling back to the epoch observed right now.
+    const explicitToken = typeof options?.expectedToken === 'string' && options.expectedToken
+        ? options.expectedToken
+        : null;
+    const expectedToken = explicitToken
+        || entries.map(entryLifecycleToken).find(Boolean)
+        || currentPendingLifecycleToken(sessionId);
+    // An entry from a DIFFERENT epoch is not part of this ack: its rows belong
+    // to another owner and must not be deleted by this transaction.
+    const ids = new Set(entries
+        .filter((entry) => {
+            const token = entryLifecycleToken(entry);
+            return !token || token === expectedToken;
+        })
         .map(pendingMessageId).filter(Boolean));
     if (ids.size === 0) return Promise.resolve(false);
+    // The epoch already moved before we touched anything: leave the reopened
+    // owner's memory, ledger and spool rows exactly as they are.
+    if (pendingLifecycleEpochMoved(sessionId, expectedToken)) return Promise.resolve(false);
     const inDelivery = pendingIdSet(_inDeliveryPendingIds, sessionId);
     const acked = pendingIdSet(_ackedPendingIds, sessionId);
     for (const id of ids) { inDelivery.delete(id); acked.add(id); }
+    // Delivered for good: the claim copy is no longer a recovery source.
+    dropPendingClaims(sessionId, ids);
     const purgeMemory = () => {
         for (const map of [_pendingPersistBuffers, _sessionPendingMessages, _hydratedPendingMessages]) {
             const q = map.get(sessionId);
@@ -377,11 +589,22 @@ export function acknowledgePendingMessages(sessionId, deliveredEntries) {
         }
     };
     purgeMemory();
+    let epochMoved = false;
     const precedingPersist = _pendingPersistTails.get(sessionId) || Promise.resolve();
     const operation = precedingPersist.catch(() => {}).then(() => {
         // A failed preceding persist may have requeued after the first purge.
-        purgeMemory();
+        // Skipped once the epoch moved: those queue entries are the new
+        // owner's (a hydrate can republish the very same durable ids).
+        if (!pendingLifecycleEpochMoved(sessionId, expectedToken)) purgeMemory();
         return updateJsonAtomic(pendingMessagesPath(), (raw) => {
+        // Atomic re-validation: we now hold the spool lock. A generation
+        // movement while we waited means the rows are the reopened owner's —
+        // delete nothing and report failure so no ledger prune follows.
+        runPendingTestHook('ack:beforeCommit', { sessionId });
+        if (pendingLifecycleEpochMoved(sessionId, expectedToken)) {
+            epochMoved = true;
+            return undefined;
+        }
         const next = normalizePendingStore(raw);
         const q = Array.isArray(next.sessions[sessionId]) ? next.sessions[sessionId] : [];
         const kept = q.filter((entry) => !ids.has(pendingMessageId(entry)));
@@ -397,11 +620,20 @@ export function acknowledgePendingMessages(sessionId, deliveredEntries) {
         }, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
     });
     _pendingPersistTails.set(sessionId, operation);
-    const reported = operation.then(() => true).catch((err) => {
+    const reported = operation.then(() => !epochMoved).catch((err) => {
         try { process.stderr.write(`[session] pending-message ack failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
         return false;
     }).then((ok) => {
         if (_pendingPersistTails.get(sessionId) === operation) _pendingPersistTails.delete(sessionId);
+        // Refused because the epoch moved: this ack never happened. Drop the
+        // acked-id suppression again — those durable rows now belong to the
+        // reopened owner and its hydrate must be able to take them.
+        if (epochMoved) {
+            const movedAcked = pendingIdSet(_ackedPendingIds, sessionId);
+            for (const id of ids) movedAcked.delete(id);
+            if (movedAcked.size === 0) _ackedPendingIds.delete(sessionId);
+            return false;
+        }
         // Keep the acked ids when the spool cleanup FAILED: the durable entry
         // still exists, and forgetting that it was already delivered let the
         // foreign-injection drain re-take it as a "new" cross-surface submit
@@ -429,10 +661,16 @@ export function recordPendingMessageDelivery(session, deliveredEntries) {
     session.deliveredPendingMessageIds = [...new Set([...ledger, ...added])];
 }
 
-async function pruneCleanupConfirmedLedger(sessionId, confirmedEntries, session = null, persist = null) {
+async function pruneCleanupConfirmedLedger(sessionId, confirmedEntries, session = null, persist = null, expectedToken = null) {
     const confirmedIds = new Set((Array.isArray(confirmedEntries) ? confirmedEntries : [])
         .map(pendingMessageId).filter(Boolean));
     if (confirmedIds.size === 0) return false;
+    // Same atomic ownership rule as the spool ack: the ledger of a session
+    // whose durable epoch moved belongs to the reopened owner.
+    const token = expectedToken
+        || (Array.isArray(confirmedEntries) ? confirmedEntries.map(entryLifecycleToken).find(Boolean) : null)
+        || null;
+    if (token && pendingLifecycleEpochMoved(sessionId, token)) return false;
     const target = session || loadSession(sessionId);
     if (!target) return false;
     const ledger = Array.isArray(target.deliveredPendingMessageIds)
@@ -443,6 +681,9 @@ async function pruneCleanupConfirmedLedger(sessionId, confirmedEntries, session
     // Confirmed IDs need no replay protection and are removed immediately
     // (therefore bounded below any finite confirmed-ID retention cap).
     // Unconfirmed IDs are never size-evicted.
+    // Re-checked immediately before the mutation: the load above is an await
+    // boundary for the caller's chain.
+    if (token && pendingLifecycleEpochMoved(sessionId, token)) return false;
     target.deliveredPendingMessageIds = kept;
     if (typeof persist === 'function') await persist();
     else await saveSessionAsync(target, { expectedGeneration: target.generation });
@@ -470,10 +711,47 @@ export function finalizePendingMessageDelivery(session, deliveredEntries, durabl
 
 export function releasePendingMessages(sessionId, deliveredEntries) {
     const inDelivery = pendingIdSet(_inDeliveryPendingIds, sessionId);
+    const acked = pendingIdSet(_ackedPendingIds, sessionId);
+    const claim = _claimedPendingMessages.get(sessionId);
+    // Restoring is a WRITE of pending state: allowed only while the DURABLE
+    // lifecycle is still open AND still the exact epoch THIS delivery was
+    // claimed under. The token is read from the released entry itself and only
+    // falls back to the map when the two agree — a newer claim/ack that
+    // replaced or removed the map state can never re-open an old release.
+    const claimToken = (entry) => entryLifecycleToken(entry)
+        || (claim && claim.entries.has(pendingMessageId(entry)) ? claim.token : null);
+    const releasedIds = [];
+    const restored = [];
     for (const entry of Array.isArray(deliveredEntries) ? deliveredEntries : []) {
         const id = pendingMessageId(entry);
-        if (id) inDelivery.delete(id);
+        if (!id) continue;
+        inDelivery.delete(id);
+        releasedIds.push(id);
+        // The turn did NOT deliver this input. Restore the claimed entry so it
+        // is queued (and durable) again: the drain that claimed it may have
+        // consumed the not-yet-flushed persist buffer, in which case clearing
+        // the id alone would lose the message from memory, spool and replay.
+        if (acked.has(id)) continue;
+        const token = claimToken(entry);
+        // No token at all (never claimed through a drain) → fail closed.
+        if (!token || pendingLifecycleInvalidated(sessionId, token)) continue;
+        if (pendingIdStillQueued(sessionId, id)) continue;
+        const claimed = claim?.entries.get(id) || pendingMessageQueueEntry(entry);
+        if (claimed) restored.push(stampLifecycleToken(claimed, token));
     }
+    if (restored.length > 0) {
+        const q = _sessionPendingMessages.get(sessionId) || [];
+        q.push(...restored);
+        // FIFO by original enqueue time — a restored entry predates anything
+        // queued while the failed turn was running.
+        q.sort((a, b) => (Number(a?.enqueuedAt) || 0) - (Number(b?.enqueuedAt) || 0));
+        _sessionPendingMessages.set(sessionId, q);
+        // Durability: persistPendingMessages is id-idempotent, so an entry that
+        // was already flushed keeps its single spool row while one that never
+        // reached disk gets written now (restart replays it exactly once).
+        for (const entry of restored) schedulePendingMessagePersist(sessionId, entry);
+    }
+    dropPendingClaims(sessionId, releasedIds);
     if (inDelivery.size === 0) _inDeliveryPendingIds.delete(sessionId);
 }
 
@@ -489,6 +767,15 @@ export function hydratePendingMessages(sessionId, options = {}) {
       let staleLedgerEntries = [];
       const staleUserEntries = [];
       const ledgerSession = loadSession(sessionId);
+      // Durable lifecycle epoch this hydration started under. Publishing claims the
+      // durable entries into THIS process's memory, so it must be revalidated
+      // after every await (spool IO, test beforePublish, ack/prune): a close
+      // (tombstone) or a close/detach generation bump in between means the
+      // entries belong to the next owner, not to us. Aborting publishes
+      // nothing and leaves the durable rows untouched for that owner.
+      const startLifecycle = pendingSessionLifecycle(sessionId);
+      const startToken = pendingLifecycleToken(startLifecycle);
+      if (startLifecycle.closed) return 0;
       try {
         const deliveredLedger = new Set(ledgerSession?.deliveredPendingMessageIds || []);
         const inDelivery = pendingIdSet(_inDeliveryPendingIds, sessionId);
@@ -524,35 +811,51 @@ export function hydratePendingMessages(sessionId, options = {}) {
             // acknowledges these exact ids. A crash here therefore redelivers.
             return undefined;
         }, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
         await options.beforePublish?.(hydrated);
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
       } catch (err) {
         try { process.stderr.write(`[session] pending-message hydrate failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
         return 0;
       }
       const cleanupConfirmed = staleLedgerEntries.slice();
       if (alreadyDelivered.length > 0) {
-        const cleaned = await acknowledgePendingMessages(sessionId, alreadyDelivered);
+        const cleaned = await acknowledgePendingMessages(sessionId, alreadyDelivered, { expectedToken: startToken });
         if (cleaned) cleanupConfirmed.push(...alreadyDelivered);
+        // Every individual cleanup await is its own window: stop before the
+        // next mutation once the lifecycle moved (the remaining spool rows and
+        // the ledger belong to the new owner from here on).
+        runPendingTestHook('hydrate:betweenCleanups', { sessionId });
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
       }
       if (staleUserEntries.length > 0) {
         try { process.stderr.write(`[session] dropped ${staleUserEntries.length} stale queued message(s) (older than ${Math.round(STALE_USER_INJECTION_TTL_MS / 60000)}m) sessionId=${sessionId}\n`); } catch {}
-        const cleaned = await acknowledgePendingMessages(sessionId, staleUserEntries);
+        const cleaned = await acknowledgePendingMessages(sessionId, staleUserEntries, { expectedToken: startToken });
         if (cleaned) cleanupConfirmed.push(...staleUserEntries);
+        runPendingTestHook('hydrate:betweenCleanups', { sessionId });
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
       }
       if (cleanupConfirmed.length > 0) {
         try {
             // One session save prunes both IDs whose replay spool was removed
             // now and IDs whose spool was already absent at hydration start.
-            await pruneCleanupConfirmedLedger(sessionId, cleanupConfirmed, ledgerSession);
+            await pruneCleanupConfirmedLedger(sessionId, cleanupConfirmed, ledgerSession, null, startToken);
         } catch (err) {
             try { process.stderr.write(`[session] pending-message ledger prune failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
         }
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
       }
       if (hydrated.length > 0) {
+        // Last revalidation: the ack/prune awaits above are another window.
+        if (pendingLifecycleInvalidated(sessionId, startToken)) return 0;
         const inDelivery = pendingIdSet(_inDeliveryPendingIds, sessionId);
         for (const entry of hydrated) {
             const id = pendingMessageId(entry);
             if (id) inDelivery.add(id);
+            // Durable rows carry no symbol: stamp the epoch this hydration was
+            // validated under, so every later clone/claim/ack judges itself by
+            // that immutable token instead of the epoch current at the time.
+            stampLifecycleToken(entry, startToken);
         }
         const existing = _hydratedPendingMessages.get(sessionId) || [];
         existing.push(...hydrated);
@@ -578,7 +881,12 @@ function clearPersistedPendingMessages(sessionId) {
     // dropped synchronously by _dropPendingMessageState; only the durable
     // spool row lags behind, exactly like the ack path.
     const preceding = _pendingPersistTails.get(sessionId) || Promise.resolve();
+    // Epoch this clear was ordered under (normally the tombstone that just
+    // landed). Re-validated inside the lock: a reopen/detach that moves the
+    // generation while this async clear waits must leave the new owner's rows.
+    const epochToken = currentPendingLifecycleToken(sessionId);
     const operation = preceding.catch(() => {}).then(() => updateJsonAtomic(pendingMessagesPath(), (raw) => {
+        if (pendingLifecycleEpochMoved(sessionId, epochToken)) return undefined;
         const next = normalizePendingStore(raw);
         if (!Object.prototype.hasOwnProperty.call(next.sessions, sessionId)) return undefined;
         delete next.sessions[sessionId];
@@ -695,6 +1003,17 @@ export function enqueuePendingMessage(sessionId, message) {
     // Caller-provided ids are never trusted across sessions/processes.
     const entry = normalized ? { ...normalized, id: newPendingMessageId() } : null;
     if (!sessionId || !entry) return 0;
+    // A provider/tool completion that lands after the session was tombstoned
+    // must NOT recreate memory/spool state (and never clears the tombstone):
+    // only an explicit create/reopen record makes the session writable again.
+    const token = currentPendingLifecycleToken(sessionId);
+    if (pendingLifecycleInvalidated(sessionId)) return 0;
+    stampLifecycleToken(entry, token);
+    // Re-validate immediately before the memory publish: a close/detach landing
+    // in this window must reject the input instead of recreating state for a
+    // session the new owner already took over.
+    runPendingTestHook('enqueue:beforePublish', { sessionId });
+    if (pendingLifecycleInvalidated(sessionId, token)) return 0;
     let q = _sessionPendingMessages.get(sessionId);
     if (!q) { q = []; _sessionPendingMessages.set(sessionId, q); }
     q.push(entry);
@@ -712,9 +1031,19 @@ export function enqueuePendingMessage(sessionId, message) {
  */
 export function enqueueRemotePendingMessage(sessionId, message) {
     if (!isValidPendingSessionId(sessionId)) return 0;
+    // Same durable-lifecycle authority as the local enqueue: a cross-surface
+    // submit that lands after the session was closed is refused before any
+    // spool mutation.
+    const token = currentPendingLifecycleToken(sessionId);
+    if (pendingLifecycleInvalidated(sessionId)) return 0;
     const normalized = pendingMessageQueueEntry(message);
     if (!normalized) return 0;
-    return persistPendingMessages(sessionId, [{ ...normalized, id: newPendingMessageId() }]);
+    // The token rides into the async spool commit, which revalidates under the
+    // spool lock (see persistPendingMessages).
+    runPendingTestHook('enqueue:beforePublish', { sessionId });
+    if (pendingLifecycleInvalidated(sessionId, token)) return 0;
+    const remoteEntry = stampLifecycleToken({ ...normalized, id: newPendingMessageId() }, token);
+    return persistPendingMessages(sessionId, [remoteEntry]);
 }
 
 // Spool-file mtime gate so the owner's idle poller costs one stat per tick,
@@ -745,10 +1074,18 @@ function _rememberForeignSpoolScan(sessionId, mtime) {
  */
 export function drainForeignUserInjections(sessionId) {
     if (!isValidPendingSessionId(sessionId)) return [];
+    // DELIVERY path, not cleanup: this takes rows out of the spool and hands
+    // them to a live turn. A tombstoned (or otherwise invalidated) session may
+    // neither receive nor lose input, so the tombstone-TOLERANT epoch check
+    // used by the explicit cleanup mutations (ack / clear) is wrong here — the
+    // token is refused outright when it was captured from a closed session.
+    // Checked BEFORE the mtime memo so a later reopen still sees this spool
+    // state as unscanned.
+    const epochToken = currentPendingLifecycleToken(sessionId);
+    if (pendingLifecycleInvalidated(sessionId)) return [];
     let mtime = 0;
     try { mtime = statSync(pendingMessagesPath()).mtimeMs || 0; } catch { return []; }
     if (_foreignSpoolScanMtimes.get(sessionId) === mtime) return [];
-    _rememberForeignSpoolScan(sessionId, mtime);
     const localIds = new Set();
     for (const map of [_sessionPendingMessages, _pendingPersistBuffers, _hydratedPendingMessages]) {
         for (const entry of map.get(sessionId) || []) {
@@ -770,8 +1107,25 @@ export function drainForeignUserInjections(sessionId) {
     } catch { /* ledger unavailable — in-memory sets still guard */ }
     const taken = [];
     let droppedStale = 0;
+    // The mtime memo may ONLY be armed by a scan that actually reached a
+    // lifecycle-valid decision under the spool lock. Arming it upfront meant a
+    // drain refused in-lock (a close/detach/reopen landing in the window) still
+    // recorded this spool state as "seen": after the reopen the new owner
+    // polled the same unchanged mtime and skipped it, stranding the rows until
+    // some unrelated spool WRITE bumped the file. A refusal must leave the
+    // spool immediately pollable again.
+    let lifecycleDecided = false;
     try {
         updateJsonAtomicSync(pendingMessagesPath(), (raw) => {
+            // Same authority re-read INSIDE the spool lock: a close/detach
+            // landing in this window must neither deliver to the old owner nor
+            // remove the reopened owner's rows.
+            runPendingTestHook('foreignDrain:beforeCommit', { sessionId });
+            if (pendingLifecycleInvalidated(sessionId, epochToken)) return undefined;
+            // Authoritative from here on: whatever this pass decides (take,
+            // drop-stale, or "nothing foreign here") is a real answer for THIS
+            // spool state, so the memo may skip re-scanning it.
+            lifecycleDecided = true;
             const next = normalizePendingStore(raw);
             const q = Array.isArray(next.sessions[sessionId]) ? next.sessions[sessionId] : [];
             if (q.length === 0) return undefined;
@@ -803,6 +1157,7 @@ export function drainForeignUserInjections(sessionId) {
         try { process.stderr.write(`[session] foreign-injection drain failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
         return [];
     }
+    if (lifecycleDecided) _rememberForeignSpoolScan(sessionId, mtime);
     if (droppedStale > 0) {
         try { process.stderr.write(`[session] dropped ${droppedStale} stale foreign submit(s) (older than ${Math.round(STALE_USER_INJECTION_TTL_MS / 60000)}m) sessionId=${sessionId}\n`); } catch {}
     }
@@ -874,9 +1229,35 @@ export function drainPendingMessages(sessionId) {
     const visible = modelVisiblePendingMessages(ordered
         .filter(({ entry }) => !dropped.includes(entry))
         .map(({ entry }) => entry));
+    // Epoch gate. Normalization above rebuilds every entry, but each copy
+    // inherits the IMMUTABLE token the entry was accepted under. An entry from
+    // an older epoch (a cross-process close/detach moved the generation while
+    // it sat in memory) belongs to the previous owner: it is REJECTED here,
+    // never restamped with the current epoch — which would legitimize it and
+    // let a failed turn restore it into the new owner's queue.
+    const lifecycleNow = pendingSessionLifecycle(sessionId);
+    const accepted = [];
+    let rejected = 0;
+    for (const entry of visible) {
+        const token = entryLifecycleToken(entry);
+        if (token && (lifecycleNow.closed || lifecycleTokenStale(lifecycleNow, token))) {
+            rejected += 1;
+            continue;
+        }
+        accepted.push(entry);
+    }
+    if (rejected > 0) {
+        try {
+            process.stderr.write(
+                `[session] rejected ${rejected} queued message(s) from a superseded lifecycle epoch sessionId=${sessionId}\n`,
+            );
+        } catch { /* best-effort */ }
+    }
     const inDelivery = pendingIdSet(_inDeliveryPendingIds, sessionId);
-    for (const entry of visible) if (entry.id) inDelivery.add(entry.id);
-    return visible;
+    for (const entry of accepted) if (entry.id) inDelivery.add(entry.id);
+    // Keep the exact claimed payloads until ack (delivered) or release.
+    claimPendingEntries(sessionId, accepted);
+    return accepted;
 }
 
 // Snapshot queued entries without draining them. Compaction uses this to keep
@@ -913,6 +1294,12 @@ export function _dropPendingMessageState(id, { clearPersisted = true } = {}) {
             try { persistPendingMessages(id, buffered); } catch { /* ignore */ }
         }
     }
+    // Tombstoned close: the claim copies are dead (the session record itself
+    // now refuses any restore). Detach keeps them so a still-in-flight ask can
+    // release and restore its claimed entries after this cleanup runs.
+    if (clearPersisted) {
+        try { _claimedPendingMessages.delete(id); } catch { /* ignore */ }
+    }
     try { _sessionPendingMessages.delete(id); } catch { /* ignore */ }
     try { _hydratedPendingMessages.delete(id); } catch { /* ignore */ }
     try { _pendingPersistBuffers.delete(id); } catch { /* ignore */ }
@@ -924,12 +1311,86 @@ export function _dropPendingMessageState(id, { clearPersisted = true } = {}) {
     }
 }
 
+// TEST-ONLY determinism seam (never called by runtime code): flush this
+// module's own buffered persist batch and await its own async spool tails
+// (persist / ack / clear) and in-flight hydrations, so tests can wait for the
+// asynchronous persistence tail instead of guessing a sleep duration. Tails
+// can re-chain (an ack chains behind a persist), so this loops until the maps
+// are empty, bounded by timeoutMs — a genuinely stuck write still fails loudly
+// instead of hanging forever.
+export async function _settlePendingMessageWrites({ timeoutMs = 5000 } = {}) {
+    const budget = Math.max(0, Number(timeoutMs) || 0);
+    const deadline = Date.now() + budget;
+    const expired = () => new Error(
+        `pending-message writes did not settle within ${budget}ms `
+        + `(tails=${_pendingPersistTails.size}, hydrations=${_pendingHydrations.size}, buffers=${_pendingPersistBuffers.size})`,
+    );
+    let timer = null;
+    const expiry = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => reject(expired()), Math.max(0, deadline - Date.now()));
+    });
+    // Deliberately NOT unref'd: while settle waits on a stuck tail there may be
+    // no other handle, and an unref'd timer would let the loop drain (the wait
+    // would then never reject). The finally below always clears it.
+    expiry.catch(() => {});
+    try {
+        for (;;) {
+            // Snapshot BEFORE flushing: persistPendingMessages OVERWRITES this
+            // session's map entry with the new operation, so an in-flight tail
+            // captured only after the flush would be silently dropped and never
+            // awaited. Both the pre-flush tails and the newly chained ones must
+            // be awaited, and the loop repeats because acks/clears chain further
+            // tails behind them.
+            const preFlushTails = [..._pendingPersistTails.values()];
+            flushPendingMessagePersistsSync();
+            const inflight = new Set([
+                ...preFlushTails,
+                ..._pendingPersistTails.values(),
+                ..._pendingHydrations.values(),
+            ]);
+            if (inflight.size === 0 && _pendingPersistBuffers.size === 0) return;
+            if (Date.now() > deadline) throw expired();
+            // Raced against the absolute deadline: a never-settling tail must
+            // reject within timeoutMs instead of blocking here forever.
+            await Promise.race([Promise.allSettled([...inflight]), expiry]);
+            await Promise.race([new Promise((resolve) => setImmediate(resolve)), expiry]);
+        }
+    } finally {
+        if (timer) clearTimeout(timer);
+    }
+}
+
+// TEST-ONLY: publish a promise as this session's persist tail using the same
+// set + self-removal pattern as persistPendingMessages, so settle tests can
+// model a slow (or stuck) spool write deterministically. Runtime code never
+// calls this and behavior is unchanged.
+export function _setPendingPersistTailForTest(sessionId, promise) {
+    const operation = Promise.resolve(promise).catch(() => {});
+    _pendingPersistTails.set(sessionId, operation);
+    operation.finally(() => {
+        if (_pendingPersistTails.get(sessionId) === operation) _pendingPersistTails.delete(sessionId);
+    }).catch(() => {});
+    return operation;
+}
+
+// TEST-ONLY: deterministic seams for the check→mutate windows. Runtime code
+// never installs hooks; behavior is unchanged when none are set.
+//   'enqueue:beforePublish'    — between the lifecycle check and the memory /
+//                                spool publish of a local/remote enqueue
+//   'persist:beforeCommit'     — inside the spool lock, before the durable
+//                                commit re-validates each entry's token
+//   'ack:beforeCommit'         — inside the spool lock, before the ack
+//                                re-validates its epoch and deletes rows
+//   'hydrate:betweenCleanups'  — after each acknowledgement/prune await
+export function _setPendingTestHooks(hooks) {
+    _pendingTestHooks = hooks && typeof hooks === 'object' ? hooks : null;
+    return _pendingTestHooks;
+}
+
 setImmediate(() => {
-    // Spool hygiene belongs to the lead/daemon process. Worker-mode children
-    // (channels worker, agent shard) share the same spool file; a boot-time
-    // SYNC sweep in every child contends on the cross-process lock against
-    // the lead's writes (observed in the first live shard run: "atomic lock
-    // timeout after 8000ms: session-pending-messages.json.lock") for zero
+    // Spool hygiene belongs to the lead/daemon process. Channel workers share
+    // the same spool file; a boot-time SYNC sweep in every child contends on
+    // the cross-process lock against the lead's writes for zero
     // benefit — the lead already sweeps.
     if (process.env.MIXDOG_WORKER_MODE === '1') return;
     try { sweepOrphanedPendingMessages(); } catch { /* ignore */ }