mixdog 0.9.19 → 0.9.21

package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs

@@ -58,6 +58,50 @@ function addCompactUsageToSession(session, usage) {
     session.totalUncachedInputTokens = (session.totalUncachedInputTokens || 0) + uncachedInputTokens;
     session.tokensCumulative = (session.tokensCumulative || 0) + inputTokens + outputTokens;
 }
+// A dead/unreachable memory runtime makes each proxy RPC wait ~30s (waitForPort)
+// and retry once (memory-runtime-proxy.mjs) — that must NEVER wedge the
+// compact//clear path. Bound every recall-fasttrack memory call with a short
+// local timeout: on timeout we abort (best-effort cancel via a chained signal)
+// and treat it exactly like an RPC failure, so compaction proceeds WITHOUT
+// recall-fasttrack instead of hanging.
+const RECALL_MEMORY_CALL_TIMEOUT_MS = Math.max(
+    250,
+    Number(process.env.MIXDOG_AGENT_COMPACT_RECALL_TIMEOUT_MS) || 4000,
+);
+function recallMemoryTimeoutMs(session) {
+    const configured = positiveContextWindow(session?.compaction?.recallMemoryTimeoutMs);
+    // Clamp ALL sources (session-config included) to the 250ms floor so a
+    // misconfigured tiny value can't turn the bound into a busy no-wait.
+    return Math.max(250, configured || RECALL_MEMORY_CALL_TIMEOUT_MS);
+}
+async function callMemoryBounded(args, callerCtx, timeoutMs) {
+    const ac = new AbortController();
+    const outer = callerCtx?.signal;
+    const onOuterAbort = () => { try { ac.abort(); } catch {} };
+    if (outer) {
+        if (outer.aborted) ac.abort();
+        else { try { outer.addEventListener?.('abort', onOuterAbort, { once: true }); } catch {} }
+    }
+    let timer = null;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => {
+            try { ac.abort(); } catch {}
+            reject(new Error(`memory ${args?.action || 'call'} timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+        try { timer.unref?.(); } catch {}
+    });
+    try {
+        return await Promise.race([
+            executeInternalTool('memory', args, { ...callerCtx, signal: ac.signal }),
+            timeout,
+        ]);
+    } finally {
+        if (timer) clearTimeout(timer);
+        // Drop the chained-abort listener when the call settles first, so a
+        // later outer abort can't fire into a dead controller / leak.
+        try { outer?.removeEventListener?.('abort', onOuterAbort); } catch {}
+    }
+}
 async function runRecallFastTrackForSession(session, messages, opts = {}) {
     const sessionId = opts.sessionId || session?.id || null;
     if (!sessionId) throw new Error('recall-fasttrack requires a session id');
@@ -72,16 +116,22 @@ async function runRecallFastTrackForSession(session, messages, opts = {}) {
     };
     const hydrateLimit = positiveContextWindow(session?.compaction?.recallIngestLimit)
         || Math.max(500, Math.min(5000, messages.length || 0));
+    const memoryTimeoutMs = recallMemoryTimeoutMs(session);
     try {
-        await executeInternalTool('memory', {
+        await callMemoryBounded({
             action: 'ingest_session',
             sessionId,
             messages,
             cwd: session?.cwd,
             limit: hydrateLimit,
-        }, callerCtx);
+        }, callerCtx, memoryTimeoutMs);
     } catch (err) {
-        try { process.stderr.write(`[session] recall-fasttrack ingest skipped (sess=${sessionId}): ${err?.message || err}\n`); } catch {}
+        // Ingest failed (dead/timed-out memory runtime). The transcript is NOT
+        // in memory, so recall-fasttrack MUST NOT proceed with a false "history
+        // is in memory" digest — that would drop un-ingested head history.
+        // Throw so the caller falls back to the normal compaction path.
+        try { process.stderr.write(`[session] recall-fasttrack ingest failed — bailing (sess=${sessionId}): ${err?.message || err}\n`); } catch {}
+        throw new Error(`recall-fasttrack ingest failed: ${err?.message || err}`);
     }
     // Digest injection (mirrors loop/recall-fasttrack.mjs): no dump + cycle1
     // drain — that ran memory-pipeline LLM chunking inside the compaction.
@@ -89,24 +139,25 @@ async function runRecallFastTrackForSession(session, messages, opts = {}) {
     // chunks it on its own schedule and recall serves anything beyond the
     // digest.
     let recallText = '';
-    let cycle1Error = null;
     try {
-        const browsed = await executeInternalTool('memory', {
+        const browsed = await callMemoryBounded({
             action: 'search',
             sessionId,
             limit: positiveContextWindow(session?.compaction?.recallDigestLimit) || 30,
             includeMembers: true,
             includeRaw: true,
-        }, callerCtx);
+        }, callerCtx, memoryTimeoutMs);
         recallText = typeof browsed === 'string' ? browsed : String(browsed?.text ?? browsed ?? '');
     } catch (err) {
-        cycle1Error = err?.message || String(err);
-        try { process.stderr.write(`[session] recall-digest browse failed (sess=${sessionId}): ${err?.message || err}\n`); } catch {}
+        // Search failed (dead/timed-out memory runtime). Same hazard as a failed
+        // ingest: without a real recall dump we can't safely replace head
+        // history — bail out to the normal compaction path.
+        try { process.stderr.write(`[session] recall-digest browse failed — bailing (sess=${sessionId}): ${err?.message || err}\n`); } catch {}
+        throw new Error(`recall-fasttrack search failed: ${err?.message || err}`);
     }
     return {
         query,
         querySha,
-        cycle1Error,
         recallText: [
             `session_id=${sessionId}`,
             `Full history is in memory — use the recall tool for details beyond this digest.`,
@@ -221,9 +272,6 @@ export async function runSessionCompaction(session, opts = {}) {
             if (Array.isArray(recallFastTrackResult?.messages)) {
                 compacted = recallFastTrackResult.messages;
             }
-            if (recallPayload.cycle1Error) {
-                recallFastTrackError = new Error(String(recallPayload.cycle1Error));
-            }
         } catch (err) {
             recallFastTrackError = err;
             compactError = err;

package/src/runtime/agent/orchestrator/session/manager/env-utils.mjs

@@ -0,0 +1,8 @@
+// manager/env-utils.mjs
+// Shared non-negative integer env parser. Extracted verbatim from manager.mjs
+// so ask-session (terminal save timeout) and idle-cleanup (interval constants)
+// read the identical contract without importing manager.mjs back.
+export function nonNegativeIntEnv(name, fallback) {
+    const value = Number(process.env[name]);
+    return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
+}

package/src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs

@@ -0,0 +1,159 @@
+// manager/idle-cleanup.mjs
+// Periodic idle-session + tombstone sweep extracted verbatim from manager.mjs.
+// Drives sweepStaleSessions on an unref'd interval; closeSession is imported
+// from session-close.mjs (one-way dependency, no cycle).
+import { sweepStaleSessions } from '../store.mjs';
+import { sweepOrphanedPendingMessages } from './pending-messages.mjs';
+import { _getRuntimeEntry, _clearSessionRuntime, _runtimeEntries } from './runtime-liveness.mjs';
+import { _closeBashSessionLazy } from './runtime-loaders.mjs';
+import { closeSession } from './session-close.mjs';
+import { nonNegativeIntEnv } from './env-utils.mjs';
+
+// --- Periodic idle session cleanup ---
+const CLEANUP_INTERVAL_MS = nonNegativeIntEnv('MIXDOG_SESSION_CLEANUP_INTERVAL_MS', 5 * 60 * 1000); // check every 5 minutes
+const CLEANUP_INITIAL_DELAY_MS = nonNegativeIntEnv('MIXDOG_SESSION_CLEANUP_INITIAL_DELAY_MS', CLEANUP_INTERVAL_MS > 0 ? CLEANUP_INTERVAL_MS : 0);
+const CLEANUP_SLOW_LOG_MS = nonNegativeIntEnv('MIXDOG_SESSION_CLEANUP_SLOW_LOG_MS', 250);
+const TOMBSTONE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24h — far longer than any realistic ask race window
+let _cleanupTimer = null;
+let _cleanupInitialTimer = null;
+
+function _previewIds(items, limit = 5) {
+    const ids = (items || []).slice(0, limit).map((item) => item.id).filter(Boolean);
+    if (ids.length === 0) return '';
+    const more = items.length > limit ? `, +${items.length - limit} more` : '';
+    return ` (${ids.join(', ')}${more})`;
+}
+
+function sweepIdleSessions({ includeTombstones = true } = {}) {
+    const startedAt = Date.now();
+    try {
+        const result = sweepStaleSessions({
+            tombstoneMaxAgeMs: includeTombstones ? TOMBSTONE_MAX_AGE_MS : 0,
+        });
+        const {
+            cleaned,
+            remaining,
+            details,
+            tombstonesCleaned = 0,
+            tombstoneDetails = [],
+            tombstoneErrors = [],
+        } = result;
+        if (cleaned > 0) {
+            for (const d of details) {
+                // Skip entries with an active in-flight controller — aborting
+                // them via closeSession() is the safe path; clearing the runtime
+                // without signalling the controller leaves orphan provider work.
+                const rtEntry = _getRuntimeEntry(d.id);
+                if (rtEntry && rtEntry.controller && !rtEntry.controller.signal?.aborted) {
+                    try { closeSession(d.id, 'idle-sweep'); } catch { /* ignore */ }
+                } else {
+                    _clearSessionRuntime(d.id);
+                    if (d.bashSessionId) {
+                        try { _closeBashSessionLazy(d.bashSessionId, `idle-sweep:${d.id}`); } catch { /* ignore */ }
+                    }
+                }
+                process.stderr.write(`[agent-session] idle cleanup: closed ${d.id} (idle ${d.idleMinutes}m, owner=${d.owner})\n`);
+            }
+            process.stderr.write(`[agent-session] idle sweep: cleaned ${cleaned} session(s), ${remaining} remaining\n`);
+        }
+        if (tombstonesCleaned > 0) {
+            for (const d of tombstoneDetails) {
+                if (d?.id) _clearSessionRuntime(d.id);
+            }
+            process.stderr.write(`[session-sweep] unlinked ${tombstonesCleaned} tombstone(s)${_previewIds(tombstoneDetails)}\n`);
+        }
+        if (tombstoneErrors.length > 0) {
+            const first = tombstoneErrors[0];
+            process.stderr.write(`[session-sweep] tombstone unlink failed for ${tombstoneErrors.length} session(s): ${first?.id || 'unknown'} ${first?.message || ''}\n`);
+        }
+        const elapsed = Date.now() - startedAt;
+        if (elapsed >= CLEANUP_SLOW_LOG_MS) {
+            process.stderr.write(`[session-sweep] cleanup took ${elapsed}ms (idle=${cleaned}, tombstones=${tombstonesCleaned}, remaining=${remaining})\n`);
+        }
+    } catch (e) {
+        process.stderr.write(`[agent-session] idle sweep error: ${e && e.message || e}\n`);
+    }
+}
+
+/**
+ * Unlink tombstone session files (closed=true) older than TOMBSTONE_MAX_AGE_MS.
+ *
+ * Rationale: closeSession() leaves the tombstone on disk as the authoritative
+ * resurrection-blocker for racing saveSession() calls. That race resolves in
+ * microseconds (the window inside _doSave between temp write and rename), so
+ * 24h is vastly safe. After the TTL expires we reclaim the disk slot.
+ *
+ * Uses `getStoredSessionsRaw()` rather than `listStoredSessions()` because the
+ * latter's inline 30-min idle cleanup would race-unlink tombstones before we
+ * get to log them — we want to own the unlink decision and stderr line here.
+ */
+export function sweepTombstones() {
+    try {
+        const { tombstonesCleaned = 0, tombstoneDetails = [], tombstoneErrors = [] } = sweepStaleSessions({
+            sweepIdle: false,
+            tombstoneMaxAgeMs: TOMBSTONE_MAX_AGE_MS,
+        });
+        for (const d of tombstoneDetails) {
+            if (d?.id) _clearSessionRuntime(d.id);
+        }
+        if (tombstonesCleaned > 0) {
+            process.stderr.write(`[session-sweep] unlinked ${tombstonesCleaned} tombstone(s)${_previewIds(tombstoneDetails)}\n`);
+        }
+        if (tombstoneErrors.length > 0) {
+            const first = tombstoneErrors[0];
+            process.stderr.write(`[session-sweep] tombstone unlink failed for ${tombstoneErrors.length} session(s): ${first?.id || 'unknown'} ${first?.message || ''}\n`);
+        }
+        return tombstonesCleaned;
+    } catch (e) {
+        process.stderr.write(`[session-sweep] tombstone sweep error: ${e && e.message || e}\n`);
+        return 0;
+    }
+}
+
+function hasActiveRuntimeWork() {
+    for (const [, entry] of _runtimeEntries()) {
+        if (!entry || entry.closed === true) continue;
+        if (entry.controller && !entry.controller.signal?.aborted) return true;
+        if (['connecting', 'requesting', 'streaming', 'tool_running', 'cancelling'].includes(entry.stage)) return true;
+    }
+    return false;
+}
+
+function _runCleanupCycle() {
+    if (hasActiveRuntimeWork()) return;
+    sweepOrphanedPendingMessages();
+    sweepIdleSessions({ includeTombstones: true });
+}
+
+function _startCleanupInterval() {
+    if (_cleanupTimer) return;
+    if (CLEANUP_INTERVAL_MS <= 0) return;
+    _cleanupTimer = setInterval(_runCleanupCycle, CLEANUP_INTERVAL_MS);
+    if (_cleanupTimer.unref) _cleanupTimer.unref(); // don't block process exit
+}
+
+export function startIdleCleanup() {
+    if (_cleanupTimer || _cleanupInitialTimer) return;
+    if (CLEANUP_INITIAL_DELAY_MS <= 0) {
+        _runCleanupCycle();
+        _startCleanupInterval();
+        return;
+    }
+    _cleanupInitialTimer = setTimeout(() => {
+        _cleanupInitialTimer = null;
+        _runCleanupCycle();
+        _startCleanupInterval();
+    }, CLEANUP_INITIAL_DELAY_MS);
+    if (_cleanupInitialTimer.unref) _cleanupInitialTimer.unref();
+}
+
+export function stopIdleCleanup() {
+    if (_cleanupInitialTimer) {
+        clearTimeout(_cleanupInitialTimer);
+        _cleanupInitialTimer = null;
+    }
+    if (_cleanupTimer) {
+        clearInterval(_cleanupTimer);
+        _cleanupTimer = null;
+    }
+}

package/src/runtime/agent/orchestrator/session/manager/message-sanitize.mjs

@@ -0,0 +1,143 @@
+// manager/message-sanitize.mjs
+// Message sanitization + compaction-failure persistence helpers extracted
+// verbatim from manager.mjs. sanitizeSessionMessagesForModel drops internal
+// runtime-notification turns and cancelled-assistant stubs while keeping image
+// content intact (reference-agent parity); the compact-failure helpers persist
+// a compacted outgoing transcript when an ask throws mid-turn.
+import { promptContentText, isInternalRuntimeNotificationText } from './prompt-utils.mjs';
+import { saveSessionAsync } from '../store.mjs';
+import { _getRuntimeEntry } from './runtime-liveness.mjs';
+
+function isInternalCancelledAssistantMessage(message) {
+    if (!message || message.role !== 'assistant') return false;
+    if (message.cancelled === true) return true;
+    const text = promptContentText(message.content).trim();
+    return /^\[cancelled\]\s+This turn was interrupted before completion\./i.test(text)
+        || /Preserve the user request above as the active task context/i.test(text);
+}
+
+export function sanitizeSessionMessagesForModel(messages) {
+    // Drop internal runtime-notification turns and cancelled-assistant stubs so
+    // they never reach the model, but KEEP image content intact. Reference-agent
+    // parity: the live transcript and every model request retain attached
+    // images across turns; only the compaction-summary call strips them. The
+    // disk-stored session JSON replaces image bytes with a text placeholder at
+    // serialization time (see store.mjs), so this no longer touches images.
+    if (!Array.isArray(messages) || messages.length === 0) return [];
+    const out = [];
+    let droppingInternalTurn = false;
+    for (const message of messages) {
+        if (isInternalCancelledAssistantMessage(message)) {
+            droppingInternalTurn = false;
+            continue;
+        }
+        if (message?.role === 'user' && isInternalRuntimeNotificationText(message.content)) {
+            droppingInternalTurn = true;
+            continue;
+        }
+        if (droppingInternalTurn) {
+            if (message?.role === 'user') {
+                droppingInternalTurn = false;
+            } else {
+                continue;
+            }
+        }
+        out.push(message);
+    }
+    return out;
+}
+
+function sessionMessagesSnapshotChanged(before, after) {
+    if (!Array.isArray(before) || !Array.isArray(after)) return before !== after;
+    if (before.length !== after.length) return true;
+    for (let i = 0; i < before.length; i += 1) {
+        if (before[i] !== after[i]) return true;
+        try {
+            if (JSON.stringify(before[i]) !== JSON.stringify(after[i])) return true;
+        } catch {
+            return true;
+        }
+    }
+    return false;
+}
+
+function isCompactedOutgoingFinalAssistantMessage(message) {
+    if (!message || message.role !== 'assistant') return false;
+    if (message.emptyFinal === true) return true;
+    return true;
+}
+
+function sessionMessagesAdvancedBeyondCompactedOutgoing(currentSanitized, compactedSanitized) {
+    if (!Array.isArray(currentSanitized) || !Array.isArray(compactedSanitized)) return false;
+    if (currentSanitized.length !== compactedSanitized.length + 1) return false;
+    const prefix = currentSanitized.slice(0, compactedSanitized.length);
+    if (sessionMessagesSnapshotChanged(compactedSanitized, prefix)) return false;
+    return isCompactedOutgoingFinalAssistantMessage(currentSanitized[currentSanitized.length - 1]);
+}
+
+export const _sessionMessagesAdvancedBeyondCompactedOutgoing = sessionMessagesAdvancedBeyondCompactedOutgoing;
+
+function applyCompactFailurePersistToSession(activeSession, {
+    priorSanitized,
+    sanitized,
+    messagesAdvanced,
+    error = null,
+}) {
+    if (!messagesAdvanced && !sessionMessagesSnapshotChanged(priorSanitized, sanitized)) return false;
+    if (!messagesAdvanced) {
+        activeSession.messages = sanitized;
+        activeSession.providerState = undefined;
+    }
+    activeSession.updatedAt = Date.now();
+    activeSession.lastUsedAt = Date.now();
+    if (activeSession.compaction && typeof activeSession.compaction === 'object'
+        && (activeSession.compaction.lastStage === 'compacting'
+            || activeSession.compaction.lastStage === 'overflow_failed')) {
+        const prev = activeSession.compaction;
+        const cause = error?.cause;
+        const overflow = error?.code === 'AGENT_CONTEXT_OVERFLOW';
+        activeSession.compaction = {
+            ...prev,
+            lastStage: prev.lastStage === 'overflow_failed'
+                ? 'overflow_failed'
+                : (overflow ? 'overflow_failed' : 'failed'),
+            lastCheckedAt: Date.now(),
+            lastError: prev.lastError || cause?.message || error?.message || null,
+            lastSemanticError: prev.lastSemanticError || cause?.message || null,
+            lastRecallFastTrackError: prev.lastRecallFastTrackError
+                || (cause?.message && String(cause?.name || '').includes('Recall') ? cause.message : null),
+        };
+    }
+    return true;
+}
+
+export const _applyCompactFailurePersistToSession = applyCompactFailurePersistToSession;
+
+export async function persistCompactedOutgoingAfterAskFailure({
+    sessionId,
+    activeSession,
+    askGeneration,
+    turnOutgoing,
+    error = null,
+}) {
+    if (!activeSession || activeSession.closed === true) return;
+    if (!Array.isArray(turnOutgoing) || turnOutgoing.length === 0) return;
+    const currentRuntime = _getRuntimeEntry(sessionId);
+    if (currentRuntime?.closed || currentRuntime?.generation !== askGeneration) return;
+    const sanitized = sanitizeSessionMessagesForModel(turnOutgoing);
+    const priorSanitized = sanitizeSessionMessagesForModel(
+        Array.isArray(activeSession.messages) ? activeSession.messages : [],
+    );
+    const messagesAdvanced = sessionMessagesAdvancedBeyondCompactedOutgoing(priorSanitized, sanitized);
+    const applied = applyCompactFailurePersistToSession(activeSession, {
+        priorSanitized,
+        sanitized,
+        messagesAdvanced,
+        error,
+    });
+    if (!applied) return;
+    try {
+        await saveSessionAsync(activeSession, { expectedGeneration: askGeneration });
+    } catch { /* best-effort: preserve in-memory compaction even if disk is slow */ }
+    if (currentRuntime) currentRuntime.session = activeSession;
+}