mixdog 0.9.3 → 0.9.4

package/src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs

@@ -0,0 +1,406 @@
+// ── Runtime liveness map ──────────────────────────────────────────────
+// In-memory only. Tracks per-session stage + stream heartbeat so agent type=list
+// can surface whether a session is actually alive vs stuck. Never persisted —
+// heartbeats would otherwise churn the session JSON on every SSE delta.
+//
+// Extracted from manager.mjs (pass-3). The Map and its timer companion are
+// module-level singletons — preserving the baseline's single-process shape.
+// manager.mjs owns askSession's controller/generation lifecycle and calls the
+// accessors below via imports; a small set of accessors that need the session
+// store / provider registry are injected through configureRuntimeLiveness() to
+// avoid a circular import back into manager.mjs.
+//
+// Entry shape: {
+//   stage, lastStreamDeltaAt, lastToolCall, lastError, updatedAt,
+//   controller?: AbortController,  // set while an ask is in flight
+//   generation?: number,            // snapshot taken at ask start
+//   closed?: boolean,               // flipped by closeSession()
+// }
+import { createAbortController } from '../../../../shared/abort-controller.mjs';
+import { publishHeartbeat, deleteHeartbeat } from '../store.mjs';
+import { DEFAULT_ACTIVITY_HEARTBEAT_MS } from '../../stall-policy.mjs';
+import {
+    configureUsageMetricsRuntime,
+    dropMetricSeenState,
+    bumpUsageMetricsTurnId,
+} from './usage-metrics.mjs';
+
+const HEARTBEAT_THROTTLE_MS = 60_000; // 60s
+
+const _runtimeState = new Map();
+const _toolActivityHeartbeats = new Map();
+const VALID_STAGES = new Set([
+    'connecting', 'requesting', 'streaming', 'tool_running', 'idle', 'error', 'done', 'cancelling',
+]);
+
+// Injected deps that would otherwise pull manager.mjs's store/provider surface
+// back into this module (circular). Wired once from manager.mjs at load time.
+let _deps = {
+    loadSession: () => null,
+    saveSessionAsync: async () => {},
+};
+export function configureRuntimeLiveness(deps = {}) {
+    _deps = { ..._deps, ...deps };
+}
+
+export function _touchRuntime(id) {
+    let entry = _runtimeState.get(id);
+    if (!entry) {
+        entry = { stage: 'idle', lastStreamDeltaAt: null, lastToolCall: null, lastError: null, updatedAt: Date.now() };
+        _runtimeState.set(id, entry);
+    }
+    return entry;
+}
+
+export function _stopToolActivityHeartbeat(id) {
+    if (!id) return;
+    const timer = _toolActivityHeartbeats.get(id);
+    if (!timer) return;
+    try { clearInterval(timer); } catch { /* ignore */ }
+    _toolActivityHeartbeats.delete(id);
+}
+
+function _touchSessionActivityProgress(id) {
+    const entry = _runtimeState.get(id);
+    if (!entry || entry.closed || entry.controller?.signal?.aborted) return;
+    if (entry.stage !== 'tool_running') return;
+    const now = Date.now();
+    entry.lastProgressAt = now;
+    entry.updatedAt = now;
+    publishHeartbeat(id, now);
+}
+
+function _startToolActivityHeartbeat(id) {
+    _stopToolActivityHeartbeat(id);
+    if (!(DEFAULT_ACTIVITY_HEARTBEAT_MS > 0)) return;
+    const timer = setInterval(() => _touchSessionActivityProgress(id), DEFAULT_ACTIVITY_HEARTBEAT_MS);
+    if (typeof timer.unref === 'function') timer.unref();
+    _toolActivityHeartbeats.set(id, timer);
+}
+
+export function updateSessionStage(id, stage) {
+    if (!id || !VALID_STAGES.has(stage)) return;
+    const entry = _touchRuntime(id);
+    const now = Date.now();
+    entry.stage = stage;
+    if (stage === 'connecting' || stage === 'requesting') {
+        entry.modelRequestStartedAt = now;
+    }
+    entry.lastProgressAt = now;
+    entry.updatedAt = now;
+    if (stage !== 'tool_running') _stopToolActivityHeartbeat(id);
+}
+
+/**
+ * Reset heartbeat-visible fields for a new ask. Preserves controller/generation/
+ * closed (lifecycle) but clears the previous run's streaming state so stale
+ * lastToolCall / lastStreamDeltaAt from the previous ask don't leak into the
+ * new one.
+ */
+export function markSessionAskStart(id) {
+    if (!id) return;
+    _stopToolActivityHeartbeat(id);
+    const entry = _touchRuntime(id);
+    entry.usageMetricsTurnIncremental = false;
+    const sessionForTurn = entry.session ?? _deps.loadSession(id);
+    if (sessionForTurn) bumpUsageMetricsTurnId(sessionForTurn);
+    entry.stage = 'connecting';
+    entry.lastStreamDeltaAt = null;
+    entry.lastToolCall = null;
+    entry.toolStartedAt = null;
+    entry.lastError = null;
+    // A new ask starts a fresh turn lifecycle — clear any stale empty-final
+    // classification from the prior turn so inspectBridgeEntry doesn't keep
+    // short-circuiting to 'empty-synthesis' (which would disable stall
+    // detection for the entire new turn).
+    entry.emptyFinal = false;
+    entry.emptyFinalAt = null;
+    // askStartedAt is the watchdog's fallback reference when a session
+    // hangs before any stream delta arrives. Without it, a provider that
+    // never returns a first token would stall forever because the watchdog
+    // keys solely on lastStreamDeltaAt.
+    const now = Date.now();
+    entry.askStartedAt = now;
+    entry.modelRequestStartedAt = now;
+    entry.lastProgressAt = now;
+    entry.updatedAt = now;
+    // Publish heartbeat immediately so the status aggregator picks the
+    // session up in the connecting / requesting window. Without this the
+    // .hb file only landed on the first stream chunk — producing a 3–10s
+    // (xhigh: 30s+) invisible gap where agent sessions ran but the CC
+    // statusline showed no maintenance/agent badge. STREAM_FRESH_MS (5 min)
+    // still drops a session whose provider truly never returns a chunk;
+    // markSessionStreamDelta keeps refreshing once chunks arrive.
+    publishHeartbeat(id, now);
+}
+export async function markSessionStreamDelta(id) {
+    if (!id) return;
+    // Non-creating lookup: a live ask ALWAYS has a runtime entry (markSessionAskStart
+    // creates it before streaming begins). _touchRuntime would instead resurrect a
+    // blank entry — and closeSession()/idle-sweep clear _runtimeState on a deferred
+    // tick while a detached provider stream may still be trickling deltas. A delta
+    // arriving after that clear must NOT re-create an entry or it would republish the
+    // .hb heartbeat that markSessionClosed deleted, orphaning a dead session's
+    // heartbeat indefinitely (the disk tombstone blocks ask resumption but not this
+    // path). Skip a missing, tombstoned, or aborted entry — never refresh liveness.
+    const entry = _runtimeState.get(id);
+    if (!entry || entry.closed || entry.controller?.signal?.aborted) return;
+    _stopToolActivityHeartbeat(id);
+    const now = Date.now();
+    entry.lastStreamDeltaAt = now;
+    entry.lastProgressAt = now;
+    // Only promote to 'streaming' if we were in a pre-stream stage; never downgrade
+    // mid-tool (tool_running has its own delta source if the tool streams back).
+    if (entry.stage === 'connecting' || entry.stage === 'requesting') {
+        entry.stage = 'streaming';
+    }
+    // Lightweight heartbeat (≤5s self-throttled) for the status aggregator.
+    // Disk-side session.lastHeartbeatAt below is the heavy 60s zombie-reaper
+    // signal; the .hb file is the fast fresh-session signal consumed by the
+    // status line.
+    publishHeartbeat(id, now);
+    const session = entry.session;
+    if (session && now - (session.lastHeartbeatAt || 0) > HEARTBEAT_THROTTLE_MS) {
+        session.lastHeartbeatAt = now;
+        await _deps.saveSessionAsync(session, { expectedGeneration: session.generation });
+    }
+    entry.updatedAt = now;
+}
+export function markSessionToolCall(id, toolName) {
+    if (!id) return;
+    const entry = _touchRuntime(id);
+    entry.stage = 'tool_running';
+    entry.lastToolCall = toolName || null;
+    entry.toolStartedAt = Date.now();
+    entry.lastProgressAt = entry.toolStartedAt;
+    entry.updatedAt = entry.toolStartedAt;
+    publishHeartbeat(id, entry.toolStartedAt);
+    _startToolActivityHeartbeat(id);
+}
+// Parent AbortSignal listeners are dropped on askSession unwind (finally /
+// terminal return) and on error/cancel/close — not in markSessionDone, which
+// also runs between queued follow-up turns within one ask.
+export function markSessionDone(id, { empty = false } = {}) {
+    if (!id) return;
+    _stopToolActivityHeartbeat(id);
+    const entry = _touchRuntime(id);
+    entry.stage = 'done';
+    entry.lastError = null;
+    entry.askStartedAt = null;
+    entry.toolStartedAt = null;
+    // Non-empty completion: drop any stale empty-final flag so a subsequent
+    // ask on the same reusable runtime entry starts clean. Empty-final
+    // completions preserve the flag (set by markSessionEmptyFinal just prior).
+    if (!empty) {
+        entry.emptyFinal = false;
+        entry.emptyFinalAt = null;
+    }
+    const doneTs = Date.now();
+    entry.doneAt = doneTs;
+    entry.lastProgressAt = doneTs;
+    entry.updatedAt = doneTs;
+    // Terminal stage — drop the heartbeat so the status badge releases
+    // immediately. A subsequent ask on the same session re-publishes via
+    // markSessionStreamDelta on the first chunk.
+    deleteHeartbeat(id);
+}
+// Tag a session as having completed with empty final synthesis (no
+// content/reasoning). Distinct from `markSessionDone`: still a success
+// (no abort), but the stall watchdog and post-mortem tools can
+// distinguish "finished empty" from "finished with content" without
+// mistaking the silence for a stall.
+export function markSessionEmptyFinal(id) {
+    if (!id) return;
+    const entry = _touchRuntime(id);
+    entry.emptyFinal = true;
+    entry.emptyFinalAt = Date.now();
+}
+export function markSessionError(id, msg) {
+    if (!id) return;
+    _stopToolActivityHeartbeat(id);
+    const entry = _touchRuntime(id);
+    entry.stage = 'error';
+    entry.lastError = msg ? String(msg).slice(0, 200) : null;
+    entry.askStartedAt = null;
+    entry.toolStartedAt = null;
+    // Error path is a non-empty completion (we have an error message, not a
+    // silent empty final). Clear the flag so the next ask starts clean.
+    entry.emptyFinal = false;
+    entry.emptyFinalAt = null;
+    const errTs = Date.now();
+    entry.doneAt = errTs;
+    entry.lastProgressAt = errTs;
+    entry.updatedAt = errTs;
+    deleteHeartbeat(id);
+    _unlinkParentAbortListener(entry);
+}
+export function markSessionCancelled(id) {
+    if (!id) return;
+    _stopToolActivityHeartbeat(id);
+    const entry = _touchRuntime(id);
+    entry.stage = 'done';
+    entry.lastError = null;
+    entry.askStartedAt = null;
+    entry.toolStartedAt = null;
+    entry.emptyFinal = false;
+    entry.emptyFinalAt = null;
+    const doneTs = Date.now();
+    entry.doneAt = doneTs;
+    entry.lastProgressAt = doneTs;
+    entry.updatedAt = doneTs;
+    deleteHeartbeat(id);
+    _unlinkParentAbortListener(entry);
+}
+export function getSessionRuntime(id) {
+    return id ? (_runtimeState.get(id) || null) : null;
+}
+
+const _COMPACTION_BLOCKED_STAGES = new Set([
+    'connecting', 'requesting', 'streaming', 'tool_running', 'cancelling',
+]);
+
+export function isSessionCompactionBlocked(sessionId) {
+    if (!sessionId) return false;
+    const entry = _runtimeState.get(sessionId);
+    if (!entry || entry.closed === true) return false;
+    if (entry.controller && !entry.controller.signal?.aborted) return true;
+    return _COMPACTION_BLOCKED_STAGES.has(entry.stage);
+}
+
+export function getSessionProgressSnapshot(sessionId) {
+    const entry = _runtimeState.get(sessionId);
+    if (!entry) return null;
+    const askStartedAt = entry.askStartedAt || 0;
+    const modelRequestStartedAt = entry.modelRequestStartedAt || askStartedAt;
+    const firstActivityAt = Math.max(
+        entry.lastStreamDeltaAt || 0,
+        entry.toolStartedAt || 0,
+    );
+    const stage = entry.stage || 'idle';
+    const waitingForFirstActivity = Boolean(
+        modelRequestStartedAt
+        && (stage === 'connecting' || stage === 'requesting')
+        && firstActivityAt <= modelRequestStartedAt
+    );
+    return {
+        stage,
+        askStartedAt,
+        modelRequestStartedAt,
+        firstActivityAt,
+        lastStreamDeltaAt: entry.lastStreamDeltaAt || 0,
+        toolStartedAt: entry.toolStartedAt || 0,
+        lastProgressAt: entry.lastProgressAt || 0,
+        updatedAt: entry.updatedAt || 0,
+        hasFirstActivity: Boolean(firstActivityAt && (!askStartedAt || firstActivityAt >= askStartedAt)),
+        waitingForFirstActivity,
+    };
+}
+
+/**
+ * Iterate all active session runtimes. Used by the stream watchdog.
+ * Returns an iterable of [sessionId, entry] pairs; consumers should
+ * treat entries as read-only snapshots and avoid mutating them.
+ */
+export function forEachSessionRuntime() {
+    return _runtimeState.entries();
+}
+
+// Wire the usage-metrics runtime accessor to this module's _runtimeState so
+// persistIterationMetrics can read the live in-memory session and flag
+// usageMetricsTurnIncremental. (Moved with the map from manager.mjs.)
+configureUsageMetricsRuntime({ getRuntimeEntry: (id) => _runtimeState.get(id) });
+
+/** Mark session hidden so listSessions() filters it out (runtime-only). */
+export function hideSessionFromList(sessionId) {
+    if (!sessionId) return;
+    const entry = _runtimeState.get(sessionId);
+    if (entry) entry.listHidden = true;
+}
+
+export function getSessionAbortSignal(sessionId) {
+    return _runtimeState.get(sessionId)?.controller?.signal ?? null;
+}
+
+/**
+ * Return the most recent "session is making progress" timestamp.
+ *
+ * Combines three independent progress signals so an idle watchdog can stay
+ * alive across both streaming and long tool calls:
+ *   - lastStreamDeltaAt: provider stream chunk landed
+ *   - toolStartedAt: a tool call just kicked off (nested tool work may
+ *     stall the outer stream for a while; this keeps the watchdog from
+ *     killing legitimate sub-agent runs)
+ *   - askStartedAt: ask just started; covers the pre-stream connect window
+ *
+ * Returns 0 when the runtime entry is unknown so callers can decide to
+ * either skip the watchdog or treat 0 as "no progress yet".
+ */
+export function getSessionLastProgressAt(sessionId) {
+    const entry = _runtimeState.get(sessionId);
+    if (!entry) return 0;
+    return Math.max(
+        entry.lastProgressAt || 0,
+        entry.lastStreamDeltaAt || 0,
+        entry.toolStartedAt || 0,
+        entry.askStartedAt || 0,
+    );
+}
+
+/**
+ * Link a parent AbortSignal to a sub-session's controller so that aborting
+ * the parent (fan-out deadline or caller ESC) tears down the agent role's
+ * provider call promptly. Safe to call after prepareAgentSession but before
+ * askSession completes. No-op if the session runtime isn't found.
+ *
+ * @param {string} sessionId — the sub-session to abort
+ * @param {AbortSignal} parentSignal — upstream signal (from fan-out coordinator)
+ */
+export function linkParentSignalToSession(sessionId, parentSignal) {
+    if (!(parentSignal instanceof AbortSignal)) return;
+    const entry = _touchRuntime(sessionId);
+    if (!entry.controller) entry.controller = createAbortController();
+    const abortReason = () => {
+        const reason = parentSignal.reason;
+        if (reason instanceof Error) return reason;
+        if (reason !== undefined && reason !== null && reason !== '') return new Error(String(reason));
+        return new Error('parent signal aborted');
+    };
+    if (parentSignal.aborted) {
+        _unlinkParentAbortListener(entry);
+        try { entry.controller.abort(abortReason()); } catch { /* ignore */ }
+        return;
+    }
+    _unlinkParentAbortListener(entry);
+    const onParentAbort = () => {
+        try { entry.controller?.abort(abortReason()); } catch { /* ignore */ }
+    };
+    parentSignal.addEventListener('abort', onParentAbort, { once: true });
+    entry.parentAbortLink = { signal: parentSignal, listener: onParentAbort };
+}
+export function _unlinkParentAbortListener(entry) {
+    const link = entry?.parentAbortLink;
+    if (!link) return;
+    try { link.signal.removeEventListener('abort', link.listener); } catch { /* ignore */ }
+    entry.parentAbortLink = null;
+}
+export function _clearSessionRuntime(id) {
+    if (id) {
+        _stopToolActivityHeartbeat(id);
+        _unlinkParentAbortListener(_runtimeState.get(id));
+        _runtimeState.delete(id);
+        // R15: also drop the per-session metric-idempotency Set; otherwise it
+        // grows O(sessions x iterations) for the whole server lifetime since
+        // nothing else deletes from _metricSeenIter on session close.
+        dropMetricSeenState(id);
+    }
+}
+
+// Direct-Map accessors for manager.mjs internals (askSession / closeSession /
+// idle-sweep) that mutate entries in place. Kept as thin exports so the Map
+// stays private to this module.
+export function _getRuntimeEntry(id) {
+    return _runtimeState.get(id);
+}
+export function _runtimeEntries() {
+    return _runtimeState.entries();
+}

package/src/runtime/agent/orchestrator/session/manager/status-telemetry.mjs

@@ -0,0 +1,80 @@
+// Standalone-status route info + gateway usage telemetry for sessions.
+// Extracted verbatim from manager.mjs (behavior-preserving). Fire-and-forget
+// statusline telemetry — must never affect the model turn.
+import { getProvider } from '../../providers/registry.mjs';
+import { fetchOAuthUsageSnapshot } from '../../providers/oauth-usage.mjs';
+import {
+    buildGatewayLimits,
+    recordGatewayUsageEvent,
+    summarizeGatewayUsage,
+} from '../../providers/statusline-route-meta.mjs';
+import { estimateTranscriptContextUsage } from '../context-utils.mjs';
+
+export function standaloneStatusRouteInfo(session) {
+    if (!session) return null;
+    // autoCompactTokenLimit is an EXPLICIT sub-boundary auto-compact limit only.
+    // Do NOT fall back to compactBoundaryTokens/contextWindow here — that
+    // labels a derived full-window value as an explicit limit, which the
+    // runtime would treat as the compaction trigger and collapse the buffer.
+    // The boundary/window stays available via contextWindow/rawContextWindow.
+    const _boundary = Number(session.compactBoundaryTokens || session.contextWindow || 0);
+    const _limit = Number(session.autoCompactTokenLimit || 0);
+    const explicitAutoCompactTokenLimit = _limit > 0 && (!_boundary || _limit < _boundary) ? _limit : null;
+    return {
+        provider: session.provider,
+        model: session.model,
+        modelDisplay: session.modelDisplay || session.displayName || session.model,
+        effort: session.effort || '',
+        fast: session.fast === true,
+        contextWindow: session.contextWindow || null,
+        rawContextWindow: session.rawContextWindow || session.contextWindow || null,
+        effectiveContextWindowPercent: session.effectiveContextWindowPercent || null,
+        autoCompactTokenLimit: explicitAutoCompactTokenLimit,
+        presetId: session.presetId || null,
+        presetName: session.presetName || null,
+    };
+}
+
+export function recordStandaloneStatusTelemetry(session, result, durationMs) {
+    if (!session || !result?.usage) return;
+    const routeInfo = standaloneStatusRouteInfo(session);
+    if (!routeInfo?.provider || !routeInfo?.model) return;
+    const providerOut = {
+        usage: result.usage,
+        model: result.model,
+        serviceTier: result.serviceTier,
+    };
+    // The transcript estimate is the SSOT for the displayed context footprint.
+    // agentLoop()'s result has no `compact` field, so build a synthetic compact
+    // arg carrying the live monotonic estimate (estimateMessagesTokens+reserve)
+    // as afterTokens. This lights up summarizeGatewayUsage's estimate-based
+    // contextUsedPct branch (provider input_tokens swing wildly / unbounded on
+    // e.g. OpenAI gpt-5.5), and lets a genuine >100% pass through.
+    const _estTokens = estimateTranscriptContextUsage(session.messages, session.tools || []);
+    const _compactArg = { ...(result.compact && typeof result.compact === 'object' ? result.compact : {}), afterTokens: _estTokens };
+    try {
+        const summary = {
+            ...summarizeGatewayUsage(routeInfo, providerOut, _compactArg, durationMs),
+            requestKind: 'chat',
+            sessionId: session.id || null,
+            toolCount: result.toolCallsTotal ?? null,
+            messageCount: Array.isArray(session.messages) ? session.messages.length : null,
+            cacheStrategy: session.providerCacheOpts?.cacheStrategy || null,
+        };
+        recordGatewayUsageEvent(summary);
+    } catch {
+        // Statusline telemetry must never affect the model turn.
+    }
+
+    const provider = getProvider(routeInfo.provider);
+    if (!provider) return;
+    fetchOAuthUsageSnapshot(routeInfo, provider, (message) => {
+        if (process.env.MIXDOG_STATUSLINE_TRACE) {
+            process.stderr.write(`[statusline] ${message}\n`);
+        }
+    })
+        .then((snapshot) => {
+            try { buildGatewayLimits(routeInfo, providerOut, snapshot); } catch {}
+        })
+        .catch(() => {});
+}