mixdog 0.9.55 → 0.9.59

package/src/runtime/agent/orchestrator/session/context-utils.mjs

@@ -1,7 +1,8 @@
 import { isOffloadedToolResultText } from './tool-result-offload.mjs';
 import { createHash } from 'node:crypto';
+import { createRequire } from 'node:module';
 import { providerNativeToolPrefixCount } from '../../../../session-runtime/provider-request-tools.mjs';
-import { contentImageDescriptors, contentToText } from '../providers/media-normalization.mjs';
+import { contentFileDescriptors, contentImageDescriptors, contentToText } from '../providers/media-normalization.mjs';
 export {
     DEFAULT_COMPACTION_BUFFER_TOKENS,
     DEFAULT_COMPACTION_BUFFER_RATIO,
@@ -24,31 +25,137 @@ export {
 } from './context-compaction-policy.mjs';
 
 // ---------------------------------------------------------------------------
-// Conservative, Unicode-aware token estimator.
+// Token estimation: real o200k_base BPE (tiktoken) + per-provider calibration.
 //
-// This is a SAFETY estimator, NOT exact tokenizer parity. We cannot run the
-// provider's real BPE tokenizer here (no network, no bundled vocab), so the
-// goal is to never UNDERcount: a transcript the estimator says "fits" must
-// genuinely fit the model context once compaction has run. The legacy chars/4
-// heuristic badly undercounts Korean/CJK/kana/emoji and dense JSON/tool-call
-// payloads (which BPE often splits into >=1 token per character or per byte),
-// so a "fits" verdict was optimistic exactly where it mattered most.
+// estimateTokens() encodes the provider-visible text projection with the real
+// o200k_base BPE (tiktoken, same approach as Roo Code / goose / chatbox), so
+// estimates track actual billing instead of a weighted-chars heuristic that
+// diverged -25%..+60% depending on script and provider. The residual
+// provider-specific gap between an o200k count and billed prompt tokens is
+// reconciled by providerTokenCalibration(): measured on prefix-verified live
+// sessions, OpenAI billing matches o200k directly (median ratio 1.00) while
+// Anthropic bills the identical projection at ~1.7x o200k (Claude tokenizer +
+// wire framing, median 1.71). Calibration is applied at the provider-aware
+// aggregation boundary (compaction pressure / context gauge), never inside
+// the provider-agnostic per-message memo.
 //
-// Strategy: weight each code point by how expensive it tends to be under a
-// modern BPE tokenizer (cl100k/o200k-class), then take the MAX of that weighted
-// sum and the chars/4 ASCII lower bound, then apply a small safety multiplier.
-// Weights deliberately lean high (overcount) for CJK/Hangul/emoji.
+// The previous conservative Unicode-weight heuristic is retained verbatim as
+// legacyEstimateTokens(), used only when the tiktoken WASM cannot be loaded
+// (stripped installs / exotic platforms) so estimation degrades safely.
 //
-// MIXDOG_TOKEN_ESTIMATE_SAFETY_MULTIPLIER (default 1.1, clamped 1.0..2.0) lets
-// operators dial extra headroom without code changes.
+// MIXDOG_TOKEN_ESTIMATE_SAFETY_MULTIPLIER (default 1.0, clamped 1.0..2.0)
+// still lets operators dial extra headroom without code changes; the
+// provider-usage baseline remains the primary pressure source when aligned.
 function readSafetyMultiplier() {
     const raw = Number(process.env.MIXDOG_TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
     if (Number.isFinite(raw)) return Math.min(2.0, Math.max(1.0, raw));
-    return 1.1;
+    return 1.0;
 }
 const TOKEN_ESTIMATE_SAFETY_MULTIPLIER = readSafetyMultiplier();
-export const IMAGE_VISUAL_TOKEN_ALLOWANCE = 4_096;
-const IMAGE_TILE_TOKEN_ALLOWANCE = 512;
+
+// Lazy singleton o200k_base encoder. createRequire keeps the load dynamic for
+// bundlers (TUI esbuild) and lets a missing/broken WASM fall back cleanly.
+const _requireForTokenizer = createRequire(import.meta.url);
+let _bpeEncoder; // undefined = not attempted, null = unavailable
+function bpeEncoder() {
+    if (_bpeEncoder !== undefined) return _bpeEncoder;
+    try {
+        const { Tiktoken } = _requireForTokenizer('tiktoken/lite');
+        const o200k = _requireForTokenizer('tiktoken/encoders/o200k_base.json');
+        _bpeEncoder = new Tiktoken(o200k.bpe_ranks, o200k.special_tokens, o200k.pat_str);
+    } catch {
+        _bpeEncoder = null;
+    }
+    return _bpeEncoder;
+}
+
+// Compaction budget loops re-estimate overlapping transcript slices; identical
+// large strings (tool results, system blocks) would re-encode every pass.
+// Hash-keyed LRU (goose token_counter.rs pattern) absorbs the repeats.
+const TOKEN_COUNT_CACHE_MIN_CHARS = 512;
+const TOKEN_COUNT_CACHE_MAX_ENTRIES = 1_024;
+const tokenCountCache = new Map();
+// BPE merge cost is quadratic in WORD length, not text length: a degenerate
+// single-word run ('p'.repeat(250k), base64/minified blobs) forms one giant
+// pat_str word and encode() spins for minutes while the event loop is blocked
+// (observed live via inspector: compact-smoke hung inside tiktoken encode).
+// Encoding fixed-size slices caps the worst-case word at the chunk size, so
+// cost stays linear. Chunk cuts snap back to the nearest whitespace so the
+// next slice starts at a natural ` word` boundary — o200k tokenizes that
+// identically to the unsliced text, keeping estimates EXACT for prose
+// (compact-smoke asserts est === real o200k count). Only a whitespace-free
+// degenerate run falls back to a hard cut (±1 token per boundary, safely
+// inside the estimate's safety multiplier). Hard cuts avoid splitting a
+// surrogate pair so sliced emoji/CJK-ext code points still encode cleanly.
+const BPE_ENCODE_CHUNK_CHARS = 4_096;
+const BPE_CHUNK_BOUNDARY_SCAN = 512;
+function bpeEncodeCount(enc, s) {
+    if (s.length <= BPE_ENCODE_CHUNK_CHARS) return enc.encode(s, undefined, []).length;
+    let total = 0;
+    let i = 0;
+    while (i < s.length) {
+        let end = Math.min(s.length, i + BPE_ENCODE_CHUNK_CHARS);
+        if (end < s.length) {
+            // Prefer cutting BEFORE a whitespace run: the next chunk then
+            // starts with ` word`, which o200k merges exactly as it would
+            // mid-text. Scan a bounded window so degenerate inputs stay O(1).
+            let ws = -1;
+            const scanFloor = Math.max(i + 1, end - BPE_CHUNK_BOUNDARY_SCAN);
+            for (let j = end - 1; j >= scanFloor; j -= 1) {
+                const c = s.charCodeAt(j);
+                if (c === 0x20 || c === 0x0A || c === 0x0D || c === 0x09) { ws = j; break; }
+            }
+            if (ws > i) {
+                end = ws; // next chunk starts at the whitespace
+            } else {
+                const last = s.charCodeAt(end - 1);
+                if (last >= 0xD800 && last <= 0xDBFF) end += 1;
+            }
+        }
+        total += enc.encode(s.slice(i, end), undefined, []).length;
+        i = end;
+    }
+    return total;
+}
+function bpeTokenCount(enc, s) {
+    if (s.length < TOKEN_COUNT_CACHE_MIN_CHARS) return bpeEncodeCount(enc, s);
+    const key = `${s.length}:${createHash('sha1').update(s).digest('base64')}`;
+    const hit = tokenCountCache.get(key);
+    if (hit !== undefined) {
+        tokenCountCache.delete(key);
+        tokenCountCache.set(key, hit);
+        return hit;
+    }
+    const count = bpeEncodeCount(enc, s);
+    if (tokenCountCache.size >= TOKEN_COUNT_CACHE_MAX_ENTRIES) {
+        tokenCountCache.delete(tokenCountCache.keys().next().value);
+    }
+    tokenCountCache.set(key, count);
+    return count;
+}
+
+// Billed-prompt / o200k-estimate ratio per provider family, measured against
+// prefix-signature-verified provider baselines from real sessions (opaque
+// signature/encrypted payloads excluded from the projection — see
+// stripOpaquePayloads). Env overrides let a deployment recalibrate without a
+// code change; values are clamped to a plausible band.
+function calibrationEnv(name) {
+    const n = Number(process.env[name]);
+    return Number.isFinite(n) && n > 0 ? Math.min(3, Math.max(0.25, n)) : null;
+}
+export function providerTokenCalibration(provider) {
+    const p = String(provider || '').toLowerCase();
+    if (p.startsWith('anthropic')) return calibrationEnv('MIXDOG_TOKEN_CALIBRATION_ANTHROPIC') ?? 1.7;
+    if (p.startsWith('gemini') || p.startsWith('google')) return calibrationEnv('MIXDOG_TOKEN_CALIBRATION_GEMINI') ?? 1.15;
+    return calibrationEnv('MIXDOG_TOKEN_CALIBRATION_DEFAULT') ?? 1.0;
+}
+// Claude Code parity (services/tokenEstimation.ts): images/documents count a
+// flat 2000 tokens when dimensions are unknown — the conservative constant CC
+// shares with microCompact's IMAGE_MAX_TOKEN_SIZE. Known dimensions may only
+// RAISE the allowance via Anthropic's real vision formula (w*h/750), capped at
+// the 2000x2000 resize ceiling (5333 tokens).
+export const IMAGE_VISUAL_TOKEN_ALLOWANCE = 2_000;
+const IMAGE_MAX_TOKEN_ALLOWANCE = 5_333;
 
 // Per-code-point token-cost weight. Tuned to overcount, not match exactly.
 function codePointTokenWeight(cp) {
@@ -86,13 +193,23 @@ function codePointTokenWeight(cp) {
     return 0.8;
 }
 
-// Conservative Unicode-aware token estimate. Iterates by code point (so
-// surrogate-pair emoji are scored once, at the high emoji weight), takes the
-// max of the weighted sum and the chars/4 ASCII floor, then applies the safety
-// multiplier. Always overcounts relative to chars/4 for non-ASCII text.
+// Real-BPE token estimate with heuristic fallback. See module header.
 export function estimateTokens(text) {
     const s = String(text ?? '');
     if (s.length === 0) return 0;
+    const enc = bpeEncoder();
+    if (enc) {
+        try {
+            return Math.ceil(bpeTokenCount(enc, s) * TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
+        } catch { /* corrupt input — degrade to the heuristic below */ }
+    }
+    return legacyEstimateTokens(s);
+}
+
+// Legacy conservative Unicode-aware estimate (fallback only). Iterates by
+// code point, takes the max of the weighted sum and the chars/4 ASCII floor,
+// then applies the safety multiplier.
+function legacyEstimateTokens(s) {
     let weighted = 0;
     for (const ch of s) weighted += codePointTokenWeight(ch.codePointAt(0));
     // Encoded blobs, minified JSON and generated identifiers do not get the
@@ -101,7 +218,9 @@ export function estimateTokens(text) {
     // for those runs without penalizing ordinary spaced prose.
     let denseAsciiFloor = 0;
     for (const match of s.matchAll(/[\x21-\x7e]{16,}/g)) {
-        denseAsciiFloor += match[0].length * 0.75;
+        // Claude Code prices dense JSON/JSONL at chars/2 (bytesPerTokenForFileType);
+        // 0.5/char keeps that parity for long unmerged printable runs.
+        denseAsciiFloor += match[0].length * 0.5;
     }
     const encodedWords = s.match(/\b(?=[A-Za-z0-9]{8,}\b)(?=[A-Za-z0-9]*[A-Za-z])(?=[A-Za-z0-9]*\d)[A-Za-z0-9]+\b/g) || [];
     if (encodedWords.length >= 3) {
@@ -111,7 +230,7 @@ export function estimateTokens(text) {
         const encodedChars = encodedWords.reduce((sum, word) => sum + word.length, 0);
         denseAsciiFloor = Math.max(
             denseAsciiFloor,
-            (encodedChars * 0.75) + ((s.length - encodedChars) * 0.25),
+            (encodedChars * 0.5) + ((s.length - encodedChars) * 0.25),
         );
     }
     const lines = s.split(/\r?\n/).filter(line => line.trim());
@@ -122,12 +241,35 @@ export function estimateTokens(text) {
         && (jsonLikeLines >= Math.ceil(lines.length / 2) || structural / nonWhitespace >= 0.12)) {
         // JSONL, compact tables and generated line protocols can consist
         // entirely of short runs while still tokenizing like minified data.
-        denseAsciiFloor = Math.max(denseAsciiFloor, (nonWhitespace * 0.7) + ((s.length - nonWhitespace) * 0.25));
+        // chars/2 on the dense payload chars (Claude Code JSON parity).
+        denseAsciiFloor = Math.max(denseAsciiFloor, (nonWhitespace * 0.5) + ((s.length - nonWhitespace) * 0.25));
     }
     const asciiFloor = s.length / 4; // never below the legacy chars/4 lower bound
     return Math.ceil(Math.max(weighted, asciiFloor, denseAsciiFloor) * TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
 }
 
+// Opaque replay payloads (Anthropic thinking signatures, OpenAI encrypted
+// reasoning blobs, redacted data) are long base64-ish strings that are NOT
+// billed proportionally to their serialized length — including them made
+// estimates swing wildly per turn. Replace them with a fixed marker before
+// token counting; calibration factors were measured against this projection.
+// Stripping is KEY-SCOPED: only fields that structurally carry opaque replay
+// material (signatures / encrypted / redacted / raw data blobs) are eligible,
+// so genuine long model text (e.g. Gemini thought text) keeps its real cost.
+const OPAQUE_PAYLOAD_KEY_RE = /signature|encrypted|redacted|^data$|^blob$/i;
+const OPAQUE_PAYLOAD_RE = /^[A-Za-z0-9+/_=-]{64,}$/;
+function stripOpaquePayloads(value, depth = 0, keyHint = '') {
+    if (typeof value === 'string') {
+        return OPAQUE_PAYLOAD_KEY_RE.test(keyHint) && value.length >= 64 && OPAQUE_PAYLOAD_RE.test(value)
+            ? '[opaque]'
+            : value;
+    }
+    if (depth >= 8 || !value || typeof value !== 'object') return value;
+    if (Array.isArray(value)) return value.map((v) => stripOpaquePayloads(v, depth + 1, keyHint));
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = stripOpaquePayloads(v, depth + 1, k);
+    return out;
+}
 function nativeBlocksEstimateText(value) {
     const list = Array.isArray(value) ? value : [value];
     return list.map((block) => {
@@ -137,7 +279,7 @@ function nativeBlocksEstimateText(value) {
                 type: 'image', width, height, detail,
             })));
         }
-        try { return JSON.stringify(block); }
+        try { return JSON.stringify(stripOpaquePayloads(block)); }
         catch { return String(block ?? ''); }
     }).join('\n');
 }
@@ -181,15 +323,18 @@ export function messageEstimateText(m) {
 }
 function imageDescriptorAllowance(descriptor) {
     if (descriptor.width && descriptor.height) {
-        const tiles = Math.ceil(descriptor.width / 512) * Math.ceil(descriptor.height / 512);
-        // Caller-supplied dimensions/detail are not uniformly preserved by all
-        // provider normalizers, so they may raise the allowance for a known
-        // multi-tile image but can never lower the conservative unknown-image
-        // fallback. This avoids trusting metadata the provider never sees.
-        return Math.max(IMAGE_VISUAL_TOKEN_ALLOWANCE, IMAGE_TILE_TOKEN_ALLOWANCE * (tiles + 1));
-    }
-    // Unknown-size auto/high images may be tiled by the provider. A single
-    // 1k allowance is optimistic for common screenshots and documents.
+        // Anthropic vision cost: tokens = (width * height) / 750, with images
+        // resized down to at most 2000x2000 (5333 tokens) — Claude Code uses
+        // the same formula/cap. Caller-supplied dimensions may RAISE the
+        // allowance above the unknown-image floor but never lower it (the
+        // provider normalizer may not preserve caller metadata).
+        const formula = Math.ceil((descriptor.width * descriptor.height) / 750);
+        return Math.min(
+            IMAGE_MAX_TOKEN_ALLOWANCE,
+            Math.max(IMAGE_VISUAL_TOKEN_ALLOWANCE, formula),
+        );
+    }
+    // Unknown-size images: flat conservative allowance (Claude Code parity).
     return IMAGE_VISUAL_TOKEN_ALLOWANCE;
 }
 function messageImageDescriptors(m) {
@@ -203,8 +348,21 @@ function messageImageAllowance(m) {
     if (!m || typeof m !== 'object') return 0;
     return messageImageDescriptors(m).reduce((sum, descriptor) => sum + imageDescriptorAllowance(descriptor), 0);
 }
+// Inline document (PDF) allowance: Anthropic bills ~1,500-3,000 tokens per
+// page and a typical PDF page is ~50-100KB, so ~bytes/16 with a floor and a
+// cap. The base64 payload itself is excluded from text estimates (see
+// jsonFallbackFromPart), so this allowance is the document's entire cost.
+const FILE_TOKEN_ALLOWANCE_FLOOR = 1_500;
+const FILE_MAX_TOKEN_ALLOWANCE = 300_000;
+function messageFileAllowance(m) {
+    if (!m || typeof m !== 'object') return 0;
+    return contentFileDescriptors(m.content).reduce((sum, descriptor) => sum + Math.min(
+        FILE_MAX_TOKEN_ALLOWANCE,
+        Math.max(FILE_TOKEN_ALLOWANCE_FLOOR, Math.ceil((descriptor.sizeBytes || 0) / 16)),
+    ), 0);
+}
 export function estimateMessageTokens(m) {
-    return estimateTokens(messageEstimateText(m)) + messageImageAllowance(m) + 4;
+    return estimateTokens(messageEstimateText(m)) + messageImageAllowance(m) + messageFileAllowance(m) + 4;
 }
 export function estimateMessagesTokens(messages) {
     return messages.reduce((sum, m) => sum + estimateMessageTokens(m), 0);
@@ -479,11 +637,6 @@ export function contextMessagesSignature(messages, count = messages?.length) {
     return hash.digest('hex');
 }
 
-// Per-request overhead the provider injects that never appears in the
-// `messages` array: function-calling preamble + system-prompt framing the
-// provider wraps around the request. The chars/4 message estimate misses all
-// of it, so a "fits" verdict computed from messages alone is optimistic.
-const REQUEST_OVERHEAD_TOKENS = 512;
 const toolSchemaTokenMemo = new WeakMap();
 const requestReserveTokenMemo = new WeakMap();
 
@@ -531,17 +684,16 @@ export function estimateToolSchemaTokens(tools) {
 }
 
 /**
- * Total headroom the caller should reserve out of the context window before
- * compaction: tool-schema bytes + fixed request framing overhead. Pass this as
- * `opts.reserveTokens` so semantic/recall compaction budgets account for
- * request-side bytes the message estimate cannot see.
+ * Total request-side bytes the caller should reserve out of the context window
+ * before compaction. Only serialized tool schemas are counted; providers do
+ * not expose a stable framing cost, so no synthetic fixed allowance is added.
  */
 export function estimateRequestReserveTokens(tools) {
-    if (!Array.isArray(tools)) return estimateToolSchemaTokens(tools) + REQUEST_OVERHEAD_TOKENS;
+    if (!Array.isArray(tools)) return estimateToolSchemaTokens(tools);
     const signature = toolSchemaSignature(tools);
     const cached = requestReserveTokenMemo.get(tools);
     if (cached?.signature === signature) return cached.value;
-    const reserve = estimateToolSchemaTokens(tools) + REQUEST_OVERHEAD_TOKENS;
+    const reserve = estimateToolSchemaTokens(tools);
     requestReserveTokenMemo.set(tools, { signature, value: reserve });
     return reserve;
 }
@@ -554,7 +706,7 @@ export function estimateRequestReserveTokens(tools) {
  *
  * @param {unknown[]} messages
  * @param {unknown[]|number} toolsOrReserve tool list or precomputed reserve tokens
- * @param {{ messageCount?: number, estimatedMessageTokens?: number }} [opts]
+ * @param {{ messageCount?: number, estimatedMessageTokens?: number, provider?: string }} [opts]
  */
 export function estimateTranscriptContextUsage(messages, toolsOrReserve, opts = {}) {
     const list = Array.isArray(messages) ? messages : [];
@@ -566,7 +718,9 @@ export function estimateTranscriptContextUsage(messages, toolsOrReserve, opts =
     const reserve = typeof toolsOrReserve === 'number' && Number.isFinite(toolsOrReserve)
         ? Math.max(0, toolsOrReserve)
         : estimateRequestReserveTokens(toolsOrReserve);
-    return messageTokens + reserve;
+    // Provider-aware calibration reconciles the o200k estimate with actual
+    // billing (see providerTokenCalibration). No provider → neutral 1.0.
+    return Math.round((messageTokens + reserve) * providerTokenCalibration(opts.provider));
 }
 
 const TOOL_MISSING_STUB = '[Older tool result unavailable after context compaction]';

package/src/runtime/agent/orchestrator/session/loop/compact-debug.mjs

@@ -1,8 +1,12 @@
 // Compaction debug/diagnostic helpers extracted from loop.mjs.
-import { estimateMessagesTokens } from '../context-utils.mjs';
+import { summarizeContextMessages } from '../context-utils.mjs';
 
 export function estimateMessagesTokensSafe(messages) {
-    try { return estimateMessagesTokens(messages); }
+    // summarizeContextMessages is the fingerprint-validated cached form of
+    // estimateMessagesTokens (same per-message estimator, accumulated on the
+    // live array). The uncached sum cost ~30ms per call on a long session and
+    // ran on EVERY pre-send iteration; the cached path is ~1ms warm.
+    try { return summarizeContextMessages(messages).estimatedTokens; }
     catch { return null; }
 }
 

package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs

@@ -6,6 +6,7 @@ import {
     contextMessagesSignature,
     estimateMessagesTokens,
     estimateRequestReserveTokens,
+    providerTokenCalibration,
     resolveSessionCompactPolicy,
     toolSchemaSignature,
 } from '../context-utils.mjs';
@@ -170,13 +171,32 @@ export function resolveWorkerCompactPolicy(sessionRef, tools) {
         reserveTokens,
         requestReserveTokens: requestReserve,
         configuredReserveTokens: configuredReserve,
+        provider: sessionRef.provider || null,
+        tokenCalibration: providerTokenCalibration(sessionRef.provider),
         toolSchemaSignature: toolSchemaSignature(tools),
     };
 }
-/** Transcript + request reserve fallback used until an aligned provider baseline exists. */
+/**
+ * Transcript + request reserve fallback used until an aligned provider
+ * baseline exists. The transcript estimate and the serialized-tool-schema
+ * reserve are both text the provider tokenizes, so the per-provider billing
+ * calibration applies to them; a configured operator reserve is a raw token
+ * allowance and stays uncalibrated. With no calibration on the policy the
+ * result is the exact legacy sum (estimate + reserveTokens).
+ */
 function compactPressureTokens(messageTokensEst, policy) {
     if (messageTokensEst === null) return 0;
-    return Math.max(0, messageTokensEst + (policy?.reserveTokens || 0));
+    const calibration = Number(policy?.tokenCalibration) > 0 ? Number(policy.tokenCalibration) : 1;
+    const configured = Math.max(0, Number(policy?.configuredReserveTokens) || 0);
+    const totalReserve = Math.max(0, Number(policy?.reserveTokens) || 0);
+    // reserveTokens is the caller-facing override (tests/policies may zero it);
+    // the request-schema share can never exceed it.
+    const requestReserve = Math.min(
+        totalReserve,
+        Math.max(0, Number(policy?.requestReserveTokens ?? (totalReserve - configured)) || 0),
+    );
+    const otherReserve = Math.max(0, totalReserve - requestReserve);
+    return Math.max(0, Math.round((messageTokensEst + requestReserve) * calibration) + otherReserve);
 }
 
 function providerPressureTokens(sessionRef, usage) {
@@ -241,6 +261,13 @@ export function invalidateProviderContextBaseline(sessionRef) {
     sessionRef.lastContextTokensStaleAfterCompact = true;
 }
 
+// A baseline is refreshed on every provider_send/turn-end. When the transcript
+// has grown but no fresh recording landed for this long, recording is failing
+// (zero/absent usage keeps the OLD baseline because record() only returns
+// false) — distrust it and fall back to the estimate. An idle session whose
+// transcript did NOT grow keeps its baseline regardless of age.
+const BASELINE_MAX_STALE_GROWTH_MS = 30 * 60 * 1000;
+
 function providerBaselinePressureTokens(messages, sessionRef, policy) {
     if (!Array.isArray(messages) || !sessionRef
         || sessionRef.lastContextTokensStaleAfterCompact === true) return null;
@@ -255,6 +282,8 @@ function providerBaselinePressureTokens(messages, sessionRef, policy) {
         || sessionRef.contextPressureBaselineModel !== (sessionRef.model || null)
         || sessionRef.contextPressureBaselineToolSignature !== policy?.toolSchemaSignature
         || sessionRef.contextPressureBaselinePrefixSignature !== contextMessagesSignature(messages, count)) return null;
+    if (messages.length > count && baselineAt > 0
+        && (Date.now() - baselineAt) > BASELINE_MAX_STALE_GROWTH_MS) return null;
     if (sessionRef.contextPressureBaselineBoundary === 'request') {
         const assistantOffset = messages.slice(count).findIndex(message => message?.role === 'assistant');
         if (assistantOffset >= 0) {
@@ -268,8 +297,11 @@ function providerBaselinePressureTokens(messages, sessionRef, policy) {
         }
     }
     try {
+        // Baseline tokens are authoritative provider billing; only the growth
+        // after the baseline is a local estimate and needs billing calibration.
+        const calibration = Number(policy?.tokenCalibration) > 0 ? Number(policy.tokenCalibration) : 1;
         const growth = count < messages.length
-            ? estimateMessagesTokens(messages.slice(count))
+            ? Math.round(estimateMessagesTokens(messages.slice(count)) * calibration)
             : 0;
         return Math.max(0, tokens + growth + Math.max(0, Number(policy?.configuredReserveTokens) || 0));
     } catch {
@@ -278,8 +310,20 @@ function providerBaselinePressureTokens(messages, sessionRef, policy) {
 }
 
 export function resolveCompactionPressureTokens(messageTokensEst, policy, { messages, sessionRef } = {}) {
-    return providerBaselinePressureTokens(messages, sessionRef, policy)
-        ?? compactPressureTokens(messageTokensEst, policy);
+    const baseline = providerBaselinePressureTokens(messages, sessionRef, policy);
+    const estimate = compactPressureTokens(messageTokensEst, policy);
+    if (baseline == null) return estimate;
+    // Sanity band: the baseline exists to correct OVER-counting estimates
+    // (dense-data floors can inflate the estimate up to ~2x real usage), so a
+    // lower baseline is normally preferred. But a corrupt/stale baseline below
+    // HALF the estimate is no longer plausible as an overcount correction —
+    // a live session showed ~70% on the gauge while the provider was billing
+    // 111% of a 1M window. Beyond that band, trust the transcript estimate for
+    // both the gauge and the compaction decision. Erring toward the estimate
+    // may compact somewhat early; erring toward a rotten baseline blows past
+    // the context window at full token cost.
+    if (Number.isFinite(estimate) && estimate > 0 && baseline * 2 < estimate) return estimate;
+    return baseline;
 }
 
 /** Telemetry pressure when a reactive overflow retry forces the next compact. */
@@ -331,7 +375,16 @@ export function shouldCompactForSession(messageTokensEst, policy, {
     const pressure = Number.isFinite(Number(pressureTokens))
         ? Number(pressureTokens)
         : resolveCompactionPressureTokens(messageTokensEst, policy, { messages, sessionRef });
-    return pressure >= (policy.triggerTokens || policy.boundaryTokens);
+    const trigger = policy.triggerTokens || policy.boundaryTokens;
+    if (pressure >= trigger) return true;
+    // Safety net: the provider-usage baseline exists to correct OVER-counting
+    // transcript estimates, so a lower baseline-derived pressure is normally
+    // preferred. But a stale/wrong baseline must never SUPPRESS compaction
+    // once the raw transcript estimate itself has crossed the trigger — that
+    // failure mode let a live session sail past a 950k trigger to 1.1M+ real
+    // tokens without a single auto compact. The trigger decision (not the
+    // gauge) therefore takes the max of both pressure sources.
+    return compactPressureTokens(messageTokensEst, policy) >= trigger;
 }
 export function countPrunedToolOutputs(before, after) {
     if (!Array.isArray(before) || !Array.isArray(after)) return 0;
@@ -349,6 +402,11 @@ export function rememberCompactTelemetry(sessionRef, policy, meta = {}) {
         ? sessionRef.compaction
         : {};
     const changed = meta.compactChanged === true || meta.pruneCount > 0;
+    // Both are successful terminal pre-send states. In particular,
+    // pre_send_check is the no-op path after a prior recovered/failing compact;
+    // retaining its old component error makes status report a failure although
+    // this send's compaction stage completed successfully.
+    const terminalSuccess = meta.stage === 'pre_send' || meta.stage === 'pre_send_check';
     sessionRef.compaction = {
         ...prev,
         auto: policy.auto !== false,
@@ -383,16 +441,22 @@ export function rememberCompactTelemetry(sessionRef, policy, meta = {}) {
         lastChanged: changed,
         lastTrigger: meta.trigger || prev.lastTrigger || null,
         lastSemantic: meta.semanticCompact === true,
-        lastSemanticError: Object.hasOwn(meta, 'semanticError')
-            ? (meta.semanticError ?? null)
-            : (prev.lastSemanticError ?? null),
+        lastSemanticError: terminalSuccess
+            ? null
+            : Object.hasOwn(meta, 'semanticError')
+                ? (meta.semanticError ?? null)
+                : (prev.lastSemanticError ?? null),
         lastRecallFastTrack: meta.recallFastTrack === true,
-        lastRecallFastTrackError: Object.hasOwn(meta, 'recallFastTrackError')
-            ? (meta.recallFastTrackError ?? null)
-            : (prev.lastRecallFastTrackError ?? null),
-        lastError: Object.hasOwn(meta, 'compactError') || Object.hasOwn(meta, 'lastError')
-            ? (meta.compactError ?? meta.lastError ?? null)
-            : (prev.lastError ?? null),
+        lastRecallFastTrackError: terminalSuccess
+            ? null
+            : Object.hasOwn(meta, 'recallFastTrackError')
+                ? (meta.recallFastTrackError ?? null)
+                : (prev.lastRecallFastTrackError ?? null),
+        lastError: terminalSuccess
+            ? null
+            : Object.hasOwn(meta, 'compactError') || Object.hasOwn(meta, 'lastError')
+                ? (meta.compactError ?? meta.lastError ?? null)
+                : (prev.lastError ?? null),
         lastPruneCount: meta.pruneCount || 0,
         lastDurationMs: meta.durationMs != null && Number.isFinite(Number(meta.durationMs))
             ? Math.max(0, Math.round(Number(meta.durationMs)))
@@ -402,6 +466,23 @@ export function rememberCompactTelemetry(sessionRef, policy, meta = {}) {
             ? true
             : prev.singleShotConsumed === true,
     };
+    // Postmortem ring buffer: the per-check telemetry above is overwritten on
+    // every stage change, which erased all pre-compact evidence when a session
+    // blew past its trigger without compacting. Keep the last few decisions
+    // (pressure vs estimate vs trigger plus the live baseline) on the session
+    // so a missed-trigger incident is diagnosable after the fact.
+    {
+        const prior = Array.isArray(prev.recentChecks) ? prev.recentChecks : [];
+        sessionRef.compaction.recentChecks = [...prior, {
+            at: Date.now(),
+            stage: meta.stage || null,
+            pressure: meta.pressureTokens ?? null,
+            est: meta.beforeTokens ?? null,
+            trigger: policy.triggerTokens || policy.boundaryTokens || null,
+            baseline: positiveTokenInt(sessionRef.contextPressureBaselineTokens) || null,
+            baselineAt: Number(sessionRef.contextPressureBaselineUpdatedAt) || null,
+        }].slice(-8);
+    }
     if (changed) {
         const changedAt = Date.now();
         sessionRef.compaction.lastChangedAt = changedAt;
@@ -435,3 +516,17 @@ export function emitCompactEvent(opts, event = {}) {
 export function compactEventType(policy, fallback = DEFAULT_COMPACT_TYPE) {
     return policy?.compactType || policy?.type || fallback;
 }
+
+// Semantic-summary model override. NO automatic downshift: the runtime cannot
+// know which models a given account/gateway can actually serve (an OAuth plan
+// or relay may not expose Haiku at all), so guessing a cheaper model risks a
+// hard compact failure. The summary runs on the session's own model unless an
+// operator explicitly configures compaction.semanticModel (or the
+// MIXDOG_AGENT_COMPACT_SEMANTIC_MODEL env) — that opt-in is the only override.
+export function resolveSemanticSummaryModel(sessionRef, _opts = {}) {
+    const cfg = sessionRef?.compaction && typeof sessionRef.compaction === 'object' ? sessionRef.compaction : {};
+    const explicit = String(cfg.semanticModel || '').trim()
+        || String(process.env.MIXDOG_AGENT_COMPACT_SEMANTIC_MODEL || '').trim();
+    if (explicit) return explicit;
+    return null;
+}

package/src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs

@@ -161,6 +161,7 @@ export async function runRecallFastTrackCompact({ sessionRef, messages, compactB
             limit: positiveTokenInt(sessionRef?.compaction?.recallDigestLimit) || 30,
             includeMembers: true,
             includeRaw: true,
+            compactDigest: true,
         }, callerCtx);
         digestBody = typeof browsed === 'string' ? browsed : String(browsed?.text ?? browsed ?? '');
     } catch (err) {
@@ -213,6 +214,7 @@ export async function runRecallFastTrackCompact({ sessionRef, messages, compactB
         recallText: digestText,
         query,
         querySha,
+        cwd: sessionRef?.cwd,
         // Ingest + search both succeeded above, so the memory DB genuinely holds
         // this transcript and the recall notice is truthful even when the digest
         // body is small. A failure would have aborted before reaching here.

package/src/runtime/agent/orchestrator/session/loop/tool-exec.mjs

@@ -9,7 +9,7 @@ import { executeBashSessionTool } from '../../tools/bash-session.mjs';
 import { executePatchTool } from '../../tools/patch.mjs';
 import { executeInternalTool, isInternalTool } from '../../internal-tools.mjs';
 import { normalizeToolEnvelope, makeToolEnvelope } from '../tool-envelope.mjs';
-import { getSessionAbortSignal, enqueuePendingMessage, markCompletionEntry } from '../manager.mjs';
+import { getSessionAbortSignal, enqueuePendingMessage, markCompletionEntry, markSessionToolOutputTail } from '../manager.mjs';
 import { createScopedCacheOutcome } from '../cache/scoped-cache-outcome.mjs';
 import { modelVisibleToolCompletionMessage } from '../../../../shared/tool-execution-contract.mjs';
 import { _isScopedCacheableTool } from './tool-classify.mjs';
@@ -81,6 +81,12 @@ export async function executeTool(name, args, cwd, callerSessionId, sessionRef,
         routingSessionId: callerSessionId,
         clientHostPid: sessionRef?.clientHostPid,
         notifyFn,
+        // Live shell-output tail → session liveness (~1 s cadence from the
+        // shell tool's tail timer), surfaced to transcript consumers (desktop
+        // running tool cards) via getSessionProgressSnapshot.
+        onOutputTail: (tail) => {
+            try { markSessionToolOutputTail(callerSessionId, tail); } catch { /* best effort */ }
+        },
     };
     const beforeToolHook = typeof executeOpts.beforeToolHook === 'function'
         ? executeOpts.beforeToolHook