mixdog 0.8.0 → 0.9.0

package/src/runtime/agent/orchestrator/session/context-utils.mjs

@@ -1,9 +1,78 @@
 import { isOffloadedToolResultText } from './tool-result-offload.mjs';
 import { createHash } from 'node:crypto';
 
-// Rough token estimate: ~4 chars per token
+// ---------------------------------------------------------------------------
+// Conservative, Unicode-aware token estimator.
+//
+// This is a SAFETY estimator, NOT exact tokenizer parity. We cannot run the
+// provider's real BPE tokenizer here (no network, no bundled vocab), so the
+// goal is to never UNDERcount: a transcript the estimator says "fits" must
+// genuinely fit the model context once compaction has run. The legacy chars/4
+// heuristic badly undercounts Korean/CJK/kana/emoji and dense JSON/tool-call
+// payloads (which BPE often splits into >=1 token per character or per byte),
+// so a "fits" verdict was optimistic exactly where it mattered most.
+//
+// Strategy: weight each code point by how expensive it tends to be under a
+// modern BPE tokenizer (cl100k/o200k-class), then take the MAX of that weighted
+// sum and the chars/4 ASCII lower bound, then apply a small safety multiplier.
+// Weights deliberately lean high (overcount) for CJK/Hangul/emoji.
+//
+// MIXDOG_TOKEN_ESTIMATE_SAFETY_MULTIPLIER (default 1.1, clamped 1.0..2.0) lets
+// operators dial extra headroom without code changes.
+function readSafetyMultiplier() {
+    const raw = Number(process.env.MIXDOG_TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
+    if (Number.isFinite(raw)) return Math.min(2.0, Math.max(1.0, raw));
+    return 1.1;
+}
+const TOKEN_ESTIMATE_SAFETY_MULTIPLIER = readSafetyMultiplier();
+
+// Per-code-point token-cost weight. Tuned to overcount, not match exactly.
+function codePointTokenWeight(cp) {
+    // ASCII (latin letters, digits, punctuation, whitespace, control): the one
+    // region where chars/4 is roughly right — keep the cheap 0.25/char cost.
+    if (cp < 0x80) return 0.25;
+    // Hangul syllables + Jamo + compatibility Jamo. Korean is the worst case
+    // for chars/4: a single syllable frequently costs 1.5–3 BPE tokens, and
+    // rarer syllables fall back to multi-byte splits. Weight high for safety.
+    if (cp >= 0xAC00 && cp <= 0xD7A3) return 1.5;
+    if (cp >= 0x1100 && cp <= 0x11FF) return 1.5;
+    if (cp >= 0x3130 && cp <= 0x318F) return 1.5;
+    if (cp >= 0xA960 && cp <= 0xA97F) return 1.5;
+    if (cp >= 0xD7B0 && cp <= 0xD7FF) return 1.5;
+    // Hiragana / Katakana / Katakana phonetic extensions.
+    if (cp >= 0x3040 && cp <= 0x30FF) return 1.2;
+    if (cp >= 0x31F0 && cp <= 0x31FF) return 1.2;
+    // CJK unified ideographs (incl. Ext A) + compatibility ideographs.
+    if (cp >= 0x3400 && cp <= 0x4DBF) return 1.2;
+    if (cp >= 0x4E00 && cp <= 0x9FFF) return 1.2;
+    if (cp >= 0xF900 && cp <= 0xFAFF) return 1.2;
+    // CJK Extension B and beyond (supplementary ideographic plane).
+    if (cp >= 0x20000 && cp <= 0x2FA1F) return 1.2;
+    // Emoji / pictographs / dingbats / symbols — these explode under BPE
+    // (surrogate pairs, ZWJ sequences, variation selectors), so weight highest.
+    if (cp >= 0x2600 && cp <= 0x27BF) return 2.0;
+    if (cp >= 0x1F000 && cp <= 0x1FAFF) return 2.0;
+    if (cp >= 0x2190 && cp <= 0x21FF) return 1.5; // arrows
+    if (cp >= 0x2300 && cp <= 0x23FF) return 1.5; // technical symbols
+    // Latin-1 supplement / extended latin / IPA — pricier than ASCII (often a
+    // token per accented char) but cheaper than CJK.
+    if (cp < 0x0400) return 0.6;
+    // Everything else non-ASCII (Cyrillic, Greek, Arabic, Hebrew, Thai, …):
+    // multi-byte UTF-8, typically ~0.5–1 token/char. Stay conservative.
+    return 0.8;
+}
+
+// Conservative Unicode-aware token estimate. Iterates by code point (so
+// surrogate-pair emoji are scored once, at the high emoji weight), takes the
+// max of the weighted sum and the chars/4 ASCII floor, then applies the safety
+// multiplier. Always overcounts relative to chars/4 for non-ASCII text.
 function estimateTokens(text) {
-    return Math.ceil(String(text ?? '').length / 4);
+    const s = String(text ?? '');
+    if (s.length === 0) return 0;
+    let weighted = 0;
+    for (const ch of s) weighted += codePointTokenWeight(ch.codePointAt(0));
+    const asciiFloor = s.length / 4; // never below the legacy chars/4 lower bound
+    return Math.ceil(Math.max(weighted, asciiFloor) * TOKEN_ESTIMATE_SAFETY_MULTIPLIER);
 }
 function messageEstimateText(m) {
     if (!m || typeof m !== 'object') return '';
@@ -46,56 +115,122 @@ export function estimateToolSchemaTokens(tools) {
 
 /**
  * Total headroom the caller should reserve out of the context window before
- * compaction: tool-schema bytes + fixed request framing overhead. Pass the
- * result as `opts.reserveTokens` to compactMessages so the working budget
- * accounts for request-side bytes the message estimate cannot see.
+ * compaction: tool-schema bytes + fixed request framing overhead. Pass this as
+ * `opts.reserveTokens` so semantic/recall compaction budgets account for
+ * request-side bytes the message estimate cannot see.
  */
 export function estimateRequestReserveTokens(tools) {
     return estimateToolSchemaTokens(tools) + REQUEST_OVERHEAD_TOKENS;
 }
+
+/**
+ * Live/current context numerator SSOT: transcript estimate + request reserve.
+ * Provider-reported usage is excluded (secondary metadata only).
+ * Empty / no-activity transcript returns 0 so fresh sessions do not show
+ * reserve-only phantom usage.
+ *
+ * @param {unknown[]} messages
+ * @param {unknown[]|number} toolsOrReserve tool list or precomputed reserve tokens
+ * @param {{ messageCount?: number }} [opts]
+ */
+export function estimateTranscriptContextUsage(messages, toolsOrReserve, opts = {}) {
+    const list = Array.isArray(messages) ? messages : [];
+    const count = Number.isFinite(Number(opts.messageCount)) ? Number(opts.messageCount) : list.length;
+    if (count <= 0 || list.length === 0) return 0;
+    const messageTokens = estimateMessagesTokens(list);
+    const reserve = typeof toolsOrReserve === 'number' && Number.isFinite(toolsOrReserve)
+        ? Math.max(0, toolsOrReserve)
+        : estimateRequestReserveTokens(toolsOrReserve);
+    return messageTokens + reserve;
+}
+
 const TOOL_MISSING_STUB = '[Older tool result unavailable after context compaction]';
+function collectAssistantToolCallIds(message) {
+    if (!message || message.role !== 'assistant') return [];
+    const ids = [];
+    const seen = new Set();
+    const add = (id) => {
+        if (!id || seen.has(id)) return;
+        seen.add(id);
+        ids.push(id);
+    };
+    if (Array.isArray(message.toolCalls)) {
+        for (const tc of message.toolCalls) add(tc?.id);
+    }
+    const blocksFrom = (blocks) => {
+        if (!Array.isArray(blocks)) return;
+        for (const b of blocks) {
+            if (b?.type === 'tool_use' && b.id) add(b.id);
+        }
+    };
+    blocksFrom(message.assistantBlocks);
+    blocksFrom(message.content);
+    return ids;
+}
 /**
- * Tool-pair sanitization (Hermes `_sanitize_tool_pairs`):
+ * Tool-pair sanitization (unmatched tool_use / tool_result repair):
+ *   - Drop malformed `tool` messages without toolCallId.
  *   - Drop `tool` messages whose toolCallId has no surviving assistant tool_call.
- *   - For surviving assistant tool_calls whose results are missing, insert a
- *     stub tool message so the provider doesn't reject the request for
- *     unmatched tool_use_id.
- * Messages ordering is preserved; stubs are inserted immediately after the
- * assistant message so the tool pair sits adjacent.
+ *   - For each surviving assistant tool_call, reattach the matching `tool`
+ *     message (if any) immediately after that assistant; duplicate ids prefer
+ *     the contiguous post-assistant block, then later matches, then earlier.
+ *   - For tool_calls with no matching result, insert a stub tool message so
+ *     the provider doesn't reject the request for unmatched tool_use_id.
+ * Non-tool message order is preserved; tool results are not duplicated.
  */
 export function sanitizeToolPairs(messages) {
     if (!Array.isArray(messages) || messages.length === 0) return messages;
     const assistantCallIds = new Set();
     for (const m of messages) {
-        if (m.role === 'assistant' && Array.isArray(m.toolCalls)) {
-            for (const tc of m.toolCalls) {
-                if (tc && tc.id) assistantCallIds.add(tc.id);
-            }
-        }
-    }
-    const toolById = new Map();
-    for (const m of messages) {
-        if (m.role === 'tool' && m.toolCallId) toolById.set(m.toolCallId, m);
+        for (const id of collectAssistantToolCallIds(m)) assistantCallIds.add(id);
     }
-    const filtered = messages.filter(m => {
-        if (m.role !== 'tool') return true;
-        if (!m.toolCallId) return true;
-        return assistantCallIds.has(m.toolCallId);
-    });
+    const pickToolResultForAssistant = (assistantIdx, toolCallId) => {
+        let i = assistantIdx + 1;
+        while (i < messages.length && messages[i]?.role === 'tool') {
+            const tm = messages[i];
+            if (tm.toolCallId === toolCallId) return tm;
+            i += 1;
+        }
+        let afterBlock = assistantIdx + 1;
+        while (afterBlock < messages.length && messages[afterBlock]?.role === 'tool') afterBlock += 1;
+        for (let j = afterBlock; j < messages.length; j += 1) {
+            const tm = messages[j];
+            if (tm?.role === 'tool' && tm.toolCallId === toolCallId) return tm;
+        }
+        for (let j = 0; j < assistantIdx; j += 1) {
+            const tm = messages[j];
+            if (tm?.role === 'tool' && tm.toolCallId === toolCallId) return tm;
+        }
+        return null;
+    };
+    const placedToolIds = new Set();
     const result = [];
-    for (const m of filtered) {
+    for (let idx = 0; idx < messages.length; idx += 1) {
+        const m = messages[idx];
+        if (m.role === 'tool') {
+            if (!m.toolCallId) continue;
+            if (!assistantCallIds.has(m.toolCallId)) continue;
+            if (placedToolIds.has(m.toolCallId)) continue;
+            continue;
+        }
         result.push(m);
-        if (m.role !== 'assistant' || !Array.isArray(m.toolCalls)) continue;
-        for (const tc of m.toolCalls) {
-            if (!tc?.id) continue;
-            const existing = toolById.get(tc.id);
-            if (existing && filtered.includes(existing)) continue;
-            const preserved = existing?.content;
+        if (m.role !== 'assistant') continue;
+        const callIds = collectAssistantToolCallIds(m);
+        if (callIds.length === 0) continue;
+        for (const callId of callIds) {
+            if (placedToolIds.has(callId)) continue;
+            const existing = pickToolResultForAssistant(idx, callId);
+            if (existing) {
+                result.push(existing);
+                placedToolIds.add(callId);
+                continue;
+            }
             result.push({
                 role: 'tool',
-                content: isOffloadedToolResultText(preserved) ? preserved : TOOL_MISSING_STUB,
-                toolCallId: tc.id,
+                content: TOOL_MISSING_STUB,
+                toolCallId: callId,
             });
+            placedToolIds.add(callId);
         }
     }
     return result;
@@ -177,8 +312,27 @@ export function sanitizeAnthropicContentPairs(messages) {
     if (!Array.isArray(messages)) return messages;
     const work = messages.slice();
     const out = [];
+    let pendingToolUseIds = new Set();
+    const stripOrphanToolResults = (userMsg, allowedIds) => {
+        if (userMsg?.role !== 'user' || !Array.isArray(userMsg.content)) return userMsg;
+        const hasToolResults = userMsg.content.some((b) => b?.type === 'tool_result');
+        if (!hasToolResults) return userMsg;
+        const filtered = userMsg.content.filter((b) => {
+            if (b?.type !== 'tool_result') return true;
+            if (!b.tool_use_id) return false;
+            return allowedIds.size > 0 && allowedIds.has(b.tool_use_id);
+        });
+        if (filtered.length === userMsg.content.length) return userMsg;
+        return { ...userMsg, content: filtered };
+    };
     for (let i = 0; i < work.length; i++) {
         let m = work[i];
+        if (m?.role === 'user' && Array.isArray(m.content)) {
+            const hadToolResults = m.content.some((b) => b?.type === 'tool_result');
+            m = stripOrphanToolResults(m, pendingToolUseIds);
+            work[i] = m;
+            if (hadToolResults) pendingToolUseIds = new Set();
+        }
         // Drop tool_use blocks without an id from assistant messages — these
         // come from partial streaming chunks that never finalised, and the
         // provider rejects them as `tool_use ids were found without
@@ -192,13 +346,22 @@ export function sanitizeAnthropicContentPairs(messages) {
                 work[i] = m;
             }
         }
+        if (m?.role === 'user' && Array.isArray(m.content) && m.content.length === 0) continue;
         out.push(m);
         if (m?.role !== 'assistant' || !Array.isArray(m.content)) continue;
         const toolUseIds = m.content
             .filter((b) => b?.type === 'tool_use' && b.id)
             .map((b) => b.id);
-        if (toolUseIds.length === 0) continue;
-        const next = work[i + 1];
+        if (toolUseIds.length === 0) {
+            pendingToolUseIds = new Set();
+            continue;
+        }
+        pendingToolUseIds = new Set(toolUseIds);
+        let next = work[i + 1];
+        if (next?.role === 'user' && Array.isArray(next.content)) {
+            next = stripOrphanToolResults(next, pendingToolUseIds);
+            work[i + 1] = next;
+        }
         const nextResultIds = (next?.role === 'user' && Array.isArray(next.content))
             ? new Set(
                 next.content