mixdog 0.9.55 → 0.9.58

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mixdog",
-  "version": "0.9.55",
+  "version": "0.9.58",
   "private": false,
   "type": "module",
   "description": "Standalone mixdog coding-agent CLI/TUI workspace.",
@@ -70,6 +70,7 @@
     "test:tui-queue": "node --test scripts/submit-commandbusy-race-test.mjs scripts/steering-drain-buckets-test.mjs scripts/abort-recovery-test.mjs scripts/execution-pending-resume-kick-test.mjs scripts/execution-resume-esc-integration-test.mjs",
     "test:tui-input-render": "node --test scripts/prompt-immediate-render-test.mjs",
     "test:tui-streaming-window": "node --test scripts/streaming-tail-window-test.mjs",
+    "test:tui-ambiguous-width": "node --test scripts/tui-ambiguous-width-test.mjs",
     "test:release-assets": "node --check scripts/verify-release-assets.mjs && node --check scripts/verify-release-assets-test.mjs && node --test scripts/verify-release-assets-test.mjs",
     "test:release-focused": "npm run test:release-assets && npm run test:patch-binary-cache && npm run test:providers && npm run test:deferred-tools && npm run smoke:compact && node --test scripts/code-graph-aggregate-cwd-test.mjs && npm run test:code-graph-dispatch && node --test scripts/code-graph-disk-hit-test.mjs && npm run test:shellhardening && node --test scripts/windows-hide-spawn-options-test.mjs && node --test scripts/tui-transcript-perf-test.mjs",
     "test:native-edit-wire": "node --test scripts/native-edit-wire-test.mjs",
@@ -117,6 +118,7 @@
     "react": "^19.2.7",
     "string-width": "^8.2.1",
     "strip-ansi": "^7.2.0",
+    "tiktoken": "^1.0.22",
     "undici": "^8.5.0",
     "wrap-ansi": "^10.0.0",
     "ws": "^8.21.0",

package/scripts/_smoke_wd.mjs

@@ -0,0 +1,7 @@
+const wd = setTimeout(() => {
+  console.error('=== WATCHDOG 45s: active resources:', JSON.stringify(process.getActiveResourcesInfo()));
+  process.exit(99);
+}, 45000);
+await import('./compact-smoke.mjs');
+clearTimeout(wd);
+console.error('=== script import completed normally');

package/scripts/agent-terminal-reap-test.mjs

@@ -1,5 +1,5 @@
 import assert from 'node:assert/strict';
-import { mkdtempSync, mkdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import {
@@ -41,7 +41,12 @@ try {
   writeFileSync(join(root, 'mixdog-config.json'), JSON.stringify({
     agent: { autoClear: overrideConfig.autoClear },
   }));
-  const { sweepStaleSessions } = await import('../src/runtime/agent/orchestrator/session/store.mjs');
+  const {
+    deleteSession,
+    markSessionClosed,
+    saveSession,
+    sweepStaleSessions,
+  } = await import('../src/runtime/agent/orchestrator/session/store.mjs');
   const old = Date.now() - 181_000;
   const known = {
     id: 'sess_known_reap',
@@ -83,6 +88,126 @@ try {
   assert.ok(defaultSweep.details.some((detail) => detail.id === known.id), 'store reaps a listed provider at its Advanced duration');
   assert.ok(defaultSweep.details.some((detail) => detail.id === unknown.id), 'store reaps an unlisted provider at the default duration');
 
+  const locallyLive = {
+    ...known,
+    id: 'sess_locally_live_keep',
+  };
+  writeFileSync(join(root, 'sessions', `${locallyLive.id}.json`), JSON.stringify(locallyLive));
+  utimesSync(join(root, 'sessions', `${locallyLive.id}.json`), old / 1000, old / 1000);
+  const protectedSweep = sweepStaleSessions({
+    retainOpenSessions: false,
+    isSessionLive: (id) => id === locallyLive.id,
+  });
+  assert.ok(!protectedSweep.details.some((detail) => detail.id === locallyLive.id), 'store does not reap a locally live stale session');
+  assert.notEqual(JSON.parse(readFileSync(join(root, 'sessions', `${locallyLive.id}.json`), 'utf8')).closed, true);
+  const settledSweep = sweepStaleSessions({ retainOpenSessions: false });
+  assert.ok(settledSweep.details.some((detail) => detail.id === locallyLive.id), 'store reaps the session once local work settles');
+
+  const heartbeatRace = {
+    ...known,
+    id: 'sess_heartbeat_race_keep',
+  };
+  const heartbeatRacePath = join(root, 'sessions', `${heartbeatRace.id}.json`);
+  writeFileSync(heartbeatRacePath, JSON.stringify(heartbeatRace));
+  utimesSync(heartbeatRacePath, old / 1000, old / 1000);
+  let livenessChecks = 0;
+  const heartbeatRaceSweep = sweepStaleSessions({
+    retainOpenSessions: false,
+    isSessionLive: (id) => {
+      if (id !== heartbeatRace.id) return false;
+      livenessChecks++;
+      if (livenessChecks === 3) {
+        writeFileSync(join(root, 'sessions', `${id}.hb`), '');
+      }
+      return false;
+    },
+  });
+  assert.ok(livenessChecks >= 3, 'store performs the final runtime probe before taking the close lock');
+  assert.ok(!heartbeatRaceSweep.details.some((detail) => detail.id === heartbeatRace.id), 'heartbeat landing before the locked re-stat vetoes close');
+  assert.notEqual(JSON.parse(readFileSync(heartbeatRacePath, 'utf8')).closed, true);
+
+  const retentionHeartbeat = {
+    ...known,
+    id: 'sess_retention_heartbeat_keep',
+    createdAt: Date.now() - 1_000,
+    updatedAt: Date.now() - 1_000,
+  };
+  const retentionPath = join(root, 'sessions', `${retentionHeartbeat.id}.json`);
+  writeFileSync(retentionPath, JSON.stringify(retentionHeartbeat));
+  let retentionLivenessChecks = 0;
+  const retentionSweep = sweepStaleSessions({
+    ttlMs: 60_000,
+    openMaxAgeMs: 24 * 60 * 60 * 1000,
+    openMaxCount: 0,
+    isSessionLive: (id) => {
+      if (id !== retentionHeartbeat.id) return false;
+      retentionLivenessChecks++;
+      if (retentionLivenessChecks === 2) {
+        writeFileSync(join(root, 'sessions', `${id}.hb`), '');
+      }
+      return false;
+    },
+  });
+  assert.ok(retentionLivenessChecks >= 2, 'retention performs its runtime probe before taking the delete lock');
+  assert.ok(!retentionSweep.openPrunedDetails.some((detail) => detail.id === retentionHeartbeat.id), 'commit-edge heartbeat vetoes retention hard-delete');
+  assert.ok(existsSync(retentionPath), 'heartbeating retention candidate survives');
+
+  const pendingClose = { ...known, id: 'sess_vetoed_close_pending', messages: [] };
+  const pendingDelete = { ...known, id: 'sess_vetoed_delete_pending', messages: [] };
+  for (const session of [pendingClose, pendingDelete]) {
+    writeFileSync(join(root, 'sessions', `${session.id}.json`), JSON.stringify(session));
+    saveSession({ ...session, messages: [{ role: 'user', content: 'pending save survived' }] });
+  }
+  assert.equal(markSessionClosed(pendingClose.id, 'idle-sweep', { isSessionLive: () => true }), null);
+  assert.equal(deleteSession(pendingDelete.id, { isSessionLive: () => true }), false);
+  await new Promise((resolve) => setTimeout(resolve, 250));
+  for (const session of [pendingClose, pendingDelete]) {
+    const saved = JSON.parse(readFileSync(join(root, 'sessions', `${session.id}.json`), 'utf8'));
+    assert.equal(saved.messages[0]?.content, 'pending save survived', 'veto leaves debounce persistence intact');
+    assert.notEqual(saved.closed, true);
+  }
+
+  const {
+    _clearSessionRuntime,
+    _getRuntimeEntry,
+    markSessionAskStart,
+  } = await import('../src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs');
+  const {
+    _finalizeSweptSessionRuntime,
+    _runCleanupCycle,
+  } = await import('../src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs');
+
+  const busy = {
+    ...known,
+    id: 'sess_unrelated_busy_keep',
+    status: 'running',
+  };
+  const terminalDuringBusy = {
+    ...known,
+    id: 'sess_terminal_during_busy_reap',
+  };
+  for (const session of [busy, terminalDuringBusy]) {
+    const path = join(root, 'sessions', `${session.id}.json`);
+    writeFileSync(path, JSON.stringify(session));
+    utimesSync(path, old / 1000, old / 1000);
+  }
+  markSessionAskStart(busy.id);
+  const busyEntry = _getRuntimeEntry(busy.id);
+  busyEntry.controller = new AbortController();
+  _runCleanupCycle();
+  assert.notEqual(JSON.parse(readFileSync(join(root, 'sessions', `${busy.id}.json`), 'utf8')).closed, true, 'unrelated busy runtime survives idle cleanup');
+  assert.equal(JSON.parse(readFileSync(join(root, 'sessions', `${terminalDuringBusy.id}.json`), 'utf8')).closed, true, 'terminal session is reaped while unrelated runtime is busy');
+
+  const postScanRaceId = 'sess_post_scan_active_veto';
+  markSessionAskStart(postScanRaceId);
+  const postScanEntry = _getRuntimeEntry(postScanRaceId);
+  postScanEntry.controller = new AbortController();
+  assert.equal(_finalizeSweptSessionRuntime({ id: postScanRaceId }), false, 'post-scan activity vetoes runtime cleanup');
+  assert.equal(postScanEntry.controller.signal.aborted, false, 'post-scan controller is not aborted');
+  assert.equal(_getRuntimeEntry(postScanRaceId), postScanEntry, 'post-scan runtime remains owned');
+  _clearSessionRuntime(busy.id);
+  _clearSessionRuntime(postScanRaceId);
+
   const dataDir = join(root, 'worker-index');
   mkdirSync(dataDir, { recursive: true });
   writeFileSync(join(dataDir, 'agent-workers.json'), JSON.stringify({

package/scripts/compact-file-reattach-test.mjs

@@ -0,0 +1,88 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { recallFastTrackCompactMessages, semanticCompactMessages } from '../src/runtime/agent/orchestrator/session/compact.mjs';
+import {
+  buildPostCompactFileAttachment,
+  MAX_REATTACH_FILES,
+  REATTACH_MAX_TOTAL_TOKENS,
+} from '../src/runtime/agent/orchestrator/session/compact/file-reattach.mjs';
+import { estimateTokens, sanitizeToolPairs } from '../src/runtime/agent/orchestrator/session/context-utils.mjs';
+
+const dir = mkdtempSync(join(tmpdir(), 'reattach-'));
+const fileA = join(dir, 'a.mjs'); writeFileSync(fileA, 'export const A = 1;\n'.repeat(50));
+const fileB = join(dir, 'b.mjs'); writeFileSync(fileB, 'export const B = 2;\n'.repeat(50));
+const fileHuge = join(dir, 'huge.txt'); writeFileSync(fileHuge, 'h'.repeat(600 * 1024)); // > 512KB cap
+const budgetFiles = Array.from({ length: 5 }, (_, i) => {
+  const p = join(dir, `budget-${i}.mjs`);
+  writeFileSync(p, `export const budget${i} = "${'token payload '.repeat(4000)}";\n`);
+  return p;
+});
+
+const readCall = (id, p) => ({ role: 'assistant', content: '', toolCalls: [{ id, name: 'read', arguments: JSON.stringify({ path: p }) }] });
+const toolRes = (id) => ({ role: 'tool', toolCallId: id, content: 'old cached body '.repeat(100) });
+
+function transcript() {
+  const msgs = [{ role: 'system', content: 'rules' }];
+  msgs.push({ role: 'user', content: 'fix bug in a.mjs' });
+  msgs.push(readCall('c1', fileA), toolRes('c1'));
+  msgs.push(readCall('c2', fileHuge), toolRes('c2'));
+  msgs.push(readCall('c3', join(dir, 'missing.mjs')), toolRes('c3'));
+  for (let i = 0; i < 12; i++) { msgs.push({ role: 'user', content: `iterate ${i} ` + 'pad '.repeat(300) }, { role: 'assistant', content: `ok ${i}` }); }
+  // newest turn reads fileB — must survive in tail and be skipped
+  msgs.push({ role: 'user', content: 'now check b.mjs' });
+  msgs.push(readCall('c9', fileB), toolRes('c9'));
+  msgs.push({ role: 'assistant', content: 'checked' });
+  return msgs;
+}
+
+// 1) fasttrack: A re-attached, B (in tail) skipped, huge+missing skipped
+{
+  const r = recallFastTrackCompactMessages(transcript(), 4000, { force: true, recallText: 'digest', allowEmptyRecall: true, tailTurns: 2, keepTokens: 2000, cwd: dir });
+  const ref = r.messages.find((m) => m.role === 'user' && typeof m.content === 'string' && m.content.startsWith('Reference files:'));
+  assert.ok(ref, 'fasttrack: reference-files message injected');
+  assert.ok(ref.content.includes(fileA), 'fileA re-attached');
+  assert.ok(ref.content.includes('export const A'), 'fileA fresh content present');
+  assert.ok(!ref.content.includes(fileHuge), 'huge file skipped');
+  assert.ok(!ref.content.includes('missing.mjs'), 'missing file skipped');
+  assert.ok(!ref.content.includes(fileB) || !ref.content.includes('export const B'), 'tail-surviving fileB not re-attached');
+  const refIdx = r.messages.indexOf(ref);
+  assert.equal(r.messages[refIdx + 1]?.role, 'assistant', 'ack follows reference message');
+  assert.equal(JSON.stringify(sanitizeToolPairs(r.messages)), JSON.stringify(r.messages), 'pairing valid');
+  assert.equal(r.diagnostics.fileReattached, true, 'diagnostics flag set');
+}
+// 2) semantic path with fake provider
+{
+  const provider = { name: 'fake', async send() { return { content: '## Goal\n- g\n\n## Constraints & Preferences\n- (none)\n\n## Progress\n### Done\n- d\n\n### In Progress\n- (none)\n\n### Blocked\n- (none)\n\n## Key Decisions\n- (none)\n\n## Next Steps\n- n\n\n## Critical Context\n- c\n\n## Relevant Files\n- a.mjs' }; } };
+  const r = await semanticCompactMessages(provider, transcript(), 'fake-model', 4000, { force: true, tailTurns: 1, cwd: dir });
+  const ref = r.messages.find((m) => m.role === 'user' && typeof m.content === 'string' && m.content.startsWith('Reference files:'));
+  assert.ok(ref, 'semantic: reference-files message injected');
+  assert.ok(ref.content.includes('export const A'), 'semantic: fileA fresh content');
+  assert.equal(r.diagnostics.fileReattached, true, 'semantic diagnostics flag');
+}
+// 3) no room -> no injection, still valid compact
+{
+  const r = recallFastTrackCompactMessages(transcript(), 1500, { force: true, recallText: 'digest', allowEmptyRecall: true, tailTurns: 1, keepTokens: 1200, cwd: dir });
+  assert.ok(Array.isArray(r.messages), 'tight budget compact still succeeds');
+  const ref = r.messages.find((m) => m.role === 'user' && typeof m.content === 'string' && m.content.startsWith('Reference files:'));
+  if (ref) assert.ok(r.diagnostics.finalTokens <= r.diagnostics.budgetTokens, 'reattach never exceeds budget');
+}
+// 4) env off-switch
+{
+  process.env.MIXDOG_COMPACT_FILE_REATTACH = '0';
+  const off = buildPostCompactFileAttachment([readCall('x', fileA)], [], 10000, { cwd: dir });
+  assert.equal(off, null, 'env kill-switch disables reattach');
+  delete process.env.MIXDOG_COMPACT_FILE_REATTACH;
+}
+// 5) generous context room is still bounded by the compact re-attach envelope
+{
+  const calls = budgetFiles.map((p, i) => readCall(`budget-${i}`, p));
+  const ref = buildPostCompactFileAttachment(calls, [], 50_000, { cwd: dir });
+  assert.ok(ref, 'bounded attachment should include at least one recent file');
+  assert.ok((ref.content.match(/^### /gm) || []).length <= MAX_REATTACH_FILES, 'attachment file count must stay capped');
+  assert.ok(estimateTokens(ref.content) <= REATTACH_MAX_TOTAL_TOKENS, 'attachment total tokens must stay capped');
+}
+rmSync(dir, { recursive: true, force: true });
+console.log('compact file-reattach test passed \u2713');
+

package/scripts/compact-pressure-test.mjs

@@ -1,11 +1,13 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import { createRequire } from 'node:module';
 import {
     estimateMessagesTokens,
     estimateRequestReserveTokens,
     estimateToolSchemaTokens,
     estimateTokens,
     IMAGE_VISUAL_TOKEN_ALLOWANCE,
+    providerTokenCalibration,
     summarizeContextMessages,
 } from '../src/runtime/agent/orchestrator/session/context-utils.mjs';
 import { createContextStatus } from '../src/session-runtime/context-status.mjs';
@@ -176,9 +178,10 @@ test('degenerate main budgets use a documented single-shot fallback', () => {
 
     const oneToken = { contextWindow: 1, compaction: {}, tools: [] };
     const oneTokenPolicy = policyFor(oneToken);
-    assert.ok(oneTokenPolicy.reserveTokens >= oneTokenPolicy.triggerTokens);
-    assert.equal(oneTokenPolicy.singleShot, true,
-        'the request reserve, not the one-token boundary itself, makes this degenerate');
+    assert.equal(oneTokenPolicy.reserveTokens, 0,
+        'an empty request has no synthetic fixed reserve');
+    assert.equal(oneTokenPolicy.singleShot, false,
+        'the one-token boundary alone does not require a single-shot fallback');
     assert.equal(oneTokenPolicy.triggerTokens, 1);
     assert.equal(compactTargetBudget(oneTokenPolicy), 1);
 
@@ -280,21 +283,32 @@ test('image payload bytes do not inflate live gauge or fallback compaction estim
         'estimating live context must not sanitize or remove image content');
 });
 
-test('dense ASCII, encoded, and minified payloads do not retain the prose chars/4 estimate', () => {
-    for (const text of [
-        'A1b2C3d4E5f6G7h8'.repeat(100),
-        '%7B%22path%22%3A%22very%2Flong%2Fencoded%2Fvalue%22%7D'.repeat(40),
-        JSON.stringify({ rows: Array.from({ length: 100 }, (_, i) => ({ id: i, value: `item_${i}_abcdef` })) }),
-    ]) {
-        assert.ok(estimateTokens(text) >= Math.ceil(text.length * 0.7),
-            `dense ASCII estimate must be conservative (${estimateTokens(text)}/${text.length})`);
+test('token estimates match the real o200k BPE tokenizer at the default multiplier', () => {
+    const requireForTest = createRequire(import.meta.url);
+    const { Tiktoken } = requireForTest('tiktoken/lite');
+    const o200k = requireForTest('tiktoken/encoders/o200k_base.json');
+    const enc = new Tiktoken(o200k.bpe_ranks, o200k.special_tokens, o200k.pat_str);
+    try {
+        for (const text of [
+            'A1b2C3d4E5f6G7h8'.repeat(100),
+            '%7B%22path%22%3A%22very%2Flong%2Fencoded%2Fvalue%22%7D'.repeat(40),
+            JSON.stringify({ rows: Array.from({ length: 100 }, (_, i) => ({ id: i, value: `item_${i}_abcdef` })) }),
+            Array.from({ length: 200 }, (_, i) => `{"i":${i}}`).join('\n'),
+            'plain english prose with ordinary words that merge well. '.repeat(50),
+            '한국어 컨텍스트 추정 정확도 검증 문장입니다. '.repeat(30),
+            '😀🚀🔥✅'.repeat(50),
+        ]) {
+            assert.equal(estimateTokens(text), enc.encode(text, undefined, []).length,
+                'estimateTokens must equal the real o200k token count');
+        }
+        // Repeat call exercises the hash-keyed LRU for long strings — the
+        // cached count must be identical to the fresh encode.
+        const longText = 'cached long payload '.repeat(200);
+        assert.equal(estimateTokens(longText), estimateTokens(longText),
+            'cached long-string counts must be stable');
+    } finally {
+        enc.free();
     }
-    const shortJsonl = Array.from({ length: 200 }, (_, i) => `{"i":${i}}`).join('\n');
-    assert.ok(estimateTokens(shortJsonl) >= shortJsonl.replace(/\s/g, '').length * 0.7,
-        'short JSONL records must receive the structured multiline floor');
-    const spacedShortIdentifiers = 'A1b2C3d4E5 F6G7H8I9J0 '.repeat(100);
-    assert.ok(estimateTokens(spacedShortIdentifiers) >= spacedShortIdentifiers.length * 0.7,
-        'space-separated encoded identifiers below the long-run threshold must receive the dense floor');
 });
 
 test('assistantBlocks and reasoningItems count provider-visible data but not image bytes', () => {
@@ -508,7 +522,9 @@ test('provider baselines fingerprint actual sendTools and reject provider, model
         const policy = { ...resolveWorkerCompactPolicy(session, session.tools), reserveTokens: 0 };
         return {
             pressure: resolveCompactionPressureTokens(estimateMessagesTokens(messages), policy, { messages, sessionRef: session }),
-            fallback: estimateMessagesTokens(messages),
+            // The estimate fallback includes provider billing calibration, so
+            // compare against the same pressure computation without a baseline.
+            fallback: resolveCompactionPressureTokens(estimateMessagesTokens(messages), policy, { messages, sessionRef: null }),
         };
     };
     for (const mutate of [
@@ -591,7 +607,8 @@ test('provider callback usage counts assistant output once and estimates only la
     const wholeEstimate = estimateMessagesTokens(messages);
     assert.ok(wholeEstimate < policy.triggerTokens, 'fixture must reproduce local estimator undercount');
     const pressure = compactionTelemetryPressureTokens(wholeEstimate, policy, { messages, sessionRef: session });
-    const expectedPressure = 94_800 + estimateMessagesTokens([laterToolResult]);
+    const expectedPressure = 94_800
+        + Math.round(estimateMessagesTokens([laterToolResult]) * providerTokenCalibration('anthropic'));
     assert.equal(pressure, expectedPressure, 'assistant output/reasoning must stay in actual usage, not be estimated again');
     assert.ok(pressure >= 95_000, `actual usage plus later tool growth should cross trigger, got ${pressure}`);
     assert.equal(shouldCompactForSession(wholeEstimate, policy, {
@@ -619,7 +636,8 @@ test('thinking-only continuation without assistant replay excludes provider outp
 
     const wholeEstimate = estimateMessagesTokens(messages);
     const pressure = compactionTelemetryPressureTokens(wholeEstimate, policy, { messages, sessionRef: session });
-    const expectedPressure = 74_000 + estimateMessagesTokens([nudge]);
+    const expectedPressure = 74_000
+        + Math.round(estimateMessagesTokens([nudge]) * providerTokenCalibration('anthropic'));
     assert.equal(pressure, expectedPressure, 'unreplayed output must be removed while the later nudge is estimated');
     assert.equal(shouldCompactForSession(wholeEstimate, policy, {
         messages,
@@ -639,22 +657,25 @@ test('OpenAI OAuth WS warmup remains billed but does not double the main context
     const usage = _combineUsageWithWarmup(main, main, { separateMainContext: true });
     assert.equal(usage.inputTokens, 335_270, 'billing usage must retain warmup plus main input');
     const { initProviders, getProvider } = await import('../src/runtime/agent/orchestrator/providers/registry.mjs');
-    const { createSession, askSession, getSession } = await import('../src/runtime/agent/orchestrator/session/manager.mjs');
+    const { createSession, askSession, getSession, closeSession } = await import('../src/runtime/agent/orchestrator/session/manager.mjs');
     await initProviders({ 'openai-oauth': { enabled: true } });
     const provider = getProvider('openai-oauth');
     const originalSend = provider.send;
     const compactEvents = [];
+    let regressionSessionId = '';
     provider.send = async () => ({ content: 'done', usage });
     try {
         const session = createSession({
             provider: 'openai-oauth',
             model: 'warmup-context-regression',
+            owner: 'test',
             tools: [],
             cwd: process.cwd(),
             skipAgentRules: true,
             skipSkills: true,
             compaction: { auto: true },
         });
+        regressionSessionId = session.id;
         session.contextWindow = 272_000;
         session.rawContextWindow = 272_000;
         await askSession(session.id, 'large OAuth WebSocket request', null, null, process.cwd(), null, {
@@ -680,6 +701,9 @@ test('OpenAI OAuth WS warmup remains billed but does not double the main context
         }), false, 'main-request pressure must not trigger compaction');
         assert.equal(compactEvents.length, 0, 'warmup must not cause an auto-compaction');
     } finally {
+        if (regressionSessionId) {
+            closeSession(regressionSessionId, 'warmup-context-regression-cleanup', { tombstone: true });
+        }
         provider.send = originalSend;
     }
 });

package/scripts/compact-recall-digest-test.mjs

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import {
+  compactDigestRows,
+  renderEntryLines,
+} from '../src/runtime/memory/lib/recall-format.mjs';
+import { createQueryHandlers } from '../src/runtime/memory/lib/query-handlers.mjs';
+
+const longPlan = 'cache recent session snapshots and display immediately while the runtime initializes in the background without a blocking veil';
+const nearPlan = `${longPlan} and keep the newest click authoritative`;
+const rows = [
+  { id: 10, ts: 300, role: 'assistant', content: nearPlan, is_root: 0, chunk_root: null },
+  { id: 9, ts: 290, role: 'assistant', content: longPlan, is_root: 0, chunk_root: null },
+  { id: 8, ts: 280, role: 'assistant', content: longPlan, is_root: 0, chunk_root: null },
+  { id: 7, ts: 270, role: 'user', content: '오케이', is_root: 0, chunk_root: null },
+  { id: 6, ts: 260, role: 'user', content: '오케이', is_root: 0, chunk_root: null },
+  { id: 5, ts: 250, role: 'assistant', content: 'distinct completed result', is_root: 0, chunk_root: null },
+];
+
+const compact = compactDigestRows(rows, 30);
+assert.equal(compact.filter((row) => row.content === '오케이').length, 1, 'short exact duplicates collapse');
+assert.equal(compact.filter((row) => row.content === longPlan).length, 0, 'older near-duplicate plans collapse');
+assert.ok(compact.some((row) => row.content === nearPlan), 'newest near-duplicate plan is retained');
+assert.ok(compact.some((row) => row.content === 'distinct completed result'), 'distinct state survives');
+
+const normalText = renderEntryLines(compact);
+assert.match(normalText, /\[pending\]/, 'normal recall keeps raw-row pipeline status');
+const digestText = renderEntryLines(compact, { pendingMarks: false });
+assert.doesNotMatch(digestText, /\[pending\]/, 'compact digest omits misleading pipeline status');
+
+const fakeDb = {
+  async query(sql) {
+    if (/SELECT source_turn t/.test(sql)) return { rows: [] };
+    if (/id <> ALL/.test(sql)) return { rows: [] };
+    if (/FROM entries/.test(sql)) return { rows };
+    return { rows: [] };
+  },
+};
+const { handleSearch } = createQueryHandlers({
+  getDb: () => fakeDb,
+  log: () => {},
+  resolveProjectScope: () => null,
+  embeddingWarmupCanStart: () => false,
+  getBootTimestamp: () => 0,
+  getTraceDb: () => null,
+});
+const integrated = await handleSearch({
+  sessionId: 'compact-digest-session',
+  limit: 30,
+  includeMembers: true,
+  includeRaw: true,
+  compactDigest: true,
+});
+assert.doesNotMatch(integrated.text, /\[pending\]/, 'compact search path suppresses pipeline status');
+assert.equal((integrated.text.match(/오케이/g) || []).length, 1, 'compact search path removes exact duplicates');
+assert.equal((integrated.text.match(/cache recent session snapshots/g) || []).length, 1, 'compact search path removes near duplicates');
+
+console.log('compact recall digest test passed \u2713');

package/scripts/compact-smoke.mjs

@@ -1,7 +1,8 @@
 #!/usr/bin/env node
+import { createRequire } from 'node:module';
 import { semanticCompactMessages, recallFastTrackCompactMessages, SUMMARY_PREFIX, COMPACT_TYPE_SEMANTIC, COMPACT_TYPE_RECALL_FASTTRACK, normalizeCompactType } from '../src/runtime/agent/orchestrator/session/compact.mjs';
 import { agentLoop } from '../src/runtime/agent/orchestrator/session/loop.mjs';
-import { estimateMessagesTokens, estimateToolSchemaTokens } from '../src/runtime/agent/orchestrator/session/context-utils.mjs';
+import { estimateMessagesTokens, estimateToolSchemaTokens, providerTokenCalibration } from '../src/runtime/agent/orchestrator/session/context-utils.mjs';
 import { runSessionCompaction } from '../src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs';
 import { autoCompactWindowForRoute, summarizeGatewayUsage } from '../src/vendor/statusline/src/gateway/route-meta.mjs';
 
@@ -928,58 +929,53 @@ assert(!tinyCapUserContent.startsWith('[...'), 'tiny cap recall must not use a p
 // ---------------------------------------------------------------------------
 // Conservative Unicode-aware estimator (strict-fit hardening).
 //
-// The estimator is a SAFETY device, not exact tokenizer parity: it must never
-// undercount Korean/CJK/emoji/JSON the way the legacy chars/4 heuristic did, or
-// a transcript that "fits" locally can still overflow the real model window.
-// These probes prove the conservative estimate is meaningfully higher than a
-// chars/4 style count for non-ASCII heavy content while staying near chars/4
-// for plain ASCII.
-function chars4(text) {
-  return Math.ceil(String(text ?? '').length / 4);
+// The estimator now encodes the provider-visible projection with the real
+// o200k_base BPE (tiktoken), so local estimates equal actual tokenizer counts
+// for every script (Korean/CJK/emoji/ASCII alike). Provider-specific billing
+// deltas (e.g. Anthropic ~1.7x o200k) are applied separately via
+// providerTokenCalibration at the pressure/gauge aggregation boundary.
+const requireForTokenizer = createRequire(import.meta.url);
+const { Tiktoken: SmokeTiktoken } = requireForTokenizer('tiktoken/lite');
+const smokeO200k = requireForTokenizer('tiktoken/encoders/o200k_base.json');
+const smokeEncoder = new SmokeTiktoken(smokeO200k.bpe_ranks, smokeO200k.special_tokens, smokeO200k.pat_str);
+const bpeCount = (text) => smokeEncoder.encode(String(text), undefined, []).length;
+
+for (const [label, probe] of [
+  ['korean', '\uD55C\uAD6D\uC5B4 \uCEF4\uD329\uC158 \uACBD\uACC4 \uD14C\uC2A4\uD2B8 '.repeat(200)],
+  ['cjk', '上下文压缩边界测试令牌预算'.repeat(200)],
+  ['emoji', '😀🚀🔥✅'.repeat(200)],
+  ['ascii', 'plain ascii sentence with normal words. '.repeat(200)],
+]) {
+  const est = estimateMessagesTokens([{ role: 'user', content: probe }]);
+  const expected = bpeCount(probe) + 4;
+  assert(
+    est === expected,
+    `${label} message estimate must equal the real o200k count plus framing (est=${est}, expected=${expected})`,
+  );
 }
 
-const koHeavy = '\uD55C\uAD6D\uC5B4 \uCEF4\uD329\uC158 \uACBD\uACC4 \uD14C\uC2A4\uD2B8 '.repeat(200);
-const koEst = estimateMessagesTokens([{ role: 'user', content: koHeavy }]);
-assert(
-  koEst > chars4(koHeavy) * 2,
-  `Korean-heavy content must estimate well above chars/4 (est=${koEst}, chars/4=${chars4(koHeavy)})`,
-);
-
-const cjkHeavy = '上下文压缩边界测试令牌预算'.repeat(200);
-const cjkEst = estimateMessagesTokens([{ role: 'user', content: cjkHeavy }]);
-assert(
-  cjkEst > chars4(cjkHeavy) * 1.5,
-  `CJK-heavy content must estimate above chars/4 (est=${cjkEst}, chars/4=${chars4(cjkHeavy)})`,
-);
-
-const emojiHeavy = '😀🚀🔥✅'.repeat(200);
-const emojiEst = estimateMessagesTokens([{ role: 'user', content: emojiHeavy }]);
-assert(
-  emojiEst > chars4(emojiHeavy) * 2,
-  `Emoji-heavy content must estimate well above chars/4 (est=${emojiEst}, chars/4=${chars4(emojiHeavy)})`,
-);
-
-// Plain ASCII must stay close to the chars/4 floor (within the safety
-// multiplier band) — the estimator overcounts non-ASCII, not everything.
-const asciiPlain = 'plain ascii sentence with normal words. '.repeat(200);
-const asciiEst = estimateMessagesTokens([{ role: 'user', content: asciiPlain }]);
+// Provider calibration reconciles o200k counts with actual billing.
 assert(
-  asciiEst >= chars4(asciiPlain) && asciiEst <= chars4(asciiPlain) * 1.5,
-  `ASCII must stay near chars/4 (est=${asciiEst}, chars/4=${chars4(asciiPlain)})`,
+  providerTokenCalibration('anthropic-oauth') > 1.5 && providerTokenCalibration('anthropic-oauth') < 2,
+  'anthropic calibration must reflect the measured ~1.7x billed/o200k ratio',
 );
+assert(providerTokenCalibration('openai-oauth') === 1.0, 'openai calibration must stay neutral');
+assert(providerTokenCalibration(undefined) === 1.0, 'unknown provider calibration must stay neutral');
 
 // Tool schemas are serialized into the request body, so their estimate must
-// also use the conservative estimator (>= chars/4 of the serialized JSON).
+// track the real tokenizer over the wire-shape serialization.
 const toolSchema = [{
   name: 'apply_patch',
   description: '\uD328\uCE58\uB97C \uD30C\uC77C\uC5D0 \uC801\uC6A9 (\uD55C\uAD6D\uC5B4 \uC124\uBA85) — applies a patch',
   parameters: { type: 'object', properties: { patch: { type: 'string', description: '\uD328\uCE58 \uBCF8\uBB38 \uD14D\uC2A4\uD2B8' } } },
 }];
 const toolSchemaEst = estimateToolSchemaTokens(toolSchema);
+const toolWireJson = JSON.stringify(toolSchema.map((t) => ({ name: t.name, description: t.description, input_schema: t.parameters })));
 assert(
-  toolSchemaEst >= chars4(JSON.stringify(toolSchema)),
-  `tool schema estimate must be >= chars/4 of serialized JSON (est=${toolSchemaEst})`,
+  toolSchemaEst === bpeCount(toolWireJson),
+  `tool schema estimate must equal the o200k count of the serialized wire JSON (est=${toolSchemaEst}, expected=${bpeCount(toolWireJson)})`,
 );
+smokeEncoder.free();
 
 // Strict-fit smoke: a Korean/CJK-heavy newest turn far larger than the preserve
 // budget must be summarized/truncated safely — never preserved verbatim — and