mixdog 0.9.40 → 0.9.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mixdog",
-  "version": "0.9.40",
+  "version": "0.9.41",
   "private": false,
   "type": "module",
   "description": "Standalone mixdog coding-agent CLI/TUI workspace.",
@@ -57,6 +57,7 @@
     "test:shellhardening": "node --test scripts/shell-hardening-test.mjs",
     "test:placeholder": "node --test scripts/compacted-placeholder-scrub-test.mjs",
     "test:providers": "node --test scripts/provider-toolcall-test.mjs",
+    "test:anthropic-oauth-race": "node --test scripts/anthropic-oauth-refresh-race-test.mjs",
     "test:atomiclock": "node --test scripts/atomic-lock-tryonce-test.mjs",
     "test:memory-routing": "node --test scripts/memory-cycle-routing-test.mjs scripts/memory-rule-contract-test.mjs",
     "test:tui-queue": "node --test scripts/submit-commandbusy-race-test.mjs scripts/steering-drain-buckets-test.mjs scripts/abort-recovery-test.mjs scripts/execution-pending-resume-kick-test.mjs scripts/execution-resume-esc-integration-test.mjs",

package/scripts/anthropic-oauth-refresh-race-test.mjs

@@ -0,0 +1,338 @@
+import test, { after } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+    existsSync,
+    mkdirSync,
+    mkdtempSync,
+    readFileSync,
+    readdirSync,
+    rmSync,
+    writeFileSync,
+} from 'node:fs';
+import { spawn } from 'node:child_process';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const root = mkdtempSync(join(tmpdir(), 'mixdog-anthropic-race-'));
+const credentialsPath = join(root, 'anthropic-oauth-credentials.json');
+process.env.MIXDOG_DATA_DIR = root;
+process.env.ANTHROPIC_OAUTH_CREDENTIALS_PATH = credentialsPath;
+
+const credentialsModule = await import(
+    '../src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs'
+);
+const { AnthropicOAuthProvider } = await import(
+    '../src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs'
+);
+const {
+    _saveCredentialsFile,
+    loadCredentials,
+    loadCredentialsFromPath,
+    preflightAnthropicOAuthCredentials,
+    refreshOAuthCredentials,
+} = credentialsModule;
+
+after(() => {
+    delete process.env.MIXDOG_ANTHROPIC_OAUTH_REFRESH_DISABLED;
+    rmSync(root, { recursive: true, force: true });
+});
+
+function writeCredentials({ accessToken, refreshToken, expiresAt }, path = credentialsPath) {
+    _saveCredentialsFile(path, {
+        claudeAiOauth: {
+            accessToken,
+            refreshToken,
+            expiresAt,
+            scopes: ['user:inference'],
+        },
+    });
+}
+
+test('parallel host preflights refresh once and snapshot the leased generation', async () => {
+    writeCredentials({
+        accessToken: 'fixture-access-old',
+        refreshToken: 'fixture-refresh-old',
+        expiresAt: Date.now() + 1_000,
+    });
+    let refreshCalls = 0;
+    const refreshFn = async (current) => {
+        refreshCalls += 1;
+        await new Promise((resolve) => setTimeout(resolve, 30));
+        const raw = JSON.parse(readFileSync(current.path, 'utf8'));
+        raw.claudeAiOauth.accessToken = 'fixture-access-new';
+        raw.claudeAiOauth.refreshToken = 'fixture-refresh-new';
+        raw.claudeAiOauth.expiresAt = Date.now() + 10 * 60_000;
+        _saveCredentialsFile(current.path, raw);
+        return loadCredentials();
+    };
+
+    const snapshots = Array.from(
+        { length: 12 },
+        (_, index) => join(root, `container-${index}.json`),
+    );
+    await Promise.all(snapshots.map((snapshotPath) => (
+        preflightAnthropicOAuthCredentials({
+            minimumValidityMs: 5 * 60_000,
+            snapshotPath,
+            refreshFn,
+        })
+    )));
+
+    assert.equal(refreshCalls, 1);
+    for (const snapshotPath of snapshots) {
+        const oauth = JSON.parse(readFileSync(snapshotPath, 'utf8')).claudeAiOauth;
+        assert.equal(oauth.accessToken, 'fixture-access-new');
+        assert.equal(Object.hasOwn(oauth, 'refreshToken'), false);
+        assert.equal(Object.hasOwn(oauth, 'refresh_token'), false);
+    }
+});
+
+test('explicit credential path is pinned over a newer default candidate', async () => {
+    const explicitPath = join(root, 'explicit', 'credentials.json');
+    mkdirSync(join(root, 'explicit'), { recursive: true });
+    writeCredentials({
+        accessToken: 'fixture-default-newer',
+        refreshToken: 'fixture-default-refresh',
+        expiresAt: Date.now() + 60 * 60_000,
+    });
+    writeCredentials({
+        accessToken: 'fixture-explicit',
+        refreshToken: 'fixture-explicit-refresh',
+        expiresAt: Date.now() + 20 * 60_000,
+    }, explicitPath);
+    process.env.ANTHROPIC_OAUTH_CREDENTIALS_PATH = explicitPath;
+    const snapshotPath = join(root, 'explicit-snapshot.json');
+    try {
+        assert.equal(loadCredentials().path, explicitPath);
+        assert.equal(loadCredentials().accessToken, 'fixture-explicit');
+        await preflightAnthropicOAuthCredentials({
+            credentialsPath: explicitPath,
+            minimumValidityMs: 5 * 60_000,
+            snapshotPath,
+        });
+        const snapshot = loadCredentialsFromPath(snapshotPath);
+        assert.equal(snapshot.accessToken, 'fixture-explicit');
+        assert.equal(snapshot.refreshToken, null);
+        assert.equal(
+            loadCredentialsFromPath(credentialsPath).accessToken,
+            'fixture-default-newer',
+        );
+    } finally {
+        process.env.ANTHROPIC_OAUTH_CREDENTIALS_PATH = credentialsPath;
+    }
+});
+
+test('10 contending host processes share one token exchange lock', async () => {
+    const processCredentials = join(root, 'process-credentials.json');
+    const exchangeDir = join(root, 'exchange-owners');
+    const startPath = join(root, 'children-start');
+    const readyDir = join(root, 'children-ready');
+    const snapshotDir = join(root, 'child-snapshots');
+    mkdirSync(readyDir);
+    mkdirSync(snapshotDir);
+    mkdirSync(exchangeDir);
+    writeCredentials({
+        accessToken: 'fixture-process-old',
+        refreshToken: 'fixture-process-refresh-old',
+        expiresAt: Date.now() + 1_000,
+    }, processCredentials);
+
+    const moduleUrl = new URL(
+        '../src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs',
+        import.meta.url,
+    ).href;
+    const childSource = `
+import { existsSync, writeFileSync } from 'node:fs';
+const mod = await import(${JSON.stringify(moduleUrl)});
+const stale = mod.loadCredentials();
+writeFileSync(process.env.READY_PATH, 'ready');
+while (!existsSync(process.env.START_PATH)) {
+  await new Promise((resolve) => setTimeout(resolve, 10));
+}
+globalThis.fetch = async () => {
+  const owner = String(process.pid);
+  writeFileSync(process.env.EXCHANGE_DIR + '/' + owner + '.exchange', owner, { flag: 'wx' });
+  await new Promise((resolve) => setTimeout(resolve, 75));
+  return {
+    ok: true,
+    status: 200,
+    async text() {
+      return JSON.stringify({
+        access_token: 'fixture-process-new-' + owner,
+        refresh_token: 'fixture-process-refresh-new-' + owner,
+        expires_in: 600
+      });
+    }
+  };
+};
+if (process.env.CHILD_MODE === 'preflight') {
+  await mod.preflightAnthropicOAuthCredentials({
+    credentialsPath: process.env.ANTHROPIC_OAUTH_CREDENTIALS_PATH,
+    minimumValidityMs: 300000,
+    snapshotPath: process.env.SNAPSHOT_PATH
+  });
+} else {
+  await mod.refreshOAuthCredentials(stale);
+}
+`;
+
+    const exits = [];
+    for (let index = 0; index < 10; index += 1) {
+        const child = spawn(
+            process.execPath,
+            ['--input-type=module', '--eval', childSource],
+            {
+                env: {
+                    ...process.env,
+                    MIXDOG_DATA_DIR: root,
+                    ANTHROPIC_OAUTH_CREDENTIALS_PATH: processCredentials,
+                    READY_PATH: join(readyDir, String(index)),
+                    START_PATH: startPath,
+                    EXCHANGE_DIR: exchangeDir,
+                    CHILD_MODE: index % 2 === 0 ? 'preflight' : 'refresh',
+                    SNAPSHOT_PATH: join(snapshotDir, `${index}.json`),
+                },
+                stdio: ['ignore', 'ignore', 'pipe'],
+            },
+        );
+        exits.push(new Promise((resolveExit, rejectExit) => {
+            let stderr = '';
+            child.stderr.setEncoding('utf8');
+            child.stderr.on('data', (chunk) => { stderr += chunk; });
+            child.on('error', rejectExit);
+            child.on('exit', (code) => {
+                if (code === 0) resolveExit();
+                else rejectExit(new Error(`contention child ${index} exited ${code}: ${stderr}`));
+            });
+        }));
+    }
+
+    const readyDeadline = Date.now() + 30_000;
+    while (
+        Array.from({ length: 10 }, (_, index) => existsSync(join(readyDir, String(index))))
+            .some((ready) => !ready)
+    ) {
+        if (Date.now() >= readyDeadline) throw new Error('contention children did not become ready');
+        await new Promise((resolveWait) => setTimeout(resolveWait, 20));
+    }
+    writeFileSync(startPath, 'start');
+    await Promise.all(exits);
+
+    const exchangeMarkers = readdirSync(exchangeDir)
+        .filter((name) => name.endsWith('.exchange'));
+    assert.equal(exchangeMarkers.length, 1);
+    const exchangeOwner = exchangeMarkers[0].slice(0, -'.exchange'.length);
+    assert.equal(
+        readFileSync(join(exchangeDir, exchangeMarkers[0]), 'utf8'),
+        exchangeOwner,
+    );
+    assert.equal(
+        loadCredentialsFromPath(processCredentials).accessToken,
+        `fixture-process-new-${exchangeOwner}`,
+    );
+    for (let index = 0; index < 10; index += 2) {
+        const oauth = JSON.parse(
+            readFileSync(join(snapshotDir, `${index}.json`), 'utf8'),
+        ).claudeAiOauth;
+        assert.equal(oauth.accessToken, `fixture-process-new-${exchangeOwner}`);
+        assert.equal(Object.hasOwn(oauth, 'refreshToken'), false);
+    }
+});
+
+test('host preflight fails clearly when refresh cannot satisfy the lease', async () => {
+    writeCredentials({
+        accessToken: 'fixture-access-short-old',
+        refreshToken: 'fixture-refresh-short-old',
+        expiresAt: Date.now() + 1_000,
+    });
+    const snapshotPath = join(root, 'short-lease-container.json');
+    const refreshFn = async (current) => {
+        const raw = JSON.parse(readFileSync(current.path, 'utf8'));
+        raw.claudeAiOauth.accessToken = 'fixture-access-short-new';
+        raw.claudeAiOauth.refreshToken = 'fixture-refresh-short-new';
+        raw.claudeAiOauth.expiresAt = Date.now() + 30_000;
+        _saveCredentialsFile(current.path, raw);
+        return loadCredentials();
+    };
+
+    await assert.rejects(
+        preflightAnthropicOAuthCredentials({
+            minimumValidityMs: 60_000,
+            snapshotPath,
+            refreshFn,
+        }),
+        /cannot satisfy the 60s credential lease/,
+    );
+});
+
+test('expiring, forced, and 401 containers cause zero token exchanges', async () => {
+    writeCredentials({
+        accessToken: 'fixture-access-expiring',
+        refreshToken: 'fixture-refresh-must-not-be-used',
+        expiresAt: Date.now() + 1_000,
+    });
+    process.env.MIXDOG_ANTHROPIC_OAUTH_REFRESH_DISABLED = '1';
+    let fetchCalls = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => {
+        fetchCalls += 1;
+        throw new Error('refresh transport must not run');
+    };
+    try {
+        const providers = Array.from({ length: 12 }, () => {
+            const provider = Object.create(AnthropicOAuthProvider.prototype);
+            provider.credentials = loadCredentials();
+            provider.config = {};
+            return provider;
+        });
+        const results = await Promise.allSettled(
+            providers.map((provider) => provider.ensureAuth()),
+        );
+        assert.equal(fetchCalls, 0);
+        assert.ok(results.every((result) => (
+            result.status === 'rejected'
+            && /refresh is disabled.*Host credential preflight/s.test(result.reason.message)
+        )));
+        await assert.rejects(
+            refreshOAuthCredentials(loadCredentials()),
+            /host credential preflight must provide a fresh snapshot/,
+        );
+        writeCredentials({
+            accessToken: 'fixture-access-valid',
+            expiresAt: Date.now() + 10 * 60_000,
+        });
+        const forcedProvider = Object.create(AnthropicOAuthProvider.prototype);
+        forcedProvider.credentials = loadCredentials();
+        forcedProvider.config = {};
+        forcedProvider.fastModeBetaHeaderLatched = false;
+        await assert.rejects(
+            forcedProvider.ensureAuth({ forceRefresh: true, reason: 'test' }),
+            /refresh is disabled.*Host credential preflight/s,
+        );
+
+        let apiRequests = 0;
+        await assert.rejects(
+            forcedProvider.send([], 'claude-sonnet-4-5', [], {
+                _doRequestFn: async () => {
+                    apiRequests += 1;
+                    return {
+                        response: {
+                            status: 401,
+                            ok: false,
+                            headers: new Map(),
+                            async text() { return ''; },
+                        },
+                        controller: null,
+                        cancelHandler: null,
+                    };
+                },
+            }),
+            /refresh is disabled.*Host credential preflight/s,
+        );
+        assert.equal(apiRequests, 1);
+        assert.equal(fetchCalls, 0);
+    } finally {
+        globalThis.fetch = originalFetch;
+        delete process.env.MIXDOG_ANTHROPIC_OAUTH_REFRESH_DISABLED;
+    }
+});

package/scripts/compact-pressure-test.mjs

@@ -0,0 +1,128 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { estimateMessagesTokens } from '../src/runtime/agent/orchestrator/session/context-utils.mjs';
+import {
+    compactionTelemetryPressureTokens,
+    recordProviderContextBaseline,
+    rememberCompactTelemetry,
+    resolveWorkerCompactPolicy,
+    shouldCompactForSession,
+} from '../src/runtime/agent/orchestrator/session/loop/compact-policy.mjs';
+
+function policyFor(session) {
+    return resolveWorkerCompactPolicy(session, []);
+}
+
+test('Lead/main auto-compaction triggers at 95% of the effective boundary', () => {
+    const leadPolicy = policyFor({ contextWindow: 100_000, compaction: {} });
+    assert.equal(leadPolicy.triggerTokens, 95_000);
+    assert.equal(leadPolicy.bufferTokens, 5_000);
+
+    const noReserve = { ...leadPolicy, reserveTokens: 0 };
+    assert.equal(shouldCompactForSession(94_999, noReserve), false);
+    assert.equal(shouldCompactForSession(95_000, noReserve), true);
+
+    const agentPolicy = policyFor({ owner: 'agent', contextWindow: 100_000, compaction: {} });
+    assert.equal(agentPolicy.triggerTokens, 90_000, 'agent default 10% headroom must remain unchanged');
+});
+
+test('provider callback usage counts assistant output once and estimates only later tool results', () => {
+    const session = { provider: 'anthropic', contextWindow: 100_000, compaction: {} };
+    const policy = { ...policyFor(session), reserveTokens: 0 };
+    const messages = [{ role: 'user', content: 'production-shaped tool request' }];
+    const onProviderUsage = d => recordProviderContextBaseline(session, messages, {
+        inputTokens: d.deltaInput,
+        outputTokens: d.deltaOutput,
+        promptTokens: d.deltaPrompt,
+        cachedTokens: d.deltaCachedRead,
+        cacheWriteTokens: d.deltaCacheWrite,
+    }, { boundary: 'request' });
+    assert.equal(onProviderUsage({
+        source: 'provider_send',
+        deltaInput: 5_000,
+        deltaOutput: 800,
+        deltaPrompt: 0,
+        deltaCachedRead: 88_000,
+        deltaCacheWrite: 1_000,
+    }), true);
+    messages.push({
+        role: 'assistant',
+        content: 'Calling the requested tool.',
+        reasoningItems: [{ type: 'reasoning', encrypted_content: 'opaque-provider-reasoning' }],
+        toolCalls: [{ id: 'call_1', name: 'read', arguments: '{"path":"large.txt"}' }],
+    });
+    const laterToolResult = { role: 'tool', toolCallId: 'call_1', content: 'x'.repeat(4_000) };
+    messages.push(laterToolResult);
+
+    const wholeEstimate = estimateMessagesTokens(messages);
+    assert.ok(wholeEstimate < policy.triggerTokens, 'fixture must reproduce local estimator undercount');
+    const pressure = compactionTelemetryPressureTokens(wholeEstimate, policy, { messages, sessionRef: session });
+    const expectedPressure = 94_800 + estimateMessagesTokens([laterToolResult]);
+    assert.equal(pressure, expectedPressure, 'assistant output/reasoning must stay in actual usage, not be estimated again');
+    assert.ok(pressure >= 95_000, `actual usage plus later tool growth should cross trigger, got ${pressure}`);
+    assert.equal(shouldCompactForSession(wholeEstimate, policy, {
+        messages,
+        sessionRef: session,
+        pressureTokens: pressure,
+    }), true);
+});
+
+test('thinking-only continuation without assistant replay excludes provider output and estimates the nudge', () => {
+    const session = { provider: 'anthropic', contextWindow: 100_000, compaction: {} };
+    const policy = { ...policyFor(session), reserveTokens: 0 };
+    const messages = [{ role: 'user', content: 'request that returns thinking but no replayable assistant message' }];
+    recordProviderContextBaseline(session, messages, {
+        inputTokens: 5_000,
+        outputTokens: 2_000,
+        cachedTokens: 88_000,
+        cacheWriteTokens: 1_000,
+    }, { boundary: 'request' });
+    const nudge = {
+        role: 'user',
+        content: '[mixdog-runtime] Previous response was empty. Continue with a final answer or tool call.',
+    };
+    messages.push(nudge);
+
+    const wholeEstimate = estimateMessagesTokens(messages);
+    const pressure = compactionTelemetryPressureTokens(wholeEstimate, policy, { messages, sessionRef: session });
+    const expectedPressure = 94_000 + estimateMessagesTokens([nudge]);
+    assert.equal(pressure, expectedPressure, 'unreplayed output must be removed while the later nudge is estimated');
+    assert.equal(shouldCompactForSession(wholeEstimate, policy, {
+        messages,
+        sessionRef: session,
+        pressureTokens: pressure,
+    }), false, 'unreplayed thinking output must not cause an early compact');
+});
+
+test('successful compact invalidates stale usage and cannot immediately compact again', () => {
+    const session = { provider: 'openai', contextWindow: 100_000, compaction: {} };
+    const policy = { ...policyFor(session), reserveTokens: 0 };
+    const before = [{ role: 'user', content: 'old context' }];
+    recordProviderContextBaseline(session, before, { inputTokens: 99_000, outputTokens: 500 });
+    assert.equal(shouldCompactForSession(estimateMessagesTokens(before), policy, {
+        messages: before,
+        sessionRef: session,
+    }), true);
+
+    rememberCompactTelemetry(session, policy, {
+        compactChanged: true,
+        beforeTokens: 99_500,
+        afterTokens: 20,
+        pressureTokens: 99_500,
+    });
+    const compacted = [{ role: 'user', content: 'short summary' }];
+    const compactedEstimate = estimateMessagesTokens(compacted);
+    assert.equal(session.contextPressureBaselineTokens, null);
+    assert.equal(session.lastContextTokensStaleAfterCompact, true);
+    assert.equal(shouldCompactForSession(compactedEstimate, policy, {
+        messages: compacted,
+        sessionRef: session,
+    }), false, 'stale pre-compact usage must not trigger a consecutive compact');
+
+    recordProviderContextBaseline(session, compacted, { inputTokens: 10_000, outputTokens: 100 });
+    assert.equal(session.lastContextTokensStaleAfterCompact, false);
+    assert.equal(shouldCompactForSession(compactedEstimate, policy, {
+        messages: compacted,
+        sessionRef: session,
+    }), false, 'fresh post-compact usage may be reused safely');
+});

package/scripts/compact-trigger-migration-smoke.mjs

@@ -55,14 +55,14 @@ function assert(condition, message) {
     `sub-boundary explicit limit should be preserved, got ${meta.autoCompactTokenLimit}`);
 }
 
-// 3) compactTriggerForSession: main/user (non-agent) recall-fasttrack compacts
-//    on the boundary itself (100%, no early-trigger buffer); a truly-explicit
+// 3) compactTriggerForSession: main/user (non-agent) recall-fasttrack keeps 5%
+//    headroom (95% trigger); a truly-explicit
 //    sub-boundary autoCompactTokenLimit still wins. Agent-owned semantic
 //    sessions keep the default 10% buffer (90%).
 {
   const boundary = 200000;
   const legacy = compactTriggerForSession({ autoCompactTokenLimit: boundary, compaction: {} }, boundary);
-  assert(legacy === 200000, `main/user boundary-equal limit should yield 100% trigger 200000, got ${legacy}`);
+  assert(legacy === 190000, `main/user boundary-equal limit should yield 95% trigger 190000, got ${legacy}`);
   const explicit = compactTriggerForSession({ autoCompactTokenLimit: 150000, compaction: {} }, boundary);
   assert(explicit === 150000, `sub-boundary explicit limit should be the trigger, got ${explicit}`);
   const agent = compactTriggerForSession({ owner: 'agent', autoCompactTokenLimit: boundary, compaction: {} }, boundary);
@@ -72,19 +72,19 @@ function assert(condition, message) {
 }
 
 // 4) A configured bufferPercent flows into the trigger for agent-owned sessions;
-//    main/user sessions ignore the buffer and stay on the boundary (100%).
+//    main/user sessions use their fixed 5% headroom.
 {
   const boundary = 200000;
   const agentTrigger = compactTriggerForSession({ owner: 'agent', compaction: { bufferPercent: 5 } }, boundary);
   assert(agentTrigger === 190000, `agent bufferPercent 5 should yield trigger 190000, got ${agentTrigger}`);
   const userTrigger = compactTriggerForSession({ compaction: { bufferPercent: 5 } }, boundary);
-  assert(userTrigger === 200000, `main/user should ignore bufferPercent and stay at 100% 200000, got ${userTrigger}`);
+  assert(userTrigger === 190000, `main/user should keep 5% headroom and trigger at 190000, got ${userTrigger}`);
 }
 
 // 5) Legacy/zero buffer telemetry migration is an agent-path concern (the
 //    default-buffer sanitizer). Agent-owned sessions reapply the current 10%
 //    default (trigger 180000); explicit bufferTokens still lowers the agent
-//    trigger. Main/user sessions ignore all of it and compact at 100%.
+//    trigger. Main/user sessions ignore all of it and compact at 95%.
 {
   const boundary = 200000;
   const agentLegacy = compactTriggerForSession({
@@ -105,8 +105,8 @@ function assert(condition, message) {
   const userTelemetry = compactTriggerForSession({
     compaction: { boundaryTokens: boundary, triggerTokens: 180000, bufferTokens: 20000, bufferRatio: 0.1 },
   }, boundary);
-  assert(userTelemetry === 200000,
-    `main/user should ignore buffer telemetry and stay at 100% 200000, got ${userTelemetry}`);
+  assert(userTelemetry === 190000,
+    `main/user should ignore buffer telemetry and keep 5% headroom at 190000, got ${userTelemetry}`);
 }
 
 // 6) preserveBufferConfigFields copies only finite-positive percent/ratio fields.