mixdog 0.9.49 → 0.9.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/package.json +7 -7
- package/scripts/embedding-worker-exit-test.mjs +76 -0
- package/scripts/hook-bus-test.mjs +23 -0
- package/scripts/internal-tools-normalization-test.mjs +10 -0
- package/scripts/openai-oauth-ws-1006-retry-test.mjs +67 -1
- package/scripts/provider-toolcall-test.mjs +200 -2
- package/scripts/session-bench-cache-break-test.mjs +102 -0
- package/scripts/session-bench.mjs +101 -42
- package/scripts/shell-failure-diagnostics-test.mjs +73 -4
- package/scripts/smoke-loop-failure-summary-test.mjs +38 -0
- package/scripts/smoke-loop-failure-summary.mjs +16 -0
- package/scripts/smoke-loop.mjs +4 -3
- package/scripts/smoke.mjs +5 -106
- package/scripts/tool-failures.mjs +15 -1
- package/scripts/tui-transcript-perf-test.mjs +38 -0
- package/scripts/verify-release-assets-test.mjs +15 -381
- package/scripts/web-fetch-routing-test.mjs +158 -0
- package/src/runtime/agent/orchestrator/agent-trace-format.mjs +12 -0
- package/src/runtime/agent/orchestrator/internal-tools.mjs +2 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +14 -4
- package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs +74 -1
- package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs +75 -3
- package/src/runtime/agent/orchestrator/session/agent-loop.mjs +13 -0
- package/src/runtime/agent/orchestrator/session/loop/pre-dispatch-deny.mjs +38 -0
- package/src/runtime/agent/orchestrator/session/loop/tool-exec.mjs +15 -0
- package/src/runtime/agent/orchestrator/session/manager/tool-resolution.mjs +5 -1
- package/src/runtime/agent/orchestrator/session/pre-send-compact.mjs +4 -0
- package/src/runtime/memory/lib/embedding-provider.mjs +3 -2
- package/src/runtime/search/index.mjs +41 -0
- package/src/runtime/search/lib/http-fetch.mjs +154 -0
- package/src/runtime/search/tool-defs.mjs +23 -0
- package/src/session-runtime/runtime-core.mjs +37 -15
- package/src/standalone/hook-bus/handlers.mjs +4 -0
- package/src/standalone/hook-bus/rules.mjs +7 -1
- package/src/standalone/hook-bus.mjs +10 -2
- package/src/tui/App.jsx +17 -11
- package/src/tui/app/live-spinner-visibility.mjs +20 -0
- package/src/tui/dist/index.mjs +28 -3
package/README.md
CHANGED
|
@@ -152,12 +152,12 @@ current Anthropic models.
|
|
|
152
152
|
## Scripts
|
|
153
153
|
|
|
154
154
|
```bash
|
|
155
|
-
npm run smoke
|
|
156
|
-
npm run smoke:all
|
|
157
|
-
npm run smoke:tui
|
|
158
|
-
npm run
|
|
159
|
-
npm run build:tui
|
|
160
|
-
npm run audit:models
|
|
155
|
+
npm run smoke # fast core feature smoke
|
|
156
|
+
npm run smoke:all # feature-surface smoke suite
|
|
157
|
+
npm run smoke:tui # TUI feature smoke
|
|
158
|
+
npm run test:tool-contracts # optional tool contract suite
|
|
159
|
+
npm run build:tui # build the bundled Ink TUI
|
|
160
|
+
npm run audit:models # inspect model catalog metadata
|
|
161
161
|
```
|
|
162
162
|
|
|
163
163
|
Additional diagnostics and benchmarks live under `scripts/`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixdog",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.51",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Standalone mixdog coding-agent CLI/TUI workspace.",
|
|
@@ -36,14 +36,14 @@
|
|
|
36
36
|
"prepack": "npm run build:tui",
|
|
37
37
|
"prepublishOnly": "node -e \"if(!process.env.CI){console.error('local npm publish is disabled — run npm run release:patch');process.exit(1)}\"",
|
|
38
38
|
"start": "node src/cli.mjs",
|
|
39
|
-
"smoke": "node scripts/smoke.mjs
|
|
40
|
-
"smoke:all": "npm run smoke && npm run smoke:boot && npm run smoke:
|
|
39
|
+
"smoke": "node scripts/smoke.mjs",
|
|
40
|
+
"smoke:all": "npm run smoke && npm run smoke:boot && npm run smoke:patch && npm run smoke:output && npm run smoke:tui && npm run smoke:live-worker",
|
|
41
41
|
"smoke:boot": "node scripts/boot-smoke.mjs",
|
|
42
42
|
"smoke:compact": "node scripts/compact-smoke.mjs",
|
|
43
43
|
"smoke:loop": "node scripts/smoke-loop.mjs",
|
|
44
|
-
"smoke:loop:final": "node scripts/smoke-loop-report.mjs --require-complete --min-elapsed 5h --min-iterations 400 --max-gap 60s --max-smoke-ms 10000 --max-avg-smoke-ms 8500 --max-step-ms
|
|
44
|
+
"smoke:loop:final": "node scripts/smoke-loop-report.mjs --require-complete --min-elapsed 5h --min-iterations 400 --max-gap 60s --max-smoke-ms 10000 --max-avg-smoke-ms 8500 --max-step-ms smoke.mjs=4000 --max-step-ms boot-smoke.mjs=8000 --max-rss-mb 140 --max-rss-growth-mb 50",
|
|
45
45
|
"smoke:loop:report": "node scripts/smoke-loop-report.mjs",
|
|
46
|
-
"
|
|
46
|
+
"test:tool-contracts": "node scripts/tool-smoke.mjs",
|
|
47
47
|
"smoke:patch": "node scripts/apply-patch-edit-smoke.mjs",
|
|
48
48
|
"smoke:output": "node scripts/output-style-smoke.mjs",
|
|
49
49
|
"smoke:tui": "node scripts/tui-render-smoke.mjs && npm run test:tui-input-render && npm run test:tui-streaming-window && npm run test:tui-queue",
|
|
@@ -62,14 +62,14 @@
|
|
|
62
62
|
"test:anthropic-oauth-race": "node --test scripts/anthropic-oauth-refresh-race-test.mjs",
|
|
63
63
|
"test:grok-oauth-race": "node --test scripts/grok-oauth-refresh-race-test.mjs",
|
|
64
64
|
"test:atomiclock": "node --test scripts/atomic-lock-tryonce-test.mjs",
|
|
65
|
-
"test:memory-routing": "node --test scripts/memory-cycle-routing-test.mjs scripts/
|
|
65
|
+
"test:memory-routing": "node --test scripts/memory-cycle-routing-test.mjs scripts/maintenance-default-routes-test.mjs scripts/embedding-worker-exit-test.mjs",
|
|
66
66
|
"test:code-graph-dispatch": "node --test scripts/code-graph-dispatch-test.mjs",
|
|
67
67
|
"test:code-graph-clean-cache": "node --test scripts/code-graph-dispatch-test.mjs",
|
|
68
68
|
"test:tui-queue": "node --test scripts/submit-commandbusy-race-test.mjs scripts/steering-drain-buckets-test.mjs scripts/abort-recovery-test.mjs scripts/execution-pending-resume-kick-test.mjs scripts/execution-resume-esc-integration-test.mjs",
|
|
69
69
|
"test:tui-input-render": "node --test scripts/prompt-immediate-render-test.mjs",
|
|
70
70
|
"test:tui-streaming-window": "node --test scripts/streaming-tail-window-test.mjs",
|
|
71
71
|
"test:release-assets": "node --check scripts/verify-release-assets.mjs && node --check scripts/verify-release-assets-test.mjs && node --test scripts/verify-release-assets-test.mjs",
|
|
72
|
-
"test:release-focused": "npm run test:release-assets && npm run test:patch-binary-cache && npm run test:providers && npm run test:deferred-tools && npm run smoke:compact &&
|
|
72
|
+
"test:release-focused": "npm run test:release-assets && npm run test:patch-binary-cache && npm run test:providers && npm run test:deferred-tools && npm run smoke:compact && node --test scripts/code-graph-aggregate-cwd-test.mjs && npm run test:code-graph-dispatch && node --test scripts/code-graph-disk-hit-test.mjs && npm run test:shellhardening && node --test scripts/windows-hide-spawn-options-test.mjs && node --test scripts/tui-transcript-perf-test.mjs",
|
|
73
73
|
"test:native-edit-wire": "node --test scripts/native-edit-wire-test.mjs",
|
|
74
74
|
"test:patch-binary-cache": "node --test scripts/patch-binary-cache-test.mjs",
|
|
75
75
|
"test:session": "node --test scripts/session-orphan-sweep-test.mjs",
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import test from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { spawn } from 'node:child_process'
|
|
4
|
+
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import { join } from 'node:path'
|
|
7
|
+
|
|
8
|
+
const providerUrl = new URL('../src/runtime/memory/lib/embedding-provider.mjs', import.meta.url)
|
|
9
|
+
const CHILD_DEADLINE_MS = 3_000
|
|
10
|
+
|
|
11
|
+
async function createExitWorkerFixture() {
|
|
12
|
+
const tempDir = await mkdtemp(join(tmpdir(), 'mixdog-embed-worker-exit-'))
|
|
13
|
+
const workerPath = join(tempDir, 'exit-before-reply.mjs')
|
|
14
|
+
await writeFile(workerPath, `
|
|
15
|
+
import { parentPort } from 'node:worker_threads'
|
|
16
|
+
parentPort.once('message', () => process.exit(0))
|
|
17
|
+
`)
|
|
18
|
+
const source = await readFile(providerUrl, 'utf8')
|
|
19
|
+
const instrumented = source
|
|
20
|
+
.replace("'./model-profile.mjs'", JSON.stringify(new URL('../src/runtime/memory/lib/model-profile.mjs', import.meta.url).href))
|
|
21
|
+
.replace("'./embedding-model-config.mjs'", JSON.stringify(new URL('../src/runtime/memory/lib/embedding-model-config.mjs', import.meta.url).href))
|
|
22
|
+
.replace(/^const WORKER_PATH = .*$/m, `const WORKER_PATH = ${JSON.stringify(workerPath)}`)
|
|
23
|
+
const providerModule = `data:text/javascript;base64,${Buffer.from(instrumented).toString('base64')}#${Date.now()}`
|
|
24
|
+
const fixturePath = join(tempDir, 'run-exit-test.mjs')
|
|
25
|
+
await writeFile(fixturePath, `
|
|
26
|
+
import { embedText } from ${JSON.stringify(providerModule)}
|
|
27
|
+
const started = Date.now()
|
|
28
|
+
const settled = await Promise.allSettled([embedText('first'), embedText('second')])
|
|
29
|
+
if (Date.now() - started >= 2_000) throw new Error('worker exit did not reject pending embeds promptly')
|
|
30
|
+
if (settled.some(({ status }) => status !== 'rejected')) throw new Error('worker exit resolved a pending embed')
|
|
31
|
+
for (const result of settled) {
|
|
32
|
+
if (result.reason?.message !== 'Worker exited with code 0') throw result.reason
|
|
33
|
+
}
|
|
34
|
+
process.stdout.write('ok')
|
|
35
|
+
`)
|
|
36
|
+
return { fixturePath, tempDir }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function runFixture(fixturePath) {
|
|
40
|
+
return new Promise((resolve, reject) => {
|
|
41
|
+
const child = spawn(process.execPath, [fixturePath], { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
|
|
42
|
+
let stdout = ''
|
|
43
|
+
let stderr = ''
|
|
44
|
+
let timedOut = false
|
|
45
|
+
const deadline = setTimeout(() => {
|
|
46
|
+
timedOut = true
|
|
47
|
+
child.kill()
|
|
48
|
+
}, CHILD_DEADLINE_MS)
|
|
49
|
+
child.stdout.on('data', (chunk) => { stdout += chunk })
|
|
50
|
+
child.stderr.on('data', (chunk) => { stderr += chunk })
|
|
51
|
+
child.once('error', (error) => {
|
|
52
|
+
clearTimeout(deadline)
|
|
53
|
+
reject(error)
|
|
54
|
+
})
|
|
55
|
+
child.once('close', (code, signal) => {
|
|
56
|
+
clearTimeout(deadline)
|
|
57
|
+
if (timedOut) {
|
|
58
|
+
reject(new Error(`exit-worker fixture exceeded ${CHILD_DEADLINE_MS}ms and was killed`))
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
resolve({ code, signal, stdout, stderr })
|
|
62
|
+
})
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
test('code-zero worker exit before reply promptly rejects every pending embed', async () => {
|
|
67
|
+
const { fixturePath, tempDir } = await createExitWorkerFixture()
|
|
68
|
+
try {
|
|
69
|
+
const result = await runFixture(fixturePath)
|
|
70
|
+
assert.equal(result.code, 0, result.stderr)
|
|
71
|
+
assert.equal(result.signal, null, result.stderr)
|
|
72
|
+
assert.equal(result.stdout, 'ok')
|
|
73
|
+
} finally {
|
|
74
|
+
await rm(tempDir, { recursive: true, force: true })
|
|
75
|
+
}
|
|
76
|
+
})
|
|
@@ -60,6 +60,29 @@ if (command.includes('rm -rf')) {
|
|
|
60
60
|
}
|
|
61
61
|
});
|
|
62
62
|
|
|
63
|
+
test('standard PreToolUse hook can replace the tool name', async () => {
|
|
64
|
+
const root = tempRoot();
|
|
65
|
+
const hookScript = join(root, 'rename.mjs');
|
|
66
|
+
writeFileSync(hookScript, `console.log(JSON.stringify({ hookSpecificOutput: {
|
|
67
|
+
hookEventName: 'PreToolUse',
|
|
68
|
+
updatedToolName: 'local_fetch'
|
|
69
|
+
}}));\n`, 'utf8');
|
|
70
|
+
const hooksFile = join(root, 'hooks.json');
|
|
71
|
+
writeJson(hooksFile, { hooks: { PreToolUse: [{ matcher: 'web_fetch', hooks: [{ type: 'command', command: process.execPath, args: [hookScript] }] }] } });
|
|
72
|
+
const prev = process.env.MIXDOG_HOOKS_FILE;
|
|
73
|
+
process.env.MIXDOG_HOOKS_FILE = hooksFile;
|
|
74
|
+
try {
|
|
75
|
+
const decision = await createStandaloneHookBus({ dataDir: root }).beforeTool({
|
|
76
|
+
cwd: root, name: 'web_fetch', args: { url: 'http://localhost:3000/' },
|
|
77
|
+
});
|
|
78
|
+
assert.equal(decision.action, 'modify');
|
|
79
|
+
assert.equal(decision.name, 'local_fetch');
|
|
80
|
+
} finally {
|
|
81
|
+
if (prev == null) delete process.env.MIXDOG_HOOKS_FILE;
|
|
82
|
+
else process.env.MIXDOG_HOOKS_FILE = prev;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
63
86
|
test('UserPromptSubmit hook returns additional context', async () => {
|
|
64
87
|
const root = tempRoot();
|
|
65
88
|
const hookScript = join(root, 'prompt.mjs');
|
|
@@ -50,3 +50,13 @@ test('isError:false output stays normal', async () => {
|
|
|
50
50
|
assert.equal(result, 'search complete');
|
|
51
51
|
assert.equal(classifyResultKind(result), 'normal');
|
|
52
52
|
});
|
|
53
|
+
|
|
54
|
+
test('structured image output survives internal tool normalization', async () => {
|
|
55
|
+
const input = {
|
|
56
|
+
content: [
|
|
57
|
+
{ type: 'text', text: 'downloaded' },
|
|
58
|
+
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'YWJj' } },
|
|
59
|
+
],
|
|
60
|
+
};
|
|
61
|
+
assert.deepEqual(await normalizeToolResult(input), input);
|
|
62
|
+
});
|
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
// Regression: OpenAI OAuth WS abnormal-close recovery before visible output.
|
|
2
2
|
import test from 'node:test';
|
|
3
3
|
import assert from 'node:assert/strict';
|
|
4
|
+
import { EventEmitter } from 'node:events';
|
|
4
5
|
import { OpenAIOAuthProvider } from '../src/runtime/agent/orchestrator/providers/openai-oauth.mjs';
|
|
5
6
|
import { _acquireWithRetry, sendViaWebSocket } from '../src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs';
|
|
6
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
_clearWebSocketPoolForTest,
|
|
9
|
+
_seedWebSocketEntryForTest,
|
|
10
|
+
acquireWebSocket,
|
|
11
|
+
releaseWebSocket,
|
|
12
|
+
_sendFrame,
|
|
13
|
+
} from '../src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs';
|
|
7
14
|
|
|
8
15
|
function close1006() {
|
|
9
16
|
const err = new Error('WebSocket closed abnormally');
|
|
@@ -28,6 +35,65 @@ function wsArgs(overrides = {}) {
|
|
|
28
35
|
};
|
|
29
36
|
}
|
|
30
37
|
|
|
38
|
+
class PoolSocket extends EventEmitter {
|
|
39
|
+
constructor() {
|
|
40
|
+
super();
|
|
41
|
+
this.readyState = 1;
|
|
42
|
+
this._socket = { ref() {}, unref() {} };
|
|
43
|
+
}
|
|
44
|
+
close() { this.readyState = 3; }
|
|
45
|
+
ping() { queueMicrotask(() => this.emit('pong')); }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function poolEntry(responseId) {
|
|
49
|
+
return {
|
|
50
|
+
socket: new PoolSocket(),
|
|
51
|
+
busy: true,
|
|
52
|
+
closing: false,
|
|
53
|
+
ephemeral: false,
|
|
54
|
+
lastResponseId: responseId,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
test('pool release/acquire preserves latest compatible chain and auth/cache boundaries', async (t) => {
|
|
59
|
+
t.after(() => _clearWebSocketPoolForTest());
|
|
60
|
+
const poolKey = 'reused-session-id';
|
|
61
|
+
const authA = { account_id: 'account-a', access_token: 'token-a' };
|
|
62
|
+
const authB = { account_id: 'account-b', access_token: 'token-b' };
|
|
63
|
+
const oldest = _seedWebSocketEntryForTest({
|
|
64
|
+
poolKey, auth: authA, cacheKey: 'cache-a', entry: poolEntry('resp-old'),
|
|
65
|
+
});
|
|
66
|
+
const latest = _seedWebSocketEntryForTest({
|
|
67
|
+
poolKey, auth: authA, cacheKey: 'cache-a', entry: poolEntry('resp-latest'),
|
|
68
|
+
});
|
|
69
|
+
const otherAccount = _seedWebSocketEntryForTest({
|
|
70
|
+
poolKey, auth: authB, cacheKey: 'cache-a', entry: poolEntry('resp-account-b'),
|
|
71
|
+
});
|
|
72
|
+
const otherCache = _seedWebSocketEntryForTest({
|
|
73
|
+
poolKey, auth: authA, cacheKey: 'cache-b', entry: poolEntry('resp-cache-b'),
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
releaseWebSocket({ entry: oldest, poolKey, keep: true });
|
|
77
|
+
releaseWebSocket({ entry: otherAccount, poolKey, keep: true });
|
|
78
|
+
releaseWebSocket({ entry: otherCache, poolKey, keep: true });
|
|
79
|
+
releaseWebSocket({ entry: latest, poolKey, keep: true });
|
|
80
|
+
|
|
81
|
+
const first = await acquireWebSocket({
|
|
82
|
+
auth: authA, poolKey, cacheKey: 'cache-a', forceFresh: false,
|
|
83
|
+
});
|
|
84
|
+
assert.equal(first.entry, latest, 'latest completed compatible chain wins');
|
|
85
|
+
assert.equal(first.reused, true);
|
|
86
|
+
assert.equal(latest.busy, true, 'acquire reserves the selected entry');
|
|
87
|
+
|
|
88
|
+
const second = await acquireWebSocket({
|
|
89
|
+
auth: authA, poolKey, cacheKey: 'cache-a', forceFresh: false,
|
|
90
|
+
});
|
|
91
|
+
assert.equal(second.entry, oldest, 'reserved latest entry cannot be acquired concurrently');
|
|
92
|
+
assert.equal(second.entry.lastResponseId, 'resp-old');
|
|
93
|
+
assert.notEqual(second.entry, otherAccount);
|
|
94
|
+
assert.notEqual(second.entry, otherCache);
|
|
95
|
+
});
|
|
96
|
+
|
|
31
97
|
test('acquire timeout reconnects successfully with progress, not a terminal WS error', async () => {
|
|
32
98
|
const oldWrite = process.stderr.write;
|
|
33
99
|
const oldQuiet = process.env.MIXDOG_QUIET_PROVIDER_LOG;
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
import {
|
|
38
38
|
_cacheObservationForTest,
|
|
39
39
|
_cacheContinuityResetReasonForTest,
|
|
40
|
+
sendViaWebSocket,
|
|
40
41
|
} from '../src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs';
|
|
41
42
|
import {
|
|
42
43
|
_withCodexWsClientMetadata,
|
|
@@ -1786,13 +1787,23 @@ test('openai oauth ws delta: ws-delta mode uses previous_response_id without tur
|
|
|
1786
1787
|
entry: {
|
|
1787
1788
|
...entry,
|
|
1788
1789
|
lastRequestInput: [body.input[0]],
|
|
1789
|
-
lastResponseItems: [{
|
|
1790
|
+
lastResponseItems: [{
|
|
1791
|
+
type: 'function_call',
|
|
1792
|
+
call_id: 'call_1',
|
|
1793
|
+
name: 'tool',
|
|
1794
|
+
arguments: '{"api_key":"provider-function-secret","nested":{"z":2,"a":1}}',
|
|
1795
|
+
}],
|
|
1790
1796
|
},
|
|
1791
1797
|
body: {
|
|
1792
1798
|
...body,
|
|
1793
1799
|
input: [
|
|
1794
1800
|
body.input[0],
|
|
1795
|
-
{
|
|
1801
|
+
{
|
|
1802
|
+
type: 'function_call',
|
|
1803
|
+
call_id: 'call_other',
|
|
1804
|
+
name: 'tool',
|
|
1805
|
+
arguments: '{"nested":{"a":1,"z":2},"api_key":"replayed-function-secret"}',
|
|
1806
|
+
},
|
|
1796
1807
|
body.input[1],
|
|
1797
1808
|
],
|
|
1798
1809
|
},
|
|
@@ -1800,6 +1811,193 @@ test('openai oauth ws delta: ws-delta mode uses previous_response_id without tur
|
|
|
1800
1811
|
assert.equal(responseMismatch.mode, 'full');
|
|
1801
1812
|
assert.equal(responseMismatch.reason, 'response_output_mismatch:function_call');
|
|
1802
1813
|
assert.equal(responseMismatch.frame.previous_response_id, undefined);
|
|
1814
|
+
const mismatch = responseMismatch.responseOutputMismatch;
|
|
1815
|
+
assert.deepEqual({
|
|
1816
|
+
expectedType: mismatch.response_output_mismatch_expected_type,
|
|
1817
|
+
actualType: mismatch.response_output_mismatch_actual_type,
|
|
1818
|
+
expectedCount: mismatch.response_output_mismatch_expected_response_item_count,
|
|
1819
|
+
actualCount: mismatch.response_output_mismatch_actual_replayed_input_item_count,
|
|
1820
|
+
}, {
|
|
1821
|
+
expectedType: 'function_call',
|
|
1822
|
+
actualType: 'function_call',
|
|
1823
|
+
expectedCount: 1,
|
|
1824
|
+
actualCount: 2,
|
|
1825
|
+
});
|
|
1826
|
+
assert.match(mismatch.response_output_mismatch_expected_hash, /^[a-f0-9]{64}$/);
|
|
1827
|
+
assert.match(mismatch.response_output_mismatch_actual_hash, /^[a-f0-9]{64}$/);
|
|
1828
|
+
const traceSafe = JSON.stringify(mismatch);
|
|
1829
|
+
assert.equal(traceSafe.includes('call_other'), false);
|
|
1830
|
+
assert.equal(traceSafe.includes('provider-function-secret'), false);
|
|
1831
|
+
assert.equal(traceSafe.includes('replayed-function-secret'), false);
|
|
1832
|
+
} finally {
|
|
1833
|
+
if (prevTransport == null) delete process.env.MIXDOG_OAI_TRANSPORT;
|
|
1834
|
+
else process.env.MIXDOG_OAI_TRANSPORT = prevTransport;
|
|
1835
|
+
}
|
|
1836
|
+
});
|
|
1837
|
+
|
|
1838
|
+
test('openai oauth ws delta: message mismatch diagnostics hash normalized content without tracing it', () => {
|
|
1839
|
+
const prevTransport = process.env.MIXDOG_OAI_TRANSPORT;
|
|
1840
|
+
try {
|
|
1841
|
+
process.env.MIXDOG_OAI_TRANSPORT = 'ws-delta';
|
|
1842
|
+
const prior = { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'prior' }] };
|
|
1843
|
+
const expected = { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'provider secret' }] };
|
|
1844
|
+
const actual = { type: 'message', role: 'assistant', content: [{ type: 'input_text', text: 'replayed secret' }] };
|
|
1845
|
+
const delta = _computeDelta({
|
|
1846
|
+
entry: {
|
|
1847
|
+
lastRequestSansInput: '{"model":"gpt-5.5"}',
|
|
1848
|
+
lastResponseId: 'resp_prev',
|
|
1849
|
+
lastRequestInput: [prior],
|
|
1850
|
+
lastResponseItems: [expected],
|
|
1851
|
+
},
|
|
1852
|
+
body: { model: 'gpt-5.5', input: [prior, actual] },
|
|
1853
|
+
});
|
|
1854
|
+
const mismatch = delta.responseOutputMismatch;
|
|
1855
|
+
assert.equal(delta.reason, 'response_output_mismatch:message');
|
|
1856
|
+
assert.equal(mismatch.response_output_mismatch_expected_type, 'message');
|
|
1857
|
+
assert.equal(mismatch.response_output_mismatch_actual_type, 'message');
|
|
1858
|
+
assert.notEqual(mismatch.response_output_mismatch_expected_hash, mismatch.response_output_mismatch_actual_hash);
|
|
1859
|
+
const traceSafe = JSON.stringify(mismatch);
|
|
1860
|
+
assert.equal(traceSafe.includes('provider secret'), false);
|
|
1861
|
+
assert.equal(traceSafe.includes('replayed secret'), false);
|
|
1862
|
+
} finally {
|
|
1863
|
+
if (prevTransport == null) delete process.env.MIXDOG_OAI_TRANSPORT;
|
|
1864
|
+
else process.env.MIXDOG_OAI_TRANSPORT = prevTransport;
|
|
1865
|
+
}
|
|
1866
|
+
});
|
|
1867
|
+
|
|
1868
|
+
test('openai oauth ws delta: diagnostic hashes normalize equivalent message and function-call forms', () => {
|
|
1869
|
+
const prevTransport = process.env.MIXDOG_OAI_TRANSPORT;
|
|
1870
|
+
try {
|
|
1871
|
+
process.env.MIXDOG_OAI_TRANSPORT = 'ws-delta';
|
|
1872
|
+
const prior = { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'prior' }] };
|
|
1873
|
+
const mismatch = (expected, actual) => _computeDelta({
|
|
1874
|
+
entry: {
|
|
1875
|
+
lastRequestSansInput: '{"model":"gpt-5.5"}',
|
|
1876
|
+
lastResponseId: 'resp_prev',
|
|
1877
|
+
lastRequestInput: [prior],
|
|
1878
|
+
lastResponseItems: [expected],
|
|
1879
|
+
},
|
|
1880
|
+
body: { model: 'gpt-5.5', input: [prior, actual] },
|
|
1881
|
+
}).responseOutputMismatch;
|
|
1882
|
+
const message = { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'same sensitive message' }] };
|
|
1883
|
+
const messageExpected = mismatch(message, {
|
|
1884
|
+
type: 'message',
|
|
1885
|
+
role: 'assistant',
|
|
1886
|
+
content: [{ type: 'output_text', text: 'different message' }],
|
|
1887
|
+
});
|
|
1888
|
+
const messageActual = mismatch({
|
|
1889
|
+
type: 'message',
|
|
1890
|
+
role: 'assistant',
|
|
1891
|
+
content: [{ type: 'output_text', text: 'different message' }],
|
|
1892
|
+
}, {
|
|
1893
|
+
type: 'message',
|
|
1894
|
+
role: 'assistant',
|
|
1895
|
+
content: [{ type: 'input_text', text: 'same sensitive message' }],
|
|
1896
|
+
});
|
|
1897
|
+
assert.equal(
|
|
1898
|
+
messageExpected.response_output_mismatch_expected_hash,
|
|
1899
|
+
messageActual.response_output_mismatch_actual_hash,
|
|
1900
|
+
);
|
|
1901
|
+
|
|
1902
|
+
const functionExpected = (argumentsValue) => mismatch({
|
|
1903
|
+
type: 'function_call',
|
|
1904
|
+
call_id: 'call_same',
|
|
1905
|
+
name: 'tool',
|
|
1906
|
+
arguments: argumentsValue,
|
|
1907
|
+
}, {
|
|
1908
|
+
type: 'function_call',
|
|
1909
|
+
call_id: 'call_different',
|
|
1910
|
+
name: 'tool',
|
|
1911
|
+
arguments: '{"ignored":"replayed-function-secret"}',
|
|
1912
|
+
});
|
|
1913
|
+
assert.equal(
|
|
1914
|
+
functionExpected('{"api_key":"function-secret","nested":{"z":2,"a":1}}').response_output_mismatch_expected_hash,
|
|
1915
|
+
functionExpected('{"nested":{"a":1,"z":2},"api_key":"function-secret"}').response_output_mismatch_expected_hash,
|
|
1916
|
+
);
|
|
1917
|
+
} finally {
|
|
1918
|
+
if (prevTransport == null) delete process.env.MIXDOG_OAI_TRANSPORT;
|
|
1919
|
+
else process.env.MIXDOG_OAI_TRANSPORT = prevTransport;
|
|
1920
|
+
}
|
|
1921
|
+
});
|
|
1922
|
+
|
|
1923
|
+
test('openai oauth ws delta: mismatch diagnostics reach transport and cache_break without sensitive values', async () => {
|
|
1924
|
+
const prevTransport = process.env.MIXDOG_OAI_TRANSPORT;
|
|
1925
|
+
try {
|
|
1926
|
+
process.env.MIXDOG_OAI_TRANSPORT = 'ws-delta';
|
|
1927
|
+
const rows = [];
|
|
1928
|
+
const prior = { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'prior' }] };
|
|
1929
|
+
const cases = [{
|
|
1930
|
+
expected: {
|
|
1931
|
+
type: 'function_call',
|
|
1932
|
+
call_id: 'call_expected',
|
|
1933
|
+
name: 'tool',
|
|
1934
|
+
arguments: '{"api_key":"provider-function-secret","nested":{"z":2,"a":1}}',
|
|
1935
|
+
},
|
|
1936
|
+
actual: {
|
|
1937
|
+
type: 'function_call',
|
|
1938
|
+
call_id: 'call_actual',
|
|
1939
|
+
name: 'tool',
|
|
1940
|
+
arguments: '{"nested":{"a":1,"z":2},"api_key":"replayed-function-secret"}',
|
|
1941
|
+
},
|
|
1942
|
+
}, {
|
|
1943
|
+
expected: {
|
|
1944
|
+
type: 'custom_tool_call',
|
|
1945
|
+
call_id: 'custom_expected',
|
|
1946
|
+
name: 'apply_patch',
|
|
1947
|
+
input: 'provider-custom-input-secret',
|
|
1948
|
+
},
|
|
1949
|
+
actual: {
|
|
1950
|
+
type: 'custom_tool_call',
|
|
1951
|
+
call_id: 'custom_actual',
|
|
1952
|
+
name: 'apply_patch',
|
|
1953
|
+
input: 'replayed-custom-input-secret',
|
|
1954
|
+
},
|
|
1955
|
+
}];
|
|
1956
|
+
for (const { expected, actual } of cases) {
|
|
1957
|
+
const body = { model: 'gpt-5.5', input: [prior, actual] };
|
|
1958
|
+
const entry = {
|
|
1959
|
+
socket: { close() {} },
|
|
1960
|
+
lastRequestSansInput: _stableStringify(_sansInput(body)),
|
|
1961
|
+
lastResponseId: 'resp_prev',
|
|
1962
|
+
lastRequestInput: [prior],
|
|
1963
|
+
lastResponseItems: [expected],
|
|
1964
|
+
};
|
|
1965
|
+
const expectedDiagnostics = _computeDelta({ entry, body }).responseOutputMismatch;
|
|
1966
|
+
await sendViaWebSocket({
|
|
1967
|
+
auth: { type: 'xai', access_token: 'test-token' },
|
|
1968
|
+
body,
|
|
1969
|
+
poolKey: `mismatch-trace-${expected.type}`,
|
|
1970
|
+
cacheKey: 'mismatch-trace-test',
|
|
1971
|
+
iteration: 1,
|
|
1972
|
+
useModel: 'gpt-5.5',
|
|
1973
|
+
traceProvider: 'xai',
|
|
1974
|
+
_acquireWithRetryFn: async () => ({ entry, reused: false }),
|
|
1975
|
+
_sendFrameFn: async () => {},
|
|
1976
|
+
_streamFn: async () => ({
|
|
1977
|
+
content: 'ok',
|
|
1978
|
+
model: 'gpt-5.5',
|
|
1979
|
+
toolCalls: [],
|
|
1980
|
+
usage: {},
|
|
1981
|
+
responseId: 'resp_next',
|
|
1982
|
+
responseItems: [],
|
|
1983
|
+
closeSocket: true,
|
|
1984
|
+
}),
|
|
1985
|
+
_agentTraceFn: (row) => rows.push(row),
|
|
1986
|
+
});
|
|
1987
|
+
const emitted = rows.filter((row) => row.payload?.response_output_mismatch_expected_type === expected.type);
|
|
1988
|
+
assert.deepEqual(emitted.map((row) => row.kind).sort(), ['cache_break', 'transport']);
|
|
1989
|
+
for (const row of emitted) assert.deepEqual(
|
|
1990
|
+
Object.fromEntries(Object.keys(expectedDiagnostics).map((key) => [key, row.payload[key]])),
|
|
1991
|
+
expectedDiagnostics,
|
|
1992
|
+
);
|
|
1993
|
+
}
|
|
1994
|
+
const traceText = JSON.stringify(rows);
|
|
1995
|
+
for (const secret of [
|
|
1996
|
+
'provider-function-secret',
|
|
1997
|
+
'replayed-function-secret',
|
|
1998
|
+
'provider-custom-input-secret',
|
|
1999
|
+
'replayed-custom-input-secret',
|
|
2000
|
+
]) assert.equal(traceText.includes(secret), false);
|
|
1803
2001
|
} finally {
|
|
1804
2002
|
if (prevTransport == null) delete process.env.MIXDOG_OAI_TRANSPORT;
|
|
1805
2003
|
else process.env.MIXDOG_OAI_TRANSPORT = prevTransport;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { execFileSync } from 'node:child_process';
|
|
4
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
|
|
8
|
+
test('root session bench promotes actionable compact mismatches without hiding intentional transitions', () => {
|
|
9
|
+
const dir = mkdtempSync(join(tmpdir(), 'mixdog-session-bench-'));
|
|
10
|
+
const trace = join(dir, 'trace.jsonl');
|
|
11
|
+
const sessionId = 'cache-break-observability';
|
|
12
|
+
const rows = [
|
|
13
|
+
...Array.from({ length: 4 }, (_, index) => ({
|
|
14
|
+
ts: index + 1, sessionId, iteration: index + 1, kind: 'cache_break',
|
|
15
|
+
payload: { reason: 'input_prefix_mismatch', intentional_transition: 'automatic_compaction' },
|
|
16
|
+
})),
|
|
17
|
+
...Array.from({ length: 2 }, (_, index) => ({
|
|
18
|
+
ts: index + 5, sessionId, iteration: index + 5, kind: 'cache_break',
|
|
19
|
+
payload: { reason: 'input_prefix_mismatch', intentional_transition: 'transcript_rebuild' },
|
|
20
|
+
})),
|
|
21
|
+
...Array.from({ length: 2 }, (_, index) => ({
|
|
22
|
+
ts: index + 7, sessionId, iteration: index + 7, kind: 'cache_break',
|
|
23
|
+
payload: {
|
|
24
|
+
reason: 'request_properties_changed',
|
|
25
|
+
intentional_transition: 'explorer_hard_cap_final_tool_choice_none',
|
|
26
|
+
request_tool_choice: 'none',
|
|
27
|
+
},
|
|
28
|
+
})),
|
|
29
|
+
{
|
|
30
|
+
ts: 9, sessionId, iteration: 9, kind: 'cache_break',
|
|
31
|
+
payload: { reason: 'response_output_mismatch:function_call' },
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
ts: 10, sessionId, iteration: 10, kind: 'cache_break',
|
|
35
|
+
payload: { reason: 'input_prefix_mismatch', intentional_transition: 'unknown_transition' },
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
ts: 11, sessionId, iteration: 11, kind: 'cache_break',
|
|
39
|
+
payload: {
|
|
40
|
+
reason: 'response_output_mismatch:function_call',
|
|
41
|
+
intentional_transition: 'automatic_compaction',
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
ts: 12, sessionId: `${sessionId}:compact`, iteration: 1, kind: 'cache_break',
|
|
46
|
+
payload: { reason: 'input_prefix_mismatch' },
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
ts: 13, sessionId, iteration: 13, kind: 'cache_break',
|
|
50
|
+
payload: {
|
|
51
|
+
reason: 'request_properties_changed',
|
|
52
|
+
intentional_transition: 'explorer_hard_cap_final_tool_choice_none',
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
...Array.from({ length: 6 }, (_, index) => ({
|
|
56
|
+
ts: index + 14, sessionId, iteration: index + 14, kind: 'cache_break',
|
|
57
|
+
payload: { reason: 'response_output_mismatch:function_call' },
|
|
58
|
+
})),
|
|
59
|
+
];
|
|
60
|
+
writeFileSync(trace, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`);
|
|
61
|
+
try {
|
|
62
|
+
const output = execFileSync(process.execPath, [
|
|
63
|
+
'scripts/session-bench.mjs', '--json', '--trace', trace, '--session', sessionId,
|
|
64
|
+
], { cwd: process.cwd(), encoding: 'utf8' });
|
|
65
|
+
const report = JSON.parse(output);
|
|
66
|
+
assert.equal(report.cache.cache_breaks.length, 19, 'raw evidence is retained');
|
|
67
|
+
assert.equal(report.cache.intentional_cache_breaks.length, 8);
|
|
68
|
+
assert.equal(report.cache.actionable_cache_breaks.length, 11);
|
|
69
|
+
assert.deepEqual(
|
|
70
|
+
report.cache.intentional_cache_breaks.map((row) => row.phase),
|
|
71
|
+
[
|
|
72
|
+
...Array(4).fill('intentional_automatic_compaction'),
|
|
73
|
+
...Array(2).fill('intentional_transcript_rebuild'),
|
|
74
|
+
...Array(2).fill('intentional_explorer_hard_cap_final_tool_choice_none'),
|
|
75
|
+
],
|
|
76
|
+
);
|
|
77
|
+
assert.equal(report.cache.actionable_cache_breaks[0].reason, 'response_output_mismatch:function_call');
|
|
78
|
+
const actionableByTs = new Map(report.cache.actionable_cache_breaks.map((row) => [row.ts, row]));
|
|
79
|
+
assert.equal(actionableByTs.get(10).intentional_transition, 'unknown_transition');
|
|
80
|
+
assert.equal(actionableByTs.get(11).intentional_transition, 'automatic_compaction');
|
|
81
|
+
assert.equal(actionableByTs.get(12).session_id, `${sessionId}:compact`);
|
|
82
|
+
assert.equal(actionableByTs.get(13).intentional_transition, 'explorer_hard_cap_final_tool_choice_none');
|
|
83
|
+
const cacheBreakIssues = report.issues.filter((issue) => issue.type === 'cache_break');
|
|
84
|
+
assert.equal(cacheBreakIssues.length, 10, 'the cache-break issue cap remains bounded');
|
|
85
|
+
assert.equal(cacheBreakIssues.filter((issue) => issue.session_id === sessionId).length, 9);
|
|
86
|
+
assert.ok(
|
|
87
|
+
report.issues.some((issue) => issue.type === 'cache_break' && issue.session_id === `${sessionId}:compact`),
|
|
88
|
+
'the root report must surface a child compact mismatch as an issue',
|
|
89
|
+
);
|
|
90
|
+
assert.ok(
|
|
91
|
+
report.rankings.some((ranking) => ranking.type === 'cache_breaks' && ranking.message.includes('11 actionable cache break(s)')),
|
|
92
|
+
'the root report must rank child compact mismatches with other actionable breaks',
|
|
93
|
+
);
|
|
94
|
+
const text = execFileSync(process.execPath, [
|
|
95
|
+
'scripts/session-bench.mjs', '--trace', trace, '--session', sessionId,
|
|
96
|
+
], { cwd: process.cwd(), encoding: 'utf8' });
|
|
97
|
+
assert.match(text, /compact cache breaks \(actionable\):/);
|
|
98
|
+
assert.doesNotMatch(text, /compact cache resets \(intentional\):/);
|
|
99
|
+
} finally {
|
|
100
|
+
rmSync(dir, { recursive: true, force: true });
|
|
101
|
+
}
|
|
102
|
+
});
|