mixdog 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -3
- package/scripts/_bench-cwc.json +20 -0
- package/scripts/agent-loop-policy-test.mjs +37 -0
- package/scripts/agent-parallel-smoke.mjs +54 -10
- package/scripts/background-task-meta-smoke.mjs +1 -1
- package/scripts/bench-run.mjs +262 -0
- package/scripts/compact-smoke.mjs +12 -0
- package/scripts/compact-trigger-migration-smoke.mjs +67 -1
- package/scripts/ingest-pure-conversation-smoke.mjs +148 -0
- package/scripts/internal-comms-bench.mjs +727 -0
- package/scripts/internal-comms-smoke.mjs +75 -0
- package/scripts/lead-workflow-smoke.mjs +4 -4
- package/scripts/live-worker-smoke.mjs +9 -9
- package/scripts/output-style-bench.mjs +285 -0
- package/scripts/output-style-smoke.mjs +13 -10
- package/scripts/patch-replay.mjs +90 -0
- package/scripts/provider-stream-stall-test.mjs +276 -0
- package/scripts/provider-toolcall-test.mjs +599 -1
- package/scripts/routing-corpus.mjs +281 -0
- package/scripts/session-bench.mjs +1526 -0
- package/scripts/session-diag.mjs +595 -0
- package/scripts/session-ingest-smoke.mjs +2 -2
- package/scripts/task-bench.mjs +207 -0
- package/scripts/tool-failures.mjs +6 -6
- package/scripts/tool-smoke.mjs +306 -66
- package/scripts/toolcall-args-test.mjs +81 -0
- package/src/agents/debugger/AGENT.md +4 -4
- package/src/agents/heavy-worker/AGENT.md +4 -2
- package/src/agents/reviewer/AGENT.md +4 -4
- package/src/agents/worker/AGENT.md +4 -2
- package/src/app.mjs +10 -6
- package/src/defaults/{hidden-roles.json → agents.json} +7 -7
- package/src/examples/schedules/SCHEDULE.example.md +32 -0
- package/src/examples/webhooks/WEBHOOK.example.md +40 -0
- package/src/headless-role.mjs +14 -14
- package/src/help.mjs +1 -0
- package/src/lib/mixdog-debug.cjs +0 -22
- package/src/lib/plugin-paths.cjs +1 -7
- package/src/lib/rules-builder.cjs +34 -56
- package/src/mixdog-session-runtime.mjs +710 -319
- package/src/output-styles/default.md +12 -7
- package/src/output-styles/minimal.md +25 -0
- package/src/output-styles/oneline.md +21 -0
- package/src/output-styles/simple.md +10 -9
- package/src/repl.mjs +12 -4
- package/src/rules/agent/00-common.md +7 -5
- package/src/rules/agent/30-explorer.md +7 -8
- package/src/rules/lead/01-general.md +3 -1
- package/src/rules/lead/lead-tool.md +7 -0
- package/src/rules/shared/01-tool.md +17 -12
- package/src/runtime/agent/orchestrator/agent-runtime/agent-dispatch.mjs +90 -32
- package/src/runtime/agent/orchestrator/agent-runtime/agent-loop-policy.mjs +32 -0
- package/src/runtime/agent/orchestrator/agent-runtime/agent-progress-watchdog.mjs +18 -6
- package/src/runtime/agent/orchestrator/agent-runtime/cache-strategy.mjs +23 -20
- package/src/runtime/agent/orchestrator/agent-runtime/session-builder.mjs +48 -14
- package/src/runtime/agent/orchestrator/agent-trace.mjs +87 -12
- package/src/runtime/agent/orchestrator/config.mjs +3 -0
- package/src/runtime/agent/orchestrator/context/collect.mjs +131 -67
- package/src/runtime/agent/orchestrator/{internal-roles.mjs → internal-agents.mjs} +72 -72
- package/src/runtime/agent/orchestrator/internal-tools.mjs +13 -26
- package/src/runtime/agent/orchestrator/mcp/client.mjs +94 -16
- package/src/runtime/agent/orchestrator/providers/anthropic-betas.mjs +7 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +188 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-leaked-toolcall.mjs +444 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +359 -106
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +63 -51
- package/src/runtime/agent/orchestrator/providers/api-usage.mjs +27 -20
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +184 -17
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +8 -1
- package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +18 -8
- package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +210 -21
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +86 -30
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +254 -280
- package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +191 -50
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +18 -0
- package/src/runtime/agent/orchestrator/providers/opencode-go-usage.mjs +11 -5
- package/src/runtime/agent/orchestrator/providers/registry.mjs +2 -1
- package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs +265 -1
- package/src/runtime/agent/orchestrator/session/compact.mjs +560 -51
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +250 -3
- package/src/runtime/agent/orchestrator/session/loop.mjs +394 -132
- package/src/runtime/agent/orchestrator/session/manager.mjs +217 -170
- package/src/runtime/agent/orchestrator/session/store.mjs +4 -4
- package/src/runtime/agent/orchestrator/session/tool-envelope.mjs +61 -0
- package/src/runtime/agent/orchestrator/session/tool-result-offload.mjs +5 -0
- package/src/runtime/agent/orchestrator/stall-policy.mjs +63 -15
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +1 -1
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.mjs +194 -32
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.test.mjs +143 -0
- package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +1 -44
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +34 -18
- package/src/runtime/agent/orchestrator/tools/builtin/external-tool-adapters.mjs +0 -0
- package/src/runtime/agent/orchestrator/tools/builtin/list-formatting.mjs +10 -0
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +5 -4
- package/src/runtime/agent/orchestrator/tools/builtin/path-utils.mjs +15 -0
- package/src/runtime/agent/orchestrator/tools/builtin/read-args.mjs +9 -44
- package/src/runtime/agent/orchestrator/tools/builtin/read-constants.mjs +2 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-formatting.mjs +13 -4
- package/src/runtime/agent/orchestrator/tools/builtin/read-tool.mjs +10 -17
- package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +18 -2
- package/src/runtime/agent/orchestrator/tools/builtin/shell-output.mjs +3 -2
- package/src/runtime/agent/orchestrator/tools/builtin/tool-output-limit.mjs +10 -0
- package/src/runtime/agent/orchestrator/tools/builtin.mjs +59 -1
- package/src/runtime/agent/orchestrator/tools/code-graph-tool-defs.mjs +5 -5
- package/src/runtime/agent/orchestrator/tools/code-graph.mjs +4076 -3985
- package/src/runtime/agent/orchestrator/tools/patch.mjs +116 -2
- package/src/runtime/channels/backends/discord.mjs +99 -9
- package/src/runtime/channels/backends/telegram.mjs +501 -0
- package/src/runtime/channels/index.mjs +441 -1254
- package/src/runtime/channels/lib/cli-worker-host.mjs +1 -8
- package/src/runtime/channels/lib/config.mjs +54 -3
- package/src/runtime/channels/lib/drop-trace.mjs +1 -1
- package/src/runtime/channels/lib/executor.mjs +0 -3
- package/src/runtime/channels/lib/format.mjs +4 -2
- package/src/runtime/channels/lib/memory-client.mjs +0 -38
- package/src/runtime/channels/lib/output-forwarder.mjs +77 -71
- package/src/runtime/channels/lib/runtime-paths.mjs +29 -6
- package/src/runtime/channels/lib/scheduler.mjs +1 -1
- package/src/runtime/channels/lib/session-discovery.mjs +0 -4
- package/src/runtime/channels/lib/telegram-format.mjs +283 -0
- package/src/runtime/channels/lib/tool-format.mjs +1 -2
- package/src/runtime/channels/lib/transcript-discovery.mjs +20 -11
- package/src/runtime/channels/lib/webhook.mjs +59 -31
- package/src/runtime/channels/tool-defs.mjs +1 -1
- package/src/runtime/lib/keychain-cjs.cjs +0 -1
- package/src/runtime/memory/data/runtime-manifest.json +6 -7
- package/src/runtime/memory/index.mjs +187 -43
- package/src/runtime/memory/lib/agent-ipc.mjs +2 -2
- package/src/runtime/memory/lib/core-memory-store.mjs +1 -1
- package/src/runtime/memory/lib/llm-worker-host.mjs +0 -4
- package/src/runtime/memory/lib/memory-cycle1.mjs +1 -1
- package/src/runtime/memory/lib/memory-cycle2.mjs +9 -6
- package/src/runtime/memory/lib/memory-cycle3.mjs +1 -1
- package/src/runtime/memory/lib/memory-ops-policy.mjs +0 -1
- package/src/runtime/memory/lib/memory.mjs +101 -4
- package/src/runtime/memory/lib/pg/adapter.mjs +139 -15
- package/src/runtime/memory/lib/runtime-fetcher.mjs +43 -18
- package/src/runtime/memory/lib/session-ingest.mjs +116 -7
- package/src/runtime/memory/lib/trace-store.mjs +69 -22
- package/src/runtime/memory/tool-defs.mjs +6 -3
- package/src/runtime/search/index.mjs +2 -7
- package/src/runtime/search/lib/config.mjs +0 -4
- package/src/runtime/search/lib/state.mjs +1 -15
- package/src/runtime/search/lib/web-tools.mjs +0 -1
- package/src/runtime/shared/channel-notification-routing.mjs +12 -0
- package/src/runtime/shared/channel-notification-routing.test.mjs +45 -0
- package/src/runtime/shared/child-spawn-gate.mjs +0 -6
- package/src/runtime/shared/config.mjs +9 -0
- package/src/runtime/shared/llm/http-agent.mjs +12 -5
- package/src/runtime/shared/schedules-store.mjs +21 -19
- package/src/runtime/shared/tool-surface.mjs +98 -13
- package/src/runtime/shared/transcript-writer.mjs +129 -0
- package/src/runtime/shared/update-checker.mjs +214 -0
- package/src/standalone/agent-tool.mjs +255 -109
- package/src/standalone/channel-admin.mjs +133 -40
- package/src/standalone/channel-worker.mjs +8 -291
- package/src/standalone/explore-tool.mjs +2 -2
- package/src/standalone/memory-runtime-proxy.mjs +3 -1
- package/src/standalone/provider-admin.mjs +11 -0
- package/src/standalone/seeds.mjs +1 -11
- package/src/standalone/usage-dashboard.mjs +1 -1
- package/src/tui/App.jsx +2137 -750
- package/src/tui/components/ConfirmBar.jsx +47 -0
- package/src/tui/components/ContextPanel.jsx +5 -3
- package/src/tui/components/ItemRightHintOverprint.jsx +54 -0
- package/src/tui/components/Markdown.jsx +22 -98
- package/src/tui/components/Message.jsx +14 -35
- package/src/tui/components/Picker.jsx +87 -12
- package/src/tui/components/PromptInput.jsx +146 -9
- package/src/tui/components/QueuedCommands.jsx +1 -1
- package/src/tui/components/SlashCommandPalette.jsx +8 -5
- package/src/tui/components/Spinner.jsx +7 -7
- package/src/tui/components/StatusLine.jsx +40 -21
- package/src/tui/components/TextEntryPanel.jsx +51 -7
- package/src/tui/components/ToolExecution.jsx +177 -100
- package/src/tui/components/TurnDone.jsx +4 -4
- package/src/tui/components/UsagePanel.jsx +1 -1
- package/src/tui/components/tool-output-format.mjs +312 -40
- package/src/tui/components/tool-output-format.test.mjs +180 -1
- package/src/tui/display-width.mjs +69 -0
- package/src/tui/display-width.test.mjs +35 -0
- package/src/tui/dist/index.mjs +7324 -2393
- package/src/tui/engine.mjs +287 -126
- package/src/tui/index.jsx +117 -7
- package/src/tui/keyboard-protocol.mjs +42 -0
- package/src/tui/lib/voice-recorder.mjs +453 -0
- package/src/tui/markdown/format-token.mjs +354 -142
- package/src/tui/markdown/format-token.test.mjs +155 -17
- package/src/tui/markdown/measure-rendered-rows.mjs +85 -0
- package/src/tui/markdown/render-ansi.test.mjs +1 -1
- package/src/tui/markdown/streaming-markdown.mjs +167 -0
- package/src/tui/markdown/streaming-markdown.test.mjs +70 -0
- package/src/tui/markdown/table-layout.mjs +9 -9
- package/src/tui/paste-attachments.mjs +0 -11
- package/src/tui/prompt-history-store.mjs +129 -0
- package/src/tui/prompt-history-store.test.mjs +52 -0
- package/src/tui/statusline-ansi-bridge.test.mjs +3 -3
- package/src/tui/theme.mjs +41 -647
- package/src/tui/themes/base.mjs +86 -0
- package/src/tui/themes/basic.mjs +85 -0
- package/src/tui/themes/catppuccin.mjs +72 -0
- package/src/tui/themes/dracula.mjs +70 -0
- package/src/tui/themes/everforest.mjs +71 -0
- package/src/tui/themes/gruvbox.mjs +71 -0
- package/src/tui/themes/index.mjs +71 -0
- package/src/tui/themes/indigo.mjs +78 -0
- package/src/tui/themes/kanagawa.mjs +80 -0
- package/src/tui/themes/light.mjs +81 -0
- package/src/tui/themes/nord.mjs +72 -0
- package/src/tui/themes/onedark.mjs +16 -0
- package/src/tui/themes/rosepine.mjs +70 -0
- package/src/tui/themes/teal.mjs +81 -0
- package/src/tui/themes/tokyonight.mjs +79 -0
- package/src/tui/themes/utils.mjs +106 -0
- package/src/tui/themes/warm.mjs +79 -0
- package/src/tui/transcript-tool-failures.mjs +13 -2
- package/src/ui/markdown.mjs +1 -1
- package/src/ui/model-display.mjs +2 -2
- package/src/ui/statusline.mjs +26 -27
- package/src/vendor/statusline/bin/statusline-lib.mjs +0 -623
- package/src/vendor/statusline/bin/statusline-route.mjs +5 -12
- package/src/vendor/statusline/src/gateway/claude-current.mjs +3 -3
- package/src/vendor/statusline/src/gateway/route-meta.mjs +30 -16
- package/src/workflows/default/WORKFLOW.md +39 -12
- package/src/workflows/sequential/WORKFLOW.md +46 -0
- package/src/workflows/solo/WORKFLOW.md +7 -0
- package/vendor/ink/build/display-width.js +62 -0
- package/vendor/ink/build/ink.js +154 -20
- package/vendor/ink/build/measure-text.js +4 -1
- package/vendor/ink/build/output.js +115 -9
- package/vendor/ink/build/render-node-to-output.js +4 -1
- package/vendor/ink/build/render.js +4 -0
- package/src/hooks/lib/permission-rules.cjs +0 -170
- package/src/hooks/lib/settings-loader.cjs +0 -112
- package/src/lib/hook-pipe-path.cjs +0 -10
- package/src/output-styles/extreme-simple.md +0 -20
- package/src/rules/lead/04-workflow.md +0 -51
- package/src/runtime/channels/lib/hook-pipe-server.mjs +0 -671
- package/src/workflows/default/workflow.json +0 -13
- package/src/workflows/solo/workflow.json +0 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixdog",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Standalone mixdog coding-agent CLI/TUI workspace.",
|
|
@@ -45,6 +45,13 @@
|
|
|
45
45
|
"test:providers": "node --test scripts/provider-toolcall-test.mjs",
|
|
46
46
|
"failures": "node scripts/tool-failures.mjs",
|
|
47
47
|
"trace:llm": "node scripts/llm-trace-summary.mjs",
|
|
48
|
+
"diag:sessions": "node scripts/session-diag.mjs",
|
|
49
|
+
"diag:session": "node scripts/session-diag.mjs",
|
|
50
|
+
"bench:session": "node scripts/session-bench.mjs",
|
|
51
|
+
"bench:task": "node scripts/task-bench.mjs",
|
|
52
|
+
"bench:corpus": "node scripts/routing-corpus.mjs",
|
|
53
|
+
"bench:run": "node scripts/bench-run.mjs",
|
|
54
|
+
"patch:replay": "node scripts/patch-replay.mjs",
|
|
48
55
|
"build:tui": "node scripts/build-tui.mjs"
|
|
49
56
|
},
|
|
50
57
|
"engines": {
|
|
@@ -57,6 +64,7 @@
|
|
|
57
64
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
58
65
|
"@mozilla/readability": "^0.6.0",
|
|
59
66
|
"chalk": "^5.6.2",
|
|
67
|
+
"cli-highlight": "^2.1.11",
|
|
60
68
|
"diff": "^9.0.0",
|
|
61
69
|
"discord.js": "^14.26.4",
|
|
62
70
|
"ink": "^7.1.0",
|
|
@@ -72,8 +80,7 @@
|
|
|
72
80
|
"undici": "^8.5.0",
|
|
73
81
|
"wrap-ansi": "^10.0.0",
|
|
74
82
|
"ws": "^8.21.0",
|
|
75
|
-
"zod": "^3.25.76"
|
|
76
|
-
"zod-to-json-schema": "^3.25.2"
|
|
83
|
+
"zod": "^3.25.76"
|
|
77
84
|
},
|
|
78
85
|
"overrides": {
|
|
79
86
|
"discord.js": {
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "cwc-hookflow",
|
|
4
|
+
"agent": "worker",
|
|
5
|
+
"prompt": "In this repo (mixdog), what happens to a tool call when no hooks are configured? Verify the default observer/bypass path: where beforeTool() is invoked, what it returns with no configured rules, and where a configured custom hook rule could instead deny the call. Give file paths and function names. Do not modify anything.",
|
|
6
|
+
"cwd": "."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"id": "cwc-cacheusage",
|
|
10
|
+
"agent": "worker",
|
|
11
|
+
"prompt": "In this repo (mixdog), how are cached token counts recorded and surfaced in usage metrics? Find where cached_tokens comes from and how it flows into the usage record. Give file paths and function names. Do not modify anything.",
|
|
12
|
+
"cwd": "."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "cwc-compacttrigger",
|
|
16
|
+
"agent": "worker",
|
|
17
|
+
"prompt": "In this repo (mixdog), what triggers a context compaction and where is that threshold decided? Trace the trigger condition to the compaction entry point. Give file paths and function names. Do not modify anything.",
|
|
18
|
+
"cwd": "."
|
|
19
|
+
}
|
|
20
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import {
|
|
5
|
+
resolveSessionMaxLoopIterations,
|
|
6
|
+
LEAD_MAX_LOOP_ITERATIONS,
|
|
7
|
+
} from '../src/runtime/agent/orchestrator/agent-runtime/agent-loop-policy.mjs';
|
|
8
|
+
|
|
9
|
+
// No low per-agent caps: every public/delegated agent shares the single high
|
|
10
|
+
// runaway guard (LEAD_MAX_LOOP_ITERATIONS) unless a session pins its own value.
|
|
11
|
+
test('agent owner session falls through to the shared runaway guard when unset', () => {
|
|
12
|
+
const cap = resolveSessionMaxLoopIterations({
|
|
13
|
+
owner: 'agent',
|
|
14
|
+
agent: 'heavy-worker',
|
|
15
|
+
permission: 'read-write',
|
|
16
|
+
maxLoopIterations: null,
|
|
17
|
+
});
|
|
18
|
+
assert.equal(cap, LEAD_MAX_LOOP_ITERATIONS);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test('lead session keeps the same shared ceiling when unset', () => {
|
|
22
|
+
assert.equal(resolveSessionMaxLoopIterations({ owner: 'user', agent: null }), LEAD_MAX_LOOP_ITERATIONS);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test('a session-pinned maxLoopIterations is honored', () => {
|
|
26
|
+
assert.equal(
|
|
27
|
+
resolveSessionMaxLoopIterations({ owner: 'agent', agent: 'heavy-worker', maxLoopIterations: 4 }),
|
|
28
|
+
4,
|
|
29
|
+
);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test('explicit override wins over everything', () => {
|
|
33
|
+
assert.equal(
|
|
34
|
+
resolveSessionMaxLoopIterations({ owner: 'agent', agent: 'heavy-worker', maxLoopIterations: 4 }, 50),
|
|
35
|
+
50,
|
|
36
|
+
);
|
|
37
|
+
});
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
// first-response watchdog is explicitly disabled (firstResponseTimeoutMs:0).
|
|
16
16
|
// 5. spawnPrepTimeoutMs:0 disables the prep cap even when an env default cap
|
|
17
17
|
// is set (explicit per-call override beats the env default).
|
|
18
|
+
// 6. A same-tag retry after a prep timeout cannot be blocked by the original
|
|
19
|
+
// timed-out prep binding the tag during late provider-init cleanup.
|
|
18
20
|
//
|
|
19
21
|
// Run: node scripts/agent-parallel-smoke.mjs
|
|
20
22
|
import { mkdtempSync, mkdirSync, rmSync } from 'node:fs';
|
|
@@ -103,7 +105,7 @@ async function fakeAskSession(sessionId, prompt) {
|
|
|
103
105
|
maxAskConcurrency = Math.max(maxAskConcurrency, askConcurrency);
|
|
104
106
|
try {
|
|
105
107
|
await sleep(300); // each agent's "work" lasts noticeably longer than spawn return
|
|
106
|
-
return { content: `ack ${session?.
|
|
108
|
+
return { content: `ack ${session?.agent || 'worker'}` };
|
|
107
109
|
} finally {
|
|
108
110
|
askConcurrency -= 1;
|
|
109
111
|
if (session) { session.status = 'idle'; session.lastStreamDeltaAt = Date.now(); }
|
|
@@ -145,7 +147,7 @@ async function main() {
|
|
|
145
147
|
const outs = await Promise.all(
|
|
146
148
|
Array.from({ length: batchSize }, (_, i) => agent.execute({
|
|
147
149
|
type: 'spawn',
|
|
148
|
-
|
|
150
|
+
agent: i % 2 === 0 ? 'worker' : 'reviewer',
|
|
149
151
|
tag: `par${i}`,
|
|
150
152
|
cwd: root,
|
|
151
153
|
prompt: `parallel task ${i}`,
|
|
@@ -176,7 +178,7 @@ async function main() {
|
|
|
176
178
|
const initBatch = await Promise.all(
|
|
177
179
|
Array.from({ length: 4 }, (_, i) => agent.execute({
|
|
178
180
|
type: 'spawn',
|
|
179
|
-
|
|
181
|
+
agent: 'worker',
|
|
180
182
|
tag: `init${i}`,
|
|
181
183
|
cwd: root,
|
|
182
184
|
prompt: `init task ${i}`,
|
|
@@ -195,14 +197,14 @@ async function main() {
|
|
|
195
197
|
const callsBeforeChange = initProvidersCalls;
|
|
196
198
|
// Same config first: must be skipped (no new init).
|
|
197
199
|
await waitJob(await agent.execute({
|
|
198
|
-
type: 'spawn',
|
|
200
|
+
type: 'spawn', agent: 'worker', tag: 'cfgSame', cwd: root, prompt: 'cfg same',
|
|
199
201
|
}, { invocationSource: 'model-tool', cwd: root }), /ack worker/, 'cfg same');
|
|
200
202
|
assert(initProvidersCalls === callsBeforeChange,
|
|
201
203
|
`unchanged config must not re-init; before=${callsBeforeChange} after=${initProvidersCalls}`);
|
|
202
204
|
// Now mutate the provider config and spawn again → must re-init once.
|
|
203
205
|
providerConfig = { 'openai-oauth': { enabled: true, baseUrl: 'https://changed.example' } };
|
|
204
206
|
await waitJob(await agent.execute({
|
|
205
|
-
type: 'spawn',
|
|
207
|
+
type: 'spawn', agent: 'worker', tag: 'cfgChanged', cwd: root, prompt: 'cfg changed',
|
|
206
208
|
}, { invocationSource: 'model-tool', cwd: root }), /ack worker/, 'cfg changed');
|
|
207
209
|
assert(initProvidersCalls === callsBeforeChange + 1,
|
|
208
210
|
`config change must trigger exactly one re-init; before=${callsBeforeChange} after=${initProvidersCalls}`);
|
|
@@ -267,7 +269,7 @@ async function main() {
|
|
|
267
269
|
// Fire A (slow), then B and C back-to-back while A is still running. B is
|
|
268
270
|
// superseded by C before A's chain link releases.
|
|
269
271
|
const aOut = await raceAgent.execute({
|
|
270
|
-
type: 'spawn',
|
|
272
|
+
type: 'spawn', agent: 'worker', tag: 'raceA', cwd: raceRoot, prompt: 'race A',
|
|
271
273
|
spawnPrepTimeoutMs: 0, // testing init serialization, not the prep cap
|
|
272
274
|
}, { invocationSource: 'model-tool', cwd: raceRoot });
|
|
273
275
|
// Let A's deferred job reach ensureProvider and START its slow (200ms) init,
|
|
@@ -277,12 +279,12 @@ async function main() {
|
|
|
277
279
|
await sleep(60);
|
|
278
280
|
raceConfig = { providers: { 'openai-oauth': { enabled: true, baseUrl: 'gen-b' } } };
|
|
279
281
|
const bOut = await raceAgent.execute({
|
|
280
|
-
type: 'spawn',
|
|
282
|
+
type: 'spawn', agent: 'worker', tag: 'raceB', cwd: raceRoot, prompt: 'race B',
|
|
281
283
|
spawnPrepTimeoutMs: 0,
|
|
282
284
|
}, { invocationSource: 'model-tool', cwd: raceRoot });
|
|
283
285
|
raceConfig = { providers: { 'openai-oauth': { enabled: true, baseUrl: 'gen-c' } } };
|
|
284
286
|
const cOut = await raceAgent.execute({
|
|
285
|
-
type: 'spawn',
|
|
287
|
+
type: 'spawn', agent: 'worker', tag: 'raceC', cwd: raceRoot, prompt: 'race C',
|
|
286
288
|
spawnPrepTimeoutMs: 0,
|
|
287
289
|
}, { invocationSource: 'model-tool', cwd: raceRoot });
|
|
288
290
|
await Promise.all([
|
|
@@ -325,7 +327,7 @@ async function main() {
|
|
|
325
327
|
const wedgeStart = Date.now();
|
|
326
328
|
const wedged = await hangAgent.execute({
|
|
327
329
|
type: 'spawn',
|
|
328
|
-
|
|
330
|
+
agent: 'worker',
|
|
329
331
|
tag: 'wedge1',
|
|
330
332
|
cwd: hangRoot,
|
|
331
333
|
prompt: 'wedge prep',
|
|
@@ -341,6 +343,48 @@ async function main() {
|
|
|
341
343
|
try { hangAgent.closeAll('agent-parallel-smoke-hang-end'); } catch {}
|
|
342
344
|
rmSync(hangRoot, { recursive: true, force: true });
|
|
343
345
|
|
|
346
|
+
// --- 4b: same-tag retry after prep timeout must not race late bind/cleanup ---
|
|
347
|
+
const retryRoot = mkdtempSync(join(tmpdir(), 'mixdog-agent-retry-prep-'));
|
|
348
|
+
const retryDataDir = join(retryRoot, '.mixdog-data');
|
|
349
|
+
mkdirSync(retryDataDir, { recursive: true });
|
|
350
|
+
let releaseRetryInit;
|
|
351
|
+
const retryInitGate = new Promise((resolve) => { releaseRetryInit = resolve; });
|
|
352
|
+
let retryInitCalls = 0;
|
|
353
|
+
const retryReg = {
|
|
354
|
+
getProvider() { return undefined; }, // force every spawn through shared init
|
|
355
|
+
async initProviders(config) {
|
|
356
|
+
retryInitCalls += 1;
|
|
357
|
+
await retryInitGate;
|
|
358
|
+
return realInitProviders({ 'openai-oauth': { enabled: true }, ...config });
|
|
359
|
+
},
|
|
360
|
+
};
|
|
361
|
+
const retryAgent = createStandaloneAgent({ cfgMod, reg: retryReg, mgr, dataDir: retryDataDir, cwd: retryRoot });
|
|
362
|
+
const retryFirst = await retryAgent.execute({
|
|
363
|
+
type: 'spawn',
|
|
364
|
+
agent: 'worker',
|
|
365
|
+
tag: 'retrySameTag',
|
|
366
|
+
cwd: retryRoot,
|
|
367
|
+
prompt: 'retry first should timeout before provider init releases',
|
|
368
|
+
spawnPrepTimeoutMs: 50,
|
|
369
|
+
}, { invocationSource: 'model-tool', cwd: retryRoot });
|
|
370
|
+
await waitJob(retryFirst, /timed out|status: failed/, 'same-tag first prep timeout', 20);
|
|
371
|
+
assert(retryInitCalls === 1, `first retry spawn should start one gated init; calls=${retryInitCalls}`);
|
|
372
|
+
const retrySecond = await retryAgent.execute({
|
|
373
|
+
type: 'spawn',
|
|
374
|
+
agent: 'worker',
|
|
375
|
+
tag: 'retrySameTag',
|
|
376
|
+
cwd: retryRoot,
|
|
377
|
+
prompt: 'retry second should succeed after init releases',
|
|
378
|
+
spawnPrepTimeoutMs: 0,
|
|
379
|
+
}, { invocationSource: 'model-tool', cwd: retryRoot });
|
|
380
|
+
assert(/agent task:/.test(retrySecond), `same-tag retry should return a task: ${retrySecond}`);
|
|
381
|
+
await sleep(30); // let the retry join the gated provider init before release
|
|
382
|
+
releaseRetryInit();
|
|
383
|
+
const retrySecondResult = await waitJob(retrySecond, /ack worker/, 'same-tag retry success', 60);
|
|
384
|
+
assert(!/already exists/i.test(retrySecondResult), `same-tag retry hit stale bind: ${retrySecondResult}`);
|
|
385
|
+
try { retryAgent.closeAll('agent-parallel-smoke-retry-end'); } catch {}
|
|
386
|
+
rmSync(retryRoot, { recursive: true, force: true });
|
|
387
|
+
|
|
344
388
|
// --- 5: spawnPrepTimeoutMs:0 disables the prep cap even when an env default
|
|
345
389
|
// is set. The module captured ENV_PREP_CAP_MS (150ms) at load. A prep step
|
|
346
390
|
// that runs longer than that (300ms) would normally be aborted by the env
|
|
@@ -362,7 +406,7 @@ async function main() {
|
|
|
362
406
|
const slowStart = Date.now();
|
|
363
407
|
const slowOut = await slowAgent.execute({
|
|
364
408
|
type: 'spawn',
|
|
365
|
-
|
|
409
|
+
agent: 'worker',
|
|
366
410
|
tag: 'slowprep1',
|
|
367
411
|
cwd: slowRoot,
|
|
368
412
|
prompt: 'slow prep override',
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// bench-run.mjs — repeatable internal A/B bench runner (mixdog).
|
|
3
|
+
//
|
|
4
|
+
// Runs a FIXED task set through the LIVE headless path (runHeadlessRole in a
|
|
5
|
+
// fresh child node process = identical to `mixdog <agent> <msg>` and loads the
|
|
6
|
+
// latest on-disk code), captures each task's session id, scores the round with
|
|
7
|
+
// task-bench, and freezes it to a round file. Change rules/briefs between
|
|
8
|
+
// rounds; the task set stays constant so rounds are comparable.
|
|
9
|
+
//
|
|
10
|
+
// node scripts/bench-run.mjs --tasks tasks.json --round 1 --save round1.json
|
|
11
|
+
// (edit rules/briefs)
|
|
12
|
+
// node scripts/bench-run.mjs --tasks tasks.json --round 2 --save round2.json
|
|
13
|
+
// node scripts/task-bench.mjs --vs round1.json round2.json
|
|
14
|
+
//
|
|
15
|
+
// tasks.json: [{ "id":"...", "agent":"worker", "prompt":"...", "cwd":"." }, ...]
|
|
16
|
+
//
|
|
17
|
+
// This is INTERNAL A/B only (mixdog vs its own prior round). Cross-CLI compare
|
|
18
|
+
// vs codex/claude (tree-total vs solo) is a separate future mode; codex/claude
|
|
19
|
+
// runners are left as slots.
|
|
20
|
+
import { execFileSync } from 'node:child_process';
|
|
21
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
22
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
23
|
+
import { dirname, resolve } from 'node:path';
|
|
24
|
+
|
|
25
|
+
const __dir = dirname(fileURLToPath(import.meta.url));
|
|
26
|
+
const HEADLESS = pathToFileURL(resolve(__dir, '../src/headless-role.mjs')).href;
|
|
27
|
+
const TASK_BENCH = resolve(__dir, 'task-bench.mjs');
|
|
28
|
+
|
|
29
|
+
function argValue(name, fallback = null) {
|
|
30
|
+
const idx = process.argv.indexOf(name);
|
|
31
|
+
if (idx >= 0 && idx + 1 < process.argv.length) return process.argv[idx + 1];
|
|
32
|
+
const pref = `${name}=`;
|
|
33
|
+
const hit = process.argv.find((a) => a.startsWith(pref));
|
|
34
|
+
return hit ? hit.slice(pref.length) : fallback;
|
|
35
|
+
}
|
|
36
|
+
function hasFlag(name) { return process.argv.includes(name); }
|
|
37
|
+
|
|
38
|
+
// Model aliases so a round switches model+provider with one flag
|
|
39
|
+
// (--model opus|gpt|grok). Full provider/model pairs still work verbatim.
|
|
40
|
+
const MODEL_ALIASES = {
|
|
41
|
+
opus: { provider: 'anthropic-oauth', model: 'claude-opus-4-8' },
|
|
42
|
+
sonnet: { provider: 'anthropic-oauth', model: 'claude-sonnet-5' },
|
|
43
|
+
gpt: { provider: 'openai-oauth', model: 'gpt-5.5' },
|
|
44
|
+
'gpt-5.5': { provider: 'openai-oauth', model: 'gpt-5.5' },
|
|
45
|
+
grok: { provider: 'grok-oauth', model: 'grok-composer-2.5-fast' },
|
|
46
|
+
};
|
|
47
|
+
function resolveModelOpts(modelArg, providerArg) {
|
|
48
|
+
const key = String(modelArg || '').trim().toLowerCase();
|
|
49
|
+
if (MODEL_ALIASES[key] && !providerArg) return { ...MODEL_ALIASES[key] };
|
|
50
|
+
return { provider: providerArg || null, model: modelArg || null };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function extractSessionId(text) {
|
|
54
|
+
const s = String(text || '');
|
|
55
|
+
const m = s.match(/sessionId:\s*(sess_[A-Za-z0-9_]+)/) || s.match(/\b(sess_[A-Za-z0-9_]+)/);
|
|
56
|
+
return m ? m[1] : null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const RUNNERS = {
|
|
60
|
+
mixdog(task, opts) {
|
|
61
|
+
const driver = [
|
|
62
|
+
`import { runHeadlessRole } from ${JSON.stringify(HEADLESS)};`,
|
|
63
|
+
`const out = [];`,
|
|
64
|
+
`const code = await runHeadlessRole({`,
|
|
65
|
+
` agent: ${JSON.stringify(task.agent || 'worker')},`,
|
|
66
|
+
` message: ${JSON.stringify(task.prompt || '')},`,
|
|
67
|
+
` provider: ${JSON.stringify(opts.provider || null)},`,
|
|
68
|
+
` model: ${JSON.stringify(opts.model || null)},`,
|
|
69
|
+
` cwd: ${JSON.stringify(task.cwd ? resolve(task.cwd) : process.cwd())},`,
|
|
70
|
+
` write: (t) => out.push(t),`,
|
|
71
|
+
` writeErr: (t) => process.stderr.write(t),`,
|
|
72
|
+
`});`,
|
|
73
|
+
`process.stdout.write(out.join(''));`,
|
|
74
|
+
`process.exit(code);`,
|
|
75
|
+
].join('\n');
|
|
76
|
+
const started = Date.now();
|
|
77
|
+
let raw = '';
|
|
78
|
+
let ok = false;
|
|
79
|
+
try {
|
|
80
|
+
raw = execFileSync('node', ['--input-type=module', '-e', driver], {
|
|
81
|
+
encoding: 'utf8',
|
|
82
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
83
|
+
env: {
|
|
84
|
+
...process.env,
|
|
85
|
+
...(opts.effort ? { MIXDOG_AGENT_EFFORT: opts.effort } : {}),
|
|
86
|
+
...(opts.fast ? { MIXDOG_AGENT_FAST: '1' } : {}),
|
|
87
|
+
...(opts.env || {}),
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
ok = true;
|
|
91
|
+
} catch (e) {
|
|
92
|
+
raw = String(e.stdout || '') + String(e.stderr || '');
|
|
93
|
+
ok = false;
|
|
94
|
+
}
|
|
95
|
+
return { sessionId: extractSessionId(raw), ok, ms: Date.now() - started, raw };
|
|
96
|
+
},
|
|
97
|
+
codex() {
|
|
98
|
+
throw new Error('runner "codex" not implemented (slot: codex exec --json). Cross-CLI compare is a separate mode.');
|
|
99
|
+
},
|
|
100
|
+
claude() {
|
|
101
|
+
throw new Error('runner "claude" not implemented (slot: claude -p --output-format json). Cross-CLI compare is a separate mode.');
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
function scoreSessions(ids) {
|
|
106
|
+
if (!ids.length) return null;
|
|
107
|
+
const raw = execFileSync('node', [TASK_BENCH, '--session', ids.join(','), '--group', '--json'], {
|
|
108
|
+
encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
|
|
109
|
+
});
|
|
110
|
+
const s = raw.replace(/^\uFEFF/, '');
|
|
111
|
+
const i = s.indexOf('{');
|
|
112
|
+
return JSON.parse(i >= 0 ? s.slice(i) : s);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function sleepMs(ms) {
|
|
116
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, ms));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function scoreSession(sessionId, { attempts = 5 } = {}) {
|
|
120
|
+
let lastError = null;
|
|
121
|
+
for (let i = 0; i < attempts; i += 1) {
|
|
122
|
+
try {
|
|
123
|
+
const raw = execFileSync('node', [TASK_BENCH, '--session', sessionId, '--json'], {
|
|
124
|
+
encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
|
|
125
|
+
});
|
|
126
|
+
const s = raw.replace(/^\uFEFF/, '');
|
|
127
|
+
const j = JSON.parse(s.slice(Math.max(0, s.indexOf('{'))));
|
|
128
|
+
const card = Array.isArray(j.cards) ? j.cards.find((c) => c?.session === sessionId) || j.cards[0] : null;
|
|
129
|
+
if (card) return { ok: true, card };
|
|
130
|
+
lastError = new Error(`no scorecard returned for ${sessionId}`);
|
|
131
|
+
} catch (e) {
|
|
132
|
+
lastError = e;
|
|
133
|
+
}
|
|
134
|
+
sleepMs(250 * (i + 1));
|
|
135
|
+
}
|
|
136
|
+
return { ok: false, error: lastError?.message || String(lastError || 'score failed') };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function num(v) {
|
|
140
|
+
const n = Number(v);
|
|
141
|
+
return Number.isFinite(n) ? n : 0;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function averageCards(cards) {
|
|
145
|
+
if (!cards.length) return null;
|
|
146
|
+
const keys = Object.keys(cards[0]).filter((k) => cards.some((c) => typeof c[k] === 'number' && Number.isFinite(c[k])));
|
|
147
|
+
const out = { n: cards.length };
|
|
148
|
+
for (const k of keys) {
|
|
149
|
+
const vals = cards.map((c) => num(c[k]));
|
|
150
|
+
out[k] = Math.round((vals.reduce((a, b) => a + b, 0) / vals.length) * 10) / 10;
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ---- main ----
|
|
156
|
+
const tasksPath = argValue('--tasks', null);
|
|
157
|
+
if (!tasksPath) {
|
|
158
|
+
console.error('usage: --tasks <tasks.json> [--round N] [--runner mixdog] [--provider P] [--model M] [--effort E] [--fast] [--save round.json] [--json]');
|
|
159
|
+
process.exit(1);
|
|
160
|
+
}
|
|
161
|
+
if (!existsSync(resolve(tasksPath))) { console.error(`tasks file not found: ${tasksPath}`); process.exit(1); }
|
|
162
|
+
const tasks = JSON.parse(readFileSync(resolve(tasksPath), 'utf8'));
|
|
163
|
+
if (!Array.isArray(tasks) || !tasks.length) { console.error('tasks.json must be a non-empty array'); process.exit(1); }
|
|
164
|
+
|
|
165
|
+
const runnerName = argValue('--runner', 'mixdog');
|
|
166
|
+
const runner = RUNNERS[runnerName];
|
|
167
|
+
if (!runner) { console.error(`unknown runner "${runnerName}" (mixdog|codex|claude)`); process.exit(1); }
|
|
168
|
+
const round = argValue('--round', '1');
|
|
169
|
+
const savePath = argValue('--save', null);
|
|
170
|
+
const jsonMode = hasFlag('--json');
|
|
171
|
+
const _mo = resolveModelOpts(argValue('--model', null), argValue('--provider', null));
|
|
172
|
+
// A/B knobs -> child env. --read-max-lines is a convenience for the read-policy
|
|
173
|
+
// bench; --env KEY=VAL (repeatable) passes any override verbatim.
|
|
174
|
+
const _env = {};
|
|
175
|
+
const _readMax = argValue('--read-max-lines', null);
|
|
176
|
+
if (_readMax) _env.MIXDOG_READ_MAX_LINES = String(_readMax);
|
|
177
|
+
for (let i = 0; i < process.argv.length; i++) {
|
|
178
|
+
if (process.argv[i] === '--env' && process.argv[i + 1]) {
|
|
179
|
+
const eq = process.argv[i + 1].indexOf('=');
|
|
180
|
+
if (eq > 0) _env[process.argv[i + 1].slice(0, eq)] = process.argv[i + 1].slice(eq + 1);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
const opts = {
|
|
184
|
+
provider: _mo.provider,
|
|
185
|
+
model: _mo.model,
|
|
186
|
+
effort: argValue('--effort', null),
|
|
187
|
+
fast: hasFlag('--fast'),
|
|
188
|
+
env: _env,
|
|
189
|
+
};
|
|
190
|
+
process.stderr.write(`[bench-run] model=${opts.model || '(agent default)'} provider=${opts.provider || '(agent default)'} effort=${opts.effort || '-'} fast=${opts.fast} env=${JSON.stringify(_env)}\n`);
|
|
191
|
+
|
|
192
|
+
const results = [];
|
|
193
|
+
for (const task of tasks) {
|
|
194
|
+
process.stderr.write(`[bench-run] round=${round} runner=${runnerName} task=${task.id || task.agent} ...\n`);
|
|
195
|
+
let r;
|
|
196
|
+
try { r = runner(task, opts); }
|
|
197
|
+
catch (e) { console.error(`[bench-run] runner error: ${e.message}`); process.exit(1); }
|
|
198
|
+
process.stderr.write(`[bench-run] -> ${r.ok ? 'ok' : 'FAIL'} ${Math.round(r.ms / 1000)}s session=${r.sessionId || '(none)'}\n`);
|
|
199
|
+
const result = { id: task.id || null, agent: task.agent || null, ok: r.ok, ms: r.ms, sessionId: r.sessionId };
|
|
200
|
+
if (r.sessionId) {
|
|
201
|
+
const scored = scoreSession(r.sessionId);
|
|
202
|
+
if (scored.ok) {
|
|
203
|
+
result.card = scored.card;
|
|
204
|
+
process.stderr.write(`[bench-run] -> scored ${r.sessionId.slice(0, 22)} turns=${scored.card.turns} tools=${scored.card.tool_calls}\n`);
|
|
205
|
+
} else {
|
|
206
|
+
result.scoreError = scored.error;
|
|
207
|
+
process.stderr.write(`[bench-run] -> SCORE FAIL ${r.sessionId.slice(0, 22)} ${scored.error}\n`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
results.push(result);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const sessionIds = results.map((r) => r.sessionId).filter(Boolean);
|
|
214
|
+
const scoredCards = results.map((r) => r.card).filter(Boolean);
|
|
215
|
+
const scoreErrors = results
|
|
216
|
+
.filter((r) => !r.card)
|
|
217
|
+
.map((r) => ({ id: r.id, sessionId: r.sessionId || null, error: r.sessionId ? (r.scoreError || 'missing scorecard') : 'missing sessionId' }));
|
|
218
|
+
const score = scoredCards.length ? { cards: scoredCards, group: averageCards(scoredCards.map(({ session, ...c }) => c)) } : null;
|
|
219
|
+
const completed = results.filter((r) => r.ok).length;
|
|
220
|
+
const taskErrors = results
|
|
221
|
+
.filter((r) => !r.ok)
|
|
222
|
+
.map((r) => ({ id: r.id, sessionId: r.sessionId || null, error: 'task failed' }));
|
|
223
|
+
const roundResult = {
|
|
224
|
+
round, runner: runnerName, opts,
|
|
225
|
+
tasks: results.length, completed,
|
|
226
|
+
completion_rate: results.length ? Math.round((completed / results.length) * 100) : 0,
|
|
227
|
+
sessions: sessionIds, results,
|
|
228
|
+
task_complete: results.length > 0 && completed === results.length,
|
|
229
|
+
task_errors: taskErrors,
|
|
230
|
+
score_complete: results.length > 0 && taskErrors.length === 0 && scoreErrors.length === 0 && (score?.cards?.length || 0) === results.length,
|
|
231
|
+
score_errors: scoreErrors,
|
|
232
|
+
group: score?.group || null,
|
|
233
|
+
cards: score?.cards || null,
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
if (roundResult.task_complete === false) {
|
|
237
|
+
console.error(`[bench-run] incomplete tasks: completed=${completed}/${results.length}`);
|
|
238
|
+
for (const e of taskErrors) console.error(`[bench-run] task error ${e.id || '-'} ${e.sessionId || '(no session)'}`);
|
|
239
|
+
}
|
|
240
|
+
if (roundResult.score_complete === false) {
|
|
241
|
+
console.error(`[bench-run] incomplete scoring: scored=${roundResult.cards?.length || 0}/${results.length}`);
|
|
242
|
+
for (const e of scoreErrors) console.error(`[bench-run] score error ${e.id || '-'} ${e.sessionId || '(no session)'}: ${e.error}`);
|
|
243
|
+
}
|
|
244
|
+
if (savePath) {
|
|
245
|
+
if (roundResult.task_complete === false || roundResult.score_complete === false) {
|
|
246
|
+
console.error(`[bench-run] not saving incomplete round -> ${resolve(savePath)}`);
|
|
247
|
+
} else {
|
|
248
|
+
writeFileSync(resolve(savePath), JSON.stringify(roundResult, null, 2));
|
|
249
|
+
console.error(`[bench-run] saved round ${round} -> ${resolve(savePath)}`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
if (jsonMode) {
|
|
253
|
+
console.log(JSON.stringify(roundResult, null, 2));
|
|
254
|
+
} else {
|
|
255
|
+
console.log(`round=${round} runner=${runnerName} tasks=${results.length} completed=${completed} (${roundResult.completion_rate}%)`);
|
|
256
|
+
for (const r of results) console.log(`- ${r.id || r.agent}: ${r.ok ? 'ok' : 'FAIL'} ${Math.round(r.ms / 1000)}s ${r.sessionId ? r.sessionId.slice(0, 22) : '(no session)'}`);
|
|
257
|
+
if (roundResult.group) {
|
|
258
|
+
const g = roundResult.group;
|
|
259
|
+
console.log(`group: wall=${Math.round((g.wall_ms || 0) / 1000)}s turns=${g.turns} tools=${g.tool_calls} tpt=${g.tools_per_turn} tool_ms=${Math.round((g.total_tool_ms || 0) / 1000)}s cache=${Math.round((g.cache_ratio || 0) * 100)}% antipatterns=${g.antipatterns}`);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
if (roundResult.task_complete === false || roundResult.score_complete === false) process.exit(1);
|
|
@@ -92,6 +92,18 @@ assert(semanticForced.semantic === true && semanticCalls === 1, 'forced semantic
|
|
|
92
92
|
assert(semanticForced.compactType === COMPACT_TYPE_SEMANTIC, 'semantic compact should report compact type 1');
|
|
93
93
|
assert(findSummary(semanticForced.messages), 'forced semantic compact should insert an anchored summary');
|
|
94
94
|
|
|
95
|
+
let wholeTranscriptForcedCalls = 0;
|
|
96
|
+
const wholeTranscriptProvider = {
|
|
97
|
+
name: 'whole-transcript-forced-smoke',
|
|
98
|
+
async send() {
|
|
99
|
+
wholeTranscriptForcedCalls += 1;
|
|
100
|
+
return { content: '## Goal\n- continue whole-transcript forced compact\n\n## Constraints & Preferences\n- (none)\n\n## Progress\n### Done\n- oldest turn summarized\n\n### In Progress\n- (none)\n\n### Blocked\n- (none)\n\n## Key Decisions\n- (none)\n\n## Next Steps\n- continue\n\n## Critical Context\n- compact.mjs\n\n## Relevant Files\n- compact.mjs' };
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
const wholeTranscriptForced = await semanticCompactMessages(wholeTranscriptProvider, semanticMessages, 'fake-model', 5_000, { force: true });
|
|
104
|
+
assert(wholeTranscriptForced.semantic === true && wholeTranscriptForcedCalls === 1, 'forced semantic compact must summarize when tail budget would preserve the whole live transcript (default tailTurns)');
|
|
105
|
+
assert(findSummary(wholeTranscriptForced.messages), 'whole-transcript forced semantic compact should insert an anchored summary');
|
|
106
|
+
|
|
95
107
|
assert(normalizeCompactType('type1') === COMPACT_TYPE_SEMANTIC, 'type1 should resolve to semantic compact');
|
|
96
108
|
assert(normalizeCompactType('recall-fasttrack') === COMPACT_TYPE_RECALL_FASTTRACK, 'type2 should resolve to recall fast-track compact');
|
|
97
109
|
// Alias parity: snake_case + dash + no-dash spellings of fast-track must all
|
|
@@ -7,10 +7,16 @@ import {
|
|
|
7
7
|
_compactTriggerForSession as compactTriggerForSession,
|
|
8
8
|
_preserveBufferConfigFields as preserveBufferConfigFields,
|
|
9
9
|
} from '../src/runtime/agent/orchestrator/session/manager.mjs';
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
_autoCompactTokenLimit as routeMetaAutoCompactTokenLimit,
|
|
12
|
+
autoCompactWindowForRoute,
|
|
13
|
+
compactBoundaryDenominator,
|
|
14
|
+
summarizeGatewayUsage,
|
|
15
|
+
} from '../src/vendor/statusline/src/gateway/route-meta.mjs';
|
|
11
16
|
import {
|
|
12
17
|
_routeContextMeta as routeContextMeta,
|
|
13
18
|
_resolveStatusAutoCompactTokenLimit as resolveStatusAutoCompactTokenLimit,
|
|
19
|
+
compactBoundaryForStatus,
|
|
14
20
|
} from '../src/vendor/statusline/bin/statusline-route.mjs';
|
|
15
21
|
|
|
16
22
|
function assert(condition, message) {
|
|
@@ -184,4 +190,64 @@ function assert(condition, message) {
|
|
|
184
190
|
`status: configured explicit limit should surface, got ${configuredExplicit}`);
|
|
185
191
|
}
|
|
186
192
|
|
|
193
|
+
// 10) Boundary / window denominators ignore autoCompactTokenLimit (trigger-only).
|
|
194
|
+
{
|
|
195
|
+
const boundary = 200000;
|
|
196
|
+
const trigger = 150000;
|
|
197
|
+
const route = {
|
|
198
|
+
contextWindow: boundary,
|
|
199
|
+
compactBoundaryTokens: boundary,
|
|
200
|
+
rawContextWindow: boundary,
|
|
201
|
+
autoCompactTokenLimit: trigger,
|
|
202
|
+
};
|
|
203
|
+
assert(compactBoundaryForStatus(route) === boundary,
|
|
204
|
+
`status boundary should be ${boundary}, got ${compactBoundaryForStatus(route)}`);
|
|
205
|
+
assert(compactBoundaryForStatus({ contextWindow: boundary, autoCompactTokenLimit: trigger }) === boundary,
|
|
206
|
+
`status boundary should use contextWindow ${boundary}, got ${compactBoundaryForStatus({ contextWindow: boundary, autoCompactTokenLimit: trigger })}`);
|
|
207
|
+
assert(autoCompactWindowForRoute({ contextWindow: boundary, autoCompactTokenLimit: trigger }) === boundary,
|
|
208
|
+
`route compact window should be ${boundary}, got ${autoCompactWindowForRoute({ contextWindow: boundary, autoCompactTokenLimit: trigger })}`);
|
|
209
|
+
const trig = compactTriggerForSession({ autoCompactTokenLimit: trigger, compaction: {} }, boundary);
|
|
210
|
+
assert(trig === trigger, `trigger resolver should stay ${trigger}, got ${trig}`);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// 11) Boundary fallback order and compact boundary fields.
|
|
214
|
+
{
|
|
215
|
+
const context = 200000;
|
|
216
|
+
const smallerBudget = 150000;
|
|
217
|
+
assert(
|
|
218
|
+
compactBoundaryForStatus({ contextWindow: context }, { budgetWindow: smallerBudget, compactLimitTokens: smallerBudget }) === context,
|
|
219
|
+
`contextWindow ${context} must win over smaller budgetWindow ${smallerBudget}`,
|
|
220
|
+
);
|
|
221
|
+
assert(
|
|
222
|
+
compactBoundaryDenominator({ contextWindow: context }, { budgetWindow: smallerBudget }) === context,
|
|
223
|
+
`route denominator: contextWindow must win over budgetWindow`,
|
|
224
|
+
);
|
|
225
|
+
assert(
|
|
226
|
+
compactBoundaryForStatus({ contextWindow: context }, { boundaryTokens: 180000 }) === 180000,
|
|
227
|
+
'compact.boundaryTokens should be honored',
|
|
228
|
+
);
|
|
229
|
+
assert(
|
|
230
|
+
compactBoundaryForStatus({ contextWindow: context }, { compactBoundaryTokens: 190000 }) === 190000,
|
|
231
|
+
'compact.compactBoundaryTokens should be honored',
|
|
232
|
+
);
|
|
233
|
+
assert(
|
|
234
|
+
compactBoundaryForStatus({ boundaryTokens: 175000, contextWindow: context }, { budgetWindow: smallerBudget }) === 175000,
|
|
235
|
+
'routeInfo.boundaryTokens should be honored',
|
|
236
|
+
);
|
|
237
|
+
const withLimitOnly = compactBoundaryForStatus(
|
|
238
|
+
{ contextWindow: context, autoCompactTokenLimit: smallerBudget },
|
|
239
|
+
{ compactLimitTokens: smallerBudget, budgetWindow: smallerBudget },
|
|
240
|
+
);
|
|
241
|
+
assert(withLimitOnly === context,
|
|
242
|
+
`compactLimitTokens/budget must not shrink boundary; expected ${context}, got ${withLimitOnly}`);
|
|
243
|
+
const usage = summarizeGatewayUsage(
|
|
244
|
+
{ provider: 'openai-oauth', model: 'gpt-5.5', contextWindow: context },
|
|
245
|
+
{ usage: { inputTokens: 1, outputTokens: 1 } },
|
|
246
|
+
{ boundaryTokens: 180000, budgetWindow: smallerBudget, compactLimitTokens: smallerBudget, afterTokens: 90000 },
|
|
247
|
+
1,
|
|
248
|
+
);
|
|
249
|
+
assert(usage.contextUsedPct === 50,
|
|
250
|
+
`gateway usage pct should use compact.boundaryTokens denominator 180000; got ${usage.contextUsedPct}`);
|
|
251
|
+
}
|
|
252
|
+
|
|
187
253
|
process.stdout.write('compact-trigger-migration smoke passed ✓\n');
|