@oh-my-pi/pi-coding-agent 15.7.5 → 15.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -1
- package/dist/types/config/settings-schema.d.ts +9 -0
- package/dist/types/eval/__tests__/heartbeat.test.d.ts +1 -0
- package/dist/types/eval/heartbeat.d.ts +45 -0
- package/dist/types/extensibility/extensions/loader.d.ts +2 -2
- package/dist/types/extensibility/extensions/runner.d.ts +2 -1
- package/dist/types/extensibility/extensions/types.d.ts +24 -5
- package/dist/types/lsp/diagnostics-ledger.d.ts +10 -0
- package/dist/types/lsp/index.d.ts +2 -0
- package/dist/types/lsp/utils.d.ts +4 -0
- package/dist/types/modes/acp/acp-agent.d.ts +1 -1
- package/dist/types/modes/components/assistant-message.d.ts +3 -1
- package/dist/types/modes/components/custom-editor.d.ts +0 -1
- package/dist/types/modes/components/hook-selector.d.ts +7 -1
- package/dist/types/modes/controllers/extension-ui-controller.d.ts +2 -2
- package/dist/types/modes/interactive-mode.d.ts +2 -2
- package/dist/types/modes/rpc/rpc-mode.d.ts +1 -1
- package/dist/types/modes/theme/theme.d.ts +1 -1
- package/dist/types/modes/types.d.ts +2 -2
- package/dist/types/session/agent-session.d.ts +8 -1
- package/dist/types/tools/ask.d.ts +8 -6
- package/dist/types/tools/eval-backends.d.ts +12 -0
- package/dist/types/tools/eval-render.d.ts +52 -0
- package/dist/types/tools/eval.d.ts +2 -35
- package/dist/types/tools/index.d.ts +4 -11
- package/dist/types/tui/output-block.d.ts +4 -3
- package/examples/extensions/README.md +1 -0
- package/examples/extensions/thinking-note.ts +13 -0
- package/package.json +9 -9
- package/src/config/model-registry.ts +33 -4
- package/src/config/settings-schema.ts +10 -0
- package/src/discovery/claude.ts +41 -22
- package/src/edit/index.ts +23 -3
- package/src/eval/__tests__/agent-bridge.test.ts +90 -0
- package/src/eval/__tests__/heartbeat.test.ts +84 -0
- package/src/eval/__tests__/llm-bridge.test.ts +30 -0
- package/src/eval/agent-bridge.ts +44 -38
- package/src/eval/heartbeat.ts +74 -0
- package/src/eval/js/executor.ts +13 -9
- package/src/eval/llm-bridge.ts +20 -14
- package/src/eval/py/executor.ts +14 -18
- package/src/exec/bash-executor.ts +31 -5
- package/src/extensibility/extensions/loader.ts +16 -18
- package/src/extensibility/extensions/runner.ts +22 -17
- package/src/extensibility/extensions/types.ts +39 -5
- package/src/internal-urls/docs-index.generated.ts +3 -3
- package/src/lsp/diagnostics-ledger.ts +51 -0
- package/src/lsp/index.ts +9 -22
- package/src/lsp/utils.ts +21 -0
- package/src/modes/acp/acp-agent.ts +8 -4
- package/src/modes/components/assistant-message.ts +28 -1
- package/src/modes/components/custom-editor.ts +9 -7
- package/src/modes/components/hook-selector.ts +159 -32
- package/src/modes/components/tool-execution.ts +20 -4
- package/src/modes/controllers/event-controller.ts +5 -2
- package/src/modes/controllers/extension-ui-controller.ts +3 -2
- package/src/modes/controllers/input-controller.ts +0 -15
- package/src/modes/interactive-mode.ts +2 -1
- package/src/modes/rpc/rpc-mode.ts +17 -6
- package/src/modes/theme/theme-schema.json +30 -0
- package/src/modes/theme/theme.ts +39 -2
- package/src/modes/types.ts +2 -1
- package/src/modes/utils/ui-helpers.ts +5 -2
- package/src/prompts/tools/ask.md +2 -1
- package/src/prompts/tools/eval.md +1 -1
- package/src/session/agent-session.ts +65 -11
- package/src/tools/ask.ts +74 -32
- package/src/tools/eval-backends.ts +38 -0
- package/src/tools/eval-render.ts +750 -0
- package/src/tools/eval.ts +27 -754
- package/src/tools/index.ts +7 -37
- package/src/tools/renderers.ts +1 -1
- package/src/tools/write.ts +9 -1
- package/src/tui/output-block.ts +5 -4
|
@@ -14,9 +14,10 @@ export interface OutputBlockOptions {
|
|
|
14
14
|
animate?: boolean;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
* Monotonic frame counter for animated borders
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* Monotonic frame counter for animated borders, quantized to the TUI's ~16ms
|
|
18
|
+
* render cap so the cache key advances once per ~60fps frame — fine enough for a
|
|
19
|
+
* smooth segment sweep, coarse enough to coalesce multiple render passes that
|
|
20
|
+
* land inside the same frame.
|
|
20
21
|
*/
|
|
21
22
|
export declare function borderShimmerTick(): number;
|
|
22
23
|
/**
|
|
@@ -41,6 +41,7 @@ cp permission-gate.ts ~/.omp/agent/extensions/
|
|
|
41
41
|
| `handoff.ts` | Transfer context to a new focused session via `/handoff <goal>` |
|
|
42
42
|
| `qna.ts` | Extracts questions from last response into editor via `ctx.ui.setEditorText()` |
|
|
43
43
|
| `status-line.ts` | Shows turn progress in footer via `ctx.ui.setStatus()` with themed colors |
|
|
44
|
+
| `thinking-note.ts` | Adds display-only supplemental UI below assistant thinking blocks |
|
|
44
45
|
| `snake.ts` | Snake game with custom UI, keyboard handling, and session persistence |
|
|
45
46
|
|
|
46
47
|
### Git Integration
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ExtensionFactory } from "@oh-my-pi/pi-coding-agent";
|
|
2
|
+
import { Container, Text } from "@oh-my-pi/pi-tui";
|
|
3
|
+
|
|
4
|
+
const extension: ExtensionFactory = pi => {
|
|
5
|
+
pi.setLabel("Thinking note");
|
|
6
|
+
pi.registerAssistantThinkingRenderer((context, theme) => {
|
|
7
|
+
const container = new Container();
|
|
8
|
+
container.addChild(new Text(theme.fg("dim", `thinking chars: ${context.text.length}`), 1, 0));
|
|
9
|
+
return container;
|
|
10
|
+
});
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export default extension;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-coding-agent",
|
|
4
|
-
"version": "15.7.
|
|
4
|
+
"version": "15.7.6",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -47,14 +47,14 @@
|
|
|
47
47
|
"@agentclientprotocol/sdk": "0.22.1",
|
|
48
48
|
"@babel/parser": "^7.29.7",
|
|
49
49
|
"@mozilla/readability": "^0.6.0",
|
|
50
|
-
"@oh-my-pi/hashline": "15.7.
|
|
51
|
-
"@oh-my-pi/omp-stats": "15.7.
|
|
52
|
-
"@oh-my-pi/pi-agent-core": "15.7.
|
|
53
|
-
"@oh-my-pi/pi-ai": "15.7.
|
|
54
|
-
"@oh-my-pi/pi-mnemopi": "15.7.
|
|
55
|
-
"@oh-my-pi/pi-natives": "15.7.
|
|
56
|
-
"@oh-my-pi/pi-tui": "15.7.
|
|
57
|
-
"@oh-my-pi/pi-utils": "15.7.
|
|
50
|
+
"@oh-my-pi/hashline": "15.7.6",
|
|
51
|
+
"@oh-my-pi/omp-stats": "15.7.6",
|
|
52
|
+
"@oh-my-pi/pi-agent-core": "15.7.6",
|
|
53
|
+
"@oh-my-pi/pi-ai": "15.7.6",
|
|
54
|
+
"@oh-my-pi/pi-mnemopi": "15.7.6",
|
|
55
|
+
"@oh-my-pi/pi-natives": "15.7.6",
|
|
56
|
+
"@oh-my-pi/pi-tui": "15.7.6",
|
|
57
|
+
"@oh-my-pi/pi-utils": "15.7.6",
|
|
58
58
|
"@puppeteer/browsers": "^3.0.4",
|
|
59
59
|
"@types/turndown": "5.0.6",
|
|
60
60
|
"@xterm/headless": "^6.0.0",
|
|
@@ -27,6 +27,32 @@ import {
|
|
|
27
27
|
// any provider module at startup. Must match `DEFAULT_LOCAL_TOKEN` in oauth/lm-studio.ts.
|
|
28
28
|
const DEFAULT_LOCAL_TOKEN = "lm-studio-local";
|
|
29
29
|
|
|
30
|
+
// Default cap on `max_tokens` for auto-discovered models that do not advertise
|
|
31
|
+
// their own output limit (OpenAI-models-list, Ollama, llama.cpp, new-api/
|
|
32
|
+
// one-api proxies). 32K matches the upper end of what mainstream
|
|
33
|
+
// OpenAI-compatible providers (DeepSeek, MiMo, OpenRouter, etc.) actually
|
|
34
|
+
// accept and keeps `min(contextWindow, …)` honoring smaller local windows.
|
|
35
|
+
// Conservative caps below this caused providers to drop the connection
|
|
36
|
+
// mid-stream when models hit the cap on legitimate large tool calls (see
|
|
37
|
+
// issue #1528: `write` payloads >~5KB on deepseek-v4-pro surfaced as
|
|
38
|
+
// "socket connection was closed unexpectedly").
|
|
39
|
+
const DISCOVERY_DEFAULT_MAX_TOKENS = 32_768;
|
|
40
|
+
|
|
41
|
+
// Anthropic-safe variant of the discovery cap. The Anthropic stream converter
|
|
42
|
+
// in `packages/ai/src/providers/anthropic.ts` derives the request limit as
|
|
43
|
+
// `(model.maxTokens / 3) | 0`, so the 32K default would surface as 10,922
|
|
44
|
+
// requested output tokens — above the 8,192 hard cap on classic Claude 3.x
|
|
45
|
+
// Sonnet/Haiku/Opus endpoints. Discovered models routed through
|
|
46
|
+
// `anthropic-messages` (proxy `supported_endpoint_types: ["anthropic"]` or a
|
|
47
|
+
// custom provider with `api: anthropic-messages` + openai-models-list
|
|
48
|
+
// discovery) fall back to this conservative value.
|
|
49
|
+
const DISCOVERY_DEFAULT_MAX_TOKENS_ANTHROPIC = 8_192;
|
|
50
|
+
|
|
51
|
+
/** Routes discovered-model `maxTokens` defaults around Anthropic's 3× output divisor. */
|
|
52
|
+
function discoveryDefaultMaxTokens(api: Api | undefined): number {
|
|
53
|
+
return api === "anthropic-messages" ? DISCOVERY_DEFAULT_MAX_TOKENS_ANTHROPIC : DISCOVERY_DEFAULT_MAX_TOKENS;
|
|
54
|
+
}
|
|
55
|
+
|
|
30
56
|
import { registerOAuthProvider, unregisterOAuthProviders } from "@oh-my-pi/pi-ai/utils/oauth";
|
|
31
57
|
import type { OAuthCredentials, OAuthLoginCallbacks } from "@oh-my-pi/pi-ai/utils/oauth/types";
|
|
32
58
|
import { isRecord, logger } from "@oh-my-pi/pi-utils";
|
|
@@ -1690,7 +1716,7 @@ export class ModelRegistry {
|
|
|
1690
1716
|
input: metadata?.input ?? ["text"],
|
|
1691
1717
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
1692
1718
|
contextWindow: metadata?.contextWindow ?? 128000,
|
|
1693
|
-
maxTokens: Math.min(metadata?.contextWindow ?? Number.POSITIVE_INFINITY,
|
|
1719
|
+
maxTokens: Math.min(metadata?.contextWindow ?? Number.POSITIVE_INFINITY, DISCOVERY_DEFAULT_MAX_TOKENS),
|
|
1694
1720
|
headers: providerConfig.headers,
|
|
1695
1721
|
});
|
|
1696
1722
|
});
|
|
@@ -1760,7 +1786,10 @@ export class ModelRegistry {
|
|
|
1760
1786
|
input: serverMetadata?.input ?? ["text"],
|
|
1761
1787
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
1762
1788
|
contextWindow: serverMetadata?.contextWindow ?? 128000,
|
|
1763
|
-
maxTokens: Math.min(
|
|
1789
|
+
maxTokens: Math.min(
|
|
1790
|
+
serverMetadata?.contextWindow ?? Number.POSITIVE_INFINITY,
|
|
1791
|
+
DISCOVERY_DEFAULT_MAX_TOKENS,
|
|
1792
|
+
),
|
|
1764
1793
|
headers,
|
|
1765
1794
|
compat: {
|
|
1766
1795
|
supportsStore: false,
|
|
@@ -1807,7 +1836,7 @@ export class ModelRegistry {
|
|
|
1807
1836
|
input: ["text"],
|
|
1808
1837
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
1809
1838
|
contextWindow: 128000,
|
|
1810
|
-
maxTokens:
|
|
1839
|
+
maxTokens: discoveryDefaultMaxTokens(providerConfig.api),
|
|
1811
1840
|
headers,
|
|
1812
1841
|
compat: {
|
|
1813
1842
|
supportsStore: false,
|
|
@@ -1890,7 +1919,7 @@ export class ModelRegistry {
|
|
|
1890
1919
|
// we successfully recover the upstream model identity.
|
|
1891
1920
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
1892
1921
|
contextWindow: reference?.contextWindow ?? 128000,
|
|
1893
|
-
maxTokens: reference?.maxTokens ??
|
|
1922
|
+
maxTokens: reference?.maxTokens ?? discoveryDefaultMaxTokens(api),
|
|
1894
1923
|
headers,
|
|
1895
1924
|
// OpenAI-compat fields are no-ops on anthropic models; the
|
|
1896
1925
|
// Anthropic SDK ignores them. Provider-level disableStrictTools
|
|
@@ -1966,6 +1966,16 @@ export const SETTINGS_SCHEMA = {
|
|
|
1966
1966
|
},
|
|
1967
1967
|
},
|
|
1968
1968
|
|
|
1969
|
+
"lsp.diagnosticsDeduplicate": {
|
|
1970
|
+
type: "boolean",
|
|
1971
|
+
default: true,
|
|
1972
|
+
ui: {
|
|
1973
|
+
tab: "editing",
|
|
1974
|
+
label: "Deduplicate Diagnostics",
|
|
1975
|
+
description: "Suppress post-edit LSP diagnostics already shown for a file; only surface new or changed ones",
|
|
1976
|
+
},
|
|
1977
|
+
},
|
|
1978
|
+
|
|
1969
1979
|
// Bash interceptor
|
|
1970
1980
|
"bashInterceptor.enabled": {
|
|
1971
1981
|
type: "boolean",
|
package/src/discovery/claude.ts
CHANGED
|
@@ -269,6 +269,29 @@ function readClaudeCommandToggles(): { enableUser: boolean; enableProject: boole
|
|
|
269
269
|
}
|
|
270
270
|
}
|
|
271
271
|
|
|
272
|
+
function getClaudeRelativeCommandName(commandsDir: string, filePath: string): string {
|
|
273
|
+
return path.relative(commandsDir, filePath).replace(/\.md$/, "");
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function addClaudeCommandNamespaceAliases(commands: SlashCommand[], commandsDir: string): SlashCommand[] {
|
|
277
|
+
const rootCommands: SlashCommand[] = [];
|
|
278
|
+
const nestedCommands: SlashCommand[] = [];
|
|
279
|
+
const aliases: SlashCommand[] = [];
|
|
280
|
+
|
|
281
|
+
for (const command of commands) {
|
|
282
|
+
const relativeName = getClaudeRelativeCommandName(commandsDir, command.path);
|
|
283
|
+
if (!/[\\/]/.test(relativeName)) {
|
|
284
|
+
rootCommands.push(command);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
nestedCommands.push(command);
|
|
289
|
+
aliases.push({ ...command, name: relativeName.replace(/[\\/]+/g, ":") });
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
return nestedCommands.length === 0 ? commands : [...rootCommands, ...nestedCommands, ...aliases];
|
|
293
|
+
}
|
|
294
|
+
|
|
272
295
|
async function loadSlashCommands(ctx: LoadContext): Promise<LoadResult<SlashCommand>> {
|
|
273
296
|
const items: SlashCommand[] = [];
|
|
274
297
|
const warnings: string[] = [];
|
|
@@ -280,19 +303,17 @@ async function loadSlashCommands(ctx: LoadContext): Promise<LoadResult<SlashComm
|
|
|
280
303
|
|
|
281
304
|
const userResult = await loadFilesFromDir<SlashCommand>(ctx, userCommandsDir, PROVIDER_ID, "user", {
|
|
282
305
|
extensions: ["md"],
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
};
|
|
292
|
-
},
|
|
306
|
+
recursive: true,
|
|
307
|
+
transform: (name, content, filePath, source) => ({
|
|
308
|
+
name: name.replace(/\.md$/, ""),
|
|
309
|
+
path: filePath,
|
|
310
|
+
content,
|
|
311
|
+
level: "user",
|
|
312
|
+
_source: source,
|
|
313
|
+
}),
|
|
293
314
|
});
|
|
294
315
|
|
|
295
|
-
items.push(...userResult.items);
|
|
316
|
+
items.push(...addClaudeCommandNamespaceAliases(userResult.items, userCommandsDir));
|
|
296
317
|
if (userResult.warnings) warnings.push(...userResult.warnings);
|
|
297
318
|
}
|
|
298
319
|
|
|
@@ -301,19 +322,17 @@ async function loadSlashCommands(ctx: LoadContext): Promise<LoadResult<SlashComm
|
|
|
301
322
|
|
|
302
323
|
const projectResult = await loadFilesFromDir<SlashCommand>(ctx, projectCommandsDir, PROVIDER_ID, "project", {
|
|
303
324
|
extensions: ["md"],
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
};
|
|
313
|
-
},
|
|
325
|
+
recursive: true,
|
|
326
|
+
transform: (name, content, filePath, source) => ({
|
|
327
|
+
name: name.replace(/\.md$/, ""),
|
|
328
|
+
path: filePath,
|
|
329
|
+
content,
|
|
330
|
+
level: "project",
|
|
331
|
+
_source: source,
|
|
332
|
+
}),
|
|
314
333
|
});
|
|
315
334
|
|
|
316
|
-
items.push(...projectResult.items);
|
|
335
|
+
items.push(...addClaudeCommandNamespaceAliases(projectResult.items, projectCommandsDir));
|
|
317
336
|
if (projectResult.warnings) warnings.push(...projectResult.warnings);
|
|
318
337
|
}
|
|
319
338
|
|
package/src/edit/index.ts
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
type WritethroughDeferredHandle,
|
|
11
11
|
writethroughNoop,
|
|
12
12
|
} from "../lsp";
|
|
13
|
+
import { getDiagnosticsLedger } from "../lsp/diagnostics-ledger";
|
|
13
14
|
import applyPatchDescription from "../prompts/tools/apply-patch.md" with { type: "text" };
|
|
14
15
|
import patchDescription from "../prompts/tools/patch.md" with { type: "text" };
|
|
15
16
|
import replaceDescription from "../prompts/tools/replace.md" with { type: "text" };
|
|
@@ -102,7 +103,16 @@ function createEditWritethrough(session: ToolSession): WritethroughCallback {
|
|
|
102
103
|
const enableLsp = session.enableLsp ?? true;
|
|
103
104
|
const enableDiagnostics = enableLsp && session.settings.get("lsp.diagnosticsOnEdit");
|
|
104
105
|
const enableFormat = enableLsp && session.settings.get("lsp.formatOnWrite");
|
|
105
|
-
|
|
106
|
+
const dedup = enableDiagnostics && session.settings.get("lsp.diagnosticsDeduplicate");
|
|
107
|
+
return enableLsp
|
|
108
|
+
? createLspWritethrough(session.cwd, {
|
|
109
|
+
enableFormat,
|
|
110
|
+
enableDiagnostics,
|
|
111
|
+
transformDiagnostics: dedup
|
|
112
|
+
? (path, result) => getDiagnosticsLedger(session).reduce(path, result)
|
|
113
|
+
: undefined,
|
|
114
|
+
})
|
|
115
|
+
: writethroughNoop;
|
|
106
116
|
}
|
|
107
117
|
|
|
108
118
|
/** Run apply_patch file operations and aggregate their multi-file result. */
|
|
@@ -294,6 +304,7 @@ export class EditTool implements AgentTool<TInput> {
|
|
|
294
304
|
readonly #fuzzyThreshold: number;
|
|
295
305
|
readonly #writethrough: WritethroughCallback;
|
|
296
306
|
readonly #editMode?: EditMode;
|
|
307
|
+
readonly #dedupDiagnostics: boolean;
|
|
297
308
|
readonly #pendingDeferredFetches = new Map<string, AbortController>();
|
|
298
309
|
|
|
299
310
|
constructor(private readonly session: ToolSession) {
|
|
@@ -306,6 +317,10 @@ export class EditTool implements AgentTool<TInput> {
|
|
|
306
317
|
this.#editMode = resolveConfiguredEditMode(envEditVariant);
|
|
307
318
|
this.#allowFuzzy = resolveAllowFuzzy(session, editFuzzy);
|
|
308
319
|
this.#fuzzyThreshold = resolveFuzzyThreshold(session, editFuzzyThreshold);
|
|
320
|
+
this.#dedupDiagnostics =
|
|
321
|
+
(session.enableLsp ?? true) &&
|
|
322
|
+
session.settings.get("lsp.diagnosticsOnEdit") &&
|
|
323
|
+
session.settings.get("lsp.diagnosticsDeduplicate");
|
|
309
324
|
this.#writethrough = createEditWritethrough(session);
|
|
310
325
|
}
|
|
311
326
|
|
|
@@ -495,8 +510,13 @@ export class EditTool implements AgentTool<TInput> {
|
|
|
495
510
|
}
|
|
496
511
|
|
|
497
512
|
#injectLateDiagnostics(path: string, diagnostics: FileDiagnosticsResult): void {
|
|
498
|
-
const
|
|
499
|
-
|
|
513
|
+
const effective = this.#dedupDiagnostics
|
|
514
|
+
? getDiagnosticsLedger(this.session).reduce(path, diagnostics)
|
|
515
|
+
: diagnostics;
|
|
516
|
+
if (this.#dedupDiagnostics && effective.messages.length === 0) return;
|
|
517
|
+
|
|
518
|
+
const summary = effective.summary ?? "";
|
|
519
|
+
const lines = effective.messages ?? [];
|
|
500
520
|
const body = [`Late LSP diagnostics for ${path} (arrived after the edit tool returned):`, summary, ...lines]
|
|
501
521
|
.filter(Boolean)
|
|
502
522
|
.join("\n");
|
|
@@ -10,6 +10,8 @@ import { AgentOutputManager } from "../../task/output-manager";
|
|
|
10
10
|
import type { AgentDefinition, AgentProgress, SingleResult } from "../../task/types";
|
|
11
11
|
import type { ToolSession } from "../../tools";
|
|
12
12
|
import { EVAL_AGENT_MAX_DEPTH, runEvalAgent } from "../agent-bridge";
|
|
13
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs } from "../heartbeat";
|
|
14
|
+
import { IdleTimeout } from "../idle-timeout";
|
|
13
15
|
import { disposeAllVmContexts } from "../js/context-manager";
|
|
14
16
|
import { executeJs } from "../js/executor";
|
|
15
17
|
import { disposeAllKernelSessions, executePython } from "../py/executor";
|
|
@@ -232,6 +234,7 @@ describe("runEvalAgent", () => {
|
|
|
232
234
|
describe("agent() through eval runtimes", () => {
|
|
233
235
|
afterEach(() => {
|
|
234
236
|
vi.restoreAllMocks();
|
|
237
|
+
setBridgeHeartbeatIntervalMs();
|
|
235
238
|
});
|
|
236
239
|
|
|
237
240
|
afterAll(async () => {
|
|
@@ -430,4 +433,91 @@ describe("agent() through eval runtimes", () => {
|
|
|
430
433
|
);
|
|
431
434
|
expect(displayAgentEvents.length).toBe(2);
|
|
432
435
|
});
|
|
436
|
+
|
|
437
|
+
it("keeps the idle watchdog armed while a quiet agent() runs past the budget", async () => {
|
|
438
|
+
using tempDir = TempDir.createSync("@omp-eval-agent-heartbeat-");
|
|
439
|
+
const { session } = makeEvalSession(tempDir, "js-agent-heartbeat");
|
|
440
|
+
mockAgents();
|
|
441
|
+
// Heartbeat cadence well under the idle budget so a working-but-silent
|
|
442
|
+
// subagent re-arms the watchdog several times before it could expire.
|
|
443
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
444
|
+
|
|
445
|
+
// runSubprocess runs far past the budget and emits NO progress of its own
|
|
446
|
+
// — the only thing standing between the subagent and a spurious idle abort
|
|
447
|
+
// is the heartbeat keepalive the bridge pumps while it awaits.
|
|
448
|
+
vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => {
|
|
449
|
+
await Bun.sleep(200);
|
|
450
|
+
return singleResult(options, { output: "done" });
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
// Mirror the eval tool's wiring: an IdleTimeout drives cancellation and
|
|
454
|
+
// ONLY a bridge heartbeat re-arms it.
|
|
455
|
+
using idle = new IdleTimeout(60);
|
|
456
|
+
const result = await runEvalAgent(
|
|
457
|
+
{ prompt: "investigate" },
|
|
458
|
+
{
|
|
459
|
+
session,
|
|
460
|
+
signal: idle.signal,
|
|
461
|
+
emitStatus: event => {
|
|
462
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
463
|
+
},
|
|
464
|
+
},
|
|
465
|
+
);
|
|
466
|
+
|
|
467
|
+
expect(idle.signal.aborted).toBe(false);
|
|
468
|
+
expect(result.text).toBe("done");
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
it("does not let agent() progress snapshots re-arm the watchdog without a heartbeat", async () => {
|
|
472
|
+
using tempDir = TempDir.createSync("@omp-eval-agent-progress-no-rearm-");
|
|
473
|
+
const { session } = makeEvalSession(tempDir, "js-agent-progress-no-rearm");
|
|
474
|
+
mockAgents();
|
|
475
|
+
// Heartbeat slower than the budget: only the immediate beat at call start
|
|
476
|
+
// fires, so after the budget elapses nothing re-arms the watchdog.
|
|
477
|
+
setBridgeHeartbeatIntervalMs(10_000);
|
|
478
|
+
|
|
479
|
+
// Stream frequent progress snapshots (op:"agent") for well past the budget.
|
|
480
|
+
// Progress is rendered but MUST NOT count as activity — only heartbeats do.
|
|
481
|
+
vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => {
|
|
482
|
+
for (let i = 0; i < 40; i++) {
|
|
483
|
+
options.onProgress?.({
|
|
484
|
+
index: options.index,
|
|
485
|
+
id: options.id,
|
|
486
|
+
agent: options.agent.name,
|
|
487
|
+
agentSource: options.agent.source,
|
|
488
|
+
status: "running",
|
|
489
|
+
task: options.task,
|
|
490
|
+
assignment: options.assignment,
|
|
491
|
+
description: options.description,
|
|
492
|
+
recentTools: [],
|
|
493
|
+
recentOutput: [],
|
|
494
|
+
toolCount: i,
|
|
495
|
+
tokens: 0,
|
|
496
|
+
cost: 0,
|
|
497
|
+
durationMs: i * 10,
|
|
498
|
+
});
|
|
499
|
+
await Bun.sleep(10);
|
|
500
|
+
}
|
|
501
|
+
return singleResult(options, { output: "done" });
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
const ops: string[] = [];
|
|
505
|
+
using idle = new IdleTimeout(80);
|
|
506
|
+
await runEvalAgent(
|
|
507
|
+
{ prompt: "investigate" },
|
|
508
|
+
{
|
|
509
|
+
session,
|
|
510
|
+
signal: idle.signal,
|
|
511
|
+
emitStatus: event => {
|
|
512
|
+
ops.push(event.op);
|
|
513
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
514
|
+
},
|
|
515
|
+
},
|
|
516
|
+
);
|
|
517
|
+
|
|
518
|
+
// Progress streamed, but the watchdog still fired: agent snapshots never
|
|
519
|
+
// re-armed it, and the lone start heartbeat lapsed before the call ended.
|
|
520
|
+
expect(ops).toContain("agent");
|
|
521
|
+
expect(idle.signal.aborted).toBe(true);
|
|
522
|
+
});
|
|
433
523
|
});
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs, withBridgeHeartbeat } from "../heartbeat";
|
|
3
|
+
import type { JsStatusEvent } from "../js/shared/types";
|
|
4
|
+
|
|
5
|
+
describe("withBridgeHeartbeat", () => {
|
|
6
|
+
afterEach(() => {
|
|
7
|
+
setBridgeHeartbeatIntervalMs();
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
it("pumps heartbeat events on cadence while the operation is pending, then stops", async () => {
|
|
11
|
+
setBridgeHeartbeatIntervalMs(20);
|
|
12
|
+
const events: JsStatusEvent[] = [];
|
|
13
|
+
|
|
14
|
+
const value = await withBridgeHeartbeat(
|
|
15
|
+
event => events.push(event),
|
|
16
|
+
async () => {
|
|
17
|
+
await Bun.sleep(130);
|
|
18
|
+
return "done";
|
|
19
|
+
},
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
expect(value).toBe("done");
|
|
23
|
+
// ~6 ticks fit in 130ms at a 20ms cadence; assert it ticked repeatedly
|
|
24
|
+
// without pinning the exact count (scheduler jitter).
|
|
25
|
+
expect(events.length).toBeGreaterThanOrEqual(3);
|
|
26
|
+
expect(events.every(event => event.op === EVAL_HEARTBEAT_OP)).toBe(true);
|
|
27
|
+
|
|
28
|
+
// The interval is cleared once the operation settles: no further ticks.
|
|
29
|
+
const settledCount = events.length;
|
|
30
|
+
await Bun.sleep(80);
|
|
31
|
+
expect(events.length).toBe(settledCount);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("emits a heartbeat immediately so a bridge call extends the budget at once", async () => {
|
|
35
|
+
// Interval far longer than the operation: the only beat that can fire is
|
|
36
|
+
// the immediate one at call start. It must still reach the sink.
|
|
37
|
+
setBridgeHeartbeatIntervalMs(10_000);
|
|
38
|
+
const events: JsStatusEvent[] = [];
|
|
39
|
+
|
|
40
|
+
await withBridgeHeartbeat(
|
|
41
|
+
event => events.push(event),
|
|
42
|
+
async () => {
|
|
43
|
+
await Bun.sleep(30);
|
|
44
|
+
return "done";
|
|
45
|
+
},
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
expect(events.length).toBe(1);
|
|
49
|
+
expect(events[0]?.op).toBe(EVAL_HEARTBEAT_OP);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("runs the operation without emitting when no status sink is wired", async () => {
|
|
53
|
+
setBridgeHeartbeatIntervalMs(5);
|
|
54
|
+
let ran = 0;
|
|
55
|
+
|
|
56
|
+
const value = await withBridgeHeartbeat(undefined, async () => {
|
|
57
|
+
ran++;
|
|
58
|
+
await Bun.sleep(40);
|
|
59
|
+
return 42;
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
expect(value).toBe(42);
|
|
63
|
+
expect(ran).toBe(1);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("clears the heartbeat even when the operation throws", async () => {
|
|
67
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
68
|
+
const events: JsStatusEvent[] = [];
|
|
69
|
+
|
|
70
|
+
await expect(
|
|
71
|
+
withBridgeHeartbeat(
|
|
72
|
+
event => events.push(event),
|
|
73
|
+
async () => {
|
|
74
|
+
await Bun.sleep(60);
|
|
75
|
+
throw new Error("boom");
|
|
76
|
+
},
|
|
77
|
+
),
|
|
78
|
+
).rejects.toThrow("boom");
|
|
79
|
+
|
|
80
|
+
const afterThrow = events.length;
|
|
81
|
+
await Bun.sleep(60);
|
|
82
|
+
expect(events.length).toBe(afterThrow);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
@@ -8,6 +8,8 @@ import type { ModelRegistry } from "../../config/model-registry";
|
|
|
8
8
|
import { Settings } from "../../config/settings";
|
|
9
9
|
import type { ToolSession } from "../../tools";
|
|
10
10
|
import { ToolError } from "../../tools/tool-errors";
|
|
11
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs } from "../heartbeat";
|
|
12
|
+
import { IdleTimeout } from "../idle-timeout";
|
|
11
13
|
import { disposeAllVmContexts } from "../js/context-manager";
|
|
12
14
|
import { executeJs } from "../js/executor";
|
|
13
15
|
import { runEvalLlm } from "../llm-bridge";
|
|
@@ -97,6 +99,7 @@ function assistant(opts: {
|
|
|
97
99
|
describe("runEvalLlm", () => {
|
|
98
100
|
afterEach(() => {
|
|
99
101
|
vi.restoreAllMocks();
|
|
102
|
+
setBridgeHeartbeatIntervalMs();
|
|
100
103
|
});
|
|
101
104
|
|
|
102
105
|
it("resolves each tier to its expected model", async () => {
|
|
@@ -213,6 +216,33 @@ describe("runEvalLlm", () => {
|
|
|
213
216
|
ToolError,
|
|
214
217
|
);
|
|
215
218
|
});
|
|
219
|
+
|
|
220
|
+
it("keeps the idle watchdog armed while a slow llm() request is in flight", async () => {
|
|
221
|
+
// A oneshot completion emits no status until it returns; a slow request
|
|
222
|
+
// must not look like a stalled cell. The bridge pumps a heartbeat while it
|
|
223
|
+
// awaits, re-arming the watchdog through emitStatus.
|
|
224
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
225
|
+
vi.spyOn(ai, "completeSimple").mockImplementation(async () => {
|
|
226
|
+
await Bun.sleep(200);
|
|
227
|
+
return assistant({ text: "the answer" });
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
using idle = new IdleTimeout(60);
|
|
231
|
+
const result = await runEvalLlm(
|
|
232
|
+
{ prompt: "q", model: "smol" },
|
|
233
|
+
{
|
|
234
|
+
session: makeSession(),
|
|
235
|
+
signal: idle.signal,
|
|
236
|
+
// Mirror the eval tool: only a bridge heartbeat re-arms the watchdog.
|
|
237
|
+
emitStatus: event => {
|
|
238
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
);
|
|
242
|
+
|
|
243
|
+
expect(idle.signal.aborted).toBe(false);
|
|
244
|
+
expect(result.text).toBe("the answer");
|
|
245
|
+
});
|
|
216
246
|
});
|
|
217
247
|
|
|
218
248
|
describe("llm() through eval runtimes", () => {
|
package/src/eval/agent-bridge.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { AgentOutputManager } from "../task/output-manager";
|
|
|
16
16
|
import type { AgentDefinition, AgentProgress } from "../task/types";
|
|
17
17
|
import type { ToolSession } from "../tools";
|
|
18
18
|
import { ToolError } from "../tools/tool-errors";
|
|
19
|
+
import { withBridgeHeartbeat } from "./heartbeat";
|
|
19
20
|
import type { JsStatusEvent } from "./js/shared/types";
|
|
20
21
|
// Import review tools for side effects (registers subagent tool handlers).
|
|
21
22
|
import "../tools/review";
|
|
@@ -231,44 +232,49 @@ export async function runEvalAgent(args: unknown, options: EvalAgentBridgeOption
|
|
|
231
232
|
const id = await outputManager.allocate(outputIdBase(parsed.label, agentName));
|
|
232
233
|
const assignment = parsed.prompt.trim();
|
|
233
234
|
const context = trimToUndefined(parsed.context);
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
235
|
+
// Pump a heartbeat while the subagent runs so the eval idle watchdog stays
|
|
236
|
+
// armed across quiet stretches (time-to-first-token, long nested tools)
|
|
237
|
+
// where `onProgress` would otherwise emit no status to re-arm it.
|
|
238
|
+
const result = await withBridgeHeartbeat(options.emitStatus, () =>
|
|
239
|
+
taskExecutor.runSubprocess({
|
|
240
|
+
cwd: options.session.cwd,
|
|
241
|
+
agent: effectiveAgent,
|
|
242
|
+
task: renderSubagentPrompt(assignment),
|
|
243
|
+
assignment,
|
|
244
|
+
context,
|
|
245
|
+
description: trimToUndefined(parsed.label),
|
|
246
|
+
index: 0,
|
|
247
|
+
id,
|
|
248
|
+
taskDepth: options.session.taskDepth ?? 0,
|
|
249
|
+
modelOverride,
|
|
250
|
+
parentActiveModelPattern,
|
|
251
|
+
thinkingLevel: effectiveAgent.thinkingLevel,
|
|
252
|
+
outputSchema: structured ? parsed.schema : undefined,
|
|
253
|
+
sessionFile,
|
|
254
|
+
persistArtifacts: Boolean(sessionFile),
|
|
255
|
+
artifactsDir,
|
|
256
|
+
contextFile,
|
|
257
|
+
enableLsp: (options.session.enableLsp ?? true) && options.session.settings.get("task.enableLsp"),
|
|
258
|
+
signal: options.signal,
|
|
259
|
+
eventBus: options.session.eventBus,
|
|
260
|
+
onProgress: progress => emitProgressStatus(options.emitStatus, progress),
|
|
261
|
+
authStorage: options.session.authStorage,
|
|
262
|
+
modelRegistry: options.session.modelRegistry,
|
|
263
|
+
settings: options.session.settings,
|
|
264
|
+
mcpManager,
|
|
265
|
+
contextFiles,
|
|
266
|
+
skills: availableSkills,
|
|
267
|
+
autoloadSkills: resolvedAutoloadSkills,
|
|
268
|
+
workspaceTree: options.session.workspaceTree,
|
|
269
|
+
promptTemplates: options.session.promptTemplates,
|
|
270
|
+
localProtocolOptions,
|
|
271
|
+
parentArtifactManager,
|
|
272
|
+
parentHindsightSessionState: options.session.getHindsightSessionState?.(),
|
|
273
|
+
parentMnemopiSessionState: options.session.getMnemopiSessionState?.(),
|
|
274
|
+
parentTelemetry: options.session.getTelemetry?.(),
|
|
275
|
+
parentEvalSessionId,
|
|
276
|
+
}),
|
|
277
|
+
);
|
|
272
278
|
|
|
273
279
|
if (result.exitCode !== 0 || result.error) {
|
|
274
280
|
const failureMessage =
|