@oh-my-pi/pi-coding-agent 15.7.5 → 15.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +159 -182
- package/dist/types/async/job-manager.d.ts +3 -3
- package/dist/types/cli/args.d.ts +1 -0
- package/dist/types/cli/claude-trace-cli.d.ts +54 -0
- package/dist/types/cli/session-picker.d.ts +10 -3
- package/dist/types/cli/update-cli.d.ts +17 -0
- package/dist/types/commands/launch.d.ts +3 -0
- package/dist/types/config/keybindings.d.ts +5 -0
- package/dist/types/config/settings-schema.d.ts +11 -2
- package/dist/types/config/settings.d.ts +13 -0
- package/dist/types/eval/__tests__/heartbeat.test.d.ts +1 -0
- package/dist/types/eval/concurrency-bridge.d.ts +26 -0
- package/dist/types/eval/heartbeat.d.ts +45 -0
- package/dist/types/eval/js/tool-bridge.d.ts +2 -1
- package/dist/types/extensibility/extensions/loader.d.ts +2 -2
- package/dist/types/extensibility/extensions/runner.d.ts +2 -1
- package/dist/types/extensibility/extensions/types.d.ts +24 -5
- package/dist/types/extensibility/plugins/legacy-pi-compat.d.ts +11 -0
- package/dist/types/lsp/diagnostics-ledger.d.ts +10 -0
- package/dist/types/lsp/index.d.ts +2 -0
- package/dist/types/lsp/utils.d.ts +4 -0
- package/dist/types/main.d.ts +5 -0
- package/dist/types/modes/acp/acp-agent.d.ts +1 -1
- package/dist/types/modes/components/assistant-message.d.ts +3 -1
- package/dist/types/modes/components/custom-editor.d.ts +3 -2
- package/dist/types/modes/components/hook-selector.d.ts +10 -1
- package/dist/types/modes/components/session-selector.d.ts +32 -5
- package/dist/types/modes/components/tool-execution.d.ts +8 -0
- package/dist/types/modes/controllers/extension-ui-controller.d.ts +3 -3
- package/dist/types/modes/controllers/input-controller.d.ts +1 -0
- package/dist/types/modes/interactive-mode.d.ts +10 -3
- package/dist/types/modes/rpc/rpc-mode.d.ts +1 -1
- package/dist/types/modes/theme/theme.d.ts +1 -1
- package/dist/types/modes/types.d.ts +5 -3
- package/dist/types/registry/agent-registry.d.ts +1 -1
- package/dist/types/sdk.d.ts +2 -2
- package/dist/types/session/agent-session.d.ts +12 -3
- package/dist/types/session/history-storage.d.ts +16 -1
- package/dist/types/session/session-manager.d.ts +4 -0
- package/dist/types/task/output-manager.d.ts +6 -15
- package/dist/types/tools/ask.d.ts +8 -6
- package/dist/types/tools/eval-backends.d.ts +12 -0
- package/dist/types/tools/eval-render.d.ts +52 -0
- package/dist/types/tools/eval.d.ts +2 -35
- package/dist/types/tools/find.d.ts +0 -9
- package/dist/types/tools/index.d.ts +5 -12
- package/dist/types/tools/path-utils.d.ts +16 -0
- package/dist/types/tools/sqlite-reader.d.ts +25 -8
- package/dist/types/tui/output-block.d.ts +4 -3
- package/dist/types/utils/clipboard.d.ts +4 -0
- package/dist/types/web/kagi.d.ts +76 -0
- package/dist/types/web/search/providers/exa.d.ts +7 -1
- package/dist/types/web/search/providers/kagi.d.ts +1 -0
- package/examples/extensions/README.md +1 -0
- package/examples/extensions/thinking-note.ts +13 -0
- package/package.json +9 -9
- package/src/async/job-manager.ts +3 -3
- package/src/cli/args.ts +6 -2
- package/src/cli/claude-trace-cli.ts +783 -0
- package/src/cli/session-picker.ts +36 -10
- package/src/cli/update-cli.ts +35 -2
- package/src/commands/launch.ts +3 -0
- package/src/config/keybindings.ts +6 -0
- package/src/config/model-registry.ts +33 -4
- package/src/config/settings-schema.ts +12 -2
- package/src/config/settings.ts +23 -0
- package/src/discovery/claude-plugins.ts +7 -9
- package/src/discovery/claude.ts +41 -22
- package/src/edit/index.ts +23 -3
- package/src/eval/__tests__/agent-bridge.test.ts +148 -4
- package/src/eval/__tests__/heartbeat.test.ts +84 -0
- package/src/eval/__tests__/llm-bridge.test.ts +30 -0
- package/src/eval/agent-bridge.ts +44 -38
- package/src/eval/concurrency-bridge.ts +34 -0
- package/src/eval/heartbeat.ts +74 -0
- package/src/eval/js/executor.ts +13 -9
- package/src/eval/js/shared/prelude.txt +20 -17
- package/src/eval/js/tool-bridge.ts +5 -0
- package/src/eval/llm-bridge.ts +20 -14
- package/src/eval/py/executor.ts +14 -18
- package/src/eval/py/prelude.py +23 -15
- package/src/exec/bash-executor.ts +31 -5
- package/src/extensibility/extensions/loader.ts +16 -18
- package/src/extensibility/extensions/runner.ts +22 -17
- package/src/extensibility/extensions/types.ts +39 -5
- package/src/extensibility/plugins/legacy-pi-compat.ts +115 -131
- package/src/extensibility/skills.ts +0 -1
- package/src/internal-urls/docs-index.generated.ts +14 -13
- package/src/lsp/diagnostics-ledger.ts +51 -0
- package/src/lsp/index.ts +9 -22
- package/src/lsp/utils.ts +21 -0
- package/src/main.ts +92 -24
- package/src/modes/acp/acp-agent.ts +8 -4
- package/src/modes/acp/acp-event-mapper.ts +54 -4
- package/src/modes/components/assistant-message.ts +28 -1
- package/src/modes/components/custom-editor.ts +19 -7
- package/src/modes/components/hook-selector.ts +229 -44
- package/src/modes/components/oauth-selector.ts +12 -6
- package/src/modes/components/session-selector.ts +179 -24
- package/src/modes/components/tool-execution.ts +36 -7
- package/src/modes/controllers/command-controller.ts +2 -11
- package/src/modes/controllers/event-controller.ts +5 -2
- package/src/modes/controllers/extension-ui-controller.ts +6 -4
- package/src/modes/controllers/input-controller.ts +19 -16
- package/src/modes/controllers/selector-controller.ts +61 -21
- package/src/modes/interactive-mode.ts +127 -16
- package/src/modes/rpc/rpc-mode.ts +17 -6
- package/src/modes/theme/theme-schema.json +30 -0
- package/src/modes/theme/theme.ts +39 -2
- package/src/modes/types.ts +7 -3
- package/src/modes/utils/ui-helpers.ts +5 -2
- package/src/prompts/system/orchestrate-notice.md +5 -3
- package/src/prompts/system/workflow-notice.md +2 -2
- package/src/prompts/tools/ask.md +2 -1
- package/src/prompts/tools/eval.md +6 -6
- package/src/prompts/tools/find.md +1 -1
- package/src/prompts/tools/irc.md +6 -6
- package/src/prompts/tools/search.md +1 -1
- package/src/prompts/tools/task.md +1 -1
- package/src/registry/agent-registry.ts +1 -1
- package/src/sdk.ts +85 -31
- package/src/session/agent-session.ts +127 -57
- package/src/session/history-storage.ts +56 -12
- package/src/session/session-manager.ts +34 -0
- package/src/task/output-manager.ts +40 -48
- package/src/task/render.ts +3 -8
- package/src/tools/ask.ts +74 -32
- package/src/tools/browser/tab-worker.ts +8 -5
- package/src/tools/eval-backends.ts +38 -0
- package/src/tools/eval-render.ts +750 -0
- package/src/tools/eval.ts +27 -754
- package/src/tools/find.ts +5 -29
- package/src/tools/index.ts +8 -38
- package/src/tools/path-utils.ts +144 -1
- package/src/tools/read.ts +47 -0
- package/src/tools/renderers.ts +1 -1
- package/src/tools/search.ts +2 -27
- package/src/tools/sqlite-reader.ts +92 -9
- package/src/tools/write.ts +9 -1
- package/src/tui/output-block.ts +5 -4
- package/src/utils/clipboard.ts +38 -1
- package/src/utils/open.ts +37 -2
- package/src/web/kagi.ts +168 -49
- package/src/web/search/providers/anthropic.ts +1 -1
- package/src/web/search/providers/exa.ts +20 -86
- package/src/web/search/providers/kagi.ts +4 -0
|
@@ -10,6 +10,8 @@ import { AgentOutputManager } from "../../task/output-manager";
|
|
|
10
10
|
import type { AgentDefinition, AgentProgress, SingleResult } from "../../task/types";
|
|
11
11
|
import type { ToolSession } from "../../tools";
|
|
12
12
|
import { EVAL_AGENT_MAX_DEPTH, runEvalAgent } from "../agent-bridge";
|
|
13
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs } from "../heartbeat";
|
|
14
|
+
import { IdleTimeout } from "../idle-timeout";
|
|
13
15
|
import { disposeAllVmContexts } from "../js/context-manager";
|
|
14
16
|
import { executeJs } from "../js/executor";
|
|
15
17
|
import { disposeAllKernelSessions, executePython } from "../py/executor";
|
|
@@ -104,6 +106,7 @@ function singleResult(options: ExecutorOptions, overrides: Partial<SingleResult>
|
|
|
104
106
|
function makeEvalSession(
|
|
105
107
|
tempDir: TempDir,
|
|
106
108
|
prefix: string,
|
|
109
|
+
settings?: Settings,
|
|
107
110
|
): { session: ToolSession; sessionFile: string; sessionId: string } {
|
|
108
111
|
const sessionFile = path.join(tempDir.path(), "session.jsonl");
|
|
109
112
|
const artifactsDir = sessionFile.slice(0, -6);
|
|
@@ -111,6 +114,7 @@ function makeEvalSession(
|
|
|
111
114
|
cwd: tempDir.path(),
|
|
112
115
|
sessionFile,
|
|
113
116
|
artifactsDir,
|
|
117
|
+
settings,
|
|
114
118
|
outputManager: new AgentOutputManager(() => artifactsDir),
|
|
115
119
|
});
|
|
116
120
|
return { session, sessionFile, sessionId: `${prefix}:${crypto.randomUUID()}` };
|
|
@@ -232,6 +236,7 @@ describe("runEvalAgent", () => {
|
|
|
232
236
|
describe("agent() through eval runtimes", () => {
|
|
233
237
|
afterEach(() => {
|
|
234
238
|
vi.restoreAllMocks();
|
|
239
|
+
setBridgeHeartbeatIntervalMs();
|
|
235
240
|
});
|
|
236
241
|
|
|
237
242
|
afterAll(async () => {
|
|
@@ -258,9 +263,15 @@ describe("agent() through eval runtimes", () => {
|
|
|
258
263
|
expect(JSON.parse(result.output.trim())).toEqual(["hello from agent", { ok: true, n: 3 }]);
|
|
259
264
|
});
|
|
260
265
|
|
|
261
|
-
it("
|
|
266
|
+
it("bounds JavaScript parallel() by the task.maxConcurrency setting while preserving order", async () => {
|
|
262
267
|
using tempDir = TempDir.createSync("@omp-eval-agent-js-parallel-");
|
|
263
|
-
const
|
|
268
|
+
const settings = Settings.isolated({
|
|
269
|
+
"async.enabled": false,
|
|
270
|
+
"task.isolation.mode": "none",
|
|
271
|
+
"task.enableLsp": true,
|
|
272
|
+
"task.maxConcurrency": 2,
|
|
273
|
+
});
|
|
274
|
+
const { session, sessionFile, sessionId } = makeEvalSession(tempDir, "js-agent-parallel", settings);
|
|
264
275
|
mockAgents();
|
|
265
276
|
let inFlight = 0;
|
|
266
277
|
let maxInFlight = 0;
|
|
@@ -276,7 +287,7 @@ describe("agent() through eval runtimes", () => {
|
|
|
276
287
|
});
|
|
277
288
|
|
|
278
289
|
const result = await executeJs(
|
|
279
|
-
'const values = await parallel(["a", "b", "c", "d"].map(name => () => agent(name))
|
|
290
|
+
'const values = await parallel(["a", "b", "c", "d"].map(name => () => agent(name))); return JSON.stringify(values);',
|
|
280
291
|
{ cwd: tempDir.path(), sessionId, session, sessionFile },
|
|
281
292
|
);
|
|
282
293
|
|
|
@@ -297,7 +308,7 @@ describe("agent() through eval runtimes", () => {
|
|
|
297
308
|
return singleResult(options, { output: options.assignment ?? "" });
|
|
298
309
|
});
|
|
299
310
|
|
|
300
|
-
const result = await executeJs('await parallel([() => agent("ok"), () => agent("bad")]
|
|
311
|
+
const result = await executeJs('await parallel([() => agent("ok"), () => agent("bad")]);', {
|
|
301
312
|
cwd: tempDir.path(),
|
|
302
313
|
sessionId,
|
|
303
314
|
session,
|
|
@@ -340,6 +351,52 @@ describe("agent() through eval runtimes", () => {
|
|
|
340
351
|
expect(result.output.trim()).toBe("hello from python");
|
|
341
352
|
});
|
|
342
353
|
|
|
354
|
+
it("bounds Python parallel() by the task.maxConcurrency setting while preserving order", async () => {
|
|
355
|
+
using tempDir = TempDir.createSync("@omp-eval-agent-py-parallel-");
|
|
356
|
+
const settings = Settings.isolated({
|
|
357
|
+
"async.enabled": false,
|
|
358
|
+
"task.isolation.mode": "none",
|
|
359
|
+
"task.enableLsp": true,
|
|
360
|
+
"task.maxConcurrency": 2,
|
|
361
|
+
});
|
|
362
|
+
const { session, sessionFile, sessionId } = makeEvalSession(tempDir, "py-agent-parallel", settings);
|
|
363
|
+
mockAgents();
|
|
364
|
+
let inFlight = 0;
|
|
365
|
+
let maxInFlight = 0;
|
|
366
|
+
vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => {
|
|
367
|
+
inFlight++;
|
|
368
|
+
maxInFlight = Math.max(maxInFlight, inFlight);
|
|
369
|
+
try {
|
|
370
|
+
await Bun.sleep(options.assignment === "a" ? 30 : 10);
|
|
371
|
+
return singleResult(options, { output: options.assignment ?? "" });
|
|
372
|
+
} finally {
|
|
373
|
+
inFlight--;
|
|
374
|
+
}
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
const probe = await executePython('print("probe")', {
|
|
378
|
+
cwd: tempDir.path(),
|
|
379
|
+
sessionId: `${sessionId}:probe`,
|
|
380
|
+
sessionFile,
|
|
381
|
+
kernelMode: "per-call",
|
|
382
|
+
});
|
|
383
|
+
if (probe.exitCode === undefined && probe.cancelled) {
|
|
384
|
+
expect(probe.output).toBe("");
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
expect(probe.exitCode).toBe(0);
|
|
388
|
+
|
|
389
|
+
const result = await executePython(
|
|
390
|
+
'import json\nprint(json.dumps(parallel([lambda n=n: agent(n) for n in ["a", "b", "c", "d"]])))',
|
|
391
|
+
{ cwd: tempDir.path(), sessionId, sessionFile, kernelMode: "per-call", toolSession: session },
|
|
392
|
+
);
|
|
393
|
+
|
|
394
|
+
expect(result.exitCode).toBe(0);
|
|
395
|
+
expect(JSON.parse(result.output.trim())).toEqual(["a", "b", "c", "d"]);
|
|
396
|
+
expect(maxInFlight).toBeGreaterThan(1);
|
|
397
|
+
expect(maxInFlight).toBeLessThanOrEqual(2);
|
|
398
|
+
});
|
|
399
|
+
|
|
343
400
|
it("streams enriched agent progress through onStatus before the cell finishes", async () => {
|
|
344
401
|
using tempDir = TempDir.createSync("@omp-eval-agent-progress-");
|
|
345
402
|
const { session, sessionFile, sessionId } = makeEvalSession(tempDir, "js-agent-progress");
|
|
@@ -430,4 +487,91 @@ describe("agent() through eval runtimes", () => {
|
|
|
430
487
|
);
|
|
431
488
|
expect(displayAgentEvents.length).toBe(2);
|
|
432
489
|
});
|
|
490
|
+
|
|
491
|
+
it("keeps the idle watchdog armed while a quiet agent() runs past the budget", async () => {
|
|
492
|
+
using tempDir = TempDir.createSync("@omp-eval-agent-heartbeat-");
|
|
493
|
+
const { session } = makeEvalSession(tempDir, "js-agent-heartbeat");
|
|
494
|
+
mockAgents();
|
|
495
|
+
// Heartbeat cadence well under the idle budget so a working-but-silent
|
|
496
|
+
// subagent re-arms the watchdog several times before it could expire.
|
|
497
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
498
|
+
|
|
499
|
+
// runSubprocess runs far past the budget and emits NO progress of its own
|
|
500
|
+
// — the only thing standing between the subagent and a spurious idle abort
|
|
501
|
+
// is the heartbeat keepalive the bridge pumps while it awaits.
|
|
502
|
+
vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => {
|
|
503
|
+
await Bun.sleep(200);
|
|
504
|
+
return singleResult(options, { output: "done" });
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
// Mirror the eval tool's wiring: an IdleTimeout drives cancellation and
|
|
508
|
+
// ONLY a bridge heartbeat re-arms it.
|
|
509
|
+
using idle = new IdleTimeout(60);
|
|
510
|
+
const result = await runEvalAgent(
|
|
511
|
+
{ prompt: "investigate" },
|
|
512
|
+
{
|
|
513
|
+
session,
|
|
514
|
+
signal: idle.signal,
|
|
515
|
+
emitStatus: event => {
|
|
516
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
517
|
+
},
|
|
518
|
+
},
|
|
519
|
+
);
|
|
520
|
+
|
|
521
|
+
expect(idle.signal.aborted).toBe(false);
|
|
522
|
+
expect(result.text).toBe("done");
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it("does not let agent() progress snapshots re-arm the watchdog without a heartbeat", async () => {
|
|
526
|
+
using tempDir = TempDir.createSync("@omp-eval-agent-progress-no-rearm-");
|
|
527
|
+
const { session } = makeEvalSession(tempDir, "js-agent-progress-no-rearm");
|
|
528
|
+
mockAgents();
|
|
529
|
+
// Heartbeat slower than the budget: only the immediate beat at call start
|
|
530
|
+
// fires, so after the budget elapses nothing re-arms the watchdog.
|
|
531
|
+
setBridgeHeartbeatIntervalMs(10_000);
|
|
532
|
+
|
|
533
|
+
// Stream frequent progress snapshots (op:"agent") for well past the budget.
|
|
534
|
+
// Progress is rendered but MUST NOT count as activity — only heartbeats do.
|
|
535
|
+
vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => {
|
|
536
|
+
for (let i = 0; i < 40; i++) {
|
|
537
|
+
options.onProgress?.({
|
|
538
|
+
index: options.index,
|
|
539
|
+
id: options.id,
|
|
540
|
+
agent: options.agent.name,
|
|
541
|
+
agentSource: options.agent.source,
|
|
542
|
+
status: "running",
|
|
543
|
+
task: options.task,
|
|
544
|
+
assignment: options.assignment,
|
|
545
|
+
description: options.description,
|
|
546
|
+
recentTools: [],
|
|
547
|
+
recentOutput: [],
|
|
548
|
+
toolCount: i,
|
|
549
|
+
tokens: 0,
|
|
550
|
+
cost: 0,
|
|
551
|
+
durationMs: i * 10,
|
|
552
|
+
});
|
|
553
|
+
await Bun.sleep(10);
|
|
554
|
+
}
|
|
555
|
+
return singleResult(options, { output: "done" });
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
const ops: string[] = [];
|
|
559
|
+
using idle = new IdleTimeout(80);
|
|
560
|
+
await runEvalAgent(
|
|
561
|
+
{ prompt: "investigate" },
|
|
562
|
+
{
|
|
563
|
+
session,
|
|
564
|
+
signal: idle.signal,
|
|
565
|
+
emitStatus: event => {
|
|
566
|
+
ops.push(event.op);
|
|
567
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
568
|
+
},
|
|
569
|
+
},
|
|
570
|
+
);
|
|
571
|
+
|
|
572
|
+
// Progress streamed, but the watchdog still fired: agent snapshots never
|
|
573
|
+
// re-armed it, and the lone start heartbeat lapsed before the call ended.
|
|
574
|
+
expect(ops).toContain("agent");
|
|
575
|
+
expect(idle.signal.aborted).toBe(true);
|
|
576
|
+
});
|
|
433
577
|
});
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs, withBridgeHeartbeat } from "../heartbeat";
|
|
3
|
+
import type { JsStatusEvent } from "../js/shared/types";
|
|
4
|
+
|
|
5
|
+
describe("withBridgeHeartbeat", () => {
|
|
6
|
+
afterEach(() => {
|
|
7
|
+
setBridgeHeartbeatIntervalMs();
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
it("pumps heartbeat events on cadence while the operation is pending, then stops", async () => {
|
|
11
|
+
setBridgeHeartbeatIntervalMs(20);
|
|
12
|
+
const events: JsStatusEvent[] = [];
|
|
13
|
+
|
|
14
|
+
const value = await withBridgeHeartbeat(
|
|
15
|
+
event => events.push(event),
|
|
16
|
+
async () => {
|
|
17
|
+
await Bun.sleep(130);
|
|
18
|
+
return "done";
|
|
19
|
+
},
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
expect(value).toBe("done");
|
|
23
|
+
// ~6 ticks fit in 130ms at a 20ms cadence; assert it ticked repeatedly
|
|
24
|
+
// without pinning the exact count (scheduler jitter).
|
|
25
|
+
expect(events.length).toBeGreaterThanOrEqual(3);
|
|
26
|
+
expect(events.every(event => event.op === EVAL_HEARTBEAT_OP)).toBe(true);
|
|
27
|
+
|
|
28
|
+
// The interval is cleared once the operation settles: no further ticks.
|
|
29
|
+
const settledCount = events.length;
|
|
30
|
+
await Bun.sleep(80);
|
|
31
|
+
expect(events.length).toBe(settledCount);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("emits a heartbeat immediately so a bridge call extends the budget at once", async () => {
|
|
35
|
+
// Interval far longer than the operation: the only beat that can fire is
|
|
36
|
+
// the immediate one at call start. It must still reach the sink.
|
|
37
|
+
setBridgeHeartbeatIntervalMs(10_000);
|
|
38
|
+
const events: JsStatusEvent[] = [];
|
|
39
|
+
|
|
40
|
+
await withBridgeHeartbeat(
|
|
41
|
+
event => events.push(event),
|
|
42
|
+
async () => {
|
|
43
|
+
await Bun.sleep(30);
|
|
44
|
+
return "done";
|
|
45
|
+
},
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
expect(events.length).toBe(1);
|
|
49
|
+
expect(events[0]?.op).toBe(EVAL_HEARTBEAT_OP);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("runs the operation without emitting when no status sink is wired", async () => {
|
|
53
|
+
setBridgeHeartbeatIntervalMs(5);
|
|
54
|
+
let ran = 0;
|
|
55
|
+
|
|
56
|
+
const value = await withBridgeHeartbeat(undefined, async () => {
|
|
57
|
+
ran++;
|
|
58
|
+
await Bun.sleep(40);
|
|
59
|
+
return 42;
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
expect(value).toBe(42);
|
|
63
|
+
expect(ran).toBe(1);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("clears the heartbeat even when the operation throws", async () => {
|
|
67
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
68
|
+
const events: JsStatusEvent[] = [];
|
|
69
|
+
|
|
70
|
+
await expect(
|
|
71
|
+
withBridgeHeartbeat(
|
|
72
|
+
event => events.push(event),
|
|
73
|
+
async () => {
|
|
74
|
+
await Bun.sleep(60);
|
|
75
|
+
throw new Error("boom");
|
|
76
|
+
},
|
|
77
|
+
),
|
|
78
|
+
).rejects.toThrow("boom");
|
|
79
|
+
|
|
80
|
+
const afterThrow = events.length;
|
|
81
|
+
await Bun.sleep(60);
|
|
82
|
+
expect(events.length).toBe(afterThrow);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
@@ -8,6 +8,8 @@ import type { ModelRegistry } from "../../config/model-registry";
|
|
|
8
8
|
import { Settings } from "../../config/settings";
|
|
9
9
|
import type { ToolSession } from "../../tools";
|
|
10
10
|
import { ToolError } from "../../tools/tool-errors";
|
|
11
|
+
import { EVAL_HEARTBEAT_OP, setBridgeHeartbeatIntervalMs } from "../heartbeat";
|
|
12
|
+
import { IdleTimeout } from "../idle-timeout";
|
|
11
13
|
import { disposeAllVmContexts } from "../js/context-manager";
|
|
12
14
|
import { executeJs } from "../js/executor";
|
|
13
15
|
import { runEvalLlm } from "../llm-bridge";
|
|
@@ -97,6 +99,7 @@ function assistant(opts: {
|
|
|
97
99
|
describe("runEvalLlm", () => {
|
|
98
100
|
afterEach(() => {
|
|
99
101
|
vi.restoreAllMocks();
|
|
102
|
+
setBridgeHeartbeatIntervalMs();
|
|
100
103
|
});
|
|
101
104
|
|
|
102
105
|
it("resolves each tier to its expected model", async () => {
|
|
@@ -213,6 +216,33 @@ describe("runEvalLlm", () => {
|
|
|
213
216
|
ToolError,
|
|
214
217
|
);
|
|
215
218
|
});
|
|
219
|
+
|
|
220
|
+
it("keeps the idle watchdog armed while a slow llm() request is in flight", async () => {
|
|
221
|
+
// A oneshot completion emits no status until it returns; a slow request
|
|
222
|
+
// must not look like a stalled cell. The bridge pumps a heartbeat while it
|
|
223
|
+
// awaits, re-arming the watchdog through emitStatus.
|
|
224
|
+
setBridgeHeartbeatIntervalMs(15);
|
|
225
|
+
vi.spyOn(ai, "completeSimple").mockImplementation(async () => {
|
|
226
|
+
await Bun.sleep(200);
|
|
227
|
+
return assistant({ text: "the answer" });
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
using idle = new IdleTimeout(60);
|
|
231
|
+
const result = await runEvalLlm(
|
|
232
|
+
{ prompt: "q", model: "smol" },
|
|
233
|
+
{
|
|
234
|
+
session: makeSession(),
|
|
235
|
+
signal: idle.signal,
|
|
236
|
+
// Mirror the eval tool: only a bridge heartbeat re-arms the watchdog.
|
|
237
|
+
emitStatus: event => {
|
|
238
|
+
if (event.op === EVAL_HEARTBEAT_OP) idle.bump();
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
);
|
|
242
|
+
|
|
243
|
+
expect(idle.signal.aborted).toBe(false);
|
|
244
|
+
expect(result.text).toBe("the answer");
|
|
245
|
+
});
|
|
216
246
|
});
|
|
217
247
|
|
|
218
248
|
describe("llm() through eval runtimes", () => {
|
package/src/eval/agent-bridge.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { AgentOutputManager } from "../task/output-manager";
|
|
|
16
16
|
import type { AgentDefinition, AgentProgress } from "../task/types";
|
|
17
17
|
import type { ToolSession } from "../tools";
|
|
18
18
|
import { ToolError } from "../tools/tool-errors";
|
|
19
|
+
import { withBridgeHeartbeat } from "./heartbeat";
|
|
19
20
|
import type { JsStatusEvent } from "./js/shared/types";
|
|
20
21
|
// Import review tools for side effects (registers subagent tool handlers).
|
|
21
22
|
import "../tools/review";
|
|
@@ -231,44 +232,49 @@ export async function runEvalAgent(args: unknown, options: EvalAgentBridgeOption
|
|
|
231
232
|
const id = await outputManager.allocate(outputIdBase(parsed.label, agentName));
|
|
232
233
|
const assignment = parsed.prompt.trim();
|
|
233
234
|
const context = trimToUndefined(parsed.context);
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
235
|
+
// Pump a heartbeat while the subagent runs so the eval idle watchdog stays
|
|
236
|
+
// armed across quiet stretches (time-to-first-token, long nested tools)
|
|
237
|
+
// where `onProgress` would otherwise emit no status to re-arm it.
|
|
238
|
+
const result = await withBridgeHeartbeat(options.emitStatus, () =>
|
|
239
|
+
taskExecutor.runSubprocess({
|
|
240
|
+
cwd: options.session.cwd,
|
|
241
|
+
agent: effectiveAgent,
|
|
242
|
+
task: renderSubagentPrompt(assignment),
|
|
243
|
+
assignment,
|
|
244
|
+
context,
|
|
245
|
+
description: trimToUndefined(parsed.label),
|
|
246
|
+
index: 0,
|
|
247
|
+
id,
|
|
248
|
+
taskDepth: options.session.taskDepth ?? 0,
|
|
249
|
+
modelOverride,
|
|
250
|
+
parentActiveModelPattern,
|
|
251
|
+
thinkingLevel: effectiveAgent.thinkingLevel,
|
|
252
|
+
outputSchema: structured ? parsed.schema : undefined,
|
|
253
|
+
sessionFile,
|
|
254
|
+
persistArtifacts: Boolean(sessionFile),
|
|
255
|
+
artifactsDir,
|
|
256
|
+
contextFile,
|
|
257
|
+
enableLsp: (options.session.enableLsp ?? true) && options.session.settings.get("task.enableLsp"),
|
|
258
|
+
signal: options.signal,
|
|
259
|
+
eventBus: options.session.eventBus,
|
|
260
|
+
onProgress: progress => emitProgressStatus(options.emitStatus, progress),
|
|
261
|
+
authStorage: options.session.authStorage,
|
|
262
|
+
modelRegistry: options.session.modelRegistry,
|
|
263
|
+
settings: options.session.settings,
|
|
264
|
+
mcpManager,
|
|
265
|
+
contextFiles,
|
|
266
|
+
skills: availableSkills,
|
|
267
|
+
autoloadSkills: resolvedAutoloadSkills,
|
|
268
|
+
workspaceTree: options.session.workspaceTree,
|
|
269
|
+
promptTemplates: options.session.promptTemplates,
|
|
270
|
+
localProtocolOptions,
|
|
271
|
+
parentArtifactManager,
|
|
272
|
+
parentHindsightSessionState: options.session.getHindsightSessionState?.(),
|
|
273
|
+
parentMnemopiSessionState: options.session.getMnemopiSessionState?.(),
|
|
274
|
+
parentTelemetry: options.session.getTelemetry?.(),
|
|
275
|
+
parentEvalSessionId,
|
|
276
|
+
}),
|
|
277
|
+
);
|
|
272
278
|
|
|
273
279
|
if (result.exitCode !== 0 || result.error) {
|
|
274
280
|
const failureMessage =
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side handler for the eval `parallel()` / `pipeline()` worker pool.
|
|
3
|
+
*
|
|
4
|
+
* The pool ceiling is not a kernel-side knob: it tracks the `task.maxConcurrency`
|
|
5
|
+
* setting so an eval fan-out runs as wide as a `task` tool batch would. `0` means
|
|
6
|
+
* unbounded — run every item at once, exactly like `task.maxConcurrency = 0`.
|
|
7
|
+
*/
|
|
8
|
+
import type { ToolSession } from "../tools";
|
|
9
|
+
import type { JsStatusEvent } from "./js/shared/types";
|
|
10
|
+
|
|
11
|
+
/** Synthetic bridge name reserved for the parallel-pool ceiling across both runtimes. */
|
|
12
|
+
export const EVAL_CONCURRENCY_BRIDGE_NAME = "__concurrency__";
|
|
13
|
+
|
|
14
|
+
export interface EvalConcurrencyBridgeOptions {
|
|
15
|
+
session: ToolSession;
|
|
16
|
+
signal?: AbortSignal;
|
|
17
|
+
emitStatus?: (event: JsStatusEvent) => void;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface EvalConcurrencyResult {
|
|
21
|
+
/** Worker-pool ceiling; `0` means unbounded (run every item at once). */
|
|
22
|
+
limit: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the worker-pool ceiling for an eval cell's `parallel()`/`pipeline()`
|
|
27
|
+
* helpers from the live `task.maxConcurrency` setting. Negative/non-finite
|
|
28
|
+
* values collapse to `0` (unbounded), matching the `task` tool's own handling.
|
|
29
|
+
*/
|
|
30
|
+
export function runEvalConcurrency(_args: unknown, options: EvalConcurrencyBridgeOptions): EvalConcurrencyResult {
|
|
31
|
+
const raw = options.session.settings.get("task.maxConcurrency");
|
|
32
|
+
const limit = Number.isFinite(raw) ? Math.trunc(raw) : 0;
|
|
33
|
+
return { limit: limit > 0 ? limit : 0 };
|
|
34
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keepalive for in-flight host-side eval bridge calls.
|
|
3
|
+
*
|
|
4
|
+
* The eval watchdog ({@link ../tools/eval IdleTimeout}) caps a cell's `timeout`
|
|
5
|
+
* as a wall-clock budget on the cell's *own* work, but pauses that budget while
|
|
6
|
+
* a host-side `agent()`/`parallel()` (via `runSubprocess`) or `llm()` (a single
|
|
7
|
+
* completion) call is in flight. Those calls are the only thing that re-arms the
|
|
8
|
+
* watchdog — and they can run for long stretches with **no** status of their own
|
|
9
|
+
* (a subagent's time-to-first-token on a reasoning model, a long quiet nested
|
|
10
|
+
* tool, or the entire body of a oneshot `llm()` call). Without a keepalive the
|
|
11
|
+
* watchdog would mistake that delegated work for the cell stalling and abort it
|
|
12
|
+
* mid-flight, killing the subagent.
|
|
13
|
+
*
|
|
14
|
+
* {@link withBridgeHeartbeat} bridges that gap by emitting a synthetic
|
|
15
|
+
* {@link EVAL_HEARTBEAT_OP} status event immediately when the call begins and
|
|
16
|
+
* then on a fixed cadence until it settles. The event rides the same
|
|
17
|
+
* `emitStatus → onStatus` channel both runtimes already forward, so it re-arms
|
|
18
|
+
* the watchdog without any new plumbing. The heartbeat is the *sole* signal that
|
|
19
|
+
* extends the budget: consumers MUST treat it as a pure keepalive — bump the
|
|
20
|
+
* watchdog and drop it (never persist or render it) — see the executor display
|
|
21
|
+
* sinks and the eval tool's `onStatus` handler. Every other status event
|
|
22
|
+
* (compute helpers, `log()`/`phase()`, tool results) counts against the budget.
|
|
23
|
+
*/
|
|
24
|
+
import type { JsStatusEvent } from "./js/shared/types";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Synthetic status op emitted purely to keep the eval idle watchdog alive while
|
|
28
|
+
* a host-side bridge call is in flight. Carries no payload.
|
|
29
|
+
*/
|
|
30
|
+
export const EVAL_HEARTBEAT_OP = "heartbeat";
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Heartbeat cadence. Comfortably below the default 30s idle budget (and the
|
|
34
|
+
* larger budgets long fanouts run under), so a working bridge call always bumps
|
|
35
|
+
* the watchdog before it expires, while a genuine stall is still bounded once
|
|
36
|
+
* the call settles and the heartbeat stops.
|
|
37
|
+
*/
|
|
38
|
+
const HEARTBEAT_INTERVAL_MS = 5_000;
|
|
39
|
+
|
|
40
|
+
let heartbeatIntervalMs = HEARTBEAT_INTERVAL_MS;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Test seam: override the heartbeat cadence so integration tests can exercise
|
|
44
|
+
* the keepalive within a sub-second idle budget. Pass no value to restore the
|
|
45
|
+
* production default.
|
|
46
|
+
*/
|
|
47
|
+
export function setBridgeHeartbeatIntervalMs(ms?: number): void {
|
|
48
|
+
heartbeatIntervalMs = ms === undefined ? HEARTBEAT_INTERVAL_MS : Math.max(1, Math.floor(ms));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Run {@link operation}, pumping {@link EVAL_HEARTBEAT_OP} status events through
|
|
53
|
+
* {@link emitStatus} — one immediately, then on a fixed cadence — until it
|
|
54
|
+
* settles. The immediate beat pauses the watchdog the instant the call begins,
|
|
55
|
+
* so a bridge call that starts close to the budget edge (after the cell already
|
|
56
|
+
* spent most of it computing) is not aborted before the first interval tick. A
|
|
57
|
+
* no-op wrapper when no `emitStatus` sink is wired (the heartbeat would reach
|
|
58
|
+
* nobody).
|
|
59
|
+
*/
|
|
60
|
+
export async function withBridgeHeartbeat<T>(
|
|
61
|
+
emitStatus: ((event: JsStatusEvent) => void) | undefined,
|
|
62
|
+
operation: () => Promise<T>,
|
|
63
|
+
): Promise<T> {
|
|
64
|
+
if (!emitStatus) return operation();
|
|
65
|
+
emitStatus({ op: EVAL_HEARTBEAT_OP });
|
|
66
|
+
const timer = setInterval(() => emitStatus({ op: EVAL_HEARTBEAT_OP }), heartbeatIntervalMs);
|
|
67
|
+
// Never keep the event loop alive for the heartbeat alone.
|
|
68
|
+
timer.unref?.();
|
|
69
|
+
try {
|
|
70
|
+
return await operation();
|
|
71
|
+
} finally {
|
|
72
|
+
clearInterval(timer);
|
|
73
|
+
}
|
|
74
|
+
}
|
package/src/eval/js/executor.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DEFAULT_MAX_BYTES, OutputSink } from "../../session/streaming-output";
|
|
2
2
|
import type { ToolSession } from "../../tools";
|
|
3
3
|
import { resolveOutputMaxColumns, resolveOutputSinkHeadBytes } from "../../tools/output-meta";
|
|
4
|
+
import { EVAL_HEARTBEAT_OP } from "../heartbeat";
|
|
4
5
|
import { executeInVmContext, type JsDisplayOutput } from "./context-manager";
|
|
5
6
|
import type { JsStatusEvent } from "./shared/types";
|
|
6
7
|
|
|
@@ -59,11 +60,10 @@ function isTimeoutReason(reason: unknown): boolean {
|
|
|
59
60
|
);
|
|
60
61
|
}
|
|
61
62
|
|
|
62
|
-
function formatJsTimeoutAnnotation(timeoutMs: number | undefined
|
|
63
|
-
const suffix = idle ? " of inactivity" : "";
|
|
63
|
+
function formatJsTimeoutAnnotation(timeoutMs: number | undefined): string {
|
|
64
64
|
if (timeoutMs === undefined) return "Command timed out";
|
|
65
65
|
const secs = Math.max(1, Math.round(timeoutMs / 1000));
|
|
66
|
-
return `Command timed out after ${secs} seconds
|
|
66
|
+
return `Command timed out after ${secs} seconds`;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
export async function executeJs(code: string, options: JsExecutorOptions): Promise<JsResult> {
|
|
@@ -85,10 +85,9 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
85
85
|
options.signal && timeoutSignal
|
|
86
86
|
? AbortSignal.any([options.signal, timeoutSignal])
|
|
87
87
|
: (options.signal ?? timeoutSignal);
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
const idleMode = legacyTimeoutMs === undefined && options.idleTimeoutMs !== undefined;
|
|
88
|
+
// The eval tool drives cancellation via an idle-aware `signal` and passes only
|
|
89
|
+
// an inactivity budget; use it solely as worker cold-start headroom and never
|
|
90
|
+
// derive a competing fixed timer from it.
|
|
92
91
|
const acquireBudgetMs = legacyTimeoutMs ?? options.idleTimeoutMs;
|
|
93
92
|
|
|
94
93
|
try {
|
|
@@ -105,8 +104,13 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
105
104
|
signal,
|
|
106
105
|
onText: chunk => outputSink.push(chunk),
|
|
107
106
|
onDisplay: output => {
|
|
107
|
+
if (output.type === "status") {
|
|
108
|
+
// Heartbeats are pure idle-watchdog keepalives: forward them so
|
|
109
|
+
// the eval tool re-arms its timer, but never store or render them.
|
|
110
|
+
options.onStatus?.(output.event);
|
|
111
|
+
if (output.event.op === EVAL_HEARTBEAT_OP) return;
|
|
112
|
+
}
|
|
108
113
|
displayOutputs.push(output);
|
|
109
|
-
if (output.type === "status") options.onStatus?.(output.event);
|
|
110
114
|
},
|
|
111
115
|
},
|
|
112
116
|
});
|
|
@@ -127,7 +131,7 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
127
131
|
if (signal?.aborted || isAbortError(error)) {
|
|
128
132
|
const timedOut = Boolean(timeoutSignal?.aborted) || isTimeoutReason(options.signal?.reason);
|
|
129
133
|
if (timedOut) {
|
|
130
|
-
outputSink.push(formatJsTimeoutAnnotation(legacyTimeoutMs ?? options.idleTimeoutMs
|
|
134
|
+
outputSink.push(formatJsTimeoutAnnotation(legacyTimeoutMs ?? options.idleTimeoutMs));
|
|
131
135
|
}
|
|
132
136
|
const summary = await outputSink.dump();
|
|
133
137
|
return {
|
|
@@ -55,15 +55,24 @@ if (!globalThis.__omp_js_prelude_loaded__) {
|
|
|
55
55
|
return hasOwn(o, "schema") ? JSON.parse(text) : text;
|
|
56
56
|
};
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
// Pool ceiling mirrors the task tool's `task.maxConcurrency` setting so an
|
|
59
|
+
// eval fan-out runs as wide as a `task` batch would. 0 (and any non-finite
|
|
60
|
+
// reply) means unbounded — run every item at once.
|
|
61
|
+
const __concurrencyLimit = async () => {
|
|
62
|
+
try {
|
|
63
|
+
const r = await globalThis.__omp_call_tool__("__concurrency__", {});
|
|
64
|
+
const n = Math.trunc(Number(r && typeof r === "object" ? r.limit : r));
|
|
65
|
+
return Number.isFinite(n) && n > 0 ? n : 0;
|
|
66
|
+
} catch {
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
62
69
|
};
|
|
63
70
|
|
|
64
|
-
const __pool = async (items,
|
|
71
|
+
const __pool = async (items, fn) => {
|
|
65
72
|
const list = Array.from(items ?? []);
|
|
66
|
-
|
|
73
|
+
if (list.length === 0) return [];
|
|
74
|
+
const limit = await __concurrencyLimit();
|
|
75
|
+
const concurrency = limit > 0 ? Math.min(limit, list.length) : list.length;
|
|
67
76
|
const results = new Array(list.length);
|
|
68
77
|
let next = 0;
|
|
69
78
|
const worker = async () => {
|
|
@@ -77,23 +86,17 @@ if (!globalThis.__omp_js_prelude_loaded__) {
|
|
|
77
86
|
return results;
|
|
78
87
|
};
|
|
79
88
|
|
|
80
|
-
const parallel = async
|
|
81
|
-
__pool(thunks,
|
|
89
|
+
const parallel = async thunks =>
|
|
90
|
+
__pool(thunks, (thunk, index) => {
|
|
82
91
|
if (typeof thunk !== "function") throw new TypeError("parallel() expects an iterable of functions");
|
|
83
92
|
return thunk(index);
|
|
84
93
|
});
|
|
85
94
|
|
|
86
|
-
const pipeline = async (items, ...
|
|
87
|
-
let opts = {};
|
|
88
|
-
const last = stagesAndOptions.at(-1);
|
|
89
|
-
if (last && typeof last === "object" && !Array.isArray(last)) {
|
|
90
|
-
opts = last;
|
|
91
|
-
stagesAndOptions = stagesAndOptions.slice(0, -1);
|
|
92
|
-
}
|
|
95
|
+
const pipeline = async (items, ...stages) => {
|
|
93
96
|
let current = Array.from(items ?? []);
|
|
94
|
-
for (const stage of
|
|
97
|
+
for (const stage of stages) {
|
|
95
98
|
if (typeof stage !== "function") throw new TypeError("pipeline() stages must be functions");
|
|
96
|
-
current = await __pool(current,
|
|
99
|
+
current = await __pool(current, stage);
|
|
97
100
|
}
|
|
98
101
|
return current;
|
|
99
102
|
};
|