@oh-my-pi/pi-coding-agent 15.7.1 → 15.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -6
- package/dist/types/auto-thinking/classifier.d.ts +35 -0
- package/dist/types/cli/args.d.ts +1 -1
- package/dist/types/cli/extension-flags.d.ts +36 -0
- package/dist/types/config/config-file.d.ts +4 -0
- package/dist/types/config/file-lock.d.ts +23 -0
- package/dist/types/config/keybindings.d.ts +2 -1
- package/dist/types/config/model-registry.d.ts +6 -0
- package/dist/types/config/settings-schema.d.ts +112 -69
- package/dist/types/edit/hashline/diff.d.ts +9 -3
- package/dist/types/eval/__tests__/agent-bridge.test.d.ts +1 -0
- package/dist/types/eval/__tests__/budget-bridge.test.d.ts +1 -0
- package/dist/types/eval/__tests__/idle-timeout.test.d.ts +1 -0
- package/dist/types/eval/agent-bridge.d.ts +25 -0
- package/dist/types/eval/backend.d.ts +17 -2
- package/dist/types/eval/budget-bridge.d.ts +29 -0
- package/dist/types/eval/idle-timeout.d.ts +28 -0
- package/dist/types/eval/js/executor.d.ts +8 -0
- package/dist/types/eval/js/tool-bridge.d.ts +2 -1
- package/dist/types/eval/py/executor.d.ts +13 -0
- package/dist/types/exec/bash-executor.d.ts +1 -0
- package/dist/types/extensibility/custom-tools/types.d.ts +2 -2
- package/dist/types/extensibility/extensions/runner.d.ts +7 -0
- package/dist/types/extensibility/plugins/git-url.d.ts +11 -1
- package/dist/types/extensibility/plugins/manager.d.ts +12 -1
- package/dist/types/extensibility/shared-events.d.ts +2 -2
- package/dist/types/memory-backend/index.d.ts +1 -1
- package/dist/types/memory-backend/resolve.d.ts +1 -1
- package/dist/types/memory-backend/types.d.ts +3 -3
- package/dist/types/mnemopi/backend.d.ts +4 -0
- package/dist/types/mnemopi/config.d.ts +29 -0
- package/dist/types/mnemopi/state.d.ts +72 -0
- package/dist/types/modes/components/custom-editor.d.ts +2 -2
- package/dist/types/modes/components/model-selector.d.ts +3 -2
- package/dist/types/modes/components/omfg-panel.d.ts +19 -0
- package/dist/types/modes/controllers/command-controller.d.ts +7 -0
- package/dist/types/modes/controllers/input-controller.d.ts +1 -3
- package/dist/types/modes/controllers/omfg-controller.d.ts +10 -0
- package/dist/types/modes/controllers/omfg-rule.d.ts +26 -0
- package/dist/types/modes/gradient-highlight.d.ts +5 -1
- package/dist/types/modes/interactive-mode.d.ts +7 -3
- package/dist/types/modes/magic-keywords.d.ts +14 -0
- package/dist/types/modes/markdown-prose.d.ts +27 -0
- package/dist/types/modes/orchestrate.d.ts +7 -2
- package/dist/types/modes/shared.d.ts +1 -1
- package/dist/types/modes/theme/theme.d.ts +2 -1
- package/dist/types/modes/turn-budget.d.ts +18 -0
- package/dist/types/modes/types.d.ts +7 -3
- package/dist/types/modes/ultrathink.d.ts +7 -2
- package/dist/types/modes/workflow.d.ts +15 -0
- package/dist/types/sdk.d.ts +15 -4
- package/dist/types/session/agent-session.d.ts +59 -23
- package/dist/types/session/session-manager.d.ts +18 -0
- package/dist/types/session/session-storage.d.ts +6 -0
- package/dist/types/session/shake-types.d.ts +24 -0
- package/dist/types/task/executor.d.ts +2 -2
- package/dist/types/thinking.d.ts +39 -1
- package/dist/types/tiny/device.d.ts +3 -3
- package/dist/types/tiny/models.d.ts +34 -1
- package/dist/types/tiny/title-protocol.d.ts +4 -0
- package/dist/types/tools/index.d.ts +19 -3
- package/dist/types/tools/memory-edit.d.ts +1 -1
- package/examples/sdk/09-api-keys-and-oauth.ts +2 -2
- package/package.json +10 -10
- package/src/auto-thinking/classifier.ts +180 -0
- package/src/autoresearch/tools/run-experiment.ts +45 -113
- package/src/cli/args.ts +39 -16
- package/src/cli/extension-flags.ts +48 -0
- package/src/cli/plugin-cli.ts +11 -2
- package/src/config/config-file.ts +98 -13
- package/src/config/file-lock.ts +60 -17
- package/src/config/keybindings.ts +78 -27
- package/src/config/model-registry.ts +7 -1
- package/src/config/settings-schema.ts +118 -71
- package/src/config/settings.ts +12 -0
- package/src/edit/hashline/diff.ts +87 -22
- package/src/edit/renderer.ts +16 -12
- package/src/edit/streaming.ts +17 -6
- package/src/eval/__tests__/agent-bridge.test.ts +433 -0
- package/src/eval/__tests__/budget-bridge.test.ts +69 -0
- package/src/eval/__tests__/idle-timeout.test.ts +66 -0
- package/src/eval/__tests__/shared-executors.test.ts +53 -0
- package/src/eval/agent-bridge.ts +295 -0
- package/src/eval/backend.ts +17 -2
- package/src/eval/budget-bridge.ts +48 -0
- package/src/eval/idle-timeout.ts +80 -0
- package/src/eval/js/executor.ts +35 -7
- package/src/eval/js/index.ts +2 -1
- package/src/eval/js/shared/local-module-loader.ts +75 -10
- package/src/eval/js/shared/prelude.txt +85 -1
- package/src/eval/js/tool-bridge.ts +9 -0
- package/src/eval/py/executor.ts +41 -14
- package/src/eval/py/index.ts +2 -1
- package/src/eval/py/prelude.py +132 -1
- package/src/exec/bash-executor.ts +2 -3
- package/src/extensibility/custom-tools/types.ts +2 -2
- package/src/extensibility/extensions/runner.ts +12 -2
- package/src/extensibility/plugins/git-url.ts +90 -4
- package/src/extensibility/plugins/manager.ts +103 -7
- package/src/extensibility/shared-events.ts +2 -2
- package/src/internal-urls/docs-index.generated.ts +88 -88
- package/src/main.ts +50 -56
- package/src/memory-backend/index.ts +1 -1
- package/src/memory-backend/resolve.ts +3 -3
- package/src/memory-backend/types.ts +3 -3
- package/src/{mnemosyne → mnemopi}/backend.ts +70 -70
- package/src/{mnemosyne → mnemopi}/config.ts +36 -36
- package/src/{mnemosyne → mnemopi}/state.ts +67 -67
- package/src/modes/acp/acp-agent.ts +13 -3
- package/src/modes/components/agent-dashboard.ts +6 -6
- package/src/modes/components/custom-editor.ts +4 -11
- package/src/modes/components/extensions/state-manager.ts +3 -4
- package/src/modes/components/footer.ts +18 -12
- package/src/modes/components/hook-selector.ts +86 -20
- package/src/modes/components/model-selector.ts +20 -11
- package/src/modes/components/oauth-selector.ts +93 -21
- package/src/modes/components/omfg-panel.ts +141 -0
- package/src/modes/components/settings-defs.ts +9 -2
- package/src/modes/components/settings-selector.ts +4 -1
- package/src/modes/components/status-line/segments.ts +13 -5
- package/src/modes/components/tips.txt +2 -1
- package/src/modes/components/tool-execution.ts +38 -19
- package/src/modes/components/tree-selector.ts +4 -3
- package/src/modes/components/user-message-selector.ts +94 -19
- package/src/modes/components/user-message.ts +8 -1
- package/src/modes/controllers/command-controller.ts +57 -0
- package/src/modes/controllers/event-controller.ts +65 -3
- package/src/modes/controllers/input-controller.ts +14 -11
- package/src/modes/controllers/omfg-controller.ts +283 -0
- package/src/modes/controllers/omfg-rule.ts +647 -0
- package/src/modes/controllers/selector-controller.ts +21 -6
- package/src/modes/gradient-highlight.ts +23 -6
- package/src/modes/interactive-mode.ts +41 -7
- package/src/modes/magic-keywords.ts +20 -0
- package/src/modes/markdown-prose.ts +247 -0
- package/src/modes/orchestrate.ts +17 -11
- package/src/modes/shared.ts +3 -11
- package/src/modes/theme/theme.ts +6 -0
- package/src/modes/turn-budget.ts +31 -0
- package/src/modes/types.ts +7 -1
- package/src/modes/ultrathink.ts +16 -10
- package/src/modes/utils/hotkeys-markdown.ts +1 -1
- package/src/modes/workflow.ts +42 -0
- package/src/prompts/system/auto-thinking-difficulty-local.md +14 -0
- package/src/prompts/system/auto-thinking-difficulty.md +12 -0
- package/src/prompts/system/omfg-user.md +51 -0
- package/src/prompts/system/system-prompt.md +1 -0
- package/src/prompts/system/workflow-notice.md +70 -0
- package/src/prompts/tools/eval.md +13 -1
- package/src/prompts/tools/memory-edit.md +1 -1
- package/src/sdk.ts +86 -38
- package/src/session/agent-session.ts +558 -80
- package/src/session/session-manager.ts +32 -0
- package/src/session/session-storage.ts +68 -8
- package/src/session/shake-types.ts +44 -0
- package/src/slash-commands/builtin-registry.ts +41 -16
- package/src/task/executor.ts +3 -3
- package/src/task/index.ts +6 -6
- package/src/thinking.ts +73 -1
- package/src/tiny/device.ts +4 -10
- package/src/tiny/models.ts +54 -2
- package/src/tiny/title-protocol.ts +11 -1
- package/src/tiny/worker.ts +19 -7
- package/src/tools/eval.ts +202 -26
- package/src/tools/grouped-file-output.ts +9 -2
- package/src/tools/index.ts +17 -5
- package/src/tools/memory-edit.ts +4 -4
- package/src/tools/memory-recall.ts +5 -5
- package/src/tools/memory-reflect.ts +5 -5
- package/src/tools/memory-retain.ts +4 -4
- package/src/tools/render-utils.ts +2 -1
- package/src/tools/search.ts +480 -76
- package/dist/types/mnemosyne/backend.d.ts +0 -4
- package/dist/types/mnemosyne/config.d.ts +0 -29
- package/dist/types/mnemosyne/state.d.ts +0 -72
- /package/dist/types/{mnemosyne → mnemopi}/index.d.ts +0 -0
- /package/src/{mnemosyne → mnemopi}/index.ts +0 -0
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side handler for the eval `agent()` helper.
|
|
3
|
+
*/
|
|
4
|
+
import * as fs from "node:fs/promises";
|
|
5
|
+
import * as os from "node:os";
|
|
6
|
+
import * as path from "node:path";
|
|
7
|
+
import { prompt, Snowflake } from "@oh-my-pi/pi-utils";
|
|
8
|
+
import * as z from "zod/v4";
|
|
9
|
+
import { resolveAgentModelPatterns } from "../config/model-resolver";
|
|
10
|
+
import type { LocalProtocolOptions } from "../internal-urls";
|
|
11
|
+
import { MCPManager } from "../mcp/manager";
|
|
12
|
+
import subagentUserPromptTemplate from "../prompts/system/subagent-user-prompt.md" with { type: "text" };
|
|
13
|
+
import * as taskDiscovery from "../task/discovery";
|
|
14
|
+
import * as taskExecutor from "../task/executor";
|
|
15
|
+
import { AgentOutputManager } from "../task/output-manager";
|
|
16
|
+
import type { AgentDefinition, AgentProgress } from "../task/types";
|
|
17
|
+
import type { ToolSession } from "../tools";
|
|
18
|
+
import { ToolError } from "../tools/tool-errors";
|
|
19
|
+
import type { JsStatusEvent } from "./js/shared/types";
|
|
20
|
+
// Import review tools for side effects (registers subagent tool handlers).
|
|
21
|
+
import "../tools/review";
|
|
22
|
+
|
|
23
|
+
/** Synthetic bridge name reserved for the `agent()` helper across both runtimes. */
|
|
24
|
+
export const EVAL_AGENT_BRIDGE_NAME = "__agent__";
|
|
25
|
+
|
|
26
|
+
/** Hard recursion limit for eval-driven subagents. */
|
|
27
|
+
export const EVAL_AGENT_MAX_DEPTH = 3;
|
|
28
|
+
|
|
29
|
+
const DEFAULT_AGENT_TYPE = "task";
|
|
30
|
+
const DEFAULT_AGENT_LABEL = "EvalAgent";
|
|
31
|
+
|
|
32
|
+
const agentArgsSchema = z.object({
|
|
33
|
+
prompt: z.string().min(1, "prompt must be a non-empty string"),
|
|
34
|
+
agentType: z.string().min(1).optional(),
|
|
35
|
+
model: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]).optional(),
|
|
36
|
+
context: z.string().optional(),
|
|
37
|
+
label: z.string().optional(),
|
|
38
|
+
schema: z.unknown().optional(),
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
interface EvalAgentArgs {
|
|
42
|
+
prompt: string;
|
|
43
|
+
agentType?: string;
|
|
44
|
+
model?: string | string[];
|
|
45
|
+
context?: string;
|
|
46
|
+
label?: string;
|
|
47
|
+
schema?: unknown;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface EvalAgentBridgeOptions {
|
|
51
|
+
session: ToolSession;
|
|
52
|
+
signal?: AbortSignal;
|
|
53
|
+
emitStatus?: (event: JsStatusEvent) => void;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface EvalAgentResult {
|
|
57
|
+
text: string;
|
|
58
|
+
details: {
|
|
59
|
+
agent: string;
|
|
60
|
+
id: string;
|
|
61
|
+
model?: string | string[];
|
|
62
|
+
structured: boolean;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function parseAgentArgs(args: unknown): EvalAgentArgs {
|
|
67
|
+
const parsed = agentArgsSchema.safeParse(args);
|
|
68
|
+
if (!parsed.success) {
|
|
69
|
+
const issue = parsed.error.issues[0];
|
|
70
|
+
const where = issue?.path.length ? `${issue.path.join(".")}: ` : "";
|
|
71
|
+
throw new ToolError(`agent() received invalid arguments: ${where}${issue?.message ?? "bad input"}`);
|
|
72
|
+
}
|
|
73
|
+
return parsed.data;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function assertDepthAllowed(session: ToolSession): void {
|
|
77
|
+
const taskDepth = session.taskDepth ?? 0;
|
|
78
|
+
if (taskDepth >= EVAL_AGENT_MAX_DEPTH) {
|
|
79
|
+
throw new ToolError(
|
|
80
|
+
`agent() cannot spawn another agent at task depth ${taskDepth}; maximum depth is ${EVAL_AGENT_MAX_DEPTH}.`,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function assertSpawnAllowed(session: ToolSession, agentName: string): void {
|
|
86
|
+
const parentSpawns = session.getSessionSpawns() ?? "*";
|
|
87
|
+
if (parentSpawns === "*") return;
|
|
88
|
+
if (parentSpawns === "") {
|
|
89
|
+
throw new ToolError(`Cannot spawn '${agentName}'. Allowed: none (spawns disabled for this agent)`);
|
|
90
|
+
}
|
|
91
|
+
const allowedSpawns = parentSpawns.split(",").map(spawn => spawn.trim());
|
|
92
|
+
if (!allowedSpawns.includes(agentName)) {
|
|
93
|
+
throw new ToolError(`Cannot spawn '${agentName}'. Allowed: ${parentSpawns}`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function assertAgentEnabled(session: ToolSession, agentName: string, agents: AgentDefinition[]): void {
|
|
98
|
+
const disabledAgents = session.settings.get("task.disabledAgents") as string[];
|
|
99
|
+
if (!disabledAgents.includes(agentName)) return;
|
|
100
|
+
const enabled = agents.filter(agent => !disabledAgents.includes(agent.name)).map(agent => agent.name);
|
|
101
|
+
throw new ToolError(
|
|
102
|
+
`Agent "${agentName}" is disabled in settings. Enable it via /agents, or use a different agent type.${enabled.length > 0 ? ` Available: ${enabled.join(", ")}` : ""}`,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function assertNotPlanMode(session: ToolSession): void {
|
|
107
|
+
if (session.getPlanModeState?.()?.enabled) {
|
|
108
|
+
throw new ToolError("agent() is unavailable in plan mode.");
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function renderSubagentPrompt(assignment: string): string {
|
|
113
|
+
return prompt.render(subagentUserPromptTemplate, { assignment: assignment.trim(), independentMode: false });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function trimToUndefined(value: string | undefined): string | undefined {
|
|
117
|
+
const trimmed = value?.trim();
|
|
118
|
+
return trimmed ? trimmed : undefined;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function outputIdBase(label: string | undefined, agentName: string): string {
|
|
122
|
+
const source = trimToUndefined(label) ?? agentName ?? DEFAULT_AGENT_LABEL;
|
|
123
|
+
const sanitized = source.replace(/[^A-Za-z0-9_-]+/g, "").slice(0, 48);
|
|
124
|
+
return sanitized || DEFAULT_AGENT_LABEL;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function getOutputManager(session: ToolSession): AgentOutputManager {
|
|
128
|
+
if (session.agentOutputManager) return session.agentOutputManager;
|
|
129
|
+
const manager = new AgentOutputManager(session.getArtifactsDir ?? (() => null));
|
|
130
|
+
session.agentOutputManager = manager;
|
|
131
|
+
return manager;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function getArtifacts(session: ToolSession): Promise<{
|
|
135
|
+
sessionFile: string | null;
|
|
136
|
+
artifactsDir: string;
|
|
137
|
+
contextFile?: string;
|
|
138
|
+
}> {
|
|
139
|
+
const sessionFile = session.getSessionFile();
|
|
140
|
+
const sessionArtifactsDir = sessionFile ? sessionFile.slice(0, -6) : null;
|
|
141
|
+
const artifactsDir = sessionArtifactsDir ?? path.join(os.tmpdir(), `omp-eval-agent-${Snowflake.next()}`);
|
|
142
|
+
await fs.mkdir(artifactsDir, { recursive: true });
|
|
143
|
+
|
|
144
|
+
const shouldWriteConversationContext = session.settings.get("irc.enabled") !== true;
|
|
145
|
+
const compactContext = shouldWriteConversationContext ? session.getCompactContext?.() : undefined;
|
|
146
|
+
if (!compactContext) return { sessionFile, artifactsDir };
|
|
147
|
+
|
|
148
|
+
const contextFile = path.join(artifactsDir, "context.md");
|
|
149
|
+
await Bun.write(contextFile, compactContext);
|
|
150
|
+
return { sessionFile, artifactsDir, contextFile };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function emitProgressStatus(emitStatus: ((event: JsStatusEvent) => void) | undefined, progress: AgentProgress): void {
|
|
154
|
+
if (!emitStatus) return;
|
|
155
|
+
const preview = (progress.assignment ?? progress.task ?? "").split("\n")[0]?.slice(0, 120);
|
|
156
|
+
emitStatus({
|
|
157
|
+
op: "agent",
|
|
158
|
+
id: progress.id,
|
|
159
|
+
agent: progress.agent,
|
|
160
|
+
status: progress.status,
|
|
161
|
+
lastIntent: progress.lastIntent,
|
|
162
|
+
currentTool: progress.currentTool,
|
|
163
|
+
currentToolArgs: progress.currentToolArgs,
|
|
164
|
+
taskPreview: preview || undefined,
|
|
165
|
+
toolCount: progress.toolCount,
|
|
166
|
+
tokens: progress.tokens,
|
|
167
|
+
contextTokens: progress.contextTokens,
|
|
168
|
+
contextWindow: progress.contextWindow,
|
|
169
|
+
cost: progress.cost,
|
|
170
|
+
durationMs: progress.durationMs,
|
|
171
|
+
model: progress.resolvedModel,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Run a single subagent on behalf of an eval cell's `agent()` call.
|
|
177
|
+
*/
|
|
178
|
+
export async function runEvalAgent(args: unknown, options: EvalAgentBridgeOptions): Promise<EvalAgentResult> {
|
|
179
|
+
const parsed = parseAgentArgs(args);
|
|
180
|
+
const agentName = parsed.agentType ?? DEFAULT_AGENT_TYPE;
|
|
181
|
+
const structured = Object.hasOwn(parsed, "schema");
|
|
182
|
+
|
|
183
|
+
assertNotPlanMode(options.session);
|
|
184
|
+
assertDepthAllowed(options.session);
|
|
185
|
+
assertSpawnAllowed(options.session, agentName);
|
|
186
|
+
|
|
187
|
+
const turnBudget = options.session.getTurnBudget?.();
|
|
188
|
+
if (turnBudget?.hard && turnBudget.total !== null && turnBudget.spent >= turnBudget.total) {
|
|
189
|
+
throw new ToolError(
|
|
190
|
+
`agent() blocked: turn token budget exhausted (${turnBudget.spent}/${turnBudget.total} output tokens). Raise or drop the +Nk! ceiling to continue.`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const { agents } = await taskDiscovery.discoverAgents(options.session.cwd);
|
|
195
|
+
const agent = taskDiscovery.getAgent(agents, agentName);
|
|
196
|
+
if (!agent) {
|
|
197
|
+
const available = agents.map(candidate => candidate.name).join(", ") || "none";
|
|
198
|
+
throw new ToolError(`Unknown agent "${agentName}". Available: ${available}`);
|
|
199
|
+
}
|
|
200
|
+
assertAgentEnabled(options.session, agentName, agents);
|
|
201
|
+
|
|
202
|
+
const effectiveAgent = agent;
|
|
203
|
+
const parentActiveModelPattern = options.session.getActiveModelString?.();
|
|
204
|
+
const agentModelOverrides = options.session.settings.get("task.agentModelOverrides");
|
|
205
|
+
const modelOverride = resolveAgentModelPatterns({
|
|
206
|
+
settingsOverride: parsed.model ?? agentModelOverrides[agentName],
|
|
207
|
+
agentModel: effectiveAgent.model,
|
|
208
|
+
settings: options.session.settings,
|
|
209
|
+
activeModelPattern: parentActiveModelPattern,
|
|
210
|
+
fallbackModelPattern: options.session.getModelString?.(),
|
|
211
|
+
});
|
|
212
|
+
const availableSkills = [...(options.session.skills ?? [])];
|
|
213
|
+
const resolvedAutoloadSkills =
|
|
214
|
+
effectiveAgent.autoloadSkills?.length && availableSkills.length > 0
|
|
215
|
+
? effectiveAgent.autoloadSkills
|
|
216
|
+
.map(name => availableSkills.find(skill => skill.name === name))
|
|
217
|
+
.filter((skill): skill is NonNullable<typeof skill> => skill !== undefined)
|
|
218
|
+
: [];
|
|
219
|
+
const contextFiles = options.session.contextFiles?.filter(
|
|
220
|
+
file => path.basename(file.path).toLowerCase() !== "agents.md",
|
|
221
|
+
);
|
|
222
|
+
const localProtocolOptions: LocalProtocolOptions = options.session.localProtocolOptions ?? {
|
|
223
|
+
getArtifactsDir: options.session.getArtifactsDir ?? (() => null),
|
|
224
|
+
getSessionId: options.session.getSessionId ?? (() => null),
|
|
225
|
+
};
|
|
226
|
+
const parentArtifactManager = options.session.getArtifactManager?.() ?? undefined;
|
|
227
|
+
const parentEvalSessionId = options.session.getEvalSessionId?.() ?? undefined;
|
|
228
|
+
const mcpManager = options.session.mcpManager ?? MCPManager.instance();
|
|
229
|
+
const { sessionFile, artifactsDir, contextFile } = await getArtifacts(options.session);
|
|
230
|
+
const outputManager = getOutputManager(options.session);
|
|
231
|
+
const id = await outputManager.allocate(outputIdBase(parsed.label, agentName));
|
|
232
|
+
const assignment = parsed.prompt.trim();
|
|
233
|
+
const context = trimToUndefined(parsed.context);
|
|
234
|
+
const result = await taskExecutor.runSubprocess({
|
|
235
|
+
cwd: options.session.cwd,
|
|
236
|
+
agent: effectiveAgent,
|
|
237
|
+
task: renderSubagentPrompt(assignment),
|
|
238
|
+
assignment,
|
|
239
|
+
context,
|
|
240
|
+
description: trimToUndefined(parsed.label),
|
|
241
|
+
index: 0,
|
|
242
|
+
id,
|
|
243
|
+
taskDepth: options.session.taskDepth ?? 0,
|
|
244
|
+
modelOverride,
|
|
245
|
+
parentActiveModelPattern,
|
|
246
|
+
thinkingLevel: effectiveAgent.thinkingLevel,
|
|
247
|
+
outputSchema: structured ? parsed.schema : undefined,
|
|
248
|
+
sessionFile,
|
|
249
|
+
persistArtifacts: Boolean(sessionFile),
|
|
250
|
+
artifactsDir,
|
|
251
|
+
contextFile,
|
|
252
|
+
enableLsp: (options.session.enableLsp ?? true) && options.session.settings.get("task.enableLsp"),
|
|
253
|
+
signal: options.signal,
|
|
254
|
+
eventBus: options.session.eventBus,
|
|
255
|
+
onProgress: progress => emitProgressStatus(options.emitStatus, progress),
|
|
256
|
+
authStorage: options.session.authStorage,
|
|
257
|
+
modelRegistry: options.session.modelRegistry,
|
|
258
|
+
settings: options.session.settings,
|
|
259
|
+
mcpManager,
|
|
260
|
+
contextFiles,
|
|
261
|
+
skills: availableSkills,
|
|
262
|
+
autoloadSkills: resolvedAutoloadSkills,
|
|
263
|
+
workspaceTree: options.session.workspaceTree,
|
|
264
|
+
promptTemplates: options.session.promptTemplates,
|
|
265
|
+
localProtocolOptions,
|
|
266
|
+
parentArtifactManager,
|
|
267
|
+
parentHindsightSessionState: options.session.getHindsightSessionState?.(),
|
|
268
|
+
parentMnemopiSessionState: options.session.getMnemopiSessionState?.(),
|
|
269
|
+
parentTelemetry: options.session.getTelemetry?.(),
|
|
270
|
+
parentEvalSessionId,
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
if (result.exitCode !== 0 || result.error) {
|
|
274
|
+
const failureMessage =
|
|
275
|
+
result.error ?? result.stderr ?? result.abortReason ?? `agent() subagent '${agentName}' failed.`;
|
|
276
|
+
throw new ToolError(failureMessage);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
options.session.recordEvalSubagentUsage?.(result.usage?.output ?? 0);
|
|
280
|
+
|
|
281
|
+
// The final `onProgress` flush from `runSubprocess` already emits a
|
|
282
|
+
// status:"completed" event carrying full stats (toolCount, cost, context),
|
|
283
|
+
// so we don't emit a second, sparser completion event here — it would
|
|
284
|
+
// coalesce over the richer one and drop those stats.
|
|
285
|
+
|
|
286
|
+
return {
|
|
287
|
+
text: result.output,
|
|
288
|
+
details: {
|
|
289
|
+
agent: result.agent,
|
|
290
|
+
id: result.id,
|
|
291
|
+
model: result.resolvedModel ?? modelOverride,
|
|
292
|
+
structured,
|
|
293
|
+
},
|
|
294
|
+
};
|
|
295
|
+
}
|
package/src/eval/backend.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ToolSession } from "../tools";
|
|
2
|
-
import type { EvalDisplayOutput, EvalLanguage } from "./types";
|
|
2
|
+
import type { EvalDisplayOutput, EvalLanguage, EvalStatusEvent } from "./types";
|
|
3
3
|
|
|
4
4
|
/** Per-cell execute() options. */
|
|
5
5
|
export interface ExecutorBackendExecOptions {
|
|
@@ -9,11 +9,26 @@ export interface ExecutorBackendExecOptions {
|
|
|
9
9
|
kernelOwnerId: string | undefined;
|
|
10
10
|
signal?: AbortSignal;
|
|
11
11
|
session: ToolSession;
|
|
12
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Inactivity budget in milliseconds (the cell's `timeout`). Cancellation is
|
|
14
|
+
* driven entirely by `signal`, which the eval tool arms as an idle watchdog
|
|
15
|
+
* that fires a `TimeoutError` reason after this much time with no progress
|
|
16
|
+
* (status) events. Backends use this value only for timeout-annotation text
|
|
17
|
+
* and as cold-start headroom; they MUST NOT derive a competing wall-clock
|
|
18
|
+
* timer from it.
|
|
19
|
+
*/
|
|
20
|
+
idleTimeoutMs: number;
|
|
13
21
|
reset: boolean;
|
|
14
22
|
artifactPath: string | undefined;
|
|
15
23
|
artifactId: string | undefined;
|
|
16
24
|
onChunk: (chunk: string) => void;
|
|
25
|
+
/**
|
|
26
|
+
* Live status events (read/write/agent/…) delivered as they are emitted,
|
|
27
|
+
* before the cell finishes. The same events are also returned in
|
|
28
|
+
* `displayOutputs`; this channel exists so callers can stream long-running
|
|
29
|
+
* progress (e.g. `agent()` subagents) into the UI mid-execution.
|
|
30
|
+
*/
|
|
31
|
+
onStatus?: (event: EvalStatusEvent) => void;
|
|
17
32
|
}
|
|
18
33
|
|
|
19
34
|
/** Result returned by a backend's execute(). */
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side handler for the eval `budget` helper.
|
|
3
|
+
*
|
|
4
|
+
* Reports the active token ceiling and amount spent so kernel helpers can
|
|
5
|
+
* compute remaining budget. Precedence: a `+Nk`/`+Nk!` per-turn directive (the
|
|
6
|
+
* user's immediate intent) wins; otherwise an active Goal Mode budget; otherwise
|
|
7
|
+
* no ceiling, with `spent` still reflecting this turn's output where available.
|
|
8
|
+
*/
|
|
9
|
+
import type { ToolSession } from "../tools";
|
|
10
|
+
import type { JsStatusEvent } from "./js/shared/types";
|
|
11
|
+
|
|
12
|
+
/** Synthetic bridge name reserved for the `budget` helper across both runtimes. */
|
|
13
|
+
export const EVAL_BUDGET_BRIDGE_NAME = "__budget__";
|
|
14
|
+
|
|
15
|
+
export interface EvalBudgetBridgeOptions {
|
|
16
|
+
session: ToolSession;
|
|
17
|
+
signal?: AbortSignal;
|
|
18
|
+
emitStatus?: (event: JsStatusEvent) => void;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface EvalBudgetResult {
|
|
22
|
+
total: number | null;
|
|
23
|
+
spent: number;
|
|
24
|
+
/** Whether the ceiling is enforced (eval `agent()` throws past it) vs advisory. */
|
|
25
|
+
hard: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Resolve the current token budget snapshot for an eval cell's `budget` helper.
|
|
30
|
+
* The returned object is JSON-passed verbatim by the bridge transport; kernel
|
|
31
|
+
* helpers read `.total`/`.spent`/`.hard` directly.
|
|
32
|
+
*/
|
|
33
|
+
export async function runEvalBudget(_args: unknown, options: EvalBudgetBridgeOptions): Promise<EvalBudgetResult> {
|
|
34
|
+
const turn = options.session.getTurnBudget?.();
|
|
35
|
+
if (turn && turn.total !== null) {
|
|
36
|
+
return { total: turn.total, spent: turn.spent, hard: turn.hard };
|
|
37
|
+
}
|
|
38
|
+
const goal = options.session.getGoalModeState?.();
|
|
39
|
+
if (goal?.enabled && goal.goal) {
|
|
40
|
+
return {
|
|
41
|
+
total: goal.goal.tokenBudget ?? null,
|
|
42
|
+
spent: goal.goal.tokensUsed ?? 0,
|
|
43
|
+
hard: goal.goal.tokenBudget != null,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const spent = turn?.spent ?? options.session.getUsageStatistics?.()?.output ?? 0;
|
|
47
|
+
return { total: null, spent, hard: false };
|
|
48
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inactivity watchdog for eval cells.
|
|
3
|
+
*
|
|
4
|
+
* A cell's `timeout` is treated as an *idle* budget rather than a hard
|
|
5
|
+
* wall-clock deadline: the watchdog aborts {@link signal} (with a
|
|
6
|
+
* `TimeoutError` reason, matching `AbortSignal.timeout`) only once `idleMs`
|
|
7
|
+
* elapses with no {@link bump}. Every progress signal re-arms it, so a
|
|
8
|
+
* long-running fanout that keeps reporting progress (e.g. `agent()` status
|
|
9
|
+
* updates, `log()`/`phase()`) never trips the timeout, while a genuinely
|
|
10
|
+
* stalled cell still gets interrupted.
|
|
11
|
+
*
|
|
12
|
+
* The timer self-reschedules instead of being torn down and recreated on every
|
|
13
|
+
* bump, so a high-frequency stream of bumps (sub-second agent progress) costs
|
|
14
|
+
* one timestamp write per event rather than churning a timer each time.
|
|
15
|
+
*/
|
|
16
|
+
export class IdleTimeout {
|
|
17
|
+
readonly #controller = new AbortController();
|
|
18
|
+
readonly #idleMs: number;
|
|
19
|
+
/** Absolute time (epoch ms) at which inactivity is considered to have expired. */
|
|
20
|
+
#deadlineMs: number;
|
|
21
|
+
#timer: NodeJS.Timeout | undefined;
|
|
22
|
+
#settled = false;
|
|
23
|
+
|
|
24
|
+
constructor(idleMs: number) {
|
|
25
|
+
this.#idleMs = Math.max(1, Math.floor(idleMs));
|
|
26
|
+
this.#deadlineMs = Date.now() + this.#idleMs;
|
|
27
|
+
this.#arm(this.#idleMs);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Aborts with a `TimeoutError` reason once the inactivity budget is exhausted. */
|
|
31
|
+
get signal(): AbortSignal {
|
|
32
|
+
return this.#controller.signal;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Configured inactivity budget in milliseconds. */
|
|
36
|
+
get idleMs(): number {
|
|
37
|
+
return this.#idleMs;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Record activity, pushing the inactivity deadline forward by `idleMs`. */
|
|
41
|
+
bump(): void {
|
|
42
|
+
if (this.#settled) return;
|
|
43
|
+
this.#deadlineMs = Date.now() + this.#idleMs;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Stop the watchdog. Safe to call multiple times. */
|
|
47
|
+
dispose(): void {
|
|
48
|
+
if (this.#settled) return;
|
|
49
|
+
this.#settled = true;
|
|
50
|
+
if (this.#timer) {
|
|
51
|
+
clearTimeout(this.#timer);
|
|
52
|
+
this.#timer = undefined;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
[Symbol.dispose](): void {
|
|
57
|
+
this.dispose();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#arm(delayMs: number): void {
|
|
61
|
+
const timer = setTimeout(() => this.#onExpire(), Math.max(0, delayMs));
|
|
62
|
+
// Never keep the event loop alive for the watchdog itself.
|
|
63
|
+
timer.unref?.();
|
|
64
|
+
this.#timer = timer;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
#onExpire(): void {
|
|
68
|
+
if (this.#settled) return;
|
|
69
|
+
const remainingMs = this.#deadlineMs - Date.now();
|
|
70
|
+
if (remainingMs > 0) {
|
|
71
|
+
// A bump moved the deadline forward after this timer was armed; wait
|
|
72
|
+
// out the remaining window instead of firing early.
|
|
73
|
+
this.#arm(remainingMs);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.#settled = true;
|
|
77
|
+
this.#timer = undefined;
|
|
78
|
+
this.#controller.abort(new DOMException(`Idle for ${Math.round(this.#idleMs / 1000)}s`, "TimeoutError"));
|
|
79
|
+
}
|
|
80
|
+
}
|
package/src/eval/js/executor.ts
CHANGED
|
@@ -2,12 +2,20 @@ import { DEFAULT_MAX_BYTES, OutputSink } from "../../session/streaming-output";
|
|
|
2
2
|
import type { ToolSession } from "../../tools";
|
|
3
3
|
import { resolveOutputMaxColumns, resolveOutputSinkHeadBytes } from "../../tools/output-meta";
|
|
4
4
|
import { executeInVmContext, type JsDisplayOutput } from "./context-manager";
|
|
5
|
+
import type { JsStatusEvent } from "./shared/types";
|
|
5
6
|
|
|
6
7
|
export interface JsExecutorOptions {
|
|
7
8
|
cwd?: string;
|
|
8
9
|
timeoutMs?: number;
|
|
9
10
|
deadlineMs?: number;
|
|
11
|
+
/**
|
|
12
|
+
* Inactivity budget (ms). Used for worker cold-start headroom and
|
|
13
|
+
* timeout-annotation text when the caller drives cancellation via an
|
|
14
|
+
* idle-aware `signal` instead of `deadlineMs`/`timeoutMs`. Never arms a timer.
|
|
15
|
+
*/
|
|
16
|
+
idleTimeoutMs?: number;
|
|
10
17
|
onChunk?: (chunk: string) => Promise<void> | void;
|
|
18
|
+
onStatus?: (event: JsStatusEvent) => void;
|
|
11
19
|
signal?: AbortSignal;
|
|
12
20
|
sessionId: string;
|
|
13
21
|
reset?: boolean;
|
|
@@ -44,6 +52,20 @@ function isAbortError(error: unknown): boolean {
|
|
|
44
52
|
);
|
|
45
53
|
}
|
|
46
54
|
|
|
55
|
+
function isTimeoutReason(reason: unknown): boolean {
|
|
56
|
+
return (
|
|
57
|
+
(reason instanceof DOMException && reason.name === "TimeoutError") ||
|
|
58
|
+
(reason instanceof Error && reason.name === "TimeoutError")
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function formatJsTimeoutAnnotation(timeoutMs: number | undefined, idle: boolean): string {
|
|
63
|
+
const suffix = idle ? " of inactivity" : "";
|
|
64
|
+
if (timeoutMs === undefined) return "Command timed out";
|
|
65
|
+
const secs = Math.max(1, Math.round(timeoutMs / 1000));
|
|
66
|
+
return `Command timed out after ${secs} seconds${suffix}`;
|
|
67
|
+
}
|
|
68
|
+
|
|
47
69
|
export async function executeJs(code: string, options: JsExecutorOptions): Promise<JsResult> {
|
|
48
70
|
const displayOutputs: JsDisplayOutput[] = [];
|
|
49
71
|
const outputSink = new OutputSink({
|
|
@@ -54,15 +76,20 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
54
76
|
maxColumns: resolveOutputMaxColumns(options.session.settings),
|
|
55
77
|
onChunk: chunk => options.onChunk?.(chunk),
|
|
56
78
|
});
|
|
57
|
-
const
|
|
79
|
+
const legacyTimeoutMs = getExecutionTimeoutMs(options);
|
|
58
80
|
const timeoutSignal =
|
|
59
|
-
typeof
|
|
60
|
-
? AbortSignal.timeout(
|
|
81
|
+
typeof legacyTimeoutMs === "number" && Number.isFinite(legacyTimeoutMs) && legacyTimeoutMs > 0
|
|
82
|
+
? AbortSignal.timeout(legacyTimeoutMs)
|
|
61
83
|
: undefined;
|
|
62
84
|
const signal =
|
|
63
85
|
options.signal && timeoutSignal
|
|
64
86
|
? AbortSignal.any([options.signal, timeoutSignal])
|
|
65
87
|
: (options.signal ?? timeoutSignal);
|
|
88
|
+
// Idle mode: the eval tool drives cancellation via an idle-aware `signal` and
|
|
89
|
+
// passes only an inactivity budget. Use it for worker cold-start headroom and
|
|
90
|
+
// timeout-annotation text; never derive a competing fixed timer from it.
|
|
91
|
+
const idleMode = legacyTimeoutMs === undefined && options.idleTimeoutMs !== undefined;
|
|
92
|
+
const acquireBudgetMs = legacyTimeoutMs ?? options.idleTimeoutMs;
|
|
66
93
|
|
|
67
94
|
try {
|
|
68
95
|
await executeInVmContext({
|
|
@@ -73,12 +100,13 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
73
100
|
reset: options.reset,
|
|
74
101
|
code,
|
|
75
102
|
filename: `js-cell-${crypto.randomUUID()}.js`,
|
|
76
|
-
timeoutMs,
|
|
103
|
+
timeoutMs: acquireBudgetMs,
|
|
77
104
|
runState: {
|
|
78
105
|
signal,
|
|
79
106
|
onText: chunk => outputSink.push(chunk),
|
|
80
107
|
onDisplay: output => {
|
|
81
108
|
displayOutputs.push(output);
|
|
109
|
+
if (output.type === "status") options.onStatus?.(output.event);
|
|
82
110
|
},
|
|
83
111
|
},
|
|
84
112
|
});
|
|
@@ -97,9 +125,9 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
|
|
|
97
125
|
};
|
|
98
126
|
} catch (error) {
|
|
99
127
|
if (signal?.aborted || isAbortError(error)) {
|
|
100
|
-
const
|
|
101
|
-
if (
|
|
102
|
-
outputSink.push(
|
|
128
|
+
const timedOut = Boolean(timeoutSignal?.aborted) || isTimeoutReason(options.signal?.reason);
|
|
129
|
+
if (timedOut) {
|
|
130
|
+
outputSink.push(formatJsTimeoutAnnotation(legacyTimeoutMs ?? options.idleTimeoutMs, idleMode));
|
|
103
131
|
}
|
|
104
132
|
const summary = await outputSink.dump();
|
|
105
133
|
return {
|
package/src/eval/js/index.ts
CHANGED
|
@@ -20,7 +20,7 @@ export default {
|
|
|
20
20
|
async execute(code: string, opts: ExecutorBackendExecOptions): Promise<ExecutorBackendResult> {
|
|
21
21
|
const result = await executeJs(code, {
|
|
22
22
|
cwd: opts.cwd,
|
|
23
|
-
|
|
23
|
+
idleTimeoutMs: opts.idleTimeoutMs,
|
|
24
24
|
signal: opts.signal,
|
|
25
25
|
sessionId: namespaceSessionId(opts.sessionId),
|
|
26
26
|
sessionFile: opts.sessionFile,
|
|
@@ -28,6 +28,7 @@ export default {
|
|
|
28
28
|
artifactPath: opts.artifactPath,
|
|
29
29
|
artifactId: opts.artifactId,
|
|
30
30
|
onChunk: opts.onChunk,
|
|
31
|
+
onStatus: opts.onStatus,
|
|
31
32
|
session: opts.session,
|
|
32
33
|
});
|
|
33
34
|
return {
|