@hexdecimal16/nexo 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/cli/src/common.js +1 -1
- package/cli/src/main.js +17 -2
- package/core/src/config/schema.js +3 -1
- package/core/src/engine/context-trim.js +39 -0
- package/core/src/engine/errors.js +10 -2
- package/core/src/engine/messages.js +14 -1
- package/core/src/engine/provider-gate.js +5 -3
- package/core/src/engine/run.js +105 -14
- package/core/src/providers/claude-p.js +129 -23
- package/core/src/providers/codex-p.js +16 -3
- package/core/src/providers/kiro-p.js +115 -21
- package/core/src/providers/reply-text.js +110 -0
- package/core/src/session/mentions.js +32 -8
- package/core/src/session/session.js +17 -13
- package/core/src/settings/config-edit.js +6 -0
- package/package.json +1 -1
- package/sdk/src/index.js +2 -0
- package/store/src/run-dir.js +10 -0
- package/tools/src/builtin.js +39 -8
- package/tools/src/images.js +174 -0
- package/tools/src/index.js +1 -0
- package/tui/src/app.js +71 -5
- package/tui/src/commands.js +1 -1
- package/tui/src/components.js +73 -0
- package/tui/src/image-attachments.js +142 -0
- package/web/public/lib/span-view.ts +21 -1
- package/web/public/pages/run.ts +8 -1
- package/web/public/run.html +1 -0
- package/web/public/style.css +2 -0
package/README.md
CHANGED
|
@@ -28,6 +28,8 @@ nexo doctor # check Node, config, keys and t
|
|
|
28
28
|
|
|
29
29
|
`nexo --help` lists every command. Data lives in `~/.nexo`. Set `NEXO_HOME` to move it.
|
|
30
30
|
|
|
31
|
+
Images (PNG, JPEG, GIF, WebP) work with every vision model, including the `claude-p`, `codex-p` and `kiro-p` CLI providers. In the chat, drag an image file into the terminal or press ctrl+v to paste a screenshot; it shows as `[Image #1]`. `@shot.png` attaches an image in the project, `nexo run "…" --image shot.png` attaches one to a task, and the agent's `read` tool opens image files. Large images are scaled down to 2000 px, older images in a long run are replaced by a note (the file can be read again), and `nexo ui` shows the images each run saw.
|
|
32
|
+
|
|
31
33
|
## Settings and keys
|
|
32
34
|
|
|
33
35
|
Every option is in `/settings` inside a chat, or in `nexo settings` from the shell:
|
|
@@ -76,6 +78,7 @@ try {
|
|
|
76
78
|
- `nexo.keys()`, `nexo.setKey()` and `nexo.removeKey()` manage stored keys. `nexo.webSearch(query)` searches without a model.
|
|
77
79
|
- The SDK denies every tool approval by default, so a backend never waits on a person. Pass `allowAll: true` to `createNexo` or `run` to auto-approve tool actions in sandboxed environments. Always pass `budgetUsd`.
|
|
78
80
|
- `checkSetup()` runs the doctor checks without starting the engine, for a start-up pre-flight.
|
|
81
|
+
- Send images with `run({ instruction, images: [loadImageFile("shot.png").image] })`, `session.turn(text, { images })` or `session.steer(text, images)`.
|
|
79
82
|
|
|
80
83
|
## Security
|
|
81
84
|
|
package/cli/src/common.js
CHANGED
|
@@ -19,7 +19,7 @@ export const USAGE = `Usage:
|
|
|
19
19
|
nexo settings remove-key <provider> delete a key from the credentials file
|
|
20
20
|
nexo settings keys key status of the model and web search providers (never the keys)
|
|
21
21
|
nexo search <query...> [--count <n>] [--json] search the web with the configured Brave/Tavily keys
|
|
22
|
-
nexo run <instruction...> [--skill <name>] [--model <s1|s2|provider/id>] [--effort <level>] [--workspace <dir>] [--budget <usd>] [--no-review] [--allow-all]
|
|
22
|
+
nexo run <instruction...> [--skill <name>] [--model <s1|s2|provider/id>] [--effort <level>] [--workspace <dir>] [--budget <usd>] [--image <path>]... [--no-review] [--allow-all]
|
|
23
23
|
nexo cost [--by model|tier|day|run|skill] [--since YYYY-MM-DD]
|
|
24
24
|
nexo reindex
|
|
25
25
|
nexo review-queue [--limit <n>] [--days <n>] recent runs without a verdict, worst first
|
package/cli/src/main.js
CHANGED
|
@@ -6,7 +6,8 @@ import { EFFORTS } from "../../router/src/index.js";
|
|
|
6
6
|
import { parseSkillInvocation } from "../../skills/src/index.js";
|
|
7
7
|
import { costReport, ensureNexoDirs, openDb, reindex, resolveNexoPaths } from "../../store/src/index.js";
|
|
8
8
|
import { NexoError, toErrorPayload } from "../../telemetry/src/index.js";
|
|
9
|
-
import { nonInteractivePrompter } from "../../tools/src/index.js";
|
|
9
|
+
import { describeImage, loadImageFile, nonInteractivePrompter, withAttachedImagesNote } from "../../tools/src/index.js";
|
|
10
|
+
import { basename, resolve as resolvePath } from "node:path";
|
|
10
11
|
import { EXIT_SIGINT, EXIT_SIGTERM, exitCodeFor, printError, USAGE, usageError } from "./common.js";
|
|
11
12
|
import { evalCommand } from "./eval.js";
|
|
12
13
|
import { learnCommand, learnEventLine, memoryCommand, playbookCommand } from "./learn.js";
|
|
@@ -97,11 +98,24 @@ async function runCommand(args, io, options) {
|
|
|
97
98
|
budget: { type: "string" },
|
|
98
99
|
"no-review": { type: "boolean" },
|
|
99
100
|
"allow-all": { type: "boolean" },
|
|
101
|
+
image: { type: "string", multiple: true },
|
|
100
102
|
},
|
|
101
103
|
});
|
|
102
104
|
const instruction = positionals.join(" ").trim();
|
|
103
105
|
if (!instruction)
|
|
104
106
|
throw usageError("nexo run needs an instruction");
|
|
107
|
+
const images = [];
|
|
108
|
+
const imageLabels = [];
|
|
109
|
+
for (const path of values.image ?? []) {
|
|
110
|
+
try {
|
|
111
|
+
const loaded = loadImageFile(resolvePath(options.cwd ?? process.cwd(), path));
|
|
112
|
+
images.push(loaded.image);
|
|
113
|
+
imageLabels.push(describeImage(basename(path), loaded));
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
throw usageError(`--image ${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
105
119
|
if (values.effort !== undefined && !EFFORTS.includes(values.effort)) {
|
|
106
120
|
throw usageError(`--effort must be one of ${EFFORTS.join(", ")}`);
|
|
107
121
|
}
|
|
@@ -144,7 +158,8 @@ async function runCommand(args, io, options) {
|
|
|
144
158
|
let result;
|
|
145
159
|
try {
|
|
146
160
|
result = await runtime.engine.run({
|
|
147
|
-
instruction: invoked?.instruction ?? instruction,
|
|
161
|
+
instruction: withAttachedImagesNote(invoked?.instruction ?? instruction, imageLabels),
|
|
162
|
+
...(images.length > 0 ? { images } : {}),
|
|
148
163
|
skill: values.skill ?? invoked?.skill,
|
|
149
164
|
model: values.model,
|
|
150
165
|
effort: values.effort,
|
|
@@ -79,6 +79,8 @@ export const ConfigSchema = Type.Object({
|
|
|
79
79
|
compactAt: Type.Number({ exclusiveMinimum: 0, maximum: 1 }),
|
|
80
80
|
/** Also compact when one S2 request with the history would cost more than this share of the turn's budget; 0 turns it off. */
|
|
81
81
|
compactAtBudgetShare: Type.Number({ minimum: 0, maximum: 1 }),
|
|
82
|
+
/** kiro-p keeps a run's Kiro session open between model calls and sends only the new entries (see `SessionCache`). */
|
|
83
|
+
reuseKiroSession: Type.Boolean(),
|
|
82
84
|
}, { additionalProperties: false }),
|
|
83
85
|
/** Guards against a run that loops: a cap on tool calls and a limit on identical consecutive calls; 0 turns a guard off. */
|
|
84
86
|
loop: Type.Object({ maxToolCalls: Type.Integer({ minimum: 0 }), repeatLimit: Type.Integer({ minimum: 0 }) }, { additionalProperties: false }),
|
|
@@ -181,7 +183,7 @@ export const DEFAULT_CONFIG = {
|
|
|
181
183
|
escalationCutoff: 0.8,
|
|
182
184
|
estimateOutputTokens: 4096,
|
|
183
185
|
},
|
|
184
|
-
session: { compactAt: 0.8, compactAtBudgetShare: 0.2 },
|
|
186
|
+
session: { compactAt: 0.8, compactAtBudgetShare: 0.2, reuseKiroSession: false },
|
|
185
187
|
loop: { maxToolCalls: 200, repeatLimit: 3 },
|
|
186
188
|
spawn: { maxParallelSubagents: 4, maxDepth: 4 },
|
|
187
189
|
instructions: [],
|
|
@@ -24,3 +24,42 @@ export function clearOldToolResults(messages, keepRecent) {
|
|
|
24
24
|
});
|
|
25
25
|
return { messages: cleared > 0 ? out : messages, cleared };
|
|
26
26
|
}
|
|
27
|
+
/** Images the model keeps seeing; older ones are replaced by a note (Anthropic many-image limits start above 20). */
|
|
28
|
+
export const KEEP_RECENT_IMAGES = 8;
|
|
29
|
+
/** Older images are dropped only once this many more have built up, so the cached prompt prefix changes rarely. */
|
|
30
|
+
const IMAGE_DROP_BATCH = 4;
|
|
31
|
+
export const DROPPED_IMAGE_NOTE = "[nexo] Earlier image removed to save context. Read the file again (or ask for it) to see it.";
|
|
32
|
+
/**
|
|
33
|
+
* Keeps the newest `keep` images in the conversation and replaces older ones with a short note; the text around them
|
|
34
|
+
* (paths, sizes, the tool calls) stays, so the agent can read an image again. Nothing changes until `keep + batch`
|
|
35
|
+
* images have built up, so a long screenshot-heavy run rewrites its history (and loses its prompt cache) only once
|
|
36
|
+
* every `batch` images.
|
|
37
|
+
*/
|
|
38
|
+
export function limitImages(messages, keep = KEEP_RECENT_IMAGES, batch = IMAGE_DROP_BATCH) {
|
|
39
|
+
let total = 0;
|
|
40
|
+
for (const m of messages) {
|
|
41
|
+
const content = m.content;
|
|
42
|
+
if (Array.isArray(content))
|
|
43
|
+
for (const block of content)
|
|
44
|
+
if (block?.type === "image")
|
|
45
|
+
total++;
|
|
46
|
+
}
|
|
47
|
+
if (total <= keep + batch)
|
|
48
|
+
return { messages, dropped: 0 };
|
|
49
|
+
let toDrop = total - keep;
|
|
50
|
+
const dropped = toDrop;
|
|
51
|
+
const out = messages.map((m) => {
|
|
52
|
+
const content = m.content;
|
|
53
|
+
if (toDrop === 0 || !Array.isArray(content) || !content.some((b) => b?.type === "image"))
|
|
54
|
+
return m;
|
|
55
|
+
const next = content.map((block) => {
|
|
56
|
+
if (toDrop > 0 && block?.type === "image") {
|
|
57
|
+
toDrop--;
|
|
58
|
+
return { type: "text", text: DROPPED_IMAGE_NOTE };
|
|
59
|
+
}
|
|
60
|
+
return block;
|
|
61
|
+
});
|
|
62
|
+
return { ...m, content: next };
|
|
63
|
+
});
|
|
64
|
+
return { messages: out, dropped };
|
|
65
|
+
}
|
|
@@ -46,11 +46,19 @@ const MAX_RETRY_ATTEMPTS = 200;
|
|
|
46
46
|
// Transient failures pi's classifier does not know: Kiro's capacity message and the kiro-p idle stall.
|
|
47
47
|
const EXTRA_TRANSIENT = /high traffic|try again|no output for \d+\s*s/i;
|
|
48
48
|
const NOT_TRANSIENT = /quota|billing|usage limit/i;
|
|
49
|
-
|
|
49
|
+
// The model answered outside the reply format: it called a tool natively, or its JSON reply could not be used.
|
|
50
|
+
const FORMAT_FAILURE = /native tool call|reply was not a JSON object/i;
|
|
51
|
+
/** Retries a streak of format failures gets: the same request usually comes back right, but a model that keeps derailing must not loop. */
|
|
52
|
+
export const FORMAT_RETRIES = 2;
|
|
53
|
+
/** Whether a failed model call ended on a reply the provider could not use, as opposed to a transport or capacity error. */
|
|
54
|
+
export function isFormatFailure(message) {
|
|
55
|
+
return message.stopReason === "error" && FORMAT_FAILURE.test(message.errorMessage ?? "");
|
|
56
|
+
}
|
|
57
|
+
/** Whether a failed model call is worth retrying: pi's transient patterns, the kiro-cli ones above and format failures. */
|
|
50
58
|
export function isRetryableProviderMessage(message) {
|
|
51
59
|
if (message.stopReason !== "error" || !message.errorMessage)
|
|
52
60
|
return false;
|
|
53
|
-
if (isRetryableAssistantError(message))
|
|
61
|
+
if (isRetryableAssistantError(message) || isFormatFailure(message))
|
|
54
62
|
return true;
|
|
55
63
|
return EXTRA_TRANSIENT.test(message.errorMessage) && !NOT_TRANSIENT.test(message.errorMessage);
|
|
56
64
|
}
|
|
@@ -81,6 +81,19 @@ export function foreignToolCallsAsText(messages, target) {
|
|
|
81
81
|
});
|
|
82
82
|
return changed ? out : messages;
|
|
83
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Characters an image is counted as: about 1,600 visual tokens (a downscaled screenshot on Claude's standard tier) at
|
|
86
|
+
* 4 characters per token. Counting its base64 data instead would make one screenshot look like a million tokens.
|
|
87
|
+
*/
|
|
88
|
+
export const IMAGE_ESTIMATE_CHARS = 6400;
|
|
84
89
|
export function estimatePromptChars(messages) {
|
|
85
|
-
|
|
90
|
+
let images = 0;
|
|
91
|
+
const json = JSON.stringify(messages, (_key, value) => {
|
|
92
|
+
if (value && typeof value === "object" && value.type === "image" && typeof value.data === "string") {
|
|
93
|
+
images++;
|
|
94
|
+
return { type: "image" };
|
|
95
|
+
}
|
|
96
|
+
return value;
|
|
97
|
+
});
|
|
98
|
+
return json.length + images * IMAGE_ESTIMATE_CHARS;
|
|
86
99
|
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Semaphore } from "./semaphore.js";
|
|
2
2
|
/**
|
|
3
3
|
* Wraps Models.streamSimple so each provider never exceeds its configured number of concurrent requests. When the
|
|
4
|
-
* request carries no `apiKey`, the key from `resolveApiKey` is passed as the request's `apiKey` option
|
|
4
|
+
* request carries no `apiKey`, the key from `resolveApiKey` is passed as the request's `apiKey` option, and
|
|
5
|
+
* `metadataFor`'s entries are added under the request's own `metadata`.
|
|
5
6
|
*/
|
|
6
|
-
export function createGatedStreamFn(models, limits, resolveApiKey) {
|
|
7
|
+
export function createGatedStreamFn(models, limits, resolveApiKey, metadataFor) {
|
|
7
8
|
const semaphores = new Map();
|
|
8
9
|
const semaphoreFor = (provider) => {
|
|
9
10
|
const limit = limits[provider];
|
|
@@ -21,7 +22,8 @@ export function createGatedStreamFn(models, limits, resolveApiKey) {
|
|
|
21
22
|
const release = sem ? await sem.acquire(options?.signal) : () => { };
|
|
22
23
|
try {
|
|
23
24
|
const apiKey = options?.apiKey ?? resolveApiKey?.(model.provider);
|
|
24
|
-
const
|
|
25
|
+
const metadata = metadataFor?.(model.provider);
|
|
26
|
+
const stream = models.streamSimple(model, context, apiKey === undefined && !metadata ? options : { ...options, ...(apiKey === undefined ? {} : { apiKey }), ...(metadata ? { metadata: { ...metadata, ...options?.metadata } } : {}) });
|
|
25
27
|
stream.result().then(release, release);
|
|
26
28
|
return stream;
|
|
27
29
|
}
|
package/core/src/engine/run.js
CHANGED
|
@@ -4,20 +4,21 @@ import { createModelRegistry, DEFAULT_ROUTER_POLICY, ESCALATION_BLOCK_REASONS, e
|
|
|
4
4
|
import { createLoadSkillTool, formatSkillIndex, parseSkillFile } from "../../../skills/src/index.js";
|
|
5
5
|
import { createArtifactSet, enqueueLearnJob, getActiveArtifact, insertRun, isPidAlive, listPlaybookItems, listSpans, newId, recoverCrashedRuns, resolveSourcedArtifact, RunDir, sessionRunDirs, updateRun, } from "../../../store/src/index.js";
|
|
6
6
|
import { NexoError, toErrorPayload } from "../../../telemetry/src/index.js";
|
|
7
|
-
import { builtinToolDescriptions, createAskUserTool, createBuiltinTools, createChangeTracker, createPolicy, createWebSearchTool, FileLeases, SEARCH_PROVIDER_IDS, TOOL_OUTPUT_PREFIX, } from "../../../tools/src/index.js";
|
|
7
|
+
import { builtinToolDescriptions, createAskUserTool, createBuiltinTools, createChangeTracker, createPolicy, createWebSearchTool, FileLeases, isReadOnlyCommand, matchesSessionRule, SEARCH_PROVIDER_IDS, TOOL_OUTPUT_PREFIX, } from "../../../tools/src/index.js";
|
|
8
8
|
import { realpathSync, statSync } from "node:fs";
|
|
9
9
|
import { join } from "node:path";
|
|
10
10
|
import { createApprovalClassifier } from "../approval/classifier.js";
|
|
11
11
|
import { budgetExhaustedError, checkAdmission, recordSpend, runBudgetLimit } from "../cost/admission.js";
|
|
12
12
|
import { budgetLine, RunBudget } from "../cost/budget.js";
|
|
13
13
|
import { estimateCallCostUsd, priceUsage } from "../cost/pricing.js";
|
|
14
|
-
import { clearOldToolResults } from "./context-trim.js";
|
|
15
|
-
import { classifyProviderError, contextOverflowError, isRetryableProviderMessage, providerRetryDelayMs, toolErrorText, withToolErrorCodes, withinRetryLimits } from "./errors.js";
|
|
14
|
+
import { clearOldToolResults, limitImages } from "./context-trim.js";
|
|
15
|
+
import { classifyProviderError, contextOverflowError, FORMAT_RETRIES, isFormatFailure, isRetryableProviderMessage, providerRetryDelayMs, toolErrorText, withToolErrorCodes, withinRetryLimits, } from "./errors.js";
|
|
16
16
|
import { assistantText, contentText, estimatePromptChars, foreignToolCallsAsText, lastAssistant, recordedContent, withBudgetLine } from "./messages.js";
|
|
17
17
|
import { DEFAULT_PLANNER_PROMPT, DEFAULT_SYSTEM_PROMPT, renderSystemPrompt } from "./prompts.js";
|
|
18
18
|
import { loadProjectInstructions } from "../session/project-instructions.js";
|
|
19
19
|
import { projectScope } from "../session/project-root.js";
|
|
20
20
|
import { environmentBlock } from "./environment.js";
|
|
21
|
+
import { KIRO_P_PROVIDER, KIRO_P_REUSE_META } from "../providers/kiro-p.js";
|
|
21
22
|
import { createGatedStreamFn } from "./provider-gate.js";
|
|
22
23
|
import { Recorder } from "./recorder.js";
|
|
23
24
|
import { activePlaybook, emptyMemoryContext, playbookSection, retrieveRunMemories, skillAppendix } from "../learn/context.js";
|
|
@@ -25,6 +26,8 @@ import { continuationPrompt } from "../scoring/continuation.js";
|
|
|
25
26
|
import { planScoring, runScoringRound, runSelfCheck, summarizeSpans } from "../scoring/runner.js";
|
|
26
27
|
import { judgeFamilyWarnings, loadJudgeRubric, loadRouterPolicy } from "../scoring/sources.js";
|
|
27
28
|
const TOOL_RESULT_INLINE_LIMIT = 32_768;
|
|
29
|
+
/** Approver calls that may be started ahead of their tool call at once (one assistant message with many calls). */
|
|
30
|
+
const APPROVAL_PREFETCH_MAX = 6;
|
|
28
31
|
const TOOL_RESULT_EXCERPT_CHARS = 8_000;
|
|
29
32
|
/** The head and tail of a large tool result, with a note pointing at the saved full text. */
|
|
30
33
|
function toolOutputExcerpt(text, pointer) {
|
|
@@ -86,7 +89,7 @@ export function createEngineEnv(deps) {
|
|
|
86
89
|
sleep: deps.sleep ?? defaultSleep,
|
|
87
90
|
registry: createModelRegistry(deps.models),
|
|
88
91
|
policy: createPolicy(deps.config.tools),
|
|
89
|
-
streamFn: createGatedStreamFn(deps.models, limits, deps.resolveApiKey),
|
|
92
|
+
streamFn: createGatedStreamFn(deps.models, limits, deps.resolveApiKey, (provider) => provider === KIRO_P_PROVIDER ? { [KIRO_P_REUSE_META]: deps.config.session.reuseKiroSession } : undefined),
|
|
90
93
|
};
|
|
91
94
|
const recovered = recoverCrashedRuns(deps.db, isPidAlive, now().toISOString());
|
|
92
95
|
if (recovered.length > 0)
|
|
@@ -413,6 +416,10 @@ export async function executeRun(env, input) {
|
|
|
413
416
|
let runSpan = null;
|
|
414
417
|
let turnSpan = null;
|
|
415
418
|
let llmSpan = null;
|
|
419
|
+
/** When the open model call started, and how long its first output and first answer text took (null until seen). */
|
|
420
|
+
let llmStartedAt = 0;
|
|
421
|
+
let llmFirstChunkMs = null;
|
|
422
|
+
let llmFirstTextMs = null;
|
|
416
423
|
let agent = null;
|
|
417
424
|
let budgetExceeded = false;
|
|
418
425
|
let seededCount = 0;
|
|
@@ -422,7 +429,7 @@ export async function executeRun(env, input) {
|
|
|
422
429
|
kind: "run",
|
|
423
430
|
name: request.skill ? `skill:${request.skill}` : "run",
|
|
424
431
|
parentSpanId: null,
|
|
425
|
-
attrs: { instruction: deps.redact(request.instruction), parentSpanId: parent?.spanId ?? null, depth },
|
|
432
|
+
attrs: { instruction: deps.redact(request.instruction), parentSpanId: parent?.spanId ?? null, depth, ...(request.images?.length ? { images: saveImages(runDir, "inputs/image", request.images) } : {}) },
|
|
426
433
|
});
|
|
427
434
|
const rootSpan = runSpan;
|
|
428
435
|
runDir.writeRunJson(runJson("running", "", null));
|
|
@@ -740,6 +747,36 @@ export async function executeRun(env, input) {
|
|
|
740
747
|
}
|
|
741
748
|
const approvedBy = new Map();
|
|
742
749
|
let classify = null;
|
|
750
|
+
/** Tool calls whose `beforeToolCall` has begun, and approver verdicts started ahead of theirs, by tool call id. */
|
|
751
|
+
const startedCalls = new Set();
|
|
752
|
+
const earlyVerdicts = new Map();
|
|
753
|
+
/**
|
|
754
|
+
* pi authorizes the tool calls of one assistant message one after another, so a message with several calls would
|
|
755
|
+
* wait for one approver call per tool in sequence. While the first is being authorized, the approver is started
|
|
756
|
+
* for the rest that will need it; each call's own authorization then picks its verdict up. Policy, session rules
|
|
757
|
+
* and the loop guards still run per call, in order, and a verdict nobody collects is only a wasted approver call.
|
|
758
|
+
*/
|
|
759
|
+
const prefetchApprovals = (message, signal) => {
|
|
760
|
+
const classifier = classify;
|
|
761
|
+
if (!classifier || !approvals || approvals.allowAll)
|
|
762
|
+
return;
|
|
763
|
+
const rules = approvals.sessionRules();
|
|
764
|
+
for (const block of message.content) {
|
|
765
|
+
if (earlyVerdicts.size >= APPROVAL_PREFETCH_MAX)
|
|
766
|
+
return;
|
|
767
|
+
if (block.type !== "toolCall" || startedCalls.has(block.id) || earlyVerdicts.has(block.id))
|
|
768
|
+
continue;
|
|
769
|
+
const args = block.arguments;
|
|
770
|
+
if (env.policy.decide(block.name, args).action !== "ask")
|
|
771
|
+
continue;
|
|
772
|
+
if (rules.some((rule) => matchesSessionRule(rule, block.name, args)))
|
|
773
|
+
continue;
|
|
774
|
+
const command = block.name === "bash" && typeof block.arguments.command === "string" ? block.arguments.command : null;
|
|
775
|
+
if (command !== null && isReadOnlyCommand(command))
|
|
776
|
+
continue;
|
|
777
|
+
earlyVerdicts.set(block.id, { signature: `${block.name}:${stableJson(args)}`, verdict: classifier(block.name, args, block.id, signal) });
|
|
778
|
+
}
|
|
779
|
+
};
|
|
743
780
|
let lastTrimCleared = 0;
|
|
744
781
|
/** completedCalls when the tool-call cap was first passed; tool calls from a later model reply end the run. */
|
|
745
782
|
let loopCapWarnedAt = null;
|
|
@@ -776,9 +813,11 @@ export async function executeRun(env, input) {
|
|
|
776
813
|
lastTrimCleared = Math.max(lastTrimCleared, trimmed.cleared);
|
|
777
814
|
current = trimmed.messages;
|
|
778
815
|
}
|
|
816
|
+
current = limitImages(current).messages;
|
|
779
817
|
return withBudgetLine(foreignToolCallsAsText(current, active.model), budgetLine(budget));
|
|
780
818
|
},
|
|
781
|
-
beforeToolCall: async ({ toolCall, args }, signal) => {
|
|
819
|
+
beforeToolCall: async ({ toolCall, args, assistantMessage }, signal) => {
|
|
820
|
+
startedCalls.add(toolCall.id);
|
|
782
821
|
if (budget.isExhausted()) {
|
|
783
822
|
budgetExceeded = true;
|
|
784
823
|
return { block: true, reason: "BUDGET_EXCEEDED: the run budget is exhausted", terminate: true };
|
|
@@ -804,11 +843,21 @@ export async function executeRun(env, input) {
|
|
|
804
843
|
return { block: true, reason: `LOOP_REPEAT: ${toolCall.name} was called ${repeatCount} times in a row with identical arguments (loop.repeatLimit ${repeatLimit}). Change your approach or report the blocker.` };
|
|
805
844
|
}
|
|
806
845
|
const classifier = classify;
|
|
846
|
+
prefetchApprovals(assistantMessage, signal);
|
|
807
847
|
const auth = await env.policy.authorize(deps.prompter, toolCall.name, args, signal, depth > 0 ? `subagent ${runId.slice(-6)}` : undefined, approvals
|
|
808
848
|
? {
|
|
809
849
|
sessionRules: approvals.sessionRules,
|
|
810
850
|
...(approvals.addSessionRule ? { addSessionRule: approvals.addSessionRule } : {}),
|
|
811
|
-
...(classifier
|
|
851
|
+
...(classifier
|
|
852
|
+
? {
|
|
853
|
+
classify: (name, a, s) => {
|
|
854
|
+
const early = earlyVerdicts.get(toolCall.id);
|
|
855
|
+
earlyVerdicts.delete(toolCall.id);
|
|
856
|
+
// Only for the arguments it judged: validation may have changed them since.
|
|
857
|
+
return early && early.signature === `${name}:${stableJson(a)}` ? early.verdict : classifier(name, a, toolCall.id, s);
|
|
858
|
+
},
|
|
859
|
+
}
|
|
860
|
+
: {}),
|
|
812
861
|
allowAll: approvals.allowAll,
|
|
813
862
|
}
|
|
814
863
|
: undefined);
|
|
@@ -860,9 +909,9 @@ export async function executeRun(env, input) {
|
|
|
860
909
|
const pendingSteer = [];
|
|
861
910
|
let sawPrompt = false;
|
|
862
911
|
request.onSteeringReady?.({
|
|
863
|
-
steer: (text) => {
|
|
912
|
+
steer: (text, images) => {
|
|
864
913
|
pendingSteer.push(text);
|
|
865
|
-
runAgent.steer({ role: "user", content: [{ type: "text", text }], timestamp: Date.now() });
|
|
914
|
+
runAgent.steer({ role: "user", content: [{ type: "text", text }, ...(images ?? [])], timestamp: Date.now() });
|
|
866
915
|
},
|
|
867
916
|
clear: () => {
|
|
868
917
|
pendingSteer.length = 0;
|
|
@@ -889,6 +938,9 @@ export async function executeRun(env, input) {
|
|
|
889
938
|
if (event.message.role === "assistant") {
|
|
890
939
|
if (llmSpan)
|
|
891
940
|
recorder.finish(llmSpan, { status: "error", attrs: { reason: "superseded by a new assistant message" } });
|
|
941
|
+
llmStartedAt = now().getTime();
|
|
942
|
+
llmFirstChunkMs = null;
|
|
943
|
+
llmFirstTextMs = null;
|
|
892
944
|
llmSpan = recorder.start({
|
|
893
945
|
kind: "llm_call",
|
|
894
946
|
name: active.model.id,
|
|
@@ -901,11 +953,16 @@ export async function executeRun(env, input) {
|
|
|
901
953
|
});
|
|
902
954
|
}
|
|
903
955
|
break;
|
|
904
|
-
case "message_update":
|
|
956
|
+
case "message_update": {
|
|
957
|
+
const kind = event.assistantMessageEvent.type;
|
|
958
|
+
if (kind === "text_delta" || kind === "thinking_delta" || kind === "toolcall_delta")
|
|
959
|
+
llmFirstChunkMs ??= now().getTime() - llmStartedAt;
|
|
905
960
|
if (event.assistantMessageEvent.type === "text_delta") {
|
|
961
|
+
llmFirstTextMs ??= now().getTime() - llmStartedAt;
|
|
906
962
|
deps.onEvent?.({ type: "text_delta", runId, depth, delta: event.assistantMessageEvent.delta });
|
|
907
963
|
}
|
|
908
964
|
break;
|
|
965
|
+
}
|
|
909
966
|
case "message_end": {
|
|
910
967
|
if (event.message.role !== "assistant" || !llmSpan)
|
|
911
968
|
break;
|
|
@@ -929,6 +986,9 @@ export async function executeRun(env, input) {
|
|
|
929
986
|
stopReason: msg.stopReason,
|
|
930
987
|
errorMessage: msg.errorMessage,
|
|
931
988
|
responseModel: msg.responseModel,
|
|
989
|
+
// Time to the first streamed output (thinking included) and to the first answer text; absent when nothing streamed.
|
|
990
|
+
firstChunkMs: llmFirstChunkMs ?? undefined,
|
|
991
|
+
firstTextMs: llmFirstTextMs ?? undefined,
|
|
932
992
|
content: contentRef ? undefined : content,
|
|
933
993
|
},
|
|
934
994
|
});
|
|
@@ -961,7 +1021,12 @@ export async function executeRun(env, input) {
|
|
|
961
1021
|
recorder.finish(spanId, {
|
|
962
1022
|
status: event.isError ? "error" : "ok",
|
|
963
1023
|
payloadRef,
|
|
964
|
-
attrs: {
|
|
1024
|
+
attrs: {
|
|
1025
|
+
durationMs: event.durationMs,
|
|
1026
|
+
result: payloadRef ? undefined : output,
|
|
1027
|
+
approvedBy: approvedBy.get(event.toolCallId),
|
|
1028
|
+
...imagesAttr(runDir, `tool_outputs/${spanId}-image`, event.result?.content),
|
|
1029
|
+
},
|
|
965
1030
|
});
|
|
966
1031
|
}
|
|
967
1032
|
const failures = monitor.toolResult(event.isError);
|
|
@@ -988,11 +1053,12 @@ export async function executeRun(env, input) {
|
|
|
988
1053
|
}
|
|
989
1054
|
});
|
|
990
1055
|
/** Sends one prompt (the instruction, or a continuation), retries retryable provider errors, and sets status and output. */
|
|
991
|
-
const drive = async (text) => {
|
|
992
|
-
await runAgent.prompt(text);
|
|
1056
|
+
const drive = async (text, images) => {
|
|
1057
|
+
await runAgent.prompt(text, images && images.length > 0 ? images : undefined);
|
|
993
1058
|
// Retries are counted per streak of consecutive failures: a successful call since the last retry starts over.
|
|
994
1059
|
let attempt = 0;
|
|
995
1060
|
let waitedMs = 0;
|
|
1061
|
+
let formatFailures = 0;
|
|
996
1062
|
let succeededCalls = completedCalls;
|
|
997
1063
|
for (;;) {
|
|
998
1064
|
const last = lastAssistant(runAgent.state.messages);
|
|
@@ -1001,8 +1067,11 @@ export async function executeRun(env, input) {
|
|
|
1001
1067
|
if (completedCalls !== succeededCalls) {
|
|
1002
1068
|
attempt = 0;
|
|
1003
1069
|
waitedMs = 0;
|
|
1070
|
+
formatFailures = 0;
|
|
1004
1071
|
succeededCalls = completedCalls;
|
|
1005
1072
|
}
|
|
1073
|
+
if (isFormatFailure(last) && ++formatFailures > FORMAT_RETRIES)
|
|
1074
|
+
break;
|
|
1006
1075
|
attempt++;
|
|
1007
1076
|
const providerCfg = config.providers[active.model.provider] ?? {};
|
|
1008
1077
|
if (input.signal?.aborted || budgetExceeded)
|
|
@@ -1044,7 +1113,7 @@ export async function executeRun(env, input) {
|
|
|
1044
1113
|
status = "aborted";
|
|
1045
1114
|
}
|
|
1046
1115
|
else {
|
|
1047
|
-
await drive(request.instruction);
|
|
1116
|
+
await drive(request.instruction, request.images);
|
|
1048
1117
|
// Scoring (spec §7.1 step 5): each failing round may start one S2 continuation, up to router.maxEscalations.
|
|
1049
1118
|
const selfCheckOn = policy.triggers.self_check.enabled;
|
|
1050
1119
|
if (status === "succeeded" && (plan.specs.length > 0 || plan.specError !== null || selfCheckOn)) {
|
|
@@ -1213,3 +1282,25 @@ function modelInfo(model, resolved) {
|
|
|
1213
1282
|
function newMessages(messages, seededCount) {
|
|
1214
1283
|
return messages.slice(seededCount).filter((m) => m.role === "user" || m.role === "assistant" || m.role === "toolResult");
|
|
1215
1284
|
}
|
|
1285
|
+
const IMAGE_FILE_EXTENSIONS = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp" };
|
|
1286
|
+
/** Saves images into the run directory as `<prefix>-<n>.<ext>` and returns their relative paths; a failed write is skipped. */
|
|
1287
|
+
function saveImages(runDir, prefix, images) {
|
|
1288
|
+
return images.flatMap((image, i) => {
|
|
1289
|
+
try {
|
|
1290
|
+
return [runDir.writeImage(`${prefix}-${i + 1}.${IMAGE_FILE_EXTENSIONS[image.mimeType] ?? "png"}`, image.data)];
|
|
1291
|
+
}
|
|
1292
|
+
catch {
|
|
1293
|
+
return [];
|
|
1294
|
+
}
|
|
1295
|
+
});
|
|
1296
|
+
}
|
|
1297
|
+
/** `{ images: [paths] }` for a tool result that returned images (saved under `prefix`), else nothing. */
|
|
1298
|
+
function imagesAttr(runDir, prefix, content) {
|
|
1299
|
+
if (!Array.isArray(content))
|
|
1300
|
+
return {};
|
|
1301
|
+
const images = content.filter((b) => b?.type === "image" && typeof b.data === "string");
|
|
1302
|
+
if (images.length === 0)
|
|
1303
|
+
return {};
|
|
1304
|
+
const paths = saveImages(runDir, prefix, images);
|
|
1305
|
+
return paths.length > 0 ? { images: paths } : {};
|
|
1306
|
+
}
|