@hexdecimal16/nexo 0.4.12 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -28,6 +28,8 @@ nexo doctor # check Node, config, keys and t
28
28
 
29
29
  `nexo --help` lists every command. Data lives in `~/.nexo`. Set `NEXO_HOME` to move it.
30
30
 
31
+ Images (PNG, JPEG, GIF, WebP) work with every vision model, including the `claude-p`, `codex-p` and `kiro-p` CLI providers. In the chat, drag an image file into the terminal or press ctrl+v to paste a screenshot; it shows as `[Image #1]`. `@shot.png` attaches an image in the project, `nexo run "…" --image shot.png` attaches one to a task, and the agent's `read` tool opens image files. Large images are scaled down to 2000 px, older images in a long run are replaced by a note (the file can be read again), and `nexo ui` shows the images each run saw.
32
+
31
33
  ## Settings and keys
32
34
 
33
35
  Every option is in `/settings` inside a chat, or in `nexo settings` from the shell:
@@ -76,6 +78,7 @@ try {
76
78
  - `nexo.keys()`, `nexo.setKey()` and `nexo.removeKey()` manage stored keys. `nexo.webSearch(query)` searches without a model.
77
79
  - The SDK denies every tool approval by default, so a backend never waits on a person. Pass `allowAll: true` to `createNexo` or `run` to auto-approve tool actions in sandboxed environments. Always pass `budgetUsd`.
78
80
  - `checkSetup()` runs the doctor checks without starting the engine, for a start-up pre-flight.
81
+ - Send images with `run({ instruction, images: [loadImageFile("shot.png").image] })`, `session.turn(text, { images })` or `session.steer(text, images)`.
79
82
 
80
83
  ## Security
81
84
 
package/cli/src/common.js CHANGED
@@ -19,7 +19,7 @@ export const USAGE = `Usage:
19
19
  nexo settings remove-key <provider> delete a key from the credentials file
20
20
  nexo settings keys key status of the model and web search providers (never the keys)
21
21
  nexo search <query...> [--count <n>] [--json] search the web with the configured Brave/Tavily keys
22
- nexo run <instruction...> [--skill <name>] [--model <s1|s2|provider/id>] [--effort <level>] [--workspace <dir>] [--budget <usd>] [--no-review] [--allow-all]
22
+ nexo run <instruction...> [--skill <name>] [--model <s1|s2|provider/id>] [--effort <level>] [--workspace <dir>] [--budget <usd>] [--image <path>]... [--no-review] [--allow-all]
23
23
  nexo cost [--by model|tier|day|run|skill] [--since YYYY-MM-DD]
24
24
  nexo reindex
25
25
  nexo review-queue [--limit <n>] [--days <n>] recent runs without a verdict, worst first
package/cli/src/main.js CHANGED
@@ -6,7 +6,8 @@ import { EFFORTS } from "../../router/src/index.js";
6
6
  import { parseSkillInvocation } from "../../skills/src/index.js";
7
7
  import { costReport, ensureNexoDirs, openDb, reindex, resolveNexoPaths } from "../../store/src/index.js";
8
8
  import { NexoError, toErrorPayload } from "../../telemetry/src/index.js";
9
- import { nonInteractivePrompter } from "../../tools/src/index.js";
9
+ import { describeImage, loadImageFile, nonInteractivePrompter, withAttachedImagesNote } from "../../tools/src/index.js";
10
+ import { basename, resolve as resolvePath } from "node:path";
10
11
  import { EXIT_SIGINT, EXIT_SIGTERM, exitCodeFor, printError, USAGE, usageError } from "./common.js";
11
12
  import { evalCommand } from "./eval.js";
12
13
  import { learnCommand, learnEventLine, memoryCommand, playbookCommand } from "./learn.js";
@@ -97,11 +98,24 @@ async function runCommand(args, io, options) {
97
98
  budget: { type: "string" },
98
99
  "no-review": { type: "boolean" },
99
100
  "allow-all": { type: "boolean" },
101
+ image: { type: "string", multiple: true },
100
102
  },
101
103
  });
102
104
  const instruction = positionals.join(" ").trim();
103
105
  if (!instruction)
104
106
  throw usageError("nexo run needs an instruction");
107
+ const images = [];
108
+ const imageLabels = [];
109
+ for (const path of values.image ?? []) {
110
+ try {
111
+ const loaded = loadImageFile(resolvePath(options.cwd ?? process.cwd(), path));
112
+ images.push(loaded.image);
113
+ imageLabels.push(describeImage(basename(path), loaded));
114
+ }
115
+ catch (error) {
116
+ throw usageError(`--image ${path}: ${error instanceof Error ? error.message : String(error)}`);
117
+ }
118
+ }
105
119
  if (values.effort !== undefined && !EFFORTS.includes(values.effort)) {
106
120
  throw usageError(`--effort must be one of ${EFFORTS.join(", ")}`);
107
121
  }
@@ -144,7 +158,8 @@ async function runCommand(args, io, options) {
144
158
  let result;
145
159
  try {
146
160
  result = await runtime.engine.run({
147
- instruction: invoked?.instruction ?? instruction,
161
+ instruction: withAttachedImagesNote(invoked?.instruction ?? instruction, imageLabels),
162
+ ...(images.length > 0 ? { images } : {}),
148
163
  skill: values.skill ?? invoked?.skill,
149
164
  model: values.model,
150
165
  effort: values.effort,
@@ -79,6 +79,8 @@ export const ConfigSchema = Type.Object({
79
79
  compactAt: Type.Number({ exclusiveMinimum: 0, maximum: 1 }),
80
80
  /** Also compact when one S2 request with the history would cost more than this share of the turn's budget; 0 turns it off. */
81
81
  compactAtBudgetShare: Type.Number({ minimum: 0, maximum: 1 }),
82
+ /** kiro-p keeps a run's Kiro session open between model calls and sends only the new entries (see `SessionCache`). */
83
+ reuseKiroSession: Type.Boolean(),
82
84
  }, { additionalProperties: false }),
83
85
  /** Guards against a run that loops: a cap on tool calls and a limit on identical consecutive calls; 0 turns a guard off. */
84
86
  loop: Type.Object({ maxToolCalls: Type.Integer({ minimum: 0 }), repeatLimit: Type.Integer({ minimum: 0 }) }, { additionalProperties: false }),
@@ -181,7 +183,7 @@ export const DEFAULT_CONFIG = {
181
183
  escalationCutoff: 0.8,
182
184
  estimateOutputTokens: 4096,
183
185
  },
184
- session: { compactAt: 0.8, compactAtBudgetShare: 0.2 },
186
+ session: { compactAt: 0.8, compactAtBudgetShare: 0.2, reuseKiroSession: false },
185
187
  loop: { maxToolCalls: 200, repeatLimit: 3 },
186
188
  spawn: { maxParallelSubagents: 4, maxDepth: 4 },
187
189
  instructions: [],
@@ -24,3 +24,42 @@ export function clearOldToolResults(messages, keepRecent) {
24
24
  });
25
25
  return { messages: cleared > 0 ? out : messages, cleared };
26
26
  }
27
+ /** Images the model keeps seeing; older ones are replaced by a note (Anthropic many-image limits start above 20). */
28
+ export const KEEP_RECENT_IMAGES = 8;
29
+ /** Older images are dropped only once this many more have built up, so the cached prompt prefix changes rarely. */
30
+ const IMAGE_DROP_BATCH = 4;
31
+ export const DROPPED_IMAGE_NOTE = "[nexo] Earlier image removed to save context. Read the file again (or ask for it) to see it.";
32
+ /**
33
+ * Keeps the newest `keep` images in the conversation and replaces older ones with a short note; the text around them
34
+ * (paths, sizes, the tool calls) stays, so the agent can read an image again. Nothing changes until `keep + batch`
35
+ * images have built up, so a long screenshot-heavy run rewrites its history (and loses its prompt cache) only once
36
+ * every `batch` images.
37
+ */
38
+ export function limitImages(messages, keep = KEEP_RECENT_IMAGES, batch = IMAGE_DROP_BATCH) {
39
+ let total = 0;
40
+ for (const m of messages) {
41
+ const content = m.content;
42
+ if (Array.isArray(content))
43
+ for (const block of content)
44
+ if (block?.type === "image")
45
+ total++;
46
+ }
47
+ if (total <= keep + batch)
48
+ return { messages, dropped: 0 };
49
+ let toDrop = total - keep;
50
+ const dropped = toDrop;
51
+ const out = messages.map((m) => {
52
+ const content = m.content;
53
+ if (toDrop === 0 || !Array.isArray(content) || !content.some((b) => b?.type === "image"))
54
+ return m;
55
+ const next = content.map((block) => {
56
+ if (toDrop > 0 && block?.type === "image") {
57
+ toDrop--;
58
+ return { type: "text", text: DROPPED_IMAGE_NOTE };
59
+ }
60
+ return block;
61
+ });
62
+ return { ...m, content: next };
63
+ });
64
+ return { messages: out, dropped };
65
+ }
@@ -46,11 +46,19 @@ const MAX_RETRY_ATTEMPTS = 200;
46
46
  // Transient failures pi's classifier does not know: Kiro's capacity message and the kiro-p idle stall.
47
47
  const EXTRA_TRANSIENT = /high traffic|try again|no output for \d+\s*s/i;
48
48
  const NOT_TRANSIENT = /quota|billing|usage limit/i;
49
- /** Whether a failed model call is worth retrying: pi's transient patterns plus the kiro-cli ones above. */
49
+ // The model answered outside the reply format: it called a tool natively, or its JSON reply could not be used.
50
+ const FORMAT_FAILURE = /native tool call|reply was not a JSON object/i;
51
+ /** Retries a streak of format failures gets: the same request usually comes back right, but a model that keeps derailing must not loop. */
52
+ export const FORMAT_RETRIES = 2;
53
+ /** Whether a failed model call ended on a reply the provider could not use, as opposed to a transport or capacity error. */
54
+ export function isFormatFailure(message) {
55
+ return message.stopReason === "error" && FORMAT_FAILURE.test(message.errorMessage ?? "");
56
+ }
57
+ /** Whether a failed model call is worth retrying: pi's transient patterns, the kiro-cli ones above and format failures. */
50
58
  export function isRetryableProviderMessage(message) {
51
59
  if (message.stopReason !== "error" || !message.errorMessage)
52
60
  return false;
53
- if (isRetryableAssistantError(message))
61
+ if (isRetryableAssistantError(message) || isFormatFailure(message))
54
62
  return true;
55
63
  return EXTRA_TRANSIENT.test(message.errorMessage) && !NOT_TRANSIENT.test(message.errorMessage);
56
64
  }
@@ -81,6 +81,19 @@ export function foreignToolCallsAsText(messages, target) {
81
81
  });
82
82
  return changed ? out : messages;
83
83
  }
84
+ /**
85
+ * Characters an image is counted as: about 1,600 visual tokens (a downscaled screenshot on Claude's standard tier) at
86
+ * 4 characters per token. Counting its base64 data instead would make one screenshot look like a million tokens.
87
+ */
88
+ export const IMAGE_ESTIMATE_CHARS = 6400;
84
89
  export function estimatePromptChars(messages) {
85
- return JSON.stringify(messages).length;
90
+ let images = 0;
91
+ const json = JSON.stringify(messages, (_key, value) => {
92
+ if (value && typeof value === "object" && value.type === "image" && typeof value.data === "string") {
93
+ images++;
94
+ return { type: "image" };
95
+ }
96
+ return value;
97
+ });
98
+ return json.length + images * IMAGE_ESTIMATE_CHARS;
86
99
  }
@@ -1,9 +1,10 @@
1
1
  import { Semaphore } from "./semaphore.js";
2
2
  /**
3
3
  * Wraps Models.streamSimple so each provider never exceeds its configured number of concurrent requests. When the
4
- * request carries no `apiKey`, the key from `resolveApiKey` is passed as the request's `apiKey` option.
4
+ * request carries no `apiKey`, the key from `resolveApiKey` is passed as the request's `apiKey` option, and
5
+ * `metadataFor`'s entries are added under the request's own `metadata`.
5
6
  */
6
- export function createGatedStreamFn(models, limits, resolveApiKey) {
7
+ export function createGatedStreamFn(models, limits, resolveApiKey, metadataFor) {
7
8
  const semaphores = new Map();
8
9
  const semaphoreFor = (provider) => {
9
10
  const limit = limits[provider];
@@ -21,7 +22,8 @@ export function createGatedStreamFn(models, limits, resolveApiKey) {
21
22
  const release = sem ? await sem.acquire(options?.signal) : () => { };
22
23
  try {
23
24
  const apiKey = options?.apiKey ?? resolveApiKey?.(model.provider);
24
- const stream = models.streamSimple(model, context, apiKey === undefined ? options : { ...options, apiKey });
25
+ const metadata = metadataFor?.(model.provider);
26
+ const stream = models.streamSimple(model, context, apiKey === undefined && !metadata ? options : { ...options, ...(apiKey === undefined ? {} : { apiKey }), ...(metadata ? { metadata: { ...metadata, ...options?.metadata } } : {}) });
25
27
  stream.result().then(release, release);
26
28
  return stream;
27
29
  }
@@ -4,20 +4,21 @@ import { createModelRegistry, DEFAULT_ROUTER_POLICY, ESCALATION_BLOCK_REASONS, e
4
4
  import { createLoadSkillTool, formatSkillIndex, parseSkillFile } from "../../../skills/src/index.js";
5
5
  import { createArtifactSet, enqueueLearnJob, getActiveArtifact, insertRun, isPidAlive, listPlaybookItems, listSpans, newId, recoverCrashedRuns, resolveSourcedArtifact, RunDir, sessionRunDirs, updateRun, } from "../../../store/src/index.js";
6
6
  import { NexoError, toErrorPayload } from "../../../telemetry/src/index.js";
7
- import { builtinToolDescriptions, createAskUserTool, createBuiltinTools, createChangeTracker, createPolicy, createWebSearchTool, FileLeases, SEARCH_PROVIDER_IDS, TOOL_OUTPUT_PREFIX, } from "../../../tools/src/index.js";
7
+ import { builtinToolDescriptions, createAskUserTool, createBuiltinTools, createChangeTracker, createPolicy, createWebSearchTool, FileLeases, isReadOnlyCommand, matchesSessionRule, SEARCH_PROVIDER_IDS, TOOL_OUTPUT_PREFIX, } from "../../../tools/src/index.js";
8
8
  import { realpathSync, statSync } from "node:fs";
9
9
  import { join } from "node:path";
10
10
  import { createApprovalClassifier } from "../approval/classifier.js";
11
11
  import { budgetExhaustedError, checkAdmission, recordSpend, runBudgetLimit } from "../cost/admission.js";
12
12
  import { budgetLine, RunBudget } from "../cost/budget.js";
13
13
  import { estimateCallCostUsd, priceUsage } from "../cost/pricing.js";
14
- import { clearOldToolResults } from "./context-trim.js";
15
- import { classifyProviderError, contextOverflowError, isRetryableProviderMessage, providerRetryDelayMs, toolErrorText, withToolErrorCodes, withinRetryLimits } from "./errors.js";
14
+ import { clearOldToolResults, limitImages } from "./context-trim.js";
15
+ import { classifyProviderError, contextOverflowError, FORMAT_RETRIES, isFormatFailure, isRetryableProviderMessage, providerRetryDelayMs, toolErrorText, withToolErrorCodes, withinRetryLimits, } from "./errors.js";
16
16
  import { assistantText, contentText, estimatePromptChars, foreignToolCallsAsText, lastAssistant, recordedContent, withBudgetLine } from "./messages.js";
17
17
  import { DEFAULT_PLANNER_PROMPT, DEFAULT_SYSTEM_PROMPT, renderSystemPrompt } from "./prompts.js";
18
18
  import { loadProjectInstructions } from "../session/project-instructions.js";
19
19
  import { projectScope } from "../session/project-root.js";
20
20
  import { environmentBlock } from "./environment.js";
21
+ import { KIRO_P_PROVIDER, KIRO_P_REUSE_META } from "../providers/kiro-p.js";
21
22
  import { createGatedStreamFn } from "./provider-gate.js";
22
23
  import { Recorder } from "./recorder.js";
23
24
  import { activePlaybook, emptyMemoryContext, playbookSection, retrieveRunMemories, skillAppendix } from "../learn/context.js";
@@ -25,6 +26,8 @@ import { continuationPrompt } from "../scoring/continuation.js";
25
26
  import { planScoring, runScoringRound, runSelfCheck, summarizeSpans } from "../scoring/runner.js";
26
27
  import { judgeFamilyWarnings, loadJudgeRubric, loadRouterPolicy } from "../scoring/sources.js";
27
28
  const TOOL_RESULT_INLINE_LIMIT = 32_768;
29
+ /** Approver calls that may be started ahead of their tool call at once (one assistant message with many calls). */
30
+ const APPROVAL_PREFETCH_MAX = 6;
28
31
  const TOOL_RESULT_EXCERPT_CHARS = 8_000;
29
32
  /** The head and tail of a large tool result, with a note pointing at the saved full text. */
30
33
  function toolOutputExcerpt(text, pointer) {
@@ -86,7 +89,7 @@ export function createEngineEnv(deps) {
86
89
  sleep: deps.sleep ?? defaultSleep,
87
90
  registry: createModelRegistry(deps.models),
88
91
  policy: createPolicy(deps.config.tools),
89
- streamFn: createGatedStreamFn(deps.models, limits, deps.resolveApiKey),
92
+ streamFn: createGatedStreamFn(deps.models, limits, deps.resolveApiKey, (provider) => provider === KIRO_P_PROVIDER ? { [KIRO_P_REUSE_META]: deps.config.session.reuseKiroSession } : undefined),
90
93
  };
91
94
  const recovered = recoverCrashedRuns(deps.db, isPidAlive, now().toISOString());
92
95
  if (recovered.length > 0)
@@ -413,6 +416,10 @@ export async function executeRun(env, input) {
413
416
  let runSpan = null;
414
417
  let turnSpan = null;
415
418
  let llmSpan = null;
419
+ /** When the open model call started, and how long its first output and first answer text took (null until seen). */
420
+ let llmStartedAt = 0;
421
+ let llmFirstChunkMs = null;
422
+ let llmFirstTextMs = null;
416
423
  let agent = null;
417
424
  let budgetExceeded = false;
418
425
  let seededCount = 0;
@@ -422,7 +429,7 @@ export async function executeRun(env, input) {
422
429
  kind: "run",
423
430
  name: request.skill ? `skill:${request.skill}` : "run",
424
431
  parentSpanId: null,
425
- attrs: { instruction: deps.redact(request.instruction), parentSpanId: parent?.spanId ?? null, depth },
432
+ attrs: { instruction: deps.redact(request.instruction), parentSpanId: parent?.spanId ?? null, depth, ...(request.images?.length ? { images: saveImages(runDir, "inputs/image", request.images) } : {}) },
426
433
  });
427
434
  const rootSpan = runSpan;
428
435
  runDir.writeRunJson(runJson("running", "", null));
@@ -740,6 +747,36 @@ export async function executeRun(env, input) {
740
747
  }
741
748
  const approvedBy = new Map();
742
749
  let classify = null;
750
+ /** Tool calls whose `beforeToolCall` has begun, and approver verdicts started ahead of theirs, by tool call id. */
751
+ const startedCalls = new Set();
752
+ const earlyVerdicts = new Map();
753
+ /**
754
+ * pi authorizes the tool calls of one assistant message one after another, so a message with several calls would
755
+ * wait for one approver call per tool in sequence. While the first is being authorized, the approver is started
756
+ * for the rest that will need it; each call's own authorization then picks its verdict up. Policy, session rules
757
+ * and the loop guards still run per call, in order, and a verdict nobody collects is only a wasted approver call.
758
+ */
759
+ const prefetchApprovals = (message, signal) => {
760
+ const classifier = classify;
761
+ if (!classifier || !approvals || approvals.allowAll)
762
+ return;
763
+ const rules = approvals.sessionRules();
764
+ for (const block of message.content) {
765
+ if (earlyVerdicts.size >= APPROVAL_PREFETCH_MAX)
766
+ return;
767
+ if (block.type !== "toolCall" || startedCalls.has(block.id) || earlyVerdicts.has(block.id))
768
+ continue;
769
+ const args = block.arguments;
770
+ if (env.policy.decide(block.name, args).action !== "ask")
771
+ continue;
772
+ if (rules.some((rule) => matchesSessionRule(rule, block.name, args)))
773
+ continue;
774
+ const command = block.name === "bash" && typeof block.arguments.command === "string" ? block.arguments.command : null;
775
+ if (command !== null && isReadOnlyCommand(command))
776
+ continue;
777
+ earlyVerdicts.set(block.id, { signature: `${block.name}:${stableJson(args)}`, verdict: classifier(block.name, args, block.id, signal) });
778
+ }
779
+ };
743
780
  let lastTrimCleared = 0;
744
781
  /** completedCalls when the tool-call cap was first passed; tool calls from a later model reply end the run. */
745
782
  let loopCapWarnedAt = null;
@@ -776,9 +813,11 @@ export async function executeRun(env, input) {
776
813
  lastTrimCleared = Math.max(lastTrimCleared, trimmed.cleared);
777
814
  current = trimmed.messages;
778
815
  }
816
+ current = limitImages(current).messages;
779
817
  return withBudgetLine(foreignToolCallsAsText(current, active.model), budgetLine(budget));
780
818
  },
781
- beforeToolCall: async ({ toolCall, args }, signal) => {
819
+ beforeToolCall: async ({ toolCall, args, assistantMessage }, signal) => {
820
+ startedCalls.add(toolCall.id);
782
821
  if (budget.isExhausted()) {
783
822
  budgetExceeded = true;
784
823
  return { block: true, reason: "BUDGET_EXCEEDED: the run budget is exhausted", terminate: true };
@@ -804,11 +843,21 @@ export async function executeRun(env, input) {
804
843
  return { block: true, reason: `LOOP_REPEAT: ${toolCall.name} was called ${repeatCount} times in a row with identical arguments (loop.repeatLimit ${repeatLimit}). Change your approach or report the blocker.` };
805
844
  }
806
845
  const classifier = classify;
846
+ prefetchApprovals(assistantMessage, signal);
807
847
  const auth = await env.policy.authorize(deps.prompter, toolCall.name, args, signal, depth > 0 ? `subagent ${runId.slice(-6)}` : undefined, approvals
808
848
  ? {
809
849
  sessionRules: approvals.sessionRules,
810
850
  ...(approvals.addSessionRule ? { addSessionRule: approvals.addSessionRule } : {}),
811
- ...(classifier ? { classify: (name, a, s) => classifier(name, a, toolCall.id, s) } : {}),
851
+ ...(classifier
852
+ ? {
853
+ classify: (name, a, s) => {
854
+ const early = earlyVerdicts.get(toolCall.id);
855
+ earlyVerdicts.delete(toolCall.id);
856
+ // Only for the arguments it judged: validation may have changed them since.
857
+ return early && early.signature === `${name}:${stableJson(a)}` ? early.verdict : classifier(name, a, toolCall.id, s);
858
+ },
859
+ }
860
+ : {}),
812
861
  allowAll: approvals.allowAll,
813
862
  }
814
863
  : undefined);
@@ -860,9 +909,9 @@ export async function executeRun(env, input) {
860
909
  const pendingSteer = [];
861
910
  let sawPrompt = false;
862
911
  request.onSteeringReady?.({
863
- steer: (text) => {
912
+ steer: (text, images) => {
864
913
  pendingSteer.push(text);
865
- runAgent.steer({ role: "user", content: [{ type: "text", text }], timestamp: Date.now() });
914
+ runAgent.steer({ role: "user", content: [{ type: "text", text }, ...(images ?? [])], timestamp: Date.now() });
866
915
  },
867
916
  clear: () => {
868
917
  pendingSteer.length = 0;
@@ -889,6 +938,9 @@ export async function executeRun(env, input) {
889
938
  if (event.message.role === "assistant") {
890
939
  if (llmSpan)
891
940
  recorder.finish(llmSpan, { status: "error", attrs: { reason: "superseded by a new assistant message" } });
941
+ llmStartedAt = now().getTime();
942
+ llmFirstChunkMs = null;
943
+ llmFirstTextMs = null;
892
944
  llmSpan = recorder.start({
893
945
  kind: "llm_call",
894
946
  name: active.model.id,
@@ -901,11 +953,16 @@ export async function executeRun(env, input) {
901
953
  });
902
954
  }
903
955
  break;
904
- case "message_update":
956
+ case "message_update": {
957
+ const kind = event.assistantMessageEvent.type;
958
+ if (kind === "text_delta" || kind === "thinking_delta" || kind === "toolcall_delta")
959
+ llmFirstChunkMs ??= now().getTime() - llmStartedAt;
905
960
  if (event.assistantMessageEvent.type === "text_delta") {
961
+ llmFirstTextMs ??= now().getTime() - llmStartedAt;
906
962
  deps.onEvent?.({ type: "text_delta", runId, depth, delta: event.assistantMessageEvent.delta });
907
963
  }
908
964
  break;
965
+ }
909
966
  case "message_end": {
910
967
  if (event.message.role !== "assistant" || !llmSpan)
911
968
  break;
@@ -929,6 +986,9 @@ export async function executeRun(env, input) {
929
986
  stopReason: msg.stopReason,
930
987
  errorMessage: msg.errorMessage,
931
988
  responseModel: msg.responseModel,
989
+ // Time to the first streamed output (thinking included) and to the first answer text; absent when nothing streamed.
990
+ firstChunkMs: llmFirstChunkMs ?? undefined,
991
+ firstTextMs: llmFirstTextMs ?? undefined,
932
992
  content: contentRef ? undefined : content,
933
993
  },
934
994
  });
@@ -961,7 +1021,12 @@ export async function executeRun(env, input) {
961
1021
  recorder.finish(spanId, {
962
1022
  status: event.isError ? "error" : "ok",
963
1023
  payloadRef,
964
- attrs: { durationMs: event.durationMs, result: payloadRef ? undefined : output, approvedBy: approvedBy.get(event.toolCallId) },
1024
+ attrs: {
1025
+ durationMs: event.durationMs,
1026
+ result: payloadRef ? undefined : output,
1027
+ approvedBy: approvedBy.get(event.toolCallId),
1028
+ ...imagesAttr(runDir, `tool_outputs/${spanId}-image`, event.result?.content),
1029
+ },
965
1030
  });
966
1031
  }
967
1032
  const failures = monitor.toolResult(event.isError);
@@ -988,11 +1053,12 @@ export async function executeRun(env, input) {
988
1053
  }
989
1054
  });
990
1055
  /** Sends one prompt (the instruction, or a continuation), retries retryable provider errors, and sets status and output. */
991
- const drive = async (text) => {
992
- await runAgent.prompt(text);
1056
+ const drive = async (text, images) => {
1057
+ await runAgent.prompt(text, images && images.length > 0 ? images : undefined);
993
1058
  // Retries are counted per streak of consecutive failures: a successful call since the last retry starts over.
994
1059
  let attempt = 0;
995
1060
  let waitedMs = 0;
1061
+ let formatFailures = 0;
996
1062
  let succeededCalls = completedCalls;
997
1063
  for (;;) {
998
1064
  const last = lastAssistant(runAgent.state.messages);
@@ -1001,8 +1067,11 @@ export async function executeRun(env, input) {
1001
1067
  if (completedCalls !== succeededCalls) {
1002
1068
  attempt = 0;
1003
1069
  waitedMs = 0;
1070
+ formatFailures = 0;
1004
1071
  succeededCalls = completedCalls;
1005
1072
  }
1073
+ if (isFormatFailure(last) && ++formatFailures > FORMAT_RETRIES)
1074
+ break;
1006
1075
  attempt++;
1007
1076
  const providerCfg = config.providers[active.model.provider] ?? {};
1008
1077
  if (input.signal?.aborted || budgetExceeded)
@@ -1044,7 +1113,7 @@ export async function executeRun(env, input) {
1044
1113
  status = "aborted";
1045
1114
  }
1046
1115
  else {
1047
- await drive(request.instruction);
1116
+ await drive(request.instruction, request.images);
1048
1117
  // Scoring (spec §7.1 step 5): each failing round may start one S2 continuation, up to router.maxEscalations.
1049
1118
  const selfCheckOn = policy.triggers.self_check.enabled;
1050
1119
  if (status === "succeeded" && (plan.specs.length > 0 || plan.specError !== null || selfCheckOn)) {
@@ -1213,3 +1282,25 @@ function modelInfo(model, resolved) {
1213
1282
  function newMessages(messages, seededCount) {
1214
1283
  return messages.slice(seededCount).filter((m) => m.role === "user" || m.role === "assistant" || m.role === "toolResult");
1215
1284
  }
1285
+ const IMAGE_FILE_EXTENSIONS = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp" };
1286
+ /** Saves images into the run directory as `<prefix>-<n>.<ext>` and returns their relative paths; a failed write is skipped. */
1287
+ function saveImages(runDir, prefix, images) {
1288
+ return images.flatMap((image, i) => {
1289
+ try {
1290
+ return [runDir.writeImage(`${prefix}-${i + 1}.${IMAGE_FILE_EXTENSIONS[image.mimeType] ?? "png"}`, image.data)];
1291
+ }
1292
+ catch {
1293
+ return [];
1294
+ }
1295
+ });
1296
+ }
1297
+ /** `{ images: [paths] }` for a tool result that returned images (saved under `prefix`), else nothing. */
1298
+ function imagesAttr(runDir, prefix, content) {
1299
+ if (!Array.isArray(content))
1300
+ return {};
1301
+ const images = content.filter((b) => b?.type === "image" && typeof b.data === "string");
1302
+ if (images.length === 0)
1303
+ return {};
1304
+ const paths = saveImages(runDir, prefix, images);
1305
+ return paths.length > 0 ? { images: paths } : {};
1306
+ }