@bismawy/pi-vision-watcher 1.0.15 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/describer.ts CHANGED
@@ -1,561 +1,459 @@
1
- /**
2
- * The vision describer calls a vision-capable model through its registered
3
- * provider stream, falling back to pi-ai's `completeSimple()` for built-ins.
4
- *
5
- * `completeSimple` (not `complete`) is the path that translates the `reasoning`
6
- * ThinkingLevel into each provider's `reasoningEffort`/budget. `complete()` →
7
- * `stream()` reads only the pre-translated `reasoningEffort` field and silently
8
- * drops a bare `reasoning`, so the describer's thinking setting would be a no-op
9
- * through `complete()` — the bug this swap fixes. The agent loop, SDK, and
10
- * compaction all route thinking through `completeSimple`/`streamSimple` for the
11
- * same reason.
12
- *
13
- * Two entry points:
14
- * - {@link runBatch}: ONE batched call describing N images at once (the
15
- * dataloader's dispatch path). The model is asked to emit one delimited
16
- * `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
17
- * split back into per-image descriptions.
18
- * - {@link describeSingle}: one call for one image (no delimiters). The
19
- * robust per-image fallback used when a batched response couldn't be split.
20
- *
21
- * Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
22
- * managed with the `using` keyword via the {@link Disposable} guards in
23
- * `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
24
- * carried. Disposing is lexical and exception-safe: a thrown `completeSimple()`
25
- * still tears down the timer, uninstalls the interceptor, and detaches the
26
- * abort listener.
27
- */
28
-
29
- import type {
30
- Api,
31
- AssistantMessage,
32
- Context,
33
- ImageContent,
34
- Message,
35
- Model,
36
- SimpleStreamOptions,
37
- TextContent,
38
- ThinkingLevel,
39
- } from "@earendil-works/pi-ai";
40
- import { completeSimple } from "@earendil-works/pi-ai/compat";
41
- import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
42
- import {
43
- batchUserPrompt,
44
- DEFAULT_USER_PROMPT_PREFIX,
45
- DEFAULT_VISION_PROMPT,
46
- describeTimeoutMs,
47
- formatModelRef,
48
- markDescriptionTruncated,
49
- parseBatchedDescriptions,
50
- type ExtractedImage,
51
- type VisionHandoffConfig,
52
- } from "./index.js";
53
- import { appendVisionError } from "./error-log.js";
54
- import {
55
- buildUsageRecord,
56
- describeAls,
57
- EMPTY_ENERGY_CAPTURE,
58
- type DescribeContext,
59
- type VisionHandoffEnergyCapture,
60
- type VisionHandoffUsageRecord,
61
- } from "./usage.js";
62
- import { imageHash } from "./image.js";
63
- import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
64
-
65
- /** Tokens reserved for the describer's input so the requested output budget
66
- * can't exceed the model's context window. The describer's input is bounded —
67
- * one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
68
- * math simple while leaving generous room for the image + prompt tokens.
69
- * Providers reject a request when `inputTokens + maxTokens > contextWindow`
70
- * (e.g. a model whose declared maxTokens equals its full context window), so
71
- * this clamp prevents that 400 without meaningfully limiting a high-output
72
- * model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
73
- const INPUT_RESERVE_TOKENS = 8192;
74
-
75
- /** Resolve the `maxTokens` to pass to `completeSimple()` for a describer call.
76
- *
77
- * - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
78
- * fit the context window.
79
- * - Otherwise use the vision model's declared max output (`model.maxTokens`),
80
- * clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
81
- * requested output can't exceed the context window minus a reserve for the
82
- * input. This is the real fix for models whose declared `maxTokens` equals
83
- * their full `contextWindow` (which providers reject with any non-empty
84
- * input): we cap at `contextWindow − reserve` instead.
85
- * - If the model declares no usable max (`maxTokens <= 0`) and no cap is
86
- * configured, return `undefined` so the provider applies its own default. */
87
- export function resolveMaxTokens(
88
- cfg: VisionHandoffConfig,
89
- visionModel: Model<Api>,
90
- ): number | undefined {
91
- const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
92
- const cap = ctx - INPUT_RESERVE_TOKENS;
93
- const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
94
- if (requested === undefined) return undefined;
95
- // Clamp to the context-window-derived ceiling; never below 1.
96
- return Math.max(1, Math.min(requested, cap));
97
- }
98
-
99
- /** Resolve the `reasoning` (thinking) level to pass to `completeSimple()`, or
100
- * `undefined` to leave thinking off.
101
- *
102
- * - Returns `undefined` when thinking is disabled in config.
103
- * - Returns `undefined` when the vision model doesn't declare reasoning
104
- * support (`model.reasoning === false`) — sending a `reasoning` level to a
105
- * non-reasoning model would be ignored at best and rejected at worst.
106
- * - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
107
- *
108
- * Mirrors pi core's `createSummarizationOptions` guard so the describer's
109
- * thinking behaviour matches the rest of pi. */
110
- export function resolveReasoning(
111
- cfg: VisionHandoffConfig,
112
- visionModel: Model<Api>,
113
- ): ThinkingLevel | undefined {
114
- if (!cfg.thinking) return undefined;
115
- if (!visionModel.reasoning) return undefined;
116
- return cfg.thinkingLevel;
117
- }
118
-
119
- /** Small config snapshot embedded in every error-log entry — the fields most
120
- * relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
121
- * reasoning level the model rejects, etc.). */
122
- function configSnapshot(cfg: VisionHandoffConfig): {
123
- maxTokens?: number;
124
- thinking: boolean;
125
- thinkingLevel: string;
126
- } {
127
- return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
128
- }
129
-
130
- /** Complete through the provider registered in Pi's model registry when one
131
- * supplies a custom stream. The post-0.80 ModelRuntime keeps extension streams
132
- * out of pi-ai's deprecated global compatibility registry, so calling
133
- * completeSimple() directly cannot resolve custom API ids such as `makora`.
134
- * The optional access preserves compatibility with older ModelRegistry versions,
135
- * which registered custom streams globally and do not expose the config getter. */
136
- const NEURALWATT_CONVERSATION_HEADER = "x-nw-conversation-id";
137
-
138
- function isolateVisionRequest(
139
- model: Model<Api>,
140
- options: SimpleStreamOptions,
141
- ): SimpleStreamOptions {
142
- // Stable per-model identity (not randomUUID): every describer call shares the
143
- // same system prompt (DEFAULT_VISION_PROMPT), so a stable session id lets the
144
- // provider's prompt cache reuse that prefix across calls (cacheRead/cacheWrite
145
- // > 0) instead of re-billing it every turn. Still distinct from the main
146
- // agent's conversation id (the inherited header we replace), so unrelated
147
- // image prompts never become the main agent's newest cache lineage. A
148
- // per-model id keeps different vision models (fallback chains) on separate
149
- // lineages.
150
- const sessionId = `pi-vision-watcher:${model.provider}:${model.id}`;
151
- const headers = { ...(options.headers ?? {}) };
152
- const inheritedConversationHeader = Object.keys(headers).find(
153
- (name) => name.toLowerCase() === NEURALWATT_CONVERSATION_HEADER,
154
- );
155
-
156
- if (inheritedConversationHeader) {
157
- headers[inheritedConversationHeader] = sessionId;
158
- } else if (model.provider.toLowerCase() === "neuralwatt") {
159
- headers["X-NW-Conversation-ID"] = sessionId;
160
- }
161
-
162
- return {
163
- ...options,
164
- sessionId,
165
- ...(Object.keys(headers).length > 0 ? { headers } : {}),
166
- };
167
- }
168
-
169
- export async function completeVisionModel(
170
- model: Model<Api>,
171
- modelRegistry: ModelRegistry,
172
- context: Context,
173
- options: SimpleStreamOptions,
174
- ): Promise<AssistantMessage> {
175
- const isolatedOptions = isolateVisionRequest(model, options);
176
- const provider = modelRegistry.getRegisteredProviderConfig?.(model.provider);
177
- if (provider?.streamSimple && provider.api === model.api) {
178
- return provider.streamSimple(model, context, isolatedOptions).result();
179
- }
180
- return completeSimple(model, context, isolatedOptions);
181
- }
182
-
183
- /** Dependencies the describer can't own itself (held by the engine). */
184
- export interface DescriberDeps {
185
- /** Report a usage+energy record for one real describer call (cache hits emit none). */
186
- reportUsage(record: VisionHandoffUsageRecord): void;
187
- /** Set the most-recent describer failure message (surfaced to the user by the engine).
188
- * Pass `null` to clear before a fresh attempt. */
189
- setLastError(msg: string | null): void;
190
- /** Record the model ref ACTUALLY attempted for the latest describer call.
191
- * Differs from `config.visionModel` when a failover fallback ran — the
192
- * warning must name the model that failed, not the configured primary.
193
- * Optional so older callers/tests compile unchanged. */
194
- setAttemptedModel?(ref: string): void;
195
- }
196
-
197
- /** The result of a batched describer call: per-image raw descriptions keyed by hash. */
198
- export type BatchResult = Map<string, string>;
199
-
200
- /** Describe N images with ONE batched `completeSimple()` call and split the response
201
- * back into per-image descriptions. Returns a map keyed by image hash; an
202
- * image whose section failed to parse is omitted (the caller treats omission
203
- * as "description unavailable"). On a genuine call failure (auth, abort,
204
- * empty) the map is empty and `deps.setLastError` records the reason.
205
- *
206
- * A failed delimiter-parse is a failed BATCH, not a failed description: the
207
- * unparsed images fall back to parallel single-image calls (no delimiters to
208
- * cooperate with), so the common cooperative case stays one call while an
209
- * uncooperative model still gets every image described. */
210
- export async function runBatch(
211
- misses: { img: ExtractedImage; hash: string }[],
212
- userPrompt: string,
213
- visionModel: Model<Api>,
214
- modelRegistry: ModelRegistry,
215
- cfg: VisionHandoffConfig,
216
- deps: DescriberDeps,
217
- turnSignal?: AbortSignal,
218
- ): Promise<BatchResult> {
219
- const out: BatchResult = new Map();
220
- deps.setAttemptedModel?.(formatModelRef(visionModel.provider, visionModel.id));
221
- const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
222
- if (!auth.ok || !auth.apiKey) {
223
- const reason = !auth.ok
224
- ? auth.error
225
- : `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
226
- deps.setLastError(reason);
227
- appendVisionError({
228
- phase: "batch",
229
- reason,
230
- visionModel: cfg.visionModel,
231
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
232
- imageHashes: misses.map((m) => m.hash),
233
- imageCount: misses.length,
234
- config: configSnapshot(cfg),
235
- });
236
- return out;
237
- }
238
-
239
- const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
240
- const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
241
- const content: (TextContent | ImageContent)[] = [
242
- { type: "text", text: batchUserPrompt(misses.length, userPrompt, prefix) },
243
- ...misses.map((m) => ({ type: "image", data: m.img.data, mimeType: m.img.mimeType } satisfies ImageContent)),
244
- ];
245
- const userMessage: Message = { role: "user", content, timestamp: Date.now() };
246
-
247
- const timeoutMs = describeTimeoutMs(misses.length, cfg.describeTimeoutMs);
248
- const maxTokens = resolveMaxTokens(cfg, visionModel);
249
- const reasoning = resolveReasoning(cfg, visionModel);
250
- const controller = new AbortController();
251
- let timedOut = false;
252
- using fetchGuard = fetchInterceptorGuard();
253
- using timer = timeoutGuard(timeoutMs, () => {
254
- timedOut = true;
255
- controller.abort();
256
- });
257
- using abortWire = abortWireGuard(turnSignal, controller);
258
-
259
- const describeCtx: DescribeContext = { energyReader: undefined };
260
- try {
261
- const response = await describeAls.run(describeCtx, async () =>
262
- completeVisionModel(
263
- visionModel,
264
- modelRegistry,
265
- { systemPrompt, messages: [userMessage] },
266
- { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
267
- ),
268
- );
269
- const capture = await readCapture(describeCtx);
270
- const hashes = misses.map((m) => m.hash);
271
- const record = buildUsageRecord(response, capture, visionModel, hashes[0], hashes.length > 1 ? hashes : undefined);
272
- if (record) deps.reportUsage(record);
273
- if (response.stopReason === "aborted" || response.stopReason === "error") {
274
- const reason = setStopReasonError(
275
- deps,
276
- response.stopReason,
277
- response.errorMessage,
278
- abortWire,
279
- timedOut,
280
- timeoutMs,
281
- );
282
- if (reason) {
283
- appendVisionError({
284
- phase: "batch",
285
- reason,
286
- visionModel: cfg.visionModel,
287
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
288
- imageHashes: hashes,
289
- imageCount: misses.length,
290
- stopReason: response.stopReason,
291
- timedOut,
292
- timeoutMs,
293
- errorMessage: response.errorMessage,
294
- config: configSnapshot(cfg),
295
- });
296
- }
297
- return out;
298
- }
299
- const text = response.content
300
- .filter((c): c is TextContent => c.type === "text")
301
- .map((c) => c.text)
302
- .join("\n")
303
- .trim();
304
- if (!text) {
305
- deps.setLastError("vision model returned an empty description");
306
- appendVisionError({
307
- phase: "batch",
308
- reason: "vision model returned an empty description",
309
- visionModel: cfg.visionModel,
310
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
311
- imageHashes: hashes,
312
- imageCount: misses.length,
313
- config: configSnapshot(cfg),
314
- });
315
- return out;
316
- }
317
- const parsed = parseBatchedDescriptions(text, misses.length);
318
- for (let i = 0; i < misses.length; i++) {
319
- const d = parsed[i];
320
- if (d) out.set(misses[i].hash, d);
321
- }
322
- // stopReason "length" = the batch response was cut off mid-stream. The last
323
- // image being emitted when the cap hit is the one whose section is truncated
324
- // (parseBatchedDescriptions still captures it via the end-of-text lookahead,
325
- // just with partial content). Mark that one — the highest-index parsed slot —
326
- // so the agent/user know it's incomplete. Earlier sections had `<<<END>>>`
327
- // markers and are complete. We deliberately do NOT re-describe the truncated
328
- // image: a single call for the same complex image would likely hit the same
329
- // limit, so the partial text + marker is the honest, no-wasted-call result.
330
- if (response.stopReason === "length") {
331
- for (let i = misses.length - 1; i >= 0; i--) {
332
- const h = misses[i].hash;
333
- if (out.has(h)) {
334
- out.set(h, markDescriptionTruncated(out.get(h)!));
335
- break;
336
- }
337
- }
338
- }
339
- // Fallback: a failed delimiter-parse is NOT a failed description — it's a
340
- // failed batching. Describe each unparsed image with its own single-image
341
- // call (no delimiters to cooperate with). The calls run in parallel so
342
- // their results still arrive together, not sequentially.
343
- const unparsed = misses.filter((m) => !out.has(m.hash));
344
- if (unparsed.length > 0) {
345
- const fallbacks = await Promise.all(
346
- unparsed.map((m) => describeSingle(m.img, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal)),
347
- );
348
- for (let i = 0; i < unparsed.length; i++) {
349
- if (fallbacks[i]) out.set(unparsed[i].hash, fallbacks[i]!);
350
- }
351
- }
352
- return out;
353
- } catch (err) {
354
- const userAborted = abortWire.userAborted();
355
- const reason = timedOut
356
- ? `describer timed out after ${timeoutMs / 1000}s`
357
- : err instanceof Error
358
- ? err.message
359
- : String(err);
360
- deps.setLastError(reason);
361
- // A deliberate user cancel isn't a troubleshooting-worthy error — skip
362
- // logging it (mirroring the no-warn-on-user-abort contract).
363
- if (!userAborted) {
364
- appendVisionError({
365
- phase: "batch",
366
- reason,
367
- visionModel: cfg.visionModel,
368
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
369
- imageHashes: misses.map((m) => m.hash),
370
- imageCount: misses.length,
371
- timedOut,
372
- timeoutMs,
373
- errorStack: err instanceof Error ? err.stack : undefined,
374
- config: configSnapshot(cfg),
375
- });
376
- }
377
- return out;
378
- } finally {
379
- // The `using` guards above already released the fetch interceptor, timer,
380
- // and abort wire. Only the energy tee needs an explicit unhandled-rejection
381
- // swallow: if the main stream aborted, the tee rejects too.
382
- describeCtx.energyReader?.catch(() => {});
383
- }
384
- }
385
-
386
- /** Describe a single image with one `completeSimple()` call and return the RAW
387
- * description (no envelope, no truncation). Returns null on any genuine
388
- * failure (auth, abort/error, empty) so the caller only caches `UNAVAILABLE`
389
- * when a real describer attempt failed. */
390
- export async function describeSingle(
391
- img: ExtractedImage,
392
- userPrompt: string,
393
- visionModel: Model<Api>,
394
- modelRegistry: ModelRegistry,
395
- cfg: VisionHandoffConfig,
396
- deps: DescriberDeps,
397
- turnSignal?: AbortSignal,
398
- ): Promise<string | null> {
399
- deps.setAttemptedModel?.(formatModelRef(visionModel.provider, visionModel.id));
400
- const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
401
- if (!auth.ok || !auth.apiKey) {
402
- const reason = !auth.ok
403
- ? auth.error
404
- : `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
405
- deps.setLastError(reason);
406
- appendVisionError({
407
- phase: "single",
408
- reason,
409
- visionModel: cfg.visionModel,
410
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
411
- imageHashes: [imageHash(img.mimeType, img.data)],
412
- imageCount: 1,
413
- config: configSnapshot(cfg),
414
- });
415
- return null;
416
- }
417
- const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
418
- const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
419
- const content: (TextContent | ImageContent)[] = [
420
- { type: "text", text: batchUserPrompt(1, userPrompt, prefix) },
421
- { type: "image", data: img.data, mimeType: img.mimeType } satisfies ImageContent,
422
- ];
423
- const userMessage: Message = { role: "user", content, timestamp: Date.now() };
424
- const timeoutMs = describeTimeoutMs(1, cfg.describeTimeoutMs);
425
- const maxTokens = resolveMaxTokens(cfg, visionModel);
426
- const reasoning = resolveReasoning(cfg, visionModel);
427
- const controller = new AbortController();
428
- let timedOut = false;
429
- using fetchGuard = fetchInterceptorGuard();
430
- using timer = timeoutGuard(timeoutMs, () => {
431
- timedOut = true;
432
- controller.abort();
433
- });
434
- using abortWire = abortWireGuard(turnSignal, controller);
435
-
436
- const describeCtx: DescribeContext = { energyReader: undefined };
437
- try {
438
- const response = await describeAls.run(describeCtx, async () =>
439
- completeVisionModel(
440
- visionModel,
441
- modelRegistry,
442
- { systemPrompt, messages: [userMessage] },
443
- { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
444
- ),
445
- );
446
- const capture = await readCapture(describeCtx);
447
- const hash = imageHash(img.mimeType, img.data);
448
- const record = buildUsageRecord(response, capture, visionModel, hash);
449
- if (record) deps.reportUsage(record);
450
- if (response.stopReason === "aborted" || response.stopReason === "error") {
451
- const reason = setStopReasonError(
452
- deps,
453
- response.stopReason,
454
- response.errorMessage,
455
- abortWire,
456
- timedOut,
457
- timeoutMs,
458
- );
459
- if (reason) {
460
- appendVisionError({
461
- phase: "single",
462
- reason,
463
- visionModel: cfg.visionModel,
464
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
465
- imageHashes: [hash],
466
- imageCount: 1,
467
- stopReason: response.stopReason,
468
- timedOut,
469
- timeoutMs,
470
- errorMessage: response.errorMessage,
471
- config: configSnapshot(cfg),
472
- });
473
- }
474
- return null;
475
- }
476
- const text = response.content
477
- .filter((c): c is TextContent => c.type === "text")
478
- .map((c) => c.text)
479
- .join("\n")
480
- .trim();
481
- if (!text) {
482
- deps.setLastError("vision model returned an empty description");
483
- appendVisionError({
484
- phase: "single",
485
- reason: "vision model returned an empty description",
486
- visionModel: cfg.visionModel,
487
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
488
- imageHashes: [hash],
489
- imageCount: 1,
490
- config: configSnapshot(cfg),
491
- });
492
- return null;
493
- }
494
- // stopReason "length" = the model hit a token limit (configured maxTokens or
495
- // the provider's hard output cap) before finishing. The partial text is still
496
- // useful, but it must not pass as complete — mark it so the agent/user know.
497
- return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
498
- } catch (err) {
499
- const userAborted = abortWire.userAborted();
500
- const reason = timedOut
501
- ? `describer timed out after ${timeoutMs / 1000}s`
502
- : err instanceof Error
503
- ? err.message
504
- : String(err);
505
- deps.setLastError(reason);
506
- // A deliberate user cancel isn't a troubleshooting-worthy error — skip
507
- // logging it (mirroring the no-warn-on-user-abort contract).
508
- if (!userAborted) {
509
- appendVisionError({
510
- phase: "single",
511
- reason,
512
- visionModel: cfg.visionModel,
513
- attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
514
- imageHashes: [imageHash(img.mimeType, img.data)],
515
- imageCount: 1,
516
- timedOut,
517
- timeoutMs,
518
- errorStack: err instanceof Error ? err.stack : undefined,
519
- config: configSnapshot(cfg),
520
- });
521
- }
522
- return null;
523
- } finally {
524
- describeCtx.energyReader?.catch(() => {});
525
- }
526
- }
527
-
528
- /** Read the energy tee for a describer call, if one was captured. Returns the
529
- * empty capture when there is no reader (non-Neuralwatt models) or the tee
530
- * aborted with the main stream. */
531
- async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
532
- if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
533
- try {
534
- return await describeCtx.energyReader;
535
- } catch {
536
- return EMPTY_ENERGY_CAPTURE;
537
- }
538
- }
539
-
540
- /** Translate a non-OK stopReason into a user-facing failure message, unless the
541
- * abort came from the user cancelling the turn (then stay silent — no warning
542
- * for a deliberate cancel). Returns the message that was set (so the caller
543
- * can log it), or null when the abort was a user cancel / nothing was set. */
544
- function setStopReasonError(
545
- deps: DescriberDeps,
546
- stopReason: string,
547
- errorMessage: string | undefined,
548
- abortWire: AbortWire,
549
- timedOut: boolean,
550
- timeoutMs: number,
551
- ): string | null {
552
- if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
553
- if (timedOut) {
554
- const reason = `describer timed out after ${timeoutMs / 1000}s`;
555
- deps.setLastError(reason);
556
- return reason;
557
- }
558
- const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
559
- deps.setLastError(reason);
560
- return reason;
561
- }
1
+ /**
2
+ * The vision describer calls a vision-capable model through its registered
3
+ * provider stream, falling back to pi-ai's `completeSimple()` for built-ins.
4
+ *
5
+ * `completeSimple` (not `complete`) is the path that translates the `reasoning`
6
+ * ThinkingLevel into each provider's `reasoningEffort`/budget. `complete()` →
7
+ * `stream()` reads only the pre-translated `reasoningEffort` field and silently
8
+ * drops a bare `reasoning`, so the describer's thinking setting would be a no-op
9
+ * through `complete()` — the bug this swap fixes. The agent loop, SDK, and
10
+ * compaction all route thinking through `completeSimple`/`streamSimple` for the
11
+ * same reason.
12
+ *
13
+ * Two entry points:
14
+ * - {@link runBatch}: ONE batched call describing N images at once (the
15
+ * dataloader's dispatch path). The model is asked to emit one delimited
16
+ * `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
17
+ * split back into per-image descriptions.
18
+ * - {@link describeSingle}: one call for one image (no delimiters). The
19
+ * robust per-image fallback used when a batched response couldn't be split.
20
+ *
21
+ * Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
22
+ * managed with the `using` keyword via the {@link Disposable} guards in
23
+ * `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
24
+ * carried. Disposing is lexical and exception-safe: a thrown `completeSimple()`
25
+ * still tears down the timer, uninstalls the interceptor, and detaches the
26
+ * abort listener.
27
+ */
28
+
29
+ import type {
30
+ Api,
31
+ AssistantMessage,
32
+ Context,
33
+ ImageContent,
34
+ Message,
35
+ Model,
36
+ SimpleStreamOptions,
37
+ TextContent,
38
+ ThinkingLevel,
39
+ } from "@earendil-works/pi-ai";
40
+ import { completeSimple } from "@earendil-works/pi-ai/compat";
41
+ import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
42
+ import {
43
+ batchUserPrompt,
44
+ DEFAULT_USER_PROMPT_PREFIX,
45
+ DEFAULT_VISION_PROMPT,
46
+ describeTimeoutMs,
47
+ formatModelRef,
48
+ markDescriptionTruncated,
49
+ parseBatchedDescriptions,
50
+ type ExtractedImage,
51
+ type VisionHandoffConfig,
52
+ } from "./index.js";
53
+ import { appendVisionError } from "./error-log.js";
54
+ import {
55
+ buildUsageRecord,
56
+ describeAls,
57
+ EMPTY_ENERGY_CAPTURE,
58
+ type DescribeContext,
59
+ type VisionHandoffEnergyCapture,
60
+ type VisionHandoffUsageRecord,
61
+ } from "./usage.js";
62
+ import { imageHash } from "./image.js";
63
+ import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
64
+
65
+ /** Tokens reserved for the describer's input so the requested output budget
66
+ * can't exceed the model's context window. The describer's input is bounded —
67
+ * one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
68
+ * math simple while leaving generous room for the image + prompt tokens.
69
+ * Providers reject a request when `inputTokens + maxTokens > contextWindow`
70
+ * (e.g. a model whose declared maxTokens equals its full context window), so
71
+ * this clamp prevents that 400 without meaningfully limiting a high-output
72
+ * model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
73
+ const INPUT_RESERVE_TOKENS = 8192;
74
+
75
+ /** Resolve the `maxTokens` to pass to `completeSimple()` for a describer call.
76
+ *
77
+ * - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
78
+ * fit the context window.
79
+ * - Otherwise use the vision model's declared max output (`model.maxTokens`),
80
+ * clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
81
+ * requested output can't exceed the context window minus a reserve for the
82
+ * input. This is the real fix for models whose declared `maxTokens` equals
83
+ * their full `contextWindow` (which providers reject with any non-empty
84
+ * input): we cap at `contextWindow − reserve` instead.
85
+ * - If the model declares no usable max (`maxTokens <= 0`) and no cap is
86
+ * configured, return `undefined` so the provider applies its own default. */
87
+ export function resolveMaxTokens(
88
+ cfg: VisionHandoffConfig,
89
+ visionModel: Model<Api>,
90
+ ): number | undefined {
91
+ const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
92
+ const cap = ctx - INPUT_RESERVE_TOKENS;
93
+ const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
94
+ if (requested === undefined) return undefined;
95
+ // Clamp to the context-window-derived ceiling; never below 1.
96
+ return Math.max(1, Math.min(requested, cap));
97
+ }
98
+
99
+ /** Resolve the `reasoning` (thinking) level to pass to `completeSimple()`, or
100
+ * `undefined` to leave thinking off.
101
+ *
102
+ * - Returns `undefined` when thinking is disabled in config.
103
+ * - Returns `undefined` when the vision model doesn't declare reasoning
104
+ * support (`model.reasoning === false`) — sending a `reasoning` level to a
105
+ * non-reasoning model would be ignored at best and rejected at worst.
106
+ * - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
107
+ *
108
+ * Mirrors pi core's `createSummarizationOptions` guard so the describer's
109
+ * thinking behaviour matches the rest of pi. */
110
+ export function resolveReasoning(
111
+ cfg: VisionHandoffConfig,
112
+ visionModel: Model<Api>,
113
+ ): ThinkingLevel | undefined {
114
+ if (!cfg.thinking) return undefined;
115
+ if (!visionModel.reasoning) return undefined;
116
+ return cfg.thinkingLevel;
117
+ }
118
+
119
+ /** Small config snapshot embedded in every error-log entry — the fields most
120
+ * relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
121
+ * reasoning level the model rejects, etc.). */
122
+ function configSnapshot(cfg: VisionHandoffConfig): {
123
+ maxTokens?: number;
124
+ thinking: boolean;
125
+ thinkingLevel: string;
126
+ } {
127
+ return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
128
+ }
129
+
130
+ /** Complete through the provider registered in Pi's model registry when one
131
+ * supplies a custom stream. The post-0.80 ModelRuntime keeps extension streams
132
+ * out of pi-ai's deprecated global compatibility registry, so calling
133
+ * completeSimple() directly cannot resolve custom API ids such as `makora`.
134
+ * The optional access preserves compatibility with older ModelRegistry versions,
135
+ * which registered custom streams globally and do not expose the config getter. */
136
+ const NEURALWATT_CONVERSATION_HEADER = "x-nw-conversation-id";
137
+
138
+ function isolateVisionRequest(
139
+ model: Model<Api>,
140
+ options: SimpleStreamOptions,
141
+ ): SimpleStreamOptions {
142
+ // Stable per-model identity (not randomUUID): every describer call shares the
143
+ // same system prompt (DEFAULT_VISION_PROMPT), so a stable session id lets the
144
+ // provider's prompt cache reuse that prefix across calls (cacheRead/cacheWrite
145
+ // > 0) instead of re-billing it every turn. Still distinct from the main
146
+ // agent's conversation id (the inherited header we replace), so unrelated
147
+ // image prompts never become the main agent's newest cache lineage. A
148
+ // per-model id keeps different vision models (fallback chains) on separate
149
+ // lineages.
150
+ const sessionId = `pi-vision-watcher:${model.provider}:${model.id}`;
151
+ const headers = { ...(options.headers ?? {}) };
152
+ const inheritedConversationHeader = Object.keys(headers).find(
153
+ (name) => name.toLowerCase() === NEURALWATT_CONVERSATION_HEADER,
154
+ );
155
+
156
+ if (inheritedConversationHeader) {
157
+ headers[inheritedConversationHeader] = sessionId;
158
+ } else if (model.provider.toLowerCase() === "neuralwatt") {
159
+ headers["X-NW-Conversation-ID"] = sessionId;
160
+ }
161
+
162
+ return {
163
+ ...options,
164
+ sessionId,
165
+ ...(Object.keys(headers).length > 0 ? { headers } : {}),
166
+ };
167
+ }
168
+
169
+ export async function completeVisionModel(
170
+ model: Model<Api>,
171
+ modelRegistry: ModelRegistry,
172
+ context: Context,
173
+ options: SimpleStreamOptions,
174
+ ): Promise<AssistantMessage> {
175
+ const isolatedOptions = isolateVisionRequest(model, options);
176
+ const provider = modelRegistry.getRegisteredProviderConfig?.(model.provider);
177
+ if (provider?.streamSimple && provider.api === model.api) {
178
+ return provider.streamSimple(model, context, isolatedOptions).result();
179
+ }
180
+ return completeSimple(model, context, isolatedOptions);
181
+ }
182
+
183
+ /** Dependencies the describer can't own itself (held by the engine). */
184
+ export interface DescriberDeps {
185
+ /** Report a usage+energy record for one real describer call (cache hits emit none). */
186
+ reportUsage(record: VisionHandoffUsageRecord): void;
187
+ /** Set the most-recent describer failure message (surfaced to the user by the engine).
188
+ * Pass `null` to clear before a fresh attempt. */
189
+ setLastError(msg: string | null): void;
190
+ /** Record the model ref ACTUALLY attempted for the latest describer call.
191
+ * Differs from `config.visionModel` when a failover fallback ran — the
192
+ * warning must name the model that failed, not the configured primary.
193
+ * Optional so older callers/tests compile unchanged. */
194
+ setAttemptedModel?(ref: string): void;
195
+ }
196
+
197
+ /** The result of a batched describer call: per-image raw descriptions keyed by hash. */
198
+ export type BatchResult = Map<string, string>;
199
+
200
+ /** Describe N images with ONE batched `completeSimple()` call and split the response
201
+ * back into per-image descriptions. Returns a map keyed by image hash; an
202
+ * image whose section failed to parse is omitted (the caller treats omission
203
+ * as "description unavailable"). On a genuine call failure (auth, abort,
204
+ * empty) the map is empty and `deps.setLastError` records the reason.
205
+ *
206
+ * A failed delimiter-parse is a failed BATCH, not a failed description: the
207
+ * unparsed images fall back to parallel single-image calls (no delimiters to
208
+ * cooperate with), so the common cooperative case stays one call while an
209
+ * uncooperative model still gets every image described. */
210
+ export async function runBatch(
211
+ misses: { img: ExtractedImage; hash: string }[],
212
+ userPrompt: string,
213
+ visionModel: Model<Api>,
214
+ modelRegistry: ModelRegistry,
215
+ cfg: VisionHandoffConfig,
216
+ deps: DescriberDeps,
217
+ turnSignal?: AbortSignal,
218
+ ): Promise<BatchResult> {
219
+ const out: BatchResult = new Map();
220
+ // Parse and fallback run inside callVision's try so a throw there is logged as
221
+ // a batch failure and the partially filled map is still returned.
222
+ await callVision(
223
+ misses.map((m) => m.img),
224
+ misses.map((m) => m.hash),
225
+ "batch",
226
+ userPrompt,
227
+ visionModel,
228
+ modelRegistry,
229
+ cfg,
230
+ deps,
231
+ (text, truncated) => fillBatch(out, text, truncated, misses, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal),
232
+ turnSignal,
233
+ );
234
+ return out;
235
+ }
236
+
237
+ async function fillBatch(
238
+ out: BatchResult,
239
+ text: string,
240
+ truncated: boolean,
241
+ misses: { img: ExtractedImage; hash: string }[],
242
+ userPrompt: string,
243
+ visionModel: Model<Api>,
244
+ modelRegistry: ModelRegistry,
245
+ cfg: VisionHandoffConfig,
246
+ deps: DescriberDeps,
247
+ turnSignal?: AbortSignal,
248
+ ): Promise<void> {
249
+ const parsed = parseBatchedDescriptions(text, misses.length);
250
+ for (let i = 0; i < misses.length; i++) {
251
+ const d = parsed[i];
252
+ if (d) out.set(misses[i].hash, d);
253
+ }
254
+ // stopReason length = the batch response was cut off mid-stream. Only the
255
+ // highest-index parsed section can be partial (earlier ones ended with their
256
+ // END marker), so mark that one. It is NOT re-described: a single call for
257
+ // the same complex image would likely hit the same limit.
258
+ if (truncated) {
259
+ for (let i = misses.length - 1; i >= 0; i--) {
260
+ const h = misses[i].hash;
261
+ const d = out.get(h);
262
+ if (d) {
263
+ out.set(h, markDescriptionTruncated(d));
264
+ break;
265
+ }
266
+ }
267
+ }
268
+ // A failed delimiter-parse is a failed batching, not a failed description:
269
+ // describe each unparsed image with its own single-image call, in parallel.
270
+ const unparsed = misses.filter((m) => !out.has(m.hash));
271
+ if (unparsed.length > 0) {
272
+ const fallbacks = await Promise.all(
273
+ unparsed.map((m) => describeSingle(m.img, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal)),
274
+ );
275
+ for (let i = 0; i < unparsed.length; i++) {
276
+ const d = fallbacks[i];
277
+ if (d) out.set(unparsed[i].hash, d);
278
+ }
279
+ }
280
+ }
281
+
282
+ // Describe a single image with one completeSimple() call and return the RAW
283
+ // description (no envelope). Returns null on any genuine failure (auth,
284
+ // abort/error, empty) so the caller only caches UNAVAILABLE when a real
285
+ // describer attempt failed.
286
+ export async function describeSingle(
287
+ img: ExtractedImage,
288
+ userPrompt: string,
289
+ visionModel: Model<Api>,
290
+ modelRegistry: ModelRegistry,
291
+ cfg: VisionHandoffConfig,
292
+ deps: DescriberDeps,
293
+ turnSignal?: AbortSignal,
294
+ ): Promise<string | null> {
295
+ // stopReason length = token limit hit before the model finished. The partial
296
+ // text is still useful but must not pass as complete.
297
+ return callVision(
298
+ [img],
299
+ [imageHash(img.mimeType, img.data)],
300
+ "single",
301
+ userPrompt,
302
+ visionModel,
303
+ modelRegistry,
304
+ cfg,
305
+ deps,
306
+ (text, truncated) => (truncated ? markDescriptionTruncated(text) : text),
307
+ turnSignal,
308
+ );
309
+ }
310
+
311
+ // One vision-model call shared by runBatch and describeSingle: auth, abort and
312
+ // timeout guards, usage reporting, and failure logging. Returns null on any
313
+ // genuine failure (auth, abort/error, empty, throw) after deps.setLastError
314
+ // records the reason; a user cancel stays silent.
315
+ async function callVision<T>(
316
+ images: ExtractedImage[],
317
+ hashes: string[],
318
+ phase: "batch" | "single",
319
+ userPrompt: string,
320
+ visionModel: Model<Api>,
321
+ modelRegistry: ModelRegistry,
322
+ cfg: VisionHandoffConfig,
323
+ deps: DescriberDeps,
324
+ finish: (text: string, truncated: boolean) => T | Promise<T>,
325
+ turnSignal?: AbortSignal,
326
+ ): Promise<T | null> {
327
+ const attemptedModel = formatModelRef(visionModel.provider, visionModel.id);
328
+ deps.setAttemptedModel?.(attemptedModel);
329
+ const n = images.length;
330
+ const base = { phase, visionModel: cfg.visionModel, attemptedModel, imageHashes: hashes, imageCount: n };
331
+ const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
332
+ if (!auth.ok || !auth.apiKey) {
333
+ const reason = !auth.ok ? auth.error : 'No API key for vision model "' + attemptedModel + '"';
334
+ deps.setLastError(reason);
335
+ appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
336
+ return null;
337
+ }
338
+
339
+ const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
340
+ const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
341
+ const content: (TextContent | ImageContent)[] = [
342
+ { type: "text", text: batchUserPrompt(n, userPrompt, prefix) },
343
+ ...images.map((img) => ({ type: "image", data: img.data, mimeType: img.mimeType } satisfies ImageContent)),
344
+ ];
345
+ const userMessage: Message = { role: "user", content, timestamp: Date.now() };
346
+ const timeoutMs = describeTimeoutMs(n, cfg.describeTimeoutMs);
347
+ const maxTokens = resolveMaxTokens(cfg, visionModel);
348
+ const reasoning = resolveReasoning(cfg, visionModel);
349
+ const controller = new AbortController();
350
+ let timedOut = false;
351
+ using fetchGuard = fetchInterceptorGuard();
352
+ using timer = timeoutGuard(timeoutMs, () => {
353
+ timedOut = true;
354
+ controller.abort();
355
+ });
356
+ using abortWire = abortWireGuard(turnSignal, controller);
357
+
358
+ const describeCtx: DescribeContext = { energyReader: undefined };
359
+ try {
360
+ const response = await describeAls.run(describeCtx, async () =>
361
+ completeVisionModel(
362
+ visionModel,
363
+ modelRegistry,
364
+ { systemPrompt, messages: [userMessage] },
365
+ { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
366
+ ),
367
+ );
368
+ const capture = await readCapture(describeCtx);
369
+ const record = buildUsageRecord(response, capture, visionModel, hashes[0], n > 1 ? hashes : undefined);
370
+ if (record) deps.reportUsage(record);
371
+ if (response.stopReason === "aborted" || response.stopReason === "error") {
372
+ const reason = setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
373
+ if (reason) {
374
+ appendVisionError({
375
+ ...base,
376
+ reason,
377
+ stopReason: response.stopReason,
378
+ timedOut,
379
+ timeoutMs,
380
+ errorMessage: response.errorMessage,
381
+ config: configSnapshot(cfg),
382
+ });
383
+ }
384
+ return null;
385
+ }
386
+ const text = response.content
387
+ .filter((c): c is TextContent => c.type === "text")
388
+ .map((c) => c.text)
389
+ .join("\n")
390
+ .trim();
391
+ if (!text) {
392
+ const reason = "vision model returned an empty description";
393
+ deps.setLastError(reason);
394
+ appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
395
+ return null;
396
+ }
397
+ return await finish(text, response.stopReason === "length");
398
+ } catch (err) {
399
+ const userAborted = abortWire.userAborted();
400
+ const reason = timedOut
401
+ ? "describer timed out after " + timeoutMs / 1000 + "s"
402
+ : err instanceof Error
403
+ ? err.message
404
+ : String(err);
405
+ deps.setLastError(reason);
406
+ // A deliberate user cancel is not worth logging (no-warn-on-user-abort contract).
407
+ if (!userAborted) {
408
+ appendVisionError({
409
+ ...base,
410
+ reason,
411
+ timedOut,
412
+ timeoutMs,
413
+ errorStack: err instanceof Error ? err.stack : undefined,
414
+ config: configSnapshot(cfg),
415
+ });
416
+ }
417
+ return null;
418
+ } finally {
419
+ // The using guards release the fetch interceptor, timer, and abort wire.
420
+ // Only the energy tee needs an explicit swallow: it rejects when the main
421
+ // stream aborts.
422
+ describeCtx.energyReader?.catch(() => {});
423
+ }
424
+ }
425
+
426
+ /** Read the energy tee for a describer call, if one was captured. Returns the
427
+ * empty capture when there is no reader (non-Neuralwatt models) or the tee
428
+ * aborted with the main stream. */
429
+ async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
430
+ if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
431
+ try {
432
+ return await describeCtx.energyReader;
433
+ } catch {
434
+ return EMPTY_ENERGY_CAPTURE;
435
+ }
436
+ }
437
+
438
+ /** Translate a non-OK stopReason into a user-facing failure message, unless the
439
+ * abort came from the user cancelling the turn (then stay silent — no warning
440
+ * for a deliberate cancel). Returns the message that was set (so the caller
441
+ * can log it), or null when the abort was a user cancel / nothing was set. */
442
+ function setStopReasonError(
443
+ deps: DescriberDeps,
444
+ stopReason: string,
445
+ errorMessage: string | undefined,
446
+ abortWire: AbortWire,
447
+ timedOut: boolean,
448
+ timeoutMs: number,
449
+ ): string | null {
450
+ if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
451
+ if (timedOut) {
452
+ const reason = `describer timed out after ${timeoutMs / 1000}s`;
453
+ deps.setLastError(reason);
454
+ return reason;
455
+ }
456
+ const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
457
+ deps.setLastError(reason);
458
+ return reason;
459
+ }