@bismawy/pi-vision-watcher 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +5 -4
- package/package.json +1 -1
- package/src/dataloader.ts +389 -389
- package/src/describer.ts +459 -561
- package/src/vision-model-selector.ts +123 -68
- package/vision-watcher.ts +1 -0
package/src/describer.ts
CHANGED
|
@@ -1,561 +1,459 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The vision describer calls a vision-capable model through its registered
|
|
3
|
-
* provider stream, falling back to pi-ai's `completeSimple()` for built-ins.
|
|
4
|
-
*
|
|
5
|
-
* `completeSimple` (not `complete`) is the path that translates the `reasoning`
|
|
6
|
-
* ThinkingLevel into each provider's `reasoningEffort`/budget. `complete()` →
|
|
7
|
-
* `stream()` reads only the pre-translated `reasoningEffort` field and silently
|
|
8
|
-
* drops a bare `reasoning`, so the describer's thinking setting would be a no-op
|
|
9
|
-
* through `complete()` — the bug this swap fixes. The agent loop, SDK, and
|
|
10
|
-
* compaction all route thinking through `completeSimple`/`streamSimple` for the
|
|
11
|
-
* same reason.
|
|
12
|
-
*
|
|
13
|
-
* Two entry points:
|
|
14
|
-
* - {@link runBatch}: ONE batched call describing N images at once (the
|
|
15
|
-
* dataloader's dispatch path). The model is asked to emit one delimited
|
|
16
|
-
* `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
|
|
17
|
-
* split back into per-image descriptions.
|
|
18
|
-
* - {@link describeSingle}: one call for one image (no delimiters). The
|
|
19
|
-
* robust per-image fallback used when a batched response couldn't be split.
|
|
20
|
-
*
|
|
21
|
-
* Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
|
|
22
|
-
* managed with the `using` keyword via the {@link Disposable} guards in
|
|
23
|
-
* `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
|
|
24
|
-
* carried. Disposing is lexical and exception-safe: a thrown `completeSimple()`
|
|
25
|
-
* still tears down the timer, uninstalls the interceptor, and detaches the
|
|
26
|
-
* abort listener.
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
import type {
|
|
30
|
-
Api,
|
|
31
|
-
AssistantMessage,
|
|
32
|
-
Context,
|
|
33
|
-
ImageContent,
|
|
34
|
-
Message,
|
|
35
|
-
Model,
|
|
36
|
-
SimpleStreamOptions,
|
|
37
|
-
TextContent,
|
|
38
|
-
ThinkingLevel,
|
|
39
|
-
} from "@earendil-works/pi-ai";
|
|
40
|
-
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
41
|
-
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
42
|
-
import {
|
|
43
|
-
batchUserPrompt,
|
|
44
|
-
DEFAULT_USER_PROMPT_PREFIX,
|
|
45
|
-
DEFAULT_VISION_PROMPT,
|
|
46
|
-
describeTimeoutMs,
|
|
47
|
-
formatModelRef,
|
|
48
|
-
markDescriptionTruncated,
|
|
49
|
-
parseBatchedDescriptions,
|
|
50
|
-
type ExtractedImage,
|
|
51
|
-
type VisionHandoffConfig,
|
|
52
|
-
} from "./index.js";
|
|
53
|
-
import { appendVisionError } from "./error-log.js";
|
|
54
|
-
import {
|
|
55
|
-
buildUsageRecord,
|
|
56
|
-
describeAls,
|
|
57
|
-
EMPTY_ENERGY_CAPTURE,
|
|
58
|
-
type DescribeContext,
|
|
59
|
-
type VisionHandoffEnergyCapture,
|
|
60
|
-
type VisionHandoffUsageRecord,
|
|
61
|
-
} from "./usage.js";
|
|
62
|
-
import { imageHash } from "./image.js";
|
|
63
|
-
import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
|
|
64
|
-
|
|
65
|
-
/** Tokens reserved for the describer's input so the requested output budget
|
|
66
|
-
* can't exceed the model's context window. The describer's input is bounded —
|
|
67
|
-
* one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
|
|
68
|
-
* math simple while leaving generous room for the image + prompt tokens.
|
|
69
|
-
* Providers reject a request when `inputTokens + maxTokens > contextWindow`
|
|
70
|
-
* (e.g. a model whose declared maxTokens equals its full context window), so
|
|
71
|
-
* this clamp prevents that 400 without meaningfully limiting a high-output
|
|
72
|
-
* model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
|
|
73
|
-
const INPUT_RESERVE_TOKENS = 8192;
|
|
74
|
-
|
|
75
|
-
/** Resolve the `maxTokens` to pass to `completeSimple()` for a describer call.
|
|
76
|
-
*
|
|
77
|
-
* - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
|
|
78
|
-
* fit the context window.
|
|
79
|
-
* - Otherwise use the vision model's declared max output (`model.maxTokens`),
|
|
80
|
-
* clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
|
|
81
|
-
* requested output can't exceed the context window minus a reserve for the
|
|
82
|
-
* input. This is the real fix for models whose declared `maxTokens` equals
|
|
83
|
-
* their full `contextWindow` (which providers reject with any non-empty
|
|
84
|
-
* input): we cap at `contextWindow − reserve` instead.
|
|
85
|
-
* - If the model declares no usable max (`maxTokens <= 0`) and no cap is
|
|
86
|
-
* configured, return `undefined` so the provider applies its own default. */
|
|
87
|
-
export function resolveMaxTokens(
|
|
88
|
-
cfg: VisionHandoffConfig,
|
|
89
|
-
visionModel: Model<Api>,
|
|
90
|
-
): number | undefined {
|
|
91
|
-
const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
|
|
92
|
-
const cap = ctx - INPUT_RESERVE_TOKENS;
|
|
93
|
-
const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
|
|
94
|
-
if (requested === undefined) return undefined;
|
|
95
|
-
// Clamp to the context-window-derived ceiling; never below 1.
|
|
96
|
-
return Math.max(1, Math.min(requested, cap));
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/** Resolve the `reasoning` (thinking) level to pass to `completeSimple()`, or
|
|
100
|
-
* `undefined` to leave thinking off.
|
|
101
|
-
*
|
|
102
|
-
* - Returns `undefined` when thinking is disabled in config.
|
|
103
|
-
* - Returns `undefined` when the vision model doesn't declare reasoning
|
|
104
|
-
* support (`model.reasoning === false`) — sending a `reasoning` level to a
|
|
105
|
-
* non-reasoning model would be ignored at best and rejected at worst.
|
|
106
|
-
* - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
|
|
107
|
-
*
|
|
108
|
-
* Mirrors pi core's `createSummarizationOptions` guard so the describer's
|
|
109
|
-
* thinking behaviour matches the rest of pi. */
|
|
110
|
-
export function resolveReasoning(
|
|
111
|
-
cfg: VisionHandoffConfig,
|
|
112
|
-
visionModel: Model<Api>,
|
|
113
|
-
): ThinkingLevel | undefined {
|
|
114
|
-
if (!cfg.thinking) return undefined;
|
|
115
|
-
if (!visionModel.reasoning) return undefined;
|
|
116
|
-
return cfg.thinkingLevel;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/** Small config snapshot embedded in every error-log entry — the fields most
|
|
120
|
-
* relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
|
|
121
|
-
* reasoning level the model rejects, etc.). */
|
|
122
|
-
function configSnapshot(cfg: VisionHandoffConfig): {
|
|
123
|
-
maxTokens?: number;
|
|
124
|
-
thinking: boolean;
|
|
125
|
-
thinkingLevel: string;
|
|
126
|
-
} {
|
|
127
|
-
return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/** Complete through the provider registered in Pi's model registry when one
|
|
131
|
-
* supplies a custom stream. The post-0.80 ModelRuntime keeps extension streams
|
|
132
|
-
* out of pi-ai's deprecated global compatibility registry, so calling
|
|
133
|
-
* completeSimple() directly cannot resolve custom API ids such as `makora`.
|
|
134
|
-
* The optional access preserves compatibility with older ModelRegistry versions,
|
|
135
|
-
* which registered custom streams globally and do not expose the config getter. */
|
|
136
|
-
const NEURALWATT_CONVERSATION_HEADER = "x-nw-conversation-id";
|
|
137
|
-
|
|
138
|
-
function isolateVisionRequest(
|
|
139
|
-
model: Model<Api>,
|
|
140
|
-
options: SimpleStreamOptions,
|
|
141
|
-
): SimpleStreamOptions {
|
|
142
|
-
// Stable per-model identity (not randomUUID): every describer call shares the
|
|
143
|
-
// same system prompt (DEFAULT_VISION_PROMPT), so a stable session id lets the
|
|
144
|
-
// provider's prompt cache reuse that prefix across calls (cacheRead/cacheWrite
|
|
145
|
-
// > 0) instead of re-billing it every turn. Still distinct from the main
|
|
146
|
-
// agent's conversation id (the inherited header we replace), so unrelated
|
|
147
|
-
// image prompts never become the main agent's newest cache lineage. A
|
|
148
|
-
// per-model id keeps different vision models (fallback chains) on separate
|
|
149
|
-
// lineages.
|
|
150
|
-
const sessionId = `pi-vision-watcher:${model.provider}:${model.id}`;
|
|
151
|
-
const headers = { ...(options.headers ?? {}) };
|
|
152
|
-
const inheritedConversationHeader = Object.keys(headers).find(
|
|
153
|
-
(name) => name.toLowerCase() === NEURALWATT_CONVERSATION_HEADER,
|
|
154
|
-
);
|
|
155
|
-
|
|
156
|
-
if (inheritedConversationHeader) {
|
|
157
|
-
headers[inheritedConversationHeader] = sessionId;
|
|
158
|
-
} else if (model.provider.toLowerCase() === "neuralwatt") {
|
|
159
|
-
headers["X-NW-Conversation-ID"] = sessionId;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
return {
|
|
163
|
-
...options,
|
|
164
|
-
sessionId,
|
|
165
|
-
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
export async function completeVisionModel(
|
|
170
|
-
model: Model<Api>,
|
|
171
|
-
modelRegistry: ModelRegistry,
|
|
172
|
-
context: Context,
|
|
173
|
-
options: SimpleStreamOptions,
|
|
174
|
-
): Promise<AssistantMessage> {
|
|
175
|
-
const isolatedOptions = isolateVisionRequest(model, options);
|
|
176
|
-
const provider = modelRegistry.getRegisteredProviderConfig?.(model.provider);
|
|
177
|
-
if (provider?.streamSimple && provider.api === model.api) {
|
|
178
|
-
return provider.streamSimple(model, context, isolatedOptions).result();
|
|
179
|
-
}
|
|
180
|
-
return completeSimple(model, context, isolatedOptions);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/** Dependencies the describer can't own itself (held by the engine). */
|
|
184
|
-
export interface DescriberDeps {
|
|
185
|
-
/** Report a usage+energy record for one real describer call (cache hits emit none). */
|
|
186
|
-
reportUsage(record: VisionHandoffUsageRecord): void;
|
|
187
|
-
/** Set the most-recent describer failure message (surfaced to the user by the engine).
|
|
188
|
-
* Pass `null` to clear before a fresh attempt. */
|
|
189
|
-
setLastError(msg: string | null): void;
|
|
190
|
-
/** Record the model ref ACTUALLY attempted for the latest describer call.
|
|
191
|
-
* Differs from `config.visionModel` when a failover fallback ran — the
|
|
192
|
-
* warning must name the model that failed, not the configured primary.
|
|
193
|
-
* Optional so older callers/tests compile unchanged. */
|
|
194
|
-
setAttemptedModel?(ref: string): void;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
|
|
198
|
-
export type BatchResult = Map<string, string>;
|
|
199
|
-
|
|
200
|
-
/** Describe N images with ONE batched `completeSimple()` call and split the response
|
|
201
|
-
* back into per-image descriptions. Returns a map keyed by image hash; an
|
|
202
|
-
* image whose section failed to parse is omitted (the caller treats omission
|
|
203
|
-
* as "description unavailable"). On a genuine call failure (auth, abort,
|
|
204
|
-
* empty) the map is empty and `deps.setLastError` records the reason.
|
|
205
|
-
*
|
|
206
|
-
* A failed delimiter-parse is a failed BATCH, not a failed description: the
|
|
207
|
-
* unparsed images fall back to parallel single-image calls (no delimiters to
|
|
208
|
-
* cooperate with), so the common cooperative case stays one call while an
|
|
209
|
-
* uncooperative model still gets every image described. */
|
|
210
|
-
export async function runBatch(
|
|
211
|
-
misses: { img: ExtractedImage; hash: string }[],
|
|
212
|
-
userPrompt: string,
|
|
213
|
-
visionModel: Model<Api>,
|
|
214
|
-
modelRegistry: ModelRegistry,
|
|
215
|
-
cfg: VisionHandoffConfig,
|
|
216
|
-
deps: DescriberDeps,
|
|
217
|
-
turnSignal?: AbortSignal,
|
|
218
|
-
): Promise<BatchResult> {
|
|
219
|
-
const out: BatchResult = new Map();
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
deps.setLastError(reason);
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
appendVisionError({
|
|
461
|
-
phase: "single",
|
|
462
|
-
reason,
|
|
463
|
-
visionModel: cfg.visionModel,
|
|
464
|
-
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
465
|
-
imageHashes: [hash],
|
|
466
|
-
imageCount: 1,
|
|
467
|
-
stopReason: response.stopReason,
|
|
468
|
-
timedOut,
|
|
469
|
-
timeoutMs,
|
|
470
|
-
errorMessage: response.errorMessage,
|
|
471
|
-
config: configSnapshot(cfg),
|
|
472
|
-
});
|
|
473
|
-
}
|
|
474
|
-
return null;
|
|
475
|
-
}
|
|
476
|
-
const text = response.content
|
|
477
|
-
.filter((c): c is TextContent => c.type === "text")
|
|
478
|
-
.map((c) => c.text)
|
|
479
|
-
.join("\n")
|
|
480
|
-
.trim();
|
|
481
|
-
if (!text) {
|
|
482
|
-
deps.setLastError("vision model returned an empty description");
|
|
483
|
-
appendVisionError({
|
|
484
|
-
phase: "single",
|
|
485
|
-
reason: "vision model returned an empty description",
|
|
486
|
-
visionModel: cfg.visionModel,
|
|
487
|
-
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
488
|
-
imageHashes: [hash],
|
|
489
|
-
imageCount: 1,
|
|
490
|
-
config: configSnapshot(cfg),
|
|
491
|
-
});
|
|
492
|
-
return null;
|
|
493
|
-
}
|
|
494
|
-
// stopReason "length" = the model hit a token limit (configured maxTokens or
|
|
495
|
-
// the provider's hard output cap) before finishing. The partial text is still
|
|
496
|
-
// useful, but it must not pass as complete — mark it so the agent/user know.
|
|
497
|
-
return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
|
|
498
|
-
} catch (err) {
|
|
499
|
-
const userAborted = abortWire.userAborted();
|
|
500
|
-
const reason = timedOut
|
|
501
|
-
? `describer timed out after ${timeoutMs / 1000}s`
|
|
502
|
-
: err instanceof Error
|
|
503
|
-
? err.message
|
|
504
|
-
: String(err);
|
|
505
|
-
deps.setLastError(reason);
|
|
506
|
-
// A deliberate user cancel isn't a troubleshooting-worthy error — skip
|
|
507
|
-
// logging it (mirroring the no-warn-on-user-abort contract).
|
|
508
|
-
if (!userAborted) {
|
|
509
|
-
appendVisionError({
|
|
510
|
-
phase: "single",
|
|
511
|
-
reason,
|
|
512
|
-
visionModel: cfg.visionModel,
|
|
513
|
-
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
514
|
-
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
515
|
-
imageCount: 1,
|
|
516
|
-
timedOut,
|
|
517
|
-
timeoutMs,
|
|
518
|
-
errorStack: err instanceof Error ? err.stack : undefined,
|
|
519
|
-
config: configSnapshot(cfg),
|
|
520
|
-
});
|
|
521
|
-
}
|
|
522
|
-
return null;
|
|
523
|
-
} finally {
|
|
524
|
-
describeCtx.energyReader?.catch(() => {});
|
|
525
|
-
}
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
/** Read the energy tee for a describer call, if one was captured. Returns the
|
|
529
|
-
* empty capture when there is no reader (non-Neuralwatt models) or the tee
|
|
530
|
-
* aborted with the main stream. */
|
|
531
|
-
async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
|
|
532
|
-
if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
|
|
533
|
-
try {
|
|
534
|
-
return await describeCtx.energyReader;
|
|
535
|
-
} catch {
|
|
536
|
-
return EMPTY_ENERGY_CAPTURE;
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
/** Translate a non-OK stopReason into a user-facing failure message, unless the
|
|
541
|
-
* abort came from the user cancelling the turn (then stay silent — no warning
|
|
542
|
-
* for a deliberate cancel). Returns the message that was set (so the caller
|
|
543
|
-
* can log it), or null when the abort was a user cancel / nothing was set. */
|
|
544
|
-
function setStopReasonError(
|
|
545
|
-
deps: DescriberDeps,
|
|
546
|
-
stopReason: string,
|
|
547
|
-
errorMessage: string | undefined,
|
|
548
|
-
abortWire: AbortWire,
|
|
549
|
-
timedOut: boolean,
|
|
550
|
-
timeoutMs: number,
|
|
551
|
-
): string | null {
|
|
552
|
-
if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
|
|
553
|
-
if (timedOut) {
|
|
554
|
-
const reason = `describer timed out after ${timeoutMs / 1000}s`;
|
|
555
|
-
deps.setLastError(reason);
|
|
556
|
-
return reason;
|
|
557
|
-
}
|
|
558
|
-
const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
|
|
559
|
-
deps.setLastError(reason);
|
|
560
|
-
return reason;
|
|
561
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* The vision describer calls a vision-capable model through its registered
|
|
3
|
+
* provider stream, falling back to pi-ai's `completeSimple()` for built-ins.
|
|
4
|
+
*
|
|
5
|
+
* `completeSimple` (not `complete`) is the path that translates the `reasoning`
|
|
6
|
+
* ThinkingLevel into each provider's `reasoningEffort`/budget. `complete()` →
|
|
7
|
+
* `stream()` reads only the pre-translated `reasoningEffort` field and silently
|
|
8
|
+
* drops a bare `reasoning`, so the describer's thinking setting would be a no-op
|
|
9
|
+
* through `complete()` — the bug this swap fixes. The agent loop, SDK, and
|
|
10
|
+
* compaction all route thinking through `completeSimple`/`streamSimple` for the
|
|
11
|
+
* same reason.
|
|
12
|
+
*
|
|
13
|
+
* Two entry points:
|
|
14
|
+
* - {@link runBatch}: ONE batched call describing N images at once (the
|
|
15
|
+
* dataloader's dispatch path). The model is asked to emit one delimited
|
|
16
|
+
* `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
|
|
17
|
+
* split back into per-image descriptions.
|
|
18
|
+
* - {@link describeSingle}: one call for one image (no delimiters). The
|
|
19
|
+
* robust per-image fallback used when a batched response couldn't be split.
|
|
20
|
+
*
|
|
21
|
+
* Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
|
|
22
|
+
* managed with the `using` keyword via the {@link Disposable} guards in
|
|
23
|
+
* `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
|
|
24
|
+
* carried. Disposing is lexical and exception-safe: a thrown `completeSimple()`
|
|
25
|
+
* still tears down the timer, uninstalls the interceptor, and detaches the
|
|
26
|
+
* abort listener.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type {
|
|
30
|
+
Api,
|
|
31
|
+
AssistantMessage,
|
|
32
|
+
Context,
|
|
33
|
+
ImageContent,
|
|
34
|
+
Message,
|
|
35
|
+
Model,
|
|
36
|
+
SimpleStreamOptions,
|
|
37
|
+
TextContent,
|
|
38
|
+
ThinkingLevel,
|
|
39
|
+
} from "@earendil-works/pi-ai";
|
|
40
|
+
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
41
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
42
|
+
import {
|
|
43
|
+
batchUserPrompt,
|
|
44
|
+
DEFAULT_USER_PROMPT_PREFIX,
|
|
45
|
+
DEFAULT_VISION_PROMPT,
|
|
46
|
+
describeTimeoutMs,
|
|
47
|
+
formatModelRef,
|
|
48
|
+
markDescriptionTruncated,
|
|
49
|
+
parseBatchedDescriptions,
|
|
50
|
+
type ExtractedImage,
|
|
51
|
+
type VisionHandoffConfig,
|
|
52
|
+
} from "./index.js";
|
|
53
|
+
import { appendVisionError } from "./error-log.js";
|
|
54
|
+
import {
|
|
55
|
+
buildUsageRecord,
|
|
56
|
+
describeAls,
|
|
57
|
+
EMPTY_ENERGY_CAPTURE,
|
|
58
|
+
type DescribeContext,
|
|
59
|
+
type VisionHandoffEnergyCapture,
|
|
60
|
+
type VisionHandoffUsageRecord,
|
|
61
|
+
} from "./usage.js";
|
|
62
|
+
import { imageHash } from "./image.js";
|
|
63
|
+
import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
|
|
64
|
+
|
|
65
|
+
/** Tokens reserved for the describer's input so the requested output budget
|
|
66
|
+
* can't exceed the model's context window. The describer's input is bounded —
|
|
67
|
+
* one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
|
|
68
|
+
* math simple while leaving generous room for the image + prompt tokens.
|
|
69
|
+
* Providers reject a request when `inputTokens + maxTokens > contextWindow`
|
|
70
|
+
* (e.g. a model whose declared maxTokens equals its full context window), so
|
|
71
|
+
* this clamp prevents that 400 without meaningfully limiting a high-output
|
|
72
|
+
* model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
|
|
73
|
+
const INPUT_RESERVE_TOKENS = 8192;
|
|
74
|
+
|
|
75
|
+
/** Resolve the `maxTokens` to pass to `completeSimple()` for a describer call.
|
|
76
|
+
*
|
|
77
|
+
* - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
|
|
78
|
+
* fit the context window.
|
|
79
|
+
* - Otherwise use the vision model's declared max output (`model.maxTokens`),
|
|
80
|
+
* clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
|
|
81
|
+
* requested output can't exceed the context window minus a reserve for the
|
|
82
|
+
* input. This is the real fix for models whose declared `maxTokens` equals
|
|
83
|
+
* their full `contextWindow` (which providers reject with any non-empty
|
|
84
|
+
* input): we cap at `contextWindow − reserve` instead.
|
|
85
|
+
* - If the model declares no usable max (`maxTokens <= 0`) and no cap is
|
|
86
|
+
* configured, return `undefined` so the provider applies its own default. */
|
|
87
|
+
export function resolveMaxTokens(
|
|
88
|
+
cfg: VisionHandoffConfig,
|
|
89
|
+
visionModel: Model<Api>,
|
|
90
|
+
): number | undefined {
|
|
91
|
+
const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
|
|
92
|
+
const cap = ctx - INPUT_RESERVE_TOKENS;
|
|
93
|
+
const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
|
|
94
|
+
if (requested === undefined) return undefined;
|
|
95
|
+
// Clamp to the context-window-derived ceiling; never below 1.
|
|
96
|
+
return Math.max(1, Math.min(requested, cap));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Resolve the `reasoning` (thinking) level to pass to `completeSimple()`, or
|
|
100
|
+
* `undefined` to leave thinking off.
|
|
101
|
+
*
|
|
102
|
+
* - Returns `undefined` when thinking is disabled in config.
|
|
103
|
+
* - Returns `undefined` when the vision model doesn't declare reasoning
|
|
104
|
+
* support (`model.reasoning === false`) — sending a `reasoning` level to a
|
|
105
|
+
* non-reasoning model would be ignored at best and rejected at worst.
|
|
106
|
+
* - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
|
|
107
|
+
*
|
|
108
|
+
* Mirrors pi core's `createSummarizationOptions` guard so the describer's
|
|
109
|
+
* thinking behaviour matches the rest of pi. */
|
|
110
|
+
export function resolveReasoning(
|
|
111
|
+
cfg: VisionHandoffConfig,
|
|
112
|
+
visionModel: Model<Api>,
|
|
113
|
+
): ThinkingLevel | undefined {
|
|
114
|
+
if (!cfg.thinking) return undefined;
|
|
115
|
+
if (!visionModel.reasoning) return undefined;
|
|
116
|
+
return cfg.thinkingLevel;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Small config snapshot embedded in every error-log entry — the fields most
|
|
120
|
+
* relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
|
|
121
|
+
* reasoning level the model rejects, etc.). */
|
|
122
|
+
function configSnapshot(cfg: VisionHandoffConfig): {
|
|
123
|
+
maxTokens?: number;
|
|
124
|
+
thinking: boolean;
|
|
125
|
+
thinkingLevel: string;
|
|
126
|
+
} {
|
|
127
|
+
return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Complete through the provider registered in Pi's model registry when one
|
|
131
|
+
* supplies a custom stream. The post-0.80 ModelRuntime keeps extension streams
|
|
132
|
+
* out of pi-ai's deprecated global compatibility registry, so calling
|
|
133
|
+
* completeSimple() directly cannot resolve custom API ids such as `makora`.
|
|
134
|
+
* The optional access preserves compatibility with older ModelRegistry versions,
|
|
135
|
+
* which registered custom streams globally and do not expose the config getter. */
|
|
136
|
+
const NEURALWATT_CONVERSATION_HEADER = "x-nw-conversation-id";
|
|
137
|
+
|
|
138
|
+
function isolateVisionRequest(
|
|
139
|
+
model: Model<Api>,
|
|
140
|
+
options: SimpleStreamOptions,
|
|
141
|
+
): SimpleStreamOptions {
|
|
142
|
+
// Stable per-model identity (not randomUUID): every describer call shares the
|
|
143
|
+
// same system prompt (DEFAULT_VISION_PROMPT), so a stable session id lets the
|
|
144
|
+
// provider's prompt cache reuse that prefix across calls (cacheRead/cacheWrite
|
|
145
|
+
// > 0) instead of re-billing it every turn. Still distinct from the main
|
|
146
|
+
// agent's conversation id (the inherited header we replace), so unrelated
|
|
147
|
+
// image prompts never become the main agent's newest cache lineage. A
|
|
148
|
+
// per-model id keeps different vision models (fallback chains) on separate
|
|
149
|
+
// lineages.
|
|
150
|
+
const sessionId = `pi-vision-watcher:${model.provider}:${model.id}`;
|
|
151
|
+
const headers = { ...(options.headers ?? {}) };
|
|
152
|
+
const inheritedConversationHeader = Object.keys(headers).find(
|
|
153
|
+
(name) => name.toLowerCase() === NEURALWATT_CONVERSATION_HEADER,
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
if (inheritedConversationHeader) {
|
|
157
|
+
headers[inheritedConversationHeader] = sessionId;
|
|
158
|
+
} else if (model.provider.toLowerCase() === "neuralwatt") {
|
|
159
|
+
headers["X-NW-Conversation-ID"] = sessionId;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return {
|
|
163
|
+
...options,
|
|
164
|
+
sessionId,
|
|
165
|
+
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export async function completeVisionModel(
|
|
170
|
+
model: Model<Api>,
|
|
171
|
+
modelRegistry: ModelRegistry,
|
|
172
|
+
context: Context,
|
|
173
|
+
options: SimpleStreamOptions,
|
|
174
|
+
): Promise<AssistantMessage> {
|
|
175
|
+
const isolatedOptions = isolateVisionRequest(model, options);
|
|
176
|
+
const provider = modelRegistry.getRegisteredProviderConfig?.(model.provider);
|
|
177
|
+
if (provider?.streamSimple && provider.api === model.api) {
|
|
178
|
+
return provider.streamSimple(model, context, isolatedOptions).result();
|
|
179
|
+
}
|
|
180
|
+
return completeSimple(model, context, isolatedOptions);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Dependencies the describer can't own itself (held by the engine). */
|
|
184
|
+
export interface DescriberDeps {
|
|
185
|
+
/** Report a usage+energy record for one real describer call (cache hits emit none). */
|
|
186
|
+
reportUsage(record: VisionHandoffUsageRecord): void;
|
|
187
|
+
/** Set the most-recent describer failure message (surfaced to the user by the engine).
|
|
188
|
+
* Pass `null` to clear before a fresh attempt. */
|
|
189
|
+
setLastError(msg: string | null): void;
|
|
190
|
+
/** Record the model ref ACTUALLY attempted for the latest describer call.
|
|
191
|
+
* Differs from `config.visionModel` when a failover fallback ran — the
|
|
192
|
+
* warning must name the model that failed, not the configured primary.
|
|
193
|
+
* Optional so older callers/tests compile unchanged. */
|
|
194
|
+
setAttemptedModel?(ref: string): void;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
|
|
198
|
+
export type BatchResult = Map<string, string>;
|
|
199
|
+
|
|
200
|
+
/** Describe N images with ONE batched `completeSimple()` call and split the response
|
|
201
|
+
* back into per-image descriptions. Returns a map keyed by image hash; an
|
|
202
|
+
* image whose section failed to parse is omitted (the caller treats omission
|
|
203
|
+
* as "description unavailable"). On a genuine call failure (auth, abort,
|
|
204
|
+
* empty) the map is empty and `deps.setLastError` records the reason.
|
|
205
|
+
*
|
|
206
|
+
* A failed delimiter-parse is a failed BATCH, not a failed description: the
|
|
207
|
+
* unparsed images fall back to parallel single-image calls (no delimiters to
|
|
208
|
+
* cooperate with), so the common cooperative case stays one call while an
|
|
209
|
+
* uncooperative model still gets every image described. */
|
|
210
|
+
export async function runBatch(
|
|
211
|
+
misses: { img: ExtractedImage; hash: string }[],
|
|
212
|
+
userPrompt: string,
|
|
213
|
+
visionModel: Model<Api>,
|
|
214
|
+
modelRegistry: ModelRegistry,
|
|
215
|
+
cfg: VisionHandoffConfig,
|
|
216
|
+
deps: DescriberDeps,
|
|
217
|
+
turnSignal?: AbortSignal,
|
|
218
|
+
): Promise<BatchResult> {
|
|
219
|
+
const out: BatchResult = new Map();
|
|
220
|
+
// Parse and fallback run inside callVision's try so a throw there is logged as
|
|
221
|
+
// a batch failure and the partially filled map is still returned.
|
|
222
|
+
await callVision(
|
|
223
|
+
misses.map((m) => m.img),
|
|
224
|
+
misses.map((m) => m.hash),
|
|
225
|
+
"batch",
|
|
226
|
+
userPrompt,
|
|
227
|
+
visionModel,
|
|
228
|
+
modelRegistry,
|
|
229
|
+
cfg,
|
|
230
|
+
deps,
|
|
231
|
+
(text, truncated) => fillBatch(out, text, truncated, misses, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal),
|
|
232
|
+
turnSignal,
|
|
233
|
+
);
|
|
234
|
+
return out;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async function fillBatch(
|
|
238
|
+
out: BatchResult,
|
|
239
|
+
text: string,
|
|
240
|
+
truncated: boolean,
|
|
241
|
+
misses: { img: ExtractedImage; hash: string }[],
|
|
242
|
+
userPrompt: string,
|
|
243
|
+
visionModel: Model<Api>,
|
|
244
|
+
modelRegistry: ModelRegistry,
|
|
245
|
+
cfg: VisionHandoffConfig,
|
|
246
|
+
deps: DescriberDeps,
|
|
247
|
+
turnSignal?: AbortSignal,
|
|
248
|
+
): Promise<void> {
|
|
249
|
+
const parsed = parseBatchedDescriptions(text, misses.length);
|
|
250
|
+
for (let i = 0; i < misses.length; i++) {
|
|
251
|
+
const d = parsed[i];
|
|
252
|
+
if (d) out.set(misses[i].hash, d);
|
|
253
|
+
}
|
|
254
|
+
// stopReason length = the batch response was cut off mid-stream. Only the
|
|
255
|
+
// highest-index parsed section can be partial (earlier ones ended with their
|
|
256
|
+
// END marker), so mark that one. It is NOT re-described: a single call for
|
|
257
|
+
// the same complex image would likely hit the same limit.
|
|
258
|
+
if (truncated) {
|
|
259
|
+
for (let i = misses.length - 1; i >= 0; i--) {
|
|
260
|
+
const h = misses[i].hash;
|
|
261
|
+
const d = out.get(h);
|
|
262
|
+
if (d) {
|
|
263
|
+
out.set(h, markDescriptionTruncated(d));
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// A failed delimiter-parse is a failed batching, not a failed description:
|
|
269
|
+
// describe each unparsed image with its own single-image call, in parallel.
|
|
270
|
+
const unparsed = misses.filter((m) => !out.has(m.hash));
|
|
271
|
+
if (unparsed.length > 0) {
|
|
272
|
+
const fallbacks = await Promise.all(
|
|
273
|
+
unparsed.map((m) => describeSingle(m.img, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal)),
|
|
274
|
+
);
|
|
275
|
+
for (let i = 0; i < unparsed.length; i++) {
|
|
276
|
+
const d = fallbacks[i];
|
|
277
|
+
if (d) out.set(unparsed[i].hash, d);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Describe a single image with one completeSimple() call and return the RAW
|
|
283
|
+
// description (no envelope). Returns null on any genuine failure (auth,
|
|
284
|
+
// abort/error, empty) so the caller only caches UNAVAILABLE when a real
|
|
285
|
+
// describer attempt failed.
|
|
286
|
+
export async function describeSingle(
|
|
287
|
+
img: ExtractedImage,
|
|
288
|
+
userPrompt: string,
|
|
289
|
+
visionModel: Model<Api>,
|
|
290
|
+
modelRegistry: ModelRegistry,
|
|
291
|
+
cfg: VisionHandoffConfig,
|
|
292
|
+
deps: DescriberDeps,
|
|
293
|
+
turnSignal?: AbortSignal,
|
|
294
|
+
): Promise<string | null> {
|
|
295
|
+
// stopReason length = token limit hit before the model finished. The partial
|
|
296
|
+
// text is still useful but must not pass as complete.
|
|
297
|
+
return callVision(
|
|
298
|
+
[img],
|
|
299
|
+
[imageHash(img.mimeType, img.data)],
|
|
300
|
+
"single",
|
|
301
|
+
userPrompt,
|
|
302
|
+
visionModel,
|
|
303
|
+
modelRegistry,
|
|
304
|
+
cfg,
|
|
305
|
+
deps,
|
|
306
|
+
(text, truncated) => (truncated ? markDescriptionTruncated(text) : text),
|
|
307
|
+
turnSignal,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// One vision-model call shared by runBatch and describeSingle: auth, abort and
|
|
312
|
+
// timeout guards, usage reporting, and failure logging. Returns null on any
|
|
313
|
+
// genuine failure (auth, abort/error, empty, throw) after deps.setLastError
|
|
314
|
+
// records the reason; a user cancel stays silent.
|
|
315
|
+
async function callVision<T>(
|
|
316
|
+
images: ExtractedImage[],
|
|
317
|
+
hashes: string[],
|
|
318
|
+
phase: "batch" | "single",
|
|
319
|
+
userPrompt: string,
|
|
320
|
+
visionModel: Model<Api>,
|
|
321
|
+
modelRegistry: ModelRegistry,
|
|
322
|
+
cfg: VisionHandoffConfig,
|
|
323
|
+
deps: DescriberDeps,
|
|
324
|
+
finish: (text: string, truncated: boolean) => T | Promise<T>,
|
|
325
|
+
turnSignal?: AbortSignal,
|
|
326
|
+
): Promise<T | null> {
|
|
327
|
+
const attemptedModel = formatModelRef(visionModel.provider, visionModel.id);
|
|
328
|
+
deps.setAttemptedModel?.(attemptedModel);
|
|
329
|
+
const n = images.length;
|
|
330
|
+
const base = { phase, visionModel: cfg.visionModel, attemptedModel, imageHashes: hashes, imageCount: n };
|
|
331
|
+
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
332
|
+
if (!auth.ok || !auth.apiKey) {
|
|
333
|
+
const reason = !auth.ok ? auth.error : 'No API key for vision model "' + attemptedModel + '"';
|
|
334
|
+
deps.setLastError(reason);
|
|
335
|
+
appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
|
|
340
|
+
const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
|
|
341
|
+
const content: (TextContent | ImageContent)[] = [
|
|
342
|
+
{ type: "text", text: batchUserPrompt(n, userPrompt, prefix) },
|
|
343
|
+
...images.map((img) => ({ type: "image", data: img.data, mimeType: img.mimeType } satisfies ImageContent)),
|
|
344
|
+
];
|
|
345
|
+
const userMessage: Message = { role: "user", content, timestamp: Date.now() };
|
|
346
|
+
const timeoutMs = describeTimeoutMs(n, cfg.describeTimeoutMs);
|
|
347
|
+
const maxTokens = resolveMaxTokens(cfg, visionModel);
|
|
348
|
+
const reasoning = resolveReasoning(cfg, visionModel);
|
|
349
|
+
const controller = new AbortController();
|
|
350
|
+
let timedOut = false;
|
|
351
|
+
using fetchGuard = fetchInterceptorGuard();
|
|
352
|
+
using timer = timeoutGuard(timeoutMs, () => {
|
|
353
|
+
timedOut = true;
|
|
354
|
+
controller.abort();
|
|
355
|
+
});
|
|
356
|
+
using abortWire = abortWireGuard(turnSignal, controller);
|
|
357
|
+
|
|
358
|
+
const describeCtx: DescribeContext = { energyReader: undefined };
|
|
359
|
+
try {
|
|
360
|
+
const response = await describeAls.run(describeCtx, async () =>
|
|
361
|
+
completeVisionModel(
|
|
362
|
+
visionModel,
|
|
363
|
+
modelRegistry,
|
|
364
|
+
{ systemPrompt, messages: [userMessage] },
|
|
365
|
+
{ apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
|
|
366
|
+
),
|
|
367
|
+
);
|
|
368
|
+
const capture = await readCapture(describeCtx);
|
|
369
|
+
const record = buildUsageRecord(response, capture, visionModel, hashes[0], n > 1 ? hashes : undefined);
|
|
370
|
+
if (record) deps.reportUsage(record);
|
|
371
|
+
if (response.stopReason === "aborted" || response.stopReason === "error") {
|
|
372
|
+
const reason = setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
|
|
373
|
+
if (reason) {
|
|
374
|
+
appendVisionError({
|
|
375
|
+
...base,
|
|
376
|
+
reason,
|
|
377
|
+
stopReason: response.stopReason,
|
|
378
|
+
timedOut,
|
|
379
|
+
timeoutMs,
|
|
380
|
+
errorMessage: response.errorMessage,
|
|
381
|
+
config: configSnapshot(cfg),
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
return null;
|
|
385
|
+
}
|
|
386
|
+
const text = response.content
|
|
387
|
+
.filter((c): c is TextContent => c.type === "text")
|
|
388
|
+
.map((c) => c.text)
|
|
389
|
+
.join("\n")
|
|
390
|
+
.trim();
|
|
391
|
+
if (!text) {
|
|
392
|
+
const reason = "vision model returned an empty description";
|
|
393
|
+
deps.setLastError(reason);
|
|
394
|
+
appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
|
|
395
|
+
return null;
|
|
396
|
+
}
|
|
397
|
+
return await finish(text, response.stopReason === "length");
|
|
398
|
+
} catch (err) {
|
|
399
|
+
const userAborted = abortWire.userAborted();
|
|
400
|
+
const reason = timedOut
|
|
401
|
+
? "describer timed out after " + timeoutMs / 1000 + "s"
|
|
402
|
+
: err instanceof Error
|
|
403
|
+
? err.message
|
|
404
|
+
: String(err);
|
|
405
|
+
deps.setLastError(reason);
|
|
406
|
+
// A deliberate user cancel is not worth logging (no-warn-on-user-abort contract).
|
|
407
|
+
if (!userAborted) {
|
|
408
|
+
appendVisionError({
|
|
409
|
+
...base,
|
|
410
|
+
reason,
|
|
411
|
+
timedOut,
|
|
412
|
+
timeoutMs,
|
|
413
|
+
errorStack: err instanceof Error ? err.stack : undefined,
|
|
414
|
+
config: configSnapshot(cfg),
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
return null;
|
|
418
|
+
} finally {
|
|
419
|
+
// The using guards release the fetch interceptor, timer, and abort wire.
|
|
420
|
+
// Only the energy tee needs an explicit swallow: it rejects when the main
|
|
421
|
+
// stream aborts.
|
|
422
|
+
describeCtx.energyReader?.catch(() => {});
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** Read the energy tee for a describer call, if one was captured. Returns the
|
|
427
|
+
* empty capture when there is no reader (non-Neuralwatt models) or the tee
|
|
428
|
+
* aborted with the main stream. */
|
|
429
|
+
async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
|
|
430
|
+
if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
|
|
431
|
+
try {
|
|
432
|
+
return await describeCtx.energyReader;
|
|
433
|
+
} catch {
|
|
434
|
+
return EMPTY_ENERGY_CAPTURE;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** Translate a non-OK stopReason into a user-facing failure message, unless the
|
|
439
|
+
* abort came from the user cancelling the turn (then stay silent — no warning
|
|
440
|
+
* for a deliberate cancel). Returns the message that was set (so the caller
|
|
441
|
+
* can log it), or null when the abort was a user cancel / nothing was set. */
|
|
442
|
+
function setStopReasonError(
|
|
443
|
+
deps: DescriberDeps,
|
|
444
|
+
stopReason: string,
|
|
445
|
+
errorMessage: string | undefined,
|
|
446
|
+
abortWire: AbortWire,
|
|
447
|
+
timedOut: boolean,
|
|
448
|
+
timeoutMs: number,
|
|
449
|
+
): string | null {
|
|
450
|
+
if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
|
|
451
|
+
if (timedOut) {
|
|
452
|
+
const reason = `describer timed out after ${timeoutMs / 1000}s`;
|
|
453
|
+
deps.setLastError(reason);
|
|
454
|
+
return reason;
|
|
455
|
+
}
|
|
456
|
+
const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
|
|
457
|
+
deps.setLastError(reason);
|
|
458
|
+
return reason;
|
|
459
|
+
}
|