@bismawy/pi-vision-watcher 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +140 -0
- package/package.json +68 -0
- package/src/dataloader.ts +383 -0
- package/src/describer.ts +546 -0
- package/src/dispose.ts +61 -0
- package/src/error-log.ts +131 -0
- package/src/image.ts +331 -0
- package/src/index.ts +580 -0
- package/src/prewarm-editor.ts +95 -0
- package/src/usage.ts +261 -0
- package/src/vision-model-selector.ts +452 -0
- package/vision-watcher.ts +1161 -0
- package/vitest.config.ts +15 -0
package/src/describer.ts
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The vision describer calls a vision-capable model through its registered
|
|
3
|
+
* provider stream, falling back to pi-ai's `completeSimple()` for built-ins.
|
|
4
|
+
*
|
|
5
|
+
* `completeSimple` (not `complete`) is the path that translates the `reasoning`
|
|
6
|
+
* ThinkingLevel into each provider's `reasoningEffort`/budget. `complete()` →
|
|
7
|
+
* `stream()` reads only the pre-translated `reasoningEffort` field and silently
|
|
8
|
+
* drops a bare `reasoning`, so the describer's thinking setting would be a no-op
|
|
9
|
+
* through `complete()` — the bug this swap fixes. The agent loop, SDK, and
|
|
10
|
+
* compaction all route thinking through `completeSimple`/`streamSimple` for the
|
|
11
|
+
* same reason.
|
|
12
|
+
*
|
|
13
|
+
* Two entry points:
|
|
14
|
+
* - {@link runBatch}: ONE batched call describing N images at once (the
|
|
15
|
+
* dataloader's dispatch path). The model is asked to emit one delimited
|
|
16
|
+
* `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
|
|
17
|
+
* split back into per-image descriptions.
|
|
18
|
+
* - {@link describeSingle}: one call for one image (no delimiters). The
|
|
19
|
+
* robust per-image fallback used when a batched response couldn't be split.
|
|
20
|
+
*
|
|
21
|
+
* Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
|
|
22
|
+
* managed with the `using` keyword via the {@link Disposable} guards in
|
|
23
|
+
* `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
|
|
24
|
+
* carried. Disposing is lexical and exception-safe: a thrown `completeSimple()`
|
|
25
|
+
* still tears down the timer, uninstalls the interceptor, and detaches the
|
|
26
|
+
* abort listener.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type {
|
|
30
|
+
Api,
|
|
31
|
+
AssistantMessage,
|
|
32
|
+
Context,
|
|
33
|
+
ImageContent,
|
|
34
|
+
Message,
|
|
35
|
+
Model,
|
|
36
|
+
SimpleStreamOptions,
|
|
37
|
+
TextContent,
|
|
38
|
+
ThinkingLevel,
|
|
39
|
+
} from "@earendil-works/pi-ai";
|
|
40
|
+
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
41
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
42
|
+
import {
|
|
43
|
+
batchUserPrompt,
|
|
44
|
+
DEFAULT_USER_PROMPT_PREFIX,
|
|
45
|
+
DEFAULT_VISION_PROMPT,
|
|
46
|
+
describeTimeoutMs,
|
|
47
|
+
formatModelRef,
|
|
48
|
+
markDescriptionTruncated,
|
|
49
|
+
parseBatchedDescriptions,
|
|
50
|
+
type ExtractedImage,
|
|
51
|
+
type VisionHandoffConfig,
|
|
52
|
+
} from "./index.js";
|
|
53
|
+
import { appendVisionError } from "./error-log.js";
|
|
54
|
+
import {
|
|
55
|
+
buildUsageRecord,
|
|
56
|
+
describeAls,
|
|
57
|
+
EMPTY_ENERGY_CAPTURE,
|
|
58
|
+
type DescribeContext,
|
|
59
|
+
type VisionHandoffEnergyCapture,
|
|
60
|
+
type VisionHandoffUsageRecord,
|
|
61
|
+
} from "./usage.js";
|
|
62
|
+
import { imageHash } from "./image.js";
|
|
63
|
+
import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
|
|
64
|
+
|
|
65
|
+
/** Tokens reserved for the describer's input so the requested output budget
|
|
66
|
+
* can't exceed the model's context window. The describer's input is bounded —
|
|
67
|
+
* one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
|
|
68
|
+
* math simple while leaving generous room for the image + prompt tokens.
|
|
69
|
+
* Providers reject a request when `inputTokens + maxTokens > contextWindow`
|
|
70
|
+
* (e.g. a model whose declared maxTokens equals its full context window), so
|
|
71
|
+
* this clamp prevents that 400 without meaningfully limiting a high-output
|
|
72
|
+
* model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
|
|
73
|
+
const INPUT_RESERVE_TOKENS = 8192;
|
|
74
|
+
|
|
75
|
+
/** Resolve the `maxTokens` to pass to `completeSimple()` for a describer call.
|
|
76
|
+
*
|
|
77
|
+
* - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
|
|
78
|
+
* fit the context window.
|
|
79
|
+
* - Otherwise use the vision model's declared max output (`model.maxTokens`),
|
|
80
|
+
* clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
|
|
81
|
+
* requested output can't exceed the context window minus a reserve for the
|
|
82
|
+
* input. This is the real fix for models whose declared `maxTokens` equals
|
|
83
|
+
* their full `contextWindow` (which providers reject with any non-empty
|
|
84
|
+
* input): we cap at `contextWindow − reserve` instead.
|
|
85
|
+
* - If the model declares no usable max (`maxTokens <= 0`) and no cap is
|
|
86
|
+
* configured, return `undefined` so the provider applies its own default. */
|
|
87
|
+
export function resolveMaxTokens(
|
|
88
|
+
cfg: VisionHandoffConfig,
|
|
89
|
+
visionModel: Model<Api>,
|
|
90
|
+
): number | undefined {
|
|
91
|
+
const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
|
|
92
|
+
const cap = ctx - INPUT_RESERVE_TOKENS;
|
|
93
|
+
const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
|
|
94
|
+
if (requested === undefined) return undefined;
|
|
95
|
+
// Clamp to the context-window-derived ceiling; never below 1.
|
|
96
|
+
return Math.max(1, Math.min(requested, cap));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Resolve the `reasoning` (thinking) level to pass to `completeSimple()`, or
|
|
100
|
+
* `undefined` to leave thinking off.
|
|
101
|
+
*
|
|
102
|
+
* - Returns `undefined` when thinking is disabled in config.
|
|
103
|
+
* - Returns `undefined` when the vision model doesn't declare reasoning
|
|
104
|
+
* support (`model.reasoning === false`) — sending a `reasoning` level to a
|
|
105
|
+
* non-reasoning model would be ignored at best and rejected at worst.
|
|
106
|
+
* - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
|
|
107
|
+
*
|
|
108
|
+
* Mirrors pi core's `createSummarizationOptions` guard so the describer's
|
|
109
|
+
* thinking behaviour matches the rest of pi. */
|
|
110
|
+
export function resolveReasoning(
|
|
111
|
+
cfg: VisionHandoffConfig,
|
|
112
|
+
visionModel: Model<Api>,
|
|
113
|
+
): ThinkingLevel | undefined {
|
|
114
|
+
if (!cfg.thinking) return undefined;
|
|
115
|
+
if (!visionModel.reasoning) return undefined;
|
|
116
|
+
return cfg.thinkingLevel;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Small config snapshot embedded in every error-log entry — the fields most
|
|
120
|
+
* relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
|
|
121
|
+
* reasoning level the model rejects, etc.). */
|
|
122
|
+
function configSnapshot(cfg: VisionHandoffConfig): {
|
|
123
|
+
maxTokens?: number;
|
|
124
|
+
thinking: boolean;
|
|
125
|
+
thinkingLevel: string;
|
|
126
|
+
} {
|
|
127
|
+
return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Complete through the provider registered in Pi's model registry when one
|
|
131
|
+
* supplies a custom stream. The post-0.80 ModelRuntime keeps extension streams
|
|
132
|
+
* out of pi-ai's deprecated global compatibility registry, so calling
|
|
133
|
+
* completeSimple() directly cannot resolve custom API ids such as `makora`.
|
|
134
|
+
* The optional access preserves compatibility with older ModelRegistry versions,
|
|
135
|
+
* which registered custom streams globally and do not expose the config getter. */
|
|
136
|
+
const NEURALWATT_CONVERSATION_HEADER = "x-nw-conversation-id";
|
|
137
|
+
|
|
138
|
+
function isolateVisionRequest(
|
|
139
|
+
model: Model<Api>,
|
|
140
|
+
options: SimpleStreamOptions,
|
|
141
|
+
): SimpleStreamOptions {
|
|
142
|
+
// Stable per-model identity (not randomUUID): every describer call shares the
|
|
143
|
+
// same system prompt (DEFAULT_VISION_PROMPT), so a stable session id lets the
|
|
144
|
+
// provider's prompt cache reuse that prefix across calls (cacheRead/cacheWrite
|
|
145
|
+
// > 0) instead of re-billing it every turn. Still distinct from the main
|
|
146
|
+
// agent's conversation id (the inherited header we replace), so unrelated
|
|
147
|
+
// image prompts never become the main agent's newest cache lineage. A
|
|
148
|
+
// per-model id keeps different vision models (fallback chains) on separate
|
|
149
|
+
// lineages.
|
|
150
|
+
const sessionId = `pi-vision-watcher:${model.provider}:${model.id}`;
|
|
151
|
+
const headers = { ...(options.headers ?? {}) };
|
|
152
|
+
const inheritedConversationHeader = Object.keys(headers).find(
|
|
153
|
+
(name) => name.toLowerCase() === NEURALWATT_CONVERSATION_HEADER,
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
if (inheritedConversationHeader) {
|
|
157
|
+
headers[inheritedConversationHeader] = sessionId;
|
|
158
|
+
} else if (model.provider.toLowerCase() === "neuralwatt") {
|
|
159
|
+
headers["X-NW-Conversation-ID"] = sessionId;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return {
|
|
163
|
+
...options,
|
|
164
|
+
sessionId,
|
|
165
|
+
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export async function completeVisionModel(
|
|
170
|
+
model: Model<Api>,
|
|
171
|
+
modelRegistry: ModelRegistry,
|
|
172
|
+
context: Context,
|
|
173
|
+
options: SimpleStreamOptions,
|
|
174
|
+
): Promise<AssistantMessage> {
|
|
175
|
+
const isolatedOptions = isolateVisionRequest(model, options);
|
|
176
|
+
const provider = modelRegistry.getRegisteredProviderConfig?.(model.provider);
|
|
177
|
+
if (provider?.streamSimple && provider.api === model.api) {
|
|
178
|
+
return provider.streamSimple(model, context, isolatedOptions).result();
|
|
179
|
+
}
|
|
180
|
+
return completeSimple(model, context, isolatedOptions);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Dependencies the describer can't own itself (held by the engine). */
|
|
184
|
+
export interface DescriberDeps {
|
|
185
|
+
/** Report a usage+energy record for one real describer call (cache hits emit none). */
|
|
186
|
+
reportUsage(record: VisionHandoffUsageRecord): void;
|
|
187
|
+
/** Set the most-recent describer failure message (surfaced to the user by the engine).
|
|
188
|
+
* Pass `null` to clear before a fresh attempt. */
|
|
189
|
+
setLastError(msg: string | null): void;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
|
|
193
|
+
export type BatchResult = Map<string, string>;
|
|
194
|
+
|
|
195
|
+
/** Describe N images with ONE batched `completeSimple()` call and split the response
|
|
196
|
+
* back into per-image descriptions. Returns a map keyed by image hash; an
|
|
197
|
+
* image whose section failed to parse is omitted (the caller treats omission
|
|
198
|
+
* as "description unavailable"). On a genuine call failure (auth, abort,
|
|
199
|
+
* empty) the map is empty and `deps.setLastError` records the reason.
|
|
200
|
+
*
|
|
201
|
+
* A failed delimiter-parse is a failed BATCH, not a failed description: the
|
|
202
|
+
* unparsed images fall back to parallel single-image calls (no delimiters to
|
|
203
|
+
* cooperate with), so the common cooperative case stays one call while an
|
|
204
|
+
* uncooperative model still gets every image described. */
|
|
205
|
+
export async function runBatch(
|
|
206
|
+
misses: { img: ExtractedImage; hash: string }[],
|
|
207
|
+
userPrompt: string,
|
|
208
|
+
visionModel: Model<Api>,
|
|
209
|
+
modelRegistry: ModelRegistry,
|
|
210
|
+
cfg: VisionHandoffConfig,
|
|
211
|
+
deps: DescriberDeps,
|
|
212
|
+
turnSignal?: AbortSignal,
|
|
213
|
+
): Promise<BatchResult> {
|
|
214
|
+
const out: BatchResult = new Map();
|
|
215
|
+
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
216
|
+
if (!auth.ok || !auth.apiKey) {
|
|
217
|
+
const reason = !auth.ok
|
|
218
|
+
? auth.error
|
|
219
|
+
: `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
|
|
220
|
+
deps.setLastError(reason);
|
|
221
|
+
appendVisionError({
|
|
222
|
+
phase: "batch",
|
|
223
|
+
reason,
|
|
224
|
+
visionModel: cfg.visionModel,
|
|
225
|
+
imageHashes: misses.map((m) => m.hash),
|
|
226
|
+
imageCount: misses.length,
|
|
227
|
+
config: configSnapshot(cfg),
|
|
228
|
+
});
|
|
229
|
+
return out;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
|
|
233
|
+
const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
|
|
234
|
+
const content: (TextContent | ImageContent)[] = [
|
|
235
|
+
{ type: "text", text: batchUserPrompt(misses.length, userPrompt, prefix) },
|
|
236
|
+
...misses.map((m) => ({ type: "image", data: m.img.data, mimeType: m.img.mimeType } satisfies ImageContent)),
|
|
237
|
+
];
|
|
238
|
+
const userMessage: Message = { role: "user", content, timestamp: Date.now() };
|
|
239
|
+
|
|
240
|
+
const timeoutMs = describeTimeoutMs(misses.length, cfg.describeTimeoutMs);
|
|
241
|
+
const maxTokens = resolveMaxTokens(cfg, visionModel);
|
|
242
|
+
const reasoning = resolveReasoning(cfg, visionModel);
|
|
243
|
+
const controller = new AbortController();
|
|
244
|
+
let timedOut = false;
|
|
245
|
+
using fetchGuard = fetchInterceptorGuard();
|
|
246
|
+
using timer = timeoutGuard(timeoutMs, () => {
|
|
247
|
+
timedOut = true;
|
|
248
|
+
controller.abort();
|
|
249
|
+
});
|
|
250
|
+
using abortWire = abortWireGuard(turnSignal, controller);
|
|
251
|
+
|
|
252
|
+
const describeCtx: DescribeContext = { energyReader: undefined };
|
|
253
|
+
try {
|
|
254
|
+
const response = await describeAls.run(describeCtx, async () =>
|
|
255
|
+
completeVisionModel(
|
|
256
|
+
visionModel,
|
|
257
|
+
modelRegistry,
|
|
258
|
+
{ systemPrompt, messages: [userMessage] },
|
|
259
|
+
{ apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
|
|
260
|
+
),
|
|
261
|
+
);
|
|
262
|
+
const capture = await readCapture(describeCtx);
|
|
263
|
+
const hashes = misses.map((m) => m.hash);
|
|
264
|
+
const record = buildUsageRecord(response, capture, visionModel, hashes[0], hashes.length > 1 ? hashes : undefined);
|
|
265
|
+
if (record) deps.reportUsage(record);
|
|
266
|
+
if (response.stopReason === "aborted" || response.stopReason === "error") {
|
|
267
|
+
const reason = setStopReasonError(
|
|
268
|
+
deps,
|
|
269
|
+
response.stopReason,
|
|
270
|
+
response.errorMessage,
|
|
271
|
+
abortWire,
|
|
272
|
+
timedOut,
|
|
273
|
+
timeoutMs,
|
|
274
|
+
);
|
|
275
|
+
if (reason) {
|
|
276
|
+
appendVisionError({
|
|
277
|
+
phase: "batch",
|
|
278
|
+
reason,
|
|
279
|
+
visionModel: cfg.visionModel,
|
|
280
|
+
imageHashes: hashes,
|
|
281
|
+
imageCount: misses.length,
|
|
282
|
+
stopReason: response.stopReason,
|
|
283
|
+
timedOut,
|
|
284
|
+
timeoutMs,
|
|
285
|
+
errorMessage: response.errorMessage,
|
|
286
|
+
config: configSnapshot(cfg),
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
return out;
|
|
290
|
+
}
|
|
291
|
+
const text = response.content
|
|
292
|
+
.filter((c): c is TextContent => c.type === "text")
|
|
293
|
+
.map((c) => c.text)
|
|
294
|
+
.join("\n")
|
|
295
|
+
.trim();
|
|
296
|
+
if (!text) {
|
|
297
|
+
deps.setLastError("vision model returned an empty description");
|
|
298
|
+
appendVisionError({
|
|
299
|
+
phase: "batch",
|
|
300
|
+
reason: "vision model returned an empty description",
|
|
301
|
+
visionModel: cfg.visionModel,
|
|
302
|
+
imageHashes: hashes,
|
|
303
|
+
imageCount: misses.length,
|
|
304
|
+
config: configSnapshot(cfg),
|
|
305
|
+
});
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
const parsed = parseBatchedDescriptions(text, misses.length);
|
|
309
|
+
for (let i = 0; i < misses.length; i++) {
|
|
310
|
+
const d = parsed[i];
|
|
311
|
+
if (d) out.set(misses[i].hash, d);
|
|
312
|
+
}
|
|
313
|
+
// stopReason "length" = the batch response was cut off mid-stream. The last
|
|
314
|
+
// image being emitted when the cap hit is the one whose section is truncated
|
|
315
|
+
// (parseBatchedDescriptions still captures it via the end-of-text lookahead,
|
|
316
|
+
// just with partial content). Mark that one — the highest-index parsed slot —
|
|
317
|
+
// so the agent/user know it's incomplete. Earlier sections had `<<<END>>>`
|
|
318
|
+
// markers and are complete. We deliberately do NOT re-describe the truncated
|
|
319
|
+
// image: a single call for the same complex image would likely hit the same
|
|
320
|
+
// limit, so the partial text + marker is the honest, no-wasted-call result.
|
|
321
|
+
if (response.stopReason === "length") {
|
|
322
|
+
for (let i = misses.length - 1; i >= 0; i--) {
|
|
323
|
+
const h = misses[i].hash;
|
|
324
|
+
if (out.has(h)) {
|
|
325
|
+
out.set(h, markDescriptionTruncated(out.get(h)!));
|
|
326
|
+
break;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
// Fallback: a failed delimiter-parse is NOT a failed description — it's a
|
|
331
|
+
// failed batching. Describe each unparsed image with its own single-image
|
|
332
|
+
// call (no delimiters to cooperate with). The calls run in parallel so
|
|
333
|
+
// their results still arrive together, not sequentially.
|
|
334
|
+
const unparsed = misses.filter((m) => !out.has(m.hash));
|
|
335
|
+
if (unparsed.length > 0) {
|
|
336
|
+
const fallbacks = await Promise.all(
|
|
337
|
+
unparsed.map((m) => describeSingle(m.img, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal)),
|
|
338
|
+
);
|
|
339
|
+
for (let i = 0; i < unparsed.length; i++) {
|
|
340
|
+
if (fallbacks[i]) out.set(unparsed[i].hash, fallbacks[i]!);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return out;
|
|
344
|
+
} catch (err) {
|
|
345
|
+
const userAborted = abortWire.userAborted();
|
|
346
|
+
const reason = timedOut
|
|
347
|
+
? `describer timed out after ${timeoutMs / 1000}s`
|
|
348
|
+
: err instanceof Error
|
|
349
|
+
? err.message
|
|
350
|
+
: String(err);
|
|
351
|
+
deps.setLastError(reason);
|
|
352
|
+
// A deliberate user cancel isn't a troubleshooting-worthy error — skip
|
|
353
|
+
// logging it (mirroring the no-warn-on-user-abort contract).
|
|
354
|
+
if (!userAborted) {
|
|
355
|
+
appendVisionError({
|
|
356
|
+
phase: "batch",
|
|
357
|
+
reason,
|
|
358
|
+
visionModel: cfg.visionModel,
|
|
359
|
+
imageHashes: misses.map((m) => m.hash),
|
|
360
|
+
imageCount: misses.length,
|
|
361
|
+
timedOut,
|
|
362
|
+
timeoutMs,
|
|
363
|
+
errorStack: err instanceof Error ? err.stack : undefined,
|
|
364
|
+
config: configSnapshot(cfg),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
return out;
|
|
368
|
+
} finally {
|
|
369
|
+
// The `using` guards above already released the fetch interceptor, timer,
|
|
370
|
+
// and abort wire. Only the energy tee needs an explicit unhandled-rejection
|
|
371
|
+
// swallow: if the main stream aborted, the tee rejects too.
|
|
372
|
+
describeCtx.energyReader?.catch(() => {});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Describe a single image with one `completeSimple()` call and return the RAW
|
|
377
|
+
* description (no envelope, no truncation). Returns null on any genuine
|
|
378
|
+
* failure (auth, abort/error, empty) so the caller only caches `UNAVAILABLE`
|
|
379
|
+
* when a real describer attempt failed. */
|
|
380
|
+
export async function describeSingle(
|
|
381
|
+
img: ExtractedImage,
|
|
382
|
+
userPrompt: string,
|
|
383
|
+
visionModel: Model<Api>,
|
|
384
|
+
modelRegistry: ModelRegistry,
|
|
385
|
+
cfg: VisionHandoffConfig,
|
|
386
|
+
deps: DescriberDeps,
|
|
387
|
+
turnSignal?: AbortSignal,
|
|
388
|
+
): Promise<string | null> {
|
|
389
|
+
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
390
|
+
if (!auth.ok || !auth.apiKey) {
|
|
391
|
+
const reason = !auth.ok
|
|
392
|
+
? auth.error
|
|
393
|
+
: `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
|
|
394
|
+
deps.setLastError(reason);
|
|
395
|
+
appendVisionError({
|
|
396
|
+
phase: "single",
|
|
397
|
+
reason,
|
|
398
|
+
visionModel: cfg.visionModel,
|
|
399
|
+
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
400
|
+
imageCount: 1,
|
|
401
|
+
config: configSnapshot(cfg),
|
|
402
|
+
});
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
|
|
406
|
+
const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
|
|
407
|
+
const content: (TextContent | ImageContent)[] = [
|
|
408
|
+
{ type: "text", text: batchUserPrompt(1, userPrompt, prefix) },
|
|
409
|
+
{ type: "image", data: img.data, mimeType: img.mimeType } satisfies ImageContent,
|
|
410
|
+
];
|
|
411
|
+
const userMessage: Message = { role: "user", content, timestamp: Date.now() };
|
|
412
|
+
const timeoutMs = describeTimeoutMs(1, cfg.describeTimeoutMs);
|
|
413
|
+
const maxTokens = resolveMaxTokens(cfg, visionModel);
|
|
414
|
+
const reasoning = resolveReasoning(cfg, visionModel);
|
|
415
|
+
const controller = new AbortController();
|
|
416
|
+
let timedOut = false;
|
|
417
|
+
using fetchGuard = fetchInterceptorGuard();
|
|
418
|
+
using timer = timeoutGuard(timeoutMs, () => {
|
|
419
|
+
timedOut = true;
|
|
420
|
+
controller.abort();
|
|
421
|
+
});
|
|
422
|
+
using abortWire = abortWireGuard(turnSignal, controller);
|
|
423
|
+
|
|
424
|
+
const describeCtx: DescribeContext = { energyReader: undefined };
|
|
425
|
+
try {
|
|
426
|
+
const response = await describeAls.run(describeCtx, async () =>
|
|
427
|
+
completeVisionModel(
|
|
428
|
+
visionModel,
|
|
429
|
+
modelRegistry,
|
|
430
|
+
{ systemPrompt, messages: [userMessage] },
|
|
431
|
+
{ apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
|
|
432
|
+
),
|
|
433
|
+
);
|
|
434
|
+
const capture = await readCapture(describeCtx);
|
|
435
|
+
const hash = imageHash(img.mimeType, img.data);
|
|
436
|
+
const record = buildUsageRecord(response, capture, visionModel, hash);
|
|
437
|
+
if (record) deps.reportUsage(record);
|
|
438
|
+
if (response.stopReason === "aborted" || response.stopReason === "error") {
|
|
439
|
+
const reason = setStopReasonError(
|
|
440
|
+
deps,
|
|
441
|
+
response.stopReason,
|
|
442
|
+
response.errorMessage,
|
|
443
|
+
abortWire,
|
|
444
|
+
timedOut,
|
|
445
|
+
timeoutMs,
|
|
446
|
+
);
|
|
447
|
+
if (reason) {
|
|
448
|
+
appendVisionError({
|
|
449
|
+
phase: "single",
|
|
450
|
+
reason,
|
|
451
|
+
visionModel: cfg.visionModel,
|
|
452
|
+
imageHashes: [hash],
|
|
453
|
+
imageCount: 1,
|
|
454
|
+
stopReason: response.stopReason,
|
|
455
|
+
timedOut,
|
|
456
|
+
timeoutMs,
|
|
457
|
+
errorMessage: response.errorMessage,
|
|
458
|
+
config: configSnapshot(cfg),
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
const text = response.content
|
|
464
|
+
.filter((c): c is TextContent => c.type === "text")
|
|
465
|
+
.map((c) => c.text)
|
|
466
|
+
.join("\n")
|
|
467
|
+
.trim();
|
|
468
|
+
if (!text) {
|
|
469
|
+
deps.setLastError("vision model returned an empty description");
|
|
470
|
+
appendVisionError({
|
|
471
|
+
phase: "single",
|
|
472
|
+
reason: "vision model returned an empty description",
|
|
473
|
+
visionModel: cfg.visionModel,
|
|
474
|
+
imageHashes: [hash],
|
|
475
|
+
imageCount: 1,
|
|
476
|
+
config: configSnapshot(cfg),
|
|
477
|
+
});
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
// stopReason "length" = the model hit a token limit (configured maxTokens or
|
|
481
|
+
// the provider's hard output cap) before finishing. The partial text is still
|
|
482
|
+
// useful, but it must not pass as complete — mark it so the agent/user know.
|
|
483
|
+
return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
|
|
484
|
+
} catch (err) {
|
|
485
|
+
const userAborted = abortWire.userAborted();
|
|
486
|
+
const reason = timedOut
|
|
487
|
+
? `describer timed out after ${timeoutMs / 1000}s`
|
|
488
|
+
: err instanceof Error
|
|
489
|
+
? err.message
|
|
490
|
+
: String(err);
|
|
491
|
+
deps.setLastError(reason);
|
|
492
|
+
// A deliberate user cancel isn't a troubleshooting-worthy error — skip
|
|
493
|
+
// logging it (mirroring the no-warn-on-user-abort contract).
|
|
494
|
+
if (!userAborted) {
|
|
495
|
+
appendVisionError({
|
|
496
|
+
phase: "single",
|
|
497
|
+
reason,
|
|
498
|
+
visionModel: cfg.visionModel,
|
|
499
|
+
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
500
|
+
imageCount: 1,
|
|
501
|
+
timedOut,
|
|
502
|
+
timeoutMs,
|
|
503
|
+
errorStack: err instanceof Error ? err.stack : undefined,
|
|
504
|
+
config: configSnapshot(cfg),
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
return null;
|
|
508
|
+
} finally {
|
|
509
|
+
describeCtx.energyReader?.catch(() => {});
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** Read the energy tee for a describer call, if one was captured. Returns the
|
|
514
|
+
* empty capture when there is no reader (non-Neuralwatt models) or the tee
|
|
515
|
+
* aborted with the main stream. */
|
|
516
|
+
async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
|
|
517
|
+
if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
|
|
518
|
+
try {
|
|
519
|
+
return await describeCtx.energyReader;
|
|
520
|
+
} catch {
|
|
521
|
+
return EMPTY_ENERGY_CAPTURE;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/** Translate a non-OK stopReason into a user-facing failure message, unless the
|
|
526
|
+
* abort came from the user cancelling the turn (then stay silent — no warning
|
|
527
|
+
* for a deliberate cancel). Returns the message that was set (so the caller
|
|
528
|
+
* can log it), or null when the abort was a user cancel / nothing was set. */
|
|
529
|
+
function setStopReasonError(
|
|
530
|
+
deps: DescriberDeps,
|
|
531
|
+
stopReason: string,
|
|
532
|
+
errorMessage: string | undefined,
|
|
533
|
+
abortWire: AbortWire,
|
|
534
|
+
timedOut: boolean,
|
|
535
|
+
timeoutMs: number,
|
|
536
|
+
): string | null {
|
|
537
|
+
if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
|
|
538
|
+
if (timedOut) {
|
|
539
|
+
const reason = `describer timed out after ${timeoutMs / 1000}s`;
|
|
540
|
+
deps.setLastError(reason);
|
|
541
|
+
return reason;
|
|
542
|
+
}
|
|
543
|
+
const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
|
|
544
|
+
deps.setLastError(reason);
|
|
545
|
+
return reason;
|
|
546
|
+
}
|
package/src/dispose.ts
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Disposable` guard factories for the `using` keyword (Explicit Resource
|
|
3
|
+
* Management). Each factory acquires a resource and returns a `Disposable`
|
|
4
|
+
* whose `[Symbol.dispose]` releases it, so a `using` binding replaces a
|
|
5
|
+
* manual acquire/`try`/`finally`/release pair.
|
|
6
|
+
*
|
|
7
|
+
* Used by the describer to bundle the fetch interceptor, the timeout timer,
|
|
8
|
+
* and the turn-abort wire into lexical scopes whose cleanup is automatic.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { installFetchInterceptor, uninstallFetchInterceptor } from "./usage.js";
|
|
12
|
+
|
|
13
|
+
/** A `Disposable` that uninstalls the refcounted fetch interceptor on release. */
|
|
14
|
+
export function fetchInterceptorGuard(): Disposable {
|
|
15
|
+
installFetchInterceptor();
|
|
16
|
+
return {
|
|
17
|
+
[Symbol.dispose]() {
|
|
18
|
+
uninstallFetchInterceptor();
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A `Disposable` that clears a `setTimeout` handle on release. */
|
|
24
|
+
export function timeoutGuard(ms: number, onTimeout: () => void): Disposable {
|
|
25
|
+
const handle = setTimeout(onTimeout, ms);
|
|
26
|
+
return {
|
|
27
|
+
[Symbol.dispose]() {
|
|
28
|
+
clearTimeout(handle);
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A `Disposable` abort-wire: propagates a turn abort signal into a
|
|
34
|
+
* describer's `AbortController`, and detaches the listener on release.
|
|
35
|
+
* Also exposes `userAborted()` so the describer can tell a deliberate user
|
|
36
|
+
* cancel apart from a provider/timeout abort (to suppress spurious warnings).
|
|
37
|
+
*
|
|
38
|
+
* Always returns a `Disposable` (a no-op when there is no turn signal) so a
|
|
39
|
+
* `using` binding never has to null-check. */
|
|
40
|
+
export interface AbortWire extends Disposable {
|
|
41
|
+
/** True iff the abort originated from the turn signal (a user cancel). */
|
|
42
|
+
userAborted(): boolean;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function abortWireGuard(turnSignal: AbortSignal | undefined, controller: AbortController): AbortWire {
|
|
46
|
+
if (!turnSignal) {
|
|
47
|
+
return {
|
|
48
|
+
[Symbol.dispose]() {},
|
|
49
|
+
userAborted: () => false,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
const onAbort = () => controller.abort();
|
|
53
|
+
if (turnSignal.aborted) controller.abort();
|
|
54
|
+
else turnSignal.addEventListener("abort", onAbort, { once: true });
|
|
55
|
+
return {
|
|
56
|
+
[Symbol.dispose]() {
|
|
57
|
+
turnSignal.removeEventListener("abort", onAbort);
|
|
58
|
+
},
|
|
59
|
+
userAborted: () => turnSignal.aborted,
|
|
60
|
+
};
|
|
61
|
+
}
|