@bismawy/pi-vision-watcher 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +5 -4
- package/package.json +1 -1
- package/src/dataloader.ts +389 -389
- package/src/describer.ts +459 -561
- package/src/vision-model-selector.ts +153 -96
- package/vision-watcher.ts +1 -0
package/src/dataloader.ts
CHANGED
|
@@ -1,389 +1,389 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Facebook DataLoader pattern for image descriptions.
|
|
3
|
-
*
|
|
4
|
-
* `loadDescription(img)` returns a memoized Promise for the description and
|
|
5
|
-
* pushes the image's key (hash) into the CURRENT batch. All `load()` calls in
|
|
6
|
-
* the same execution frame (+ its microtask cascade) coalesce into ONE batch
|
|
7
|
-
* object. Dispatch is scheduled via `setImmediate` (see `enqueuePostPromiseJob`),
|
|
8
|
-
* so every load in the frame — and every load from a separate I/O callback in
|
|
9
|
-
* the same poll iteration — lands in the single batch before the ONE vision
|
|
10
|
-
* call fires. Each load()'s promise then resolves with its description.
|
|
11
|
-
*
|
|
12
|
-
* Mapping: the `read` tools are the `load()` callers. Their `tool_result`
|
|
13
|
-
* handler awaits the shared batch — so N parallel reads coalesce into the SAME
|
|
14
|
-
* single vision call and all resolve together, the descriptions landing in the
|
|
15
|
-
* tool results BEFORE the agent's next turn. pi fires each read's `tool_result`
|
|
16
|
-
* as that read's I/O completes (poll phase); the loader's `setImmediate`
|
|
17
|
-
* dispatch defers to the check phase, AFTER the whole poll iteration, so reads
|
|
18
|
-
* completing together (the common case for cached local files) land in ONE
|
|
19
|
-
* batch — and reads completing in separate iterations get separate calls, but
|
|
20
|
-
* always in parallel, never sequential. The agent's tool-result wait is free
|
|
21
|
-
* time, so this adds zero latency to the critical path. `context` then sees
|
|
22
|
-
* text-described tool results (no image blocks to swap); any remaining images
|
|
23
|
-
* (user-attached, custom-injected) are cache hits.
|
|
24
|
-
*
|
|
25
|
-
* All mutable state (batch, cache, turn context) lives on the instance — no
|
|
26
|
-
* module-level globals — and the class implements `Disposable` so a `using`
|
|
27
|
-
* binding (or an explicit `reset()`) cleanly abandons an in-flight batch and
|
|
28
|
-
* clears turn context, e.g. on session reset.
|
|
29
|
-
*/
|
|
30
|
-
|
|
31
|
-
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
32
|
-
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
33
|
-
import {
|
|
34
|
-
IMAGE_PLACEHOLDER_PREFIX,
|
|
35
|
-
IMAGE_PLACEHOLDER_SUFFIX,
|
|
36
|
-
wrapDescription,
|
|
37
|
-
type ExtractedImage,
|
|
38
|
-
type VisionHandoffConfig,
|
|
39
|
-
} from "./index.js";
|
|
40
|
-
import { imageHash } from "./image.js";
|
|
41
|
-
import { runBatch,
|
|
42
|
-
|
|
43
|
-
/** Resolved when a description couldn't be obtained (graceful degradation).
|
|
44
|
-
* Failures are NOT cached, so the next turn re-attempts. */
|
|
45
|
-
export const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${IMAGE_PLACEHOLDER_SUFFIX}`;
|
|
46
|
-
|
|
47
|
-
interface DescriptionBatch {
|
|
48
|
-
keys: string[];
|
|
49
|
-
imgs: ExtractedImage[];
|
|
50
|
-
/** One per `loadDescription()` call. A duplicate load (same hash, but its
|
|
51
|
-
* first cache entry was evicted mid-frame so it couldn't short-circuit on
|
|
52
|
-
* the cache) pushes a second callback for an existing key — so
|
|
53
|
-
* `callbacks.length` can exceed `keys.length`. `dispatchBatch` resolves by
|
|
54
|
-
* hash so every callback is reached. */
|
|
55
|
-
callbacks: { hash: string; resolve: (v: string) => void; reject: (e: Error) => void }[];
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
/** Engine-provided resolver for the configured vision model. */
|
|
59
|
-
export interface VisionModelResolver {
|
|
60
|
-
(registry: ModelRegistry, ref: string): Model<Api> | null;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/** Dependencies the loader can't own itself (held by the engine). */
|
|
64
|
-
export interface LoaderDeps extends DescriberDeps {
|
|
65
|
-
/** Read the current config (reloaded on session_start / config writes). */
|
|
66
|
-
getConfig(): VisionHandoffConfig;
|
|
67
|
-
/** Resolve the configured vision model against a registry. */
|
|
68
|
-
resolveVisionModel: VisionModelResolver;
|
|
69
|
-
/** Backoff (ms) before retrying a totally-failed describer batch. Defaults
|
|
70
|
-
* to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
|
|
71
|
-
retryBackoffMs?: number;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** Defer `fn` to the next check phase (`setImmediate`), so every
|
|
75
|
-
* `loadDescription()` — whether called from sync code, a microtask cascade,
|
|
76
|
-
* or a separate I/O callback in the same poll iteration — coalesces into one
|
|
77
|
-
* batch before the single vision call fires.
|
|
78
|
-
*
|
|
79
|
-
* Why `setImmediate` and not DataLoader's classic `process.nextTick`:
|
|
80
|
-
* nextTick drains between I/O callbacks in the poll phase, so dispatch would
|
|
81
|
-
* fire after the first parallel `read`'s `tool_result` but before the
|
|
82
|
-
* second's — splitting N reads into N single-image calls. `setImmediate` runs
|
|
83
|
-
* in the check phase, AFTER the whole poll phase, so all `tool_result`
|
|
84
|
-
* handlers that fire in one poll iteration land in ONE batch. The check phase
|
|
85
|
-
* also runs after the microtask queue drains, so loads issued from a `.then`
|
|
86
|
-
* cascade (e.g. the clipboard pre-warm) still coalesce. */
|
|
87
|
-
function enqueuePostPromiseJob(fn: () => void): void {
|
|
88
|
-
setImmediate(fn);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** Default backoff (ms) before retrying a totally-failed describer batch. A
|
|
92
|
-
* short backoff covers a transient provider blip (network hiccup, momentary
|
|
93
|
-
* 429) without adding meaningful latency to the tool-result phase (free
|
|
94
|
-
* time). Generous enough to let a rate-limited provider recover, short enough
|
|
95
|
-
* that a genuinely broken vision model doesn't stall a turn. */
|
|
96
|
-
const DESCRIBE_RETRY_BACKOFF_MS = 500;
|
|
97
|
-
|
|
98
|
-
/** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
|
|
99
|
-
* during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
|
|
100
|
-
* wait the full backoff before the retry is skipped. */
|
|
101
|
-
function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
102
|
-
return new Promise((resolve) => {
|
|
103
|
-
if (ms <= 0 || signal.aborted) return resolve();
|
|
104
|
-
const onAbort = () => {
|
|
105
|
-
clearTimeout(timer);
|
|
106
|
-
resolve();
|
|
107
|
-
};
|
|
108
|
-
const timer = setTimeout(() => {
|
|
109
|
-
signal.removeEventListener("abort", onAbort);
|
|
110
|
-
resolve();
|
|
111
|
-
}, ms);
|
|
112
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
export class DescriptionLoader implements Disposable {
|
|
117
|
-
private readonly cache = new Map<string, Promise<string>>();
|
|
118
|
-
private batch: DescriptionBatch | null = null;
|
|
119
|
-
private dispatchScheduled = false;
|
|
120
|
-
private turnModelRegistry: ModelRegistry | null = null;
|
|
121
|
-
private turnVisionModel: Model<Api> | null = null;
|
|
122
|
-
/** Turn-level abort controller. In-flight describer batches (`runBatch`)
|
|
123
|
-
* are wired to `turnAbortController.signal` so a user cancel (ESC) aborts
|
|
124
|
-
* them EVEN WHEN the batch was dispatched before the run's live abort
|
|
125
|
-
* signal existed — the `before_agent_start` / paste-time prewarm case,
|
|
126
|
-
* where `ctx.signal` is `undefined` (the agent run hasn't started yet).
|
|
127
|
-
* The run's live signal, once it arrives via `bindTurnContext`, is forwarded
|
|
128
|
-
* into `turnAbortController.abort()`. Because `runBatch` holds a reference
|
|
129
|
-
* to this controller's signal (not a snapshot of `ctx.signal`), a batch
|
|
130
|
-
* that started during prewarm becomes abortable the instant a later
|
|
131
|
-
* `bindTurnContext` (from `tool_result`/`context`) brings the live signal.
|
|
132
|
-
* Reset per turn via {@link resetTurnAbort} so a previous turn's cancel
|
|
133
|
-
* can't poison the next turn's prewarm. */
|
|
134
|
-
private turnAbortController = new AbortController();
|
|
135
|
-
/** The run signal currently forwarded into {@link turnAbortController}, so
|
|
136
|
-
* the abort listener is attached at most once per signal (the same signal
|
|
137
|
-
* is bound by every `tool_result`/`context` event in a turn). */
|
|
138
|
-
private wiredSignal: AbortSignal | undefined;
|
|
139
|
-
private pendingTurnPrompt = "";
|
|
140
|
-
|
|
141
|
-
constructor(private readonly deps: LoaderDeps) {}
|
|
142
|
-
|
|
143
|
-
/** Bind the turn context (model registry, resolved vision model, abort
|
|
144
|
-
* signal) for the loader's next dispatch. Called from every handler that
|
|
145
|
-
* may trigger `loadDescription()`.
|
|
146
|
-
*
|
|
147
|
-
* The abort signal is forwarded into the loader's {@link turnAbortController}
|
|
148
|
-
* rather than stored directly. Storing `ctx.signal` directly would leave a
|
|
149
|
-
* prewarm-dispatched batch (started in `before_agent_start`, where
|
|
150
|
-
* `ctx.signal` is `undefined` because the run hasn't started) with no abort
|
|
151
|
-
* wire — ESC couldn't cancel it, so the `tool_result` handler would only
|
|
152
|
-
* discard the result AFTER the vision call ran to completion. Forwarding
|
|
153
|
-
* the live signal into a stable, loader-owned controller lets an in-flight
|
|
154
|
-
* prewarm batch be aborted the moment the live signal arrives. */
|
|
155
|
-
bindTurnContext(ctx: { modelRegistry: ModelRegistry; signal?: AbortSignal }): void {
|
|
156
|
-
this.turnModelRegistry = ctx.modelRegistry;
|
|
157
|
-
const cfg = this.deps.getConfig();
|
|
158
|
-
const resolved = this.deps.resolveVisionModel(ctx.modelRegistry, cfg.visionModel!);
|
|
159
|
-
if (resolved) {
|
|
160
|
-
this.turnVisionModel = resolved;
|
|
161
|
-
} else {
|
|
162
|
-
// Primary unresolvable (typo, removed model): pick the first resolvable
|
|
163
|
-
// fallback so failover still works instead of leaving the loader dead.
|
|
164
|
-
// Call-time failover in dispatchBatch covers the case where the primary
|
|
165
|
-
// RESOLVES but fails at call time.
|
|
166
|
-
for (const ref of cfg.fallbackModels ?? []) {
|
|
167
|
-
const fb = this.deps.resolveVisionModel(ctx.modelRegistry, ref);
|
|
168
|
-
if (fb) {
|
|
169
|
-
this.turnVisionModel = fb;
|
|
170
|
-
break;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
const signal = ctx.signal;
|
|
175
|
-
if (!signal) return;
|
|
176
|
-
if (signal.aborted) {
|
|
177
|
-
this.turnAbortController.abort();
|
|
178
|
-
this.wiredSignal = signal;
|
|
179
|
-
return;
|
|
180
|
-
}
|
|
181
|
-
if (signal === this.wiredSignal) return; // already forwarded this signal
|
|
182
|
-
this.wiredSignal = signal;
|
|
183
|
-
signal.addEventListener("abort", () => this.turnAbortController.abort(), { once: true });
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/** Reset the turn-level abort controller for a fresh turn. Call at turn
|
|
187
|
-
* boundaries (`before_agent_start`, paste-time prewarm) so a previous turn's
|
|
188
|
-
* cancel doesn't leave {@link turnAbortController} aborted — which would
|
|
189
|
-
* make every subsequent dispatch short-circuit to UNAVAILABLE. A
|
|
190
|
-
* non-aborted controller is reused (avoids orphaning an in-flight paste
|
|
191
|
-
* prewarm holding its signal); only an aborted one is replaced. `wiredSignal`
|
|
192
|
-
* is always cleared so the next live signal re-wires. Safe at a real turn
|
|
193
|
-
* boundary, where the prior turn's batch has settled. */
|
|
194
|
-
resetTurnAbort(): void {
|
|
195
|
-
if (this.turnAbortController.signal.aborted) {
|
|
196
|
-
this.turnAbortController = new AbortController();
|
|
197
|
-
}
|
|
198
|
-
this.wiredSignal = undefined;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/** Capture this turn's user prompt so every image in the turn is described in
|
|
202
|
-
* the same request context. */
|
|
203
|
-
setPendingTurnPrompt(prompt: string): void {
|
|
204
|
-
this.pendingTurnPrompt = prompt;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/** Load an image's description. Returns a memoized Promise: cache hits return
|
|
208
|
-
* the existing (in-flight or resolved) promise; misses push the image's key
|
|
209
|
-
* into the current batch and schedule dispatch. Callers in the same frame
|
|
210
|
-
* share ONE batch → ONE vision call. Failures resolve to {@link UNAVAILABLE}
|
|
211
|
-
* and are NOT cached, so the next turn re-attempts. */
|
|
212
|
-
loadDescription(img: ExtractedImage): Promise<string> {
|
|
213
|
-
const hash = imageHash(img.mimeType, img.data);
|
|
214
|
-
const cached = this.cache.get(hash);
|
|
215
|
-
if (cached) {
|
|
216
|
-
// Re-insert to refresh Map order: insertion order is FIFO, so without
|
|
217
|
-
// this a hot image ages like a cold one and gets evicted ahead of it.
|
|
218
|
-
this.cache.delete(hash);
|
|
219
|
-
this.cache.set(hash, cached);
|
|
220
|
-
return cached;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
if (!this.batch) {
|
|
224
|
-
this.batch = { keys: [], imgs: [], callbacks: [] };
|
|
225
|
-
this.scheduleDispatch();
|
|
226
|
-
}
|
|
227
|
-
const batch = this.batch;
|
|
228
|
-
let idx = batch.keys.indexOf(hash);
|
|
229
|
-
if (idx === -1) {
|
|
230
|
-
batch.keys.push(hash);
|
|
231
|
-
batch.imgs.push(img);
|
|
232
|
-
idx = batch.keys.length - 1;
|
|
233
|
-
} else {
|
|
234
|
-
// Same image loaded twice in the frame (the first load's cache entry was
|
|
235
|
-
// evicted mid-frame, else the second load would have short-circuited on
|
|
236
|
-
// the cache). Share the one key/image slot but give this caller its own
|
|
237
|
-
// promise — dispatch resolves it by hash alongside the first caller's.
|
|
238
|
-
batch.imgs[idx] = img;
|
|
239
|
-
}
|
|
240
|
-
const promise = new Promise<string>((resolve, reject) => {
|
|
241
|
-
batch.callbacks.push({ hash, resolve, reject });
|
|
242
|
-
});
|
|
243
|
-
const cfg = this.deps.getConfig();
|
|
244
|
-
if (this.cache.size >= cfg.cacheMax) {
|
|
245
|
-
const firstKey = this.cache.keys().next().value;
|
|
246
|
-
if (firstKey !== undefined) this.cache.delete(firstKey);
|
|
247
|
-
}
|
|
248
|
-
this.cache.set(hash, promise);
|
|
249
|
-
return promise;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/** Abandon any in-flight batch and clear turn context. Used on session reset
|
|
253
|
-
* (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
|
|
254
|
-
* `using` binding can scope a loader lifetime. The description cache is
|
|
255
|
-
* preserved across turns (descriptions are stable per hash). */
|
|
256
|
-
reset(): void {
|
|
257
|
-
this.batch = null;
|
|
258
|
-
this.dispatchScheduled = false;
|
|
259
|
-
this.turnModelRegistry = null;
|
|
260
|
-
this.turnVisionModel = null;
|
|
261
|
-
// Fresh controller on session reset (no in-flight batch to orphan).
|
|
262
|
-
this.turnAbortController = new AbortController();
|
|
263
|
-
this.wiredSignal = undefined;
|
|
264
|
-
this.pendingTurnPrompt = "";
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
[Symbol.dispose](): void {
|
|
268
|
-
this.reset();
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
private scheduleDispatch(): void {
|
|
272
|
-
if (this.dispatchScheduled) return;
|
|
273
|
-
this.dispatchScheduled = true;
|
|
274
|
-
enqueuePostPromiseJob(() => this.dispatchBatch());
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
/** Dispatch the current batch: ONE batched `runBatch` vision call for every
|
|
278
|
-
* key collected this frame, then resolve each load()'s promise with its
|
|
279
|
-
* description (or {@link UNAVAILABLE} on failure). Failures are evicted from
|
|
280
|
-
* the cache so the next turn re-attempts.
|
|
281
|
-
*
|
|
282
|
-
* Results are resolved BY HASH and fanned out to EVERY callback: a batch can
|
|
283
|
-
* hold more callbacks than keys when the same image was loaded twice in one
|
|
284
|
-
* frame (its first cache entry was evicted mid-frame, so the second load
|
|
285
|
-
* pushed a second callback for the same hash). Indexing callbacks by key
|
|
286
|
-
* would skip those duplicates and hang their promises; iterating callbacks
|
|
287
|
-
* and looking up each one's hash fans the one result to all of them. */
|
|
288
|
-
private async dispatchBatch(): Promise<void> {
|
|
289
|
-
this.dispatchScheduled = false;
|
|
290
|
-
const batch = this.batch;
|
|
291
|
-
this.batch = null;
|
|
292
|
-
if (!batch || batch.keys.length === 0) return;
|
|
293
|
-
if (!this.turnVisionModel || !this.turnModelRegistry) {
|
|
294
|
-
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
if (this.turnAbortController.signal.aborted) {
|
|
298
|
-
for (const key of batch.keys) this.cache.delete(key);
|
|
299
|
-
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
300
|
-
return;
|
|
301
|
-
}
|
|
302
|
-
this.deps.setLastError(null); // clear before a fresh attempt
|
|
303
|
-
const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
|
|
304
|
-
const cfg = this.deps.getConfig();
|
|
305
|
-
let parsed = await runBatch(
|
|
306
|
-
misses,
|
|
307
|
-
this.pendingTurnPrompt,
|
|
308
|
-
this.turnVisionModel,
|
|
309
|
-
this.turnModelRegistry,
|
|
310
|
-
cfg,
|
|
311
|
-
this.deps,
|
|
312
|
-
this.turnAbortController.signal,
|
|
313
|
-
);
|
|
314
|
-
// Retry once on a TOTAL batch failure (the batched call itself failed:
|
|
315
|
-
// auth, network, timeout, empty, or stopReason "error"). A transient blip
|
|
316
|
-
// would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
|
|
317
|
-
// cached, re-attempted next turn. Retrying the whole batch recovers it
|
|
318
|
-
// within the same tool-result phase (free time). Skip when the turn was
|
|
319
|
-
// cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
|
|
320
|
-
// API key before the vision call), so no vision call is wasted on a
|
|
321
|
-
// permanent auth error. Partial failures (some images described) are left
|
|
322
|
-
// as-is — only a totally-empty result retries.
|
|
323
|
-
if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
|
|
324
|
-
await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
|
|
325
|
-
if (!this.turnAbortController.signal.aborted) {
|
|
326
|
-
this.deps.setLastError(null);
|
|
327
|
-
parsed = await runBatch(
|
|
328
|
-
misses,
|
|
329
|
-
this.pendingTurnPrompt,
|
|
330
|
-
this.turnVisionModel,
|
|
331
|
-
this.turnModelRegistry,
|
|
332
|
-
cfg,
|
|
333
|
-
this.deps,
|
|
334
|
-
this.turnAbortController.signal,
|
|
335
|
-
);
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
// Failover: the same-model retry failed too. Try each configured fallback
|
|
339
|
-
// describer in order (a different provider has independent rate limits /
|
|
340
|
-
// auth / availability). No per-fallback backoff — the fallback doesn't
|
|
341
|
-
// share the primary's rate-limit state. A partial result from any
|
|
342
|
-
// fallback wins; only when every fallback also returns empty do we give
|
|
343
|
-
// up (UNAVAILABLE, evicted from cache, re-attempted next turn). User
|
|
344
|
-
// cancel (ESC) skips failover.
|
|
345
|
-
if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
|
|
346
|
-
for (const ref of cfg.fallbackModels ?? []) {
|
|
347
|
-
if (this.turnAbortController.signal.aborted) break;
|
|
348
|
-
const fallbackModel = this.deps.resolveVisionModel(this.turnModelRegistry, ref);
|
|
349
|
-
if (!fallbackModel) {
|
|
350
|
-
this.deps.setLastError(`fallback vision model "${ref}" could not be resolved`);
|
|
351
|
-
continue;
|
|
352
|
-
}
|
|
353
|
-
this.deps.setLastError(null);
|
|
354
|
-
parsed = await runBatch(
|
|
355
|
-
misses,
|
|
356
|
-
this.pendingTurnPrompt,
|
|
357
|
-
fallbackModel,
|
|
358
|
-
this.turnModelRegistry,
|
|
359
|
-
cfg,
|
|
360
|
-
this.deps,
|
|
361
|
-
this.turnAbortController.signal,
|
|
362
|
-
);
|
|
363
|
-
if (parsed.size > 0 || this.turnAbortController.signal.aborted) break;
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
// Build per-hash results, then fan them out to every callback. This reaches
|
|
367
|
-
// duplicate-hash callbacks that a key-indexed loop would have skipped (and
|
|
368
|
-
// left hanging).
|
|
369
|
-
const results = new Map<string, string>();
|
|
370
|
-
for (let i = 0; i < batch.keys.length; i++) {
|
|
371
|
-
const key = batch.keys[i];
|
|
372
|
-
const raw = parsed.get(key);
|
|
373
|
-
if (raw) {
|
|
374
|
-
const final = wrapDescription(raw, cfg);
|
|
375
|
-
results.set(key, final);
|
|
376
|
-
// Cache the resolved value so later loads (this frame or next) hit.
|
|
377
|
-
this.cache.set(key, Promise.resolve(final));
|
|
378
|
-
} else {
|
|
379
|
-
// Genuine failure — do NOT cache; next turn re-attempts (and surfaces
|
|
380
|
-
// the real error). Resolve (not reject) with UNAVAILABLE to match the
|
|
381
|
-
// graceful-degradation contract and avoid unhandled rejections.
|
|
382
|
-
this.cache.delete(key);
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
for (const cb of batch.callbacks) {
|
|
386
|
-
cb.resolve(results.get(cb.hash) ?? UNAVAILABLE);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Facebook DataLoader pattern for image descriptions.
|
|
3
|
+
*
|
|
4
|
+
* `loadDescription(img)` returns a memoized Promise for the description and
|
|
5
|
+
* pushes the image's key (hash) into the CURRENT batch. All `load()` calls in
|
|
6
|
+
* the same execution frame (+ its microtask cascade) coalesce into ONE batch
|
|
7
|
+
* object. Dispatch is scheduled via `setImmediate` (see `enqueuePostPromiseJob`),
|
|
8
|
+
* so every load in the frame — and every load from a separate I/O callback in
|
|
9
|
+
* the same poll iteration — lands in the single batch before the ONE vision
|
|
10
|
+
* call fires. Each load()'s promise then resolves with its description.
|
|
11
|
+
*
|
|
12
|
+
* Mapping: the `read` tools are the `load()` callers. Their `tool_result`
|
|
13
|
+
* handler awaits the shared batch — so N parallel reads coalesce into the SAME
|
|
14
|
+
* single vision call and all resolve together, the descriptions landing in the
|
|
15
|
+
* tool results BEFORE the agent's next turn. pi fires each read's `tool_result`
|
|
16
|
+
* as that read's I/O completes (poll phase); the loader's `setImmediate`
|
|
17
|
+
* dispatch defers to the check phase, AFTER the whole poll iteration, so reads
|
|
18
|
+
* completing together (the common case for cached local files) land in ONE
|
|
19
|
+
* batch — and reads completing in separate iterations get separate calls, but
|
|
20
|
+
* always in parallel, never sequential. The agent's tool-result wait is free
|
|
21
|
+
* time, so this adds zero latency to the critical path. `context` then sees
|
|
22
|
+
* text-described tool results (no image blocks to swap); any remaining images
|
|
23
|
+
* (user-attached, custom-injected) are cache hits.
|
|
24
|
+
*
|
|
25
|
+
* All mutable state (batch, cache, turn context) lives on the instance — no
|
|
26
|
+
* module-level globals — and the class implements `Disposable` so a `using`
|
|
27
|
+
* binding (or an explicit `reset()`) cleanly abandons an in-flight batch and
|
|
28
|
+
* clears turn context, e.g. on session reset.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
32
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
33
|
+
import {
|
|
34
|
+
IMAGE_PLACEHOLDER_PREFIX,
|
|
35
|
+
IMAGE_PLACEHOLDER_SUFFIX,
|
|
36
|
+
wrapDescription,
|
|
37
|
+
type ExtractedImage,
|
|
38
|
+
type VisionHandoffConfig,
|
|
39
|
+
} from "./index.js";
|
|
40
|
+
import { imageHash } from "./image.js";
|
|
41
|
+
import { runBatch, type DescriberDeps } from "./describer.js";
|
|
42
|
+
|
|
43
|
+
/** Resolved when a description couldn't be obtained (graceful degradation).
|
|
44
|
+
* Failures are NOT cached, so the next turn re-attempts. */
|
|
45
|
+
export const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${IMAGE_PLACEHOLDER_SUFFIX}`;
|
|
46
|
+
|
|
47
|
+
interface DescriptionBatch {
|
|
48
|
+
keys: string[];
|
|
49
|
+
imgs: ExtractedImage[];
|
|
50
|
+
/** One per `loadDescription()` call. A duplicate load (same hash, but its
|
|
51
|
+
* first cache entry was evicted mid-frame so it couldn't short-circuit on
|
|
52
|
+
* the cache) pushes a second callback for an existing key — so
|
|
53
|
+
* `callbacks.length` can exceed `keys.length`. `dispatchBatch` resolves by
|
|
54
|
+
* hash so every callback is reached. */
|
|
55
|
+
callbacks: { hash: string; resolve: (v: string) => void; reject: (e: Error) => void }[];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Engine-provided resolver for the configured vision model. */
|
|
59
|
+
export interface VisionModelResolver {
|
|
60
|
+
(registry: ModelRegistry, ref: string): Model<Api> | null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Dependencies the loader can't own itself (held by the engine). */
|
|
64
|
+
export interface LoaderDeps extends DescriberDeps {
|
|
65
|
+
/** Read the current config (reloaded on session_start / config writes). */
|
|
66
|
+
getConfig(): VisionHandoffConfig;
|
|
67
|
+
/** Resolve the configured vision model against a registry. */
|
|
68
|
+
resolveVisionModel: VisionModelResolver;
|
|
69
|
+
/** Backoff (ms) before retrying a totally-failed describer batch. Defaults
|
|
70
|
+
* to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
|
|
71
|
+
retryBackoffMs?: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Defer `fn` to the next check phase (`setImmediate`), so every
|
|
75
|
+
* `loadDescription()` — whether called from sync code, a microtask cascade,
|
|
76
|
+
* or a separate I/O callback in the same poll iteration — coalesces into one
|
|
77
|
+
* batch before the single vision call fires.
|
|
78
|
+
*
|
|
79
|
+
* Why `setImmediate` and not DataLoader's classic `process.nextTick`:
|
|
80
|
+
* nextTick drains between I/O callbacks in the poll phase, so dispatch would
|
|
81
|
+
* fire after the first parallel `read`'s `tool_result` but before the
|
|
82
|
+
* second's — splitting N reads into N single-image calls. `setImmediate` runs
|
|
83
|
+
* in the check phase, AFTER the whole poll phase, so all `tool_result`
|
|
84
|
+
* handlers that fire in one poll iteration land in ONE batch. The check phase
|
|
85
|
+
* also runs after the microtask queue drains, so loads issued from a `.then`
|
|
86
|
+
* cascade (e.g. the clipboard pre-warm) still coalesce. */
|
|
87
|
+
function enqueuePostPromiseJob(fn: () => void): void {
|
|
88
|
+
setImmediate(fn);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Default backoff (ms) before retrying a totally-failed describer batch. A
|
|
92
|
+
* short backoff covers a transient provider blip (network hiccup, momentary
|
|
93
|
+
* 429) without adding meaningful latency to the tool-result phase (free
|
|
94
|
+
* time). Generous enough to let a rate-limited provider recover, short enough
|
|
95
|
+
* that a genuinely broken vision model doesn't stall a turn. */
|
|
96
|
+
const DESCRIBE_RETRY_BACKOFF_MS = 500;
|
|
97
|
+
|
|
98
|
+
/** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
|
|
99
|
+
* during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
|
|
100
|
+
* wait the full backoff before the retry is skipped. */
|
|
101
|
+
function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
102
|
+
return new Promise((resolve) => {
|
|
103
|
+
if (ms <= 0 || signal.aborted) return resolve();
|
|
104
|
+
const onAbort = () => {
|
|
105
|
+
clearTimeout(timer);
|
|
106
|
+
resolve();
|
|
107
|
+
};
|
|
108
|
+
const timer = setTimeout(() => {
|
|
109
|
+
signal.removeEventListener("abort", onAbort);
|
|
110
|
+
resolve();
|
|
111
|
+
}, ms);
|
|
112
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export class DescriptionLoader implements Disposable {
|
|
117
|
+
private readonly cache = new Map<string, Promise<string>>();
|
|
118
|
+
private batch: DescriptionBatch | null = null;
|
|
119
|
+
private dispatchScheduled = false;
|
|
120
|
+
private turnModelRegistry: ModelRegistry | null = null;
|
|
121
|
+
private turnVisionModel: Model<Api> | null = null;
|
|
122
|
+
/** Turn-level abort controller. In-flight describer batches (`runBatch`)
|
|
123
|
+
* are wired to `turnAbortController.signal` so a user cancel (ESC) aborts
|
|
124
|
+
* them EVEN WHEN the batch was dispatched before the run's live abort
|
|
125
|
+
* signal existed — the `before_agent_start` / paste-time prewarm case,
|
|
126
|
+
* where `ctx.signal` is `undefined` (the agent run hasn't started yet).
|
|
127
|
+
* The run's live signal, once it arrives via `bindTurnContext`, is forwarded
|
|
128
|
+
* into `turnAbortController.abort()`. Because `runBatch` holds a reference
|
|
129
|
+
* to this controller's signal (not a snapshot of `ctx.signal`), a batch
|
|
130
|
+
* that started during prewarm becomes abortable the instant a later
|
|
131
|
+
* `bindTurnContext` (from `tool_result`/`context`) brings the live signal.
|
|
132
|
+
* Reset per turn via {@link resetTurnAbort} so a previous turn's cancel
|
|
133
|
+
* can't poison the next turn's prewarm. */
|
|
134
|
+
private turnAbortController = new AbortController();
|
|
135
|
+
/** The run signal currently forwarded into {@link turnAbortController}, so
|
|
136
|
+
* the abort listener is attached at most once per signal (the same signal
|
|
137
|
+
* is bound by every `tool_result`/`context` event in a turn). */
|
|
138
|
+
private wiredSignal: AbortSignal | undefined;
|
|
139
|
+
private pendingTurnPrompt = "";
|
|
140
|
+
|
|
141
|
+
constructor(private readonly deps: LoaderDeps) {}
|
|
142
|
+
|
|
143
|
+
/** Bind the turn context (model registry, resolved vision model, abort
|
|
144
|
+
* signal) for the loader's next dispatch. Called from every handler that
|
|
145
|
+
* may trigger `loadDescription()`.
|
|
146
|
+
*
|
|
147
|
+
* The abort signal is forwarded into the loader's {@link turnAbortController}
|
|
148
|
+
* rather than stored directly. Storing `ctx.signal` directly would leave a
|
|
149
|
+
* prewarm-dispatched batch (started in `before_agent_start`, where
|
|
150
|
+
* `ctx.signal` is `undefined` because the run hasn't started) with no abort
|
|
151
|
+
* wire — ESC couldn't cancel it, so the `tool_result` handler would only
|
|
152
|
+
* discard the result AFTER the vision call ran to completion. Forwarding
|
|
153
|
+
* the live signal into a stable, loader-owned controller lets an in-flight
|
|
154
|
+
* prewarm batch be aborted the moment the live signal arrives. */
|
|
155
|
+
bindTurnContext(ctx: { modelRegistry: ModelRegistry; signal?: AbortSignal }): void {
|
|
156
|
+
this.turnModelRegistry = ctx.modelRegistry;
|
|
157
|
+
const cfg = this.deps.getConfig();
|
|
158
|
+
const resolved = this.deps.resolveVisionModel(ctx.modelRegistry, cfg.visionModel!);
|
|
159
|
+
if (resolved) {
|
|
160
|
+
this.turnVisionModel = resolved;
|
|
161
|
+
} else {
|
|
162
|
+
// Primary unresolvable (typo, removed model): pick the first resolvable
|
|
163
|
+
// fallback so failover still works instead of leaving the loader dead.
|
|
164
|
+
// Call-time failover in dispatchBatch covers the case where the primary
|
|
165
|
+
// RESOLVES but fails at call time.
|
|
166
|
+
for (const ref of cfg.fallbackModels ?? []) {
|
|
167
|
+
const fb = this.deps.resolveVisionModel(ctx.modelRegistry, ref);
|
|
168
|
+
if (fb) {
|
|
169
|
+
this.turnVisionModel = fb;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
const signal = ctx.signal;
|
|
175
|
+
if (!signal) return;
|
|
176
|
+
if (signal.aborted) {
|
|
177
|
+
this.turnAbortController.abort();
|
|
178
|
+
this.wiredSignal = signal;
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (signal === this.wiredSignal) return; // already forwarded this signal
|
|
182
|
+
this.wiredSignal = signal;
|
|
183
|
+
signal.addEventListener("abort", () => this.turnAbortController.abort(), { once: true });
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Reset the turn-level abort controller for a fresh turn. Call at turn
|
|
187
|
+
* boundaries (`before_agent_start`, paste-time prewarm) so a previous turn's
|
|
188
|
+
* cancel doesn't leave {@link turnAbortController} aborted — which would
|
|
189
|
+
* make every subsequent dispatch short-circuit to UNAVAILABLE. A
|
|
190
|
+
* non-aborted controller is reused (avoids orphaning an in-flight paste
|
|
191
|
+
* prewarm holding its signal); only an aborted one is replaced. `wiredSignal`
|
|
192
|
+
* is always cleared so the next live signal re-wires. Safe at a real turn
|
|
193
|
+
* boundary, where the prior turn's batch has settled. */
|
|
194
|
+
resetTurnAbort(): void {
|
|
195
|
+
if (this.turnAbortController.signal.aborted) {
|
|
196
|
+
this.turnAbortController = new AbortController();
|
|
197
|
+
}
|
|
198
|
+
this.wiredSignal = undefined;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Capture this turn's user prompt so every image in the turn is described in
|
|
202
|
+
* the same request context. */
|
|
203
|
+
setPendingTurnPrompt(prompt: string): void {
|
|
204
|
+
this.pendingTurnPrompt = prompt;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Load an image's description. Returns a memoized Promise: cache hits return
|
|
208
|
+
* the existing (in-flight or resolved) promise; misses push the image's key
|
|
209
|
+
* into the current batch and schedule dispatch. Callers in the same frame
|
|
210
|
+
* share ONE batch → ONE vision call. Failures resolve to {@link UNAVAILABLE}
|
|
211
|
+
* and are NOT cached, so the next turn re-attempts. */
|
|
212
|
+
loadDescription(img: ExtractedImage): Promise<string> {
|
|
213
|
+
const hash = imageHash(img.mimeType, img.data);
|
|
214
|
+
const cached = this.cache.get(hash);
|
|
215
|
+
if (cached) {
|
|
216
|
+
// Re-insert to refresh Map order: insertion order is FIFO, so without
|
|
217
|
+
// this a hot image ages like a cold one and gets evicted ahead of it.
|
|
218
|
+
this.cache.delete(hash);
|
|
219
|
+
this.cache.set(hash, cached);
|
|
220
|
+
return cached;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (!this.batch) {
|
|
224
|
+
this.batch = { keys: [], imgs: [], callbacks: [] };
|
|
225
|
+
this.scheduleDispatch();
|
|
226
|
+
}
|
|
227
|
+
const batch = this.batch;
|
|
228
|
+
let idx = batch.keys.indexOf(hash);
|
|
229
|
+
if (idx === -1) {
|
|
230
|
+
batch.keys.push(hash);
|
|
231
|
+
batch.imgs.push(img);
|
|
232
|
+
idx = batch.keys.length - 1;
|
|
233
|
+
} else {
|
|
234
|
+
// Same image loaded twice in the frame (the first load's cache entry was
|
|
235
|
+
// evicted mid-frame, else the second load would have short-circuited on
|
|
236
|
+
// the cache). Share the one key/image slot but give this caller its own
|
|
237
|
+
// promise — dispatch resolves it by hash alongside the first caller's.
|
|
238
|
+
batch.imgs[idx] = img;
|
|
239
|
+
}
|
|
240
|
+
const promise = new Promise<string>((resolve, reject) => {
|
|
241
|
+
batch.callbacks.push({ hash, resolve, reject });
|
|
242
|
+
});
|
|
243
|
+
const cfg = this.deps.getConfig();
|
|
244
|
+
if (this.cache.size >= cfg.cacheMax) {
|
|
245
|
+
const firstKey = this.cache.keys().next().value;
|
|
246
|
+
if (firstKey !== undefined) this.cache.delete(firstKey);
|
|
247
|
+
}
|
|
248
|
+
this.cache.set(hash, promise);
|
|
249
|
+
return promise;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Abandon any in-flight batch and clear turn context. Used on session reset
|
|
253
|
+
* (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
|
|
254
|
+
* `using` binding can scope a loader lifetime. The description cache is
|
|
255
|
+
* preserved across turns (descriptions are stable per hash). */
|
|
256
|
+
reset(): void {
|
|
257
|
+
this.batch = null;
|
|
258
|
+
this.dispatchScheduled = false;
|
|
259
|
+
this.turnModelRegistry = null;
|
|
260
|
+
this.turnVisionModel = null;
|
|
261
|
+
// Fresh controller on session reset (no in-flight batch to orphan).
|
|
262
|
+
this.turnAbortController = new AbortController();
|
|
263
|
+
this.wiredSignal = undefined;
|
|
264
|
+
this.pendingTurnPrompt = "";
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
[Symbol.dispose](): void {
|
|
268
|
+
this.reset();
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private scheduleDispatch(): void {
|
|
272
|
+
if (this.dispatchScheduled) return;
|
|
273
|
+
this.dispatchScheduled = true;
|
|
274
|
+
enqueuePostPromiseJob(() => this.dispatchBatch());
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Dispatch the current batch: ONE batched `runBatch` vision call for every
|
|
278
|
+
* key collected this frame, then resolve each load()'s promise with its
|
|
279
|
+
* description (or {@link UNAVAILABLE} on failure). Failures are evicted from
|
|
280
|
+
* the cache so the next turn re-attempts.
|
|
281
|
+
*
|
|
282
|
+
* Results are resolved BY HASH and fanned out to EVERY callback: a batch can
|
|
283
|
+
* hold more callbacks than keys when the same image was loaded twice in one
|
|
284
|
+
* frame (its first cache entry was evicted mid-frame, so the second load
|
|
285
|
+
* pushed a second callback for the same hash). Indexing callbacks by key
|
|
286
|
+
* would skip those duplicates and hang their promises; iterating callbacks
|
|
287
|
+
* and looking up each one's hash fans the one result to all of them. */
|
|
288
|
+
private async dispatchBatch(): Promise<void> {
|
|
289
|
+
this.dispatchScheduled = false;
|
|
290
|
+
const batch = this.batch;
|
|
291
|
+
this.batch = null;
|
|
292
|
+
if (!batch || batch.keys.length === 0) return;
|
|
293
|
+
if (!this.turnVisionModel || !this.turnModelRegistry) {
|
|
294
|
+
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
if (this.turnAbortController.signal.aborted) {
|
|
298
|
+
for (const key of batch.keys) this.cache.delete(key);
|
|
299
|
+
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
this.deps.setLastError(null); // clear before a fresh attempt
|
|
303
|
+
const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
|
|
304
|
+
const cfg = this.deps.getConfig();
|
|
305
|
+
let parsed = await runBatch(
|
|
306
|
+
misses,
|
|
307
|
+
this.pendingTurnPrompt,
|
|
308
|
+
this.turnVisionModel,
|
|
309
|
+
this.turnModelRegistry,
|
|
310
|
+
cfg,
|
|
311
|
+
this.deps,
|
|
312
|
+
this.turnAbortController.signal,
|
|
313
|
+
);
|
|
314
|
+
// Retry once on a TOTAL batch failure (the batched call itself failed:
|
|
315
|
+
// auth, network, timeout, empty, or stopReason "error"). A transient blip
|
|
316
|
+
// would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
|
|
317
|
+
// cached, re-attempted next turn. Retrying the whole batch recovers it
|
|
318
|
+
// within the same tool-result phase (free time). Skip when the turn was
|
|
319
|
+
// cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
|
|
320
|
+
// API key before the vision call), so no vision call is wasted on a
|
|
321
|
+
// permanent auth error. Partial failures (some images described) are left
|
|
322
|
+
// as-is — only a totally-empty result retries.
|
|
323
|
+
if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
|
|
324
|
+
await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
|
|
325
|
+
if (!this.turnAbortController.signal.aborted) {
|
|
326
|
+
this.deps.setLastError(null);
|
|
327
|
+
parsed = await runBatch(
|
|
328
|
+
misses,
|
|
329
|
+
this.pendingTurnPrompt,
|
|
330
|
+
this.turnVisionModel,
|
|
331
|
+
this.turnModelRegistry,
|
|
332
|
+
cfg,
|
|
333
|
+
this.deps,
|
|
334
|
+
this.turnAbortController.signal,
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
// Failover: the same-model retry failed too. Try each configured fallback
|
|
339
|
+
// describer in order (a different provider has independent rate limits /
|
|
340
|
+
// auth / availability). No per-fallback backoff — the fallback doesn't
|
|
341
|
+
// share the primary's rate-limit state. A partial result from any
|
|
342
|
+
// fallback wins; only when every fallback also returns empty do we give
|
|
343
|
+
// up (UNAVAILABLE, evicted from cache, re-attempted next turn). User
|
|
344
|
+
// cancel (ESC) skips failover.
|
|
345
|
+
if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
|
|
346
|
+
for (const ref of cfg.fallbackModels ?? []) {
|
|
347
|
+
if (this.turnAbortController.signal.aborted) break;
|
|
348
|
+
const fallbackModel = this.deps.resolveVisionModel(this.turnModelRegistry, ref);
|
|
349
|
+
if (!fallbackModel) {
|
|
350
|
+
this.deps.setLastError(`fallback vision model "${ref}" could not be resolved`);
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
this.deps.setLastError(null);
|
|
354
|
+
parsed = await runBatch(
|
|
355
|
+
misses,
|
|
356
|
+
this.pendingTurnPrompt,
|
|
357
|
+
fallbackModel,
|
|
358
|
+
this.turnModelRegistry,
|
|
359
|
+
cfg,
|
|
360
|
+
this.deps,
|
|
361
|
+
this.turnAbortController.signal,
|
|
362
|
+
);
|
|
363
|
+
if (parsed.size > 0 || this.turnAbortController.signal.aborted) break;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
// Build per-hash results, then fan them out to every callback. This reaches
|
|
367
|
+
// duplicate-hash callbacks that a key-indexed loop would have skipped (and
|
|
368
|
+
// left hanging).
|
|
369
|
+
const results = new Map<string, string>();
|
|
370
|
+
for (let i = 0; i < batch.keys.length; i++) {
|
|
371
|
+
const key = batch.keys[i];
|
|
372
|
+
const raw = parsed.get(key);
|
|
373
|
+
if (raw) {
|
|
374
|
+
const final = wrapDescription(raw, cfg);
|
|
375
|
+
results.set(key, final);
|
|
376
|
+
// Cache the resolved value so later loads (this frame or next) hit.
|
|
377
|
+
this.cache.set(key, Promise.resolve(final));
|
|
378
|
+
} else {
|
|
379
|
+
// Genuine failure — do NOT cache; next turn re-attempts (and surfaces
|
|
380
|
+
// the real error). Resolve (not reject) with UNAVAILABLE to match the
|
|
381
|
+
// graceful-degradation contract and avoid unhandled rejections.
|
|
382
|
+
this.cache.delete(key);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
for (const cb of batch.callbacks) {
|
|
386
|
+
cb.resolve(results.get(cb.hash) ?? UNAVAILABLE);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|