@bismawy/pi-vision-watcher 1.0.15 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/dataloader.ts CHANGED
@@ -1,389 +1,389 @@
1
- /**
2
- * Facebook DataLoader pattern for image descriptions.
3
- *
4
- * `loadDescription(img)` returns a memoized Promise for the description and
5
- * pushes the image's key (hash) into the CURRENT batch. All `load()` calls in
6
- * the same execution frame (+ its microtask cascade) coalesce into ONE batch
7
- * object. Dispatch is scheduled via `setImmediate` (see `enqueuePostPromiseJob`),
8
- * so every load in the frame — and every load from a separate I/O callback in
9
- * the same poll iteration — lands in the single batch before the ONE vision
10
- * call fires. Each load()'s promise then resolves with its description.
11
- *
12
- * Mapping: the `read` tools are the `load()` callers. Their `tool_result`
13
- * handler awaits the shared batch — so N parallel reads coalesce into the SAME
14
- * single vision call and all resolve together, the descriptions landing in the
15
- * tool results BEFORE the agent's next turn. pi fires each read's `tool_result`
16
- * as that read's I/O completes (poll phase); the loader's `setImmediate`
17
- * dispatch defers to the check phase, AFTER the whole poll iteration, so reads
18
- * completing together (the common case for cached local files) land in ONE
19
- * batch — and reads completing in separate iterations get separate calls, but
20
- * always in parallel, never sequential. The agent's tool-result wait is free
21
- * time, so this adds zero latency to the critical path. `context` then sees
22
- * text-described tool results (no image blocks to swap); any remaining images
23
- * (user-attached, custom-injected) are cache hits.
24
- *
25
- * All mutable state (batch, cache, turn context) lives on the instance — no
26
- * module-level globals — and the class implements `Disposable` so a `using`
27
- * binding (or an explicit `reset()`) cleanly abandons an in-flight batch and
28
- * clears turn context, e.g. on session reset.
29
- */
30
-
31
- import type { Api, Model } from "@earendil-works/pi-ai";
32
- import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
33
- import {
34
- IMAGE_PLACEHOLDER_PREFIX,
35
- IMAGE_PLACEHOLDER_SUFFIX,
36
- wrapDescription,
37
- type ExtractedImage,
38
- type VisionHandoffConfig,
39
- } from "./index.js";
40
- import { imageHash } from "./image.js";
41
- import { runBatch, describeSingle, type DescriberDeps } from "./describer.js";
42
-
43
- /** Resolved when a description couldn't be obtained (graceful degradation).
44
- * Failures are NOT cached, so the next turn re-attempts. */
45
- export const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${IMAGE_PLACEHOLDER_SUFFIX}`;
46
-
47
- interface DescriptionBatch {
48
- keys: string[];
49
- imgs: ExtractedImage[];
50
- /** One per `loadDescription()` call. A duplicate load (same hash, but its
51
- * first cache entry was evicted mid-frame so it couldn't short-circuit on
52
- * the cache) pushes a second callback for an existing key — so
53
- * `callbacks.length` can exceed `keys.length`. `dispatchBatch` resolves by
54
- * hash so every callback is reached. */
55
- callbacks: { hash: string; resolve: (v: string) => void; reject: (e: Error) => void }[];
56
- }
57
-
58
- /** Engine-provided resolver for the configured vision model. */
59
- export interface VisionModelResolver {
60
- (registry: ModelRegistry, ref: string): Model<Api> | null;
61
- }
62
-
63
- /** Dependencies the loader can't own itself (held by the engine). */
64
- export interface LoaderDeps extends DescriberDeps {
65
- /** Read the current config (reloaded on session_start / config writes). */
66
- getConfig(): VisionHandoffConfig;
67
- /** Resolve the configured vision model against a registry. */
68
- resolveVisionModel: VisionModelResolver;
69
- /** Backoff (ms) before retrying a totally-failed describer batch. Defaults
70
- * to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
71
- retryBackoffMs?: number;
72
- }
73
-
74
- /** Defer `fn` to the next check phase (`setImmediate`), so every
75
- * `loadDescription()` — whether called from sync code, a microtask cascade,
76
- * or a separate I/O callback in the same poll iteration — coalesces into one
77
- * batch before the single vision call fires.
78
- *
79
- * Why `setImmediate` and not DataLoader's classic `process.nextTick`:
80
- * nextTick drains between I/O callbacks in the poll phase, so dispatch would
81
- * fire after the first parallel `read`'s `tool_result` but before the
82
- * second's — splitting N reads into N single-image calls. `setImmediate` runs
83
- * in the check phase, AFTER the whole poll phase, so all `tool_result`
84
- * handlers that fire in one poll iteration land in ONE batch. The check phase
85
- * also runs after the microtask queue drains, so loads issued from a `.then`
86
- * cascade (e.g. the clipboard pre-warm) still coalesce. */
87
- function enqueuePostPromiseJob(fn: () => void): void {
88
- setImmediate(fn);
89
- }
90
-
91
- /** Default backoff (ms) before retrying a totally-failed describer batch. A
92
- * short backoff covers a transient provider blip (network hiccup, momentary
93
- * 429) without adding meaningful latency to the tool-result phase (free
94
- * time). Generous enough to let a rate-limited provider recover, short enough
95
- * that a genuinely broken vision model doesn't stall a turn. */
96
- const DESCRIBE_RETRY_BACKOFF_MS = 500;
97
-
98
- /** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
99
- * during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
100
- * wait the full backoff before the retry is skipped. */
101
- function sleep(ms: number, signal: AbortSignal): Promise<void> {
102
- return new Promise((resolve) => {
103
- if (ms <= 0 || signal.aborted) return resolve();
104
- const onAbort = () => {
105
- clearTimeout(timer);
106
- resolve();
107
- };
108
- const timer = setTimeout(() => {
109
- signal.removeEventListener("abort", onAbort);
110
- resolve();
111
- }, ms);
112
- signal.addEventListener("abort", onAbort, { once: true });
113
- });
114
- }
115
-
116
- export class DescriptionLoader implements Disposable {
117
- private readonly cache = new Map<string, Promise<string>>();
118
- private batch: DescriptionBatch | null = null;
119
- private dispatchScheduled = false;
120
- private turnModelRegistry: ModelRegistry | null = null;
121
- private turnVisionModel: Model<Api> | null = null;
122
- /** Turn-level abort controller. In-flight describer batches (`runBatch`)
123
- * are wired to `turnAbortController.signal` so a user cancel (ESC) aborts
124
- * them EVEN WHEN the batch was dispatched before the run's live abort
125
- * signal existed — the `before_agent_start` / paste-time prewarm case,
126
- * where `ctx.signal` is `undefined` (the agent run hasn't started yet).
127
- * The run's live signal, once it arrives via `bindTurnContext`, is forwarded
128
- * into `turnAbortController.abort()`. Because `runBatch` holds a reference
129
- * to this controller's signal (not a snapshot of `ctx.signal`), a batch
130
- * that started during prewarm becomes abortable the instant a later
131
- * `bindTurnContext` (from `tool_result`/`context`) brings the live signal.
132
- * Reset per turn via {@link resetTurnAbort} so a previous turn's cancel
133
- * can't poison the next turn's prewarm. */
134
- private turnAbortController = new AbortController();
135
- /** The run signal currently forwarded into {@link turnAbortController}, so
136
- * the abort listener is attached at most once per signal (the same signal
137
- * is bound by every `tool_result`/`context` event in a turn). */
138
- private wiredSignal: AbortSignal | undefined;
139
- private pendingTurnPrompt = "";
140
-
141
- constructor(private readonly deps: LoaderDeps) {}
142
-
143
- /** Bind the turn context (model registry, resolved vision model, abort
144
- * signal) for the loader's next dispatch. Called from every handler that
145
- * may trigger `loadDescription()`.
146
- *
147
- * The abort signal is forwarded into the loader's {@link turnAbortController}
148
- * rather than stored directly. Storing `ctx.signal` directly would leave a
149
- * prewarm-dispatched batch (started in `before_agent_start`, where
150
- * `ctx.signal` is `undefined` because the run hasn't started) with no abort
151
- * wire — ESC couldn't cancel it, so the `tool_result` handler would only
152
- * discard the result AFTER the vision call ran to completion. Forwarding
153
- * the live signal into a stable, loader-owned controller lets an in-flight
154
- * prewarm batch be aborted the moment the live signal arrives. */
155
- bindTurnContext(ctx: { modelRegistry: ModelRegistry; signal?: AbortSignal }): void {
156
- this.turnModelRegistry = ctx.modelRegistry;
157
- const cfg = this.deps.getConfig();
158
- const resolved = this.deps.resolveVisionModel(ctx.modelRegistry, cfg.visionModel!);
159
- if (resolved) {
160
- this.turnVisionModel = resolved;
161
- } else {
162
- // Primary unresolvable (typo, removed model): pick the first resolvable
163
- // fallback so failover still works instead of leaving the loader dead.
164
- // Call-time failover in dispatchBatch covers the case where the primary
165
- // RESOLVES but fails at call time.
166
- for (const ref of cfg.fallbackModels ?? []) {
167
- const fb = this.deps.resolveVisionModel(ctx.modelRegistry, ref);
168
- if (fb) {
169
- this.turnVisionModel = fb;
170
- break;
171
- }
172
- }
173
- }
174
- const signal = ctx.signal;
175
- if (!signal) return;
176
- if (signal.aborted) {
177
- this.turnAbortController.abort();
178
- this.wiredSignal = signal;
179
- return;
180
- }
181
- if (signal === this.wiredSignal) return; // already forwarded this signal
182
- this.wiredSignal = signal;
183
- signal.addEventListener("abort", () => this.turnAbortController.abort(), { once: true });
184
- }
185
-
186
- /** Reset the turn-level abort controller for a fresh turn. Call at turn
187
- * boundaries (`before_agent_start`, paste-time prewarm) so a previous turn's
188
- * cancel doesn't leave {@link turnAbortController} aborted — which would
189
- * make every subsequent dispatch short-circuit to UNAVAILABLE. A
190
- * non-aborted controller is reused (avoids orphaning an in-flight paste
191
- * prewarm holding its signal); only an aborted one is replaced. `wiredSignal`
192
- * is always cleared so the next live signal re-wires. Safe at a real turn
193
- * boundary, where the prior turn's batch has settled. */
194
- resetTurnAbort(): void {
195
- if (this.turnAbortController.signal.aborted) {
196
- this.turnAbortController = new AbortController();
197
- }
198
- this.wiredSignal = undefined;
199
- }
200
-
201
- /** Capture this turn's user prompt so every image in the turn is described in
202
- * the same request context. */
203
- setPendingTurnPrompt(prompt: string): void {
204
- this.pendingTurnPrompt = prompt;
205
- }
206
-
207
- /** Load an image's description. Returns a memoized Promise: cache hits return
208
- * the existing (in-flight or resolved) promise; misses push the image's key
209
- * into the current batch and schedule dispatch. Callers in the same frame
210
- * share ONE batch → ONE vision call. Failures resolve to {@link UNAVAILABLE}
211
- * and are NOT cached, so the next turn re-attempts. */
212
- loadDescription(img: ExtractedImage): Promise<string> {
213
- const hash = imageHash(img.mimeType, img.data);
214
- const cached = this.cache.get(hash);
215
- if (cached) {
216
- // Re-insert to refresh Map order: insertion order is FIFO, so without
217
- // this a hot image ages like a cold one and gets evicted ahead of it.
218
- this.cache.delete(hash);
219
- this.cache.set(hash, cached);
220
- return cached;
221
- }
222
-
223
- if (!this.batch) {
224
- this.batch = { keys: [], imgs: [], callbacks: [] };
225
- this.scheduleDispatch();
226
- }
227
- const batch = this.batch;
228
- let idx = batch.keys.indexOf(hash);
229
- if (idx === -1) {
230
- batch.keys.push(hash);
231
- batch.imgs.push(img);
232
- idx = batch.keys.length - 1;
233
- } else {
234
- // Same image loaded twice in the frame (the first load's cache entry was
235
- // evicted mid-frame, else the second load would have short-circuited on
236
- // the cache). Share the one key/image slot but give this caller its own
237
- // promise — dispatch resolves it by hash alongside the first caller's.
238
- batch.imgs[idx] = img;
239
- }
240
- const promise = new Promise<string>((resolve, reject) => {
241
- batch.callbacks.push({ hash, resolve, reject });
242
- });
243
- const cfg = this.deps.getConfig();
244
- if (this.cache.size >= cfg.cacheMax) {
245
- const firstKey = this.cache.keys().next().value;
246
- if (firstKey !== undefined) this.cache.delete(firstKey);
247
- }
248
- this.cache.set(hash, promise);
249
- return promise;
250
- }
251
-
252
- /** Abandon any in-flight batch and clear turn context. Used on session reset
253
- * (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
254
- * `using` binding can scope a loader lifetime. The description cache is
255
- * preserved across turns (descriptions are stable per hash). */
256
- reset(): void {
257
- this.batch = null;
258
- this.dispatchScheduled = false;
259
- this.turnModelRegistry = null;
260
- this.turnVisionModel = null;
261
- // Fresh controller on session reset (no in-flight batch to orphan).
262
- this.turnAbortController = new AbortController();
263
- this.wiredSignal = undefined;
264
- this.pendingTurnPrompt = "";
265
- }
266
-
267
- [Symbol.dispose](): void {
268
- this.reset();
269
- }
270
-
271
- private scheduleDispatch(): void {
272
- if (this.dispatchScheduled) return;
273
- this.dispatchScheduled = true;
274
- enqueuePostPromiseJob(() => this.dispatchBatch());
275
- }
276
-
277
- /** Dispatch the current batch: ONE batched `runBatch` vision call for every
278
- * key collected this frame, then resolve each load()'s promise with its
279
- * description (or {@link UNAVAILABLE} on failure). Failures are evicted from
280
- * the cache so the next turn re-attempts.
281
- *
282
- * Results are resolved BY HASH and fanned out to EVERY callback: a batch can
283
- * hold more callbacks than keys when the same image was loaded twice in one
284
- * frame (its first cache entry was evicted mid-frame, so the second load
285
- * pushed a second callback for the same hash). Indexing callbacks by key
286
- * would skip those duplicates and hang their promises; iterating callbacks
287
- * and looking up each one's hash fans the one result to all of them. */
288
- private async dispatchBatch(): Promise<void> {
289
- this.dispatchScheduled = false;
290
- const batch = this.batch;
291
- this.batch = null;
292
- if (!batch || batch.keys.length === 0) return;
293
- if (!this.turnVisionModel || !this.turnModelRegistry) {
294
- for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
295
- return;
296
- }
297
- if (this.turnAbortController.signal.aborted) {
298
- for (const key of batch.keys) this.cache.delete(key);
299
- for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
300
- return;
301
- }
302
- this.deps.setLastError(null); // clear before a fresh attempt
303
- const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
304
- const cfg = this.deps.getConfig();
305
- let parsed = await runBatch(
306
- misses,
307
- this.pendingTurnPrompt,
308
- this.turnVisionModel,
309
- this.turnModelRegistry,
310
- cfg,
311
- this.deps,
312
- this.turnAbortController.signal,
313
- );
314
- // Retry once on a TOTAL batch failure (the batched call itself failed:
315
- // auth, network, timeout, empty, or stopReason "error"). A transient blip
316
- // would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
317
- // cached, re-attempted next turn. Retrying the whole batch recovers it
318
- // within the same tool-result phase (free time). Skip when the turn was
319
- // cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
320
- // API key before the vision call), so no vision call is wasted on a
321
- // permanent auth error. Partial failures (some images described) are left
322
- // as-is — only a totally-empty result retries.
323
- if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
324
- await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
325
- if (!this.turnAbortController.signal.aborted) {
326
- this.deps.setLastError(null);
327
- parsed = await runBatch(
328
- misses,
329
- this.pendingTurnPrompt,
330
- this.turnVisionModel,
331
- this.turnModelRegistry,
332
- cfg,
333
- this.deps,
334
- this.turnAbortController.signal,
335
- );
336
- }
337
- }
338
- // Failover: the same-model retry failed too. Try each configured fallback
339
- // describer in order (a different provider has independent rate limits /
340
- // auth / availability). No per-fallback backoff — the fallback doesn't
341
- // share the primary's rate-limit state. A partial result from any
342
- // fallback wins; only when every fallback also returns empty do we give
343
- // up (UNAVAILABLE, evicted from cache, re-attempted next turn). User
344
- // cancel (ESC) skips failover.
345
- if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
346
- for (const ref of cfg.fallbackModels ?? []) {
347
- if (this.turnAbortController.signal.aborted) break;
348
- const fallbackModel = this.deps.resolveVisionModel(this.turnModelRegistry, ref);
349
- if (!fallbackModel) {
350
- this.deps.setLastError(`fallback vision model "${ref}" could not be resolved`);
351
- continue;
352
- }
353
- this.deps.setLastError(null);
354
- parsed = await runBatch(
355
- misses,
356
- this.pendingTurnPrompt,
357
- fallbackModel,
358
- this.turnModelRegistry,
359
- cfg,
360
- this.deps,
361
- this.turnAbortController.signal,
362
- );
363
- if (parsed.size > 0 || this.turnAbortController.signal.aborted) break;
364
- }
365
- }
366
- // Build per-hash results, then fan them out to every callback. This reaches
367
- // duplicate-hash callbacks that a key-indexed loop would have skipped (and
368
- // left hanging).
369
- const results = new Map<string, string>();
370
- for (let i = 0; i < batch.keys.length; i++) {
371
- const key = batch.keys[i];
372
- const raw = parsed.get(key);
373
- if (raw) {
374
- const final = wrapDescription(raw, cfg);
375
- results.set(key, final);
376
- // Cache the resolved value so later loads (this frame or next) hit.
377
- this.cache.set(key, Promise.resolve(final));
378
- } else {
379
- // Genuine failure — do NOT cache; next turn re-attempts (and surfaces
380
- // the real error). Resolve (not reject) with UNAVAILABLE to match the
381
- // graceful-degradation contract and avoid unhandled rejections.
382
- this.cache.delete(key);
383
- }
384
- }
385
- for (const cb of batch.callbacks) {
386
- cb.resolve(results.get(cb.hash) ?? UNAVAILABLE);
387
- }
388
- }
389
- }
1
+ /**
2
+ * Facebook DataLoader pattern for image descriptions.
3
+ *
4
+ * `loadDescription(img)` returns a memoized Promise for the description and
5
+ * pushes the image's key (hash) into the CURRENT batch. All `load()` calls in
6
+ * the same execution frame (+ its microtask cascade) coalesce into ONE batch
7
+ * object. Dispatch is scheduled via `setImmediate` (see `enqueuePostPromiseJob`),
8
+ * so every load in the frame — and every load from a separate I/O callback in
9
+ * the same poll iteration — lands in the single batch before the ONE vision
10
+ * call fires. Each load()'s promise then resolves with its description.
11
+ *
12
+ * Mapping: the `read` tools are the `load()` callers. Their `tool_result`
13
+ * handler awaits the shared batch — so N parallel reads coalesce into the SAME
14
+ * single vision call and all resolve together, the descriptions landing in the
15
+ * tool results BEFORE the agent's next turn. pi fires each read's `tool_result`
16
+ * as that read's I/O completes (poll phase); the loader's `setImmediate`
17
+ * dispatch defers to the check phase, AFTER the whole poll iteration, so reads
18
+ * completing together (the common case for cached local files) land in ONE
19
+ * batch — and reads completing in separate iterations get separate calls, but
20
+ * always in parallel, never sequential. The agent's tool-result wait is free
21
+ * time, so this adds zero latency to the critical path. `context` then sees
22
+ * text-described tool results (no image blocks to swap); any remaining images
23
+ * (user-attached, custom-injected) are cache hits.
24
+ *
25
+ * All mutable state (batch, cache, turn context) lives on the instance — no
26
+ * module-level globals — and the class implements `Disposable` so a `using`
27
+ * binding (or an explicit `reset()`) cleanly abandons an in-flight batch and
28
+ * clears turn context, e.g. on session reset.
29
+ */
30
+
31
+ import type { Api, Model } from "@earendil-works/pi-ai";
32
+ import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
33
+ import {
34
+ IMAGE_PLACEHOLDER_PREFIX,
35
+ IMAGE_PLACEHOLDER_SUFFIX,
36
+ wrapDescription,
37
+ type ExtractedImage,
38
+ type VisionHandoffConfig,
39
+ } from "./index.js";
40
+ import { imageHash } from "./image.js";
41
+ import { runBatch, type DescriberDeps } from "./describer.js";
42
+
43
+ /** Resolved when a description couldn't be obtained (graceful degradation).
44
+ * Failures are NOT cached, so the next turn re-attempts. */
45
+ export const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${IMAGE_PLACEHOLDER_SUFFIX}`;
46
+
47
+ interface DescriptionBatch {
48
+ keys: string[];
49
+ imgs: ExtractedImage[];
50
+ /** One per `loadDescription()` call. A duplicate load (same hash, but its
51
+ * first cache entry was evicted mid-frame so it couldn't short-circuit on
52
+ * the cache) pushes a second callback for an existing key — so
53
+ * `callbacks.length` can exceed `keys.length`. `dispatchBatch` resolves by
54
+ * hash so every callback is reached. */
55
+ callbacks: { hash: string; resolve: (v: string) => void; reject: (e: Error) => void }[];
56
+ }
57
+
58
+ /** Engine-provided resolver for the configured vision model. */
59
+ export interface VisionModelResolver {
60
+ (registry: ModelRegistry, ref: string): Model<Api> | null;
61
+ }
62
+
63
+ /** Dependencies the loader can't own itself (held by the engine). */
64
+ export interface LoaderDeps extends DescriberDeps {
65
+ /** Read the current config (reloaded on session_start / config writes). */
66
+ getConfig(): VisionHandoffConfig;
67
+ /** Resolve the configured vision model against a registry. */
68
+ resolveVisionModel: VisionModelResolver;
69
+ /** Backoff (ms) before retrying a totally-failed describer batch. Defaults
70
+ * to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
71
+ retryBackoffMs?: number;
72
+ }
73
+
74
+ /** Defer `fn` to the next check phase (`setImmediate`), so every
75
+ * `loadDescription()` — whether called from sync code, a microtask cascade,
76
+ * or a separate I/O callback in the same poll iteration — coalesces into one
77
+ * batch before the single vision call fires.
78
+ *
79
+ * Why `setImmediate` and not DataLoader's classic `process.nextTick`:
80
+ * nextTick drains between I/O callbacks in the poll phase, so dispatch would
81
+ * fire after the first parallel `read`'s `tool_result` but before the
82
+ * second's — splitting N reads into N single-image calls. `setImmediate` runs
83
+ * in the check phase, AFTER the whole poll phase, so all `tool_result`
84
+ * handlers that fire in one poll iteration land in ONE batch. The check phase
85
+ * also runs after the microtask queue drains, so loads issued from a `.then`
86
+ * cascade (e.g. the clipboard pre-warm) still coalesce. */
87
+ function enqueuePostPromiseJob(fn: () => void): void {
88
+ setImmediate(fn);
89
+ }
90
+
91
+ /** Default backoff (ms) before retrying a totally-failed describer batch. A
92
+ * short backoff covers a transient provider blip (network hiccup, momentary
93
+ * 429) without adding meaningful latency to the tool-result phase (free
94
+ * time). Generous enough to let a rate-limited provider recover, short enough
95
+ * that a genuinely broken vision model doesn't stall a turn. */
96
+ const DESCRIBE_RETRY_BACKOFF_MS = 500;
97
+
98
+ /** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
99
+ * during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
100
+ * wait the full backoff before the retry is skipped. */
101
+ function sleep(ms: number, signal: AbortSignal): Promise<void> {
102
+ return new Promise((resolve) => {
103
+ if (ms <= 0 || signal.aborted) return resolve();
104
+ const onAbort = () => {
105
+ clearTimeout(timer);
106
+ resolve();
107
+ };
108
+ const timer = setTimeout(() => {
109
+ signal.removeEventListener("abort", onAbort);
110
+ resolve();
111
+ }, ms);
112
+ signal.addEventListener("abort", onAbort, { once: true });
113
+ });
114
+ }
115
+
116
+ export class DescriptionLoader implements Disposable {
117
+ private readonly cache = new Map<string, Promise<string>>();
118
+ private batch: DescriptionBatch | null = null;
119
+ private dispatchScheduled = false;
120
+ private turnModelRegistry: ModelRegistry | null = null;
121
+ private turnVisionModel: Model<Api> | null = null;
122
+ /** Turn-level abort controller. In-flight describer batches (`runBatch`)
123
+ * are wired to `turnAbortController.signal` so a user cancel (ESC) aborts
124
+ * them EVEN WHEN the batch was dispatched before the run's live abort
125
+ * signal existed — the `before_agent_start` / paste-time prewarm case,
126
+ * where `ctx.signal` is `undefined` (the agent run hasn't started yet).
127
+ * The run's live signal, once it arrives via `bindTurnContext`, is forwarded
128
+ * into `turnAbortController.abort()`. Because `runBatch` holds a reference
129
+ * to this controller's signal (not a snapshot of `ctx.signal`), a batch
130
+ * that started during prewarm becomes abortable the instant a later
131
+ * `bindTurnContext` (from `tool_result`/`context`) brings the live signal.
132
+ * Reset per turn via {@link resetTurnAbort} so a previous turn's cancel
133
+ * can't poison the next turn's prewarm. */
134
+ private turnAbortController = new AbortController();
135
+ /** The run signal currently forwarded into {@link turnAbortController}, so
136
+ * the abort listener is attached at most once per signal (the same signal
137
+ * is bound by every `tool_result`/`context` event in a turn). */
138
+ private wiredSignal: AbortSignal | undefined;
139
+ private pendingTurnPrompt = "";
140
+
141
+ constructor(private readonly deps: LoaderDeps) {}
142
+
143
+ /** Bind the turn context (model registry, resolved vision model, abort
144
+ * signal) for the loader's next dispatch. Called from every handler that
145
+ * may trigger `loadDescription()`.
146
+ *
147
+ * The abort signal is forwarded into the loader's {@link turnAbortController}
148
+ * rather than stored directly. Storing `ctx.signal` directly would leave a
149
+ * prewarm-dispatched batch (started in `before_agent_start`, where
150
+ * `ctx.signal` is `undefined` because the run hasn't started) with no abort
151
+ * wire — ESC couldn't cancel it, so the `tool_result` handler would only
152
+ * discard the result AFTER the vision call ran to completion. Forwarding
153
+ * the live signal into a stable, loader-owned controller lets an in-flight
154
+ * prewarm batch be aborted the moment the live signal arrives. */
155
+ bindTurnContext(ctx: { modelRegistry: ModelRegistry; signal?: AbortSignal }): void {
156
+ this.turnModelRegistry = ctx.modelRegistry;
157
+ const cfg = this.deps.getConfig();
158
+ const resolved = this.deps.resolveVisionModel(ctx.modelRegistry, cfg.visionModel!);
159
+ if (resolved) {
160
+ this.turnVisionModel = resolved;
161
+ } else {
162
+ // Primary unresolvable (typo, removed model): pick the first resolvable
163
+ // fallback so failover still works instead of leaving the loader dead.
164
+ // Call-time failover in dispatchBatch covers the case where the primary
165
+ // RESOLVES but fails at call time.
166
+ for (const ref of cfg.fallbackModels ?? []) {
167
+ const fb = this.deps.resolveVisionModel(ctx.modelRegistry, ref);
168
+ if (fb) {
169
+ this.turnVisionModel = fb;
170
+ break;
171
+ }
172
+ }
173
+ }
174
+ const signal = ctx.signal;
175
+ if (!signal) return;
176
+ if (signal.aborted) {
177
+ this.turnAbortController.abort();
178
+ this.wiredSignal = signal;
179
+ return;
180
+ }
181
+ if (signal === this.wiredSignal) return; // already forwarded this signal
182
+ this.wiredSignal = signal;
183
+ signal.addEventListener("abort", () => this.turnAbortController.abort(), { once: true });
184
+ }
185
+
186
+ /** Reset the turn-level abort controller for a fresh turn. Call at turn
187
+ * boundaries (`before_agent_start`, paste-time prewarm) so a previous turn's
188
+ * cancel doesn't leave {@link turnAbortController} aborted — which would
189
+ * make every subsequent dispatch short-circuit to UNAVAILABLE. A
190
+ * non-aborted controller is reused (avoids orphaning an in-flight paste
191
+ * prewarm holding its signal); only an aborted one is replaced. `wiredSignal`
192
+ * is always cleared so the next live signal re-wires. Safe at a real turn
193
+ * boundary, where the prior turn's batch has settled. */
194
+ resetTurnAbort(): void {
195
+ if (this.turnAbortController.signal.aborted) {
196
+ this.turnAbortController = new AbortController();
197
+ }
198
+ this.wiredSignal = undefined;
199
+ }
200
+
201
+ /** Capture this turn's user prompt so every image in the turn is described in
202
+ * the same request context. */
203
+ setPendingTurnPrompt(prompt: string): void {
204
+ this.pendingTurnPrompt = prompt;
205
+ }
206
+
207
+ /** Load an image's description. Returns a memoized Promise: cache hits return
208
+ * the existing (in-flight or resolved) promise; misses push the image's key
209
+ * into the current batch and schedule dispatch. Callers in the same frame
210
+ * share ONE batch → ONE vision call. Failures resolve to {@link UNAVAILABLE}
211
+ * and are NOT cached, so the next turn re-attempts. */
212
+ loadDescription(img: ExtractedImage): Promise<string> {
213
+ const hash = imageHash(img.mimeType, img.data);
214
+ const cached = this.cache.get(hash);
215
+ if (cached) {
216
+ // Re-insert to refresh Map order: insertion order is FIFO, so without
217
+ // this a hot image ages like a cold one and gets evicted ahead of it.
218
+ this.cache.delete(hash);
219
+ this.cache.set(hash, cached);
220
+ return cached;
221
+ }
222
+
223
+ if (!this.batch) {
224
+ this.batch = { keys: [], imgs: [], callbacks: [] };
225
+ this.scheduleDispatch();
226
+ }
227
+ const batch = this.batch;
228
+ let idx = batch.keys.indexOf(hash);
229
+ if (idx === -1) {
230
+ batch.keys.push(hash);
231
+ batch.imgs.push(img);
232
+ idx = batch.keys.length - 1;
233
+ } else {
234
+ // Same image loaded twice in the frame (the first load's cache entry was
235
+ // evicted mid-frame, else the second load would have short-circuited on
236
+ // the cache). Share the one key/image slot but give this caller its own
237
+ // promise — dispatch resolves it by hash alongside the first caller's.
238
+ batch.imgs[idx] = img;
239
+ }
240
+ const promise = new Promise<string>((resolve, reject) => {
241
+ batch.callbacks.push({ hash, resolve, reject });
242
+ });
243
+ const cfg = this.deps.getConfig();
244
+ if (this.cache.size >= cfg.cacheMax) {
245
+ const firstKey = this.cache.keys().next().value;
246
+ if (firstKey !== undefined) this.cache.delete(firstKey);
247
+ }
248
+ this.cache.set(hash, promise);
249
+ return promise;
250
+ }
251
+
252
+ /** Abandon any in-flight batch and clear turn context. Used on session reset
253
+ * (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
254
+ * `using` binding can scope a loader lifetime. The description cache is
255
+ * preserved across turns (descriptions are stable per hash). */
256
+ reset(): void {
257
+ this.batch = null;
258
+ this.dispatchScheduled = false;
259
+ this.turnModelRegistry = null;
260
+ this.turnVisionModel = null;
261
+ // Fresh controller on session reset (no in-flight batch to orphan).
262
+ this.turnAbortController = new AbortController();
263
+ this.wiredSignal = undefined;
264
+ this.pendingTurnPrompt = "";
265
+ }
266
+
267
+ [Symbol.dispose](): void {
268
+ this.reset();
269
+ }
270
+
271
+ private scheduleDispatch(): void {
272
+ if (this.dispatchScheduled) return;
273
+ this.dispatchScheduled = true;
274
+ enqueuePostPromiseJob(() => this.dispatchBatch());
275
+ }
276
+
277
+ /** Dispatch the current batch: ONE batched `runBatch` vision call for every
278
+ * key collected this frame, then resolve each load()'s promise with its
279
+ * description (or {@link UNAVAILABLE} on failure). Failures are evicted from
280
+ * the cache so the next turn re-attempts.
281
+ *
282
+ * Results are resolved BY HASH and fanned out to EVERY callback: a batch can
283
+ * hold more callbacks than keys when the same image was loaded twice in one
284
+ * frame (its first cache entry was evicted mid-frame, so the second load
285
+ * pushed a second callback for the same hash). Indexing callbacks by key
286
+ * would skip those duplicates and hang their promises; iterating callbacks
287
+ * and looking up each one's hash fans the one result to all of them. */
288
+ private async dispatchBatch(): Promise<void> {
289
+ this.dispatchScheduled = false;
290
+ const batch = this.batch;
291
+ this.batch = null;
292
+ if (!batch || batch.keys.length === 0) return;
293
+ if (!this.turnVisionModel || !this.turnModelRegistry) {
294
+ for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
295
+ return;
296
+ }
297
+ if (this.turnAbortController.signal.aborted) {
298
+ for (const key of batch.keys) this.cache.delete(key);
299
+ for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
300
+ return;
301
+ }
302
+ this.deps.setLastError(null); // clear before a fresh attempt
303
+ const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
304
+ const cfg = this.deps.getConfig();
305
+ let parsed = await runBatch(
306
+ misses,
307
+ this.pendingTurnPrompt,
308
+ this.turnVisionModel,
309
+ this.turnModelRegistry,
310
+ cfg,
311
+ this.deps,
312
+ this.turnAbortController.signal,
313
+ );
314
+ // Retry once on a TOTAL batch failure (the batched call itself failed:
315
+ // auth, network, timeout, empty, or stopReason "error"). A transient blip
316
+ // would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
317
+ // cached, re-attempted next turn. Retrying the whole batch recovers it
318
+ // within the same tool-result phase (free time). Skip when the turn was
319
+ // cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
320
+ // API key before the vision call), so no vision call is wasted on a
321
+ // permanent auth error. Partial failures (some images described) are left
322
+ // as-is — only a totally-empty result retries.
323
+ if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
324
+ await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
325
+ if (!this.turnAbortController.signal.aborted) {
326
+ this.deps.setLastError(null);
327
+ parsed = await runBatch(
328
+ misses,
329
+ this.pendingTurnPrompt,
330
+ this.turnVisionModel,
331
+ this.turnModelRegistry,
332
+ cfg,
333
+ this.deps,
334
+ this.turnAbortController.signal,
335
+ );
336
+ }
337
+ }
338
+ // Failover: the same-model retry failed too. Try each configured fallback
339
+ // describer in order (a different provider has independent rate limits /
340
+ // auth / availability). No per-fallback backoff — the fallback doesn't
341
+ // share the primary's rate-limit state. A partial result from any
342
+ // fallback wins; only when every fallback also returns empty do we give
343
+ // up (UNAVAILABLE, evicted from cache, re-attempted next turn). User
344
+ // cancel (ESC) skips failover.
345
+ if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
346
+ for (const ref of cfg.fallbackModels ?? []) {
347
+ if (this.turnAbortController.signal.aborted) break;
348
+ const fallbackModel = this.deps.resolveVisionModel(this.turnModelRegistry, ref);
349
+ if (!fallbackModel) {
350
+ this.deps.setLastError(`fallback vision model "${ref}" could not be resolved`);
351
+ continue;
352
+ }
353
+ this.deps.setLastError(null);
354
+ parsed = await runBatch(
355
+ misses,
356
+ this.pendingTurnPrompt,
357
+ fallbackModel,
358
+ this.turnModelRegistry,
359
+ cfg,
360
+ this.deps,
361
+ this.turnAbortController.signal,
362
+ );
363
+ if (parsed.size > 0 || this.turnAbortController.signal.aborted) break;
364
+ }
365
+ }
366
+ // Build per-hash results, then fan them out to every callback. This reaches
367
+ // duplicate-hash callbacks that a key-indexed loop would have skipped (and
368
+ // left hanging).
369
+ const results = new Map<string, string>();
370
+ for (let i = 0; i < batch.keys.length; i++) {
371
+ const key = batch.keys[i];
372
+ const raw = parsed.get(key);
373
+ if (raw) {
374
+ const final = wrapDescription(raw, cfg);
375
+ results.set(key, final);
376
+ // Cache the resolved value so later loads (this frame or next) hit.
377
+ this.cache.set(key, Promise.resolve(final));
378
+ } else {
379
+ // Genuine failure — do NOT cache; next turn re-attempts (and surfaces
380
+ // the real error). Resolve (not reject) with UNAVAILABLE to match the
381
+ // graceful-degradation contract and avoid unhandled rejections.
382
+ this.cache.delete(key);
383
+ }
384
+ }
385
+ for (const cb of batch.callbacks) {
386
+ cb.resolve(results.get(cb.hash) ?? UNAVAILABLE);
387
+ }
388
+ }
389
+ }