@bismawy/pi-vision-watcher 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/README.md +3 -0
- package/package.json +1 -1
- package/src/describer.ts +92 -12
- package/src/index.ts +605 -583
- package/src/vision-model-selector.ts +31 -29
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## [
|
|
3
|
+
## [1.3.1] - 2026-11-13
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- An image batch no longer leaves as one oversized request body. Aggregator proxies cap the whole
|
|
7
|
+
body (Remote Magpie answers HTTP 413 near 1 MB), and that failure cost every description in the
|
|
8
|
+
batch. An image above the budget is re-encoded down first, and the rest are grouped into
|
|
9
|
+
sequential calls that each fit `maxRequestBytes` (default `700000`; `0` disables splitting).
|
|
10
|
+
- A provider that rejects an image for a text-only model with a "can't see images" error now triggers
|
|
11
|
+
the existing text-only recovery. The model is marked text-only and its images hand off on the next
|
|
12
|
+
turn, rather than the rejection being logged as an unrelated failure.
|
|
13
|
+
|
|
14
|
+
## [1.3.0] - 2026-10-24
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
- The `/vision-watcher` picker adopts the pi-arnative providers model-list style: rows render a checkbox
|
|
18
|
+
(`[x]` accent when the row is the primary, `[ ]` dim otherwise) instead of a trailing `✓`, labels are the
|
|
19
|
+
bare `provider/id` ref (no `[provider]`/`✦` badges; unselected rows sit in muted), and a scrolled list
|
|
20
|
+
announces hidden rows with `↑ N more` / `↓ N more` lines instead of `(cursor/total)`. The search field and
|
|
21
|
+
the footer sit flush in the frame's one-space pad with no prefix of their own, like the providers screen.
|
|
22
|
+
Fallback members keep the `⇆` mark.
|
|
4
23
|
|
|
5
24
|
## [1.2.0] - 2026-10-21
|
|
6
25
|
|
package/README.md
CHANGED
|
@@ -84,6 +84,7 @@ pi --extension ./vision-watcher.ts
|
|
|
84
84
|
"prewarmPastedImages": false,
|
|
85
85
|
"asyncClipboardHandoff": false,
|
|
86
86
|
"maxTokens": null,
|
|
87
|
+
"maxRequestBytes": 700000,
|
|
87
88
|
"cacheMax": 50,
|
|
88
89
|
"maxDescriptionLines": 0
|
|
89
90
|
}
|
|
@@ -93,6 +94,8 @@ Most fields have sane defaults — `visionModel` is the only one you normally se
|
|
|
93
94
|
|
|
94
95
|
`fallbackModels` is the failover chain: when the primary describer fails (timeout, rate limit, auth), each entry is tried **in order** until one returns a description. The picker caps the chain at 3 entries (`tab`). Set it from the picker instead of hand-editing the file.
|
|
95
96
|
|
|
97
|
+
`maxRequestBytes` (default `700000`) is the byte budget for one describer request body, prompt text and base64 images together. Aggregator proxies such as Remote Magpie reject a body near 1 MB with HTTP 413 no matter how large the model context is, so an image above the budget is re-encoded down and the batch is split into sequential calls that each fit. Set it to your provider's limit; `0` disables the budget.
|
|
98
|
+
|
|
96
99
|
</details>
|
|
97
100
|
|
|
98
101
|
<details>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bismawy/pi-vision-watcher",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "Intelligent vision watcher. Connected models. Zero workflow interruption. Text-only Pi models get background image descriptions with failover chains.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "bismawy",
|
package/src/describer.ts
CHANGED
|
@@ -38,6 +38,7 @@ import type {
|
|
|
38
38
|
ThinkingLevel,
|
|
39
39
|
} from "@earendil-works/pi-ai";
|
|
40
40
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
41
|
+
import { resizeImage } from "@earendil-works/pi-coding-agent";
|
|
41
42
|
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
42
43
|
import {
|
|
43
44
|
batchUserPrompt,
|
|
@@ -197,12 +198,81 @@ export interface DescriberDeps {
|
|
|
197
198
|
/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
|
|
198
199
|
export type BatchResult = Map<string, string>;
|
|
199
200
|
|
|
201
|
+
/** Per-image request overhead outside the base64 payload: the JSON block
|
|
202
|
+
* wrapper, the `data:image/…;base64,` prefix some providers add, and the batch
|
|
203
|
+
* delimiter prose. One kilobyte is generous against a ~700 KB budget. */
|
|
204
|
+
const PER_IMAGE_REQUEST_OVERHEAD = 1024;
|
|
205
|
+
|
|
206
|
+
/** Reserve for the prompt text and JSON envelope of one request body. */
|
|
207
|
+
const PROMPT_REQUEST_RESERVE = 8 * 1024;
|
|
208
|
+
|
|
209
|
+
/** Largest base64 payload one image may occupy so a request carrying only that
|
|
210
|
+
* image still fits `budget`. `0` means "no limit" (budget disabled). */
|
|
211
|
+
export function perImageBudget(budget: number): number {
|
|
212
|
+
if (budget <= 0) return 0;
|
|
213
|
+
return Math.max(16 * 1024, budget - PROMPT_REQUEST_RESERVE - PER_IMAGE_REQUEST_OVERHEAD);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Shrink any image that alone exceeds `maxBytes`, so an oversized attachment
|
|
217
|
+
* cannot fail its own request. pi resizes images when they enter history, but
|
|
218
|
+
* the describer can see the raw bytes first (its `context` hook runs before the
|
|
219
|
+
* encode lands) and a pasted or pre-existing image may never be re-encoded.
|
|
220
|
+
* `maxBytes <= 0` skips the pass. A failed resize keeps the original bytes and
|
|
221
|
+
* lets the provider reject them, so the real error is surfaced, not hidden. */
|
|
222
|
+
async function fitImages(images: ExtractedImage[], maxBytes: number): Promise<ExtractedImage[]> {
|
|
223
|
+
if (maxBytes <= 0) return images;
|
|
224
|
+
return Promise.all(
|
|
225
|
+
images.map(async (img) => {
|
|
226
|
+
if (img.data.length <= maxBytes) return img;
|
|
227
|
+
const resized = await resizeImage(Buffer.from(img.data, "base64"), img.mimeType, {
|
|
228
|
+
maxWidth: 2048,
|
|
229
|
+
maxHeight: 2048,
|
|
230
|
+
maxBytes,
|
|
231
|
+
jpegQuality: 80,
|
|
232
|
+
});
|
|
233
|
+
return resized ? { data: resized.data, mimeType: resized.mimeType } : img;
|
|
234
|
+
}),
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Split a batch into as few order-preserving groups as fit `budget` base64
|
|
239
|
+
* bytes each. An image larger than the budget gets its own group: it cannot be
|
|
240
|
+
* split further, and pi's ingest resize should already have kept it small.
|
|
241
|
+
* `budget <= 0` means "never split" — one group with every image. */
|
|
242
|
+
export function splitByByteBudget<T extends { img: ExtractedImage }>(misses: T[], budget: number): T[][] {
|
|
243
|
+
if (misses.length === 0) return [];
|
|
244
|
+
if (budget <= 0 || misses.length === 1) return [misses];
|
|
245
|
+
const groups: T[][] = [];
|
|
246
|
+
let current: T[] = [];
|
|
247
|
+
let bytes = 0;
|
|
248
|
+
for (const miss of misses) {
|
|
249
|
+
const cost = miss.img.data.length + PER_IMAGE_REQUEST_OVERHEAD;
|
|
250
|
+
if (current.length > 0 && bytes + cost > budget) {
|
|
251
|
+
groups.push(current);
|
|
252
|
+
current = [];
|
|
253
|
+
bytes = 0;
|
|
254
|
+
}
|
|
255
|
+
current.push(miss);
|
|
256
|
+
bytes += cost;
|
|
257
|
+
}
|
|
258
|
+
if (current.length > 0) groups.push(current);
|
|
259
|
+
return groups;
|
|
260
|
+
}
|
|
261
|
+
|
|
200
262
|
/** Describe N images with ONE batched `completeSimple()` call and split the response
|
|
201
263
|
* back into per-image descriptions. Returns a map keyed by image hash; an
|
|
202
264
|
* image whose section failed to parse is omitted (the caller treats omission
|
|
203
265
|
* as "description unavailable"). On a genuine call failure (auth, abort,
|
|
204
266
|
* empty) the map is empty and `deps.setLastError` records the reason.
|
|
205
267
|
*
|
|
268
|
+
* A N-image batch is split into groups that each fit `cfg.maxRequestBytes`.
|
|
269
|
+
* Aggregator proxies cap the request BODY (prompt text plus base64 images),
|
|
270
|
+
* not the model context, so an oversized body is rejected outright (Remote
|
|
271
|
+
* Magpie: HTTP 413 at ~1 MB) and every description in the batch is lost. An
|
|
272
|
+
* image that alone exceeds the budget is re-encoded down first; the remaining
|
|
273
|
+
* images run in sequence, one call per group, so a large read costs a few
|
|
274
|
+
* requests instead of one fatal one.
|
|
275
|
+
*
|
|
206
276
|
* A failed delimiter-parse is a failed BATCH, not a failed description: the
|
|
207
277
|
* unparsed images fall back to parallel single-image calls (no delimiters to
|
|
208
278
|
* cooperate with), so the common cooperative case stays one call while an
|
|
@@ -217,20 +287,30 @@ export async function runBatch(
|
|
|
217
287
|
turnSignal?: AbortSignal,
|
|
218
288
|
): Promise<BatchResult> {
|
|
219
289
|
const out: BatchResult = new Map();
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
await
|
|
290
|
+
// Fit each image to the budget BEFORE grouping: an image above it would 413 on
|
|
291
|
+
// its own, and grouping cannot help a body dominated by one oversized image.
|
|
292
|
+
const images = await fitImages(
|
|
223
293
|
misses.map((m) => m.img),
|
|
224
|
-
|
|
225
|
-
"batch",
|
|
226
|
-
userPrompt,
|
|
227
|
-
visionModel,
|
|
228
|
-
modelRegistry,
|
|
229
|
-
cfg,
|
|
230
|
-
deps,
|
|
231
|
-
(text, truncated) => fillBatch(out, text, truncated, misses, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal),
|
|
232
|
-
turnSignal,
|
|
294
|
+
perImageBudget(cfg.maxRequestBytes),
|
|
233
295
|
);
|
|
296
|
+
const items = images.map((img, i) => ({ img, hash: misses[i].hash }));
|
|
297
|
+
// Parse and fallback run inside callVision's try so a throw there is logged as
|
|
298
|
+
// a batch failure and the partially filled map is still returned.
|
|
299
|
+
for (const group of splitByByteBudget(items, cfg.maxRequestBytes)) {
|
|
300
|
+
await callVision(
|
|
301
|
+
group.map((m) => m.img),
|
|
302
|
+
group.map((m) => m.hash),
|
|
303
|
+
"batch",
|
|
304
|
+
userPrompt,
|
|
305
|
+
visionModel,
|
|
306
|
+
modelRegistry,
|
|
307
|
+
cfg,
|
|
308
|
+
deps,
|
|
309
|
+
(text, truncated) =>
|
|
310
|
+
fillBatch(out, text, truncated, group, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal),
|
|
311
|
+
turnSignal,
|
|
312
|
+
);
|
|
313
|
+
}
|
|
234
314
|
return out;
|
|
235
315
|
}
|
|
236
316
|
|