dsh-tool-vision 0.9.0 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,1332 +1,1453 @@
1
- /**
2
- * dsh-tool-vision — external vision model for DeepSeek Harness.
3
- *
4
- * Two capabilities:
5
- *
6
- * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
- * any OpenAI-compatible chat/completions endpoint that supports
8
- * `image_url` content parts, and returns the vision model's text answer.
9
- *
10
- * 2. Image bridge — pasted images are bridged to text hints before they
11
- * enter a text-only model's request:
12
- *
13
- * - New images are bridged on the `agent/pre-step` waterfall (the only
14
- * seam where the harness lets a plugin replace the messages that enter
15
- * a step — they become the durable `user/message` log, so the
16
- * `llm/stream` request-reconstruction invariant stays satisfied).
17
- * - Images already logged before the plugin was installed (or before a
18
- * server restart) are repaired lazily with a surface `replace`, one
19
- * event at a time, on the first pre-step of the session.
20
- *
21
- * The bridged hint points at an exported local copy of the image, which
22
- * the agent hands to `inspect_image`. Which models skip the bridge
23
- * (v0.9.0) is decided by two independent inputs:
24
- *
25
- * - `multimodalModels` — a user-owned list whose meaning is set by
26
- * `multimodalListMode`: `whitelist` (listed models receive image blocks
27
- * directly; the historical meaning and the default), `blacklist`
28
- * (listed models are forced through the bridge), or `off` (list
29
- * ignored). Entries match the full model id, its bare id after the last
30
- * `/`, or `provider/id`, and may contain `*` / `?` globs.
31
- * - `autoDetectMultimodal` — when on, the route's own declared
32
- * `inputModalities` forms the base set that the list then adds to
33
- * (whitelist) or subtracts from (blacklist). The declaration is always
34
- * read from the *unwrapped* `resolveModelInfo`, because the admission
35
- * wrap below rewrites it for every model.
36
- *
37
- * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
38
- * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
39
- * inline thumbnails for bridged pasted images:
40
- *
41
- * - `bridgeMessages` stamps every bridged hint with an invisible marker
42
- * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
43
- * precisely instead of pattern-matching free text.
44
- * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
45
- * exported images to the same-origin page; the client inserts a
46
- * thumbnail above the hint text and opens a lightbox on click.
47
- * - Pure display layer: persisted messages, the transcript, the
48
- * model-facing text and the `inspect_image` chain are untouched.
49
- */
50
- import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
51
- import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
52
- import os from "node:os";
53
- import z from "@deepseek-ai/schemastery";
54
- import { defineTool } from "@deepseek-ai/dsh-tools";
55
- import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
56
- import { sessionHeaders } from "./lib/session-header.js";
57
- import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
58
-
59
- /** Cordis plugin name. */
60
- const name = "tool-vision";
61
- /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
62
- const inject = ["tools", "llm", "attachments", "webServer"];
63
- /** Settings namespace owned by this plugin (Web UI settings section). */
64
- const NS = "tool-vision";
65
-
66
- /**
67
- * Invisible prefix stamped onto every bridged hint text block. The browser
68
- * half uses it to recognize bridge text precisely (no free-text regex over
69
- * user messages), and the model-facing hint stays intact otherwise.
70
- */
71
- const BRIDGE_MARKER = "\u200b[bridge]";
72
-
73
- /** Loopback route serving bridged images to the same-origin page. */
74
- const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
75
- /**
76
- * Loopback JSON route the settings panel reads the configured model catalog
77
- * from (autocomplete for `multimodalModels` and the current-route readout).
78
- * Registered on the plugin's own fiber, so it stays up while the master
79
- * switch is off — a section you cannot configure until you switch it on would
80
- * be useless.
81
- */
82
- const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
83
- /** Per-provider cap on catalog entries returned to the panel. */
84
- const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
85
- /** Hard cap for a single served image (defense in depth; export is bounded). */
86
- const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
87
- /** Extensions the preview route serves (svg/ico intentionally excluded). */
88
- const BRIDGE_PREVIEW_MEDIA = {
89
- ".png": "image/png",
90
- ".jpg": "image/jpeg",
91
- ".jpeg": "image/jpeg",
92
- ".webp": "image/webp",
93
- ".gif": "image/gif",
94
- ".avif": "image/avif",
95
- ".bmp": "image/bmp",
96
- };
97
-
98
- const DEFAULT_DESCRIPTION =
99
- "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
100
- "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
101
- "optionally with a specific question. Returns the vision model's textual description or answer. " +
102
- "Use this whenever you need to read, describe, or extract information from image content, " +
103
- "including an image that reached you as a bridged text hint.";
104
-
105
- /** Runtime schema for the tool-vision row. */
106
- const Config = z.object({
107
- /**
108
- * Master switch, hot-applied. When off the plugin registers nothing at all —
109
- * no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
110
- * preview route, no llm capability wrap — while the settings section itself
111
- * stays mounted so the switch can turn it back on. Turning it off is the way
112
- * to stop every vision tool from reaching the model without uninstalling the
113
- * package or restarting dsh.
114
- */
115
- enabled: z.boolean().default(true),
116
- /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
117
- baseURL: z.string().default("https://api.openai.com/v1"),
118
- /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
119
- apiKey: z.string().default("").role("secret"),
120
- /** Environment variable holding the API key. */
121
- apiKeyEnv: z.string().default("VISION_API_KEY"),
122
- /** Vision model id served by the endpoint. */
123
- model: z.string().default("gpt-4o-mini"),
124
- /** Max output tokens for the vision call. */
125
- maxTokens: z.number().default(4096),
126
- /** Per-request timeout in milliseconds. */
127
- timeoutMs: z.number().default(60000),
128
- /** Largest local image accepted, in bytes. */
129
- maxImageBytes: z.number().default(10 * 1024 * 1024),
130
- /** Tool description shown to the model; overrides the default. */
131
- description: z.string().default(DEFAULT_DESCRIPTION),
132
- /** Bridge pasted images to text hints on models that cannot see images. */
133
- bridgeTextOnly: z.boolean().default(true),
134
- /** Export directory for bridged images; empty = system temp. */
135
- bridgeExportDir: z.string().default(""),
136
- /**
137
- * Model ids the list below refers to (v0.9.0). Each entry is matched
138
- * case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
139
- * bare id after the last `/` (`mimo-v2.5`), and `provider/id`
140
- * (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
141
- * (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
142
- * route happens to spell it.
143
- */
144
- multimodalModels: z.array(z.string()).default([]),
145
- /**
146
- * How `multimodalModels` is read (v0.9.0):
147
- *
148
- * - `whitelist` (default): a listed model receives image blocks directly
149
- * and is never bridged — the meaning the list always had, so an
150
- * existing configuration keeps behaving exactly as before.
151
- * - `blacklist`: a listed model is forced through the bridge even when its
152
- * route declares image input. This is the correction layer for a model
153
- * that claims image support its endpoint does not really have.
154
- * - `off`: the list is ignored entirely; nothing is forced either way.
155
- *
156
- * An unknown value falls back to `whitelist`.
157
- */
158
- multimodalListMode: z.string().default("whitelist"),
159
- /**
160
- * Let the current route's own declared `inputModalities` drive the bridge
161
- * decision, so only the models that cannot see images get bridged (v0.9.0,
162
- * **on by default**). This and `multimodalModels` are one decision, not two:
163
- * the declaration forms the base set, and the list unions with it
164
- * (whitelist) or subtracts from it (blacklist).
165
- *
166
- * The declaration is only whatever the profile — or the installed catalog —
167
- * says, and it CAN be wrong: profiles sometimes declare
168
- * `input: [text, image]` just to pass the host admission gate, and a wrong
169
- * "yes" sends the image straight to an endpoint that may reject it. The
170
- * harness's own text-model projection is disabled by `bridgeAutoImage`, so
171
- * nothing underneath catches that. Two things bound the damage: the first
172
- * promotion of a route logs a notice naming the escape hatch
173
- * ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
174
- * under `blacklist` mode forces the bridge back on. Set this to false to
175
- * bridge everything the list does not name.
176
- *
177
- * The value is always read from the *unwrapped* `resolveModelInfo`, so the
178
- * admission wrap can never feed its own claim back in as evidence.
179
- */
180
- autoDetectMultimodal: z.boolean().default(true),
181
- /**
182
- * Measured image capability per route (v0.9.0), keyed `"provider/model"` with
183
- * the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
184
- * never by hand — the only entry in this schema that records what a route
185
- * DOES rather than what it says.
186
- *
187
- * It outranks the route's own declaration, because a probe is a real request
188
- * and a declaration is only a claim. It does NOT outrank `multimodalModels`:
189
- * that list is explicit human intent, and a person who names a model
190
- * deserves the last word over an automated measurement.
191
- */
192
- probeResults: z.dict(z.string()).default({}),
193
- /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
194
- bridgePreview: z.boolean().default(true),
195
- /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
196
- bridgePreviewScanIntervalMs: z.number().default(2000),
197
- /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
198
- bridgePreviewHideHint: z.boolean().default(true),
199
- /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
200
- desktopScreenshot: z.boolean().default(false),
201
- /**
202
- * Advertise image input capability for every model while the bridge is on.
203
- * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
204
- * pasted images unless the current model declares `image` in its
205
- * `inputModalities`. The bridge handles those images anyway (they become
206
- * text hints the agent inspects through `inspect_image`), so this wraps the
207
- * llm service's `resolveModelInfo` to report image support for text-only
208
- * models too — users can paste images on any model without hand-editing
209
- * provider model configs.
210
- */
211
- bridgeAutoImage: z.boolean().default(true),
212
- /** Send a stable per-conversation id header (`x-opencode-session`) on vision
213
- * requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
214
- * on every request; requests without the header may error. The value is the
215
- * current dsh session id when the call runs inside one, else `sessionId`
216
- * below, else a stable per-process random id. Set false to disable. */
217
- sendSessionHeader: z.boolean().default(true),
218
- /** Header name carrying the session id. */
219
- sessionHeaderName: z.string().default("x-opencode-session"),
220
- /** Fixed session id override for callers without a dsh session context.
221
- * Empty = auto (per-process stable id). */
222
- sessionId: z.string().default(""),
223
- });
224
-
225
- const MIME_BY_EXT = {
226
- ".png": "image/png",
227
- ".jpg": "image/jpeg",
228
- ".jpeg": "image/jpeg",
229
- ".gif": "image/gif",
230
- ".webp": "image/webp",
231
- ".bmp": "image/bmp",
232
- ".avif": "image/avif",
233
- ".svg": "image/svg+xml",
234
- ".ico": "image/x-icon",
235
- };
236
-
237
- const EXT_BY_MEDIA = {
238
- "image/png": ".png",
239
- "image/jpeg": ".jpg",
240
- "image/webp": ".webp",
241
- "image/gif": ".gif",
242
- };
243
-
244
- /** True when any message carries an image content block. */
245
- function hasImageBlock(messages) {
246
- return (messages ?? []).some((m) =>
247
- Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
248
- );
249
- }
250
-
251
- /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
252
- function deepFreeze(value) {
253
- if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
254
- if (Array.isArray(value)) {
255
- for (const item of value) deepFreeze(item);
256
- return Object.freeze(value);
257
- }
258
- for (const key of Object.keys(value)) deepFreeze(value[key]);
259
- return Object.freeze(value);
260
- }
261
-
262
- /** Export one attachment to disk; returns the file path (cached per process). */
263
- const exportedPaths = new Map();
264
- async function exportImage(attachment, ctx, dir) {
265
- const cached = exportedPaths.get(attachment.attachmentId);
266
- if (cached) return cached;
267
- const { data } = await ctx.attachments.readImage(attachment);
268
- const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
269
- const safeName = attachment.name
270
- ? attachment.name
271
- .replace(/\.[^.]+$/, "")
272
- .replace(/[^\w\-]+/g, "_")
273
- .replace(/^_+|_+$/g, "")
274
- .slice(0, 40)
275
- : "";
276
- const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
277
- const path = join(dir, `${base}${ext}`);
278
- await writeFile(path, data);
279
- exportedPaths.set(attachment.attachmentId, path);
280
- return path;
281
- }
282
-
283
- /**
284
- * Replace image content blocks with text hints pointing at exported files.
285
- * Non-image messages are returned as-is (same reference); bridged messages
286
- * are fresh, deep-frozen objects with the original identity and source.
287
- * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
288
- * prefix so the browser half can recognize it precisely.
289
- * Exported for unit testing; `ctx` only needs `attachments`.
290
- */
291
- async function bridgeMessages(messages, ctx, dir) {
292
- const next = [];
293
- for (const message of messages) {
294
- const content = message?.content;
295
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
296
- next.push(message);
297
- continue;
298
- }
299
- const blocks = [];
300
- for (const block of content) {
301
- if (block?.type !== "image") {
302
- blocks.push(block);
303
- continue;
304
- }
305
- const path = await exportImage(block.attachment, ctx, dir);
306
- const name = block.attachment.name ? ` (${block.attachment.name})` : "";
307
- blocks.push({
308
- type: "text",
309
- text:
310
- `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
311
- `Inspect it with the inspect_image tool to see its content.]`,
312
- });
313
- }
314
- next.push(deepFreeze({ ...message, content: blocks }));
315
- }
316
- return next;
317
- }
318
-
319
- /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
320
- function parseQuery(rawUrl) {
321
- const query = {};
322
- const at = rawUrl.indexOf("?");
323
- if (at === -1) return query;
324
- for (const pair of rawUrl.slice(at + 1).split("&")) {
325
- const eq = pair.indexOf("=");
326
- if (eq === -1) continue;
327
- try {
328
- query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
329
- } catch {
330
- /* skip malformed pairs */
331
- }
332
- }
333
- return query;
334
- }
335
-
336
- /**
337
- * Advertise `image` in the `inputModalities` reported by the llm service for
338
- * every model, so the host admission gate lets pasted images through on
339
- * text-only models. The bridge turns those images into text hints anyway, so
340
- * this is pure admission: it never changes what the adapter actually streams
341
- * (the adapter's own stream validation reads the model's real `input` from
342
- * the provider config, untouched here).
343
- *
344
- * The wrap is installed on the shared llm service instance, so it must be
345
- * idempotent across HMR re-applies and restored on dispose. `marker` is a
346
- * per-instance record proving this plugin already owns the wrap; the
347
- * `installed` flag distinguishes "we wrapped it" from "the original was
348
- * replaced by something else" so dispose only restores what we replaced.
349
- *
350
- * Exported for unit testing; the test passes a fake llm service.
351
- */
352
- const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
353
- function installAutoImageAdmission(llm, logger) {
354
- if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
355
- logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
356
- return () => {};
357
- }
358
- if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
359
- const original = llm.resolveModelInfo.bind(llm);
360
- const wrapped = async (provider, model, signal) => {
361
- const info = await original(provider, model, signal);
362
- if (!info) return info;
363
- const mods = info.inputModalities;
364
- if (Array.isArray(mods) && mods.includes("image")) return info;
365
- return { ...info, inputModalities: [...(mods ?? []), "image"] };
366
- };
367
- let installed = false;
368
- // The marker carries BOTH functions: the wrapped one proves ownership for
369
- // dispose, and `original` is the only honest source of a route's real
370
- // modalities once the wrap is in place (`unwrappedResolveModelInfo`).
371
- llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
372
- llm.resolveModelInfo = wrapped;
373
- installed = true;
374
- logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
375
- return () => {
376
- if (installed) {
377
- if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
378
- installed = false;
379
- }
380
- delete llm[LLM_RESOLVE_WRAP_MARK];
381
- };
382
- }
383
-
384
- /**
385
- * Register the loopback route that serves bridged images to the same-origin
386
- * page (the preview thumbnails). Read-only and tightly scoped:
387
- * - only files inside the bridge export directory (no traversal);
388
- * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
389
- * - Host restricted to the local machine;
390
- * - hard 20MB cap per file.
391
- */
392
- function registerBridgePreviewRoute(ctx, exportDir, logger) {
393
- const webServer = ctx.get("webServer");
394
- if (webServer === undefined) {
395
- logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
396
- return;
397
- }
398
- const bridgeDir = resolvePath(exportDir);
399
- ctx.effect(() => webServer.register({
400
- kind: "exact",
401
- path: BRIDGE_PREVIEW_ROUTE,
402
- async handler(req, res) {
403
- try {
404
- const raw = String(req.url ?? "");
405
- const query = parseQuery(raw);
406
- const p = query.p;
407
- if (typeof p !== "string" || p.length === 0) {
408
- res.writeHead(400);
409
- res.end("bad request");
410
- return;
411
- }
412
- const lower = p.toLowerCase();
413
- const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
414
- if (mediaType === undefined) {
415
- res.writeHead(400);
416
- res.end("not an image path");
417
- return;
418
- }
419
- const host = String(req.headers?.host ?? "");
420
- if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
421
- res.writeHead(403);
422
- res.end("forbidden");
423
- return;
424
- }
425
- const target = resolvePath(p);
426
- if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
427
- res.writeHead(403);
428
- res.end("forbidden");
429
- return;
430
- }
431
- const info = await stat(target);
432
- if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
433
- res.writeHead(404);
434
- res.end("not found");
435
- return;
436
- }
437
- const bytes = await readFile(target);
438
- res.writeHead(200, {
439
- "Content-Type": mediaType,
440
- "Cache-Control": "private, max-age=60",
441
- });
442
- res.end(bytes);
443
- } catch {
444
- try {
445
- res.writeHead(404);
446
- res.end("not found");
447
- } catch {
448
- /* response already sent */
449
- }
450
- }
451
- },
452
- }), "dsh-tool-vision: bridge preview route");
453
- }
454
-
455
- /**
456
- * Register the loopback JSON route the settings panel reads:
457
- *
458
- * { current: {provider, model, direct, source, mode}, providers: [...] }
459
- *
460
- * `providers` comes from `llm.listProviders()` + `llm.listModels(provider)` —
461
- * the routes the harness actually has configured, which is exactly the set a
462
- * `multimodalModels` entry may name. `listModels` is a different seam from
463
- * `resolveModelInfo`, so these declared modalities are the profile's own and
464
- * are NOT contaminated by the admission wrap (a model the wrap advertises as
465
- * image-capable still reports its real declaration here).
466
- *
467
- * Same confinement as the image route: loopback Host only, read-only,
468
- * `no-store`, and no credentials or endpoints in the payload — provider ids,
469
- * model ids/names and one boolean each.
470
- */
471
- function registerModelCatalogRoute(ctx, logger, getConfig) {
472
- const webServer = ctx.get("webServer");
473
- if (webServer === undefined) {
474
- logger?.warn?.("[tool-vision] webServer unavailable; model catalog route not registered");
475
- return;
476
- }
477
- ctx.effect(() => webServer.register({
478
- kind: "exact",
479
- path: MODEL_CATALOG_ROUTE,
480
- async handler(req, res) {
481
- const send = (status, payload) => {
482
- res.writeHead(status, {
483
- "Content-Type": "application/json; charset=utf-8",
484
- "Cache-Control": "no-store",
485
- });
486
- res.end(JSON.stringify(payload));
487
- };
488
- const host = String(req.headers?.host ?? "");
489
- if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
490
- send(403, { error: "forbidden" });
491
- return;
492
- }
493
- const llm = ctx.get("llm");
494
- if (llm === undefined) {
495
- send(503, { error: "llm service unavailable" });
496
- return;
497
- }
498
- // `listed` is computed HERE, with the same matcher the bridge itself
499
- // uses, so the panel never reimplements the rule (and can never
500
- // disagree with the decision it is describing).
501
- const cfg = typeof getConfig === "function" ? getConfig() : undefined;
502
- const list = Array.isArray(cfg?.multimodalModels) ? cfg.multimodalModels : [];
503
- const listMode = normalizeListMode(cfg?.multimodalListMode);
504
- let routes = [];
505
- try {
506
- routes = llm.listProviders() ?? [];
507
- } catch (error) {
508
- logger?.debug?.(`[tool-vision] listProviders failed: ${String(error)}`);
509
- }
510
- const providers = [];
511
- for (const entry of routes) {
512
- const id = entry?.id;
513
- if (typeof id !== "string" || id.length === 0) continue;
514
- const record = {
515
- id,
516
- name: typeof entry?.name === "string" && entry.name.length > 0 ? entry.name : id,
517
- models: [],
518
- };
519
- try {
520
- const models = await llm.listModels(id);
521
- record.models = (models ?? []).slice(0, MODEL_CATALOG_MAX_PER_PROVIDER).map((model) => {
522
- const modelId = String(model.id);
523
- const matchedEntries = modelListMatchEntries(list, id, modelId);
524
- return {
525
- id: modelId,
526
- name: typeof model.name === "string" && model.name.length > 0 ? model.name : modelId,
527
- image: Array.isArray(model.inputModalities) && model.inputModalities.includes("image"),
528
- listed: matchedEntries.length > 0,
529
- matchedEntries,
530
- // What the route was MEASURED doing, when anyone has probed it.
531
- // `null` means "not probed", which the panel must render
532
- // differently from a probe that came back negative.
533
- probe: probeVerdict(cfg, id, modelId) ?? null,
534
- };
535
- });
536
- } catch (error) {
537
- // One broken route must never blank the whole panel: report it and
538
- // keep going, so the other providers still list.
539
- record.error = String(error?.message ?? error);
540
- }
541
- providers.push(record);
542
- }
543
- send(200, {
544
- current: { ...lastModelDecision },
545
- list,
546
- listMode,
547
- probeResults: typeof getConfig === "function" ? { ...(getConfig()?.probeResults ?? {}) } : {},
548
- providers,
549
- });
550
- },
551
- }), "dsh-tool-vision: model catalog route");
552
- }
553
-
554
- /** Accepted `multimodalListMode` values; anything else falls back to whitelist. */
555
- const MULTIMODAL_LIST_MODES = ["off", "whitelist", "blacklist"];
556
- const DEFAULT_MULTIMODAL_LIST_MODE = "whitelist";
557
-
558
- /**
559
- * Normalize a configured list mode. The settings section only ever writes one
560
- * of {@link MULTIMODAL_LIST_MODES}, but the value can also arrive from a
561
- * hand-edited `settings.yaml`, so an unknown string is clamped rather than
562
- * treated as "no mode" (which would silently bridge everything).
563
- */
564
- function normalizeListMode(value) {
565
- return MULTIMODAL_LIST_MODES.includes(value) ? value : DEFAULT_MULTIMODAL_LIST_MODE;
566
- }
567
-
568
- const listEntryCache = new Map();
569
- /**
570
- * Compile one list entry into an anchored, case-insensitive regexp. A literal
571
- * entry is just an exact match; `*` and `?` are the only metacharacters, so a
572
- * model id containing `.` or `+` still matches itself.
573
- */
574
- function listEntryRegExp(entry) {
575
- const cached = listEntryCache.get(entry);
576
- if (cached !== undefined) return cached;
577
- const source = entry
578
- .replace(/[.+^${}()|[\]\\]/g, "\\$&")
579
- .replace(/\*/g, ".*")
580
- .replace(/\?/g, ".");
581
- const compiled = new RegExp(`^${source}$`, "i");
582
- listEntryCache.set(entry, compiled);
583
- return compiled;
584
- }
585
-
586
- /**
587
- * Whether any entry in `patterns` names this route. Each entry is tested
588
- * against the full model id, its bare id after the last `/`, and
589
- * `provider/id` — so `mimo-v2.5`, `xiaomi/mimo-v2.5` and
590
- * `commandcode/xiaomi/mimo-v2.5` all address the same route, and `*vl*`
591
- * addresses every variant of one family. Exported for unit testing.
592
- */
593
- function modelListMatches(patterns, provider, model) {
594
- return modelListMatchEntries(patterns, provider, model).length > 0;
595
- }
596
-
597
- /**
598
- * The list entries that address this route. The panel needs *which* entry hit
599
- * so that unchecking a model removes exactly the entry responsible — matching
600
- * `mimo-v2.5` against `xiaomi/mimo-v2.5` must not leave an orphan behind, and
601
- * must not delete an unrelated entry either. Exported for unit testing.
602
- */
603
- function modelListMatchEntries(patterns, provider, model) {
604
- if (!Array.isArray(patterns) || patterns.length === 0) return [];
605
- if (typeof model !== "string" || model.length === 0) return [];
606
- const slash = model.lastIndexOf("/");
607
- const candidates = [model, slash === -1 ? model : model.slice(slash + 1)];
608
- if (typeof provider === "string" && provider.length > 0) candidates.push(`${provider}/${model}`);
609
- const hits = [];
610
- for (const raw of patterns) {
611
- if (raw === undefined || raw === null) continue;
612
- const entry = String(raw).trim();
613
- if (entry.length === 0) continue;
614
- const re = listEntryRegExp(entry);
615
- if (candidates.some((candidate) => re.test(candidate))) hits.push(String(raw));
616
- }
617
- return hits;
618
- }
619
-
620
- /**
621
- * The route the session is on: the logged request header first, then the
622
- * agent's own options. `provider` is optional (older logs may lack it).
623
- */
624
- function currentRoute(agent) {
625
- const header = agent?.session?.requestHeader?.();
626
- return {
627
- provider: header?.config?.provider ?? agent?.options?.provider,
628
- model: header?.config?.model ?? agent?.options?.model,
629
- };
630
- }
631
-
632
- /**
633
- * The last decision `currentModelAcceptsImage` made, for the settings panel's
634
- * "current route" readout (served over {@link MODEL_CATALOG_ROUTE}). A plain
635
- * mutable record — the panel only ever reads it.
636
- */
637
- const lastModelDecision = {
638
- provider: undefined,
639
- model: undefined,
640
- direct: false,
641
- source: "none",
642
- mode: DEFAULT_MULTIMODAL_LIST_MODE,
643
- };
644
-
645
- /**
646
- * `resolveModelInfo` as it was before {@link installAutoImageAdmission}
647
- * wrapped it. Auto-detection MUST read through this: the wrap reports image
648
- * support for *every* model, so reading the wrapped method would make the
649
- * plugin's own claim the evidence for itself.
650
- */
651
- function unwrappedResolveModelInfo(llm) {
652
- const marker = llm?.[LLM_RESOLVE_WRAP_MARK];
653
- const fn = marker?.original ?? llm?.resolveModelInfo;
654
- return typeof fn === "function" ? fn.bind(llm) : undefined;
655
- }
656
-
657
- /**
658
- * Whether the route declares `image` input, read before the admission wrap.
659
- * `false` and `undefined` are the same answer to the caller ("not known to be
660
- * multimodal"): a route nobody could resolve is bridged, never guessed.
661
- */
662
- async function routeDeclaresImage(llm, provider, model) {
663
- const resolve = unwrappedResolveModelInfo(llm);
664
- if (resolve === undefined || typeof provider !== "string" || typeof model !== "string") {
665
- return false;
666
- }
667
- try {
668
- const info = await resolve(provider, model);
669
- return info?.inputModalities?.includes("image") === true;
670
- } catch {
671
- return false;
672
- }
673
- }
674
-
675
- /** Routes already warned about, so the notice appears once per route. */
676
- const autoPromotionWarned = new Set();
677
-
678
- /** Registry key for one measured route. Provider-qualified: the same model id
679
- * can be served by two gateways with different real capabilities. */
680
- function probeKey(provider, model) {
681
- return `${typeof provider === "string" ? provider : ""}/${model}`;
682
- }
683
-
684
- /**
685
- * The measured verdict for a route, or `undefined` when it was never probed.
686
- * Anything other than the two recorded values (a hand-edited settings.yaml, a
687
- * future format) reads as "not probed" rather than as a guess.
688
- */
689
- function probeVerdict(config, provider, model) {
690
- const results = config?.probeResults;
691
- if (results === undefined || results === null || typeof model !== "string" || model.length === 0) {
692
- return undefined;
693
- }
694
- const value = results[probeKey(provider, model)];
695
- return value === "yes" || value === "no" ? value : undefined;
696
- }
697
-
698
- /**
699
- * Resolve the endpoint and credential a route actually sends to, so a probe
700
- * talks to the same place the model does instead of a copy of it.
701
- *
702
- * The pieces come from two seams that a plugin is allowed to read:
703
- * - `llm.listConfigurableProviders()` names the settings namespace and the
704
- * path a provider's profile lives at;
705
- * - `settings.get(ns)` resolves that profile (`baseURL`, `apiKeyEnv`, `api`);
706
- * - `credentials.resolve(ref)` turns `apiKeyEnv` into the secret itself
707
- * (`.value`), the same way `dsh-llm-pi-ai` does it for a real call.
708
- *
709
- * Read-only, and the secret never leaves this function's caller: the probe
710
- * result records a verdict, never an endpoint or a key.
711
- *
712
- * @throws {Error} with a caller-facing reason when the route is unprobeable.
713
- */
714
- async function resolveProbeTarget(ctx, provider, model) {
715
- const llm = ctx.get("llm");
716
- const settings = ctx.get("settings");
717
- const credentials = ctx.get("credentials");
718
- if (settings === undefined) {
719
- throw new Error("the settings service is unavailable, so the route's endpoint cannot be read");
720
- }
721
- let entry;
722
- try {
723
- entry = (llm?.listConfigurableProviders?.() ?? []).find((candidate) => candidate?.provider === provider);
724
- } catch {
725
- /* an adapter that cannot describe its providers simply has no entry */
726
- }
727
- const settingsNs = entry?.settingsNs;
728
- if (typeof settingsNs !== "string" || settingsNs.length === 0) {
729
- throw new Error(
730
- `provider "${provider}" does not publish a settings namespace, so its endpoint cannot be resolved for probing`,
731
- );
732
- }
733
- let profile = settings.get(settingsNs);
734
- const path = Array.isArray(entry.settingsPath) && entry.settingsPath.length > 0 ? entry.settingsPath : [provider];
735
- for (const segment of path) profile = profile?.[segment];
736
- if (profile === undefined || profile === null || typeof profile !== "object") {
737
- throw new Error(`no stored profile for provider "${provider}" under "${settingsNs}"`);
738
- }
739
- const baseURL = typeof profile.baseURL === "string" && profile.baseURL.length > 0 ? profile.baseURL : undefined;
740
- if (baseURL === undefined) {
741
- throw new Error(`provider "${provider}" has no baseURL, so its endpoint cannot be probed`);
742
- }
743
- let apiKey;
744
- const ref = profile.apiKeyEnv;
745
- if (typeof ref === "string" && ref.length > 0) {
746
- try {
747
- const record = await credentials?.resolve?.(ref);
748
- apiKey = typeof record?.value === "string" ? record.value : undefined;
749
- } catch (error) {
750
- throw new Error(`stored credential "${ref}" could not be read: ${String(error?.message ?? error)}`);
751
- }
752
- }
753
- return { baseURL, apiKey, api: profile.api, model };
754
- }
755
-
756
- /**
757
- * One-time notice when a route receives images directly *only* because of its
758
- * own declaration (source `auto`). This is the one failure auto-detection
759
- * cannot rule out: if the declaration is wrong, the endpoint rejects the image
760
- * after the message is already durable, and the error alone does not say what
761
- * to change. Naming the escape hatch the first time it happens turns a
762
- * confusing 400 into a one-line fix — and now names the probe too, which
763
- * settles the question instead of asking the user to guess.
764
- * Read {@link lastModelDecision}; never throws.
765
- */
766
- function warnAutoPromotion(logger) {
767
- const { provider, model, direct, source } = lastModelDecision;
768
- if (direct !== true || source !== "auto" || typeof model !== "string" || model.length === 0) return;
769
- const key = `${typeof provider === "string" ? provider : ""}\u0000${model}`;
770
- if (autoPromotionWarned.has(key)) return;
771
- autoPromotionWarned.add(key);
772
- logger?.warn?.(
773
- `[tool-vision] "${model}" receives images directly because its route declares image input, ` +
774
- `which is unverified; run vision_probe_model to measure it, or list it in multimodalModels ` +
775
- `under multimodalListMode: blacklist to force the bridge`,
776
- );
777
- }
778
-
779
- /**
780
- * Whether the session's current model may receive image blocks directly.
781
- *
782
- * Two inputs, deliberately independent:
783
- *
784
- * - the base set — the route's own declared `inputModalities`, and only when
785
- * `autoDetectMultimodal` is on. It is read through
786
- * {@link unwrappedResolveModelInfo}, never through the admission wrap.
787
- * - the list — `multimodalModels` under `multimodalListMode`, and the list is
788
- * always explicit user intent, so it outranks the declaration: whitelist
789
- * adds to the base set, blacklist subtracts from it.
790
- *
791
- * Returns true when bridging is disabled (nothing would be bridged anyway).
792
- */
793
- async function currentModelAcceptsImage(agent, config, llm) {
794
- const mode = normalizeListMode(config.multimodalListMode);
795
- if (!config.bridgeTextOnly) {
796
- Object.assign(lastModelDecision, { ...currentRoute(agent), direct: true, source: "bridge-off", mode });
797
- return true;
798
- }
799
- const { provider, model } = currentRoute(agent);
800
- if (!model) {
801
- Object.assign(lastModelDecision, { provider, model, direct: false, source: "no-route", mode });
802
- return false;
803
- }
804
- const base = config.autoDetectMultimodal
805
- ? await routeDeclaresImage(llm, provider, model)
806
- : false;
807
- let direct = base;
808
- let source = base ? "auto" : "default";
809
- // A measured verdict outranks the route's own claim: a probe is a real
810
- // request, a declaration is only a statement. Both directions count, so a
811
- // route that says "image" and answered without reading the pixels goes back
812
- // to the bridge without anyone hunting for the right list entry.
813
- const probed = probeVerdict(config, provider, model);
814
- if (probed !== undefined) {
815
- direct = probed === "yes";
816
- source = probed === "yes" ? "probe-yes" : "probe-no";
817
- }
818
- // `off` means the list carries no opinion at all — it is not "empty list
819
- // under whitelist", it is "no list semantics", so nothing is forced. The
820
- // human list keeps the last word over both other signals.
821
- const listed = mode !== "off" && modelListMatches(config.multimodalModels, provider, model);
822
- if (listed) {
823
- direct = mode !== "blacklist";
824
- source = mode === "blacklist" ? "blacklist" : "whitelist";
825
- }
826
- Object.assign(lastModelDecision, { provider, model, direct, source, mode });
827
- return direct;
828
- }
829
-
830
- /**
831
- * Lazily bridge image blocks that are already part of the session log
832
- * (pasted before the plugin was active, or before a restart). Each affected
833
- * event is rewritten once with a surface `replace`, which swaps the durable
834
- * derivation (and the transcript) to the text hint. Events that are no
835
- * longer on the surface (already shadowed) are skipped and remembered.
836
- * `repaired` tracks per-session state: a `Set` of handled seqs plus a
837
- * monotonic scan cursor.
838
- */
839
- async function repairLoggedImages(ctx, session, exportDir, repaired) {
840
- const events = session.events;
841
- for (let index = repaired.cursor; index < events.length; index += 1) {
842
- const event = events[index];
843
- if (event.type !== "user/message" || repaired.set.has(event.seq)) {
844
- repaired.set.add(event.seq);
845
- continue;
846
- }
847
- const content = event.data?.content;
848
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
849
- repaired.set.add(event.seq);
850
- continue;
851
- }
852
- const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
853
- try {
854
- session.append("user/message", bridged, {
855
- surfaceOp: { op: "replace", start: event.seq, end: event.seq },
856
- sourceEventSeqs: [event.seq],
857
- });
858
- ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
859
- } catch (error) {
860
- ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
861
- }
862
- repaired.set.add(event.seq);
863
- }
864
- repaired.cursor = events.length;
865
- }
866
-
867
- /**
868
- * Install the pre-step bridge at the root level. Agent-scoped waterfalls
869
- * admit untagged (root) listeners, so one listener serves every agent —
870
- * including sessions resumed after a server restart, which never re-fire
871
- * `session/created` for per-agent attachments. Runs before every proposed
872
- * step: new pasted images are bridged into the durable log, and stuck
873
- * logged images are repaired, before the model request is derived from it.
874
- */
875
- function attachPreStepBridge(ctx, getConfig, exportDir) {
876
- const repairedBySession = new Map();
877
- ctx.on("agent/pre-step", async (payload, next) => {
878
- const decision = await next();
879
- if (!decision || decision.kind !== "enter") return decision;
880
- const agent = payload?.agent;
881
- if (!agent?.session) return decision;
882
- try {
883
- const acceptsImage = await currentModelAcceptsImage(agent, getConfig(), ctx.get("llm"));
884
- warnAutoPromotion(ctx.logger);
885
- if (!acceptsImage) {
886
- let repaired = repairedBySession.get(agent.session.id);
887
- if (!repaired) {
888
- repaired = { set: new Set(), cursor: 0 };
889
- repairedBySession.set(agent.session.id, repaired);
890
- }
891
- await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
892
- ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
893
- });
894
- }
895
- if (acceptsImage) return decision;
896
- const messages = await bridgeMessages(decision.messages, ctx, exportDir);
897
- if (messages.every((message, index) => message === decision.messages[index])) return decision;
898
- return { ...decision, messages };
899
- } catch (error) {
900
- ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
901
- return decision;
902
- }
903
- });
904
- }
905
-
906
- function resolveApiKey(config) {
907
- if (config.apiKey) return config.apiKey;
908
- if (config.apiKeyEnv) {
909
- const fromEnv = process.env[config.apiKeyEnv];
910
- if (fromEnv) return fromEnv;
911
- }
912
- return process.env.OPENAI_API_KEY ?? "";
913
- }
914
-
915
- /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
916
- async function toImageUrl(target, cwd, config) {
917
- if (/^https?:\/\//i.test(target)) return { url: target, note: target };
918
- const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
919
- const info = await stat(abs).catch(() => null);
920
- if (!info) throw new Error(`image not found: ${abs}`);
921
- if (info.size > config.maxImageBytes) {
922
- throw new Error(
923
- `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
924
- );
925
- }
926
- const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
927
- if (!mime) {
928
- throw new Error(
929
- `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
930
- );
931
- }
932
- const data = await readFile(abs);
933
- return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
934
- }
935
-
936
- /** One OpenAI-compatible chat/completions call with an image_url content part. */
937
- async function callVision(config, imageUrl, question, detail, signal, exec) {
938
- const key = resolveApiKey(config);
939
- if (!key) {
940
- throw new Error(
941
- `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
942
- );
943
- }
944
- const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
945
- const endpoint = new URL("chat/completions", base);
946
- const sessionHeader = sessionHeaders(config, exec);
947
- const controller = new AbortController();
948
- const timer = setTimeout(
949
- () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
950
- config.timeoutMs,
951
- );
952
- const onSignalAbort = () => controller.abort(signal?.reason);
953
- if (signal) {
954
- if (signal.aborted) controller.abort();
955
- else signal.addEventListener("abort", onSignalAbort, { once: true });
956
- }
957
- const content = [
958
- { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
959
- { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
960
- ];
961
- try {
962
- const response = await fetch(endpoint, {
963
- method: "POST",
964
- headers: {
965
- "Content-Type": "application/json",
966
- Authorization: `Bearer ${key}`,
967
- ...sessionHeader,
968
- },
969
- body: JSON.stringify({
970
- model: config.model,
971
- messages: [{ role: "user", content }],
972
- max_tokens: config.maxTokens,
973
- }),
974
- signal: controller.signal,
975
- });
976
- const body = await response.json().catch(() => null);
977
- if (!response.ok) {
978
- const detailText = body?.error?.message ?? response.statusText;
979
- throw new Error(
980
- `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
981
- );
982
- }
983
- // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
984
- // `reasoning_content` first; when the final `content` is empty or was cut
985
- // off by max_tokens, fall back to the reasoning text so the answer is
986
- // still useful.
987
- const message = body?.choices?.[0]?.message;
988
- let answer = message?.content ?? "";
989
- if (!answer.trim()) answer = message?.reasoning_content ?? "";
990
- if (typeof answer !== "string" || !answer.trim()) {
991
- throw new Error("vision endpoint returned an empty response");
992
- }
993
- return answer.trim();
994
- } finally {
995
- clearTimeout(timer);
996
- signal?.removeEventListener("abort", onSignalAbort);
997
- }
998
- }
999
-
1000
- function apply(ctx, config) {
1001
- // ── settings-backed configuration ─────────────────────────────────────────
1002
- // The composition entry stays the `base` layer; a registered `tool-vision`
1003
- // settings section (Web UI section, settings.yaml) overlays it live, so
1004
- // edits hot-apply without a restart. `sourceGetter` is a GETTER
1005
- // (`() => scope.get()`), not the config object — keep it and call it at use
1006
- // time, or `getConfig()` would return a function and every `cfg.*` read
1007
- // would be undefined (apiKey included).
1008
- let current = config;
1009
- let sourceGetter = null;
1010
- const getConfig = () => (sourceGetter ? sourceGetter() : current);
1011
- // The registered settings scope, kept so tools can PERSIST what they measure.
1012
- // Null whenever no settings provider is mounted (composition-entry-only runs),
1013
- // in which case a probe still reports its verdict but cannot record it.
1014
- let settingsScope = null;
1015
- const getSettingsScope = () => settingsScope;
1016
-
1017
- // ── master switch: a child fiber owns every registration ──────────────────
1018
- // `ctx.tools.register`, `ctx.effect` and `ctx.on` are all effects on the fiber
1019
- // that makes them, so confining the tool/bridge/route registrations to a child
1020
- // fiber makes `enabled` real: disposing that fiber unregisters all 15 tools,
1021
- // removes the pre-step listener and the preview route, and restores the llm
1022
- // capability wrap. The settings registration deliberately stays on the parent
1023
- // fiber — it has to outlive the switch, or the section that turns the plugin
1024
- // back on would vanish with it.
1025
- const REGISTRATION_KEYS = [
1026
- "enabled",
1027
- "bridgeTextOnly",
1028
- "bridgeExportDir",
1029
- "bridgeAutoImage",
1030
- "bridgePreview",
1031
- ];
1032
- let featureFiber = null;
1033
- let lastRegistrationKey = null;
1034
-
1035
- function registrationKey(cfg) {
1036
- return REGISTRATION_KEYS.map((key) => String(cfg[key])).join("|");
1037
- }
1038
-
1039
- function installFeatures() {
1040
- if (featureFiber !== null) return;
1041
- featureFiber = ctx.plugin({
1042
- name: "dsh-tool-vision:features",
1043
- apply(inner) {
1044
- // ── image bridge: pasted images become inspect_image hints on text-only models ──
1045
- if (getConfig().bridgeTextOnly) {
1046
- const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
1047
- mkdir(exportDir, { recursive: true }).catch(() => {});
1048
- // Root-level listener: agent-scoped waterfalls admit untagged listeners,
1049
- // so one registration serves every agent (new and resumed alike) and the
1050
- // agent is read from the fused payload.
1051
- attachPreStepBridge(inner, getConfig, exportDir);
1052
-
1053
- // ── automatic image admission: let pasted images through on text-only models ──
1054
- // The host gate refuses images unless the model declares `image` input;
1055
- // the bridge handles them anyway, so report image support for all models.
1056
- // The wrap is installed on the shared llm service instance (idempotent,
1057
- // restored on dispose/HMR).
1058
- if (getConfig().bridgeAutoImage) {
1059
- const unwrap = installAutoImageAdmission(inner.get("llm"), inner.logger);
1060
- inner.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
1061
- }
1062
-
1063
- // ── bridge image preview: same-origin thumbnails for bridged images ──
1064
- if (getConfig().bridgePreview) {
1065
- registerBridgePreviewRoute(inner, exportDir, inner.logger);
1066
- }
1067
- }
1068
-
1069
- inner.tools.register(defineTool({
1070
- name: "inspect_image",
1071
- description: getConfig().description,
1072
- parameters: {
1073
- path: {
1074
- type: "string",
1075
- required: true,
1076
- description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
1077
- },
1078
- question: {
1079
- type: "string",
1080
- description: "Optional specific question about the image. Omit for a general detailed description.",
1081
- },
1082
- detail: {
1083
- type: "string",
1084
- enum: ["auto", "low", "high"],
1085
- description: "Optional image resolution hint for the vision API (auto by default).",
1086
- },
1087
- },
1088
- output: {
1089
- schema: { type: "string" },
1090
- render: (_args, value) => [{ type: "text", text: value }],
1091
- },
1092
- async execute(args, exec) {
1093
- const cfg = getConfig();
1094
- const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
1095
- const { url, note } = await toImageUrl(args.path, cwd, cfg);
1096
- try {
1097
- const answer = await callVision(cfg, url, args.question, args.detail, exec.signal, exec);
1098
- return note === url ? answer : `${answer}\n\n(image: ${note})`;
1099
- } catch (error) {
1100
- const raw = error && error.message ? String(error.message) : String(error);
1101
- if (CONTENT_FILTER_RE.test(raw)) {
1102
- throw new Error(
1103
- "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
1104
- "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
1105
- );
1106
- }
1107
- throw error;
1108
- }
1109
- },
1110
- }));
1111
-
1112
- // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
1113
- // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
1114
- // (baseURL/apiKey/model). No provider chain, no local models, no extra
1115
- // settings: everything comes from the existing tool-vision configuration.
1116
- registerVisionTools(inner, getConfig);
1117
-
1118
- // ── ground-truth capability probe (v0.9.0) ───────────────────────────
1119
- // Everything else decides from what a route SAYS; this measures what it
1120
- // DOES, and records the answer so the bridge decision stops guessing.
1121
- inner.tools.register(defineTool({
1122
- name: "vision_probe_model",
1123
- description:
1124
- "Measure whether a model's endpoint can actually read images, by sending it a real " +
1125
- "solid-color image and checking the answer. Use this when a model's declared image " +
1126
- "support is doubtful (it claims `image` but the provider may reject it, or it declares " +
1127
- "text-only yet the endpoint may accept images). Runs a text-only control request plus " +
1128
- "two different colors, so a model that merely guesses cannot pass. The verdict is " +
1129
- "recorded and from then on decides whether that route's images are sent directly or " +
1130
- "bridged. Costs a few requests against the model's own endpoint.",
1131
- parameters: {
1132
- model: {
1133
- type: "string",
1134
- required: true,
1135
- description: "Model id to probe, e.g. xiaomi/mimo-v2.5. Use the id exactly as dsh has it configured.",
1136
- },
1137
- provider: {
1138
- type: "string",
1139
- description:
1140
- "Provider route serving the model (e.g. commandcode). Omit to use the current " +
1141
- "session's route, or to auto-select when exactly one provider offers the model.",
1142
- },
1143
- },
1144
- output: {
1145
- schema: { type: "string" },
1146
- render: (_args, value) => [{ type: "text", text: value }],
1147
- },
1148
- async execute(args, exec) {
1149
- const cfg = getConfig();
1150
- const requested = String(args.model ?? "").trim();
1151
- if (requested.length === 0) throw new Error("vision_probe_model: model is required");
1152
- const llm = inner.get("llm");
1153
-
1154
- // Resolve which routes offer this model, so an omitted `provider`
1155
- // is still an exact choice rather than a guess.
1156
- let candidates = [];
1157
- try {
1158
- for (const entry of llm?.listProviders?.() ?? []) {
1159
- const id = entry?.id;
1160
- if (typeof id !== "string") continue;
1161
- const models = await llm.listModels(id).catch(() => []);
1162
- if ((models ?? []).some((model) => String(model?.id) === requested)) candidates.push(id);
1163
- }
1164
- } catch {
1165
- /* fall through to the route/argument fallbacks below */
1166
- }
1167
- let provider = typeof args.provider === "string" && args.provider.length > 0 ? args.provider : undefined;
1168
- if (provider === undefined) {
1169
- const here = currentRoute(exec?.agent);
1170
- if (candidates.length === 1) provider = candidates[0];
1171
- else if (here.model === requested && typeof here.provider === "string") provider = here.provider;
1172
- else if (candidates.length > 1) {
1173
- throw new Error(
1174
- `vision_probe_model: "${requested}" is served by ${candidates.join(", ")}; pass provider explicitly`,
1175
- );
1176
- } else {
1177
- provider = here.provider;
1178
- }
1179
- }
1180
- if (typeof provider !== "string" || provider.length === 0) {
1181
- throw new Error(
1182
- `vision_probe_model: cannot tell which provider serves "${requested}"; pass provider explicitly`,
1183
- );
1184
- }
1185
-
1186
- const target = await resolveProbeTarget(inner, provider, requested);
1187
- const outcome = await probeModelCapability(
1188
- {
1189
- baseURL: target.baseURL,
1190
- apiKey: target.apiKey,
1191
- api: target.api,
1192
- model: requested,
1193
- maxTokens: PROBE_MAX_TOKENS,
1194
- timeoutMs: Math.max(PROBE_TIMEOUT_MS, Number(cfg.timeoutMs) || 0),
1195
- headers: sessionHeaders(cfg, exec),
1196
- },
1197
- exec.signal,
1198
- );
1199
-
1200
- const key = probeKey(provider, requested);
1201
- let recorded = "not recorded (no settings provider)";
1202
- if (outcome.support === "yes" || outcome.support === "no") {
1203
- const scope = getSettingsScope();
1204
- if (scope !== null) {
1205
- try {
1206
- await scope.update({ probeResults: { ...(cfg.probeResults ?? {}), [key]: outcome.support } });
1207
- recorded = `recorded as "${outcome.support}"`;
1208
- } catch (error) {
1209
- recorded = `could not be recorded: ${String(error?.message ?? error)}`;
1210
- }
1211
- }
1212
- }
1213
-
1214
- const verdictText = outcome.support === "yes"
1215
- ? "SUPPORTS images"
1216
- : outcome.support === "no"
1217
- ? "does NOT read images"
1218
- : "INCONCLUSIVE (the route could not be probed)";
1219
- const effect = outcome.support === "yes"
1220
- ? "images will be sent to it directly"
1221
- : outcome.support === "no"
1222
- ? "images will be bridged to inspect_image hints"
1223
- : "the existing list/declaration rules still apply";
1224
- const evidence = Object.entries(outcome.evidence ?? {})
1225
- .map(([name, value]) => ` ${name}: ${value}`)
1226
- .join("\n");
1227
- return [
1228
- `${provider}/${requested}: ${verdictText}`,
1229
- ` ${outcome.detail}`,
1230
- ` ${recorded}; ${effect}`,
1231
- evidence ? ` evidence:\n${evidence}` : "",
1232
- ].filter(Boolean).join("\n");
1233
- },
1234
- }));
1235
- },
1236
- });
1237
- }
1238
-
1239
- function uninstallFeatures() {
1240
- if (featureFiber === null) return;
1241
- const fiber = featureFiber;
1242
- featureFiber = null;
1243
- // dispose() settles asynchronously and must never take the plugin down; the
1244
- // next install starts a fresh fiber regardless of how this one ends.
1245
- Promise.resolve(fiber.dispose()).catch((error) => {
1246
- ctx.logger?.warn?.(`[tool-vision] feature teardown failed: ${String(error)}`);
1247
- });
1248
- }
1249
-
1250
- // Re-install only when a field that gates a registration actually changed, so
1251
- // editing, say, `model` does not tear the tools down and back up mid-session.
1252
- function syncFeatures() {
1253
- const cfg = getConfig();
1254
- const key = registrationKey(cfg);
1255
- if (featureFiber !== null && key === lastRegistrationKey) return;
1256
- uninstallFeatures();
1257
- lastRegistrationKey = key;
1258
- if (cfg.enabled) installFeatures();
1259
- }
1260
-
1261
- // Compat shim: dsh-settings 0.1.2-rc.1 removed the module-level
1262
- // `installSettingsSection` export (the provider now lives at ctx.settings).
1263
- // Inline the same logic via ctx.inject(["settings"]) — works on both
1264
- // 0.1.1 (module export wrapper) and 0.1.2 (ctx.settings) hosts.
1265
- ctx.inject(["settings"], (sctx) => {
1266
- const scope = sctx.settings.register(NS, Config, { base: config });
1267
- sourceGetter = () => scope.get();
1268
- settingsScope = scope;
1269
- sctx.effect(() => () => {
1270
- sourceGetter = null;
1271
- settingsScope = null;
1272
- });
1273
- // Hot-apply: `enabled` switches the child fiber, and any field that gates a
1274
- // registration re-installs it, so none of them needs a dsh restart anymore.
1275
- scope.watch(() => syncFeatures());
1276
- sctx.effect(() => () => uninstallFeatures());
1277
- // The stored value governs from here on; the composition entry was only a
1278
- // placeholder until the provider answered.
1279
- syncFeatures();
1280
- });
1281
-
1282
- // ── model catalog route: settings support, so it lives on THIS fiber ───────
1283
- // The master switch disposes the feature fiber on purpose (tools, bridge,
1284
- // preview route), while the settings section itself deliberately stays
1285
- // mounted — and a section whose list field cannot autocomplete is a section
1286
- // you cannot configure before switching the plugin on. Registered here so it
1287
- // outlives `enabled: false`; read-only and secret-free by construction.
1288
- registerModelCatalogRoute(ctx, ctx.logger, getConfig);
1289
-
1290
- // No settings provider: the composition entry is the whole configuration.
1291
- syncFeatures();
1292
- }
1293
-
1294
- export {
1295
- BRIDGE_MARKER,
1296
- BRIDGE_PREVIEW_MAX_BYTES,
1297
- BRIDGE_PREVIEW_MEDIA,
1298
- BRIDGE_PREVIEW_ROUTE,
1299
- Config,
1300
- DEFAULT_DESCRIPTION,
1301
- DEFAULT_MULTIMODAL_LIST_MODE,
1302
- EXT_BY_MEDIA,
1303
- MODEL_CATALOG_MAX_PER_PROVIDER,
1304
- MODEL_CATALOG_ROUTE,
1305
- MULTIMODAL_LIST_MODES,
1306
- apply,
1307
- attachPreStepBridge,
1308
- autoPromotionWarned,
1309
- bridgeMessages,
1310
- currentModelAcceptsImage,
1311
- currentRoute,
1312
- deepFreeze,
1313
- exportImage,
1314
- hasImageBlock,
1315
- inject,
1316
- installAutoImageAdmission,
1317
- lastModelDecision,
1318
- modelListMatches,
1319
- modelListMatchEntries,
1320
- name,
1321
- normalizeListMode,
1322
- parseQuery,
1323
- probeKey,
1324
- probeVerdict,
1325
- registerBridgePreviewRoute,
1326
- registerModelCatalogRoute,
1327
- repairLoggedImages,
1328
- resolveProbeTarget,
1329
- routeDeclaresImage,
1330
- unwrappedResolveModelInfo,
1331
- warnAutoPromotion,
1332
- };
1
+ /**
2
+ * dsh-tool-vision — external vision model for DeepSeek Harness.
3
+ *
4
+ * Two capabilities:
5
+ *
6
+ * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
+ * any OpenAI-compatible chat/completions endpoint that supports
8
+ * `image_url` content parts, and returns the vision model's text answer.
9
+ *
10
+ * 2. Image bridge — pasted images are bridged to text hints before they
11
+ * enter a text-only model's request:
12
+ *
13
+ * - New images are bridged on the `agent/pre-step` waterfall (the only
14
+ * seam where the harness lets a plugin replace the messages that enter
15
+ * a step — they become the durable `user/message` log, so the
16
+ * `llm/stream` request-reconstruction invariant stays satisfied).
17
+ * - Images already logged before the plugin was installed (or before a
18
+ * server restart) are repaired lazily with a surface `replace`, one
19
+ * event at a time, on the first pre-step of the session.
20
+ *
21
+ * The bridged hint points at an exported local copy of the image, which
22
+ * the agent hands to `inspect_image`. Which models skip the bridge
23
+ * (v0.9.0) is decided by two independent inputs:
24
+ *
25
+ * - `multimodalModels` — a user-owned list whose meaning is set by
26
+ * `multimodalListMode`: `whitelist` (listed models receive image blocks
27
+ * directly; the historical meaning and the default), `blacklist`
28
+ * (listed models are forced through the bridge), or `off` (list
29
+ * ignored). Entries match the full model id, its bare id after the last
30
+ * `/`, or `provider/id`, and may contain `*` / `?` globs.
31
+ * - `autoDetectMultimodal` — when on, the route's own declared
32
+ * `inputModalities` forms the base set that the list then adds to
33
+ * (whitelist) or subtracts from (blacklist). The declaration is always
34
+ * read from the *unwrapped* `resolveModelInfo`, because the admission
35
+ * wrap below rewrites it for every model.
36
+ *
37
+ * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
38
+ * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
39
+ * inline thumbnails for bridged pasted images:
40
+ *
41
+ * - `bridgeMessages` stamps every bridged hint with an invisible marker
42
+ * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
43
+ * precisely instead of pattern-matching free text.
44
+ * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
45
+ * exported images to the same-origin page; the client inserts a
46
+ * thumbnail above the hint text and opens a lightbox on click.
47
+ * - Pure display layer: persisted messages, the transcript, the
48
+ * model-facing text and the `inspect_image` chain are untouched.
49
+ */
50
+ import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
51
+ import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
52
+ import os from "node:os";
53
+ import z from "@deepseek-ai/schemastery";
54
+ import { defineTool } from "@deepseek-ai/dsh-tools";
55
+ import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
56
+ import { sessionHeaders } from "./lib/session-header.js";
57
+ import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
58
+
59
+ /** Cordis plugin name. */
60
+ const name = "tool-vision";
61
+ /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
62
+ const inject = ["tools", "llm", "attachments", "webServer"];
63
+ /** Settings namespace owned by this plugin (Web UI settings section). */
64
+ const NS = "tool-vision";
65
+
66
+ /**
67
+ * Invisible prefix stamped onto every bridged hint text block. The browser
68
+ * half uses it to recognize bridge text precisely (no free-text regex over
69
+ * user messages), and the model-facing hint stays intact otherwise.
70
+ */
71
+ const BRIDGE_MARKER = "\u200b[bridge]";
72
+
73
+ /** Loopback route serving bridged images to the same-origin page. */
74
+ const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
75
+ /**
76
+ * Loopback JSON route the settings panel reads the configured model catalog
77
+ * from (autocomplete for `multimodalModels` and the current-route readout).
78
+ * Registered on the plugin's own fiber, so it stays up while the master
79
+ * switch is off — a section you cannot configure until you switch it on would
80
+ * be useless.
81
+ */
82
+ const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
83
+ /** Per-provider cap on catalog entries returned to the panel. */
84
+ const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
85
+ /** Hard cap for a single served image (defense in depth; export is bounded). */
86
+ const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
87
+ /** Extensions the preview route serves (svg/ico intentionally excluded). */
88
+ const BRIDGE_PREVIEW_MEDIA = {
89
+ ".png": "image/png",
90
+ ".jpg": "image/jpeg",
91
+ ".jpeg": "image/jpeg",
92
+ ".webp": "image/webp",
93
+ ".gif": "image/gif",
94
+ ".avif": "image/avif",
95
+ ".bmp": "image/bmp",
96
+ };
97
+
98
+ const DEFAULT_DESCRIPTION =
99
+ "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
100
+ "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
101
+ "optionally with a specific question. Returns the vision model's textual description or answer. " +
102
+ "Use this whenever you need to read, describe, or extract information from image content, " +
103
+ "including an image that reached you as a bridged text hint.";
104
+
105
+ /** Runtime schema for the tool-vision row. */
106
+ const Config = z.object({
107
+ /**
108
+ * Master switch, hot-applied. When off the plugin registers nothing at all —
109
+ * no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
110
+ * preview route, no llm capability wrap — while the settings section itself
111
+ * stays mounted so the switch can turn it back on. Turning it off is the way
112
+ * to stop every vision tool from reaching the model without uninstalling the
113
+ * package or restarting dsh.
114
+ */
115
+ enabled: z.boolean().default(true),
116
+ /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
117
+ baseURL: z.string().default("https://api.openai.com/v1"),
118
+ /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
119
+ apiKey: z.string().default("").role("secret"),
120
+ /** Environment variable holding the API key. */
121
+ apiKeyEnv: z.string().default("VISION_API_KEY"),
122
+ /** Vision model id served by the endpoint. */
123
+ model: z.string().default("gpt-4o-mini"),
124
+ /** Max output tokens for the vision call. */
125
+ maxTokens: z.number().default(4096),
126
+ /** Per-request timeout in milliseconds. */
127
+ timeoutMs: z.number().default(60000),
128
+ /** Largest local image accepted, in bytes. */
129
+ maxImageBytes: z.number().default(10 * 1024 * 1024),
130
+ /** Tool description shown to the model; overrides the default. */
131
+ description: z.string().default(DEFAULT_DESCRIPTION),
132
+ /** Bridge pasted images to text hints on models that cannot see images. */
133
+ bridgeTextOnly: z.boolean().default(true),
134
+ /** Export directory for bridged images; empty = system temp. */
135
+ bridgeExportDir: z.string().default(""),
136
+ /**
137
+ * Model ids the list below refers to (v0.9.0). Each entry is matched
138
+ * case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
139
+ * bare id after the last `/` (`mimo-v2.5`), and `provider/id`
140
+ * (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
141
+ * (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
142
+ * route happens to spell it.
143
+ */
144
+ multimodalModels: z.array(z.string()).default([]),
145
+ /**
146
+ * How `multimodalModels` is read (v0.9.0):
147
+ *
148
+ * - `whitelist` (default): a listed model receives image blocks directly
149
+ * and is never bridged — the meaning the list always had, so an
150
+ * existing configuration keeps behaving exactly as before.
151
+ * - `blacklist`: a listed model is forced through the bridge even when its
152
+ * route declares image input. This is the correction layer for a model
153
+ * that claims image support its endpoint does not really have.
154
+ * - `off`: the list is ignored entirely; nothing is forced either way.
155
+ *
156
+ * An unknown value falls back to `whitelist`.
157
+ */
158
+ multimodalListMode: z.string().default("whitelist"),
159
+ /**
160
+ * Let the current route's own declared `inputModalities` drive the bridge
161
+ * decision, so only the models that cannot see images get bridged (v0.9.0,
162
+ * **on by default**). This and `multimodalModels` are one decision, not two:
163
+ * the declaration forms the base set, and the list unions with it
164
+ * (whitelist) or subtracts from it (blacklist).
165
+ *
166
+ * The declaration is only whatever the profile — or the installed catalog —
167
+ * says, and it CAN be wrong: profiles sometimes declare
168
+ * `input: [text, image]` just to pass the host admission gate, and a wrong
169
+ * "yes" sends the image straight to an endpoint that may reject it. The
170
+ * harness's own text-model projection is disabled by `bridgeAutoImage`, so
171
+ * nothing underneath catches that. Two things bound the damage: the first
172
+ * promotion of a route logs a notice naming the escape hatch
173
+ * ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
174
+ * under `blacklist` mode forces the bridge back on. Set this to false to
175
+ * bridge everything the list does not name.
176
+ *
177
+ * The value is always read from the *unwrapped* `resolveModelInfo`, so the
178
+ * admission wrap can never feed its own claim back in as evidence.
179
+ */
180
+ autoDetectMultimodal: z.boolean().default(true),
181
+ /**
182
+ * Measured image capability per route (v0.9.0), keyed `"provider/model"` with
183
+ * the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
184
+ * never by hand — the only entry in this schema that records what a route
185
+ * DOES rather than what it says.
186
+ *
187
+ * It outranks the route's own declaration, because a probe is a real request
188
+ * and a declaration is only a claim. It does NOT outrank `multimodalModels`:
189
+ * that list is explicit human intent, and a person who names a model
190
+ * deserves the last word over an automated measurement.
191
+ */
192
+ probeResults: z.dict(z.string()).default({}),
193
+ /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
194
+ bridgePreview: z.boolean().default(true),
195
+ /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
196
+ bridgePreviewScanIntervalMs: z.number().default(2000),
197
+ /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
198
+ bridgePreviewHideHint: z.boolean().default(true),
199
+ /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
200
+ desktopScreenshot: z.boolean().default(false),
201
+ /**
202
+ * Advertise image input capability for every model while the bridge is on.
203
+ * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
204
+ * pasted images unless the current model declares `image` in its
205
+ * `inputModalities`. The bridge handles those images anyway (they become
206
+ * text hints the agent inspects through `inspect_image`), so this wraps the
207
+ * llm service's `resolveModelInfo` to report image support for text-only
208
+ * models too — users can paste images on any model without hand-editing
209
+ * provider model configs.
210
+ */
211
+ bridgeAutoImage: z.boolean().default(true),
212
+ /** Send a stable per-conversation id header (`x-opencode-session`) on vision
213
+ * requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
214
+ * on every request; requests without the header may error. The value is the
215
+ * current dsh session id when the call runs inside one, else `sessionId`
216
+ * below, else a stable per-process random id. Set false to disable. */
217
+ sendSessionHeader: z.boolean().default(true),
218
+ /** Header name carrying the session id. */
219
+ sessionHeaderName: z.string().default("x-opencode-session"),
220
+ /** Fixed session id override for callers without a dsh session context.
221
+ * Empty = auto (per-process stable id). */
222
+ sessionId: z.string().default(""),
223
+ });
224
+
225
+ const MIME_BY_EXT = {
226
+ ".png": "image/png",
227
+ ".jpg": "image/jpeg",
228
+ ".jpeg": "image/jpeg",
229
+ ".gif": "image/gif",
230
+ ".webp": "image/webp",
231
+ ".bmp": "image/bmp",
232
+ ".avif": "image/avif",
233
+ ".svg": "image/svg+xml",
234
+ ".ico": "image/x-icon",
235
+ };
236
+
237
+ const EXT_BY_MEDIA = {
238
+ "image/png": ".png",
239
+ "image/jpeg": ".jpg",
240
+ "image/webp": ".webp",
241
+ "image/gif": ".gif",
242
+ };
243
+
244
+ /** True when any message carries an image content block. */
245
+ function hasImageBlock(messages) {
246
+ return (messages ?? []).some((m) =>
247
+ Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
248
+ );
249
+ }
250
+
251
+ /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
252
+ function deepFreeze(value) {
253
+ if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
254
+ if (Array.isArray(value)) {
255
+ for (const item of value) deepFreeze(item);
256
+ return Object.freeze(value);
257
+ }
258
+ for (const key of Object.keys(value)) deepFreeze(value[key]);
259
+ return Object.freeze(value);
260
+ }
261
+
262
+ /** Export one attachment to disk; returns the file path (cached per process). */
263
+ const exportedPaths = new Map();
264
+ async function exportImage(attachment, ctx, dir) {
265
+ const cached = exportedPaths.get(attachment.attachmentId);
266
+ if (cached) return cached;
267
+ const { data } = await ctx.attachments.readImage(attachment);
268
+ const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
269
+ const safeName = attachment.name
270
+ ? attachment.name
271
+ .replace(/\.[^.]+$/, "")
272
+ .replace(/[^\w\-]+/g, "_")
273
+ .replace(/^_+|_+$/g, "")
274
+ .slice(0, 40)
275
+ : "";
276
+ const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
277
+ const path = join(dir, `${base}${ext}`);
278
+ await writeFile(path, data);
279
+ exportedPaths.set(attachment.attachmentId, path);
280
+ return path;
281
+ }
282
+
283
+ /**
284
+ * Replace image content blocks with text hints pointing at exported files.
285
+ * Non-image messages are returned as-is (same reference); bridged messages
286
+ * are fresh, deep-frozen objects with the original identity and source.
287
+ * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
288
+ * prefix so the browser half can recognize it precisely.
289
+ * Exported for unit testing; `ctx` only needs `attachments`.
290
+ */
291
+ async function bridgeMessages(messages, ctx, dir) {
292
+ const next = [];
293
+ for (const message of messages) {
294
+ const content = message?.content;
295
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
296
+ next.push(message);
297
+ continue;
298
+ }
299
+ const blocks = [];
300
+ for (const block of content) {
301
+ if (block?.type !== "image") {
302
+ blocks.push(block);
303
+ continue;
304
+ }
305
+ const path = await exportImage(block.attachment, ctx, dir);
306
+ const name = block.attachment.name ? ` (${block.attachment.name})` : "";
307
+ blocks.push({
308
+ type: "text",
309
+ text:
310
+ `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
311
+ `Inspect it with the inspect_image tool to see its content.]`,
312
+ });
313
+ }
314
+ next.push(deepFreeze({ ...message, content: blocks }));
315
+ }
316
+ return next;
317
+ }
318
+
319
+ /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
320
+ function parseQuery(rawUrl) {
321
+ const query = {};
322
+ const at = rawUrl.indexOf("?");
323
+ if (at === -1) return query;
324
+ for (const pair of rawUrl.slice(at + 1).split("&")) {
325
+ const eq = pair.indexOf("=");
326
+ if (eq === -1) continue;
327
+ try {
328
+ query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
329
+ } catch {
330
+ /* skip malformed pairs */
331
+ }
332
+ }
333
+ return query;
334
+ }
335
+
336
+ /**
337
+ * Advertise `image` in the `inputModalities` reported by the llm service for
338
+ * every model, so the host admission gate lets pasted images through on
339
+ * text-only models. The bridge turns those images into text hints anyway, so
340
+ * this is pure admission: it never changes what the adapter actually streams
341
+ * (the adapter's own stream validation reads the model's real `input` from
342
+ * the provider config, untouched here).
343
+ *
344
+ * The wrap is installed on the shared llm service instance, so it must be
345
+ * idempotent across HMR re-applies and restored on dispose. `marker` is a
346
+ * per-instance record proving this plugin already owns the wrap; the
347
+ * `installed` flag distinguishes "we wrapped it" from "the original was
348
+ * replaced by something else" so dispose only restores what we replaced.
349
+ *
350
+ * Exported for unit testing; the test passes a fake llm service.
351
+ */
352
+ const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
353
+ function installAutoImageAdmission(llm, logger) {
354
+ if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
355
+ logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
356
+ return () => {};
357
+ }
358
+ if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
359
+ const original = llm.resolveModelInfo.bind(llm);
360
+ const wrapped = async (provider, model, signal) => {
361
+ const info = await original(provider, model, signal);
362
+ if (!info) return info;
363
+ const mods = info.inputModalities;
364
+ if (Array.isArray(mods) && mods.includes("image")) return info;
365
+ return { ...info, inputModalities: [...(mods ?? []), "image"] };
366
+ };
367
+ let installed = false;
368
+ // The marker carries BOTH functions: the wrapped one proves ownership for
369
+ // dispose, and `original` is the only honest source of a route's real
370
+ // modalities once the wrap is in place (`unwrappedResolveModelInfo`).
371
+ llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
372
+ llm.resolveModelInfo = wrapped;
373
+ installed = true;
374
+ logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
375
+ return () => {
376
+ if (installed) {
377
+ if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
378
+ installed = false;
379
+ }
380
+ delete llm[LLM_RESOLVE_WRAP_MARK];
381
+ };
382
+ }
383
+
384
+ /**
385
+ * Admission is not dispatch, and only dispatch decides whether the pixels arrive.
386
+ *
387
+ * {@link installAutoImageAdmission} wraps `llm.resolveModelInfo`. That method
388
+ * governs who may *offer* an image — the host gate that refuses a pasted image,
389
+ * the model list, the settings readout — and it is deliberately "pure admission:
390
+ * it never changes what the adapter actually streams".
391
+ *
392
+ * What the adapter actually streams is decided one layer down, by
393
+ * `LlmService.generate`:
394
+ *
395
+ * const adapterCall = await adapter.prepareCall(provider, model, signal);
396
+ * modelInfo = this.normalizeModelInfo(registration, model, adapterCall.model);
397
+ * if (modelInfo.inputModalities !== undefined
398
+ * && !modelInfo.inputModalities.includes("image")
399
+ * && projectedMessages.some((message) => contentHasImage(message.content)))
400
+ * projectedMessages = projectImagesForTextModel(projectedMessages);
401
+ *
402
+ * When that list lacks `image`, every image block is rewritten into a text
403
+ * placeholder (`textOnlyImageText`) *before the adapter is called*. So on a
404
+ * route the plugin had already measured as image-capable, and whose bridge
405
+ * decision was therefore "direct, do not intercept", the image was still
406
+ * destroyed — by a check that reads the adapter's declaration and nothing else.
407
+ * `probeResults`, `multimodalModels` and `autoDetectMultimodal` were all
408
+ * invisible to it.
409
+ *
410
+ * This wraps `llm.registration` — the accessor the core itself calls — so every
411
+ * adapter, including one registered later, answers `prepareCall` through
412
+ * {@link routeDirectDecision}: the same rule the bridge uses, so the two can
413
+ * never disagree about a route.
414
+ *
415
+ * When the decision is "bridge", nothing changes: the bridge already turned the
416
+ * image into an `inspect_image` hint, and the core's projection stays as the
417
+ * correct fallback for any image that still reaches dispatch.
418
+ *
419
+ * Idempotent across HMR re-applies, and dispose restores every adapter it
420
+ * touched plus the accessor itself.
421
+ */
422
+ const LLM_REGISTRATION_WRAP_MARK = Symbol("dsh-tool-vision.registration.wrapped");
423
+ function installDispatchImageAdmission(llm, getConfig, logger) {
424
+ if (llm === undefined || llm === null || typeof llm.registration !== "function") {
425
+ logger?.warn?.("[tool-vision] llm service unavailable; dispatch image admission not installed");
426
+ return () => {};
427
+ }
428
+ if (llm[LLM_REGISTRATION_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
429
+ const original = llm.registration.bind(llm);
430
+ /** adapter -> the prepareCall it had before this wrap, for dispose. */
431
+ const patched = new Map();
432
+ const wrapped = (provider) => {
433
+ const registration = original(provider);
434
+ const adapter = registration?.adapter;
435
+ if (adapter !== undefined && adapter !== null
436
+ && !patched.has(adapter) && typeof adapter.prepareCall === "function") {
437
+ const rawPrepare = adapter.prepareCall;
438
+ const prepare = rawPrepare.bind(adapter);
439
+ adapter.prepareCall = async (route, model, signal) => {
440
+ const call = await prepare(route, model, signal);
441
+ if (call === undefined || call === null || call.model === undefined || call.model === null) return call;
442
+ const mods = call.model.inputModalities;
443
+ if (Array.isArray(mods) && mods.includes("image")) return call;
444
+ let direct = false;
445
+ try {
446
+ ({ direct } = await routeDirectDecision(route, model, getConfig(), llm));
447
+ } catch {
448
+ // A decision that cannot be made is not a licence to send pixels a
449
+ // route may reject: leave the core's projection in place.
450
+ return call;
451
+ }
452
+ if (!direct) return call;
453
+ return { ...call, model: { ...call.model, inputModalities: [...(mods ?? []), "image"] } };
454
+ };
455
+ patched.set(adapter, rawPrepare);
456
+ }
457
+ return registration;
458
+ };
459
+ llm[LLM_REGISTRATION_WRAP_MARK] = { original, wrapped };
460
+ llm.registration = wrapped;
461
+ logger?.debug?.("[tool-vision] dispatch image admission installed (llm.registration wrapped)");
462
+ return () => {
463
+ for (const [adapter, prepare] of patched) {
464
+ if (typeof prepare === "function") adapter.prepareCall = prepare;
465
+ }
466
+ patched.clear();
467
+ if (llm.registration === wrapped) llm.registration = original;
468
+ delete llm[LLM_REGISTRATION_WRAP_MARK];
469
+ };
470
+ }
471
+
472
+ /**
473
+ * Register the loopback route that serves bridged images to the same-origin
474
+ * page (the preview thumbnails). Read-only and tightly scoped:
475
+ * - only files inside the bridge export directory (no traversal);
476
+ * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
477
+ * - Host restricted to the local machine;
478
+ * - hard 20MB cap per file.
479
+ */
480
+ function registerBridgePreviewRoute(ctx, exportDir, logger) {
481
+ const webServer = ctx.get("webServer");
482
+ if (webServer === undefined) {
483
+ logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
484
+ return;
485
+ }
486
+ const bridgeDir = resolvePath(exportDir);
487
+ ctx.effect(() => webServer.register({
488
+ kind: "exact",
489
+ path: BRIDGE_PREVIEW_ROUTE,
490
+ async handler(req, res) {
491
+ try {
492
+ const raw = String(req.url ?? "");
493
+ const query = parseQuery(raw);
494
+ const p = query.p;
495
+ if (typeof p !== "string" || p.length === 0) {
496
+ res.writeHead(400);
497
+ res.end("bad request");
498
+ return;
499
+ }
500
+ const lower = p.toLowerCase();
501
+ const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
502
+ if (mediaType === undefined) {
503
+ res.writeHead(400);
504
+ res.end("not an image path");
505
+ return;
506
+ }
507
+ const host = String(req.headers?.host ?? "");
508
+ if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
509
+ res.writeHead(403);
510
+ res.end("forbidden");
511
+ return;
512
+ }
513
+ const target = resolvePath(p);
514
+ if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
515
+ res.writeHead(403);
516
+ res.end("forbidden");
517
+ return;
518
+ }
519
+ const info = await stat(target);
520
+ if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
521
+ res.writeHead(404);
522
+ res.end("not found");
523
+ return;
524
+ }
525
+ const bytes = await readFile(target);
526
+ res.writeHead(200, {
527
+ "Content-Type": mediaType,
528
+ "Cache-Control": "private, max-age=60",
529
+ });
530
+ res.end(bytes);
531
+ } catch {
532
+ try {
533
+ res.writeHead(404);
534
+ res.end("not found");
535
+ } catch {
536
+ /* response already sent */
537
+ }
538
+ }
539
+ },
540
+ }), "dsh-tool-vision: bridge preview route");
541
+ }
542
+
543
+ /**
544
+ * Register the loopback JSON route the settings panel reads:
545
+ *
546
+ * { current: {provider, model, direct, source, mode}, providers: [...] }
547
+ *
548
+ * `providers` comes from `llm.listProviders()` + `llm.listModels(provider)` —
549
+ * the routes the harness actually has configured, which is exactly the set a
550
+ * `multimodalModels` entry may name. `listModels` is a different seam from
551
+ * `resolveModelInfo`, so these declared modalities are the profile's own and
552
+ * are NOT contaminated by the admission wrap (a model the wrap advertises as
553
+ * image-capable still reports its real declaration here).
554
+ *
555
+ * Same confinement as the image route: loopback Host only, read-only,
556
+ * `no-store`, and no credentials or endpoints in the payload — provider ids,
557
+ * model ids/names and one boolean each.
558
+ */
559
+ function registerModelCatalogRoute(ctx, logger, getConfig) {
560
+ const webServer = ctx.get("webServer");
561
+ if (webServer === undefined) {
562
+ logger?.warn?.("[tool-vision] webServer unavailable; model catalog route not registered");
563
+ return;
564
+ }
565
+ ctx.effect(() => webServer.register({
566
+ kind: "exact",
567
+ path: MODEL_CATALOG_ROUTE,
568
+ async handler(req, res) {
569
+ const send = (status, payload) => {
570
+ res.writeHead(status, {
571
+ "Content-Type": "application/json; charset=utf-8",
572
+ "Cache-Control": "no-store",
573
+ });
574
+ res.end(JSON.stringify(payload));
575
+ };
576
+ const host = String(req.headers?.host ?? "");
577
+ if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
578
+ send(403, { error: "forbidden" });
579
+ return;
580
+ }
581
+ const llm = ctx.get("llm");
582
+ if (llm === undefined) {
583
+ send(503, { error: "llm service unavailable" });
584
+ return;
585
+ }
586
+ // `listed` is computed HERE, with the same matcher the bridge itself
587
+ // uses, so the panel never reimplements the rule (and can never
588
+ // disagree with the decision it is describing).
589
+ const cfg = typeof getConfig === "function" ? getConfig() : undefined;
590
+ const list = Array.isArray(cfg?.multimodalModels) ? cfg.multimodalModels : [];
591
+ const listMode = normalizeListMode(cfg?.multimodalListMode);
592
+ let routes = [];
593
+ try {
594
+ routes = llm.listProviders() ?? [];
595
+ } catch (error) {
596
+ logger?.debug?.(`[tool-vision] listProviders failed: ${String(error)}`);
597
+ }
598
+ const providers = [];
599
+ for (const entry of routes) {
600
+ const id = entry?.id;
601
+ if (typeof id !== "string" || id.length === 0) continue;
602
+ const record = {
603
+ id,
604
+ name: typeof entry?.name === "string" && entry.name.length > 0 ? entry.name : id,
605
+ models: [],
606
+ };
607
+ try {
608
+ const models = await llm.listModels(id);
609
+ record.models = (models ?? []).slice(0, MODEL_CATALOG_MAX_PER_PROVIDER).map((model) => {
610
+ const modelId = String(model.id);
611
+ const matchedEntries = modelListMatchEntries(list, id, modelId);
612
+ return {
613
+ id: modelId,
614
+ name: typeof model.name === "string" && model.name.length > 0 ? model.name : modelId,
615
+ image: Array.isArray(model.inputModalities) && model.inputModalities.includes("image"),
616
+ listed: matchedEntries.length > 0,
617
+ matchedEntries,
618
+ // What the route was MEASURED doing, when anyone has probed it.
619
+ // `null` means "not probed", which the panel must render
620
+ // differently from a probe that came back negative.
621
+ probe: probeVerdict(cfg, id, modelId) ?? null,
622
+ };
623
+ });
624
+ } catch (error) {
625
+ // One broken route must never blank the whole panel: report it and
626
+ // keep going, so the other providers still list.
627
+ record.error = String(error?.message ?? error);
628
+ }
629
+ providers.push(record);
630
+ }
631
+ send(200, {
632
+ current: { ...lastModelDecision },
633
+ list,
634
+ listMode,
635
+ probeResults: typeof getConfig === "function" ? { ...(getConfig()?.probeResults ?? {}) } : {},
636
+ providers,
637
+ });
638
+ },
639
+ }), "dsh-tool-vision: model catalog route");
640
+ }
641
+
642
+ /** Accepted `multimodalListMode` values; anything else falls back to whitelist. */
643
+ const MULTIMODAL_LIST_MODES = ["off", "whitelist", "blacklist"];
644
+ const DEFAULT_MULTIMODAL_LIST_MODE = "whitelist";
645
+
646
+ /**
647
+ * Normalize a configured list mode. The settings section only ever writes one
648
+ * of {@link MULTIMODAL_LIST_MODES}, but the value can also arrive from a
649
+ * hand-edited `settings.yaml`, so an unknown string is clamped rather than
650
+ * treated as "no mode" (which would silently bridge everything).
651
+ */
652
+ function normalizeListMode(value) {
653
+ return MULTIMODAL_LIST_MODES.includes(value) ? value : DEFAULT_MULTIMODAL_LIST_MODE;
654
+ }
655
+
656
+ const listEntryCache = new Map();
657
+ /**
658
+ * Compile one list entry into an anchored, case-insensitive regexp. A literal
659
+ * entry is just an exact match; `*` and `?` are the only metacharacters, so a
660
+ * model id containing `.` or `+` still matches itself.
661
+ */
662
+ function listEntryRegExp(entry) {
663
+ const cached = listEntryCache.get(entry);
664
+ if (cached !== undefined) return cached;
665
+ const source = entry
666
+ .replace(/[.+^${}()|[\]\\]/g, "\\$&")
667
+ .replace(/\*/g, ".*")
668
+ .replace(/\?/g, ".");
669
+ const compiled = new RegExp(`^${source}$`, "i");
670
+ listEntryCache.set(entry, compiled);
671
+ return compiled;
672
+ }
673
+
674
+ /**
675
+ * Whether any entry in `patterns` names this route. Each entry is tested
676
+ * against the full model id, its bare id after the last `/`, and
677
+ * `provider/id` — so `mimo-v2.5`, `xiaomi/mimo-v2.5` and
678
+ * `commandcode/xiaomi/mimo-v2.5` all address the same route, and `*vl*`
679
+ * addresses every variant of one family. Exported for unit testing.
680
+ */
681
+ function modelListMatches(patterns, provider, model) {
682
+ return modelListMatchEntries(patterns, provider, model).length > 0;
683
+ }
684
+
685
+ /**
686
+ * The list entries that address this route. The panel needs *which* entry hit
687
+ * so that unchecking a model removes exactly the entry responsible — matching
688
+ * `mimo-v2.5` against `xiaomi/mimo-v2.5` must not leave an orphan behind, and
689
+ * must not delete an unrelated entry either. Exported for unit testing.
690
+ */
691
+ function modelListMatchEntries(patterns, provider, model) {
692
+ if (!Array.isArray(patterns) || patterns.length === 0) return [];
693
+ if (typeof model !== "string" || model.length === 0) return [];
694
+ const slash = model.lastIndexOf("/");
695
+ const candidates = [model, slash === -1 ? model : model.slice(slash + 1)];
696
+ if (typeof provider === "string" && provider.length > 0) candidates.push(`${provider}/${model}`);
697
+ const hits = [];
698
+ for (const raw of patterns) {
699
+ if (raw === undefined || raw === null) continue;
700
+ const entry = String(raw).trim();
701
+ if (entry.length === 0) continue;
702
+ const re = listEntryRegExp(entry);
703
+ if (candidates.some((candidate) => re.test(candidate))) hits.push(String(raw));
704
+ }
705
+ return hits;
706
+ }
707
+
708
+ /**
709
+ * The route the session is on: the logged request header first, then the
710
+ * agent's own options. `provider` is optional (older logs may lack it).
711
+ */
712
+ function currentRoute(agent) {
713
+ const header = agent?.session?.requestHeader?.();
714
+ return {
715
+ provider: header?.config?.provider ?? agent?.options?.provider,
716
+ model: header?.config?.model ?? agent?.options?.model,
717
+ };
718
+ }
719
+
720
+ /**
721
+ * The last decision `currentModelAcceptsImage` made, for the settings panel's
722
+ * "current route" readout (served over {@link MODEL_CATALOG_ROUTE}). A plain
723
+ * mutable record — the panel only ever reads it.
724
+ */
725
+ const lastModelDecision = {
726
+ provider: undefined,
727
+ model: undefined,
728
+ direct: false,
729
+ source: "none",
730
+ mode: DEFAULT_MULTIMODAL_LIST_MODE,
731
+ };
732
+
733
+ /**
734
+ * `resolveModelInfo` as it was before {@link installAutoImageAdmission}
735
+ * wrapped it. Auto-detection MUST read through this: the wrap reports image
736
+ * support for *every* model, so reading the wrapped method would make the
737
+ * plugin's own claim the evidence for itself.
738
+ */
739
+ function unwrappedResolveModelInfo(llm) {
740
+ const marker = llm?.[LLM_RESOLVE_WRAP_MARK];
741
+ const fn = marker?.original ?? llm?.resolveModelInfo;
742
+ return typeof fn === "function" ? fn.bind(llm) : undefined;
743
+ }
744
+
745
+ /**
746
+ * Whether the route declares `image` input, read before the admission wrap.
747
+ * `false` and `undefined` are the same answer to the caller ("not known to be
748
+ * multimodal"): a route nobody could resolve is bridged, never guessed.
749
+ */
750
+ async function routeDeclaresImage(llm, provider, model) {
751
+ const resolve = unwrappedResolveModelInfo(llm);
752
+ if (resolve === undefined || typeof provider !== "string" || typeof model !== "string") {
753
+ return false;
754
+ }
755
+ try {
756
+ const info = await resolve(provider, model);
757
+ return info?.inputModalities?.includes("image") === true;
758
+ } catch {
759
+ return false;
760
+ }
761
+ }
762
+
763
+ /** Routes already warned about, so the notice appears once per route. */
764
+ const autoPromotionWarned = new Set();
765
+
766
+ /** Registry key for one measured route. Provider-qualified: the same model id
767
+ * can be served by two gateways with different real capabilities. */
768
+ function probeKey(provider, model) {
769
+ return `${typeof provider === "string" ? provider : ""}/${model}`;
770
+ }
771
+
772
+ /**
773
+ * The measured verdict for a route, or `undefined` when it was never probed.
774
+ * Anything other than the two recorded values (a hand-edited settings.yaml, a
775
+ * future format) reads as "not probed" rather than as a guess.
776
+ */
777
+ function probeVerdict(config, provider, model) {
778
+ const results = config?.probeResults;
779
+ if (results === undefined || results === null || typeof model !== "string" || model.length === 0) {
780
+ return undefined;
781
+ }
782
+ const value = results[probeKey(provider, model)];
783
+ return value === "yes" || value === "no" ? value : undefined;
784
+ }
785
+
786
+ /**
787
+ * Resolve the endpoint and credential a route actually sends to, so a probe
788
+ * talks to the same place the model does instead of a copy of it.
789
+ *
790
+ * The pieces come from two seams that a plugin is allowed to read:
791
+ * - `llm.listConfigurableProviders()` names the settings namespace and the
792
+ * path a provider's profile lives at;
793
+ * - `settings.get(ns)` resolves that profile (`baseURL`, `apiKeyEnv`, `api`);
794
+ * - `credentials.resolve(ref)` turns `apiKeyEnv` into the secret itself
795
+ * (`.value`), the same way `dsh-llm-pi-ai` does it for a real call.
796
+ *
797
+ * Read-only, and the secret never leaves this function's caller: the probe
798
+ * result records a verdict, never an endpoint or a key.
799
+ *
800
+ * @throws {Error} with a caller-facing reason when the route is unprobeable.
801
+ */
802
+ async function resolveProbeTarget(ctx, provider, model) {
803
+ const llm = ctx.get("llm");
804
+ const settings = ctx.get("settings");
805
+ const credentials = ctx.get("credentials");
806
+ if (settings === undefined) {
807
+ throw new Error("the settings service is unavailable, so the route's endpoint cannot be read");
808
+ }
809
+ let entry;
810
+ try {
811
+ entry = (llm?.listConfigurableProviders?.() ?? []).find((candidate) => candidate?.provider === provider);
812
+ } catch {
813
+ /* an adapter that cannot describe its providers simply has no entry */
814
+ }
815
+ const settingsNs = entry?.settingsNs;
816
+ if (typeof settingsNs !== "string" || settingsNs.length === 0) {
817
+ throw new Error(
818
+ `provider "${provider}" does not publish a settings namespace, so its endpoint cannot be resolved for probing`,
819
+ );
820
+ }
821
+ let profile = settings.get(settingsNs);
822
+ const path = Array.isArray(entry.settingsPath) && entry.settingsPath.length > 0 ? entry.settingsPath : [provider];
823
+ for (const segment of path) profile = profile?.[segment];
824
+ if (profile === undefined || profile === null || typeof profile !== "object") {
825
+ throw new Error(`no stored profile for provider "${provider}" under "${settingsNs}"`);
826
+ }
827
+ const baseURL = typeof profile.baseURL === "string" && profile.baseURL.length > 0 ? profile.baseURL : undefined;
828
+ if (baseURL === undefined) {
829
+ throw new Error(`provider "${provider}" has no baseURL, so its endpoint cannot be probed`);
830
+ }
831
+ let apiKey;
832
+ const ref = profile.apiKeyEnv;
833
+ if (typeof ref === "string" && ref.length > 0) {
834
+ try {
835
+ const record = await credentials?.resolve?.(ref);
836
+ apiKey = typeof record?.value === "string" ? record.value : undefined;
837
+ } catch (error) {
838
+ throw new Error(`stored credential "${ref}" could not be read: ${String(error?.message ?? error)}`);
839
+ }
840
+ }
841
+ return { baseURL, apiKey, api: profile.api, model };
842
+ }
843
+
844
+ /**
845
+ * One-time notice when a route receives images directly *only* because of its
846
+ * own declaration (source `auto`). This is the one failure auto-detection
847
+ * cannot rule out: if the declaration is wrong, the endpoint rejects the image
848
+ * after the message is already durable, and the error alone does not say what
849
+ * to change. Naming the escape hatch the first time it happens turns a
850
+ * confusing 400 into a one-line fix — and now names the probe too, which
851
+ * settles the question instead of asking the user to guess.
852
+ * Read {@link lastModelDecision}; never throws.
853
+ */
854
+ function warnAutoPromotion(logger) {
855
+ const { provider, model, direct, source } = lastModelDecision;
856
+ if (direct !== true || source !== "auto" || typeof model !== "string" || model.length === 0) return;
857
+ const key = `${typeof provider === "string" ? provider : ""}\u0000${model}`;
858
+ if (autoPromotionWarned.has(key)) return;
859
+ autoPromotionWarned.add(key);
860
+ logger?.warn?.(
861
+ `[tool-vision] "${model}" receives images directly because its route declares image input, ` +
862
+ `which is unverified; run vision_probe_model to measure it, or list it in multimodalModels ` +
863
+ `under multimodalListMode: blacklist to force the bridge`,
864
+ );
865
+ }
866
+
867
+ /**
868
+ * Whether images may be handed to this route directly, and why.
869
+ *
870
+ * The plugin's single precedence rule. Three inputs, deliberately independent:
871
+ *
872
+ * - the base set — the route's own declared `inputModalities`, and only when
873
+ * `autoDetectMultimodal` is on. It is read through
874
+ * {@link unwrappedResolveModelInfo}, never through the admission wrap.
875
+ * - the measurement — `probeResults`, which outranks the declaration because a
876
+ * probe is a real request and a declaration is only a statement.
877
+ * - the list — `multimodalModels` under `multimodalListMode`. Explicit human
878
+ * intent, so it outranks both: whitelist adds to the base set, blacklist
879
+ * subtracts from it.
880
+ *
881
+ * Two callers need this exact answer and must never disagree:
882
+ * - {@link currentModelAcceptsImage} — the bridge, deciding whether to
883
+ * intercept an image at all;
884
+ * - {@link installDispatchImageAdmission} — the dispatch path, where the core
885
+ * independently decides whether the model may receive the image block.
886
+ *
887
+ * Returns `direct: true` when bridging is disabled (nothing would be bridged
888
+ * anyway).
889
+ */
890
+ async function routeDirectDecision(provider, model, config, llm) {
891
+ const mode = normalizeListMode(config.multimodalListMode);
892
+ if (!config.bridgeTextOnly) return { direct: true, source: "bridge-off", mode };
893
+ if (typeof model !== "string" || model.length === 0) {
894
+ return { direct: false, source: "no-route", mode };
895
+ }
896
+ const base = config.autoDetectMultimodal
897
+ ? await routeDeclaresImage(llm, provider, model)
898
+ : false;
899
+ let direct = base;
900
+ let source = base ? "auto" : "default";
901
+ // A measured verdict outranks the route's own claim: a probe is a real
902
+ // request, a declaration is only a statement. Both directions count, so a
903
+ // route that says "image" and answered without reading the pixels goes back
904
+ // to the bridge without anyone hunting for the right list entry.
905
+ const probed = probeVerdict(config, provider, model);
906
+ if (probed !== undefined) {
907
+ direct = probed === "yes";
908
+ source = probed === "yes" ? "probe-yes" : "probe-no";
909
+ }
910
+ // `off` means the list carries no opinion at all — it is not "empty list
911
+ // under whitelist", it is "no list semantics", so nothing is forced. The
912
+ // human list keeps the last word over both other signals.
913
+ const listed = mode !== "off" && modelListMatches(config.multimodalModels, provider, model);
914
+ if (listed) {
915
+ direct = mode !== "blacklist";
916
+ source = mode === "blacklist" ? "blacklist" : "whitelist";
917
+ }
918
+ return { direct, source, mode };
919
+ }
920
+
921
+ /**
922
+ * Whether the session's current model may receive image blocks directly.
923
+ *
924
+ * {@link routeDirectDecision} keyed by the session's route, with the answer
925
+ * recorded in {@link lastModelDecision} for the settings panel.
926
+ */
927
+ async function currentModelAcceptsImage(agent, config, llm) {
928
+ const { provider, model } = currentRoute(agent);
929
+ const decision = await routeDirectDecision(provider, model, config, llm);
930
+ Object.assign(lastModelDecision, { provider, model, ...decision });
931
+ return decision.direct;
932
+ }
933
+
934
+ /**
935
+ * Lazily bridge image blocks that are already part of the session log
936
+ * (pasted before the plugin was active, or before a restart). Each affected
937
+ * event is rewritten once with a surface `replace`, which swaps the durable
938
+ * derivation (and the transcript) to the text hint. Events that are no
939
+ * longer on the surface (already shadowed) are skipped and remembered.
940
+ * `repaired` tracks per-session state: a `Set` of handled seqs plus a
941
+ * monotonic scan cursor.
942
+ */
943
+ async function repairLoggedImages(ctx, session, exportDir, repaired) {
944
+ const events = session.events;
945
+ for (let index = repaired.cursor; index < events.length; index += 1) {
946
+ const event = events[index];
947
+ if (event.type !== "user/message" || repaired.set.has(event.seq)) {
948
+ repaired.set.add(event.seq);
949
+ continue;
950
+ }
951
+ const content = event.data?.content;
952
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
953
+ repaired.set.add(event.seq);
954
+ continue;
955
+ }
956
+ const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
957
+ try {
958
+ session.append("user/message", bridged, {
959
+ surfaceOp: { op: "replace", start: event.seq, end: event.seq },
960
+ sourceEventSeqs: [event.seq],
961
+ });
962
+ ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
963
+ } catch (error) {
964
+ ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
965
+ }
966
+ repaired.set.add(event.seq);
967
+ }
968
+ repaired.cursor = events.length;
969
+ }
970
+
971
+ /**
972
+ * Install the pre-step bridge at the root level. Agent-scoped waterfalls
973
+ * admit untagged (root) listeners, so one listener serves every agent —
974
+ * including sessions resumed after a server restart, which never re-fire
975
+ * `session/created` for per-agent attachments. Runs before every proposed
976
+ * step: new pasted images are bridged into the durable log, and stuck
977
+ * logged images are repaired, before the model request is derived from it.
978
+ */
979
+ function attachPreStepBridge(ctx, getConfig, exportDir) {
980
+ const repairedBySession = new Map();
981
+ ctx.on("agent/pre-step", async (payload, next) => {
982
+ const decision = await next();
983
+ if (!decision || decision.kind !== "enter") return decision;
984
+ const agent = payload?.agent;
985
+ if (!agent?.session) return decision;
986
+ try {
987
+ const acceptsImage = await currentModelAcceptsImage(agent, getConfig(), ctx.get("llm"));
988
+ warnAutoPromotion(ctx.logger);
989
+ if (!acceptsImage) {
990
+ let repaired = repairedBySession.get(agent.session.id);
991
+ if (!repaired) {
992
+ repaired = { set: new Set(), cursor: 0 };
993
+ repairedBySession.set(agent.session.id, repaired);
994
+ }
995
+ await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
996
+ ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
997
+ });
998
+ }
999
+ if (acceptsImage) return decision;
1000
+ const messages = await bridgeMessages(decision.messages, ctx, exportDir);
1001
+ if (messages.every((message, index) => message === decision.messages[index])) return decision;
1002
+ return { ...decision, messages };
1003
+ } catch (error) {
1004
+ ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
1005
+ return decision;
1006
+ }
1007
+ });
1008
+ }
1009
+
1010
+ function resolveApiKey(config) {
1011
+ if (config.apiKey) return config.apiKey;
1012
+ if (config.apiKeyEnv) {
1013
+ const fromEnv = process.env[config.apiKeyEnv];
1014
+ if (fromEnv) return fromEnv;
1015
+ }
1016
+ return process.env.OPENAI_API_KEY ?? "";
1017
+ }
1018
+
1019
+ /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
1020
+ async function toImageUrl(target, cwd, config) {
1021
+ if (/^https?:\/\//i.test(target)) return { url: target, note: target };
1022
+ const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
1023
+ const info = await stat(abs).catch(() => null);
1024
+ if (!info) throw new Error(`image not found: ${abs}`);
1025
+ if (info.size > config.maxImageBytes) {
1026
+ throw new Error(
1027
+ `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
1028
+ );
1029
+ }
1030
+ const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
1031
+ if (!mime) {
1032
+ throw new Error(
1033
+ `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
1034
+ );
1035
+ }
1036
+ const data = await readFile(abs);
1037
+ return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
1038
+ }
1039
+
1040
+ /** One OpenAI-compatible chat/completions call with an image_url content part. */
1041
+ async function callVision(config, imageUrl, question, detail, signal, exec) {
1042
+ const key = resolveApiKey(config);
1043
+ if (!key) {
1044
+ throw new Error(
1045
+ `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
1046
+ );
1047
+ }
1048
+ const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
1049
+ const endpoint = new URL("chat/completions", base);
1050
+ const sessionHeader = sessionHeaders(config, exec);
1051
+ const controller = new AbortController();
1052
+ const timer = setTimeout(
1053
+ () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
1054
+ config.timeoutMs,
1055
+ );
1056
+ const onSignalAbort = () => controller.abort(signal?.reason);
1057
+ if (signal) {
1058
+ if (signal.aborted) controller.abort();
1059
+ else signal.addEventListener("abort", onSignalAbort, { once: true });
1060
+ }
1061
+ const content = [
1062
+ { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
1063
+ { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
1064
+ ];
1065
+ try {
1066
+ const response = await fetch(endpoint, {
1067
+ method: "POST",
1068
+ headers: {
1069
+ "Content-Type": "application/json",
1070
+ Authorization: `Bearer ${key}`,
1071
+ ...sessionHeader,
1072
+ },
1073
+ body: JSON.stringify({
1074
+ model: config.model,
1075
+ messages: [{ role: "user", content }],
1076
+ max_tokens: config.maxTokens,
1077
+ }),
1078
+ signal: controller.signal,
1079
+ });
1080
+ const body = await response.json().catch(() => null);
1081
+ if (!response.ok) {
1082
+ const detailText = body?.error?.message ?? response.statusText;
1083
+ throw new Error(
1084
+ `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
1085
+ );
1086
+ }
1087
+ // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
1088
+ // `reasoning_content` first; when the final `content` is empty or was cut
1089
+ // off by max_tokens, fall back to the reasoning text so the answer is
1090
+ // still useful.
1091
+ const message = body?.choices?.[0]?.message;
1092
+ let answer = message?.content ?? "";
1093
+ if (!answer.trim()) answer = message?.reasoning_content ?? "";
1094
+ if (typeof answer !== "string" || !answer.trim()) {
1095
+ throw new Error("vision endpoint returned an empty response");
1096
+ }
1097
+ return answer.trim();
1098
+ } finally {
1099
+ clearTimeout(timer);
1100
+ signal?.removeEventListener("abort", onSignalAbort);
1101
+ }
1102
+ }
1103
+
1104
+ function apply(ctx, config) {
1105
+ // ── settings-backed configuration ─────────────────────────────────────────
1106
+ // The composition entry stays the `base` layer; a registered `tool-vision`
1107
+ // settings section (Web UI section, settings.yaml) overlays it live, so
1108
+ // edits hot-apply without a restart. `sourceGetter` is a GETTER
1109
+ // (`() => scope.get()`), not the config object — keep it and call it at use
1110
+ // time, or `getConfig()` would return a function and every `cfg.*` read
1111
+ // would be undefined (apiKey included).
1112
+ let current = config;
1113
+ let sourceGetter = null;
1114
+ const getConfig = () => (sourceGetter ? sourceGetter() : current);
1115
+ // The registered settings scope, kept so tools can PERSIST what they measure.
1116
+ // Null whenever no settings provider is mounted (composition-entry-only runs),
1117
+ // in which case a probe still reports its verdict but cannot record it.
1118
+ let settingsScope = null;
1119
+ const getSettingsScope = () => settingsScope;
1120
+
1121
+ // ── master switch: a child fiber owns every registration ──────────────────
1122
+ // `ctx.tools.register`, `ctx.effect` and `ctx.on` are all effects on the fiber
1123
+ // that makes them, so confining the tool/bridge/route registrations to a child
1124
+ // fiber makes `enabled` real: disposing that fiber unregisters all 15 tools,
1125
+ // removes the pre-step listener and the preview route, and restores the llm
1126
+ // capability wrap. The settings registration deliberately stays on the parent
1127
+ // fiber — it has to outlive the switch, or the section that turns the plugin
1128
+ // back on would vanish with it.
1129
+ const REGISTRATION_KEYS = [
1130
+ "enabled",
1131
+ "bridgeTextOnly",
1132
+ "bridgeExportDir",
1133
+ "bridgeAutoImage",
1134
+ "bridgePreview",
1135
+ ];
1136
+ let featureFiber = null;
1137
+ let lastRegistrationKey = null;
1138
+
1139
+ function registrationKey(cfg) {
1140
+ return REGISTRATION_KEYS.map((key) => String(cfg[key])).join("|");
1141
+ }
1142
+
1143
+ function installFeatures() {
1144
+ if (featureFiber !== null) return;
1145
+ featureFiber = ctx.plugin({
1146
+ name: "dsh-tool-vision:features",
1147
+ apply(inner) {
1148
+ // ── image bridge: pasted images become inspect_image hints on text-only models ──
1149
+ if (getConfig().bridgeTextOnly) {
1150
+ const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
1151
+ mkdir(exportDir, { recursive: true }).catch(() => {});
1152
+ // Root-level listener: agent-scoped waterfalls admit untagged listeners,
1153
+ // so one registration serves every agent (new and resumed alike) and the
1154
+ // agent is read from the fused payload.
1155
+ attachPreStepBridge(inner, getConfig, exportDir);
1156
+
1157
+ // ── automatic image admission: let pasted images through on text-only models ──
1158
+ // The host gate refuses images unless the model declares `image` input;
1159
+ // the bridge handles them anyway, so report image support for all models.
1160
+ // The wrap is installed on the shared llm service instance (idempotent,
1161
+ // restored on dispose/HMR).
1162
+ if (getConfig().bridgeAutoImage) {
1163
+ const unwrap = installAutoImageAdmission(inner.get("llm"), inner.logger);
1164
+ inner.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
1165
+ }
1166
+
1167
+ // ── dispatch image admission: make the capability verdict actually land ──
1168
+ // `resolveModelInfo` above is pure admission — it never changes what
1169
+ // the adapter streams. The image the plugin decided to hand over
1170
+ // directly was still being projected into a text placeholder by
1171
+ // `LlmService.generate`, which reads the adapter's own declaration and
1172
+ // nothing else. This applies the same decision at that seam.
1173
+ //
1174
+ // Installed outside the `bridgeAutoImage` branch on purpose: the
1175
+ // question it answers is not "may the image be offered" but "does the
1176
+ // model receive it", and that is the plugin's own verdict either way.
1177
+ {
1178
+ const unwrapDispatch = installDispatchImageAdmission(inner.get("llm"), getConfig, inner.logger);
1179
+ inner.effect(() => unwrapDispatch, "dsh-tool-vision: dispatch image admission");
1180
+ }
1181
+
1182
+ // ── bridge image preview: same-origin thumbnails for bridged images ──
1183
+ if (getConfig().bridgePreview) {
1184
+ registerBridgePreviewRoute(inner, exportDir, inner.logger);
1185
+ }
1186
+ }
1187
+
1188
+ inner.tools.register(defineTool({
1189
+ name: "inspect_image",
1190
+ description: getConfig().description,
1191
+ parameters: {
1192
+ path: {
1193
+ type: "string",
1194
+ required: true,
1195
+ description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
1196
+ },
1197
+ question: {
1198
+ type: "string",
1199
+ description: "Optional specific question about the image. Omit for a general detailed description.",
1200
+ },
1201
+ detail: {
1202
+ type: "string",
1203
+ enum: ["auto", "low", "high"],
1204
+ description: "Optional image resolution hint for the vision API (auto by default).",
1205
+ },
1206
+ },
1207
+ output: {
1208
+ schema: { type: "string" },
1209
+ render: (_args, value) => [{ type: "text", text: value }],
1210
+ },
1211
+ async execute(args, exec) {
1212
+ const cfg = getConfig();
1213
+ const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
1214
+ const { url, note } = await toImageUrl(args.path, cwd, cfg);
1215
+ try {
1216
+ const answer = await callVision(cfg, url, args.question, args.detail, exec.signal, exec);
1217
+ return note === url ? answer : `${answer}\n\n(image: ${note})`;
1218
+ } catch (error) {
1219
+ const raw = error && error.message ? String(error.message) : String(error);
1220
+ if (CONTENT_FILTER_RE.test(raw)) {
1221
+ throw new Error(
1222
+ "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
1223
+ "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
1224
+ );
1225
+ }
1226
+ throw error;
1227
+ }
1228
+ },
1229
+ }));
1230
+
1231
+ // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
1232
+ // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
1233
+ // (baseURL/apiKey/model). No provider chain, no local models, no extra
1234
+ // settings: everything comes from the existing tool-vision configuration.
1235
+ registerVisionTools(inner, getConfig);
1236
+
1237
+ // ── ground-truth capability probe (v0.9.0) ───────────────────────────
1238
+ // Everything else decides from what a route SAYS; this measures what it
1239
+ // DOES, and records the answer so the bridge decision stops guessing.
1240
+ inner.tools.register(defineTool({
1241
+ name: "vision_probe_model",
1242
+ description:
1243
+ "Measure whether a model's endpoint can actually read images, by sending it a real " +
1244
+ "solid-color image and checking the answer. Use this when a model's declared image " +
1245
+ "support is doubtful (it claims `image` but the provider may reject it, or it declares " +
1246
+ "text-only yet the endpoint may accept images). Runs a text-only control request plus " +
1247
+ "two different colors, so a model that merely guesses cannot pass. The verdict is " +
1248
+ "recorded and from then on decides whether that route's images are sent directly or " +
1249
+ "bridged. Costs a few requests against the model's own endpoint.",
1250
+ parameters: {
1251
+ model: {
1252
+ type: "string",
1253
+ required: true,
1254
+ description: "Model id to probe, e.g. xiaomi/mimo-v2.5. Use the id exactly as dsh has it configured.",
1255
+ },
1256
+ provider: {
1257
+ type: "string",
1258
+ description:
1259
+ "Provider route serving the model (e.g. commandcode). Omit to use the current " +
1260
+ "session's route, or to auto-select when exactly one provider offers the model.",
1261
+ },
1262
+ },
1263
+ output: {
1264
+ schema: { type: "string" },
1265
+ render: (_args, value) => [{ type: "text", text: value }],
1266
+ },
1267
+ async execute(args, exec) {
1268
+ const cfg = getConfig();
1269
+ const requested = String(args.model ?? "").trim();
1270
+ if (requested.length === 0) throw new Error("vision_probe_model: model is required");
1271
+ const llm = inner.get("llm");
1272
+
1273
+ // Resolve which routes offer this model, so an omitted `provider`
1274
+ // is still an exact choice rather than a guess.
1275
+ let candidates = [];
1276
+ try {
1277
+ for (const entry of llm?.listProviders?.() ?? []) {
1278
+ const id = entry?.id;
1279
+ if (typeof id !== "string") continue;
1280
+ const models = await llm.listModels(id).catch(() => []);
1281
+ if ((models ?? []).some((model) => String(model?.id) === requested)) candidates.push(id);
1282
+ }
1283
+ } catch {
1284
+ /* fall through to the route/argument fallbacks below */
1285
+ }
1286
+ let provider = typeof args.provider === "string" && args.provider.length > 0 ? args.provider : undefined;
1287
+ if (provider === undefined) {
1288
+ const here = currentRoute(exec?.agent);
1289
+ if (candidates.length === 1) provider = candidates[0];
1290
+ else if (here.model === requested && typeof here.provider === "string") provider = here.provider;
1291
+ else if (candidates.length > 1) {
1292
+ throw new Error(
1293
+ `vision_probe_model: "${requested}" is served by ${candidates.join(", ")}; pass provider explicitly`,
1294
+ );
1295
+ } else {
1296
+ provider = here.provider;
1297
+ }
1298
+ }
1299
+ if (typeof provider !== "string" || provider.length === 0) {
1300
+ throw new Error(
1301
+ `vision_probe_model: cannot tell which provider serves "${requested}"; pass provider explicitly`,
1302
+ );
1303
+ }
1304
+
1305
+ const target = await resolveProbeTarget(inner, provider, requested);
1306
+ const outcome = await probeModelCapability(
1307
+ {
1308
+ baseURL: target.baseURL,
1309
+ apiKey: target.apiKey,
1310
+ api: target.api,
1311
+ model: requested,
1312
+ maxTokens: PROBE_MAX_TOKENS,
1313
+ timeoutMs: Math.max(PROBE_TIMEOUT_MS, Number(cfg.timeoutMs) || 0),
1314
+ headers: sessionHeaders(cfg, exec),
1315
+ },
1316
+ exec.signal,
1317
+ );
1318
+
1319
+ const key = probeKey(provider, requested);
1320
+ let recorded = "not recorded (no settings provider)";
1321
+ if (outcome.support === "yes" || outcome.support === "no") {
1322
+ const scope = getSettingsScope();
1323
+ if (scope !== null) {
1324
+ try {
1325
+ await scope.update({ probeResults: { ...(cfg.probeResults ?? {}), [key]: outcome.support } });
1326
+ recorded = `recorded as "${outcome.support}"`;
1327
+ } catch (error) {
1328
+ recorded = `could not be recorded: ${String(error?.message ?? error)}`;
1329
+ }
1330
+ }
1331
+ }
1332
+
1333
+ const verdictText = outcome.support === "yes"
1334
+ ? "SUPPORTS images"
1335
+ : outcome.support === "no"
1336
+ ? "does NOT read images"
1337
+ : "INCONCLUSIVE (the route could not be probed)";
1338
+ const effect = outcome.support === "yes"
1339
+ ? "images will be sent to it directly"
1340
+ : outcome.support === "no"
1341
+ ? "images will be bridged to inspect_image hints"
1342
+ : "the existing list/declaration rules still apply";
1343
+ const evidence = Object.entries(outcome.evidence ?? {})
1344
+ .map(([name, value]) => ` ${name}: ${value}`)
1345
+ .join("\n");
1346
+ return [
1347
+ `${provider}/${requested}: ${verdictText}`,
1348
+ ` ${outcome.detail}`,
1349
+ ` ${recorded}; ${effect}`,
1350
+ evidence ? ` evidence:\n${evidence}` : "",
1351
+ ].filter(Boolean).join("\n");
1352
+ },
1353
+ }));
1354
+ },
1355
+ });
1356
+ }
1357
+
1358
+ function uninstallFeatures() {
1359
+ if (featureFiber === null) return;
1360
+ const fiber = featureFiber;
1361
+ featureFiber = null;
1362
+ // dispose() settles asynchronously and must never take the plugin down; the
1363
+ // next install starts a fresh fiber regardless of how this one ends.
1364
+ Promise.resolve(fiber.dispose()).catch((error) => {
1365
+ ctx.logger?.warn?.(`[tool-vision] feature teardown failed: ${String(error)}`);
1366
+ });
1367
+ }
1368
+
1369
+ // Re-install only when a field that gates a registration actually changed, so
1370
+ // editing, say, `model` does not tear the tools down and back up mid-session.
1371
+ function syncFeatures() {
1372
+ const cfg = getConfig();
1373
+ const key = registrationKey(cfg);
1374
+ if (featureFiber !== null && key === lastRegistrationKey) return;
1375
+ uninstallFeatures();
1376
+ lastRegistrationKey = key;
1377
+ if (cfg.enabled) installFeatures();
1378
+ }
1379
+
1380
+ // Compat shim: dsh-settings 0.1.2-rc.1 removed the module-level
1381
+ // `installSettingsSection` export (the provider now lives at ctx.settings).
1382
+ // Inline the same logic via ctx.inject(["settings"]) — works on both
1383
+ // 0.1.1 (module export wrapper) and 0.1.2 (ctx.settings) hosts.
1384
+ ctx.inject(["settings"], (sctx) => {
1385
+ const scope = sctx.settings.register(NS, Config, { base: config });
1386
+ sourceGetter = () => scope.get();
1387
+ settingsScope = scope;
1388
+ sctx.effect(() => () => {
1389
+ sourceGetter = null;
1390
+ settingsScope = null;
1391
+ });
1392
+ // Hot-apply: `enabled` switches the child fiber, and any field that gates a
1393
+ // registration re-installs it, so none of them needs a dsh restart anymore.
1394
+ scope.watch(() => syncFeatures());
1395
+ sctx.effect(() => () => uninstallFeatures());
1396
+ // The stored value governs from here on; the composition entry was only a
1397
+ // placeholder until the provider answered.
1398
+ syncFeatures();
1399
+ });
1400
+
1401
+ // ── model catalog route: settings support, so it lives on THIS fiber ───────
1402
+ // The master switch disposes the feature fiber on purpose (tools, bridge,
1403
+ // preview route), while the settings section itself deliberately stays
1404
+ // mounted — and a section whose list field cannot autocomplete is a section
1405
+ // you cannot configure before switching the plugin on. Registered here so it
1406
+ // outlives `enabled: false`; read-only and secret-free by construction.
1407
+ registerModelCatalogRoute(ctx, ctx.logger, getConfig);
1408
+
1409
+ // No settings provider: the composition entry is the whole configuration.
1410
+ syncFeatures();
1411
+ }
1412
+
1413
+ export {
1414
+ BRIDGE_MARKER,
1415
+ BRIDGE_PREVIEW_MAX_BYTES,
1416
+ BRIDGE_PREVIEW_MEDIA,
1417
+ BRIDGE_PREVIEW_ROUTE,
1418
+ Config,
1419
+ DEFAULT_DESCRIPTION,
1420
+ DEFAULT_MULTIMODAL_LIST_MODE,
1421
+ EXT_BY_MEDIA,
1422
+ MODEL_CATALOG_MAX_PER_PROVIDER,
1423
+ MODEL_CATALOG_ROUTE,
1424
+ MULTIMODAL_LIST_MODES,
1425
+ apply,
1426
+ attachPreStepBridge,
1427
+ autoPromotionWarned,
1428
+ bridgeMessages,
1429
+ currentModelAcceptsImage,
1430
+ currentRoute,
1431
+ deepFreeze,
1432
+ exportImage,
1433
+ hasImageBlock,
1434
+ inject,
1435
+ installAutoImageAdmission,
1436
+ installDispatchImageAdmission,
1437
+ lastModelDecision,
1438
+ modelListMatches,
1439
+ modelListMatchEntries,
1440
+ name,
1441
+ normalizeListMode,
1442
+ parseQuery,
1443
+ probeKey,
1444
+ probeVerdict,
1445
+ registerBridgePreviewRoute,
1446
+ routeDirectDecision,
1447
+ registerModelCatalogRoute,
1448
+ repairLoggedImages,
1449
+ resolveProbeTarget,
1450
+ routeDeclaresImage,
1451
+ unwrappedResolveModelInfo,
1452
+ warnAutoPromotion,
1453
+ };