dsh-tool-vision 0.9.3 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,1500 +1,1507 @@
1
- /**
2
- * dsh-tool-vision — external vision model for DeepSeek Harness.
3
- *
4
- * Two capabilities:
5
- *
6
- * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
- * any OpenAI-compatible chat/completions endpoint that supports
8
- * `image_url` content parts, and returns the vision model's text answer.
9
- *
10
- * 2. Image bridge — pasted images are bridged to text hints before they
11
- * enter a text-only model's request:
12
- *
13
- * - New images are bridged on the `agent/pre-step` waterfall (the only
14
- * seam where the harness lets a plugin replace the messages that enter
15
- * a step — they become the durable `user/message` log, so the
16
- * `llm/stream` request-reconstruction invariant stays satisfied).
17
- * - Images already logged before the plugin was installed (or before a
18
- * server restart) are repaired lazily with a surface `replace`, one
19
- * event at a time, on the first pre-step of the session.
20
- *
21
- * The bridged hint points at an exported local copy of the image, which
22
- * the agent hands to `inspect_image`. Which models skip the bridge
23
- * (v0.9.0) is decided by two independent inputs:
24
- *
25
- * - `multimodalModels` — a user-owned list whose meaning is set by
26
- * `multimodalListMode`: `whitelist` (listed models receive image blocks
27
- * directly; the historical meaning and the default), `blacklist`
28
- * (listed models are forced through the bridge), or `off` (list
29
- * ignored). Entries match the full model id, its bare id after the last
30
- * `/`, or `provider/id`, and may contain `*` / `?` globs.
31
- * - `autoDetectMultimodal` — when on, the route's own declared
32
- * `inputModalities` forms the base set that the list then adds to
33
- * (whitelist) or subtracts from (blacklist). The declaration is always
34
- * read from the *unwrapped* `resolveModelInfo`, because the admission
35
- * wrap below rewrites it for every model.
36
- *
37
- * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
38
- * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
39
- * inline thumbnails for bridged pasted images:
40
- *
41
- * - `bridgeMessages` stamps every bridged hint with an invisible marker
42
- * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
43
- * precisely instead of pattern-matching free text.
44
- * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
45
- * exported images to the same-origin page; the client inserts a
46
- * thumbnail above the hint text and opens a lightbox on click.
47
- * - Pure display layer: persisted messages, the transcript, the
48
- * model-facing text and the `inspect_image` chain are untouched.
49
- */
50
- import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
51
- import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
52
- import os from "node:os";
53
- import z from "@deepseek-ai/schemastery";
54
- import { defineTool } from "@deepseek-ai/dsh-tools";
55
- import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
56
- import { sessionHeaders } from "./lib/session-header.js";
57
- import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
58
-
59
- /** Cordis plugin name. */
60
- const name = "tool-vision";
61
- /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
62
- const inject = ["tools", "llm", "attachments", "webServer"];
63
- /** Settings namespace owned by this plugin (Web UI settings section). */
64
- const NS = "tool-vision";
65
-
66
- /**
67
- * Invisible prefix stamped onto every bridged hint text block. The browser
68
- * half uses it to recognize bridge text precisely (no free-text regex over
69
- * user messages), and the model-facing hint stays intact otherwise.
70
- */
71
- const BRIDGE_MARKER = "\u200b[bridge]";
72
-
73
- /** Loopback route serving bridged images to the same-origin page. */
74
- const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
75
- /**
76
- * Loopback JSON route the settings panel reads the configured model catalog
77
- * from (autocomplete for `multimodalModels` and the current-route readout).
78
- * Registered on the plugin's own fiber, so it stays up while the master
79
- * switch is off — a section you cannot configure until you switch it on would
80
- * be useless.
81
- */
82
- const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
83
- /** Per-provider cap on catalog entries returned to the panel. */
84
- const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
85
- /** Hard cap for a single served image (defense in depth; export is bounded). */
86
- const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
87
- /** Extensions the preview route serves (svg/ico intentionally excluded). */
88
- const BRIDGE_PREVIEW_MEDIA = {
89
- ".png": "image/png",
90
- ".jpg": "image/jpeg",
91
- ".jpeg": "image/jpeg",
92
- ".webp": "image/webp",
93
- ".gif": "image/gif",
94
- ".avif": "image/avif",
95
- ".bmp": "image/bmp",
96
- };
97
-
98
- const DEFAULT_DESCRIPTION =
99
- "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
100
- "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
101
- "optionally with a specific question. Returns the vision model's textual description or answer. " +
102
- "Use this whenever you need to read, describe, or extract information from image content, " +
103
- "including an image that reached you as a bridged text hint.";
104
-
105
- /** Runtime schema for the tool-vision row. */
106
- const Config = z.object({
107
- /**
108
- * Master switch, hot-applied. When off the plugin registers nothing at all —
109
- * no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
110
- * preview route, no llm capability wrap — while the settings section itself
111
- * stays mounted so the switch can turn it back on. Turning it off is the way
112
- * to stop every vision tool from reaching the model without uninstalling the
113
- * package or restarting dsh.
114
- */
115
- enabled: z.boolean().default(true),
116
- /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
117
- baseURL: z.string().default("https://api.openai.com/v1"),
118
- /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
119
- apiKey: z.string().default("").role("secret"),
120
- /** Environment variable holding the API key. */
121
- apiKeyEnv: z.string().default("VISION_API_KEY"),
122
- /** Vision model id served by the endpoint. */
123
- model: z.string().default("gpt-4o-mini"),
124
- /** Max output tokens for the vision call. */
125
- maxTokens: z.number().default(4096),
126
- /** Per-request timeout in milliseconds. */
127
- timeoutMs: z.number().default(60000),
128
- /** Largest local image accepted, in bytes. */
129
- maxImageBytes: z.number().default(10 * 1024 * 1024),
130
- /** Tool description shown to the model; overrides the default. */
131
- description: z.string().default(DEFAULT_DESCRIPTION),
132
- /** Bridge pasted images to text hints on models that cannot see images. */
133
- bridgeTextOnly: z.boolean().default(true),
134
- /** Export directory for bridged images; empty = system temp. */
135
- bridgeExportDir: z.string().default(""),
136
- /**
137
- * Model ids the list below refers to (v0.9.0). Each entry is matched
138
- * case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
139
- * bare id after the last `/` (`mimo-v2.5`), and `provider/id`
140
- * (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
141
- * (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
142
- * route happens to spell it.
143
- */
144
- multimodalModels: z.array(z.string()).default([]),
145
- /**
146
- * How `multimodalModels` is read (v0.9.0):
147
- *
148
- * - `whitelist` (default): a listed model receives image blocks directly
149
- * and is never bridged — the meaning the list always had, so an
150
- * existing configuration keeps behaving exactly as before.
151
- * - `blacklist`: a listed model is forced through the bridge even when its
152
- * route declares image input. This is the correction layer for a model
153
- * that claims image support its endpoint does not really have.
154
- * - `off`: the list is ignored entirely; nothing is forced either way.
155
- *
156
- * An unknown value falls back to `whitelist`.
157
- */
158
- multimodalListMode: z.string().default("whitelist"),
159
- /**
160
- * Let the current route's own declared `inputModalities` drive the bridge
161
- * decision, so only the models that cannot see images get bridged (v0.9.0,
162
- * **on by default**). This and `multimodalModels` are one decision, not two:
163
- * the declaration forms the base set, and the list unions with it
164
- * (whitelist) or subtracts from it (blacklist).
165
- *
166
- * The declaration is only whatever the profile — or the installed catalog —
167
- * says, and it CAN be wrong: profiles sometimes declare
168
- * `input: [text, image]` just to pass the host admission gate, and a wrong
169
- * "yes" sends the image straight to an endpoint that may reject it. The
170
- * harness's own text-model projection is disabled by `bridgeAutoImage`, so
171
- * nothing underneath catches that. Two things bound the damage: the first
172
- * promotion of a route logs a notice naming the escape hatch
173
- * ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
174
- * under `blacklist` mode forces the bridge back on. Set this to false to
175
- * bridge everything the list does not name.
176
- *
177
- * The value is always read from the *unwrapped* `resolveModelInfo`, so the
178
- * admission wrap can never feed its own claim back in as evidence.
179
- */
180
- autoDetectMultimodal: z.boolean().default(true),
181
- /**
182
- * Measured image capability per route (v0.9.0), keyed `"provider/model"` with
183
- * the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
184
- * never by hand — the only entry in this schema that records what a route
185
- * DOES rather than what it says.
186
- *
187
- * It outranks the route's own declaration, because a probe is a real request
188
- * and a declaration is only a claim. It does NOT outrank `multimodalModels`:
189
- * that list is explicit human intent, and a person who names a model
190
- * deserves the last word over an automated measurement.
191
- */
192
- probeResults: z.dict(z.string()).default({}),
193
- /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
194
- bridgePreview: z.boolean().default(true),
195
- /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
196
- bridgePreviewScanIntervalMs: z.number().default(2000),
197
- /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
198
- bridgePreviewHideHint: z.boolean().default(true),
199
- /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
200
- desktopScreenshot: z.boolean().default(false),
201
- /**
202
- * Advertise image input capability for every model while the bridge is on.
203
- * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
204
- * pasted images unless the current model declares `image` in its
205
- * `inputModalities`. The bridge handles those images anyway (they become
206
- * text hints the agent inspects through `inspect_image`), so this wraps the
207
- * llm service's `resolveModelInfo` to report image support for text-only
208
- * models too — users can paste images on any model without hand-editing
209
- * provider model configs.
210
- */
211
- bridgeAutoImage: z.boolean().default(true),
212
- /** Send a stable per-conversation id header (`x-opencode-session`) on vision
213
- * requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
214
- * on every request; requests without the header may error. The value is the
215
- * current dsh session id when the call runs inside one, else `sessionId`
216
- * below, else a stable per-process random id. Set false to disable. */
217
- sendSessionHeader: z.boolean().default(true),
218
- /** Header name carrying the session id. */
219
- sessionHeaderName: z.string().default("x-opencode-session"),
220
- /** Fixed session id override for callers without a dsh session context.
221
- * Empty = auto (per-process stable id). */
222
- sessionId: z.string().default(""),
223
- });
224
-
225
- const MIME_BY_EXT = {
226
- ".png": "image/png",
227
- ".jpg": "image/jpeg",
228
- ".jpeg": "image/jpeg",
229
- ".gif": "image/gif",
230
- ".webp": "image/webp",
231
- ".bmp": "image/bmp",
232
- ".avif": "image/avif",
233
- ".svg": "image/svg+xml",
234
- ".ico": "image/x-icon",
235
- };
236
-
237
- const EXT_BY_MEDIA = {
238
- "image/png": ".png",
239
- "image/jpeg": ".jpg",
240
- "image/webp": ".webp",
241
- "image/gif": ".gif",
242
- };
243
-
244
- /** True when any message carries an image content block. */
245
- function hasImageBlock(messages) {
246
- return (messages ?? []).some((m) =>
247
- Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
248
- );
249
- }
250
-
251
- /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
252
- function deepFreeze(value) {
253
- if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
254
- if (Array.isArray(value)) {
255
- for (const item of value) deepFreeze(item);
256
- return Object.freeze(value);
257
- }
258
- for (const key of Object.keys(value)) deepFreeze(value[key]);
259
- return Object.freeze(value);
260
- }
261
-
262
- /** Export one attachment to disk; returns the file path (cached per process). */
263
- const exportedPaths = new Map();
264
- async function exportImage(attachment, ctx, dir) {
265
- const cached = exportedPaths.get(attachment.attachmentId);
266
- if (cached) return cached;
267
- const { data } = await ctx.attachments.readImage(attachment);
268
- const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
269
- const safeName = attachment.name
270
- ? attachment.name
271
- .replace(/\.[^.]+$/, "")
272
- .replace(/[^\w\-]+/g, "_")
273
- .replace(/^_+|_+$/g, "")
274
- .slice(0, 40)
275
- : "";
276
- const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
277
- const path = join(dir, `${base}${ext}`);
278
- await writeFile(path, data);
279
- exportedPaths.set(attachment.attachmentId, path);
280
- return path;
281
- }
282
-
283
- /**
284
- * Replace image content blocks with text hints pointing at exported files.
285
- * Non-image messages are returned as-is (same reference); bridged messages
286
- * are fresh, deep-frozen objects with the original identity and source.
287
- * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
288
- * prefix so the browser half can recognize it precisely.
289
- * Exported for unit testing; `ctx` only needs `attachments`.
290
- */
291
- async function bridgeMessages(messages, ctx, dir) {
292
- const next = [];
293
- for (const message of messages) {
294
- const content = message?.content;
295
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
296
- next.push(message);
297
- continue;
298
- }
299
- const blocks = [];
300
- for (const block of content) {
301
- if (block?.type !== "image") {
302
- blocks.push(block);
303
- continue;
304
- }
305
- const path = await exportImage(block.attachment, ctx, dir);
306
- const name = block.attachment.name ? ` (${block.attachment.name})` : "";
307
- blocks.push({
308
- type: "text",
309
- text:
310
- `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
311
- `Inspect it with the inspect_image tool to see its content.]`,
312
- });
313
- }
314
- next.push(deepFreeze({ ...message, content: blocks }));
315
- }
316
- return next;
317
- }
318
-
319
- /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
320
- function parseQuery(rawUrl) {
321
- const query = {};
322
- const at = rawUrl.indexOf("?");
323
- if (at === -1) return query;
324
- for (const pair of rawUrl.slice(at + 1).split("&")) {
325
- const eq = pair.indexOf("=");
326
- if (eq === -1) continue;
327
- try {
328
- query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
329
- } catch {
330
- /* skip malformed pairs */
331
- }
332
- }
333
- return query;
334
- }
335
-
336
- /**
337
- * Advertise `image` in the `inputModalities` reported by the llm service for
338
- * every model, so the host admission gate lets pasted images through on
339
- * text-only models. The bridge turns those images into text hints anyway, so
340
- * this is pure admission: it never changes what the adapter actually streams
341
- * (the adapter's own stream validation reads the model's real `input` from
342
- * the provider config, untouched here).
343
- *
344
- * The wrap is installed on the shared llm service instance, so it must be
345
- * idempotent across HMR re-applies and restored on dispose. `marker` is a
346
- * per-instance record proving this plugin already owns the wrap; the
347
- * `installed` flag distinguishes "we wrapped it" from "the original was
348
- * replaced by something else" so dispose only restores what we replaced.
349
- *
350
- * Exported for unit testing; the test passes a fake llm service.
351
- */
352
- const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
353
- function installAutoImageAdmission(llm, logger) {
354
- if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
355
- logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
356
- return () => {};
357
- }
358
- if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
359
- const original = llm.resolveModelInfo.bind(llm);
360
- const wrapped = async (provider, model, signal) => {
361
- const info = await original(provider, model, signal);
362
- if (!info) return info;
363
- const mods = info.inputModalities;
364
- if (Array.isArray(mods) && mods.includes("image")) return info;
365
- return { ...info, inputModalities: [...(mods ?? []), "image"] };
366
- };
367
- let installed = false;
368
- // The marker carries BOTH functions: the wrapped one proves ownership for
369
- // dispose, and `original` is the only honest source of a route's real
370
- // modalities once the wrap is in place (`unwrappedResolveModelInfo`).
371
- llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
372
- llm.resolveModelInfo = wrapped;
373
- installed = true;
374
- logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
375
- return () => {
376
- if (installed) {
377
- if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
378
- installed = false;
379
- }
380
- delete llm[LLM_RESOLVE_WRAP_MARK];
381
- };
382
- }
383
-
384
- /**
385
- * Admission is not dispatch, and only dispatch decides whether the pixels arrive.
386
- *
387
- * {@link installAutoImageAdmission} wraps `llm.resolveModelInfo`. That method
388
- * governs who may *offer* an image — the host gate that refuses a pasted image,
389
- * the model list, the settings readout — and it is deliberately "pure admission:
390
- * it never changes what the adapter actually streams".
391
- *
392
- * What the adapter actually streams is decided one layer down, by
393
- * `LlmService.generate`:
394
- *
395
- * const adapterCall = await adapter.prepareCall(provider, model, signal);
396
- * modelInfo = this.normalizeModelInfo(registration, model, adapterCall.model);
397
- * if (modelInfo.inputModalities !== undefined
398
- * && !modelInfo.inputModalities.includes("image")
399
- * && projectedMessages.some((message) => contentHasImage(message.content)))
400
- * projectedMessages = projectImagesForTextModel(projectedMessages);
401
- *
402
- * When that list lacks `image`, every image block is rewritten into a text
403
- * placeholder (`textOnlyImageText`) *before the adapter is called*. So on a
404
- * route the plugin had already measured as image-capable, and whose bridge
405
- * decision was therefore "direct, do not intercept", the image was still
406
- * destroyed — by a check that reads the adapter's declaration and nothing else.
407
- * `probeResults`, `multimodalModels` and `autoDetectMultimodal` were all
408
- * invisible to it.
409
- *
410
- * This wraps `llm.registration` — the accessor the core itself calls — so every
411
- * adapter, including one registered later, answers `prepareCall` through
412
- * {@link routeDirectDecision}: the same rule the bridge uses, so the two can
413
- * never disagree about a route.
414
- *
415
- * When the decision is "bridge", nothing changes: the bridge already turned the
416
- * image into an `inspect_image` hint, and the core's projection stays as the
417
- * correct fallback for any image that still reaches dispatch.
418
- *
419
- * Idempotent across HMR re-applies, and dispose restores every adapter it
420
- * touched plus the accessor itself.
421
- */
422
- const LLM_REGISTRATION_WRAP_MARK = Symbol("dsh-tool-vision.registration.wrapped");
423
- function installDispatchImageAdmission(llm, getConfig, logger) {
424
- if (llm === undefined || llm === null || typeof llm.registration !== "function") {
425
- logger?.warn?.("[tool-vision] llm service unavailable; dispatch image admission not installed");
426
- return () => {};
427
- }
428
- if (llm[LLM_REGISTRATION_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
429
- const original = llm.registration.bind(llm);
430
- /** adapter -> { prepareCall, modelOf } original methods, for dispose. */
431
- const patched = new Map();
432
- /** Set of arrays where we pushed "image", for cleanup on dispose. */
433
- const modifiedInputs = new Set();
434
- /** Keyed routes (probeKey) known to have direct dispatch permission. */
435
- const directRouteCache = new Set();
436
-
437
- const wrapped = (provider) => {
438
- const registration = original(provider);
439
- const adapter = registration?.adapter;
440
- if (adapter !== undefined && adapter !== null
441
- && !patched.has(adapter) && typeof adapter.prepareCall === "function") {
442
- const rawPrepare = adapter.prepareCall;
443
- const rawModelOf = typeof adapter.modelOf === "function" ? adapter.modelOf : undefined;
444
- const prepare = rawPrepare.bind(adapter);
445
-
446
- if (rawModelOf !== undefined) {
447
- adapter.modelOf = function (snapshot, route, model) {
448
- const resolved = rawModelOf.call(this, snapshot, route, model);
449
- if (resolved !== undefined && resolved !== null
450
- && Array.isArray(resolved.input) && !resolved.input.includes("image")
451
- && directRouteCache.has(probeKey(route, model))) {
452
- return { ...resolved, input: [...resolved.input, "image"] };
453
- }
454
- return resolved;
455
- };
456
- }
457
-
458
- adapter.prepareCall = async (route, model, signal) => {
459
- const call = await prepare(route, model, signal);
460
- if (call === undefined || call === null || call.model === undefined || call.model === null) return call;
461
- const mods = call.model.inputModalities;
462
- if (Array.isArray(mods) && mods.includes("image")) return call;
463
- let direct = false;
464
- try {
465
- ({ direct } = await routeDirectDecision(route, model, getConfig(), llm));
466
- } catch {
467
- // A decision that cannot be made is not a licence to send pixels a
468
- // route may reject: leave the core's projection in place.
469
- return call;
470
- }
471
- if (!direct) {
472
- directRouteCache.delete(probeKey(route, model));
473
- return call;
474
- }
475
- directRouteCache.add(probeKey(route, model));
476
-
477
- // In dsh-llm-pi-ai, adapter.streamWithSnapshot checks:
478
- // if (containsImage && !model.input.includes("image")) throw LlmError(...)
479
- // where model is obtained via this.modelOf(snapshot, provider, model) -> snapshot.models.getModel(route, model).
480
- // Ensure that the adapter snapshot's model also admits images so the adapter-level guard passes.
481
- try {
482
- const snapshot = typeof adapter.current === "function" ? adapter.current() : undefined;
483
- const targetModel = snapshot?.models?.getModel?.(route, model);
484
- if (targetModel !== undefined && targetModel !== null
485
- && Array.isArray(targetModel.input) && !targetModel.input.includes("image")) {
486
- targetModel.input.push("image");
487
- modifiedInputs.add(targetModel.input);
488
- }
489
- } catch {
490
- /* best-effort */
491
- }
492
-
493
- return { ...call, model: { ...call.model, inputModalities: [...(mods ?? []), "image"] } };
494
- };
495
- patched.set(adapter, { prepareCall: rawPrepare, modelOf: rawModelOf });
496
- }
497
- return registration;
498
- };
499
- llm[LLM_REGISTRATION_WRAP_MARK] = { original, wrapped };
500
- llm.registration = wrapped;
501
- logger?.debug?.("[tool-vision] dispatch image admission installed (llm.registration wrapped)");
502
- return () => {
503
- for (const [adapter, originalMethods] of patched) {
504
- if (typeof originalMethods.prepareCall === "function") adapter.prepareCall = originalMethods.prepareCall;
505
- if (typeof originalMethods.modelOf === "function") adapter.modelOf = originalMethods.modelOf;
506
- }
507
- patched.clear();
508
- for (const arr of modifiedInputs) {
509
- const idx = arr.indexOf("image");
510
- if (idx !== -1) arr.splice(idx, 1);
511
- }
512
- modifiedInputs.clear();
513
- directRouteCache.clear();
514
- if (llm.registration === wrapped) llm.registration = original;
515
- delete llm[LLM_REGISTRATION_WRAP_MARK];
516
- };
517
- }
518
-
519
- /**
520
- * Register the loopback route that serves bridged images to the same-origin
521
- * page (the preview thumbnails). Read-only and tightly scoped:
522
- * - only files inside the bridge export directory (no traversal);
523
- * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
524
- * - Host restricted to the local machine;
525
- * - hard 20MB cap per file.
526
- */
527
- function registerBridgePreviewRoute(ctx, exportDir, logger) {
528
- const webServer = ctx.get("webServer");
529
- if (webServer === undefined) {
530
- logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
531
- return;
532
- }
533
- const bridgeDir = resolvePath(exportDir);
534
- ctx.effect(() => webServer.register({
535
- kind: "exact",
536
- path: BRIDGE_PREVIEW_ROUTE,
537
- async handler(req, res) {
538
- try {
539
- const raw = String(req.url ?? "");
540
- const query = parseQuery(raw);
541
- const p = query.p;
542
- if (typeof p !== "string" || p.length === 0) {
543
- res.writeHead(400);
544
- res.end("bad request");
545
- return;
546
- }
547
- const lower = p.toLowerCase();
548
- const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
549
- if (mediaType === undefined) {
550
- res.writeHead(400);
551
- res.end("not an image path");
552
- return;
553
- }
554
- const host = String(req.headers?.host ?? "");
555
- if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
556
- res.writeHead(403);
557
- res.end("forbidden");
558
- return;
559
- }
560
- const target = resolvePath(p);
561
- if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
562
- res.writeHead(403);
563
- res.end("forbidden");
564
- return;
565
- }
566
- const info = await stat(target);
567
- if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
568
- res.writeHead(404);
569
- res.end("not found");
570
- return;
571
- }
572
- const bytes = await readFile(target);
573
- res.writeHead(200, {
574
- "Content-Type": mediaType,
575
- "Cache-Control": "private, max-age=60",
576
- });
577
- res.end(bytes);
578
- } catch {
579
- try {
580
- res.writeHead(404);
581
- res.end("not found");
582
- } catch {
583
- /* response already sent */
584
- }
585
- }
586
- },
587
- }), "dsh-tool-vision: bridge preview route");
588
- }
589
-
590
- /**
591
- * Register the loopback JSON route the settings panel reads:
592
- *
593
- * { current: {provider, model, direct, source, mode}, providers: [...] }
594
- *
595
- * `providers` comes from `llm.listProviders()` + `llm.listModels(provider)` —
596
- * the routes the harness actually has configured, which is exactly the set a
597
- * `multimodalModels` entry may name. `listModels` is a different seam from
598
- * `resolveModelInfo`, so these declared modalities are the profile's own and
599
- * are NOT contaminated by the admission wrap (a model the wrap advertises as
600
- * image-capable still reports its real declaration here).
601
- *
602
- * Same confinement as the image route: loopback Host only, read-only,
603
- * `no-store`, and no credentials or endpoints in the payload — provider ids,
604
- * model ids/names and one boolean each.
605
- */
606
- function registerModelCatalogRoute(ctx, logger, getConfig) {
607
- const webServer = ctx.get("webServer");
608
- if (webServer === undefined) {
609
- logger?.warn?.("[tool-vision] webServer unavailable; model catalog route not registered");
610
- return;
611
- }
612
- ctx.effect(() => webServer.register({
613
- kind: "exact",
614
- path: MODEL_CATALOG_ROUTE,
615
- async handler(req, res) {
616
- const send = (status, payload) => {
617
- res.writeHead(status, {
618
- "Content-Type": "application/json; charset=utf-8",
619
- "Cache-Control": "no-store",
620
- });
621
- res.end(JSON.stringify(payload));
622
- };
623
- const host = String(req.headers?.host ?? "");
624
- if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
625
- send(403, { error: "forbidden" });
626
- return;
627
- }
628
- const llm = ctx.get("llm");
629
- if (llm === undefined) {
630
- send(503, { error: "llm service unavailable" });
631
- return;
632
- }
633
- // `listed` is computed HERE, with the same matcher the bridge itself
634
- // uses, so the panel never reimplements the rule (and can never
635
- // disagree with the decision it is describing).
636
- const cfg = typeof getConfig === "function" ? getConfig() : undefined;
637
- const list = Array.isArray(cfg?.multimodalModels) ? cfg.multimodalModels : [];
638
- const listMode = normalizeListMode(cfg?.multimodalListMode);
639
- let routes = [];
640
- try {
641
- routes = llm.listProviders() ?? [];
642
- } catch (error) {
643
- logger?.debug?.(`[tool-vision] listProviders failed: ${String(error)}`);
644
- }
645
- const providers = [];
646
- for (const entry of routes) {
647
- const id = entry?.id;
648
- if (typeof id !== "string" || id.length === 0) continue;
649
- const record = {
650
- id,
651
- name: typeof entry?.name === "string" && entry.name.length > 0 ? entry.name : id,
652
- models: [],
653
- };
654
- try {
655
- const models = await llm.listModels(id);
656
- record.models = (models ?? []).slice(0, MODEL_CATALOG_MAX_PER_PROVIDER).map((model) => {
657
- const modelId = String(model.id);
658
- const matchedEntries = modelListMatchEntries(list, id, modelId);
659
- return {
660
- id: modelId,
661
- name: typeof model.name === "string" && model.name.length > 0 ? model.name : modelId,
662
- image: Array.isArray(model.inputModalities) && model.inputModalities.includes("image"),
663
- listed: matchedEntries.length > 0,
664
- matchedEntries,
665
- // What the route was MEASURED doing, when anyone has probed it.
666
- // `null` means "not probed", which the panel must render
667
- // differently from a probe that came back negative.
668
- probe: probeVerdict(cfg, id, modelId) ?? null,
669
- };
670
- });
671
- } catch (error) {
672
- // One broken route must never blank the whole panel: report it and
673
- // keep going, so the other providers still list.
674
- record.error = String(error?.message ?? error);
675
- }
676
- providers.push(record);
677
- }
678
- send(200, {
679
- current: { ...lastModelDecision },
680
- list,
681
- listMode,
682
- probeResults: typeof getConfig === "function" ? { ...(getConfig()?.probeResults ?? {}) } : {},
683
- providers,
684
- });
685
- },
686
- }), "dsh-tool-vision: model catalog route");
687
- }
688
-
689
- /** Accepted `multimodalListMode` values; anything else falls back to whitelist. */
690
- const MULTIMODAL_LIST_MODES = ["off", "whitelist", "blacklist"];
691
- const DEFAULT_MULTIMODAL_LIST_MODE = "whitelist";
692
-
693
- /**
694
- * Normalize a configured list mode. The settings section only ever writes one
695
- * of {@link MULTIMODAL_LIST_MODES}, but the value can also arrive from a
696
- * hand-edited `settings.yaml`, so an unknown string is clamped rather than
697
- * treated as "no mode" (which would silently bridge everything).
698
- */
699
- function normalizeListMode(value) {
700
- return MULTIMODAL_LIST_MODES.includes(value) ? value : DEFAULT_MULTIMODAL_LIST_MODE;
701
- }
702
-
703
- const listEntryCache = new Map();
704
- /**
705
- * Compile one list entry into an anchored, case-insensitive regexp. A literal
706
- * entry is just an exact match; `*` and `?` are the only metacharacters, so a
707
- * model id containing `.` or `+` still matches itself.
708
- */
709
- function listEntryRegExp(entry) {
710
- const cached = listEntryCache.get(entry);
711
- if (cached !== undefined) return cached;
712
- const source = entry
713
- .replace(/[.+^${}()|[\]\\]/g, "\\$&")
714
- .replace(/\*/g, ".*")
715
- .replace(/\?/g, ".");
716
- const compiled = new RegExp(`^${source}$`, "i");
717
- listEntryCache.set(entry, compiled);
718
- return compiled;
719
- }
720
-
721
- /**
722
- * Whether any entry in `patterns` names this route. Each entry is tested
723
- * against the full model id, its bare id after the last `/`, and
724
- * `provider/id` — so `mimo-v2.5`, `xiaomi/mimo-v2.5` and
725
- * `commandcode/xiaomi/mimo-v2.5` all address the same route, and `*vl*`
726
- * addresses every variant of one family. Exported for unit testing.
727
- */
728
- function modelListMatches(patterns, provider, model) {
729
- return modelListMatchEntries(patterns, provider, model).length > 0;
730
- }
731
-
732
- /**
733
- * The list entries that address this route. The panel needs *which* entry hit
734
- * so that unchecking a model removes exactly the entry responsible — matching
735
- * `mimo-v2.5` against `xiaomi/mimo-v2.5` must not leave an orphan behind, and
736
- * must not delete an unrelated entry either. Exported for unit testing.
737
- */
738
- function modelListMatchEntries(patterns, provider, model) {
739
- if (!Array.isArray(patterns) || patterns.length === 0) return [];
740
- if (typeof model !== "string" || model.length === 0) return [];
741
- const slash = model.lastIndexOf("/");
742
- const candidates = [model, slash === -1 ? model : model.slice(slash + 1)];
743
- if (typeof provider === "string" && provider.length > 0) candidates.push(`${provider}/${model}`);
744
- const hits = [];
745
- for (const raw of patterns) {
746
- if (raw === undefined || raw === null) continue;
747
- const entry = String(raw).trim();
748
- if (entry.length === 0) continue;
749
- const re = listEntryRegExp(entry);
750
- if (candidates.some((candidate) => re.test(candidate))) hits.push(String(raw));
751
- }
752
- return hits;
753
- }
754
-
755
- /**
756
- * The route the session is on: the logged request header first, then the
757
- * agent's own options. `provider` is optional (older logs may lack it).
758
- */
759
- function currentRoute(agent) {
760
- const header = agent?.session?.requestHeader?.();
761
- return {
762
- provider: header?.config?.provider ?? agent?.options?.provider,
763
- model: header?.config?.model ?? agent?.options?.model,
764
- };
765
- }
766
-
767
- /**
768
- * The last decision `currentModelAcceptsImage` made, for the settings panel's
769
- * "current route" readout (served over {@link MODEL_CATALOG_ROUTE}). A plain
770
- * mutable record — the panel only ever reads it.
771
- */
772
- const lastModelDecision = {
773
- provider: undefined,
774
- model: undefined,
775
- direct: false,
776
- source: "none",
777
- mode: DEFAULT_MULTIMODAL_LIST_MODE,
778
- };
779
-
780
- /**
781
- * `resolveModelInfo` as it was before {@link installAutoImageAdmission}
782
- * wrapped it. Auto-detection MUST read through this: the wrap reports image
783
- * support for *every* model, so reading the wrapped method would make the
784
- * plugin's own claim the evidence for itself.
785
- */
786
- function unwrappedResolveModelInfo(llm) {
787
- const marker = llm?.[LLM_RESOLVE_WRAP_MARK];
788
- const fn = marker?.original ?? llm?.resolveModelInfo;
789
- return typeof fn === "function" ? fn.bind(llm) : undefined;
790
- }
791
-
792
- /**
793
- * Whether the route declares `image` input, read before the admission wrap.
794
- * `false` and `undefined` are the same answer to the caller ("not known to be
795
- * multimodal"): a route nobody could resolve is bridged, never guessed.
796
- */
797
- async function routeDeclaresImage(llm, provider, model) {
798
- const resolve = unwrappedResolveModelInfo(llm);
799
- if (resolve === undefined || typeof provider !== "string" || typeof model !== "string") {
800
- return false;
801
- }
802
- try {
803
- const info = await resolve(provider, model);
804
- return info?.inputModalities?.includes("image") === true;
805
- } catch {
806
- return false;
807
- }
808
- }
809
-
810
- /** Routes already warned about, so the notice appears once per route. */
811
- const autoPromotionWarned = new Set();
812
-
813
- /** Registry key for one measured route. Provider-qualified: the same model id
814
- * can be served by two gateways with different real capabilities. */
815
- function probeKey(provider, model) {
816
- return `${typeof provider === "string" ? provider : ""}/${model}`;
817
- }
818
-
819
- /**
820
- * The measured verdict for a route, or `undefined` when it was never probed.
821
- * Anything other than the two recorded values (a hand-edited settings.yaml, a
822
- * future format) reads as "not probed" rather than as a guess.
823
- */
824
- function probeVerdict(config, provider, model) {
825
- const results = config?.probeResults;
826
- if (results === undefined || results === null || typeof model !== "string" || model.length === 0) {
827
- return undefined;
828
- }
829
- const value = results[probeKey(provider, model)];
830
- return value === "yes" || value === "no" ? value : undefined;
831
- }
832
-
833
- /**
834
- * Resolve the endpoint and credential a route actually sends to, so a probe
835
- * talks to the same place the model does instead of a copy of it.
836
- *
837
- * The pieces come from two seams that a plugin is allowed to read:
838
- * - `llm.listConfigurableProviders()` names the settings namespace and the
839
- * path a provider's profile lives at;
840
- * - `settings.get(ns)` resolves that profile (`baseURL`, `apiKeyEnv`, `api`);
841
- * - `credentials.resolve(ref)` turns `apiKeyEnv` into the secret itself
842
- * (`.value`), the same way `dsh-llm-pi-ai` does it for a real call.
843
- *
844
- * Read-only, and the secret never leaves this function's caller: the probe
845
- * result records a verdict, never an endpoint or a key.
846
- *
847
- * @throws {Error} with a caller-facing reason when the route is unprobeable.
848
- */
849
- async function resolveProbeTarget(ctx, provider, model) {
850
- const llm = ctx.get("llm");
851
- const settings = ctx.get("settings");
852
- const credentials = ctx.get("credentials");
853
- if (settings === undefined) {
854
- throw new Error("the settings service is unavailable, so the route's endpoint cannot be read");
855
- }
856
- let entry;
857
- try {
858
- entry = (llm?.listConfigurableProviders?.() ?? []).find((candidate) => candidate?.provider === provider);
859
- } catch {
860
- /* an adapter that cannot describe its providers simply has no entry */
861
- }
862
- const settingsNs = entry?.settingsNs;
863
- if (typeof settingsNs !== "string" || settingsNs.length === 0) {
864
- throw new Error(
865
- `provider "${provider}" does not publish a settings namespace, so its endpoint cannot be resolved for probing`,
866
- );
867
- }
868
- let profile = settings.get(settingsNs);
869
- const path = Array.isArray(entry.settingsPath) && entry.settingsPath.length > 0 ? entry.settingsPath : [provider];
870
- for (const segment of path) profile = profile?.[segment];
871
- if (profile === undefined || profile === null || typeof profile !== "object") {
872
- throw new Error(`no stored profile for provider "${provider}" under "${settingsNs}"`);
873
- }
874
- const baseURL = typeof profile.baseURL === "string" && profile.baseURL.length > 0 ? profile.baseURL : undefined;
875
- if (baseURL === undefined) {
876
- throw new Error(`provider "${provider}" has no baseURL, so its endpoint cannot be probed`);
877
- }
878
- let apiKey;
879
- const ref = profile.apiKeyEnv;
880
- if (typeof ref === "string" && ref.length > 0) {
881
- try {
882
- const record = await credentials?.resolve?.(ref);
883
- apiKey = typeof record?.value === "string" ? record.value : undefined;
884
- } catch (error) {
885
- throw new Error(`stored credential "${ref}" could not be read: ${String(error?.message ?? error)}`);
886
- }
887
- }
888
- return { baseURL, apiKey, api: profile.api, model };
889
- }
890
-
891
- /**
892
- * One-time notice when a route receives images directly *only* because of its
893
- * own declaration (source `auto`). This is the one failure auto-detection
894
- * cannot rule out: if the declaration is wrong, the endpoint rejects the image
895
- * after the message is already durable, and the error alone does not say what
896
- * to change. Naming the escape hatch the first time it happens turns a
897
- * confusing 400 into a one-line fix — and now names the probe too, which
898
- * settles the question instead of asking the user to guess.
899
- * Read {@link lastModelDecision}; never throws.
900
- */
901
- function warnAutoPromotion(logger) {
902
- const { provider, model, direct, source } = lastModelDecision;
903
- if (direct !== true || source !== "auto" || typeof model !== "string" || model.length === 0) return;
904
- const key = `${typeof provider === "string" ? provider : ""}\u0000${model}`;
905
- if (autoPromotionWarned.has(key)) return;
906
- autoPromotionWarned.add(key);
907
- logger?.warn?.(
908
- `[tool-vision] "${model}" receives images directly because its route declares image input, ` +
909
- `which is unverified; run vision_probe_model to measure it, or list it in multimodalModels ` +
910
- `under multimodalListMode: blacklist to force the bridge`,
911
- );
912
- }
913
-
914
- /**
915
- * Whether images may be handed to this route directly, and why.
916
- *
917
- * The plugin's single precedence rule. Three inputs, deliberately independent:
918
- *
919
- * - the base set — the route's own declared `inputModalities`, and only when
920
- * `autoDetectMultimodal` is on. It is read through
921
- * {@link unwrappedResolveModelInfo}, never through the admission wrap.
922
- * - the measurement — `probeResults`, which outranks the declaration because a
923
- * probe is a real request and a declaration is only a statement.
924
- * - the list — `multimodalModels` under `multimodalListMode`. Explicit human
925
- * intent, so it outranks both: whitelist adds to the base set, blacklist
926
- * subtracts from it.
927
- *
928
- * Two callers need this exact answer and must never disagree:
929
- * - {@link currentModelAcceptsImage} — the bridge, deciding whether to
930
- * intercept an image at all;
931
- * - {@link installDispatchImageAdmission} — the dispatch path, where the core
932
- * independently decides whether the model may receive the image block.
933
- *
934
- * Returns `direct: true` when bridging is disabled (nothing would be bridged
935
- * anyway).
936
- */
937
- async function routeDirectDecision(provider, model, config, llm) {
938
- const mode = normalizeListMode(config.multimodalListMode);
939
- if (!config.bridgeTextOnly) return { direct: true, source: "bridge-off", mode };
940
- if (typeof model !== "string" || model.length === 0) {
941
- return { direct: false, source: "no-route", mode };
942
- }
943
- const base = config.autoDetectMultimodal
944
- ? await routeDeclaresImage(llm, provider, model)
945
- : false;
946
- let direct = base;
947
- let source = base ? "auto" : "default";
948
- // A measured verdict outranks the route's own claim: a probe is a real
949
- // request, a declaration is only a statement. Both directions count, so a
950
- // route that says "image" and answered without reading the pixels goes back
951
- // to the bridge without anyone hunting for the right list entry.
952
- const probed = probeVerdict(config, provider, model);
953
- if (probed !== undefined) {
954
- direct = probed === "yes";
955
- source = probed === "yes" ? "probe-yes" : "probe-no";
956
- }
957
- // `off` means the list carries no opinion at all — it is not "empty list
958
- // under whitelist", it is "no list semantics", so nothing is forced. The
959
- // human list keeps the last word over both other signals.
960
- const listed = mode !== "off" && modelListMatches(config.multimodalModels, provider, model);
961
- if (listed) {
962
- direct = mode !== "blacklist";
963
- source = mode === "blacklist" ? "blacklist" : "whitelist";
964
- }
965
- return { direct, source, mode };
966
- }
967
-
968
- /**
969
- * Whether the session's current model may receive image blocks directly.
970
- *
971
- * {@link routeDirectDecision} keyed by the session's route, with the answer
972
- * recorded in {@link lastModelDecision} for the settings panel.
973
- */
974
- async function currentModelAcceptsImage(agent, config, llm) {
975
- const { provider, model } = currentRoute(agent);
976
- const decision = await routeDirectDecision(provider, model, config, llm);
977
- Object.assign(lastModelDecision, { provider, model, ...decision });
978
- return decision.direct;
979
- }
980
-
981
- /**
982
- * Lazily bridge image blocks that are already part of the session log
983
- * (pasted before the plugin was active, or before a restart). Each affected
984
- * event is rewritten once with a surface `replace`, which swaps the durable
985
- * derivation (and the transcript) to the text hint. Events that are no
986
- * longer on the surface (already shadowed) are skipped and remembered.
987
- * `repaired` tracks per-session state: a `Set` of handled seqs plus a
988
- * monotonic scan cursor.
989
- */
990
- async function repairLoggedImages(ctx, session, exportDir, repaired) {
991
- const events = session.events;
992
- for (let index = repaired.cursor; index < events.length; index += 1) {
993
- const event = events[index];
994
- if (event.type !== "user/message" || repaired.set.has(event.seq)) {
995
- repaired.set.add(event.seq);
996
- continue;
997
- }
998
- const content = event.data?.content;
999
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
1000
- repaired.set.add(event.seq);
1001
- continue;
1002
- }
1003
- const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
1004
- try {
1005
- session.append("user/message", bridged, {
1006
- surfaceOp: { op: "replace", start: event.seq, end: event.seq },
1007
- sourceEventSeqs: [event.seq],
1008
- });
1009
- ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
1010
- } catch (error) {
1011
- ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
1012
- }
1013
- repaired.set.add(event.seq);
1014
- }
1015
- repaired.cursor = events.length;
1016
- }
1017
-
1018
- /**
1019
- * Install the pre-step bridge at the root level. Agent-scoped waterfalls
1020
- * admit untagged (root) listeners, so one listener serves every agent —
1021
- * including sessions resumed after a server restart, which never re-fire
1022
- * `session/created` for per-agent attachments. Runs before every proposed
1023
- * step: new pasted images are bridged into the durable log, and stuck
1024
- * logged images are repaired, before the model request is derived from it.
1025
- */
1026
- function attachPreStepBridge(ctx, getConfig, exportDir) {
1027
- const repairedBySession = new Map();
1028
- ctx.on("agent/pre-step", async (payload, next) => {
1029
- const decision = await next();
1030
- if (!decision || decision.kind !== "enter") return decision;
1031
- const agent = payload?.agent;
1032
- if (!agent?.session) return decision;
1033
- try {
1034
- const acceptsImage = await currentModelAcceptsImage(agent, getConfig(), ctx.get("llm"));
1035
- warnAutoPromotion(ctx.logger);
1036
- if (!acceptsImage) {
1037
- let repaired = repairedBySession.get(agent.session.id);
1038
- if (!repaired) {
1039
- repaired = { set: new Set(), cursor: 0 };
1040
- repairedBySession.set(agent.session.id, repaired);
1041
- }
1042
- await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
1043
- ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
1044
- });
1045
- }
1046
- if (acceptsImage) return decision;
1047
- const messages = await bridgeMessages(decision.messages, ctx, exportDir);
1048
- if (messages.every((message, index) => message === decision.messages[index])) return decision;
1049
- return { ...decision, messages };
1050
- } catch (error) {
1051
- ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
1052
- return decision;
1053
- }
1054
- });
1055
- }
1056
-
1057
- function resolveApiKey(config) {
1058
- if (config.apiKey) return config.apiKey;
1059
- if (config.apiKeyEnv) {
1060
- const fromEnv = process.env[config.apiKeyEnv];
1061
- if (fromEnv) return fromEnv;
1062
- }
1063
- return process.env.OPENAI_API_KEY ?? "";
1064
- }
1065
-
1066
- /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
1067
- async function toImageUrl(target, cwd, config) {
1068
- if (/^https?:\/\//i.test(target)) return { url: target, note: target };
1069
- const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
1070
- const info = await stat(abs).catch(() => null);
1071
- if (!info) throw new Error(`image not found: ${abs}`);
1072
- if (info.size > config.maxImageBytes) {
1073
- throw new Error(
1074
- `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
1075
- );
1076
- }
1077
- const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
1078
- if (!mime) {
1079
- throw new Error(
1080
- `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
1081
- );
1082
- }
1083
- const data = await readFile(abs);
1084
- return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
1085
- }
1086
-
1087
- /** One OpenAI-compatible chat/completions call with an image_url content part. */
1088
- async function callVision(config, imageUrl, question, detail, signal, exec) {
1089
- const key = resolveApiKey(config);
1090
- if (!key) {
1091
- throw new Error(
1092
- `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
1093
- );
1094
- }
1095
- const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
1096
- const endpoint = new URL("chat/completions", base);
1097
- const sessionHeader = sessionHeaders(config, exec);
1098
- const controller = new AbortController();
1099
- const timer = setTimeout(
1100
- () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
1101
- config.timeoutMs,
1102
- );
1103
- const onSignalAbort = () => controller.abort(signal?.reason);
1104
- if (signal) {
1105
- if (signal.aborted) controller.abort();
1106
- else signal.addEventListener("abort", onSignalAbort, { once: true });
1107
- }
1108
- const content = [
1109
- { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
1110
- { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
1111
- ];
1112
- try {
1113
- const response = await fetch(endpoint, {
1114
- method: "POST",
1115
- headers: {
1116
- "Content-Type": "application/json",
1117
- Authorization: `Bearer ${key}`,
1118
- ...sessionHeader,
1119
- },
1120
- body: JSON.stringify({
1121
- model: config.model,
1122
- messages: [{ role: "user", content }],
1123
- max_tokens: config.maxTokens,
1124
- }),
1125
- signal: controller.signal,
1126
- });
1127
- const body = await response.json().catch(() => null);
1128
- if (!response.ok) {
1129
- const detailText = body?.error?.message ?? response.statusText;
1130
- throw new Error(
1131
- `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
1132
- );
1133
- }
1134
- // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
1135
- // `reasoning_content` first; when the final `content` is empty or was cut
1136
- // off by max_tokens, fall back to the reasoning text so the answer is
1137
- // still useful.
1138
- const message = body?.choices?.[0]?.message;
1139
- let answer = message?.content ?? "";
1140
- if (!answer.trim()) answer = message?.reasoning_content ?? "";
1141
- if (typeof answer !== "string" || !answer.trim()) {
1142
- throw new Error("vision endpoint returned an empty response");
1143
- }
1144
- return answer.trim();
1145
- } finally {
1146
- clearTimeout(timer);
1147
- signal?.removeEventListener("abort", onSignalAbort);
1148
- }
1149
- }
1150
-
1151
- function apply(ctx, config) {
1152
- // ── settings-backed configuration ─────────────────────────────────────────
1153
- // The composition entry stays the `base` layer; a registered `tool-vision`
1154
- // settings section (Web UI section, settings.yaml) overlays it live, so
1155
- // edits hot-apply without a restart. `sourceGetter` is a GETTER
1156
- // (`() => scope.get()`), not the config object — keep it and call it at use
1157
- // time, or `getConfig()` would return a function and every `cfg.*` read
1158
- // would be undefined (apiKey included).
1159
- let current = config;
1160
- let sourceGetter = null;
1161
- const getConfig = () => (sourceGetter ? sourceGetter() : current);
1162
- // The registered settings scope, kept so tools can PERSIST what they measure.
1163
- // Null whenever no settings provider is mounted (composition-entry-only runs),
1164
- // in which case a probe still reports its verdict but cannot record it.
1165
- let settingsScope = null;
1166
- const getSettingsScope = () => settingsScope;
1167
-
1168
- // ── master switch: a child fiber owns every registration ──────────────────
1169
- // `ctx.tools.register`, `ctx.effect` and `ctx.on` are all effects on the fiber
1170
- // that makes them, so confining the tool/bridge/route registrations to a child
1171
- // fiber makes `enabled` real: disposing that fiber unregisters all 15 tools,
1172
- // removes the pre-step listener and the preview route, and restores the llm
1173
- // capability wrap. The settings registration deliberately stays on the parent
1174
- // fiber — it has to outlive the switch, or the section that turns the plugin
1175
- // back on would vanish with it.
1176
- const REGISTRATION_KEYS = [
1177
- "enabled",
1178
- "bridgeTextOnly",
1179
- "bridgeExportDir",
1180
- "bridgeAutoImage",
1181
- "bridgePreview",
1182
- ];
1183
- let featureFiber = null;
1184
- let lastRegistrationKey = null;
1185
-
1186
- function registrationKey(cfg) {
1187
- return REGISTRATION_KEYS.map((key) => String(cfg[key])).join("|");
1188
- }
1189
-
1190
- function installFeatures() {
1191
- if (featureFiber !== null) return;
1192
- featureFiber = ctx.plugin({
1193
- name: "dsh-tool-vision:features",
1194
- apply(inner) {
1195
- // ── image bridge: pasted images become inspect_image hints on text-only models ──
1196
- if (getConfig().bridgeTextOnly) {
1197
- const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
1198
- mkdir(exportDir, { recursive: true }).catch(() => {});
1199
- // Root-level listener: agent-scoped waterfalls admit untagged listeners,
1200
- // so one registration serves every agent (new and resumed alike) and the
1201
- // agent is read from the fused payload.
1202
- attachPreStepBridge(inner, getConfig, exportDir);
1203
-
1204
- // ── automatic image admission: let pasted images through on text-only models ──
1205
- // The host gate refuses images unless the model declares `image` input;
1206
- // the bridge handles them anyway, so report image support for all models.
1207
- // The wrap is installed on the shared llm service instance (idempotent,
1208
- // restored on dispose/HMR).
1209
- if (getConfig().bridgeAutoImage) {
1210
- const unwrap = installAutoImageAdmission(inner.get("llm"), inner.logger);
1211
- inner.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
1212
- }
1213
-
1214
- // ── dispatch image admission: make the capability verdict actually land ──
1215
- // `resolveModelInfo` above is pure admission — it never changes what
1216
- // the adapter streams. The image the plugin decided to hand over
1217
- // directly was still being projected into a text placeholder by
1218
- // `LlmService.generate`, which reads the adapter's own declaration and
1219
- // nothing else. This applies the same decision at that seam.
1220
- //
1221
- // Installed outside the `bridgeAutoImage` branch on purpose: the
1222
- // question it answers is not "may the image be offered" but "does the
1223
- // model receive it", and that is the plugin's own verdict either way.
1224
- {
1225
- const unwrapDispatch = installDispatchImageAdmission(inner.get("llm"), getConfig, inner.logger);
1226
- inner.effect(() => unwrapDispatch, "dsh-tool-vision: dispatch image admission");
1227
- }
1228
-
1229
- // ── bridge image preview: same-origin thumbnails for bridged images ──
1230
- if (getConfig().bridgePreview) {
1231
- registerBridgePreviewRoute(inner, exportDir, inner.logger);
1232
- }
1233
- }
1234
-
1235
- inner.tools.register(defineTool({
1236
- name: "inspect_image",
1237
- description: getConfig().description,
1238
- parameters: {
1239
- path: {
1240
- type: "string",
1241
- required: true,
1242
- description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
1243
- },
1244
- question: {
1245
- type: "string",
1246
- description: "Optional specific question about the image. Omit for a general detailed description.",
1247
- },
1248
- detail: {
1249
- type: "string",
1250
- enum: ["auto", "low", "high"],
1251
- description: "Optional image resolution hint for the vision API (auto by default).",
1252
- },
1253
- },
1254
- output: {
1255
- schema: { type: "string" },
1256
- render: (_args, value) => [{ type: "text", text: value }],
1257
- },
1258
- async execute(args, exec) {
1259
- const cfg = getConfig();
1260
- const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
1261
- const { url, note } = await toImageUrl(args.path, cwd, cfg);
1262
- try {
1263
- const answer = await callVision(cfg, url, args.question, args.detail, exec.signal, exec);
1264
- return note === url ? answer : `${answer}\n\n(image: ${note})`;
1265
- } catch (error) {
1266
- const raw = error && error.message ? String(error.message) : String(error);
1267
- if (CONTENT_FILTER_RE.test(raw)) {
1268
- throw new Error(
1269
- "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
1270
- "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
1271
- );
1272
- }
1273
- throw error;
1274
- }
1275
- },
1276
- }));
1277
-
1278
- // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
1279
- // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
1280
- // (baseURL/apiKey/model). No provider chain, no local models, no extra
1281
- // settings: everything comes from the existing tool-vision configuration.
1282
- registerVisionTools(inner, getConfig);
1283
-
1284
- // ── ground-truth capability probe (v0.9.0) ───────────────────────────
1285
- // Everything else decides from what a route SAYS; this measures what it
1286
- // DOES, and records the answer so the bridge decision stops guessing.
1287
- inner.tools.register(defineTool({
1288
- name: "vision_probe_model",
1289
- description:
1290
- "Measure whether a model's endpoint can actually read images, by sending it a real " +
1291
- "solid-color image and checking the answer. Use this when a model's declared image " +
1292
- "support is doubtful (it claims `image` but the provider may reject it, or it declares " +
1293
- "text-only yet the endpoint may accept images). Runs a text-only control request plus " +
1294
- "two different colors, so a model that merely guesses cannot pass. The verdict is " +
1295
- "recorded and from then on decides whether that route's images are sent directly or " +
1296
- "bridged. Costs a few requests against the model's own endpoint.",
1297
- parameters: {
1298
- model: {
1299
- type: "string",
1300
- required: true,
1301
- description: "Model id to probe, e.g. xiaomi/mimo-v2.5. Use the id exactly as dsh has it configured.",
1302
- },
1303
- provider: {
1304
- type: "string",
1305
- description:
1306
- "Provider route serving the model (e.g. commandcode). Omit to use the current " +
1307
- "session's route, or to auto-select when exactly one provider offers the model.",
1308
- },
1309
- },
1310
- output: {
1311
- schema: { type: "string" },
1312
- render: (_args, value) => [{ type: "text", text: value }],
1313
- },
1314
- async execute(args, exec) {
1315
- const cfg = getConfig();
1316
- const requested = String(args.model ?? "").trim();
1317
- if (requested.length === 0) throw new Error("vision_probe_model: model is required");
1318
- const llm = inner.get("llm");
1319
-
1320
- // Resolve which routes offer this model, so an omitted `provider`
1321
- // is still an exact choice rather than a guess.
1322
- let candidates = [];
1323
- try {
1324
- for (const entry of llm?.listProviders?.() ?? []) {
1325
- const id = entry?.id;
1326
- if (typeof id !== "string") continue;
1327
- const models = await llm.listModels(id).catch(() => []);
1328
- if ((models ?? []).some((model) => String(model?.id) === requested)) candidates.push(id);
1329
- }
1330
- } catch {
1331
- /* fall through to the route/argument fallbacks below */
1332
- }
1333
- let provider = typeof args.provider === "string" && args.provider.length > 0 ? args.provider : undefined;
1334
- if (provider === undefined) {
1335
- const here = currentRoute(exec?.agent);
1336
- if (candidates.length === 1) provider = candidates[0];
1337
- else if (here.model === requested && typeof here.provider === "string") provider = here.provider;
1338
- else if (candidates.length > 1) {
1339
- throw new Error(
1340
- `vision_probe_model: "${requested}" is served by ${candidates.join(", ")}; pass provider explicitly`,
1341
- );
1342
- } else {
1343
- provider = here.provider;
1344
- }
1345
- }
1346
- if (typeof provider !== "string" || provider.length === 0) {
1347
- throw new Error(
1348
- `vision_probe_model: cannot tell which provider serves "${requested}"; pass provider explicitly`,
1349
- );
1350
- }
1351
-
1352
- const target = await resolveProbeTarget(inner, provider, requested);
1353
- const outcome = await probeModelCapability(
1354
- {
1355
- baseURL: target.baseURL,
1356
- apiKey: target.apiKey,
1357
- api: target.api,
1358
- model: requested,
1359
- maxTokens: PROBE_MAX_TOKENS,
1360
- timeoutMs: Math.max(PROBE_TIMEOUT_MS, Number(cfg.timeoutMs) || 0),
1361
- headers: sessionHeaders(cfg, exec),
1362
- },
1363
- exec.signal,
1364
- );
1365
-
1366
- const key = probeKey(provider, requested);
1367
- let recorded = "not recorded (no settings provider)";
1368
- if (outcome.support === "yes" || outcome.support === "no") {
1369
- const scope = getSettingsScope();
1370
- if (scope !== null) {
1371
- try {
1372
- await scope.update({ probeResults: { ...(cfg.probeResults ?? {}), [key]: outcome.support } });
1373
- recorded = `recorded as "${outcome.support}"`;
1374
- } catch (error) {
1375
- recorded = `could not be recorded: ${String(error?.message ?? error)}`;
1376
- }
1377
- }
1378
- }
1379
-
1380
- const verdictText = outcome.support === "yes"
1381
- ? "SUPPORTS images"
1382
- : outcome.support === "no"
1383
- ? "does NOT read images"
1384
- : "INCONCLUSIVE (the route could not be probed)";
1385
- const effect = outcome.support === "yes"
1386
- ? "images will be sent to it directly"
1387
- : outcome.support === "no"
1388
- ? "images will be bridged to inspect_image hints"
1389
- : "the existing list/declaration rules still apply";
1390
- const evidence = Object.entries(outcome.evidence ?? {})
1391
- .map(([name, value]) => ` ${name}: ${value}`)
1392
- .join("\n");
1393
- return [
1394
- `${provider}/${requested}: ${verdictText}`,
1395
- ` ${outcome.detail}`,
1396
- ` ${recorded}; ${effect}`,
1397
- evidence ? ` evidence:\n${evidence}` : "",
1398
- ].filter(Boolean).join("\n");
1399
- },
1400
- }));
1401
- },
1402
- });
1403
- }
1404
-
1405
- function uninstallFeatures() {
1406
- if (featureFiber === null) return;
1407
- const fiber = featureFiber;
1408
- featureFiber = null;
1409
- // dispose() settles asynchronously and must never take the plugin down; the
1410
- // next install starts a fresh fiber regardless of how this one ends.
1411
- Promise.resolve(fiber.dispose()).catch((error) => {
1412
- ctx.logger?.warn?.(`[tool-vision] feature teardown failed: ${String(error)}`);
1413
- });
1414
- }
1415
-
1416
- // Re-install only when a field that gates a registration actually changed, so
1417
- // editing, say, `model` does not tear the tools down and back up mid-session.
1418
- function syncFeatures() {
1419
- const cfg = getConfig();
1420
- const key = registrationKey(cfg);
1421
- if (featureFiber !== null && key === lastRegistrationKey) return;
1422
- uninstallFeatures();
1423
- lastRegistrationKey = key;
1424
- if (cfg.enabled) installFeatures();
1425
- }
1426
-
1427
- // Compat shim: dsh-settings 0.1.2-rc.1 removed the module-level
1428
- // `installSettingsSection` export (the provider now lives at ctx.settings).
1429
- // Inline the same logic via ctx.inject(["settings"]) — works on both
1430
- // 0.1.1 (module export wrapper) and 0.1.2 (ctx.settings) hosts.
1431
- ctx.inject(["settings"], (sctx) => {
1432
- const scope = sctx.settings.register(NS, Config, { base: config });
1433
- sourceGetter = () => scope.get();
1434
- settingsScope = scope;
1435
- sctx.effect(() => () => {
1436
- sourceGetter = null;
1437
- settingsScope = null;
1438
- });
1439
- // Hot-apply: `enabled` switches the child fiber, and any field that gates a
1440
- // registration re-installs it, so none of them needs a dsh restart anymore.
1441
- scope.watch(() => syncFeatures());
1442
- sctx.effect(() => () => uninstallFeatures());
1443
- // The stored value governs from here on; the composition entry was only a
1444
- // placeholder until the provider answered.
1445
- syncFeatures();
1446
- });
1447
-
1448
- // ── model catalog route: settings support, so it lives on THIS fiber ───────
1449
- // The master switch disposes the feature fiber on purpose (tools, bridge,
1450
- // preview route), while the settings section itself deliberately stays
1451
- // mounted — and a section whose list field cannot autocomplete is a section
1452
- // you cannot configure before switching the plugin on. Registered here so it
1453
- // outlives `enabled: false`; read-only and secret-free by construction.
1454
- registerModelCatalogRoute(ctx, ctx.logger, getConfig);
1455
-
1456
- // No settings provider: the composition entry is the whole configuration.
1457
- syncFeatures();
1458
- }
1459
-
1460
- export {
1461
- BRIDGE_MARKER,
1462
- BRIDGE_PREVIEW_MAX_BYTES,
1463
- BRIDGE_PREVIEW_MEDIA,
1464
- BRIDGE_PREVIEW_ROUTE,
1465
- Config,
1466
- DEFAULT_DESCRIPTION,
1467
- DEFAULT_MULTIMODAL_LIST_MODE,
1468
- EXT_BY_MEDIA,
1469
- MODEL_CATALOG_MAX_PER_PROVIDER,
1470
- MODEL_CATALOG_ROUTE,
1471
- MULTIMODAL_LIST_MODES,
1472
- apply,
1473
- attachPreStepBridge,
1474
- autoPromotionWarned,
1475
- bridgeMessages,
1476
- currentModelAcceptsImage,
1477
- currentRoute,
1478
- deepFreeze,
1479
- exportImage,
1480
- hasImageBlock,
1481
- inject,
1482
- installAutoImageAdmission,
1483
- installDispatchImageAdmission,
1484
- lastModelDecision,
1485
- modelListMatches,
1486
- modelListMatchEntries,
1487
- name,
1488
- normalizeListMode,
1489
- parseQuery,
1490
- probeKey,
1491
- probeVerdict,
1492
- registerBridgePreviewRoute,
1493
- routeDirectDecision,
1494
- registerModelCatalogRoute,
1495
- repairLoggedImages,
1496
- resolveProbeTarget,
1497
- routeDeclaresImage,
1498
- unwrappedResolveModelInfo,
1499
- warnAutoPromotion,
1500
- };
1
+ /**
2
+ * dsh-tool-vision — external vision model for DeepSeek Harness.
3
+ *
4
+ * Two capabilities:
5
+ *
6
+ * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
+ * any OpenAI-compatible chat/completions endpoint that supports
8
+ * `image_url` content parts, and returns the vision model's text answer.
9
+ *
10
+ * 2. Image bridge — pasted images are bridged to text hints before they
11
+ * enter a text-only model's request:
12
+ *
13
+ * - New images are bridged on the `agent/pre-step` waterfall (the only
14
+ * seam where the harness lets a plugin replace the messages that enter
15
+ * a step — they become the durable `user/message` log, so the
16
+ * `llm/stream` request-reconstruction invariant stays satisfied).
17
+ * - Images already logged before the plugin was installed (or before a
18
+ * server restart) are repaired lazily with a surface `replace`, one
19
+ * event at a time, on the first pre-step of the session.
20
+ *
21
+ * The bridged hint points at an exported local copy of the image, which
22
+ * the agent hands to `inspect_image`. Which models skip the bridge
23
+ * (v0.9.0) is decided by two independent inputs:
24
+ *
25
+ * - `multimodalModels` — a user-owned list whose meaning is set by
26
+ * `multimodalListMode`: `whitelist` (listed models receive image blocks
27
+ * directly; the historical meaning and the default), `blacklist`
28
+ * (listed models are forced through the bridge), or `off` (list
29
+ * ignored). Entries match the full model id, its bare id after the last
30
+ * `/`, or `provider/id`, and may contain `*` / `?` globs.
31
+ * - `autoDetectMultimodal` — when on, the route's own declared
32
+ * `inputModalities` forms the base set that the list then adds to
33
+ * (whitelist) or subtracts from (blacklist). The declaration is always
34
+ * read from the *unwrapped* `resolveModelInfo`, because the admission
35
+ * wrap below rewrites it for every model.
36
+ *
37
+ * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
38
+ * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
39
+ * inline thumbnails for bridged pasted images:
40
+ *
41
+ * - `bridgeMessages` stamps every bridged hint with an invisible marker
42
+ * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
43
+ * precisely instead of pattern-matching free text.
44
+ * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
45
+ * exported images to the same-origin page; the client inserts a
46
+ * thumbnail above the hint text and opens a lightbox on click.
47
+ * - Pure display layer: persisted messages, the transcript, the
48
+ * model-facing text and the `inspect_image` chain are untouched.
49
+ */
50
+ import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
51
+ import { createHash } from "node:crypto";
52
+ import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
53
+ import os from "node:os";
54
+ import z from "@deepseek-ai/schemastery";
55
+ import { defineTool } from "@deepseek-ai/dsh-tools";
56
+ import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
57
+ import { sessionHeaders } from "./lib/session-header.js";
58
+ import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
59
+
60
+ /** Cordis plugin name. */
61
+ const name = "tool-vision";
62
+ /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
63
+ const inject = ["tools", "llm", "attachments", "webServer"];
64
+ /** Settings namespace owned by this plugin (Web UI settings section). */
65
+ const NS = "tool-vision";
66
+
67
+ /**
68
+ * Invisible prefix stamped onto every bridged hint text block. The browser
69
+ * half uses it to recognize bridge text precisely (no free-text regex over
70
+ * user messages), and the model-facing hint stays intact otherwise.
71
+ */
72
+ const BRIDGE_MARKER = "\u200b[bridge]";
73
+
74
+ /** Loopback route serving bridged images to the same-origin page. */
75
+ const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
76
+ /**
77
+ * Loopback JSON route the settings panel reads the configured model catalog
78
+ * from (autocomplete for `multimodalModels` and the current-route readout).
79
+ * Registered on the plugin's own fiber, so it stays up while the master
80
+ * switch is off — a section you cannot configure until you switch it on would
81
+ * be useless.
82
+ */
83
+ const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
84
+ /** Per-provider cap on catalog entries returned to the panel. */
85
+ const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
86
+ /** Hard cap for a single served image (defense in depth; export is bounded). */
87
+ const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
88
+ /** Extensions the preview route serves (svg/ico intentionally excluded). */
89
+ const BRIDGE_PREVIEW_MEDIA = {
90
+ ".png": "image/png",
91
+ ".jpg": "image/jpeg",
92
+ ".jpeg": "image/jpeg",
93
+ ".webp": "image/webp",
94
+ ".gif": "image/gif",
95
+ ".avif": "image/avif",
96
+ ".bmp": "image/bmp",
97
+ };
98
+
99
+ const DEFAULT_DESCRIPTION =
100
+ "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
101
+ "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
102
+ "optionally with a specific question. Returns the vision model's textual description or answer. " +
103
+ "Use this whenever you need to read, describe, or extract information from image content, " +
104
+ "including an image that reached you as a bridged text hint.";
105
+
106
+ /** Runtime schema for the tool-vision row. */
107
+ const Config = z.object({
108
+ /**
109
+ * Master switch, hot-applied. When off the plugin registers nothing at all —
110
+ * no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
111
+ * preview route, no llm capability wrap — while the settings section itself
112
+ * stays mounted so the switch can turn it back on. Turning it off is the way
113
+ * to stop every vision tool from reaching the model without uninstalling the
114
+ * package or restarting dsh.
115
+ */
116
+ enabled: z.boolean().default(true),
117
+ /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
118
+ baseURL: z.string().default("https://api.openai.com/v1"),
119
+ /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
120
+ apiKey: z.string().default("").role("secret"),
121
+ /** Environment variable holding the API key. */
122
+ apiKeyEnv: z.string().default("VISION_API_KEY"),
123
+ /** Vision model id served by the endpoint. */
124
+ model: z.string().default("gpt-4o-mini"),
125
+ /** Max output tokens for the vision call. */
126
+ maxTokens: z.number().default(4096),
127
+ /** Per-request timeout in milliseconds. */
128
+ timeoutMs: z.number().default(60000),
129
+ /** Largest local image accepted, in bytes. */
130
+ maxImageBytes: z.number().default(10 * 1024 * 1024),
131
+ /** Tool description shown to the model; overrides the default. */
132
+ description: z.string().default(DEFAULT_DESCRIPTION),
133
+ /** Bridge pasted images to text hints on models that cannot see images. */
134
+ bridgeTextOnly: z.boolean().default(true),
135
+ /** Export directory for bridged images; empty = system temp. */
136
+ bridgeExportDir: z.string().default(""),
137
+ /**
138
+ * Model ids the list below refers to (v0.9.0). Each entry is matched
139
+ * case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
140
+ * bare id after the last `/` (`mimo-v2.5`), and `provider/id`
141
+ * (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
142
+ * (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
143
+ * route happens to spell it.
144
+ */
145
+ multimodalModels: z.array(z.string()).default([]),
146
+ /**
147
+ * How `multimodalModels` is read (v0.9.0):
148
+ *
149
+ * - `whitelist` (default): a listed model receives image blocks directly
150
+ * and is never bridged — the meaning the list always had, so an
151
+ * existing configuration keeps behaving exactly as before.
152
+ * - `blacklist`: a listed model is forced through the bridge even when its
153
+ * route declares image input. This is the correction layer for a model
154
+ * that claims image support its endpoint does not really have.
155
+ * - `off`: the list is ignored entirely; nothing is forced either way.
156
+ *
157
+ * An unknown value falls back to `whitelist`.
158
+ */
159
+ multimodalListMode: z.string().default("whitelist"),
160
+ /**
161
+ * Let the current route's own declared `inputModalities` drive the bridge
162
+ * decision, so only the models that cannot see images get bridged (v0.9.0,
163
+ * **on by default**). This and `multimodalModels` are one decision, not two:
164
+ * the declaration forms the base set, and the list unions with it
165
+ * (whitelist) or subtracts from it (blacklist).
166
+ *
167
+ * The declaration is only whatever the profile — or the installed catalog —
168
+ * says, and it CAN be wrong: profiles sometimes declare
169
+ * `input: [text, image]` just to pass the host admission gate, and a wrong
170
+ * "yes" sends the image straight to an endpoint that may reject it. The
171
+ * harness's own text-model projection is disabled by `bridgeAutoImage`, so
172
+ * nothing underneath catches that. Two things bound the damage: the first
173
+ * promotion of a route logs a notice naming the escape hatch
174
+ * ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
175
+ * under `blacklist` mode forces the bridge back on. Set this to false to
176
+ * bridge everything the list does not name.
177
+ *
178
+ * The value is always read from the *unwrapped* `resolveModelInfo`, so the
179
+ * admission wrap can never feed its own claim back in as evidence.
180
+ */
181
+ autoDetectMultimodal: z.boolean().default(true),
182
+ /**
183
+ * Measured image capability per route (v0.9.0), keyed `"provider/model"` with
184
+ * the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
185
+ * never by hand — the only entry in this schema that records what a route
186
+ * DOES rather than what it says.
187
+ *
188
+ * It outranks the route's own declaration, because a probe is a real request
189
+ * and a declaration is only a claim. It does NOT outrank `multimodalModels`:
190
+ * that list is explicit human intent, and a person who names a model
191
+ * deserves the last word over an automated measurement.
192
+ */
193
+ probeResults: z.dict(z.string()).default({}),
194
+ /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
195
+ bridgePreview: z.boolean().default(true),
196
+ /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
197
+ bridgePreviewScanIntervalMs: z.number().default(2000),
198
+ /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
199
+ bridgePreviewHideHint: z.boolean().default(true),
200
+ /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
201
+ desktopScreenshot: z.boolean().default(false),
202
+ /**
203
+ * Advertise image input capability for every model while the bridge is on.
204
+ * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
205
+ * pasted images unless the current model declares `image` in its
206
+ * `inputModalities`. The bridge handles those images anyway (they become
207
+ * text hints the agent inspects through `inspect_image`), so this wraps the
208
+ * llm service's `resolveModelInfo` to report image support for text-only
209
+ * models too — users can paste images on any model without hand-editing
210
+ * provider model configs.
211
+ */
212
+ bridgeAutoImage: z.boolean().default(true),
213
+ /** Send a stable per-conversation id header (`x-opencode-session`) on vision
214
+ * requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
215
+ * on every request; requests without the header may error. The value is the
216
+ * current dsh session id when the call runs inside one, else `sessionId`
217
+ * below, else a stable per-process random id. Set false to disable. */
218
+ sendSessionHeader: z.boolean().default(true),
219
+ /** Header name carrying the session id. */
220
+ sessionHeaderName: z.string().default("x-opencode-session"),
221
+ /** Fixed session id override for callers without a dsh session context.
222
+ * Empty = auto (per-process stable id). */
223
+ sessionId: z.string().default(""),
224
+ });
225
+
226
+ const MIME_BY_EXT = {
227
+ ".png": "image/png",
228
+ ".jpg": "image/jpeg",
229
+ ".jpeg": "image/jpeg",
230
+ ".gif": "image/gif",
231
+ ".webp": "image/webp",
232
+ ".bmp": "image/bmp",
233
+ ".avif": "image/avif",
234
+ ".svg": "image/svg+xml",
235
+ ".ico": "image/x-icon",
236
+ };
237
+
238
+ const EXT_BY_MEDIA = {
239
+ "image/png": ".png",
240
+ "image/jpeg": ".jpg",
241
+ "image/webp": ".webp",
242
+ "image/gif": ".gif",
243
+ };
244
+
245
+ /** True when any message carries an image content block. */
246
+ function hasImageBlock(messages) {
247
+ return (messages ?? []).some((m) =>
248
+ Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
249
+ );
250
+ }
251
+
252
+ /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
253
+ function deepFreeze(value) {
254
+ if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
255
+ if (Array.isArray(value)) {
256
+ for (const item of value) deepFreeze(item);
257
+ return Object.freeze(value);
258
+ }
259
+ for (const key of Object.keys(value)) deepFreeze(value[key]);
260
+ return Object.freeze(value);
261
+ }
262
+
263
+ /** Export one attachment to disk; returns the file path (cached per process). */
264
+ const exportedPaths = new Map();
265
+ async function exportImage(attachment, ctx, dir) {
266
+ const cacheKey = `${resolvePath(dir)}\0${attachment.attachmentId}`;
267
+ const cached = exportedPaths.get(cacheKey);
268
+ if (cached) return cached;
269
+ const { data } = await ctx.attachments.readImage(attachment);
270
+ const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
271
+ const safeName = attachment.name
272
+ ? attachment.name
273
+ .replace(/\.[^.]+$/, "")
274
+ .replace(/[^\w\-]+/g, "_")
275
+ .replace(/^_+|_+$/g, "")
276
+ .slice(0, 40)
277
+ : "";
278
+ // Attachment IDs such as `sha256:...` contain a colon. On Windows that
279
+ // addresses an alternate data stream and leaves a zero-byte visible file.
280
+ // Hash the complete ID to keep the filename portable and collision-resistant.
281
+ const idTag = createHash("sha256").update(attachment.attachmentId).digest("hex").slice(0, 24);
282
+ const base = (safeName ? `${safeName}_` : "") + idTag;
283
+ const path = join(dir, `${base}${ext}`);
284
+ await mkdir(dir, { recursive: true });
285
+ await writeFile(path, data);
286
+ exportedPaths.set(cacheKey, path);
287
+ return path;
288
+ }
289
+
290
+ /**
291
+ * Replace image content blocks with text hints pointing at exported files.
292
+ * Non-image messages are returned as-is (same reference); bridged messages
293
+ * are fresh, deep-frozen objects with the original identity and source.
294
+ * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
295
+ * prefix so the browser half can recognize it precisely.
296
+ * Exported for unit testing; `ctx` only needs `attachments`.
297
+ */
298
+ async function bridgeMessages(messages, ctx, dir) {
299
+ const next = [];
300
+ for (const message of messages) {
301
+ const content = message?.content;
302
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
303
+ next.push(message);
304
+ continue;
305
+ }
306
+ const blocks = [];
307
+ for (const block of content) {
308
+ if (block?.type !== "image") {
309
+ blocks.push(block);
310
+ continue;
311
+ }
312
+ const path = await exportImage(block.attachment, ctx, dir);
313
+ const name = block.attachment.name ? ` (${block.attachment.name})` : "";
314
+ blocks.push({
315
+ type: "text",
316
+ text:
317
+ `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
318
+ `Inspect it with the inspect_image tool to see its content.]`,
319
+ });
320
+ }
321
+ next.push(deepFreeze({ ...message, content: blocks }));
322
+ }
323
+ return next;
324
+ }
325
+
326
+ /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
327
+ function parseQuery(rawUrl) {
328
+ const query = {};
329
+ const at = rawUrl.indexOf("?");
330
+ if (at === -1) return query;
331
+ for (const pair of rawUrl.slice(at + 1).split("&")) {
332
+ const eq = pair.indexOf("=");
333
+ if (eq === -1) continue;
334
+ try {
335
+ query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
336
+ } catch {
337
+ /* skip malformed pairs */
338
+ }
339
+ }
340
+ return query;
341
+ }
342
+
343
+ /**
344
+ * Advertise `image` in the `inputModalities` reported by the llm service for
345
+ * every model, so the host admission gate lets pasted images through on
346
+ * text-only models. The bridge turns those images into text hints anyway, so
347
+ * this is pure admission: it never changes what the adapter actually streams
348
+ * (the adapter's own stream validation reads the model's real `input` from
349
+ * the provider config, untouched here).
350
+ *
351
+ * The wrap is installed on the shared llm service instance, so it must be
352
+ * idempotent across HMR re-applies and restored on dispose. `marker` is a
353
+ * per-instance record proving this plugin already owns the wrap; the
354
+ * `installed` flag distinguishes "we wrapped it" from "the original was
355
+ * replaced by something else" so dispose only restores what we replaced.
356
+ *
357
+ * Exported for unit testing; the test passes a fake llm service.
358
+ */
359
+ const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
360
+ function installAutoImageAdmission(llm, logger) {
361
+ if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
362
+ logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
363
+ return () => {};
364
+ }
365
+ if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
366
+ const original = llm.resolveModelInfo.bind(llm);
367
+ const wrapped = async (provider, model, signal) => {
368
+ const info = await original(provider, model, signal);
369
+ if (!info) return info;
370
+ const mods = info.inputModalities;
371
+ if (Array.isArray(mods) && mods.includes("image")) return info;
372
+ return { ...info, inputModalities: [...(mods ?? []), "image"] };
373
+ };
374
+ let installed = false;
375
+ // The marker carries BOTH functions: the wrapped one proves ownership for
376
+ // dispose, and `original` is the only honest source of a route's real
377
+ // modalities once the wrap is in place (`unwrappedResolveModelInfo`).
378
+ llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
379
+ llm.resolveModelInfo = wrapped;
380
+ installed = true;
381
+ logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
382
+ return () => {
383
+ if (installed) {
384
+ if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
385
+ installed = false;
386
+ }
387
+ delete llm[LLM_RESOLVE_WRAP_MARK];
388
+ };
389
+ }
390
+
391
+ /**
392
+ * Admission is not dispatch, and only dispatch decides whether the pixels arrive.
393
+ *
394
+ * {@link installAutoImageAdmission} wraps `llm.resolveModelInfo`. That method
395
+ * governs who may *offer* an image — the host gate that refuses a pasted image,
396
+ * the model list, the settings readout — and it is deliberately "pure admission:
397
+ * it never changes what the adapter actually streams".
398
+ *
399
+ * What the adapter actually streams is decided one layer down, by
400
+ * `LlmService.generate`:
401
+ *
402
+ * const adapterCall = await adapter.prepareCall(provider, model, signal);
403
+ * modelInfo = this.normalizeModelInfo(registration, model, adapterCall.model);
404
+ * if (modelInfo.inputModalities !== undefined
405
+ * && !modelInfo.inputModalities.includes("image")
406
+ * && projectedMessages.some((message) => contentHasImage(message.content)))
407
+ * projectedMessages = projectImagesForTextModel(projectedMessages);
408
+ *
409
+ * When that list lacks `image`, every image block is rewritten into a text
410
+ * placeholder (`textOnlyImageText`) *before the adapter is called*. So on a
411
+ * route the plugin had already measured as image-capable, and whose bridge
412
+ * decision was therefore "direct, do not intercept", the image was still
413
+ * destroyed — by a check that reads the adapter's declaration and nothing else.
414
+ * `probeResults`, `multimodalModels` and `autoDetectMultimodal` were all
415
+ * invisible to it.
416
+ *
417
+ * This wraps `llm.registration` — the accessor the core itself calls — so every
418
+ * adapter, including one registered later, answers `prepareCall` through
419
+ * {@link routeDirectDecision}: the same rule the bridge uses, so the two can
420
+ * never disagree about a route.
421
+ *
422
+ * When the decision is "bridge", nothing changes: the bridge already turned the
423
+ * image into an `inspect_image` hint, and the core's projection stays as the
424
+ * correct fallback for any image that still reaches dispatch.
425
+ *
426
+ * Idempotent across HMR re-applies, and dispose restores every adapter it
427
+ * touched plus the accessor itself.
428
+ */
429
+ const LLM_REGISTRATION_WRAP_MARK = Symbol("dsh-tool-vision.registration.wrapped");
430
+ function installDispatchImageAdmission(llm, getConfig, logger) {
431
+ if (llm === undefined || llm === null || typeof llm.registration !== "function") {
432
+ logger?.warn?.("[tool-vision] llm service unavailable; dispatch image admission not installed");
433
+ return () => {};
434
+ }
435
+ if (llm[LLM_REGISTRATION_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
436
+ const original = llm.registration.bind(llm);
437
+ /** adapter -> { prepareCall, modelOf } original methods, for dispose. */
438
+ const patched = new Map();
439
+ /** Set of arrays where we pushed "image", for cleanup on dispose. */
440
+ const modifiedInputs = new Set();
441
+ /** Keyed routes (probeKey) known to have direct dispatch permission. */
442
+ const directRouteCache = new Set();
443
+
444
+ const wrapped = (provider) => {
445
+ const registration = original(provider);
446
+ const adapter = registration?.adapter;
447
+ if (adapter !== undefined && adapter !== null
448
+ && !patched.has(adapter) && typeof adapter.prepareCall === "function") {
449
+ const rawPrepare = adapter.prepareCall;
450
+ const rawModelOf = typeof adapter.modelOf === "function" ? adapter.modelOf : undefined;
451
+ const prepare = rawPrepare.bind(adapter);
452
+
453
+ if (rawModelOf !== undefined) {
454
+ adapter.modelOf = function (snapshot, route, model) {
455
+ const resolved = rawModelOf.call(this, snapshot, route, model);
456
+ if (resolved !== undefined && resolved !== null
457
+ && Array.isArray(resolved.input) && !resolved.input.includes("image")
458
+ && directRouteCache.has(probeKey(route, model))) {
459
+ return { ...resolved, input: [...resolved.input, "image"] };
460
+ }
461
+ return resolved;
462
+ };
463
+ }
464
+
465
+ adapter.prepareCall = async (route, model, signal) => {
466
+ const call = await prepare(route, model, signal);
467
+ if (call === undefined || call === null || call.model === undefined || call.model === null) return call;
468
+ const mods = call.model.inputModalities;
469
+ if (Array.isArray(mods) && mods.includes("image")) return call;
470
+ let direct = false;
471
+ try {
472
+ ({ direct } = await routeDirectDecision(route, model, getConfig(), llm));
473
+ } catch {
474
+ // A decision that cannot be made is not a licence to send pixels a
475
+ // route may reject: leave the core's projection in place.
476
+ return call;
477
+ }
478
+ if (!direct) {
479
+ directRouteCache.delete(probeKey(route, model));
480
+ return call;
481
+ }
482
+ directRouteCache.add(probeKey(route, model));
483
+
484
+ // In dsh-llm-pi-ai, adapter.streamWithSnapshot checks:
485
+ // if (containsImage && !model.input.includes("image")) throw LlmError(...)
486
+ // where model is obtained via this.modelOf(snapshot, provider, model) -> snapshot.models.getModel(route, model).
487
+ // Ensure that the adapter snapshot's model also admits images so the adapter-level guard passes.
488
+ try {
489
+ const snapshot = typeof adapter.current === "function" ? adapter.current() : undefined;
490
+ const targetModel = snapshot?.models?.getModel?.(route, model);
491
+ if (targetModel !== undefined && targetModel !== null
492
+ && Array.isArray(targetModel.input) && !targetModel.input.includes("image")) {
493
+ targetModel.input.push("image");
494
+ modifiedInputs.add(targetModel.input);
495
+ }
496
+ } catch {
497
+ /* best-effort */
498
+ }
499
+
500
+ return { ...call, model: { ...call.model, inputModalities: [...(mods ?? []), "image"] } };
501
+ };
502
+ patched.set(adapter, { prepareCall: rawPrepare, modelOf: rawModelOf });
503
+ }
504
+ return registration;
505
+ };
506
+ llm[LLM_REGISTRATION_WRAP_MARK] = { original, wrapped };
507
+ llm.registration = wrapped;
508
+ logger?.debug?.("[tool-vision] dispatch image admission installed (llm.registration wrapped)");
509
+ return () => {
510
+ for (const [adapter, originalMethods] of patched) {
511
+ if (typeof originalMethods.prepareCall === "function") adapter.prepareCall = originalMethods.prepareCall;
512
+ if (typeof originalMethods.modelOf === "function") adapter.modelOf = originalMethods.modelOf;
513
+ }
514
+ patched.clear();
515
+ for (const arr of modifiedInputs) {
516
+ const idx = arr.indexOf("image");
517
+ if (idx !== -1) arr.splice(idx, 1);
518
+ }
519
+ modifiedInputs.clear();
520
+ directRouteCache.clear();
521
+ if (llm.registration === wrapped) llm.registration = original;
522
+ delete llm[LLM_REGISTRATION_WRAP_MARK];
523
+ };
524
+ }
525
+
526
+ /**
527
+ * Register the loopback route that serves bridged images to the same-origin
528
+ * page (the preview thumbnails). Read-only and tightly scoped:
529
+ * - only files inside the bridge export directory (no traversal);
530
+ * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
531
+ * - Host restricted to the local machine;
532
+ * - hard 20MB cap per file.
533
+ */
534
+ function registerBridgePreviewRoute(ctx, exportDir, logger) {
535
+ const webServer = ctx.get("webServer");
536
+ if (webServer === undefined) {
537
+ logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
538
+ return;
539
+ }
540
+ const bridgeDir = resolvePath(exportDir);
541
+ ctx.effect(() => webServer.register({
542
+ kind: "exact",
543
+ path: BRIDGE_PREVIEW_ROUTE,
544
+ async handler(req, res) {
545
+ try {
546
+ const raw = String(req.url ?? "");
547
+ const query = parseQuery(raw);
548
+ const p = query.p;
549
+ if (typeof p !== "string" || p.length === 0) {
550
+ res.writeHead(400);
551
+ res.end("bad request");
552
+ return;
553
+ }
554
+ const lower = p.toLowerCase();
555
+ const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
556
+ if (mediaType === undefined) {
557
+ res.writeHead(400);
558
+ res.end("not an image path");
559
+ return;
560
+ }
561
+ const host = String(req.headers?.host ?? "");
562
+ if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
563
+ res.writeHead(403);
564
+ res.end("forbidden");
565
+ return;
566
+ }
567
+ const target = resolvePath(p);
568
+ if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
569
+ res.writeHead(403);
570
+ res.end("forbidden");
571
+ return;
572
+ }
573
+ const info = await stat(target);
574
+ if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
575
+ res.writeHead(404);
576
+ res.end("not found");
577
+ return;
578
+ }
579
+ const bytes = await readFile(target);
580
+ res.writeHead(200, {
581
+ "Content-Type": mediaType,
582
+ "Cache-Control": "private, max-age=60",
583
+ });
584
+ res.end(bytes);
585
+ } catch {
586
+ try {
587
+ res.writeHead(404);
588
+ res.end("not found");
589
+ } catch {
590
+ /* response already sent */
591
+ }
592
+ }
593
+ },
594
+ }), "dsh-tool-vision: bridge preview route");
595
+ }
596
+
597
+ /**
598
+ * Register the loopback JSON route the settings panel reads:
599
+ *
600
+ * { current: {provider, model, direct, source, mode}, providers: [...] }
601
+ *
602
+ * `providers` comes from `llm.listProviders()` + `llm.listModels(provider)` —
603
+ * the routes the harness actually has configured, which is exactly the set a
604
+ * `multimodalModels` entry may name. `listModels` is a different seam from
605
+ * `resolveModelInfo`, so these declared modalities are the profile's own and
606
+ * are NOT contaminated by the admission wrap (a model the wrap advertises as
607
+ * image-capable still reports its real declaration here).
608
+ *
609
+ * Same confinement as the image route: loopback Host only, read-only,
610
+ * `no-store`, and no credentials or endpoints in the payload — provider ids,
611
+ * model ids/names and one boolean each.
612
+ */
613
+ function registerModelCatalogRoute(ctx, logger, getConfig) {
614
+ const webServer = ctx.get("webServer");
615
+ if (webServer === undefined) {
616
+ logger?.warn?.("[tool-vision] webServer unavailable; model catalog route not registered");
617
+ return;
618
+ }
619
+ ctx.effect(() => webServer.register({
620
+ kind: "exact",
621
+ path: MODEL_CATALOG_ROUTE,
622
+ async handler(req, res) {
623
+ const send = (status, payload) => {
624
+ res.writeHead(status, {
625
+ "Content-Type": "application/json; charset=utf-8",
626
+ "Cache-Control": "no-store",
627
+ });
628
+ res.end(JSON.stringify(payload));
629
+ };
630
+ const host = String(req.headers?.host ?? "");
631
+ if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
632
+ send(403, { error: "forbidden" });
633
+ return;
634
+ }
635
+ const llm = ctx.get("llm");
636
+ if (llm === undefined) {
637
+ send(503, { error: "llm service unavailable" });
638
+ return;
639
+ }
640
+ // `listed` is computed HERE, with the same matcher the bridge itself
641
+ // uses, so the panel never reimplements the rule (and can never
642
+ // disagree with the decision it is describing).
643
+ const cfg = typeof getConfig === "function" ? getConfig() : undefined;
644
+ const list = Array.isArray(cfg?.multimodalModels) ? cfg.multimodalModels : [];
645
+ const listMode = normalizeListMode(cfg?.multimodalListMode);
646
+ let routes = [];
647
+ try {
648
+ routes = llm.listProviders() ?? [];
649
+ } catch (error) {
650
+ logger?.debug?.(`[tool-vision] listProviders failed: ${String(error)}`);
651
+ }
652
+ const providers = [];
653
+ for (const entry of routes) {
654
+ const id = entry?.id;
655
+ if (typeof id !== "string" || id.length === 0) continue;
656
+ const record = {
657
+ id,
658
+ name: typeof entry?.name === "string" && entry.name.length > 0 ? entry.name : id,
659
+ models: [],
660
+ };
661
+ try {
662
+ const models = await llm.listModels(id);
663
+ record.models = (models ?? []).slice(0, MODEL_CATALOG_MAX_PER_PROVIDER).map((model) => {
664
+ const modelId = String(model.id);
665
+ const matchedEntries = modelListMatchEntries(list, id, modelId);
666
+ return {
667
+ id: modelId,
668
+ name: typeof model.name === "string" && model.name.length > 0 ? model.name : modelId,
669
+ image: Array.isArray(model.inputModalities) && model.inputModalities.includes("image"),
670
+ listed: matchedEntries.length > 0,
671
+ matchedEntries,
672
+ // What the route was MEASURED doing, when anyone has probed it.
673
+ // `null` means "not probed", which the panel must render
674
+ // differently from a probe that came back negative.
675
+ probe: probeVerdict(cfg, id, modelId) ?? null,
676
+ };
677
+ });
678
+ } catch (error) {
679
+ // One broken route must never blank the whole panel: report it and
680
+ // keep going, so the other providers still list.
681
+ record.error = String(error?.message ?? error);
682
+ }
683
+ providers.push(record);
684
+ }
685
+ send(200, {
686
+ current: { ...lastModelDecision },
687
+ list,
688
+ listMode,
689
+ probeResults: typeof getConfig === "function" ? { ...(getConfig()?.probeResults ?? {}) } : {},
690
+ providers,
691
+ });
692
+ },
693
+ }), "dsh-tool-vision: model catalog route");
694
+ }
695
+
696
+ /** Accepted `multimodalListMode` values; anything else falls back to whitelist. */
697
+ const MULTIMODAL_LIST_MODES = ["off", "whitelist", "blacklist"];
698
+ const DEFAULT_MULTIMODAL_LIST_MODE = "whitelist";
699
+
700
+ /**
701
+ * Normalize a configured list mode. The settings section only ever writes one
702
+ * of {@link MULTIMODAL_LIST_MODES}, but the value can also arrive from a
703
+ * hand-edited `settings.yaml`, so an unknown string is clamped rather than
704
+ * treated as "no mode" (which would silently bridge everything).
705
+ */
706
+ function normalizeListMode(value) {
707
+ return MULTIMODAL_LIST_MODES.includes(value) ? value : DEFAULT_MULTIMODAL_LIST_MODE;
708
+ }
709
+
710
+ const listEntryCache = new Map();
711
+ /**
712
+ * Compile one list entry into an anchored, case-insensitive regexp. A literal
713
+ * entry is just an exact match; `*` and `?` are the only metacharacters, so a
714
+ * model id containing `.` or `+` still matches itself.
715
+ */
716
+ function listEntryRegExp(entry) {
717
+ const cached = listEntryCache.get(entry);
718
+ if (cached !== undefined) return cached;
719
+ const source = entry
720
+ .replace(/[.+^${}()|[\]\\]/g, "\\$&")
721
+ .replace(/\*/g, ".*")
722
+ .replace(/\?/g, ".");
723
+ const compiled = new RegExp(`^${source}$`, "i");
724
+ listEntryCache.set(entry, compiled);
725
+ return compiled;
726
+ }
727
+
728
+ /**
729
+ * Whether any entry in `patterns` names this route. Each entry is tested
730
+ * against the full model id, its bare id after the last `/`, and
731
+ * `provider/id` — so `mimo-v2.5`, `xiaomi/mimo-v2.5` and
732
+ * `commandcode/xiaomi/mimo-v2.5` all address the same route, and `*vl*`
733
+ * addresses every variant of one family. Exported for unit testing.
734
+ */
735
+ function modelListMatches(patterns, provider, model) {
736
+ return modelListMatchEntries(patterns, provider, model).length > 0;
737
+ }
738
+
739
+ /**
740
+ * The list entries that address this route. The panel needs *which* entry hit
741
+ * so that unchecking a model removes exactly the entry responsible — matching
742
+ * `mimo-v2.5` against `xiaomi/mimo-v2.5` must not leave an orphan behind, and
743
+ * must not delete an unrelated entry either. Exported for unit testing.
744
+ */
745
+ function modelListMatchEntries(patterns, provider, model) {
746
+ if (!Array.isArray(patterns) || patterns.length === 0) return [];
747
+ if (typeof model !== "string" || model.length === 0) return [];
748
+ const slash = model.lastIndexOf("/");
749
+ const candidates = [model, slash === -1 ? model : model.slice(slash + 1)];
750
+ if (typeof provider === "string" && provider.length > 0) candidates.push(`${provider}/${model}`);
751
+ const hits = [];
752
+ for (const raw of patterns) {
753
+ if (raw === undefined || raw === null) continue;
754
+ const entry = String(raw).trim();
755
+ if (entry.length === 0) continue;
756
+ const re = listEntryRegExp(entry);
757
+ if (candidates.some((candidate) => re.test(candidate))) hits.push(String(raw));
758
+ }
759
+ return hits;
760
+ }
761
+
762
+ /**
763
+ * The route the session is on: the logged request header first, then the
764
+ * agent's own options. `provider` is optional (older logs may lack it).
765
+ */
766
+ function currentRoute(agent) {
767
+ const header = agent?.session?.requestHeader?.();
768
+ return {
769
+ provider: header?.config?.provider ?? agent?.options?.provider,
770
+ model: header?.config?.model ?? agent?.options?.model,
771
+ };
772
+ }
773
+
774
+ /**
775
+ * The last decision `currentModelAcceptsImage` made, for the settings panel's
776
+ * "current route" readout (served over {@link MODEL_CATALOG_ROUTE}). A plain
777
+ * mutable record — the panel only ever reads it.
778
+ */
779
+ const lastModelDecision = {
780
+ provider: undefined,
781
+ model: undefined,
782
+ direct: false,
783
+ source: "none",
784
+ mode: DEFAULT_MULTIMODAL_LIST_MODE,
785
+ };
786
+
787
+ /**
788
+ * `resolveModelInfo` as it was before {@link installAutoImageAdmission}
789
+ * wrapped it. Auto-detection MUST read through this: the wrap reports image
790
+ * support for *every* model, so reading the wrapped method would make the
791
+ * plugin's own claim the evidence for itself.
792
+ */
793
+ function unwrappedResolveModelInfo(llm) {
794
+ const marker = llm?.[LLM_RESOLVE_WRAP_MARK];
795
+ const fn = marker?.original ?? llm?.resolveModelInfo;
796
+ return typeof fn === "function" ? fn.bind(llm) : undefined;
797
+ }
798
+
799
+ /**
800
+ * Whether the route declares `image` input, read before the admission wrap.
801
+ * `false` and `undefined` are the same answer to the caller ("not known to be
802
+ * multimodal"): a route nobody could resolve is bridged, never guessed.
803
+ */
804
+ async function routeDeclaresImage(llm, provider, model) {
805
+ const resolve = unwrappedResolveModelInfo(llm);
806
+ if (resolve === undefined || typeof provider !== "string" || typeof model !== "string") {
807
+ return false;
808
+ }
809
+ try {
810
+ const info = await resolve(provider, model);
811
+ return info?.inputModalities?.includes("image") === true;
812
+ } catch {
813
+ return false;
814
+ }
815
+ }
816
+
817
+ /** Routes already warned about, so the notice appears once per route. */
818
+ const autoPromotionWarned = new Set();
819
+
820
+ /** Registry key for one measured route. Provider-qualified: the same model id
821
+ * can be served by two gateways with different real capabilities. */
822
+ function probeKey(provider, model) {
823
+ return `${typeof provider === "string" ? provider : ""}/${model}`;
824
+ }
825
+
826
+ /**
827
+ * The measured verdict for a route, or `undefined` when it was never probed.
828
+ * Anything other than the two recorded values (a hand-edited settings.yaml, a
829
+ * future format) reads as "not probed" rather than as a guess.
830
+ */
831
+ function probeVerdict(config, provider, model) {
832
+ const results = config?.probeResults;
833
+ if (results === undefined || results === null || typeof model !== "string" || model.length === 0) {
834
+ return undefined;
835
+ }
836
+ const value = results[probeKey(provider, model)];
837
+ return value === "yes" || value === "no" ? value : undefined;
838
+ }
839
+
840
+ /**
841
+ * Resolve the endpoint and credential a route actually sends to, so a probe
842
+ * talks to the same place the model does instead of a copy of it.
843
+ *
844
+ * The pieces come from two seams that a plugin is allowed to read:
845
+ * - `llm.listConfigurableProviders()` names the settings namespace and the
846
+ * path a provider's profile lives at;
847
+ * - `settings.get(ns)` resolves that profile (`baseURL`, `apiKeyEnv`, `api`);
848
+ * - `credentials.resolve(ref)` turns `apiKeyEnv` into the secret itself
849
+ * (`.value`), the same way `dsh-llm-pi-ai` does it for a real call.
850
+ *
851
+ * Read-only, and the secret never leaves this function's caller: the probe
852
+ * result records a verdict, never an endpoint or a key.
853
+ *
854
+ * @throws {Error} with a caller-facing reason when the route is unprobeable.
855
+ */
856
+ async function resolveProbeTarget(ctx, provider, model) {
857
+ const llm = ctx.get("llm");
858
+ const settings = ctx.get("settings");
859
+ const credentials = ctx.get("credentials");
860
+ if (settings === undefined) {
861
+ throw new Error("the settings service is unavailable, so the route's endpoint cannot be read");
862
+ }
863
+ let entry;
864
+ try {
865
+ entry = (llm?.listConfigurableProviders?.() ?? []).find((candidate) => candidate?.provider === provider);
866
+ } catch {
867
+ /* an adapter that cannot describe its providers simply has no entry */
868
+ }
869
+ const settingsNs = entry?.settingsNs;
870
+ if (typeof settingsNs !== "string" || settingsNs.length === 0) {
871
+ throw new Error(
872
+ `provider "${provider}" does not publish a settings namespace, so its endpoint cannot be resolved for probing`,
873
+ );
874
+ }
875
+ let profile = settings.get(settingsNs);
876
+ const path = Array.isArray(entry.settingsPath) && entry.settingsPath.length > 0 ? entry.settingsPath : [provider];
877
+ for (const segment of path) profile = profile?.[segment];
878
+ if (profile === undefined || profile === null || typeof profile !== "object") {
879
+ throw new Error(`no stored profile for provider "${provider}" under "${settingsNs}"`);
880
+ }
881
+ const baseURL = typeof profile.baseURL === "string" && profile.baseURL.length > 0 ? profile.baseURL : undefined;
882
+ if (baseURL === undefined) {
883
+ throw new Error(`provider "${provider}" has no baseURL, so its endpoint cannot be probed`);
884
+ }
885
+ let apiKey;
886
+ const ref = profile.apiKeyEnv;
887
+ if (typeof ref === "string" && ref.length > 0) {
888
+ try {
889
+ const record = await credentials?.resolve?.(ref);
890
+ apiKey = typeof record?.value === "string" ? record.value : undefined;
891
+ } catch (error) {
892
+ throw new Error(`stored credential "${ref}" could not be read: ${String(error?.message ?? error)}`);
893
+ }
894
+ }
895
+ return { baseURL, apiKey, api: profile.api, model };
896
+ }
897
+
898
+ /**
899
+ * One-time notice when a route receives images directly *only* because of its
900
+ * own declaration (source `auto`). This is the one failure auto-detection
901
+ * cannot rule out: if the declaration is wrong, the endpoint rejects the image
902
+ * after the message is already durable, and the error alone does not say what
903
+ * to change. Naming the escape hatch the first time it happens turns a
904
+ * confusing 400 into a one-line fix — and now names the probe too, which
905
+ * settles the question instead of asking the user to guess.
906
+ * Read {@link lastModelDecision}; never throws.
907
+ */
908
+ function warnAutoPromotion(logger) {
909
+ const { provider, model, direct, source } = lastModelDecision;
910
+ if (direct !== true || source !== "auto" || typeof model !== "string" || model.length === 0) return;
911
+ const key = `${typeof provider === "string" ? provider : ""}\u0000${model}`;
912
+ if (autoPromotionWarned.has(key)) return;
913
+ autoPromotionWarned.add(key);
914
+ logger?.warn?.(
915
+ `[tool-vision] "${model}" receives images directly because its route declares image input, ` +
916
+ `which is unverified; run vision_probe_model to measure it, or list it in multimodalModels ` +
917
+ `under multimodalListMode: blacklist to force the bridge`,
918
+ );
919
+ }
920
+
921
+ /**
922
+ * Whether images may be handed to this route directly, and why.
923
+ *
924
+ * The plugin's single precedence rule. Three inputs, deliberately independent:
925
+ *
926
+ * - the base set — the route's own declared `inputModalities`, and only when
927
+ * `autoDetectMultimodal` is on. It is read through
928
+ * {@link unwrappedResolveModelInfo}, never through the admission wrap.
929
+ * - the measurement — `probeResults`, which outranks the declaration because a
930
+ * probe is a real request and a declaration is only a statement.
931
+ * - the list — `multimodalModels` under `multimodalListMode`. Explicit human
932
+ * intent, so it outranks both: whitelist adds to the base set, blacklist
933
+ * subtracts from it.
934
+ *
935
+ * Two callers need this exact answer and must never disagree:
936
+ * - {@link currentModelAcceptsImage} — the bridge, deciding whether to
937
+ * intercept an image at all;
938
+ * - {@link installDispatchImageAdmission} — the dispatch path, where the core
939
+ * independently decides whether the model may receive the image block.
940
+ *
941
+ * Returns `direct: true` when bridging is disabled (nothing would be bridged
942
+ * anyway).
943
+ */
944
+ async function routeDirectDecision(provider, model, config, llm) {
945
+ const mode = normalizeListMode(config.multimodalListMode);
946
+ if (!config.bridgeTextOnly) return { direct: true, source: "bridge-off", mode };
947
+ if (typeof model !== "string" || model.length === 0) {
948
+ return { direct: false, source: "no-route", mode };
949
+ }
950
+ const base = config.autoDetectMultimodal
951
+ ? await routeDeclaresImage(llm, provider, model)
952
+ : false;
953
+ let direct = base;
954
+ let source = base ? "auto" : "default";
955
+ // A measured verdict outranks the route's own claim: a probe is a real
956
+ // request, a declaration is only a statement. Both directions count, so a
957
+ // route that says "image" and answered without reading the pixels goes back
958
+ // to the bridge without anyone hunting for the right list entry.
959
+ const probed = probeVerdict(config, provider, model);
960
+ if (probed !== undefined) {
961
+ direct = probed === "yes";
962
+ source = probed === "yes" ? "probe-yes" : "probe-no";
963
+ }
964
+ // `off` means the list carries no opinion at all — it is not "empty list
965
+ // under whitelist", it is "no list semantics", so nothing is forced. The
966
+ // human list keeps the last word over both other signals.
967
+ const listed = mode !== "off" && modelListMatches(config.multimodalModels, provider, model);
968
+ if (listed) {
969
+ direct = mode !== "blacklist";
970
+ source = mode === "blacklist" ? "blacklist" : "whitelist";
971
+ }
972
+ return { direct, source, mode };
973
+ }
974
+
975
+ /**
976
+ * Whether the session's current model may receive image blocks directly.
977
+ *
978
+ * {@link routeDirectDecision} keyed by the session's route, with the answer
979
+ * recorded in {@link lastModelDecision} for the settings panel.
980
+ */
981
+ async function currentModelAcceptsImage(agent, config, llm) {
982
+ const { provider, model } = currentRoute(agent);
983
+ const decision = await routeDirectDecision(provider, model, config, llm);
984
+ Object.assign(lastModelDecision, { provider, model, ...decision });
985
+ return decision.direct;
986
+ }
987
+
988
+ /**
989
+ * Lazily bridge image blocks that are already part of the session log
990
+ * (pasted before the plugin was active, or before a restart). Each affected
991
+ * event is rewritten once with a surface `replace`, which swaps the durable
992
+ * derivation (and the transcript) to the text hint. Events that are no
993
+ * longer on the surface (already shadowed) are skipped and remembered.
994
+ * `repaired` tracks per-session state: a `Set` of handled seqs plus a
995
+ * monotonic scan cursor.
996
+ */
997
+ async function repairLoggedImages(ctx, session, exportDir, repaired) {
998
+ const events = session.events;
999
+ for (let index = repaired.cursor; index < events.length; index += 1) {
1000
+ const event = events[index];
1001
+ if (event.type !== "user/message" || repaired.set.has(event.seq)) {
1002
+ repaired.set.add(event.seq);
1003
+ continue;
1004
+ }
1005
+ const content = event.data?.content;
1006
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
1007
+ repaired.set.add(event.seq);
1008
+ continue;
1009
+ }
1010
+ const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
1011
+ try {
1012
+ session.append("user/message", bridged, {
1013
+ surfaceOp: { op: "replace", start: event.seq, end: event.seq },
1014
+ sourceEventSeqs: [event.seq],
1015
+ });
1016
+ ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
1017
+ } catch (error) {
1018
+ ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
1019
+ }
1020
+ repaired.set.add(event.seq);
1021
+ }
1022
+ repaired.cursor = events.length;
1023
+ }
1024
+
1025
+ /**
1026
+ * Install the pre-step bridge at the root level. Agent-scoped waterfalls
1027
+ * admit untagged (root) listeners, so one listener serves every agent —
1028
+ * including sessions resumed after a server restart, which never re-fire
1029
+ * `session/created` for per-agent attachments. Runs before every proposed
1030
+ * step: new pasted images are bridged into the durable log, and stuck
1031
+ * logged images are repaired, before the model request is derived from it.
1032
+ */
1033
+ function attachPreStepBridge(ctx, getConfig, exportDir) {
1034
+ const repairedBySession = new Map();
1035
+ ctx.on("agent/pre-step", async (payload, next) => {
1036
+ const decision = await next();
1037
+ if (!decision || decision.kind !== "enter") return decision;
1038
+ const agent = payload?.agent;
1039
+ if (!agent?.session) return decision;
1040
+ try {
1041
+ const acceptsImage = await currentModelAcceptsImage(agent, getConfig(), ctx.get("llm"));
1042
+ warnAutoPromotion(ctx.logger);
1043
+ if (!acceptsImage) {
1044
+ let repaired = repairedBySession.get(agent.session.id);
1045
+ if (!repaired) {
1046
+ repaired = { set: new Set(), cursor: 0 };
1047
+ repairedBySession.set(agent.session.id, repaired);
1048
+ }
1049
+ await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
1050
+ ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
1051
+ });
1052
+ }
1053
+ if (acceptsImage) return decision;
1054
+ const messages = await bridgeMessages(decision.messages, ctx, exportDir);
1055
+ if (messages.every((message, index) => message === decision.messages[index])) return decision;
1056
+ return { ...decision, messages };
1057
+ } catch (error) {
1058
+ ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
1059
+ return decision;
1060
+ }
1061
+ });
1062
+ }
1063
+
1064
+ function resolveApiKey(config) {
1065
+ if (config.apiKey) return config.apiKey;
1066
+ if (config.apiKeyEnv) {
1067
+ const fromEnv = process.env[config.apiKeyEnv];
1068
+ if (fromEnv) return fromEnv;
1069
+ }
1070
+ return process.env.OPENAI_API_KEY ?? "";
1071
+ }
1072
+
1073
+ /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
1074
+ async function toImageUrl(target, cwd, config) {
1075
+ if (/^https?:\/\//i.test(target)) return { url: target, note: target };
1076
+ const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
1077
+ const info = await stat(abs).catch(() => null);
1078
+ if (!info) throw new Error(`image not found: ${abs}`);
1079
+ if (info.size > config.maxImageBytes) {
1080
+ throw new Error(
1081
+ `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
1082
+ );
1083
+ }
1084
+ const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
1085
+ if (!mime) {
1086
+ throw new Error(
1087
+ `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
1088
+ );
1089
+ }
1090
+ const data = await readFile(abs);
1091
+ return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
1092
+ }
1093
+
1094
+ /** One OpenAI-compatible chat/completions call with an image_url content part. */
1095
+ async function callVision(config, imageUrl, question, detail, signal, exec) {
1096
+ const key = resolveApiKey(config);
1097
+ if (!key) {
1098
+ throw new Error(
1099
+ `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
1100
+ );
1101
+ }
1102
+ const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
1103
+ const endpoint = new URL("chat/completions", base);
1104
+ const sessionHeader = sessionHeaders(config, exec);
1105
+ const controller = new AbortController();
1106
+ const timer = setTimeout(
1107
+ () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
1108
+ config.timeoutMs,
1109
+ );
1110
+ const onSignalAbort = () => controller.abort(signal?.reason);
1111
+ if (signal) {
1112
+ if (signal.aborted) controller.abort();
1113
+ else signal.addEventListener("abort", onSignalAbort, { once: true });
1114
+ }
1115
+ const content = [
1116
+ { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
1117
+ { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
1118
+ ];
1119
+ try {
1120
+ const response = await fetch(endpoint, {
1121
+ method: "POST",
1122
+ headers: {
1123
+ "Content-Type": "application/json",
1124
+ Authorization: `Bearer ${key}`,
1125
+ ...sessionHeader,
1126
+ },
1127
+ body: JSON.stringify({
1128
+ model: config.model,
1129
+ messages: [{ role: "user", content }],
1130
+ max_tokens: config.maxTokens,
1131
+ }),
1132
+ signal: controller.signal,
1133
+ });
1134
+ const body = await response.json().catch(() => null);
1135
+ if (!response.ok) {
1136
+ const detailText = body?.error?.message ?? response.statusText;
1137
+ throw new Error(
1138
+ `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
1139
+ );
1140
+ }
1141
+ // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
1142
+ // `reasoning_content` first; when the final `content` is empty or was cut
1143
+ // off by max_tokens, fall back to the reasoning text so the answer is
1144
+ // still useful.
1145
+ const message = body?.choices?.[0]?.message;
1146
+ let answer = message?.content ?? "";
1147
+ if (!answer.trim()) answer = message?.reasoning_content ?? "";
1148
+ if (typeof answer !== "string" || !answer.trim()) {
1149
+ throw new Error("vision endpoint returned an empty response");
1150
+ }
1151
+ return answer.trim();
1152
+ } finally {
1153
+ clearTimeout(timer);
1154
+ signal?.removeEventListener("abort", onSignalAbort);
1155
+ }
1156
+ }
1157
+
1158
+ function apply(ctx, config) {
1159
+ // ── settings-backed configuration ─────────────────────────────────────────
1160
+ // The composition entry stays the `base` layer; a registered `tool-vision`
1161
+ // settings section (Web UI section, settings.yaml) overlays it live, so
1162
+ // edits hot-apply without a restart. `sourceGetter` is a GETTER
1163
+ // (`() => scope.get()`), not the config object — keep it and call it at use
1164
+ // time, or `getConfig()` would return a function and every `cfg.*` read
1165
+ // would be undefined (apiKey included).
1166
+ let current = config;
1167
+ let sourceGetter = null;
1168
+ const getConfig = () => (sourceGetter ? sourceGetter() : current);
1169
+ // The registered settings scope, kept so tools can PERSIST what they measure.
1170
+ // Null whenever no settings provider is mounted (composition-entry-only runs),
1171
+ // in which case a probe still reports its verdict but cannot record it.
1172
+ let settingsScope = null;
1173
+ const getSettingsScope = () => settingsScope;
1174
+
1175
+ // ── master switch: a child fiber owns every registration ──────────────────
1176
+ // `ctx.tools.register`, `ctx.effect` and `ctx.on` are all effects on the fiber
1177
+ // that makes them, so confining the tool/bridge/route registrations to a child
1178
+ // fiber makes `enabled` real: disposing that fiber unregisters all 15 tools,
1179
+ // removes the pre-step listener and the preview route, and restores the llm
1180
+ // capability wrap. The settings registration deliberately stays on the parent
1181
+ // fiber — it has to outlive the switch, or the section that turns the plugin
1182
+ // back on would vanish with it.
1183
+ const REGISTRATION_KEYS = [
1184
+ "enabled",
1185
+ "bridgeTextOnly",
1186
+ "bridgeExportDir",
1187
+ "bridgeAutoImage",
1188
+ "bridgePreview",
1189
+ ];
1190
+ let featureFiber = null;
1191
+ let lastRegistrationKey = null;
1192
+
1193
+ function registrationKey(cfg) {
1194
+ return REGISTRATION_KEYS.map((key) => String(cfg[key])).join("|");
1195
+ }
1196
+
1197
+ function installFeatures() {
1198
+ if (featureFiber !== null) return;
1199
+ featureFiber = ctx.plugin({
1200
+ name: "dsh-tool-vision:features",
1201
+ apply(inner) {
1202
+ // ── image bridge: pasted images become inspect_image hints on text-only models ──
1203
+ if (getConfig().bridgeTextOnly) {
1204
+ const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
1205
+ mkdir(exportDir, { recursive: true }).catch(() => {});
1206
+ // Root-level listener: agent-scoped waterfalls admit untagged listeners,
1207
+ // so one registration serves every agent (new and resumed alike) and the
1208
+ // agent is read from the fused payload.
1209
+ attachPreStepBridge(inner, getConfig, exportDir);
1210
+
1211
+ // ── automatic image admission: let pasted images through on text-only models ──
1212
+ // The host gate refuses images unless the model declares `image` input;
1213
+ // the bridge handles them anyway, so report image support for all models.
1214
+ // The wrap is installed on the shared llm service instance (idempotent,
1215
+ // restored on dispose/HMR).
1216
+ if (getConfig().bridgeAutoImage) {
1217
+ const unwrap = installAutoImageAdmission(inner.get("llm"), inner.logger);
1218
+ inner.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
1219
+ }
1220
+
1221
+ // ── dispatch image admission: make the capability verdict actually land ──
1222
+ // `resolveModelInfo` above is pure admission — it never changes what
1223
+ // the adapter streams. The image the plugin decided to hand over
1224
+ // directly was still being projected into a text placeholder by
1225
+ // `LlmService.generate`, which reads the adapter's own declaration and
1226
+ // nothing else. This applies the same decision at that seam.
1227
+ //
1228
+ // Installed outside the `bridgeAutoImage` branch on purpose: the
1229
+ // question it answers is not "may the image be offered" but "does the
1230
+ // model receive it", and that is the plugin's own verdict either way.
1231
+ {
1232
+ const unwrapDispatch = installDispatchImageAdmission(inner.get("llm"), getConfig, inner.logger);
1233
+ inner.effect(() => unwrapDispatch, "dsh-tool-vision: dispatch image admission");
1234
+ }
1235
+
1236
+ // ── bridge image preview: same-origin thumbnails for bridged images ──
1237
+ if (getConfig().bridgePreview) {
1238
+ registerBridgePreviewRoute(inner, exportDir, inner.logger);
1239
+ }
1240
+ }
1241
+
1242
+ inner.tools.register(defineTool({
1243
+ name: "inspect_image",
1244
+ description: getConfig().description,
1245
+ parameters: {
1246
+ path: {
1247
+ type: "string",
1248
+ required: true,
1249
+ description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
1250
+ },
1251
+ question: {
1252
+ type: "string",
1253
+ description: "Optional specific question about the image. Omit for a general detailed description.",
1254
+ },
1255
+ detail: {
1256
+ type: "string",
1257
+ enum: ["auto", "low", "high"],
1258
+ description: "Optional image resolution hint for the vision API (auto by default).",
1259
+ },
1260
+ },
1261
+ output: {
1262
+ schema: { type: "string" },
1263
+ render: (_args, value) => [{ type: "text", text: value }],
1264
+ },
1265
+ async execute(args, exec) {
1266
+ const cfg = getConfig();
1267
+ const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
1268
+ const { url, note } = await toImageUrl(args.path, cwd, cfg);
1269
+ try {
1270
+ const answer = await callVision(cfg, url, args.question, args.detail, exec.signal, exec);
1271
+ return note === url ? answer : `${answer}\n\n(image: ${note})`;
1272
+ } catch (error) {
1273
+ const raw = error && error.message ? String(error.message) : String(error);
1274
+ if (CONTENT_FILTER_RE.test(raw)) {
1275
+ throw new Error(
1276
+ "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
1277
+ "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
1278
+ );
1279
+ }
1280
+ throw error;
1281
+ }
1282
+ },
1283
+ }));
1284
+
1285
+ // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
1286
+ // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
1287
+ // (baseURL/apiKey/model). No provider chain, no local models, no extra
1288
+ // settings: everything comes from the existing tool-vision configuration.
1289
+ registerVisionTools(inner, getConfig);
1290
+
1291
+ // ── ground-truth capability probe (v0.9.0) ───────────────────────────
1292
+ // Everything else decides from what a route SAYS; this measures what it
1293
+ // DOES, and records the answer so the bridge decision stops guessing.
1294
+ inner.tools.register(defineTool({
1295
+ name: "vision_probe_model",
1296
+ description:
1297
+ "Measure whether a model's endpoint can actually read images, by sending it a real " +
1298
+ "solid-color image and checking the answer. Use this when a model's declared image " +
1299
+ "support is doubtful (it claims `image` but the provider may reject it, or it declares " +
1300
+ "text-only yet the endpoint may accept images). Runs a text-only control request plus " +
1301
+ "two different colors, so a model that merely guesses cannot pass. The verdict is " +
1302
+ "recorded and from then on decides whether that route's images are sent directly or " +
1303
+ "bridged. Costs a few requests against the model's own endpoint.",
1304
+ parameters: {
1305
+ model: {
1306
+ type: "string",
1307
+ required: true,
1308
+ description: "Model id to probe, e.g. xiaomi/mimo-v2.5. Use the id exactly as dsh has it configured.",
1309
+ },
1310
+ provider: {
1311
+ type: "string",
1312
+ description:
1313
+ "Provider route serving the model (e.g. commandcode). Omit to use the current " +
1314
+ "session's route, or to auto-select when exactly one provider offers the model.",
1315
+ },
1316
+ },
1317
+ output: {
1318
+ schema: { type: "string" },
1319
+ render: (_args, value) => [{ type: "text", text: value }],
1320
+ },
1321
+ async execute(args, exec) {
1322
+ const cfg = getConfig();
1323
+ const requested = String(args.model ?? "").trim();
1324
+ if (requested.length === 0) throw new Error("vision_probe_model: model is required");
1325
+ const llm = inner.get("llm");
1326
+
1327
+ // Resolve which routes offer this model, so an omitted `provider`
1328
+ // is still an exact choice rather than a guess.
1329
+ let candidates = [];
1330
+ try {
1331
+ for (const entry of llm?.listProviders?.() ?? []) {
1332
+ const id = entry?.id;
1333
+ if (typeof id !== "string") continue;
1334
+ const models = await llm.listModels(id).catch(() => []);
1335
+ if ((models ?? []).some((model) => String(model?.id) === requested)) candidates.push(id);
1336
+ }
1337
+ } catch {
1338
+ /* fall through to the route/argument fallbacks below */
1339
+ }
1340
+ let provider = typeof args.provider === "string" && args.provider.length > 0 ? args.provider : undefined;
1341
+ if (provider === undefined) {
1342
+ const here = currentRoute(exec?.agent);
1343
+ if (candidates.length === 1) provider = candidates[0];
1344
+ else if (here.model === requested && typeof here.provider === "string") provider = here.provider;
1345
+ else if (candidates.length > 1) {
1346
+ throw new Error(
1347
+ `vision_probe_model: "${requested}" is served by ${candidates.join(", ")}; pass provider explicitly`,
1348
+ );
1349
+ } else {
1350
+ provider = here.provider;
1351
+ }
1352
+ }
1353
+ if (typeof provider !== "string" || provider.length === 0) {
1354
+ throw new Error(
1355
+ `vision_probe_model: cannot tell which provider serves "${requested}"; pass provider explicitly`,
1356
+ );
1357
+ }
1358
+
1359
+ const target = await resolveProbeTarget(inner, provider, requested);
1360
+ const outcome = await probeModelCapability(
1361
+ {
1362
+ baseURL: target.baseURL,
1363
+ apiKey: target.apiKey,
1364
+ api: target.api,
1365
+ model: requested,
1366
+ maxTokens: PROBE_MAX_TOKENS,
1367
+ timeoutMs: Math.max(PROBE_TIMEOUT_MS, Number(cfg.timeoutMs) || 0),
1368
+ headers: sessionHeaders(cfg, exec),
1369
+ },
1370
+ exec.signal,
1371
+ );
1372
+
1373
+ const key = probeKey(provider, requested);
1374
+ let recorded = "not recorded (no settings provider)";
1375
+ if (outcome.support === "yes" || outcome.support === "no") {
1376
+ const scope = getSettingsScope();
1377
+ if (scope !== null) {
1378
+ try {
1379
+ await scope.update({ probeResults: { ...(cfg.probeResults ?? {}), [key]: outcome.support } });
1380
+ recorded = `recorded as "${outcome.support}"`;
1381
+ } catch (error) {
1382
+ recorded = `could not be recorded: ${String(error?.message ?? error)}`;
1383
+ }
1384
+ }
1385
+ }
1386
+
1387
+ const verdictText = outcome.support === "yes"
1388
+ ? "SUPPORTS images"
1389
+ : outcome.support === "no"
1390
+ ? "does NOT read images"
1391
+ : "INCONCLUSIVE (the route could not be probed)";
1392
+ const effect = outcome.support === "yes"
1393
+ ? "images will be sent to it directly"
1394
+ : outcome.support === "no"
1395
+ ? "images will be bridged to inspect_image hints"
1396
+ : "the existing list/declaration rules still apply";
1397
+ const evidence = Object.entries(outcome.evidence ?? {})
1398
+ .map(([name, value]) => ` ${name}: ${value}`)
1399
+ .join("\n");
1400
+ return [
1401
+ `${provider}/${requested}: ${verdictText}`,
1402
+ ` ${outcome.detail}`,
1403
+ ` ${recorded}; ${effect}`,
1404
+ evidence ? ` evidence:\n${evidence}` : "",
1405
+ ].filter(Boolean).join("\n");
1406
+ },
1407
+ }));
1408
+ },
1409
+ });
1410
+ }
1411
+
1412
+ function uninstallFeatures() {
1413
+ if (featureFiber === null) return;
1414
+ const fiber = featureFiber;
1415
+ featureFiber = null;
1416
+ // dispose() settles asynchronously and must never take the plugin down; the
1417
+ // next install starts a fresh fiber regardless of how this one ends.
1418
+ Promise.resolve(fiber.dispose()).catch((error) => {
1419
+ ctx.logger?.warn?.(`[tool-vision] feature teardown failed: ${String(error)}`);
1420
+ });
1421
+ }
1422
+
1423
+ // Re-install only when a field that gates a registration actually changed, so
1424
+ // editing, say, `model` does not tear the tools down and back up mid-session.
1425
+ function syncFeatures() {
1426
+ const cfg = getConfig();
1427
+ const key = registrationKey(cfg);
1428
+ if (featureFiber !== null && key === lastRegistrationKey) return;
1429
+ uninstallFeatures();
1430
+ lastRegistrationKey = key;
1431
+ if (cfg.enabled) installFeatures();
1432
+ }
1433
+
1434
+ // Compat shim: dsh-settings 0.1.2-rc.1 removed the module-level
1435
+ // `installSettingsSection` export (the provider now lives at ctx.settings).
1436
+ // Inline the same logic via ctx.inject(["settings"]) — works on both
1437
+ // 0.1.1 (module export wrapper) and 0.1.2 (ctx.settings) hosts.
1438
+ ctx.inject(["settings"], (sctx) => {
1439
+ const scope = sctx.settings.register(NS, Config, { base: config });
1440
+ sourceGetter = () => scope.get();
1441
+ settingsScope = scope;
1442
+ sctx.effect(() => () => {
1443
+ sourceGetter = null;
1444
+ settingsScope = null;
1445
+ });
1446
+ // Hot-apply: `enabled` switches the child fiber, and any field that gates a
1447
+ // registration re-installs it, so none of them needs a dsh restart anymore.
1448
+ scope.watch(() => syncFeatures());
1449
+ sctx.effect(() => () => uninstallFeatures());
1450
+ // The stored value governs from here on; the composition entry was only a
1451
+ // placeholder until the provider answered.
1452
+ syncFeatures();
1453
+ });
1454
+
1455
+ // ── model catalog route: settings support, so it lives on THIS fiber ───────
1456
+ // The master switch disposes the feature fiber on purpose (tools, bridge,
1457
+ // preview route), while the settings section itself deliberately stays
1458
+ // mounted — and a section whose list field cannot autocomplete is a section
1459
+ // you cannot configure before switching the plugin on. Registered here so it
1460
+ // outlives `enabled: false`; read-only and secret-free by construction.
1461
+ registerModelCatalogRoute(ctx, ctx.logger, getConfig);
1462
+
1463
+ // No settings provider: the composition entry is the whole configuration.
1464
+ syncFeatures();
1465
+ }
1466
+
1467
+ export {
1468
+ BRIDGE_MARKER,
1469
+ BRIDGE_PREVIEW_MAX_BYTES,
1470
+ BRIDGE_PREVIEW_MEDIA,
1471
+ BRIDGE_PREVIEW_ROUTE,
1472
+ Config,
1473
+ DEFAULT_DESCRIPTION,
1474
+ DEFAULT_MULTIMODAL_LIST_MODE,
1475
+ EXT_BY_MEDIA,
1476
+ MODEL_CATALOG_MAX_PER_PROVIDER,
1477
+ MODEL_CATALOG_ROUTE,
1478
+ MULTIMODAL_LIST_MODES,
1479
+ apply,
1480
+ attachPreStepBridge,
1481
+ autoPromotionWarned,
1482
+ bridgeMessages,
1483
+ currentModelAcceptsImage,
1484
+ currentRoute,
1485
+ deepFreeze,
1486
+ exportImage,
1487
+ hasImageBlock,
1488
+ inject,
1489
+ installAutoImageAdmission,
1490
+ installDispatchImageAdmission,
1491
+ lastModelDecision,
1492
+ modelListMatches,
1493
+ modelListMatchEntries,
1494
+ name,
1495
+ normalizeListMode,
1496
+ parseQuery,
1497
+ probeKey,
1498
+ probeVerdict,
1499
+ registerBridgePreviewRoute,
1500
+ routeDirectDecision,
1501
+ registerModelCatalogRoute,
1502
+ repairLoggedImages,
1503
+ resolveProbeTarget,
1504
+ routeDeclaresImage,
1505
+ unwrappedResolveModelInfo,
1506
+ warnAutoPromotion,
1507
+ };