dsh-tool-vision 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -23
- package/README.md +59 -9
- package/README.zh.md +318 -277
- package/cordis.patch.yml +8 -8
- package/index.js +1453 -1332
- package/lib/model-probe.js +274 -274
- package/lib/session-header.js +57 -57
- package/lib/vision-tools.js +2485 -2485
- package/package.json +2 -2
package/index.js
CHANGED
|
@@ -1,1332 +1,1453 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* dsh-tool-vision — external vision model for DeepSeek Harness.
|
|
3
|
-
*
|
|
4
|
-
* Two capabilities:
|
|
5
|
-
*
|
|
6
|
-
* 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
|
|
7
|
-
* any OpenAI-compatible chat/completions endpoint that supports
|
|
8
|
-
* `image_url` content parts, and returns the vision model's text answer.
|
|
9
|
-
*
|
|
10
|
-
* 2. Image bridge — pasted images are bridged to text hints before they
|
|
11
|
-
* enter a text-only model's request:
|
|
12
|
-
*
|
|
13
|
-
* - New images are bridged on the `agent/pre-step` waterfall (the only
|
|
14
|
-
* seam where the harness lets a plugin replace the messages that enter
|
|
15
|
-
* a step — they become the durable `user/message` log, so the
|
|
16
|
-
* `llm/stream` request-reconstruction invariant stays satisfied).
|
|
17
|
-
* - Images already logged before the plugin was installed (or before a
|
|
18
|
-
* server restart) are repaired lazily with a surface `replace`, one
|
|
19
|
-
* event at a time, on the first pre-step of the session.
|
|
20
|
-
*
|
|
21
|
-
* The bridged hint points at an exported local copy of the image, which
|
|
22
|
-
* the agent hands to `inspect_image`. Which models skip the bridge
|
|
23
|
-
* (v0.9.0) is decided by two independent inputs:
|
|
24
|
-
*
|
|
25
|
-
* - `multimodalModels` — a user-owned list whose meaning is set by
|
|
26
|
-
* `multimodalListMode`: `whitelist` (listed models receive image blocks
|
|
27
|
-
* directly; the historical meaning and the default), `blacklist`
|
|
28
|
-
* (listed models are forced through the bridge), or `off` (list
|
|
29
|
-
* ignored). Entries match the full model id, its bare id after the last
|
|
30
|
-
* `/`, or `provider/id`, and may contain `*` / `?` globs.
|
|
31
|
-
* - `autoDetectMultimodal` — when on, the route's own declared
|
|
32
|
-
* `inputModalities` forms the base set that the list then adds to
|
|
33
|
-
* (whitelist) or subtracts from (blacklist). The declaration is always
|
|
34
|
-
* read from the *unwrapped* `resolveModelInfo`, because the admission
|
|
35
|
-
* wrap below rewrites it for every model.
|
|
36
|
-
*
|
|
37
|
-
* 3. Bridge image preview (v0.4.0, contributed by xing666173 from
|
|
38
|
-
* dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
|
|
39
|
-
* inline thumbnails for bridged pasted images:
|
|
40
|
-
*
|
|
41
|
-
* - `bridgeMessages` stamps every bridged hint with an invisible marker
|
|
42
|
-
* (`BRIDGE_MARKER`), so the client can identify bridge text blocks
|
|
43
|
-
* precisely instead of pattern-matching free text.
|
|
44
|
-
* - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
|
|
45
|
-
* exported images to the same-origin page; the client inserts a
|
|
46
|
-
* thumbnail above the hint text and opens a lightbox on click.
|
|
47
|
-
* - Pure display layer: persisted messages, the transcript, the
|
|
48
|
-
* model-facing text and the `inspect_image` chain are untouched.
|
|
49
|
-
*/
|
|
50
|
-
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
51
|
-
import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
|
|
52
|
-
import os from "node:os";
|
|
53
|
-
import z from "@deepseek-ai/schemastery";
|
|
54
|
-
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
55
|
-
import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
|
|
56
|
-
import { sessionHeaders } from "./lib/session-header.js";
|
|
57
|
-
import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
|
|
58
|
-
|
|
59
|
-
/** Cordis plugin name. */
|
|
60
|
-
const name = "tool-vision";
|
|
61
|
-
/** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
|
|
62
|
-
const inject = ["tools", "llm", "attachments", "webServer"];
|
|
63
|
-
/** Settings namespace owned by this plugin (Web UI settings section). */
|
|
64
|
-
const NS = "tool-vision";
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Invisible prefix stamped onto every bridged hint text block. The browser
|
|
68
|
-
* half uses it to recognize bridge text precisely (no free-text regex over
|
|
69
|
-
* user messages), and the model-facing hint stays intact otherwise.
|
|
70
|
-
*/
|
|
71
|
-
const BRIDGE_MARKER = "\u200b[bridge]";
|
|
72
|
-
|
|
73
|
-
/** Loopback route serving bridged images to the same-origin page. */
|
|
74
|
-
const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
|
|
75
|
-
/**
|
|
76
|
-
* Loopback JSON route the settings panel reads the configured model catalog
|
|
77
|
-
* from (autocomplete for `multimodalModels` and the current-route readout).
|
|
78
|
-
* Registered on the plugin's own fiber, so it stays up while the master
|
|
79
|
-
* switch is off — a section you cannot configure until you switch it on would
|
|
80
|
-
* be useless.
|
|
81
|
-
*/
|
|
82
|
-
const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
|
|
83
|
-
/** Per-provider cap on catalog entries returned to the panel. */
|
|
84
|
-
const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
|
|
85
|
-
/** Hard cap for a single served image (defense in depth; export is bounded). */
|
|
86
|
-
const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
|
|
87
|
-
/** Extensions the preview route serves (svg/ico intentionally excluded). */
|
|
88
|
-
const BRIDGE_PREVIEW_MEDIA = {
|
|
89
|
-
".png": "image/png",
|
|
90
|
-
".jpg": "image/jpeg",
|
|
91
|
-
".jpeg": "image/jpeg",
|
|
92
|
-
".webp": "image/webp",
|
|
93
|
-
".gif": "image/gif",
|
|
94
|
-
".avif": "image/avif",
|
|
95
|
-
".bmp": "image/bmp",
|
|
96
|
-
};
|
|
97
|
-
|
|
98
|
-
const DEFAULT_DESCRIPTION =
|
|
99
|
-
"Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
|
|
100
|
-
"Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
|
|
101
|
-
"optionally with a specific question. Returns the vision model's textual description or answer. " +
|
|
102
|
-
"Use this whenever you need to read, describe, or extract information from image content, " +
|
|
103
|
-
"including an image that reached you as a bridged text hint.";
|
|
104
|
-
|
|
105
|
-
/** Runtime schema for the tool-vision row. */
|
|
106
|
-
const Config = z.object({
|
|
107
|
-
/**
|
|
108
|
-
* Master switch, hot-applied. When off the plugin registers nothing at all —
|
|
109
|
-
* no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
|
|
110
|
-
* preview route, no llm capability wrap — while the settings section itself
|
|
111
|
-
* stays mounted so the switch can turn it back on. Turning it off is the way
|
|
112
|
-
* to stop every vision tool from reaching the model without uninstalling the
|
|
113
|
-
* package or restarting dsh.
|
|
114
|
-
*/
|
|
115
|
-
enabled: z.boolean().default(true),
|
|
116
|
-
/** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
|
|
117
|
-
baseURL: z.string().default("https://api.openai.com/v1"),
|
|
118
|
-
/** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
|
|
119
|
-
apiKey: z.string().default("").role("secret"),
|
|
120
|
-
/** Environment variable holding the API key. */
|
|
121
|
-
apiKeyEnv: z.string().default("VISION_API_KEY"),
|
|
122
|
-
/** Vision model id served by the endpoint. */
|
|
123
|
-
model: z.string().default("gpt-4o-mini"),
|
|
124
|
-
/** Max output tokens for the vision call. */
|
|
125
|
-
maxTokens: z.number().default(4096),
|
|
126
|
-
/** Per-request timeout in milliseconds. */
|
|
127
|
-
timeoutMs: z.number().default(60000),
|
|
128
|
-
/** Largest local image accepted, in bytes. */
|
|
129
|
-
maxImageBytes: z.number().default(10 * 1024 * 1024),
|
|
130
|
-
/** Tool description shown to the model; overrides the default. */
|
|
131
|
-
description: z.string().default(DEFAULT_DESCRIPTION),
|
|
132
|
-
/** Bridge pasted images to text hints on models that cannot see images. */
|
|
133
|
-
bridgeTextOnly: z.boolean().default(true),
|
|
134
|
-
/** Export directory for bridged images; empty = system temp. */
|
|
135
|
-
bridgeExportDir: z.string().default(""),
|
|
136
|
-
/**
|
|
137
|
-
* Model ids the list below refers to (v0.9.0). Each entry is matched
|
|
138
|
-
* case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
|
|
139
|
-
* bare id after the last `/` (`mimo-v2.5`), and `provider/id`
|
|
140
|
-
* (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
|
|
141
|
-
* (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
|
|
142
|
-
* route happens to spell it.
|
|
143
|
-
*/
|
|
144
|
-
multimodalModels: z.array(z.string()).default([]),
|
|
145
|
-
/**
|
|
146
|
-
* How `multimodalModels` is read (v0.9.0):
|
|
147
|
-
*
|
|
148
|
-
* - `whitelist` (default): a listed model receives image blocks directly
|
|
149
|
-
* and is never bridged — the meaning the list always had, so an
|
|
150
|
-
* existing configuration keeps behaving exactly as before.
|
|
151
|
-
* - `blacklist`: a listed model is forced through the bridge even when its
|
|
152
|
-
* route declares image input. This is the correction layer for a model
|
|
153
|
-
* that claims image support its endpoint does not really have.
|
|
154
|
-
* - `off`: the list is ignored entirely; nothing is forced either way.
|
|
155
|
-
*
|
|
156
|
-
* An unknown value falls back to `whitelist`.
|
|
157
|
-
*/
|
|
158
|
-
multimodalListMode: z.string().default("whitelist"),
|
|
159
|
-
/**
|
|
160
|
-
* Let the current route's own declared `inputModalities` drive the bridge
|
|
161
|
-
* decision, so only the models that cannot see images get bridged (v0.9.0,
|
|
162
|
-
* **on by default**). This and `multimodalModels` are one decision, not two:
|
|
163
|
-
* the declaration forms the base set, and the list unions with it
|
|
164
|
-
* (whitelist) or subtracts from it (blacklist).
|
|
165
|
-
*
|
|
166
|
-
* The declaration is only whatever the profile — or the installed catalog —
|
|
167
|
-
* says, and it CAN be wrong: profiles sometimes declare
|
|
168
|
-
* `input: [text, image]` just to pass the host admission gate, and a wrong
|
|
169
|
-
* "yes" sends the image straight to an endpoint that may reject it. The
|
|
170
|
-
* harness's own text-model projection is disabled by `bridgeAutoImage`, so
|
|
171
|
-
* nothing underneath catches that. Two things bound the damage: the first
|
|
172
|
-
* promotion of a route logs a notice naming the escape hatch
|
|
173
|
-
* ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
|
|
174
|
-
* under `blacklist` mode forces the bridge back on. Set this to false to
|
|
175
|
-
* bridge everything the list does not name.
|
|
176
|
-
*
|
|
177
|
-
* The value is always read from the *unwrapped* `resolveModelInfo`, so the
|
|
178
|
-
* admission wrap can never feed its own claim back in as evidence.
|
|
179
|
-
*/
|
|
180
|
-
autoDetectMultimodal: z.boolean().default(true),
|
|
181
|
-
/**
|
|
182
|
-
* Measured image capability per route (v0.9.0), keyed `"provider/model"` with
|
|
183
|
-
* the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
|
|
184
|
-
* never by hand — the only entry in this schema that records what a route
|
|
185
|
-
* DOES rather than what it says.
|
|
186
|
-
*
|
|
187
|
-
* It outranks the route's own declaration, because a probe is a real request
|
|
188
|
-
* and a declaration is only a claim. It does NOT outrank `multimodalModels`:
|
|
189
|
-
* that list is explicit human intent, and a person who names a model
|
|
190
|
-
* deserves the last word over an automated measurement.
|
|
191
|
-
*/
|
|
192
|
-
probeResults: z.dict(z.string()).default({}),
|
|
193
|
-
/** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
|
|
194
|
-
bridgePreview: z.boolean().default(true),
|
|
195
|
-
/** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
|
|
196
|
-
bridgePreviewScanIntervalMs: z.number().default(2000),
|
|
197
|
-
/** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
|
|
198
|
-
bridgePreviewHideHint: z.boolean().default(true),
|
|
199
|
-
/** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
|
|
200
|
-
desktopScreenshot: z.boolean().default(false),
|
|
201
|
-
/**
|
|
202
|
-
* Advertise image input capability for every model while the bridge is on.
|
|
203
|
-
* The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
|
|
204
|
-
* pasted images unless the current model declares `image` in its
|
|
205
|
-
* `inputModalities`. The bridge handles those images anyway (they become
|
|
206
|
-
* text hints the agent inspects through `inspect_image`), so this wraps the
|
|
207
|
-
* llm service's `resolveModelInfo` to report image support for text-only
|
|
208
|
-
* models too — users can paste images on any model without hand-editing
|
|
209
|
-
* provider model configs.
|
|
210
|
-
*/
|
|
211
|
-
bridgeAutoImage: z.boolean().default(true),
|
|
212
|
-
/** Send a stable per-conversation id header (`x-opencode-session`) on vision
|
|
213
|
-
* requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
|
|
214
|
-
* on every request; requests without the header may error. The value is the
|
|
215
|
-
* current dsh session id when the call runs inside one, else `sessionId`
|
|
216
|
-
* below, else a stable per-process random id. Set false to disable. */
|
|
217
|
-
sendSessionHeader: z.boolean().default(true),
|
|
218
|
-
/** Header name carrying the session id. */
|
|
219
|
-
sessionHeaderName: z.string().default("x-opencode-session"),
|
|
220
|
-
/** Fixed session id override for callers without a dsh session context.
|
|
221
|
-
* Empty = auto (per-process stable id). */
|
|
222
|
-
sessionId: z.string().default(""),
|
|
223
|
-
});
|
|
224
|
-
|
|
225
|
-
const MIME_BY_EXT = {
|
|
226
|
-
".png": "image/png",
|
|
227
|
-
".jpg": "image/jpeg",
|
|
228
|
-
".jpeg": "image/jpeg",
|
|
229
|
-
".gif": "image/gif",
|
|
230
|
-
".webp": "image/webp",
|
|
231
|
-
".bmp": "image/bmp",
|
|
232
|
-
".avif": "image/avif",
|
|
233
|
-
".svg": "image/svg+xml",
|
|
234
|
-
".ico": "image/x-icon",
|
|
235
|
-
};
|
|
236
|
-
|
|
237
|
-
const EXT_BY_MEDIA = {
|
|
238
|
-
"image/png": ".png",
|
|
239
|
-
"image/jpeg": ".jpg",
|
|
240
|
-
"image/webp": ".webp",
|
|
241
|
-
"image/gif": ".gif",
|
|
242
|
-
};
|
|
243
|
-
|
|
244
|
-
/** True when any message carries an image content block. */
|
|
245
|
-
function hasImageBlock(messages) {
|
|
246
|
-
return (messages ?? []).some((m) =>
|
|
247
|
-
Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
|
|
248
|
-
);
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
|
|
252
|
-
function deepFreeze(value) {
|
|
253
|
-
if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
|
|
254
|
-
if (Array.isArray(value)) {
|
|
255
|
-
for (const item of value) deepFreeze(item);
|
|
256
|
-
return Object.freeze(value);
|
|
257
|
-
}
|
|
258
|
-
for (const key of Object.keys(value)) deepFreeze(value[key]);
|
|
259
|
-
return Object.freeze(value);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
/** Export one attachment to disk; returns the file path (cached per process). */
|
|
263
|
-
const exportedPaths = new Map();
|
|
264
|
-
async function exportImage(attachment, ctx, dir) {
|
|
265
|
-
const cached = exportedPaths.get(attachment.attachmentId);
|
|
266
|
-
if (cached) return cached;
|
|
267
|
-
const { data } = await ctx.attachments.readImage(attachment);
|
|
268
|
-
const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
|
|
269
|
-
const safeName = attachment.name
|
|
270
|
-
? attachment.name
|
|
271
|
-
.replace(/\.[^.]+$/, "")
|
|
272
|
-
.replace(/[^\w\-]+/g, "_")
|
|
273
|
-
.replace(/^_+|_+$/g, "")
|
|
274
|
-
.slice(0, 40)
|
|
275
|
-
: "";
|
|
276
|
-
const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
|
|
277
|
-
const path = join(dir, `${base}${ext}`);
|
|
278
|
-
await writeFile(path, data);
|
|
279
|
-
exportedPaths.set(attachment.attachmentId, path);
|
|
280
|
-
return path;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
/**
|
|
284
|
-
* Replace image content blocks with text hints pointing at exported files.
|
|
285
|
-
* Non-image messages are returned as-is (same reference); bridged messages
|
|
286
|
-
* are fresh, deep-frozen objects with the original identity and source.
|
|
287
|
-
* Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
|
|
288
|
-
* prefix so the browser half can recognize it precisely.
|
|
289
|
-
* Exported for unit testing; `ctx` only needs `attachments`.
|
|
290
|
-
*/
|
|
291
|
-
async function bridgeMessages(messages, ctx, dir) {
|
|
292
|
-
const next = [];
|
|
293
|
-
for (const message of messages) {
|
|
294
|
-
const content = message?.content;
|
|
295
|
-
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
296
|
-
next.push(message);
|
|
297
|
-
continue;
|
|
298
|
-
}
|
|
299
|
-
const blocks = [];
|
|
300
|
-
for (const block of content) {
|
|
301
|
-
if (block?.type !== "image") {
|
|
302
|
-
blocks.push(block);
|
|
303
|
-
continue;
|
|
304
|
-
}
|
|
305
|
-
const path = await exportImage(block.attachment, ctx, dir);
|
|
306
|
-
const name = block.attachment.name ? ` (${block.attachment.name})` : "";
|
|
307
|
-
blocks.push({
|
|
308
|
-
type: "text",
|
|
309
|
-
text:
|
|
310
|
-
`${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
|
|
311
|
-
`Inspect it with the inspect_image tool to see its content.]`,
|
|
312
|
-
});
|
|
313
|
-
}
|
|
314
|
-
next.push(deepFreeze({ ...message, content: blocks }));
|
|
315
|
-
}
|
|
316
|
-
return next;
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
/** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
|
|
320
|
-
function parseQuery(rawUrl) {
|
|
321
|
-
const query = {};
|
|
322
|
-
const at = rawUrl.indexOf("?");
|
|
323
|
-
if (at === -1) return query;
|
|
324
|
-
for (const pair of rawUrl.slice(at + 1).split("&")) {
|
|
325
|
-
const eq = pair.indexOf("=");
|
|
326
|
-
if (eq === -1) continue;
|
|
327
|
-
try {
|
|
328
|
-
query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
|
|
329
|
-
} catch {
|
|
330
|
-
/* skip malformed pairs */
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
return query;
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
/**
|
|
337
|
-
* Advertise `image` in the `inputModalities` reported by the llm service for
|
|
338
|
-
* every model, so the host admission gate lets pasted images through on
|
|
339
|
-
* text-only models. The bridge turns those images into text hints anyway, so
|
|
340
|
-
* this is pure admission: it never changes what the adapter actually streams
|
|
341
|
-
* (the adapter's own stream validation reads the model's real `input` from
|
|
342
|
-
* the provider config, untouched here).
|
|
343
|
-
*
|
|
344
|
-
* The wrap is installed on the shared llm service instance, so it must be
|
|
345
|
-
* idempotent across HMR re-applies and restored on dispose. `marker` is a
|
|
346
|
-
* per-instance record proving this plugin already owns the wrap; the
|
|
347
|
-
* `installed` flag distinguishes "we wrapped it" from "the original was
|
|
348
|
-
* replaced by something else" so dispose only restores what we replaced.
|
|
349
|
-
*
|
|
350
|
-
* Exported for unit testing; the test passes a fake llm service.
|
|
351
|
-
*/
|
|
352
|
-
const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
|
|
353
|
-
function installAutoImageAdmission(llm, logger) {
|
|
354
|
-
if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
|
|
355
|
-
logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
|
|
356
|
-
return () => {};
|
|
357
|
-
}
|
|
358
|
-
if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
|
|
359
|
-
const original = llm.resolveModelInfo.bind(llm);
|
|
360
|
-
const wrapped = async (provider, model, signal) => {
|
|
361
|
-
const info = await original(provider, model, signal);
|
|
362
|
-
if (!info) return info;
|
|
363
|
-
const mods = info.inputModalities;
|
|
364
|
-
if (Array.isArray(mods) && mods.includes("image")) return info;
|
|
365
|
-
return { ...info, inputModalities: [...(mods ?? []), "image"] };
|
|
366
|
-
};
|
|
367
|
-
let installed = false;
|
|
368
|
-
// The marker carries BOTH functions: the wrapped one proves ownership for
|
|
369
|
-
// dispose, and `original` is the only honest source of a route's real
|
|
370
|
-
// modalities once the wrap is in place (`unwrappedResolveModelInfo`).
|
|
371
|
-
llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
|
|
372
|
-
llm.resolveModelInfo = wrapped;
|
|
373
|
-
installed = true;
|
|
374
|
-
logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
|
|
375
|
-
return () => {
|
|
376
|
-
if (installed) {
|
|
377
|
-
if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
|
|
378
|
-
installed = false;
|
|
379
|
-
}
|
|
380
|
-
delete llm[LLM_RESOLVE_WRAP_MARK];
|
|
381
|
-
};
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
/**
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
*
|
|
389
|
-
*
|
|
390
|
-
*
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
} catch {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
*
|
|
648
|
-
*
|
|
649
|
-
*
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
/**
|
|
658
|
-
*
|
|
659
|
-
*
|
|
660
|
-
*
|
|
661
|
-
*/
|
|
662
|
-
|
|
663
|
-
const
|
|
664
|
-
if (
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
*
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
*
|
|
687
|
-
*
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
const
|
|
695
|
-
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
*
|
|
710
|
-
*
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
*
|
|
788
|
-
*
|
|
789
|
-
*
|
|
790
|
-
*
|
|
791
|
-
*
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
const
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
/**
|
|
868
|
-
*
|
|
869
|
-
*
|
|
870
|
-
*
|
|
871
|
-
*
|
|
872
|
-
*
|
|
873
|
-
*
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
const
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
const
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1
|
+
/**
|
|
2
|
+
* dsh-tool-vision — external vision model for DeepSeek Harness.
|
|
3
|
+
*
|
|
4
|
+
* Two capabilities:
|
|
5
|
+
*
|
|
6
|
+
* 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
|
|
7
|
+
* any OpenAI-compatible chat/completions endpoint that supports
|
|
8
|
+
* `image_url` content parts, and returns the vision model's text answer.
|
|
9
|
+
*
|
|
10
|
+
* 2. Image bridge — pasted images are bridged to text hints before they
|
|
11
|
+
* enter a text-only model's request:
|
|
12
|
+
*
|
|
13
|
+
* - New images are bridged on the `agent/pre-step` waterfall (the only
|
|
14
|
+
* seam where the harness lets a plugin replace the messages that enter
|
|
15
|
+
* a step — they become the durable `user/message` log, so the
|
|
16
|
+
* `llm/stream` request-reconstruction invariant stays satisfied).
|
|
17
|
+
* - Images already logged before the plugin was installed (or before a
|
|
18
|
+
* server restart) are repaired lazily with a surface `replace`, one
|
|
19
|
+
* event at a time, on the first pre-step of the session.
|
|
20
|
+
*
|
|
21
|
+
* The bridged hint points at an exported local copy of the image, which
|
|
22
|
+
* the agent hands to `inspect_image`. Which models skip the bridge
|
|
23
|
+
* (v0.9.0) is decided by two independent inputs:
|
|
24
|
+
*
|
|
25
|
+
* - `multimodalModels` — a user-owned list whose meaning is set by
|
|
26
|
+
* `multimodalListMode`: `whitelist` (listed models receive image blocks
|
|
27
|
+
* directly; the historical meaning and the default), `blacklist`
|
|
28
|
+
* (listed models are forced through the bridge), or `off` (list
|
|
29
|
+
* ignored). Entries match the full model id, its bare id after the last
|
|
30
|
+
* `/`, or `provider/id`, and may contain `*` / `?` globs.
|
|
31
|
+
* - `autoDetectMultimodal` — when on, the route's own declared
|
|
32
|
+
* `inputModalities` forms the base set that the list then adds to
|
|
33
|
+
* (whitelist) or subtracts from (blacklist). The declaration is always
|
|
34
|
+
* read from the *unwrapped* `resolveModelInfo`, because the admission
|
|
35
|
+
* wrap below rewrites it for every model.
|
|
36
|
+
*
|
|
37
|
+
* 3. Bridge image preview (v0.4.0, contributed by xing666173 from
|
|
38
|
+
* dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
|
|
39
|
+
* inline thumbnails for bridged pasted images:
|
|
40
|
+
*
|
|
41
|
+
* - `bridgeMessages` stamps every bridged hint with an invisible marker
|
|
42
|
+
* (`BRIDGE_MARKER`), so the client can identify bridge text blocks
|
|
43
|
+
* precisely instead of pattern-matching free text.
|
|
44
|
+
* - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
|
|
45
|
+
* exported images to the same-origin page; the client inserts a
|
|
46
|
+
* thumbnail above the hint text and opens a lightbox on click.
|
|
47
|
+
* - Pure display layer: persisted messages, the transcript, the
|
|
48
|
+
* model-facing text and the `inspect_image` chain are untouched.
|
|
49
|
+
*/
|
|
50
|
+
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
51
|
+
import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
|
|
52
|
+
import os from "node:os";
|
|
53
|
+
import z from "@deepseek-ai/schemastery";
|
|
54
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
55
|
+
import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
|
|
56
|
+
import { sessionHeaders } from "./lib/session-header.js";
|
|
57
|
+
import { probeModelCapability, PROBE_MAX_TOKENS, PROBE_TIMEOUT_MS } from "./lib/model-probe.js";
|
|
58
|
+
|
|
59
|
+
/** Cordis plugin name. */
|
|
60
|
+
const name = "tool-vision";
|
|
61
|
+
/** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
|
|
62
|
+
const inject = ["tools", "llm", "attachments", "webServer"];
|
|
63
|
+
/** Settings namespace owned by this plugin (Web UI settings section). */
|
|
64
|
+
const NS = "tool-vision";
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Invisible prefix stamped onto every bridged hint text block. The browser
|
|
68
|
+
* half uses it to recognize bridge text precisely (no free-text regex over
|
|
69
|
+
* user messages), and the model-facing hint stays intact otherwise.
|
|
70
|
+
*/
|
|
71
|
+
const BRIDGE_MARKER = "\u200b[bridge]";
|
|
72
|
+
|
|
73
|
+
/** Loopback route serving bridged images to the same-origin page. */
|
|
74
|
+
const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
|
|
75
|
+
/**
|
|
76
|
+
* Loopback JSON route the settings panel reads the configured model catalog
|
|
77
|
+
* from (autocomplete for `multimodalModels` and the current-route readout).
|
|
78
|
+
* Registered on the plugin's own fiber, so it stays up while the master
|
|
79
|
+
* switch is off — a section you cannot configure until you switch it on would
|
|
80
|
+
* be useless.
|
|
81
|
+
*/
|
|
82
|
+
const MODEL_CATALOG_ROUTE = "/plugins/dsh-tool-vision/models";
|
|
83
|
+
/** Per-provider cap on catalog entries returned to the panel. */
|
|
84
|
+
const MODEL_CATALOG_MAX_PER_PROVIDER = 500;
|
|
85
|
+
/** Hard cap for a single served image (defense in depth; export is bounded). */
|
|
86
|
+
const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
|
|
87
|
+
/** Extensions the preview route serves (svg/ico intentionally excluded). */
|
|
88
|
+
const BRIDGE_PREVIEW_MEDIA = {
|
|
89
|
+
".png": "image/png",
|
|
90
|
+
".jpg": "image/jpeg",
|
|
91
|
+
".jpeg": "image/jpeg",
|
|
92
|
+
".webp": "image/webp",
|
|
93
|
+
".gif": "image/gif",
|
|
94
|
+
".avif": "image/avif",
|
|
95
|
+
".bmp": "image/bmp",
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const DEFAULT_DESCRIPTION =
|
|
99
|
+
"Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
|
|
100
|
+
"Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
|
|
101
|
+
"optionally with a specific question. Returns the vision model's textual description or answer. " +
|
|
102
|
+
"Use this whenever you need to read, describe, or extract information from image content, " +
|
|
103
|
+
"including an image that reached you as a bridged text hint.";
|
|
104
|
+
|
|
105
|
+
/** Runtime schema for the tool-vision row. */
|
|
106
|
+
const Config = z.object({
|
|
107
|
+
/**
|
|
108
|
+
* Master switch, hot-applied. When off the plugin registers nothing at all —
|
|
109
|
+
* no `inspect_image`, none of the 14 `vision_*` tools, no pre-step bridge, no
|
|
110
|
+
* preview route, no llm capability wrap — while the settings section itself
|
|
111
|
+
* stays mounted so the switch can turn it back on. Turning it off is the way
|
|
112
|
+
* to stop every vision tool from reaching the model without uninstalling the
|
|
113
|
+
* package or restarting dsh.
|
|
114
|
+
*/
|
|
115
|
+
enabled: z.boolean().default(true),
|
|
116
|
+
/** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
|
|
117
|
+
baseURL: z.string().default("https://api.openai.com/v1"),
|
|
118
|
+
/** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
|
|
119
|
+
apiKey: z.string().default("").role("secret"),
|
|
120
|
+
/** Environment variable holding the API key. */
|
|
121
|
+
apiKeyEnv: z.string().default("VISION_API_KEY"),
|
|
122
|
+
/** Vision model id served by the endpoint. */
|
|
123
|
+
model: z.string().default("gpt-4o-mini"),
|
|
124
|
+
/** Max output tokens for the vision call. */
|
|
125
|
+
maxTokens: z.number().default(4096),
|
|
126
|
+
/** Per-request timeout in milliseconds. */
|
|
127
|
+
timeoutMs: z.number().default(60000),
|
|
128
|
+
/** Largest local image accepted, in bytes. */
|
|
129
|
+
maxImageBytes: z.number().default(10 * 1024 * 1024),
|
|
130
|
+
/** Tool description shown to the model; overrides the default. */
|
|
131
|
+
description: z.string().default(DEFAULT_DESCRIPTION),
|
|
132
|
+
/** Bridge pasted images to text hints on models that cannot see images. */
|
|
133
|
+
bridgeTextOnly: z.boolean().default(true),
|
|
134
|
+
/** Export directory for bridged images; empty = system temp. */
|
|
135
|
+
bridgeExportDir: z.string().default(""),
|
|
136
|
+
/**
|
|
137
|
+
* Model ids the list below refers to (v0.9.0). Each entry is matched
|
|
138
|
+
* case-insensitively against the full model id (`xiaomi/mimo-v2.5`), its
|
|
139
|
+
* bare id after the last `/` (`mimo-v2.5`), and `provider/id`
|
|
140
|
+
* (`commandcode/xiaomi/mimo-v2.5`), and may use `*` / `?` globs
|
|
141
|
+
* (`*vl*`, `deepseek/*`). A bare id therefore keeps working however the
|
|
142
|
+
* route happens to spell it.
|
|
143
|
+
*/
|
|
144
|
+
multimodalModels: z.array(z.string()).default([]),
|
|
145
|
+
/**
|
|
146
|
+
* How `multimodalModels` is read (v0.9.0):
|
|
147
|
+
*
|
|
148
|
+
* - `whitelist` (default): a listed model receives image blocks directly
|
|
149
|
+
* and is never bridged — the meaning the list always had, so an
|
|
150
|
+
* existing configuration keeps behaving exactly as before.
|
|
151
|
+
* - `blacklist`: a listed model is forced through the bridge even when its
|
|
152
|
+
* route declares image input. This is the correction layer for a model
|
|
153
|
+
* that claims image support its endpoint does not really have.
|
|
154
|
+
* - `off`: the list is ignored entirely; nothing is forced either way.
|
|
155
|
+
*
|
|
156
|
+
* An unknown value falls back to `whitelist`.
|
|
157
|
+
*/
|
|
158
|
+
multimodalListMode: z.string().default("whitelist"),
|
|
159
|
+
/**
|
|
160
|
+
* Let the current route's own declared `inputModalities` drive the bridge
|
|
161
|
+
* decision, so only the models that cannot see images get bridged (v0.9.0,
|
|
162
|
+
* **on by default**). This and `multimodalModels` are one decision, not two:
|
|
163
|
+
* the declaration forms the base set, and the list unions with it
|
|
164
|
+
* (whitelist) or subtracts from it (blacklist).
|
|
165
|
+
*
|
|
166
|
+
* The declaration is only whatever the profile — or the installed catalog —
|
|
167
|
+
* says, and it CAN be wrong: profiles sometimes declare
|
|
168
|
+
* `input: [text, image]` just to pass the host admission gate, and a wrong
|
|
169
|
+
* "yes" sends the image straight to an endpoint that may reject it. The
|
|
170
|
+
* harness's own text-model projection is disabled by `bridgeAutoImage`, so
|
|
171
|
+
* nothing underneath catches that. Two things bound the damage: the first
|
|
172
|
+
* promotion of a route logs a notice naming the escape hatch
|
|
173
|
+
* ({@link warnAutoPromotion}), and naming that route in `multimodalModels`
|
|
174
|
+
* under `blacklist` mode forces the bridge back on. Set this to false to
|
|
175
|
+
* bridge everything the list does not name.
|
|
176
|
+
*
|
|
177
|
+
* The value is always read from the *unwrapped* `resolveModelInfo`, so the
|
|
178
|
+
* admission wrap can never feed its own claim back in as evidence.
|
|
179
|
+
*/
|
|
180
|
+
autoDetectMultimodal: z.boolean().default(true),
|
|
181
|
+
/**
|
|
182
|
+
* Measured image capability per route (v0.9.0), keyed `"provider/model"` with
|
|
183
|
+
* the value `"yes"` or `"no"`. Written by the `vision_probe_model` tool,
|
|
184
|
+
* never by hand — the only entry in this schema that records what a route
|
|
185
|
+
* DOES rather than what it says.
|
|
186
|
+
*
|
|
187
|
+
* It outranks the route's own declaration, because a probe is a real request
|
|
188
|
+
* and a declaration is only a claim. It does NOT outrank `multimodalModels`:
|
|
189
|
+
* that list is explicit human intent, and a person who names a model
|
|
190
|
+
* deserves the last word over an automated measurement.
|
|
191
|
+
*/
|
|
192
|
+
probeResults: z.dict(z.string()).default({}),
|
|
193
|
+
/** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
|
|
194
|
+
bridgePreview: z.boolean().default(true),
|
|
195
|
+
/** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
|
|
196
|
+
bridgePreviewScanIntervalMs: z.number().default(2000),
|
|
197
|
+
/** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
|
|
198
|
+
bridgePreviewHideHint: z.boolean().default(true),
|
|
199
|
+
/** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
|
|
200
|
+
desktopScreenshot: z.boolean().default(false),
|
|
201
|
+
/**
|
|
202
|
+
* Advertise image input capability for every model while the bridge is on.
|
|
203
|
+
* The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
|
|
204
|
+
* pasted images unless the current model declares `image` in its
|
|
205
|
+
* `inputModalities`. The bridge handles those images anyway (they become
|
|
206
|
+
* text hints the agent inspects through `inspect_image`), so this wraps the
|
|
207
|
+
* llm service's `resolveModelInfo` to report image support for text-only
|
|
208
|
+
* models too — users can paste images on any model without hand-editing
|
|
209
|
+
* provider model configs.
|
|
210
|
+
*/
|
|
211
|
+
bridgeAutoImage: z.boolean().default(true),
|
|
212
|
+
/** Send a stable per-conversation id header (`x-opencode-session`) on vision
|
|
213
|
+
* requests. OpenCode Go (and similar OpenAI-compatible gateways) require it
|
|
214
|
+
* on every request; requests without the header may error. The value is the
|
|
215
|
+
* current dsh session id when the call runs inside one, else `sessionId`
|
|
216
|
+
* below, else a stable per-process random id. Set false to disable. */
|
|
217
|
+
sendSessionHeader: z.boolean().default(true),
|
|
218
|
+
/** Header name carrying the session id. */
|
|
219
|
+
sessionHeaderName: z.string().default("x-opencode-session"),
|
|
220
|
+
/** Fixed session id override for callers without a dsh session context.
|
|
221
|
+
* Empty = auto (per-process stable id). */
|
|
222
|
+
sessionId: z.string().default(""),
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
const MIME_BY_EXT = {
|
|
226
|
+
".png": "image/png",
|
|
227
|
+
".jpg": "image/jpeg",
|
|
228
|
+
".jpeg": "image/jpeg",
|
|
229
|
+
".gif": "image/gif",
|
|
230
|
+
".webp": "image/webp",
|
|
231
|
+
".bmp": "image/bmp",
|
|
232
|
+
".avif": "image/avif",
|
|
233
|
+
".svg": "image/svg+xml",
|
|
234
|
+
".ico": "image/x-icon",
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const EXT_BY_MEDIA = {
|
|
238
|
+
"image/png": ".png",
|
|
239
|
+
"image/jpeg": ".jpg",
|
|
240
|
+
"image/webp": ".webp",
|
|
241
|
+
"image/gif": ".gif",
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
/** True when any message carries an image content block. */
|
|
245
|
+
function hasImageBlock(messages) {
|
|
246
|
+
return (messages ?? []).some((m) =>
|
|
247
|
+
Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
|
|
252
|
+
function deepFreeze(value) {
|
|
253
|
+
if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
|
|
254
|
+
if (Array.isArray(value)) {
|
|
255
|
+
for (const item of value) deepFreeze(item);
|
|
256
|
+
return Object.freeze(value);
|
|
257
|
+
}
|
|
258
|
+
for (const key of Object.keys(value)) deepFreeze(value[key]);
|
|
259
|
+
return Object.freeze(value);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Export one attachment to disk; returns the file path (cached per process). */
|
|
263
|
+
const exportedPaths = new Map();
|
|
264
|
+
async function exportImage(attachment, ctx, dir) {
|
|
265
|
+
const cached = exportedPaths.get(attachment.attachmentId);
|
|
266
|
+
if (cached) return cached;
|
|
267
|
+
const { data } = await ctx.attachments.readImage(attachment);
|
|
268
|
+
const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
|
|
269
|
+
const safeName = attachment.name
|
|
270
|
+
? attachment.name
|
|
271
|
+
.replace(/\.[^.]+$/, "")
|
|
272
|
+
.replace(/[^\w\-]+/g, "_")
|
|
273
|
+
.replace(/^_+|_+$/g, "")
|
|
274
|
+
.slice(0, 40)
|
|
275
|
+
: "";
|
|
276
|
+
const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
|
|
277
|
+
const path = join(dir, `${base}${ext}`);
|
|
278
|
+
await writeFile(path, data);
|
|
279
|
+
exportedPaths.set(attachment.attachmentId, path);
|
|
280
|
+
return path;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Replace image content blocks with text hints pointing at exported files.
|
|
285
|
+
* Non-image messages are returned as-is (same reference); bridged messages
|
|
286
|
+
* are fresh, deep-frozen objects with the original identity and source.
|
|
287
|
+
* Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
|
|
288
|
+
* prefix so the browser half can recognize it precisely.
|
|
289
|
+
* Exported for unit testing; `ctx` only needs `attachments`.
|
|
290
|
+
*/
|
|
291
|
+
async function bridgeMessages(messages, ctx, dir) {
|
|
292
|
+
const next = [];
|
|
293
|
+
for (const message of messages) {
|
|
294
|
+
const content = message?.content;
|
|
295
|
+
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
296
|
+
next.push(message);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
const blocks = [];
|
|
300
|
+
for (const block of content) {
|
|
301
|
+
if (block?.type !== "image") {
|
|
302
|
+
blocks.push(block);
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
const path = await exportImage(block.attachment, ctx, dir);
|
|
306
|
+
const name = block.attachment.name ? ` (${block.attachment.name})` : "";
|
|
307
|
+
blocks.push({
|
|
308
|
+
type: "text",
|
|
309
|
+
text:
|
|
310
|
+
`${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
|
|
311
|
+
`Inspect it with the inspect_image tool to see its content.]`,
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
next.push(deepFreeze({ ...message, content: blocks }));
|
|
315
|
+
}
|
|
316
|
+
return next;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
|
|
320
|
+
function parseQuery(rawUrl) {
|
|
321
|
+
const query = {};
|
|
322
|
+
const at = rawUrl.indexOf("?");
|
|
323
|
+
if (at === -1) return query;
|
|
324
|
+
for (const pair of rawUrl.slice(at + 1).split("&")) {
|
|
325
|
+
const eq = pair.indexOf("=");
|
|
326
|
+
if (eq === -1) continue;
|
|
327
|
+
try {
|
|
328
|
+
query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
|
|
329
|
+
} catch {
|
|
330
|
+
/* skip malformed pairs */
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return query;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Advertise `image` in the `inputModalities` reported by the llm service for
|
|
338
|
+
* every model, so the host admission gate lets pasted images through on
|
|
339
|
+
* text-only models. The bridge turns those images into text hints anyway, so
|
|
340
|
+
* this is pure admission: it never changes what the adapter actually streams
|
|
341
|
+
* (the adapter's own stream validation reads the model's real `input` from
|
|
342
|
+
* the provider config, untouched here).
|
|
343
|
+
*
|
|
344
|
+
* The wrap is installed on the shared llm service instance, so it must be
|
|
345
|
+
* idempotent across HMR re-applies and restored on dispose. `marker` is a
|
|
346
|
+
* per-instance record proving this plugin already owns the wrap; the
|
|
347
|
+
* `installed` flag distinguishes "we wrapped it" from "the original was
|
|
348
|
+
* replaced by something else" so dispose only restores what we replaced.
|
|
349
|
+
*
|
|
350
|
+
* Exported for unit testing; the test passes a fake llm service.
|
|
351
|
+
*/
|
|
352
|
+
const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
|
|
353
|
+
function installAutoImageAdmission(llm, logger) {
|
|
354
|
+
if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
|
|
355
|
+
logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
|
|
356
|
+
return () => {};
|
|
357
|
+
}
|
|
358
|
+
if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
|
|
359
|
+
const original = llm.resolveModelInfo.bind(llm);
|
|
360
|
+
const wrapped = async (provider, model, signal) => {
|
|
361
|
+
const info = await original(provider, model, signal);
|
|
362
|
+
if (!info) return info;
|
|
363
|
+
const mods = info.inputModalities;
|
|
364
|
+
if (Array.isArray(mods) && mods.includes("image")) return info;
|
|
365
|
+
return { ...info, inputModalities: [...(mods ?? []), "image"] };
|
|
366
|
+
};
|
|
367
|
+
let installed = false;
|
|
368
|
+
// The marker carries BOTH functions: the wrapped one proves ownership for
|
|
369
|
+
// dispose, and `original` is the only honest source of a route's real
|
|
370
|
+
// modalities once the wrap is in place (`unwrappedResolveModelInfo`).
|
|
371
|
+
llm[LLM_RESOLVE_WRAP_MARK] = { original, wrapped };
|
|
372
|
+
llm.resolveModelInfo = wrapped;
|
|
373
|
+
installed = true;
|
|
374
|
+
logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
|
|
375
|
+
return () => {
|
|
376
|
+
if (installed) {
|
|
377
|
+
if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
|
|
378
|
+
installed = false;
|
|
379
|
+
}
|
|
380
|
+
delete llm[LLM_RESOLVE_WRAP_MARK];
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Admission is not dispatch, and only dispatch decides whether the pixels arrive.
|
|
386
|
+
*
|
|
387
|
+
* {@link installAutoImageAdmission} wraps `llm.resolveModelInfo`. That method
|
|
388
|
+
* governs who may *offer* an image — the host gate that refuses a pasted image,
|
|
389
|
+
* the model list, the settings readout — and it is deliberately "pure admission:
|
|
390
|
+
* it never changes what the adapter actually streams".
|
|
391
|
+
*
|
|
392
|
+
* What the adapter actually streams is decided one layer down, by
|
|
393
|
+
* `LlmService.generate`:
|
|
394
|
+
*
|
|
395
|
+
* const adapterCall = await adapter.prepareCall(provider, model, signal);
|
|
396
|
+
* modelInfo = this.normalizeModelInfo(registration, model, adapterCall.model);
|
|
397
|
+
* if (modelInfo.inputModalities !== undefined
|
|
398
|
+
* && !modelInfo.inputModalities.includes("image")
|
|
399
|
+
* && projectedMessages.some((message) => contentHasImage(message.content)))
|
|
400
|
+
* projectedMessages = projectImagesForTextModel(projectedMessages);
|
|
401
|
+
*
|
|
402
|
+
* When that list lacks `image`, every image block is rewritten into a text
|
|
403
|
+
* placeholder (`textOnlyImageText`) *before the adapter is called*. So on a
|
|
404
|
+
* route the plugin had already measured as image-capable, and whose bridge
|
|
405
|
+
* decision was therefore "direct, do not intercept", the image was still
|
|
406
|
+
* destroyed — by a check that reads the adapter's declaration and nothing else.
|
|
407
|
+
* `probeResults`, `multimodalModels` and `autoDetectMultimodal` were all
|
|
408
|
+
* invisible to it.
|
|
409
|
+
*
|
|
410
|
+
* This wraps `llm.registration` — the accessor the core itself calls — so every
|
|
411
|
+
* adapter, including one registered later, answers `prepareCall` through
|
|
412
|
+
* {@link routeDirectDecision}: the same rule the bridge uses, so the two can
|
|
413
|
+
* never disagree about a route.
|
|
414
|
+
*
|
|
415
|
+
* When the decision is "bridge", nothing changes: the bridge already turned the
|
|
416
|
+
* image into an `inspect_image` hint, and the core's projection stays as the
|
|
417
|
+
* correct fallback for any image that still reaches dispatch.
|
|
418
|
+
*
|
|
419
|
+
* Idempotent across HMR re-applies, and dispose restores every adapter it
|
|
420
|
+
* touched plus the accessor itself.
|
|
421
|
+
*/
|
|
422
|
+
const LLM_REGISTRATION_WRAP_MARK = Symbol("dsh-tool-vision.registration.wrapped");
|
|
423
|
+
function installDispatchImageAdmission(llm, getConfig, logger) {
|
|
424
|
+
if (llm === undefined || llm === null || typeof llm.registration !== "function") {
|
|
425
|
+
logger?.warn?.("[tool-vision] llm service unavailable; dispatch image admission not installed");
|
|
426
|
+
return () => {};
|
|
427
|
+
}
|
|
428
|
+
if (llm[LLM_REGISTRATION_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
|
|
429
|
+
const original = llm.registration.bind(llm);
|
|
430
|
+
/** adapter -> the prepareCall it had before this wrap, for dispose. */
|
|
431
|
+
const patched = new Map();
|
|
432
|
+
const wrapped = (provider) => {
|
|
433
|
+
const registration = original(provider);
|
|
434
|
+
const adapter = registration?.adapter;
|
|
435
|
+
if (adapter !== undefined && adapter !== null
|
|
436
|
+
&& !patched.has(adapter) && typeof adapter.prepareCall === "function") {
|
|
437
|
+
const rawPrepare = adapter.prepareCall;
|
|
438
|
+
const prepare = rawPrepare.bind(adapter);
|
|
439
|
+
adapter.prepareCall = async (route, model, signal) => {
|
|
440
|
+
const call = await prepare(route, model, signal);
|
|
441
|
+
if (call === undefined || call === null || call.model === undefined || call.model === null) return call;
|
|
442
|
+
const mods = call.model.inputModalities;
|
|
443
|
+
if (Array.isArray(mods) && mods.includes("image")) return call;
|
|
444
|
+
let direct = false;
|
|
445
|
+
try {
|
|
446
|
+
({ direct } = await routeDirectDecision(route, model, getConfig(), llm));
|
|
447
|
+
} catch {
|
|
448
|
+
// A decision that cannot be made is not a licence to send pixels a
|
|
449
|
+
// route may reject: leave the core's projection in place.
|
|
450
|
+
return call;
|
|
451
|
+
}
|
|
452
|
+
if (!direct) return call;
|
|
453
|
+
return { ...call, model: { ...call.model, inputModalities: [...(mods ?? []), "image"] } };
|
|
454
|
+
};
|
|
455
|
+
patched.set(adapter, rawPrepare);
|
|
456
|
+
}
|
|
457
|
+
return registration;
|
|
458
|
+
};
|
|
459
|
+
llm[LLM_REGISTRATION_WRAP_MARK] = { original, wrapped };
|
|
460
|
+
llm.registration = wrapped;
|
|
461
|
+
logger?.debug?.("[tool-vision] dispatch image admission installed (llm.registration wrapped)");
|
|
462
|
+
return () => {
|
|
463
|
+
for (const [adapter, prepare] of patched) {
|
|
464
|
+
if (typeof prepare === "function") adapter.prepareCall = prepare;
|
|
465
|
+
}
|
|
466
|
+
patched.clear();
|
|
467
|
+
if (llm.registration === wrapped) llm.registration = original;
|
|
468
|
+
delete llm[LLM_REGISTRATION_WRAP_MARK];
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Register the loopback route that serves bridged images to the same-origin
|
|
474
|
+
* page (the preview thumbnails). Read-only and tightly scoped:
|
|
475
|
+
* - only files inside the bridge export directory (no traversal);
|
|
476
|
+
* - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
|
|
477
|
+
* - Host restricted to the local machine;
|
|
478
|
+
* - hard 20MB cap per file.
|
|
479
|
+
*/
|
|
480
|
+
function registerBridgePreviewRoute(ctx, exportDir, logger) {
|
|
481
|
+
const webServer = ctx.get("webServer");
|
|
482
|
+
if (webServer === undefined) {
|
|
483
|
+
logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
const bridgeDir = resolvePath(exportDir);
|
|
487
|
+
ctx.effect(() => webServer.register({
|
|
488
|
+
kind: "exact",
|
|
489
|
+
path: BRIDGE_PREVIEW_ROUTE,
|
|
490
|
+
async handler(req, res) {
|
|
491
|
+
try {
|
|
492
|
+
const raw = String(req.url ?? "");
|
|
493
|
+
const query = parseQuery(raw);
|
|
494
|
+
const p = query.p;
|
|
495
|
+
if (typeof p !== "string" || p.length === 0) {
|
|
496
|
+
res.writeHead(400);
|
|
497
|
+
res.end("bad request");
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
const lower = p.toLowerCase();
|
|
501
|
+
const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
|
|
502
|
+
if (mediaType === undefined) {
|
|
503
|
+
res.writeHead(400);
|
|
504
|
+
res.end("not an image path");
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
const host = String(req.headers?.host ?? "");
|
|
508
|
+
if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
|
|
509
|
+
res.writeHead(403);
|
|
510
|
+
res.end("forbidden");
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
const target = resolvePath(p);
|
|
514
|
+
if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
|
|
515
|
+
res.writeHead(403);
|
|
516
|
+
res.end("forbidden");
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
const info = await stat(target);
|
|
520
|
+
if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
|
|
521
|
+
res.writeHead(404);
|
|
522
|
+
res.end("not found");
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
525
|
+
const bytes = await readFile(target);
|
|
526
|
+
res.writeHead(200, {
|
|
527
|
+
"Content-Type": mediaType,
|
|
528
|
+
"Cache-Control": "private, max-age=60",
|
|
529
|
+
});
|
|
530
|
+
res.end(bytes);
|
|
531
|
+
} catch {
|
|
532
|
+
try {
|
|
533
|
+
res.writeHead(404);
|
|
534
|
+
res.end("not found");
|
|
535
|
+
} catch {
|
|
536
|
+
/* response already sent */
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
},
|
|
540
|
+
}), "dsh-tool-vision: bridge preview route");
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* Register the loopback JSON route the settings panel reads:
|
|
545
|
+
*
|
|
546
|
+
* { current: {provider, model, direct, source, mode}, providers: [...] }
|
|
547
|
+
*
|
|
548
|
+
* `providers` comes from `llm.listProviders()` + `llm.listModels(provider)` —
|
|
549
|
+
* the routes the harness actually has configured, which is exactly the set a
|
|
550
|
+
* `multimodalModels` entry may name. `listModels` is a different seam from
|
|
551
|
+
* `resolveModelInfo`, so these declared modalities are the profile's own and
|
|
552
|
+
* are NOT contaminated by the admission wrap (a model the wrap advertises as
|
|
553
|
+
* image-capable still reports its real declaration here).
|
|
554
|
+
*
|
|
555
|
+
* Same confinement as the image route: loopback Host only, read-only,
|
|
556
|
+
* `no-store`, and no credentials or endpoints in the payload — provider ids,
|
|
557
|
+
* model ids/names and one boolean each.
|
|
558
|
+
*/
|
|
559
|
+
function registerModelCatalogRoute(ctx, logger, getConfig) {
|
|
560
|
+
const webServer = ctx.get("webServer");
|
|
561
|
+
if (webServer === undefined) {
|
|
562
|
+
logger?.warn?.("[tool-vision] webServer unavailable; model catalog route not registered");
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
ctx.effect(() => webServer.register({
|
|
566
|
+
kind: "exact",
|
|
567
|
+
path: MODEL_CATALOG_ROUTE,
|
|
568
|
+
async handler(req, res) {
|
|
569
|
+
const send = (status, payload) => {
|
|
570
|
+
res.writeHead(status, {
|
|
571
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
572
|
+
"Cache-Control": "no-store",
|
|
573
|
+
});
|
|
574
|
+
res.end(JSON.stringify(payload));
|
|
575
|
+
};
|
|
576
|
+
const host = String(req.headers?.host ?? "");
|
|
577
|
+
if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
|
|
578
|
+
send(403, { error: "forbidden" });
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
const llm = ctx.get("llm");
|
|
582
|
+
if (llm === undefined) {
|
|
583
|
+
send(503, { error: "llm service unavailable" });
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
// `listed` is computed HERE, with the same matcher the bridge itself
|
|
587
|
+
// uses, so the panel never reimplements the rule (and can never
|
|
588
|
+
// disagree with the decision it is describing).
|
|
589
|
+
const cfg = typeof getConfig === "function" ? getConfig() : undefined;
|
|
590
|
+
const list = Array.isArray(cfg?.multimodalModels) ? cfg.multimodalModels : [];
|
|
591
|
+
const listMode = normalizeListMode(cfg?.multimodalListMode);
|
|
592
|
+
let routes = [];
|
|
593
|
+
try {
|
|
594
|
+
routes = llm.listProviders() ?? [];
|
|
595
|
+
} catch (error) {
|
|
596
|
+
logger?.debug?.(`[tool-vision] listProviders failed: ${String(error)}`);
|
|
597
|
+
}
|
|
598
|
+
const providers = [];
|
|
599
|
+
for (const entry of routes) {
|
|
600
|
+
const id = entry?.id;
|
|
601
|
+
if (typeof id !== "string" || id.length === 0) continue;
|
|
602
|
+
const record = {
|
|
603
|
+
id,
|
|
604
|
+
name: typeof entry?.name === "string" && entry.name.length > 0 ? entry.name : id,
|
|
605
|
+
models: [],
|
|
606
|
+
};
|
|
607
|
+
try {
|
|
608
|
+
const models = await llm.listModels(id);
|
|
609
|
+
record.models = (models ?? []).slice(0, MODEL_CATALOG_MAX_PER_PROVIDER).map((model) => {
|
|
610
|
+
const modelId = String(model.id);
|
|
611
|
+
const matchedEntries = modelListMatchEntries(list, id, modelId);
|
|
612
|
+
return {
|
|
613
|
+
id: modelId,
|
|
614
|
+
name: typeof model.name === "string" && model.name.length > 0 ? model.name : modelId,
|
|
615
|
+
image: Array.isArray(model.inputModalities) && model.inputModalities.includes("image"),
|
|
616
|
+
listed: matchedEntries.length > 0,
|
|
617
|
+
matchedEntries,
|
|
618
|
+
// What the route was MEASURED doing, when anyone has probed it.
|
|
619
|
+
// `null` means "not probed", which the panel must render
|
|
620
|
+
// differently from a probe that came back negative.
|
|
621
|
+
probe: probeVerdict(cfg, id, modelId) ?? null,
|
|
622
|
+
};
|
|
623
|
+
});
|
|
624
|
+
} catch (error) {
|
|
625
|
+
// One broken route must never blank the whole panel: report it and
|
|
626
|
+
// keep going, so the other providers still list.
|
|
627
|
+
record.error = String(error?.message ?? error);
|
|
628
|
+
}
|
|
629
|
+
providers.push(record);
|
|
630
|
+
}
|
|
631
|
+
send(200, {
|
|
632
|
+
current: { ...lastModelDecision },
|
|
633
|
+
list,
|
|
634
|
+
listMode,
|
|
635
|
+
probeResults: typeof getConfig === "function" ? { ...(getConfig()?.probeResults ?? {}) } : {},
|
|
636
|
+
providers,
|
|
637
|
+
});
|
|
638
|
+
},
|
|
639
|
+
}), "dsh-tool-vision: model catalog route");
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/** Accepted `multimodalListMode` values; anything else falls back to whitelist. */
|
|
643
|
+
const MULTIMODAL_LIST_MODES = ["off", "whitelist", "blacklist"];
|
|
644
|
+
const DEFAULT_MULTIMODAL_LIST_MODE = "whitelist";
|
|
645
|
+
|
|
646
|
+
/**
|
|
647
|
+
* Normalize a configured list mode. The settings section only ever writes one
|
|
648
|
+
* of {@link MULTIMODAL_LIST_MODES}, but the value can also arrive from a
|
|
649
|
+
* hand-edited `settings.yaml`, so an unknown string is clamped rather than
|
|
650
|
+
* treated as "no mode" (which would silently bridge everything).
|
|
651
|
+
*/
|
|
652
|
+
function normalizeListMode(value) {
|
|
653
|
+
return MULTIMODAL_LIST_MODES.includes(value) ? value : DEFAULT_MULTIMODAL_LIST_MODE;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const listEntryCache = new Map();
|
|
657
|
+
/**
|
|
658
|
+
* Compile one list entry into an anchored, case-insensitive regexp. A literal
|
|
659
|
+
* entry is just an exact match; `*` and `?` are the only metacharacters, so a
|
|
660
|
+
* model id containing `.` or `+` still matches itself.
|
|
661
|
+
*/
|
|
662
|
+
function listEntryRegExp(entry) {
|
|
663
|
+
const cached = listEntryCache.get(entry);
|
|
664
|
+
if (cached !== undefined) return cached;
|
|
665
|
+
const source = entry
|
|
666
|
+
.replace(/[.+^${}()|[\]\\]/g, "\\$&")
|
|
667
|
+
.replace(/\*/g, ".*")
|
|
668
|
+
.replace(/\?/g, ".");
|
|
669
|
+
const compiled = new RegExp(`^${source}$`, "i");
|
|
670
|
+
listEntryCache.set(entry, compiled);
|
|
671
|
+
return compiled;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Whether any entry in `patterns` names this route. Each entry is tested
|
|
676
|
+
* against the full model id, its bare id after the last `/`, and
|
|
677
|
+
* `provider/id` — so `mimo-v2.5`, `xiaomi/mimo-v2.5` and
|
|
678
|
+
* `commandcode/xiaomi/mimo-v2.5` all address the same route, and `*vl*`
|
|
679
|
+
* addresses every variant of one family. Exported for unit testing.
|
|
680
|
+
*/
|
|
681
|
+
function modelListMatches(patterns, provider, model) {
|
|
682
|
+
return modelListMatchEntries(patterns, provider, model).length > 0;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* The list entries that address this route. The panel needs *which* entry hit
|
|
687
|
+
* so that unchecking a model removes exactly the entry responsible — matching
|
|
688
|
+
* `mimo-v2.5` against `xiaomi/mimo-v2.5` must not leave an orphan behind, and
|
|
689
|
+
* must not delete an unrelated entry either. Exported for unit testing.
|
|
690
|
+
*/
|
|
691
|
+
function modelListMatchEntries(patterns, provider, model) {
|
|
692
|
+
if (!Array.isArray(patterns) || patterns.length === 0) return [];
|
|
693
|
+
if (typeof model !== "string" || model.length === 0) return [];
|
|
694
|
+
const slash = model.lastIndexOf("/");
|
|
695
|
+
const candidates = [model, slash === -1 ? model : model.slice(slash + 1)];
|
|
696
|
+
if (typeof provider === "string" && provider.length > 0) candidates.push(`${provider}/${model}`);
|
|
697
|
+
const hits = [];
|
|
698
|
+
for (const raw of patterns) {
|
|
699
|
+
if (raw === undefined || raw === null) continue;
|
|
700
|
+
const entry = String(raw).trim();
|
|
701
|
+
if (entry.length === 0) continue;
|
|
702
|
+
const re = listEntryRegExp(entry);
|
|
703
|
+
if (candidates.some((candidate) => re.test(candidate))) hits.push(String(raw));
|
|
704
|
+
}
|
|
705
|
+
return hits;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* The route the session is on: the logged request header first, then the
|
|
710
|
+
* agent's own options. `provider` is optional (older logs may lack it).
|
|
711
|
+
*/
|
|
712
|
+
function currentRoute(agent) {
|
|
713
|
+
const header = agent?.session?.requestHeader?.();
|
|
714
|
+
return {
|
|
715
|
+
provider: header?.config?.provider ?? agent?.options?.provider,
|
|
716
|
+
model: header?.config?.model ?? agent?.options?.model,
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/**
|
|
721
|
+
* The last decision `currentModelAcceptsImage` made, for the settings panel's
|
|
722
|
+
* "current route" readout (served over {@link MODEL_CATALOG_ROUTE}). A plain
|
|
723
|
+
* mutable record — the panel only ever reads it.
|
|
724
|
+
*/
|
|
725
|
+
const lastModelDecision = {
|
|
726
|
+
provider: undefined,
|
|
727
|
+
model: undefined,
|
|
728
|
+
direct: false,
|
|
729
|
+
source: "none",
|
|
730
|
+
mode: DEFAULT_MULTIMODAL_LIST_MODE,
|
|
731
|
+
};
|
|
732
|
+
|
|
733
|
+
/**
|
|
734
|
+
* `resolveModelInfo` as it was before {@link installAutoImageAdmission}
|
|
735
|
+
* wrapped it. Auto-detection MUST read through this: the wrap reports image
|
|
736
|
+
* support for *every* model, so reading the wrapped method would make the
|
|
737
|
+
* plugin's own claim the evidence for itself.
|
|
738
|
+
*/
|
|
739
|
+
function unwrappedResolveModelInfo(llm) {
|
|
740
|
+
const marker = llm?.[LLM_RESOLVE_WRAP_MARK];
|
|
741
|
+
const fn = marker?.original ?? llm?.resolveModelInfo;
|
|
742
|
+
return typeof fn === "function" ? fn.bind(llm) : undefined;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Whether the route declares `image` input, read before the admission wrap.
|
|
747
|
+
* `false` and `undefined` are the same answer to the caller ("not known to be
|
|
748
|
+
* multimodal"): a route nobody could resolve is bridged, never guessed.
|
|
749
|
+
*/
|
|
750
|
+
async function routeDeclaresImage(llm, provider, model) {
|
|
751
|
+
const resolve = unwrappedResolveModelInfo(llm);
|
|
752
|
+
if (resolve === undefined || typeof provider !== "string" || typeof model !== "string") {
|
|
753
|
+
return false;
|
|
754
|
+
}
|
|
755
|
+
try {
|
|
756
|
+
const info = await resolve(provider, model);
|
|
757
|
+
return info?.inputModalities?.includes("image") === true;
|
|
758
|
+
} catch {
|
|
759
|
+
return false;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/** Routes already warned about, so the notice appears once per route. */
|
|
764
|
+
const autoPromotionWarned = new Set();
|
|
765
|
+
|
|
766
|
+
/** Registry key for one measured route. Provider-qualified: the same model id
|
|
767
|
+
* can be served by two gateways with different real capabilities. */
|
|
768
|
+
function probeKey(provider, model) {
|
|
769
|
+
return `${typeof provider === "string" ? provider : ""}/${model}`;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* The measured verdict for a route, or `undefined` when it was never probed.
|
|
774
|
+
* Anything other than the two recorded values (a hand-edited settings.yaml, a
|
|
775
|
+
* future format) reads as "not probed" rather than as a guess.
|
|
776
|
+
*/
|
|
777
|
+
function probeVerdict(config, provider, model) {
|
|
778
|
+
const results = config?.probeResults;
|
|
779
|
+
if (results === undefined || results === null || typeof model !== "string" || model.length === 0) {
|
|
780
|
+
return undefined;
|
|
781
|
+
}
|
|
782
|
+
const value = results[probeKey(provider, model)];
|
|
783
|
+
return value === "yes" || value === "no" ? value : undefined;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Resolve the endpoint and credential a route actually sends to, so a probe
|
|
788
|
+
* talks to the same place the model does instead of a copy of it.
|
|
789
|
+
*
|
|
790
|
+
* The pieces come from two seams that a plugin is allowed to read:
|
|
791
|
+
* - `llm.listConfigurableProviders()` names the settings namespace and the
|
|
792
|
+
* path a provider's profile lives at;
|
|
793
|
+
* - `settings.get(ns)` resolves that profile (`baseURL`, `apiKeyEnv`, `api`);
|
|
794
|
+
* - `credentials.resolve(ref)` turns `apiKeyEnv` into the secret itself
|
|
795
|
+
* (`.value`), the same way `dsh-llm-pi-ai` does it for a real call.
|
|
796
|
+
*
|
|
797
|
+
* Read-only, and the secret never leaves this function's caller: the probe
|
|
798
|
+
* result records a verdict, never an endpoint or a key.
|
|
799
|
+
*
|
|
800
|
+
* @throws {Error} with a caller-facing reason when the route is unprobeable.
|
|
801
|
+
*/
|
|
802
|
+
async function resolveProbeTarget(ctx, provider, model) {
|
|
803
|
+
const llm = ctx.get("llm");
|
|
804
|
+
const settings = ctx.get("settings");
|
|
805
|
+
const credentials = ctx.get("credentials");
|
|
806
|
+
if (settings === undefined) {
|
|
807
|
+
throw new Error("the settings service is unavailable, so the route's endpoint cannot be read");
|
|
808
|
+
}
|
|
809
|
+
let entry;
|
|
810
|
+
try {
|
|
811
|
+
entry = (llm?.listConfigurableProviders?.() ?? []).find((candidate) => candidate?.provider === provider);
|
|
812
|
+
} catch {
|
|
813
|
+
/* an adapter that cannot describe its providers simply has no entry */
|
|
814
|
+
}
|
|
815
|
+
const settingsNs = entry?.settingsNs;
|
|
816
|
+
if (typeof settingsNs !== "string" || settingsNs.length === 0) {
|
|
817
|
+
throw new Error(
|
|
818
|
+
`provider "${provider}" does not publish a settings namespace, so its endpoint cannot be resolved for probing`,
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
let profile = settings.get(settingsNs);
|
|
822
|
+
const path = Array.isArray(entry.settingsPath) && entry.settingsPath.length > 0 ? entry.settingsPath : [provider];
|
|
823
|
+
for (const segment of path) profile = profile?.[segment];
|
|
824
|
+
if (profile === undefined || profile === null || typeof profile !== "object") {
|
|
825
|
+
throw new Error(`no stored profile for provider "${provider}" under "${settingsNs}"`);
|
|
826
|
+
}
|
|
827
|
+
const baseURL = typeof profile.baseURL === "string" && profile.baseURL.length > 0 ? profile.baseURL : undefined;
|
|
828
|
+
if (baseURL === undefined) {
|
|
829
|
+
throw new Error(`provider "${provider}" has no baseURL, so its endpoint cannot be probed`);
|
|
830
|
+
}
|
|
831
|
+
let apiKey;
|
|
832
|
+
const ref = profile.apiKeyEnv;
|
|
833
|
+
if (typeof ref === "string" && ref.length > 0) {
|
|
834
|
+
try {
|
|
835
|
+
const record = await credentials?.resolve?.(ref);
|
|
836
|
+
apiKey = typeof record?.value === "string" ? record.value : undefined;
|
|
837
|
+
} catch (error) {
|
|
838
|
+
throw new Error(`stored credential "${ref}" could not be read: ${String(error?.message ?? error)}`);
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
return { baseURL, apiKey, api: profile.api, model };
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/**
|
|
845
|
+
* One-time notice when a route receives images directly *only* because of its
|
|
846
|
+
* own declaration (source `auto`). This is the one failure auto-detection
|
|
847
|
+
* cannot rule out: if the declaration is wrong, the endpoint rejects the image
|
|
848
|
+
* after the message is already durable, and the error alone does not say what
|
|
849
|
+
* to change. Naming the escape hatch the first time it happens turns a
|
|
850
|
+
* confusing 400 into a one-line fix — and now names the probe too, which
|
|
851
|
+
* settles the question instead of asking the user to guess.
|
|
852
|
+
* Read {@link lastModelDecision}; never throws.
|
|
853
|
+
*/
|
|
854
|
+
function warnAutoPromotion(logger) {
|
|
855
|
+
const { provider, model, direct, source } = lastModelDecision;
|
|
856
|
+
if (direct !== true || source !== "auto" || typeof model !== "string" || model.length === 0) return;
|
|
857
|
+
const key = `${typeof provider === "string" ? provider : ""}\u0000${model}`;
|
|
858
|
+
if (autoPromotionWarned.has(key)) return;
|
|
859
|
+
autoPromotionWarned.add(key);
|
|
860
|
+
logger?.warn?.(
|
|
861
|
+
`[tool-vision] "${model}" receives images directly because its route declares image input, ` +
|
|
862
|
+
`which is unverified; run vision_probe_model to measure it, or list it in multimodalModels ` +
|
|
863
|
+
`under multimodalListMode: blacklist to force the bridge`,
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Whether images may be handed to this route directly, and why.
|
|
869
|
+
*
|
|
870
|
+
* The plugin's single precedence rule. Three inputs, deliberately independent:
|
|
871
|
+
*
|
|
872
|
+
* - the base set — the route's own declared `inputModalities`, and only when
|
|
873
|
+
* `autoDetectMultimodal` is on. It is read through
|
|
874
|
+
* {@link unwrappedResolveModelInfo}, never through the admission wrap.
|
|
875
|
+
* - the measurement — `probeResults`, which outranks the declaration because a
|
|
876
|
+
* probe is a real request and a declaration is only a statement.
|
|
877
|
+
* - the list — `multimodalModels` under `multimodalListMode`. Explicit human
|
|
878
|
+
* intent, so it outranks both: whitelist adds to the base set, blacklist
|
|
879
|
+
* subtracts from it.
|
|
880
|
+
*
|
|
881
|
+
* Two callers need this exact answer and must never disagree:
|
|
882
|
+
* - {@link currentModelAcceptsImage} — the bridge, deciding whether to
|
|
883
|
+
* intercept an image at all;
|
|
884
|
+
* - {@link installDispatchImageAdmission} — the dispatch path, where the core
|
|
885
|
+
* independently decides whether the model may receive the image block.
|
|
886
|
+
*
|
|
887
|
+
* Returns `direct: true` when bridging is disabled (nothing would be bridged
|
|
888
|
+
* anyway).
|
|
889
|
+
*/
|
|
890
|
+
async function routeDirectDecision(provider, model, config, llm) {
|
|
891
|
+
const mode = normalizeListMode(config.multimodalListMode);
|
|
892
|
+
if (!config.bridgeTextOnly) return { direct: true, source: "bridge-off", mode };
|
|
893
|
+
if (typeof model !== "string" || model.length === 0) {
|
|
894
|
+
return { direct: false, source: "no-route", mode };
|
|
895
|
+
}
|
|
896
|
+
const base = config.autoDetectMultimodal
|
|
897
|
+
? await routeDeclaresImage(llm, provider, model)
|
|
898
|
+
: false;
|
|
899
|
+
let direct = base;
|
|
900
|
+
let source = base ? "auto" : "default";
|
|
901
|
+
// A measured verdict outranks the route's own claim: a probe is a real
|
|
902
|
+
// request, a declaration is only a statement. Both directions count, so a
|
|
903
|
+
// route that says "image" and answered without reading the pixels goes back
|
|
904
|
+
// to the bridge without anyone hunting for the right list entry.
|
|
905
|
+
const probed = probeVerdict(config, provider, model);
|
|
906
|
+
if (probed !== undefined) {
|
|
907
|
+
direct = probed === "yes";
|
|
908
|
+
source = probed === "yes" ? "probe-yes" : "probe-no";
|
|
909
|
+
}
|
|
910
|
+
// `off` means the list carries no opinion at all — it is not "empty list
|
|
911
|
+
// under whitelist", it is "no list semantics", so nothing is forced. The
|
|
912
|
+
// human list keeps the last word over both other signals.
|
|
913
|
+
const listed = mode !== "off" && modelListMatches(config.multimodalModels, provider, model);
|
|
914
|
+
if (listed) {
|
|
915
|
+
direct = mode !== "blacklist";
|
|
916
|
+
source = mode === "blacklist" ? "blacklist" : "whitelist";
|
|
917
|
+
}
|
|
918
|
+
return { direct, source, mode };
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* Whether the session's current model may receive image blocks directly.
|
|
923
|
+
*
|
|
924
|
+
* {@link routeDirectDecision} keyed by the session's route, with the answer
|
|
925
|
+
* recorded in {@link lastModelDecision} for the settings panel.
|
|
926
|
+
*/
|
|
927
|
+
async function currentModelAcceptsImage(agent, config, llm) {
|
|
928
|
+
const { provider, model } = currentRoute(agent);
|
|
929
|
+
const decision = await routeDirectDecision(provider, model, config, llm);
|
|
930
|
+
Object.assign(lastModelDecision, { provider, model, ...decision });
|
|
931
|
+
return decision.direct;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Lazily bridge image blocks that are already part of the session log
|
|
936
|
+
* (pasted before the plugin was active, or before a restart). Each affected
|
|
937
|
+
* event is rewritten once with a surface `replace`, which swaps the durable
|
|
938
|
+
* derivation (and the transcript) to the text hint. Events that are no
|
|
939
|
+
* longer on the surface (already shadowed) are skipped and remembered.
|
|
940
|
+
* `repaired` tracks per-session state: a `Set` of handled seqs plus a
|
|
941
|
+
* monotonic scan cursor.
|
|
942
|
+
*/
|
|
943
|
+
async function repairLoggedImages(ctx, session, exportDir, repaired) {
|
|
944
|
+
const events = session.events;
|
|
945
|
+
for (let index = repaired.cursor; index < events.length; index += 1) {
|
|
946
|
+
const event = events[index];
|
|
947
|
+
if (event.type !== "user/message" || repaired.set.has(event.seq)) {
|
|
948
|
+
repaired.set.add(event.seq);
|
|
949
|
+
continue;
|
|
950
|
+
}
|
|
951
|
+
const content = event.data?.content;
|
|
952
|
+
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
953
|
+
repaired.set.add(event.seq);
|
|
954
|
+
continue;
|
|
955
|
+
}
|
|
956
|
+
const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
|
|
957
|
+
try {
|
|
958
|
+
session.append("user/message", bridged, {
|
|
959
|
+
surfaceOp: { op: "replace", start: event.seq, end: event.seq },
|
|
960
|
+
sourceEventSeqs: [event.seq],
|
|
961
|
+
});
|
|
962
|
+
ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
|
|
963
|
+
} catch (error) {
|
|
964
|
+
ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
|
|
965
|
+
}
|
|
966
|
+
repaired.set.add(event.seq);
|
|
967
|
+
}
|
|
968
|
+
repaired.cursor = events.length;
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/**
|
|
972
|
+
* Install the pre-step bridge at the root level. Agent-scoped waterfalls
|
|
973
|
+
* admit untagged (root) listeners, so one listener serves every agent —
|
|
974
|
+
* including sessions resumed after a server restart, which never re-fire
|
|
975
|
+
* `session/created` for per-agent attachments. Runs before every proposed
|
|
976
|
+
* step: new pasted images are bridged into the durable log, and stuck
|
|
977
|
+
* logged images are repaired, before the model request is derived from it.
|
|
978
|
+
*/
|
|
979
|
+
function attachPreStepBridge(ctx, getConfig, exportDir) {
|
|
980
|
+
const repairedBySession = new Map();
|
|
981
|
+
ctx.on("agent/pre-step", async (payload, next) => {
|
|
982
|
+
const decision = await next();
|
|
983
|
+
if (!decision || decision.kind !== "enter") return decision;
|
|
984
|
+
const agent = payload?.agent;
|
|
985
|
+
if (!agent?.session) return decision;
|
|
986
|
+
try {
|
|
987
|
+
const acceptsImage = await currentModelAcceptsImage(agent, getConfig(), ctx.get("llm"));
|
|
988
|
+
warnAutoPromotion(ctx.logger);
|
|
989
|
+
if (!acceptsImage) {
|
|
990
|
+
let repaired = repairedBySession.get(agent.session.id);
|
|
991
|
+
if (!repaired) {
|
|
992
|
+
repaired = { set: new Set(), cursor: 0 };
|
|
993
|
+
repairedBySession.set(agent.session.id, repaired);
|
|
994
|
+
}
|
|
995
|
+
await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
|
|
996
|
+
ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
if (acceptsImage) return decision;
|
|
1000
|
+
const messages = await bridgeMessages(decision.messages, ctx, exportDir);
|
|
1001
|
+
if (messages.every((message, index) => message === decision.messages[index])) return decision;
|
|
1002
|
+
return { ...decision, messages };
|
|
1003
|
+
} catch (error) {
|
|
1004
|
+
ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
|
|
1005
|
+
return decision;
|
|
1006
|
+
}
|
|
1007
|
+
});
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function resolveApiKey(config) {
|
|
1011
|
+
if (config.apiKey) return config.apiKey;
|
|
1012
|
+
if (config.apiKeyEnv) {
|
|
1013
|
+
const fromEnv = process.env[config.apiKeyEnv];
|
|
1014
|
+
if (fromEnv) return fromEnv;
|
|
1015
|
+
}
|
|
1016
|
+
return process.env.OPENAI_API_KEY ?? "";
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
|
|
1020
|
+
async function toImageUrl(target, cwd, config) {
|
|
1021
|
+
if (/^https?:\/\//i.test(target)) return { url: target, note: target };
|
|
1022
|
+
const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
|
|
1023
|
+
const info = await stat(abs).catch(() => null);
|
|
1024
|
+
if (!info) throw new Error(`image not found: ${abs}`);
|
|
1025
|
+
if (info.size > config.maxImageBytes) {
|
|
1026
|
+
throw new Error(
|
|
1027
|
+
`image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
|
|
1031
|
+
if (!mime) {
|
|
1032
|
+
throw new Error(
|
|
1033
|
+
`unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
const data = await readFile(abs);
|
|
1037
|
+
return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/** One OpenAI-compatible chat/completions call with an image_url content part. */
|
|
1041
|
+
async function callVision(config, imageUrl, question, detail, signal, exec) {
|
|
1042
|
+
const key = resolveApiKey(config);
|
|
1043
|
+
if (!key) {
|
|
1044
|
+
throw new Error(
|
|
1045
|
+
`vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
1048
|
+
const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
|
|
1049
|
+
const endpoint = new URL("chat/completions", base);
|
|
1050
|
+
const sessionHeader = sessionHeaders(config, exec);
|
|
1051
|
+
const controller = new AbortController();
|
|
1052
|
+
const timer = setTimeout(
|
|
1053
|
+
() => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
|
|
1054
|
+
config.timeoutMs,
|
|
1055
|
+
);
|
|
1056
|
+
const onSignalAbort = () => controller.abort(signal?.reason);
|
|
1057
|
+
if (signal) {
|
|
1058
|
+
if (signal.aborted) controller.abort();
|
|
1059
|
+
else signal.addEventListener("abort", onSignalAbort, { once: true });
|
|
1060
|
+
}
|
|
1061
|
+
const content = [
|
|
1062
|
+
{ type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
|
|
1063
|
+
{ type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
|
|
1064
|
+
];
|
|
1065
|
+
try {
|
|
1066
|
+
const response = await fetch(endpoint, {
|
|
1067
|
+
method: "POST",
|
|
1068
|
+
headers: {
|
|
1069
|
+
"Content-Type": "application/json",
|
|
1070
|
+
Authorization: `Bearer ${key}`,
|
|
1071
|
+
...sessionHeader,
|
|
1072
|
+
},
|
|
1073
|
+
body: JSON.stringify({
|
|
1074
|
+
model: config.model,
|
|
1075
|
+
messages: [{ role: "user", content }],
|
|
1076
|
+
max_tokens: config.maxTokens,
|
|
1077
|
+
}),
|
|
1078
|
+
signal: controller.signal,
|
|
1079
|
+
});
|
|
1080
|
+
const body = await response.json().catch(() => null);
|
|
1081
|
+
if (!response.ok) {
|
|
1082
|
+
const detailText = body?.error?.message ?? response.statusText;
|
|
1083
|
+
throw new Error(
|
|
1084
|
+
`vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
// Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
|
|
1088
|
+
// `reasoning_content` first; when the final `content` is empty or was cut
|
|
1089
|
+
// off by max_tokens, fall back to the reasoning text so the answer is
|
|
1090
|
+
// still useful.
|
|
1091
|
+
const message = body?.choices?.[0]?.message;
|
|
1092
|
+
let answer = message?.content ?? "";
|
|
1093
|
+
if (!answer.trim()) answer = message?.reasoning_content ?? "";
|
|
1094
|
+
if (typeof answer !== "string" || !answer.trim()) {
|
|
1095
|
+
throw new Error("vision endpoint returned an empty response");
|
|
1096
|
+
}
|
|
1097
|
+
return answer.trim();
|
|
1098
|
+
} finally {
|
|
1099
|
+
clearTimeout(timer);
|
|
1100
|
+
signal?.removeEventListener("abort", onSignalAbort);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
function apply(ctx, config) {
|
|
1105
|
+
// ── settings-backed configuration ─────────────────────────────────────────
|
|
1106
|
+
// The composition entry stays the `base` layer; a registered `tool-vision`
|
|
1107
|
+
// settings section (Web UI section, settings.yaml) overlays it live, so
|
|
1108
|
+
// edits hot-apply without a restart. `sourceGetter` is a GETTER
|
|
1109
|
+
// (`() => scope.get()`), not the config object — keep it and call it at use
|
|
1110
|
+
// time, or `getConfig()` would return a function and every `cfg.*` read
|
|
1111
|
+
// would be undefined (apiKey included).
|
|
1112
|
+
let current = config;
|
|
1113
|
+
let sourceGetter = null;
|
|
1114
|
+
const getConfig = () => (sourceGetter ? sourceGetter() : current);
|
|
1115
|
+
// The registered settings scope, kept so tools can PERSIST what they measure.
|
|
1116
|
+
// Null whenever no settings provider is mounted (composition-entry-only runs),
|
|
1117
|
+
// in which case a probe still reports its verdict but cannot record it.
|
|
1118
|
+
let settingsScope = null;
|
|
1119
|
+
const getSettingsScope = () => settingsScope;
|
|
1120
|
+
|
|
1121
|
+
// ── master switch: a child fiber owns every registration ──────────────────
|
|
1122
|
+
// `ctx.tools.register`, `ctx.effect` and `ctx.on` are all effects on the fiber
|
|
1123
|
+
// that makes them, so confining the tool/bridge/route registrations to a child
|
|
1124
|
+
// fiber makes `enabled` real: disposing that fiber unregisters all 15 tools,
|
|
1125
|
+
// removes the pre-step listener and the preview route, and restores the llm
|
|
1126
|
+
// capability wrap. The settings registration deliberately stays on the parent
|
|
1127
|
+
// fiber — it has to outlive the switch, or the section that turns the plugin
|
|
1128
|
+
// back on would vanish with it.
|
|
1129
|
+
const REGISTRATION_KEYS = [
|
|
1130
|
+
"enabled",
|
|
1131
|
+
"bridgeTextOnly",
|
|
1132
|
+
"bridgeExportDir",
|
|
1133
|
+
"bridgeAutoImage",
|
|
1134
|
+
"bridgePreview",
|
|
1135
|
+
];
|
|
1136
|
+
let featureFiber = null;
|
|
1137
|
+
let lastRegistrationKey = null;
|
|
1138
|
+
|
|
1139
|
+
function registrationKey(cfg) {
|
|
1140
|
+
return REGISTRATION_KEYS.map((key) => String(cfg[key])).join("|");
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
function installFeatures() {
|
|
1144
|
+
if (featureFiber !== null) return;
|
|
1145
|
+
featureFiber = ctx.plugin({
|
|
1146
|
+
name: "dsh-tool-vision:features",
|
|
1147
|
+
apply(inner) {
|
|
1148
|
+
// ── image bridge: pasted images become inspect_image hints on text-only models ──
|
|
1149
|
+
if (getConfig().bridgeTextOnly) {
|
|
1150
|
+
const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
|
|
1151
|
+
mkdir(exportDir, { recursive: true }).catch(() => {});
|
|
1152
|
+
// Root-level listener: agent-scoped waterfalls admit untagged listeners,
|
|
1153
|
+
// so one registration serves every agent (new and resumed alike) and the
|
|
1154
|
+
// agent is read from the fused payload.
|
|
1155
|
+
attachPreStepBridge(inner, getConfig, exportDir);
|
|
1156
|
+
|
|
1157
|
+
// ── automatic image admission: let pasted images through on text-only models ──
|
|
1158
|
+
// The host gate refuses images unless the model declares `image` input;
|
|
1159
|
+
// the bridge handles them anyway, so report image support for all models.
|
|
1160
|
+
// The wrap is installed on the shared llm service instance (idempotent,
|
|
1161
|
+
// restored on dispose/HMR).
|
|
1162
|
+
if (getConfig().bridgeAutoImage) {
|
|
1163
|
+
const unwrap = installAutoImageAdmission(inner.get("llm"), inner.logger);
|
|
1164
|
+
inner.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
// ── dispatch image admission: make the capability verdict actually land ──
|
|
1168
|
+
// `resolveModelInfo` above is pure admission — it never changes what
|
|
1169
|
+
// the adapter streams. The image the plugin decided to hand over
|
|
1170
|
+
// directly was still being projected into a text placeholder by
|
|
1171
|
+
// `LlmService.generate`, which reads the adapter's own declaration and
|
|
1172
|
+
// nothing else. This applies the same decision at that seam.
|
|
1173
|
+
//
|
|
1174
|
+
// Installed outside the `bridgeAutoImage` branch on purpose: the
|
|
1175
|
+
// question it answers is not "may the image be offered" but "does the
|
|
1176
|
+
// model receive it", and that is the plugin's own verdict either way.
|
|
1177
|
+
{
|
|
1178
|
+
const unwrapDispatch = installDispatchImageAdmission(inner.get("llm"), getConfig, inner.logger);
|
|
1179
|
+
inner.effect(() => unwrapDispatch, "dsh-tool-vision: dispatch image admission");
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
// ── bridge image preview: same-origin thumbnails for bridged images ──
|
|
1183
|
+
if (getConfig().bridgePreview) {
|
|
1184
|
+
registerBridgePreviewRoute(inner, exportDir, inner.logger);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
inner.tools.register(defineTool({
|
|
1189
|
+
name: "inspect_image",
|
|
1190
|
+
description: getConfig().description,
|
|
1191
|
+
parameters: {
|
|
1192
|
+
path: {
|
|
1193
|
+
type: "string",
|
|
1194
|
+
required: true,
|
|
1195
|
+
description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
|
|
1196
|
+
},
|
|
1197
|
+
question: {
|
|
1198
|
+
type: "string",
|
|
1199
|
+
description: "Optional specific question about the image. Omit for a general detailed description.",
|
|
1200
|
+
},
|
|
1201
|
+
detail: {
|
|
1202
|
+
type: "string",
|
|
1203
|
+
enum: ["auto", "low", "high"],
|
|
1204
|
+
description: "Optional image resolution hint for the vision API (auto by default).",
|
|
1205
|
+
},
|
|
1206
|
+
},
|
|
1207
|
+
output: {
|
|
1208
|
+
schema: { type: "string" },
|
|
1209
|
+
render: (_args, value) => [{ type: "text", text: value }],
|
|
1210
|
+
},
|
|
1211
|
+
async execute(args, exec) {
|
|
1212
|
+
const cfg = getConfig();
|
|
1213
|
+
const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
|
|
1214
|
+
const { url, note } = await toImageUrl(args.path, cwd, cfg);
|
|
1215
|
+
try {
|
|
1216
|
+
const answer = await callVision(cfg, url, args.question, args.detail, exec.signal, exec);
|
|
1217
|
+
return note === url ? answer : `${answer}\n\n(image: ${note})`;
|
|
1218
|
+
} catch (error) {
|
|
1219
|
+
const raw = error && error.message ? String(error.message) : String(error);
|
|
1220
|
+
if (CONTENT_FILTER_RE.test(raw)) {
|
|
1221
|
+
throw new Error(
|
|
1222
|
+
"inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
|
|
1223
|
+
"这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
|
|
1224
|
+
);
|
|
1225
|
+
}
|
|
1226
|
+
throw error;
|
|
1227
|
+
}
|
|
1228
|
+
},
|
|
1229
|
+
}));
|
|
1230
|
+
|
|
1231
|
+
// ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
|
|
1232
|
+
// 14 vision_* tools driven by the SAME configured endpoint as inspect_image
|
|
1233
|
+
// (baseURL/apiKey/model). No provider chain, no local models, no extra
|
|
1234
|
+
// settings: everything comes from the existing tool-vision configuration.
|
|
1235
|
+
registerVisionTools(inner, getConfig);
|
|
1236
|
+
|
|
1237
|
+
// ── ground-truth capability probe (v0.9.0) ───────────────────────────
|
|
1238
|
+
// Everything else decides from what a route SAYS; this measures what it
|
|
1239
|
+
// DOES, and records the answer so the bridge decision stops guessing.
|
|
1240
|
+
inner.tools.register(defineTool({
|
|
1241
|
+
name: "vision_probe_model",
|
|
1242
|
+
description:
|
|
1243
|
+
"Measure whether a model's endpoint can actually read images, by sending it a real " +
|
|
1244
|
+
"solid-color image and checking the answer. Use this when a model's declared image " +
|
|
1245
|
+
"support is doubtful (it claims `image` but the provider may reject it, or it declares " +
|
|
1246
|
+
"text-only yet the endpoint may accept images). Runs a text-only control request plus " +
|
|
1247
|
+
"two different colors, so a model that merely guesses cannot pass. The verdict is " +
|
|
1248
|
+
"recorded and from then on decides whether that route's images are sent directly or " +
|
|
1249
|
+
"bridged. Costs a few requests against the model's own endpoint.",
|
|
1250
|
+
parameters: {
|
|
1251
|
+
model: {
|
|
1252
|
+
type: "string",
|
|
1253
|
+
required: true,
|
|
1254
|
+
description: "Model id to probe, e.g. xiaomi/mimo-v2.5. Use the id exactly as dsh has it configured.",
|
|
1255
|
+
},
|
|
1256
|
+
provider: {
|
|
1257
|
+
type: "string",
|
|
1258
|
+
description:
|
|
1259
|
+
"Provider route serving the model (e.g. commandcode). Omit to use the current " +
|
|
1260
|
+
"session's route, or to auto-select when exactly one provider offers the model.",
|
|
1261
|
+
},
|
|
1262
|
+
},
|
|
1263
|
+
output: {
|
|
1264
|
+
schema: { type: "string" },
|
|
1265
|
+
render: (_args, value) => [{ type: "text", text: value }],
|
|
1266
|
+
},
|
|
1267
|
+
async execute(args, exec) {
|
|
1268
|
+
const cfg = getConfig();
|
|
1269
|
+
const requested = String(args.model ?? "").trim();
|
|
1270
|
+
if (requested.length === 0) throw new Error("vision_probe_model: model is required");
|
|
1271
|
+
const llm = inner.get("llm");
|
|
1272
|
+
|
|
1273
|
+
// Resolve which routes offer this model, so an omitted `provider`
|
|
1274
|
+
// is still an exact choice rather than a guess.
|
|
1275
|
+
let candidates = [];
|
|
1276
|
+
try {
|
|
1277
|
+
for (const entry of llm?.listProviders?.() ?? []) {
|
|
1278
|
+
const id = entry?.id;
|
|
1279
|
+
if (typeof id !== "string") continue;
|
|
1280
|
+
const models = await llm.listModels(id).catch(() => []);
|
|
1281
|
+
if ((models ?? []).some((model) => String(model?.id) === requested)) candidates.push(id);
|
|
1282
|
+
}
|
|
1283
|
+
} catch {
|
|
1284
|
+
/* fall through to the route/argument fallbacks below */
|
|
1285
|
+
}
|
|
1286
|
+
let provider = typeof args.provider === "string" && args.provider.length > 0 ? args.provider : undefined;
|
|
1287
|
+
if (provider === undefined) {
|
|
1288
|
+
const here = currentRoute(exec?.agent);
|
|
1289
|
+
if (candidates.length === 1) provider = candidates[0];
|
|
1290
|
+
else if (here.model === requested && typeof here.provider === "string") provider = here.provider;
|
|
1291
|
+
else if (candidates.length > 1) {
|
|
1292
|
+
throw new Error(
|
|
1293
|
+
`vision_probe_model: "${requested}" is served by ${candidates.join(", ")}; pass provider explicitly`,
|
|
1294
|
+
);
|
|
1295
|
+
} else {
|
|
1296
|
+
provider = here.provider;
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
if (typeof provider !== "string" || provider.length === 0) {
|
|
1300
|
+
throw new Error(
|
|
1301
|
+
`vision_probe_model: cannot tell which provider serves "${requested}"; pass provider explicitly`,
|
|
1302
|
+
);
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
const target = await resolveProbeTarget(inner, provider, requested);
|
|
1306
|
+
const outcome = await probeModelCapability(
|
|
1307
|
+
{
|
|
1308
|
+
baseURL: target.baseURL,
|
|
1309
|
+
apiKey: target.apiKey,
|
|
1310
|
+
api: target.api,
|
|
1311
|
+
model: requested,
|
|
1312
|
+
maxTokens: PROBE_MAX_TOKENS,
|
|
1313
|
+
timeoutMs: Math.max(PROBE_TIMEOUT_MS, Number(cfg.timeoutMs) || 0),
|
|
1314
|
+
headers: sessionHeaders(cfg, exec),
|
|
1315
|
+
},
|
|
1316
|
+
exec.signal,
|
|
1317
|
+
);
|
|
1318
|
+
|
|
1319
|
+
const key = probeKey(provider, requested);
|
|
1320
|
+
let recorded = "not recorded (no settings provider)";
|
|
1321
|
+
if (outcome.support === "yes" || outcome.support === "no") {
|
|
1322
|
+
const scope = getSettingsScope();
|
|
1323
|
+
if (scope !== null) {
|
|
1324
|
+
try {
|
|
1325
|
+
await scope.update({ probeResults: { ...(cfg.probeResults ?? {}), [key]: outcome.support } });
|
|
1326
|
+
recorded = `recorded as "${outcome.support}"`;
|
|
1327
|
+
} catch (error) {
|
|
1328
|
+
recorded = `could not be recorded: ${String(error?.message ?? error)}`;
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
const verdictText = outcome.support === "yes"
|
|
1334
|
+
? "SUPPORTS images"
|
|
1335
|
+
: outcome.support === "no"
|
|
1336
|
+
? "does NOT read images"
|
|
1337
|
+
: "INCONCLUSIVE (the route could not be probed)";
|
|
1338
|
+
const effect = outcome.support === "yes"
|
|
1339
|
+
? "images will be sent to it directly"
|
|
1340
|
+
: outcome.support === "no"
|
|
1341
|
+
? "images will be bridged to inspect_image hints"
|
|
1342
|
+
: "the existing list/declaration rules still apply";
|
|
1343
|
+
const evidence = Object.entries(outcome.evidence ?? {})
|
|
1344
|
+
.map(([name, value]) => ` ${name}: ${value}`)
|
|
1345
|
+
.join("\n");
|
|
1346
|
+
return [
|
|
1347
|
+
`${provider}/${requested}: ${verdictText}`,
|
|
1348
|
+
` ${outcome.detail}`,
|
|
1349
|
+
` ${recorded}; ${effect}`,
|
|
1350
|
+
evidence ? ` evidence:\n${evidence}` : "",
|
|
1351
|
+
].filter(Boolean).join("\n");
|
|
1352
|
+
},
|
|
1353
|
+
}));
|
|
1354
|
+
},
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
function uninstallFeatures() {
|
|
1359
|
+
if (featureFiber === null) return;
|
|
1360
|
+
const fiber = featureFiber;
|
|
1361
|
+
featureFiber = null;
|
|
1362
|
+
// dispose() settles asynchronously and must never take the plugin down; the
|
|
1363
|
+
// next install starts a fresh fiber regardless of how this one ends.
|
|
1364
|
+
Promise.resolve(fiber.dispose()).catch((error) => {
|
|
1365
|
+
ctx.logger?.warn?.(`[tool-vision] feature teardown failed: ${String(error)}`);
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
// Re-install only when a field that gates a registration actually changed, so
|
|
1370
|
+
// editing, say, `model` does not tear the tools down and back up mid-session.
|
|
1371
|
+
function syncFeatures() {
|
|
1372
|
+
const cfg = getConfig();
|
|
1373
|
+
const key = registrationKey(cfg);
|
|
1374
|
+
if (featureFiber !== null && key === lastRegistrationKey) return;
|
|
1375
|
+
uninstallFeatures();
|
|
1376
|
+
lastRegistrationKey = key;
|
|
1377
|
+
if (cfg.enabled) installFeatures();
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
// Compat shim: dsh-settings 0.1.2-rc.1 removed the module-level
|
|
1381
|
+
// `installSettingsSection` export (the provider now lives at ctx.settings).
|
|
1382
|
+
// Inline the same logic via ctx.inject(["settings"]) — works on both
|
|
1383
|
+
// 0.1.1 (module export wrapper) and 0.1.2 (ctx.settings) hosts.
|
|
1384
|
+
ctx.inject(["settings"], (sctx) => {
|
|
1385
|
+
const scope = sctx.settings.register(NS, Config, { base: config });
|
|
1386
|
+
sourceGetter = () => scope.get();
|
|
1387
|
+
settingsScope = scope;
|
|
1388
|
+
sctx.effect(() => () => {
|
|
1389
|
+
sourceGetter = null;
|
|
1390
|
+
settingsScope = null;
|
|
1391
|
+
});
|
|
1392
|
+
// Hot-apply: `enabled` switches the child fiber, and any field that gates a
|
|
1393
|
+
// registration re-installs it, so none of them needs a dsh restart anymore.
|
|
1394
|
+
scope.watch(() => syncFeatures());
|
|
1395
|
+
sctx.effect(() => () => uninstallFeatures());
|
|
1396
|
+
// The stored value governs from here on; the composition entry was only a
|
|
1397
|
+
// placeholder until the provider answered.
|
|
1398
|
+
syncFeatures();
|
|
1399
|
+
});
|
|
1400
|
+
|
|
1401
|
+
// ── model catalog route: settings support, so it lives on THIS fiber ───────
|
|
1402
|
+
// The master switch disposes the feature fiber on purpose (tools, bridge,
|
|
1403
|
+
// preview route), while the settings section itself deliberately stays
|
|
1404
|
+
// mounted — and a section whose list field cannot autocomplete is a section
|
|
1405
|
+
// you cannot configure before switching the plugin on. Registered here so it
|
|
1406
|
+
// outlives `enabled: false`; read-only and secret-free by construction.
|
|
1407
|
+
registerModelCatalogRoute(ctx, ctx.logger, getConfig);
|
|
1408
|
+
|
|
1409
|
+
// No settings provider: the composition entry is the whole configuration.
|
|
1410
|
+
syncFeatures();
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
export {
|
|
1414
|
+
BRIDGE_MARKER,
|
|
1415
|
+
BRIDGE_PREVIEW_MAX_BYTES,
|
|
1416
|
+
BRIDGE_PREVIEW_MEDIA,
|
|
1417
|
+
BRIDGE_PREVIEW_ROUTE,
|
|
1418
|
+
Config,
|
|
1419
|
+
DEFAULT_DESCRIPTION,
|
|
1420
|
+
DEFAULT_MULTIMODAL_LIST_MODE,
|
|
1421
|
+
EXT_BY_MEDIA,
|
|
1422
|
+
MODEL_CATALOG_MAX_PER_PROVIDER,
|
|
1423
|
+
MODEL_CATALOG_ROUTE,
|
|
1424
|
+
MULTIMODAL_LIST_MODES,
|
|
1425
|
+
apply,
|
|
1426
|
+
attachPreStepBridge,
|
|
1427
|
+
autoPromotionWarned,
|
|
1428
|
+
bridgeMessages,
|
|
1429
|
+
currentModelAcceptsImage,
|
|
1430
|
+
currentRoute,
|
|
1431
|
+
deepFreeze,
|
|
1432
|
+
exportImage,
|
|
1433
|
+
hasImageBlock,
|
|
1434
|
+
inject,
|
|
1435
|
+
installAutoImageAdmission,
|
|
1436
|
+
installDispatchImageAdmission,
|
|
1437
|
+
lastModelDecision,
|
|
1438
|
+
modelListMatches,
|
|
1439
|
+
modelListMatchEntries,
|
|
1440
|
+
name,
|
|
1441
|
+
normalizeListMode,
|
|
1442
|
+
parseQuery,
|
|
1443
|
+
probeKey,
|
|
1444
|
+
probeVerdict,
|
|
1445
|
+
registerBridgePreviewRoute,
|
|
1446
|
+
routeDirectDecision,
|
|
1447
|
+
registerModelCatalogRoute,
|
|
1448
|
+
repairLoggedImages,
|
|
1449
|
+
resolveProbeTarget,
|
|
1450
|
+
routeDeclaresImage,
|
|
1451
|
+
unwrappedResolveModelInfo,
|
|
1452
|
+
warnAutoPromotion,
|
|
1453
|
+
};
|