dsh-tool-vision 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -23
- package/README.md +4 -2
- package/README.zh.md +169 -166
- package/client.js +9 -4
- package/cordis.patch.yml +8 -8
- package/index.js +654 -654
- package/lib/vision-tools.js +2481 -2481
- package/package.json +3 -3
package/index.js
CHANGED
|
@@ -1,654 +1,654 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* dsh-tool-vision — external vision model for DeepSeek Harness.
|
|
3
|
-
*
|
|
4
|
-
* Two capabilities:
|
|
5
|
-
*
|
|
6
|
-
* 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
|
|
7
|
-
* any OpenAI-compatible chat/completions endpoint that supports
|
|
8
|
-
* `image_url` content parts, and returns the vision model's text answer.
|
|
9
|
-
*
|
|
10
|
-
* 2. Image bridge — pasted images are bridged to text hints before they
|
|
11
|
-
* enter a text-only model's request:
|
|
12
|
-
*
|
|
13
|
-
* - New images are bridged on the `agent/pre-step` waterfall (the only
|
|
14
|
-
* seam where the harness lets a plugin replace the messages that enter
|
|
15
|
-
* a step — they become the durable `user/message` log, so the
|
|
16
|
-
* `llm/stream` request-reconstruction invariant stays satisfied).
|
|
17
|
-
* - Images already logged before the plugin was installed (or before a
|
|
18
|
-
* server restart) are repaired lazily with a surface `replace`, one
|
|
19
|
-
* event at a time, on the first pre-step of the session.
|
|
20
|
-
*
|
|
21
|
-
* The bridged hint points at an exported local copy of the image, which
|
|
22
|
-
* the agent hands to `inspect_image`. Models listed in
|
|
23
|
-
* `multimodalModels` (or whose resolved `inputModalities` include
|
|
24
|
-
* "image") receive image blocks directly and are never bridged.
|
|
25
|
-
*
|
|
26
|
-
* 3. Bridge image preview (v0.4.0, contributed by xing666173 from
|
|
27
|
-
* dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
|
|
28
|
-
* inline thumbnails for bridged pasted images:
|
|
29
|
-
*
|
|
30
|
-
* - `bridgeMessages` stamps every bridged hint with an invisible marker
|
|
31
|
-
* (`BRIDGE_MARKER`), so the client can identify bridge text blocks
|
|
32
|
-
* precisely instead of pattern-matching free text.
|
|
33
|
-
* - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
|
|
34
|
-
* exported images to the same-origin page; the client inserts a
|
|
35
|
-
* thumbnail above the hint text and opens a lightbox on click.
|
|
36
|
-
* - Pure display layer: persisted messages, the transcript, the
|
|
37
|
-
* model-facing text and the `inspect_image` chain are untouched.
|
|
38
|
-
*/
|
|
39
|
-
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
40
|
-
import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
|
|
41
|
-
import os from "node:os";
|
|
42
|
-
import z from "@deepseek-ai/schemastery";
|
|
43
|
-
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
44
|
-
import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
|
|
45
|
-
import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
|
|
46
|
-
|
|
47
|
-
/** Cordis plugin name. */
|
|
48
|
-
const name = "tool-vision";
|
|
49
|
-
/** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
|
|
50
|
-
const inject = ["tools", "llm", "attachments", "webServer"];
|
|
51
|
-
/** Settings namespace owned by this plugin (Web UI settings section). */
|
|
52
|
-
const NS = settingsNamespace("tool-vision");
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Invisible prefix stamped onto every bridged hint text block. The browser
|
|
56
|
-
* half uses it to recognize bridge text precisely (no free-text regex over
|
|
57
|
-
* user messages), and the model-facing hint stays intact otherwise.
|
|
58
|
-
*/
|
|
59
|
-
const BRIDGE_MARKER = "\u200b[bridge]";
|
|
60
|
-
|
|
61
|
-
/** Loopback route serving bridged images to the same-origin page. */
|
|
62
|
-
const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
|
|
63
|
-
/** Hard cap for a single served image (defense in depth; export is bounded). */
|
|
64
|
-
const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
|
|
65
|
-
/** Extensions the preview route serves (svg/ico intentionally excluded). */
|
|
66
|
-
const BRIDGE_PREVIEW_MEDIA = {
|
|
67
|
-
".png": "image/png",
|
|
68
|
-
".jpg": "image/jpeg",
|
|
69
|
-
".jpeg": "image/jpeg",
|
|
70
|
-
".webp": "image/webp",
|
|
71
|
-
".gif": "image/gif",
|
|
72
|
-
".avif": "image/avif",
|
|
73
|
-
".bmp": "image/bmp",
|
|
74
|
-
};
|
|
75
|
-
|
|
76
|
-
const DEFAULT_DESCRIPTION =
|
|
77
|
-
"Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
|
|
78
|
-
"Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
|
|
79
|
-
"optionally with a specific question. Returns the vision model's textual description or answer. " +
|
|
80
|
-
"Use this whenever you need to read, describe, or extract information from image content, " +
|
|
81
|
-
"since the main model is text-only.";
|
|
82
|
-
|
|
83
|
-
/** Runtime schema for the tool-vision row. */
|
|
84
|
-
const Config = z.object({
|
|
85
|
-
/** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
|
|
86
|
-
baseURL: z.string().default("https://api.openai.com/v1"),
|
|
87
|
-
/** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
|
|
88
|
-
apiKey: z.string().default("").role("secret"),
|
|
89
|
-
/** Environment variable holding the API key. */
|
|
90
|
-
apiKeyEnv: z.string().default("VISION_API_KEY"),
|
|
91
|
-
/** Vision model id served by the endpoint. */
|
|
92
|
-
model: z.string().default("gpt-4o-mini"),
|
|
93
|
-
/** Max output tokens for the vision call. */
|
|
94
|
-
maxTokens: z.number().default(4096),
|
|
95
|
-
/** Per-request timeout in milliseconds. */
|
|
96
|
-
timeoutMs: z.number().default(60000),
|
|
97
|
-
/** Largest local image accepted, in bytes. */
|
|
98
|
-
maxImageBytes: z.number().default(10 * 1024 * 1024),
|
|
99
|
-
/** Tool description shown to the model; overrides the default. */
|
|
100
|
-
description: z.string().default(DEFAULT_DESCRIPTION),
|
|
101
|
-
/** Bridge pasted images to text hints on models that cannot see images. */
|
|
102
|
-
bridgeTextOnly: z.boolean().default(true),
|
|
103
|
-
/** Export directory for bridged images; empty = system temp. */
|
|
104
|
-
bridgeExportDir: z.string().default(""),
|
|
105
|
-
/** Model ids that receive image blocks directly (never bridged). */
|
|
106
|
-
multimodalModels: z.array(z.string()).default([]),
|
|
107
|
-
/** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
|
|
108
|
-
bridgePreview: z.boolean().default(true),
|
|
109
|
-
/** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
|
|
110
|
-
bridgePreviewScanIntervalMs: z.number().default(2000),
|
|
111
|
-
/** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
|
|
112
|
-
bridgePreviewHideHint: z.boolean().default(true),
|
|
113
|
-
/** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
|
|
114
|
-
desktopScreenshot: z.boolean().default(false),
|
|
115
|
-
/**
|
|
116
|
-
* Advertise image input capability for every model while the bridge is on.
|
|
117
|
-
* The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
|
|
118
|
-
* pasted images unless the current model declares `image` in its
|
|
119
|
-
* `inputModalities`. The bridge handles those images anyway (they become
|
|
120
|
-
* text hints the agent inspects through `inspect_image`), so this wraps the
|
|
121
|
-
* llm service's `resolveModelInfo` to report image support for text-only
|
|
122
|
-
* models too — users can paste images on any model without hand-editing
|
|
123
|
-
* provider model configs.
|
|
124
|
-
*/
|
|
125
|
-
bridgeAutoImage: z.boolean().default(true),
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
const MIME_BY_EXT = {
|
|
129
|
-
".png": "image/png",
|
|
130
|
-
".jpg": "image/jpeg",
|
|
131
|
-
".jpeg": "image/jpeg",
|
|
132
|
-
".gif": "image/gif",
|
|
133
|
-
".webp": "image/webp",
|
|
134
|
-
".bmp": "image/bmp",
|
|
135
|
-
".avif": "image/avif",
|
|
136
|
-
".svg": "image/svg+xml",
|
|
137
|
-
".ico": "image/x-icon",
|
|
138
|
-
};
|
|
139
|
-
|
|
140
|
-
const EXT_BY_MEDIA = {
|
|
141
|
-
"image/png": ".png",
|
|
142
|
-
"image/jpeg": ".jpg",
|
|
143
|
-
"image/webp": ".webp",
|
|
144
|
-
"image/gif": ".gif",
|
|
145
|
-
};
|
|
146
|
-
|
|
147
|
-
/** True when any message carries an image content block. */
|
|
148
|
-
function hasImageBlock(messages) {
|
|
149
|
-
return (messages ?? []).some((m) =>
|
|
150
|
-
Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
|
|
155
|
-
function deepFreeze(value) {
|
|
156
|
-
if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
|
|
157
|
-
if (Array.isArray(value)) {
|
|
158
|
-
for (const item of value) deepFreeze(item);
|
|
159
|
-
return Object.freeze(value);
|
|
160
|
-
}
|
|
161
|
-
for (const key of Object.keys(value)) deepFreeze(value[key]);
|
|
162
|
-
return Object.freeze(value);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/** Export one attachment to disk; returns the file path (cached per process). */
|
|
166
|
-
const exportedPaths = new Map();
|
|
167
|
-
async function exportImage(attachment, ctx, dir) {
|
|
168
|
-
const cached = exportedPaths.get(attachment.attachmentId);
|
|
169
|
-
if (cached) return cached;
|
|
170
|
-
const { data } = await ctx.attachments.readImage(attachment);
|
|
171
|
-
const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
|
|
172
|
-
const safeName = attachment.name
|
|
173
|
-
? attachment.name
|
|
174
|
-
.replace(/\.[^.]+$/, "")
|
|
175
|
-
.replace(/[^\w\-]+/g, "_")
|
|
176
|
-
.replace(/^_+|_+$/g, "")
|
|
177
|
-
.slice(0, 40)
|
|
178
|
-
: "";
|
|
179
|
-
const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
|
|
180
|
-
const path = join(dir, `${base}${ext}`);
|
|
181
|
-
await writeFile(path, data);
|
|
182
|
-
exportedPaths.set(attachment.attachmentId, path);
|
|
183
|
-
return path;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/**
|
|
187
|
-
* Replace image content blocks with text hints pointing at exported files.
|
|
188
|
-
* Non-image messages are returned as-is (same reference); bridged messages
|
|
189
|
-
* are fresh, deep-frozen objects with the original identity and source.
|
|
190
|
-
* Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
|
|
191
|
-
* prefix so the browser half can recognize it precisely.
|
|
192
|
-
* Exported for unit testing; `ctx` only needs `attachments`.
|
|
193
|
-
*/
|
|
194
|
-
async function bridgeMessages(messages, ctx, dir) {
|
|
195
|
-
const next = [];
|
|
196
|
-
for (const message of messages) {
|
|
197
|
-
const content = message?.content;
|
|
198
|
-
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
199
|
-
next.push(message);
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
const blocks = [];
|
|
203
|
-
for (const block of content) {
|
|
204
|
-
if (block?.type !== "image") {
|
|
205
|
-
blocks.push(block);
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
208
|
-
const path = await exportImage(block.attachment, ctx, dir);
|
|
209
|
-
const name = block.attachment.name ? ` (${block.attachment.name})` : "";
|
|
210
|
-
blocks.push({
|
|
211
|
-
type: "text",
|
|
212
|
-
text:
|
|
213
|
-
`${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
|
|
214
|
-
`Inspect it with the inspect_image tool to see its content.]`,
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
next.push(deepFreeze({ ...message, content: blocks }));
|
|
218
|
-
}
|
|
219
|
-
return next;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
|
|
223
|
-
function parseQuery(rawUrl) {
|
|
224
|
-
const query = {};
|
|
225
|
-
const at = rawUrl.indexOf("?");
|
|
226
|
-
if (at === -1) return query;
|
|
227
|
-
for (const pair of rawUrl.slice(at + 1).split("&")) {
|
|
228
|
-
const eq = pair.indexOf("=");
|
|
229
|
-
if (eq === -1) continue;
|
|
230
|
-
try {
|
|
231
|
-
query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
|
|
232
|
-
} catch {
|
|
233
|
-
/* skip malformed pairs */
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
return query;
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
/**
|
|
240
|
-
* Advertise `image` in the `inputModalities` reported by the llm service for
|
|
241
|
-
* every model, so the host admission gate lets pasted images through on
|
|
242
|
-
* text-only models. The bridge turns those images into text hints anyway, so
|
|
243
|
-
* this is pure admission: it never changes what the adapter actually streams
|
|
244
|
-
* (the adapter's own stream validation reads the model's real `input` from
|
|
245
|
-
* the provider config, untouched here).
|
|
246
|
-
*
|
|
247
|
-
* The wrap is installed on the shared llm service instance, so it must be
|
|
248
|
-
* idempotent across HMR re-applies and restored on dispose. `marker` is a
|
|
249
|
-
* per-instance record proving this plugin already owns the wrap; the
|
|
250
|
-
* `installed` flag distinguishes "we wrapped it" from "the original was
|
|
251
|
-
* replaced by something else" so dispose only restores what we replaced.
|
|
252
|
-
*
|
|
253
|
-
* Exported for unit testing; the test passes a fake llm service.
|
|
254
|
-
*/
|
|
255
|
-
const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
|
|
256
|
-
function installAutoImageAdmission(llm, logger) {
|
|
257
|
-
if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
|
|
258
|
-
logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
|
|
259
|
-
return () => {};
|
|
260
|
-
}
|
|
261
|
-
if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
|
|
262
|
-
const original = llm.resolveModelInfo.bind(llm);
|
|
263
|
-
const wrapped = async (provider, model, signal) => {
|
|
264
|
-
const info = await original(provider, model, signal);
|
|
265
|
-
if (!info) return info;
|
|
266
|
-
const mods = info.inputModalities;
|
|
267
|
-
if (Array.isArray(mods) && mods.includes("image")) return info;
|
|
268
|
-
return { ...info, inputModalities: [...(mods ?? []), "image"] };
|
|
269
|
-
};
|
|
270
|
-
let installed = false;
|
|
271
|
-
llm[LLM_RESOLVE_WRAP_MARK] = true;
|
|
272
|
-
llm.resolveModelInfo = wrapped;
|
|
273
|
-
installed = true;
|
|
274
|
-
logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
|
|
275
|
-
return () => {
|
|
276
|
-
if (installed) {
|
|
277
|
-
if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
|
|
278
|
-
installed = false;
|
|
279
|
-
}
|
|
280
|
-
delete llm[LLM_RESOLVE_WRAP_MARK];
|
|
281
|
-
};
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Register the loopback route that serves bridged images to the same-origin
|
|
286
|
-
* page (the preview thumbnails). Read-only and tightly scoped:
|
|
287
|
-
* - only files inside the bridge export directory (no traversal);
|
|
288
|
-
* - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
|
|
289
|
-
* - Host restricted to the local machine;
|
|
290
|
-
* - hard 20MB cap per file.
|
|
291
|
-
*/
|
|
292
|
-
function registerBridgePreviewRoute(ctx, exportDir, logger) {
|
|
293
|
-
const webServer = ctx.get("webServer");
|
|
294
|
-
if (webServer === undefined) {
|
|
295
|
-
logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
|
|
296
|
-
return;
|
|
297
|
-
}
|
|
298
|
-
const bridgeDir = resolvePath(exportDir);
|
|
299
|
-
ctx.effect(() => webServer.register({
|
|
300
|
-
kind: "exact",
|
|
301
|
-
path: BRIDGE_PREVIEW_ROUTE,
|
|
302
|
-
async handler(req, res) {
|
|
303
|
-
try {
|
|
304
|
-
const raw = String(req.url ?? "");
|
|
305
|
-
const query = parseQuery(raw);
|
|
306
|
-
const p = query.p;
|
|
307
|
-
if (typeof p !== "string" || p.length === 0) {
|
|
308
|
-
res.writeHead(400);
|
|
309
|
-
res.end("bad request");
|
|
310
|
-
return;
|
|
311
|
-
}
|
|
312
|
-
const lower = p.toLowerCase();
|
|
313
|
-
const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
|
|
314
|
-
if (mediaType === undefined) {
|
|
315
|
-
res.writeHead(400);
|
|
316
|
-
res.end("not an image path");
|
|
317
|
-
return;
|
|
318
|
-
}
|
|
319
|
-
const host = String(req.headers?.host ?? "");
|
|
320
|
-
if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
|
|
321
|
-
res.writeHead(403);
|
|
322
|
-
res.end("forbidden");
|
|
323
|
-
return;
|
|
324
|
-
}
|
|
325
|
-
const target = resolvePath(p);
|
|
326
|
-
if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
|
|
327
|
-
res.writeHead(403);
|
|
328
|
-
res.end("forbidden");
|
|
329
|
-
return;
|
|
330
|
-
}
|
|
331
|
-
const info = await stat(target);
|
|
332
|
-
if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
|
|
333
|
-
res.writeHead(404);
|
|
334
|
-
res.end("not found");
|
|
335
|
-
return;
|
|
336
|
-
}
|
|
337
|
-
const bytes = await readFile(target);
|
|
338
|
-
res.writeHead(200, {
|
|
339
|
-
"Content-Type": mediaType,
|
|
340
|
-
"Cache-Control": "private, max-age=60",
|
|
341
|
-
});
|
|
342
|
-
res.end(bytes);
|
|
343
|
-
} catch {
|
|
344
|
-
try {
|
|
345
|
-
res.writeHead(404);
|
|
346
|
-
res.end("not found");
|
|
347
|
-
} catch {
|
|
348
|
-
/* response already sent */
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
},
|
|
352
|
-
}), "dsh-tool-vision: bridge preview route");
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
/**
|
|
356
|
-
* Whether the session's current model may receive image blocks directly.
|
|
357
|
-
* Uses the last logged request header first, then the agent's own options,
|
|
358
|
-
* and consults ONLY the `multimodalModels` whitelist — never the model's
|
|
359
|
-
* declared `inputModalities`, because profiles routinely declare
|
|
360
|
-
* `input: [text, image]` on text-only models to pass the harness's prompt
|
|
361
|
-
* admission check (that declaration says nothing about whether the upstream
|
|
362
|
-
* endpoint really accepts `image_url` parts).
|
|
363
|
-
* Returns true when bridging is disabled (nothing would be bridged anyway).
|
|
364
|
-
*/
|
|
365
|
-
async function currentModelAcceptsImage(agent, config) {
|
|
366
|
-
if (!config.bridgeTextOnly) return true;
|
|
367
|
-
const header = agent?.session?.requestHeader?.();
|
|
368
|
-
const model = header?.config?.model ?? agent?.options?.model;
|
|
369
|
-
if (!model) return false;
|
|
370
|
-
return config.multimodalModels.includes(model);
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
/**
|
|
374
|
-
* Lazily bridge image blocks that are already part of the session log
|
|
375
|
-
* (pasted before the plugin was active, or before a restart). Each affected
|
|
376
|
-
* event is rewritten once with a surface `replace`, which swaps the durable
|
|
377
|
-
* derivation (and the transcript) to the text hint. Events that are no
|
|
378
|
-
* longer on the surface (already shadowed) are skipped and remembered.
|
|
379
|
-
* `repaired` tracks per-session state: a `Set` of handled seqs plus a
|
|
380
|
-
* monotonic scan cursor.
|
|
381
|
-
*/
|
|
382
|
-
async function repairLoggedImages(ctx, session, exportDir, repaired) {
|
|
383
|
-
const events = session.events;
|
|
384
|
-
for (let index = repaired.cursor; index < events.length; index += 1) {
|
|
385
|
-
const event = events[index];
|
|
386
|
-
if (event.type !== "user/message" || repaired.set.has(event.seq)) {
|
|
387
|
-
repaired.set.add(event.seq);
|
|
388
|
-
continue;
|
|
389
|
-
}
|
|
390
|
-
const content = event.data?.content;
|
|
391
|
-
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
392
|
-
repaired.set.add(event.seq);
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
395
|
-
const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
|
|
396
|
-
try {
|
|
397
|
-
session.append("user/message", bridged, {
|
|
398
|
-
surfaceOp: { op: "replace", start: event.seq, end: event.seq },
|
|
399
|
-
sourceEventSeqs: [event.seq],
|
|
400
|
-
});
|
|
401
|
-
ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
|
|
402
|
-
} catch (error) {
|
|
403
|
-
ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
|
|
404
|
-
}
|
|
405
|
-
repaired.set.add(event.seq);
|
|
406
|
-
}
|
|
407
|
-
repaired.cursor = events.length;
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
/**
|
|
411
|
-
* Install the pre-step bridge at the root level. Agent-scoped waterfalls
|
|
412
|
-
* admit untagged (root) listeners, so one listener serves every agent —
|
|
413
|
-
* including sessions resumed after a server restart, which never re-fire
|
|
414
|
-
* `session/created` for per-agent attachments. Runs before every proposed
|
|
415
|
-
* step: new pasted images are bridged into the durable log, and stuck
|
|
416
|
-
* logged images are repaired, before the model request is derived from it.
|
|
417
|
-
*/
|
|
418
|
-
function attachPreStepBridge(ctx, getConfig, exportDir) {
|
|
419
|
-
const repairedBySession = new Map();
|
|
420
|
-
ctx.on("agent/pre-step", async (payload, next) => {
|
|
421
|
-
const decision = await next();
|
|
422
|
-
if (!decision || decision.kind !== "enter") return decision;
|
|
423
|
-
const agent = payload?.agent;
|
|
424
|
-
if (!agent?.session) return decision;
|
|
425
|
-
try {
|
|
426
|
-
const acceptsImage = await currentModelAcceptsImage(agent, getConfig());
|
|
427
|
-
if (!acceptsImage) {
|
|
428
|
-
let repaired = repairedBySession.get(agent.session.id);
|
|
429
|
-
if (!repaired) {
|
|
430
|
-
repaired = { set: new Set(), cursor: 0 };
|
|
431
|
-
repairedBySession.set(agent.session.id, repaired);
|
|
432
|
-
}
|
|
433
|
-
await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
|
|
434
|
-
ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
|
|
435
|
-
});
|
|
436
|
-
}
|
|
437
|
-
if (acceptsImage) return decision;
|
|
438
|
-
const messages = await bridgeMessages(decision.messages, ctx, exportDir);
|
|
439
|
-
if (messages.every((message, index) => message === decision.messages[index])) return decision;
|
|
440
|
-
return { ...decision, messages };
|
|
441
|
-
} catch (error) {
|
|
442
|
-
ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
|
|
443
|
-
return decision;
|
|
444
|
-
}
|
|
445
|
-
});
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
function resolveApiKey(config) {
|
|
449
|
-
if (config.apiKey) return config.apiKey;
|
|
450
|
-
if (config.apiKeyEnv) {
|
|
451
|
-
const fromEnv = process.env[config.apiKeyEnv];
|
|
452
|
-
if (fromEnv) return fromEnv;
|
|
453
|
-
}
|
|
454
|
-
return process.env.OPENAI_API_KEY ?? "";
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
/** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
|
|
458
|
-
async function toImageUrl(target, cwd, config) {
|
|
459
|
-
if (/^https?:\/\//i.test(target)) return { url: target, note: target };
|
|
460
|
-
const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
|
|
461
|
-
const info = await stat(abs).catch(() => null);
|
|
462
|
-
if (!info) throw new Error(`image not found: ${abs}`);
|
|
463
|
-
if (info.size > config.maxImageBytes) {
|
|
464
|
-
throw new Error(
|
|
465
|
-
`image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
|
|
466
|
-
);
|
|
467
|
-
}
|
|
468
|
-
const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
|
|
469
|
-
if (!mime) {
|
|
470
|
-
throw new Error(
|
|
471
|
-
`unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
|
|
472
|
-
);
|
|
473
|
-
}
|
|
474
|
-
const data = await readFile(abs);
|
|
475
|
-
return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
/** One OpenAI-compatible chat/completions call with an image_url content part. */
|
|
479
|
-
async function callVision(config, imageUrl, question, detail, signal) {
|
|
480
|
-
const key = resolveApiKey(config);
|
|
481
|
-
if (!key) {
|
|
482
|
-
throw new Error(
|
|
483
|
-
`vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
|
|
484
|
-
);
|
|
485
|
-
}
|
|
486
|
-
const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
|
|
487
|
-
const endpoint = new URL("chat/completions", base);
|
|
488
|
-
const controller = new AbortController();
|
|
489
|
-
const timer = setTimeout(
|
|
490
|
-
() => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
|
|
491
|
-
config.timeoutMs,
|
|
492
|
-
);
|
|
493
|
-
const onSignalAbort = () => controller.abort(signal?.reason);
|
|
494
|
-
if (signal) {
|
|
495
|
-
if (signal.aborted) controller.abort();
|
|
496
|
-
else signal.addEventListener("abort", onSignalAbort, { once: true });
|
|
497
|
-
}
|
|
498
|
-
const content = [
|
|
499
|
-
{ type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
|
|
500
|
-
{ type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
|
|
501
|
-
];
|
|
502
|
-
try {
|
|
503
|
-
const response = await fetch(endpoint, {
|
|
504
|
-
method: "POST",
|
|
505
|
-
headers: {
|
|
506
|
-
"Content-Type": "application/json",
|
|
507
|
-
Authorization: `Bearer ${key}`,
|
|
508
|
-
},
|
|
509
|
-
body: JSON.stringify({
|
|
510
|
-
model: config.model,
|
|
511
|
-
messages: [{ role: "user", content }],
|
|
512
|
-
max_tokens: config.maxTokens,
|
|
513
|
-
}),
|
|
514
|
-
signal: controller.signal,
|
|
515
|
-
});
|
|
516
|
-
const body = await response.json().catch(() => null);
|
|
517
|
-
if (!response.ok) {
|
|
518
|
-
const detailText = body?.error?.message ?? response.statusText;
|
|
519
|
-
throw new Error(
|
|
520
|
-
`vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
|
|
521
|
-
);
|
|
522
|
-
}
|
|
523
|
-
// Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
|
|
524
|
-
// `reasoning_content` first; when the final `content` is empty or was cut
|
|
525
|
-
// off by max_tokens, fall back to the reasoning text so the answer is
|
|
526
|
-
// still useful.
|
|
527
|
-
const message = body?.choices?.[0]?.message;
|
|
528
|
-
let answer = message?.content ?? "";
|
|
529
|
-
if (!answer.trim()) answer = message?.reasoning_content ?? "";
|
|
530
|
-
if (typeof answer !== "string" || !answer.trim()) {
|
|
531
|
-
throw new Error("vision endpoint returned an empty response");
|
|
532
|
-
}
|
|
533
|
-
return answer.trim();
|
|
534
|
-
} finally {
|
|
535
|
-
clearTimeout(timer);
|
|
536
|
-
signal?.removeEventListener("abort", onSignalAbort);
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
function apply(ctx, config) {
|
|
541
|
-
// ── settings-backed configuration ─────────────────────────────────────────
|
|
542
|
-
// The composition entry stays the `base` layer; a registered `tool-vision`
|
|
543
|
-
// settings section (Web UI section, settings.yaml) overlays it live, so
|
|
544
|
-
// edits hot-apply without a restart. `installSettingsSection` hands
|
|
545
|
-
// `setSource` a GETTER (`() => scope.get()`), not the config object — keep
|
|
546
|
-
// it and call it at use time, or `getConfig()` would return a function and
|
|
547
|
-
// every `cfg.*` read would be undefined (apiKey included).
|
|
548
|
-
let current = config;
|
|
549
|
-
let sourceGetter = null;
|
|
550
|
-
const getConfig = () => (sourceGetter ? sourceGetter() : current);
|
|
551
|
-
installSettingsSection(ctx, NS, Config, config, {
|
|
552
|
-
setSource: (getter) => {
|
|
553
|
-
sourceGetter = getter;
|
|
554
|
-
},
|
|
555
|
-
onChange: () => {},
|
|
556
|
-
});
|
|
557
|
-
|
|
558
|
-
// ── image bridge: pasted images become inspect_image hints on text-only models ──
|
|
559
|
-
if (getConfig().bridgeTextOnly) {
|
|
560
|
-
const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
|
|
561
|
-
mkdir(exportDir, { recursive: true }).catch(() => {});
|
|
562
|
-
// Root-level listener: agent-scoped waterfalls admit untagged listeners,
|
|
563
|
-
// so one registration serves every agent (new and resumed alike) and the
|
|
564
|
-
// agent is read from the fused payload.
|
|
565
|
-
attachPreStepBridge(ctx, getConfig, exportDir);
|
|
566
|
-
|
|
567
|
-
// ── automatic image admission: let pasted images through on text-only models ──
|
|
568
|
-
// The host gate refuses images unless the model declares `image` input;
|
|
569
|
-
// the bridge handles them anyway, so report image support for all models.
|
|
570
|
-
// The wrap is installed on the shared llm service instance (idempotent,
|
|
571
|
-
// restored on dispose/HMR).
|
|
572
|
-
if (getConfig().bridgeAutoImage) {
|
|
573
|
-
const unwrap = installAutoImageAdmission(ctx.get("llm"), ctx.logger);
|
|
574
|
-
ctx.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
// ── bridge image preview: same-origin thumbnails for bridged images ──
|
|
578
|
-
if (getConfig().bridgePreview) {
|
|
579
|
-
registerBridgePreviewRoute(ctx, exportDir, ctx.logger);
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
ctx.tools.register(defineTool({
|
|
584
|
-
name: "inspect_image",
|
|
585
|
-
description: getConfig().description,
|
|
586
|
-
parameters: {
|
|
587
|
-
path: {
|
|
588
|
-
type: "string",
|
|
589
|
-
required: true,
|
|
590
|
-
description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
|
|
591
|
-
},
|
|
592
|
-
question: {
|
|
593
|
-
type: "string",
|
|
594
|
-
description: "Optional specific question about the image. Omit for a general detailed description.",
|
|
595
|
-
},
|
|
596
|
-
detail: {
|
|
597
|
-
type: "string",
|
|
598
|
-
enum: ["auto", "low", "high"],
|
|
599
|
-
description: "Optional image resolution hint for the vision API (auto by default).",
|
|
600
|
-
},
|
|
601
|
-
},
|
|
602
|
-
output: {
|
|
603
|
-
schema: { type: "string" },
|
|
604
|
-
render: (_args, value) => [{ type: "text", text: value }],
|
|
605
|
-
},
|
|
606
|
-
async execute(args, exec) {
|
|
607
|
-
const cfg = getConfig();
|
|
608
|
-
const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
|
|
609
|
-
const { url, note } = await toImageUrl(args.path, cwd, cfg);
|
|
610
|
-
try {
|
|
611
|
-
const answer = await callVision(cfg, url, args.question, args.detail, exec.signal);
|
|
612
|
-
return note === url ? answer : `${answer}\n\n(image: ${note})`;
|
|
613
|
-
} catch (error) {
|
|
614
|
-
const raw = error && error.message ? String(error.message) : String(error);
|
|
615
|
-
if (CONTENT_FILTER_RE.test(raw)) {
|
|
616
|
-
throw new Error(
|
|
617
|
-
"inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
|
|
618
|
-
"这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
|
|
619
|
-
);
|
|
620
|
-
}
|
|
621
|
-
throw error;
|
|
622
|
-
}
|
|
623
|
-
},
|
|
624
|
-
}));
|
|
625
|
-
|
|
626
|
-
// ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
|
|
627
|
-
// 14 vision_* tools driven by the SAME configured endpoint as inspect_image
|
|
628
|
-
// (baseURL/apiKey/model). No provider chain, no local models, no extra
|
|
629
|
-
// settings: everything comes from the existing tool-vision configuration.
|
|
630
|
-
registerVisionTools(ctx, getConfig);
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
export {
|
|
634
|
-
BRIDGE_MARKER,
|
|
635
|
-
BRIDGE_PREVIEW_MAX_BYTES,
|
|
636
|
-
BRIDGE_PREVIEW_MEDIA,
|
|
637
|
-
BRIDGE_PREVIEW_ROUTE,
|
|
638
|
-
Config,
|
|
639
|
-
DEFAULT_DESCRIPTION,
|
|
640
|
-
EXT_BY_MEDIA,
|
|
641
|
-
apply,
|
|
642
|
-
attachPreStepBridge,
|
|
643
|
-
bridgeMessages,
|
|
644
|
-
currentModelAcceptsImage,
|
|
645
|
-
deepFreeze,
|
|
646
|
-
exportImage,
|
|
647
|
-
hasImageBlock,
|
|
648
|
-
inject,
|
|
649
|
-
installAutoImageAdmission,
|
|
650
|
-
name,
|
|
651
|
-
parseQuery,
|
|
652
|
-
registerBridgePreviewRoute,
|
|
653
|
-
repairLoggedImages,
|
|
654
|
-
};
|
|
1
|
+
/**
|
|
2
|
+
* dsh-tool-vision — external vision model for DeepSeek Harness.
|
|
3
|
+
*
|
|
4
|
+
* Two capabilities:
|
|
5
|
+
*
|
|
6
|
+
* 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
|
|
7
|
+
* any OpenAI-compatible chat/completions endpoint that supports
|
|
8
|
+
* `image_url` content parts, and returns the vision model's text answer.
|
|
9
|
+
*
|
|
10
|
+
* 2. Image bridge — pasted images are bridged to text hints before they
|
|
11
|
+
* enter a text-only model's request:
|
|
12
|
+
*
|
|
13
|
+
* - New images are bridged on the `agent/pre-step` waterfall (the only
|
|
14
|
+
* seam where the harness lets a plugin replace the messages that enter
|
|
15
|
+
* a step — they become the durable `user/message` log, so the
|
|
16
|
+
* `llm/stream` request-reconstruction invariant stays satisfied).
|
|
17
|
+
* - Images already logged before the plugin was installed (or before a
|
|
18
|
+
* server restart) are repaired lazily with a surface `replace`, one
|
|
19
|
+
* event at a time, on the first pre-step of the session.
|
|
20
|
+
*
|
|
21
|
+
* The bridged hint points at an exported local copy of the image, which
|
|
22
|
+
* the agent hands to `inspect_image`. Models listed in
|
|
23
|
+
* `multimodalModels` (or whose resolved `inputModalities` include
|
|
24
|
+
* "image") receive image blocks directly and are never bridged.
|
|
25
|
+
*
|
|
26
|
+
* 3. Bridge image preview (v0.4.0, contributed by xing666173 from
|
|
27
|
+
* dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
|
|
28
|
+
* inline thumbnails for bridged pasted images:
|
|
29
|
+
*
|
|
30
|
+
* - `bridgeMessages` stamps every bridged hint with an invisible marker
|
|
31
|
+
* (`BRIDGE_MARKER`), so the client can identify bridge text blocks
|
|
32
|
+
* precisely instead of pattern-matching free text.
|
|
33
|
+
* - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
|
|
34
|
+
* exported images to the same-origin page; the client inserts a
|
|
35
|
+
* thumbnail above the hint text and opens a lightbox on click.
|
|
36
|
+
* - Pure display layer: persisted messages, the transcript, the
|
|
37
|
+
* model-facing text and the `inspect_image` chain are untouched.
|
|
38
|
+
*/
|
|
39
|
+
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
40
|
+
import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
|
|
41
|
+
import os from "node:os";
|
|
42
|
+
import z from "@deepseek-ai/schemastery";
|
|
43
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
44
|
+
import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
|
|
45
|
+
import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
|
|
46
|
+
|
|
47
|
+
/** Cordis plugin name. */
|
|
48
|
+
const name = "tool-vision";
|
|
49
|
+
/** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
|
|
50
|
+
const inject = ["tools", "llm", "attachments", "webServer"];
|
|
51
|
+
/** Settings namespace owned by this plugin (Web UI settings section). */
|
|
52
|
+
const NS = settingsNamespace("tool-vision");
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Invisible prefix stamped onto every bridged hint text block. The browser
|
|
56
|
+
* half uses it to recognize bridge text precisely (no free-text regex over
|
|
57
|
+
* user messages), and the model-facing hint stays intact otherwise.
|
|
58
|
+
*/
|
|
59
|
+
const BRIDGE_MARKER = "\u200b[bridge]";
|
|
60
|
+
|
|
61
|
+
/** Loopback route serving bridged images to the same-origin page. */
|
|
62
|
+
const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
|
|
63
|
+
/** Hard cap for a single served image (defense in depth; export is bounded). */
|
|
64
|
+
const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
|
|
65
|
+
/** Extensions the preview route serves (svg/ico intentionally excluded). */
|
|
66
|
+
const BRIDGE_PREVIEW_MEDIA = {
|
|
67
|
+
".png": "image/png",
|
|
68
|
+
".jpg": "image/jpeg",
|
|
69
|
+
".jpeg": "image/jpeg",
|
|
70
|
+
".webp": "image/webp",
|
|
71
|
+
".gif": "image/gif",
|
|
72
|
+
".avif": "image/avif",
|
|
73
|
+
".bmp": "image/bmp",
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const DEFAULT_DESCRIPTION =
|
|
77
|
+
"Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
|
|
78
|
+
"Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
|
|
79
|
+
"optionally with a specific question. Returns the vision model's textual description or answer. " +
|
|
80
|
+
"Use this whenever you need to read, describe, or extract information from image content, " +
|
|
81
|
+
"since the main model is text-only.";
|
|
82
|
+
|
|
83
|
+
/** Runtime schema for the tool-vision row. */
|
|
84
|
+
const Config = z.object({
|
|
85
|
+
/** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
|
|
86
|
+
baseURL: z.string().default("https://api.openai.com/v1"),
|
|
87
|
+
/** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
|
|
88
|
+
apiKey: z.string().default("").role("secret"),
|
|
89
|
+
/** Environment variable holding the API key. */
|
|
90
|
+
apiKeyEnv: z.string().default("VISION_API_KEY"),
|
|
91
|
+
/** Vision model id served by the endpoint. */
|
|
92
|
+
model: z.string().default("gpt-4o-mini"),
|
|
93
|
+
/** Max output tokens for the vision call. */
|
|
94
|
+
maxTokens: z.number().default(4096),
|
|
95
|
+
/** Per-request timeout in milliseconds. */
|
|
96
|
+
timeoutMs: z.number().default(60000),
|
|
97
|
+
/** Largest local image accepted, in bytes. */
|
|
98
|
+
maxImageBytes: z.number().default(10 * 1024 * 1024),
|
|
99
|
+
/** Tool description shown to the model; overrides the default. */
|
|
100
|
+
description: z.string().default(DEFAULT_DESCRIPTION),
|
|
101
|
+
/** Bridge pasted images to text hints on models that cannot see images. */
|
|
102
|
+
bridgeTextOnly: z.boolean().default(true),
|
|
103
|
+
/** Export directory for bridged images; empty = system temp. */
|
|
104
|
+
bridgeExportDir: z.string().default(""),
|
|
105
|
+
/** Model ids that receive image blocks directly (never bridged). */
|
|
106
|
+
multimodalModels: z.array(z.string()).default([]),
|
|
107
|
+
/** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
|
|
108
|
+
bridgePreview: z.boolean().default(true),
|
|
109
|
+
/** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
|
|
110
|
+
bridgePreviewScanIntervalMs: z.number().default(2000),
|
|
111
|
+
/** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
|
|
112
|
+
bridgePreviewHideHint: z.boolean().default(true),
|
|
113
|
+
/** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
|
|
114
|
+
desktopScreenshot: z.boolean().default(false),
|
|
115
|
+
/**
|
|
116
|
+
* Advertise image input capability for every model while the bridge is on.
|
|
117
|
+
* The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
|
|
118
|
+
* pasted images unless the current model declares `image` in its
|
|
119
|
+
* `inputModalities`. The bridge handles those images anyway (they become
|
|
120
|
+
* text hints the agent inspects through `inspect_image`), so this wraps the
|
|
121
|
+
* llm service's `resolveModelInfo` to report image support for text-only
|
|
122
|
+
* models too — users can paste images on any model without hand-editing
|
|
123
|
+
* provider model configs.
|
|
124
|
+
*/
|
|
125
|
+
bridgeAutoImage: z.boolean().default(true),
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
const MIME_BY_EXT = {
|
|
129
|
+
".png": "image/png",
|
|
130
|
+
".jpg": "image/jpeg",
|
|
131
|
+
".jpeg": "image/jpeg",
|
|
132
|
+
".gif": "image/gif",
|
|
133
|
+
".webp": "image/webp",
|
|
134
|
+
".bmp": "image/bmp",
|
|
135
|
+
".avif": "image/avif",
|
|
136
|
+
".svg": "image/svg+xml",
|
|
137
|
+
".ico": "image/x-icon",
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
const EXT_BY_MEDIA = {
|
|
141
|
+
"image/png": ".png",
|
|
142
|
+
"image/jpeg": ".jpg",
|
|
143
|
+
"image/webp": ".webp",
|
|
144
|
+
"image/gif": ".gif",
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/** True when any message carries an image content block. */
|
|
148
|
+
function hasImageBlock(messages) {
|
|
149
|
+
return (messages ?? []).some((m) =>
|
|
150
|
+
Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
|
|
155
|
+
function deepFreeze(value) {
|
|
156
|
+
if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
|
|
157
|
+
if (Array.isArray(value)) {
|
|
158
|
+
for (const item of value) deepFreeze(item);
|
|
159
|
+
return Object.freeze(value);
|
|
160
|
+
}
|
|
161
|
+
for (const key of Object.keys(value)) deepFreeze(value[key]);
|
|
162
|
+
return Object.freeze(value);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Export one attachment to disk; returns the file path (cached per process). */
|
|
166
|
+
const exportedPaths = new Map();
|
|
167
|
+
async function exportImage(attachment, ctx, dir) {
|
|
168
|
+
const cached = exportedPaths.get(attachment.attachmentId);
|
|
169
|
+
if (cached) return cached;
|
|
170
|
+
const { data } = await ctx.attachments.readImage(attachment);
|
|
171
|
+
const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
|
|
172
|
+
const safeName = attachment.name
|
|
173
|
+
? attachment.name
|
|
174
|
+
.replace(/\.[^.]+$/, "")
|
|
175
|
+
.replace(/[^\w\-]+/g, "_")
|
|
176
|
+
.replace(/^_+|_+$/g, "")
|
|
177
|
+
.slice(0, 40)
|
|
178
|
+
: "";
|
|
179
|
+
const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
|
|
180
|
+
const path = join(dir, `${base}${ext}`);
|
|
181
|
+
await writeFile(path, data);
|
|
182
|
+
exportedPaths.set(attachment.attachmentId, path);
|
|
183
|
+
return path;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Replace image content blocks with text hints pointing at exported files.
|
|
188
|
+
* Non-image messages are returned as-is (same reference); bridged messages
|
|
189
|
+
* are fresh, deep-frozen objects with the original identity and source.
|
|
190
|
+
* Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
|
|
191
|
+
* prefix so the browser half can recognize it precisely.
|
|
192
|
+
* Exported for unit testing; `ctx` only needs `attachments`.
|
|
193
|
+
*/
|
|
194
|
+
async function bridgeMessages(messages, ctx, dir) {
|
|
195
|
+
const next = [];
|
|
196
|
+
for (const message of messages) {
|
|
197
|
+
const content = message?.content;
|
|
198
|
+
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
199
|
+
next.push(message);
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const blocks = [];
|
|
203
|
+
for (const block of content) {
|
|
204
|
+
if (block?.type !== "image") {
|
|
205
|
+
blocks.push(block);
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const path = await exportImage(block.attachment, ctx, dir);
|
|
209
|
+
const name = block.attachment.name ? ` (${block.attachment.name})` : "";
|
|
210
|
+
blocks.push({
|
|
211
|
+
type: "text",
|
|
212
|
+
text:
|
|
213
|
+
`${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
|
|
214
|
+
`Inspect it with the inspect_image tool to see its content.]`,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
next.push(deepFreeze({ ...message, content: blocks }));
|
|
218
|
+
}
|
|
219
|
+
return next;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
|
|
223
|
+
function parseQuery(rawUrl) {
|
|
224
|
+
const query = {};
|
|
225
|
+
const at = rawUrl.indexOf("?");
|
|
226
|
+
if (at === -1) return query;
|
|
227
|
+
for (const pair of rawUrl.slice(at + 1).split("&")) {
|
|
228
|
+
const eq = pair.indexOf("=");
|
|
229
|
+
if (eq === -1) continue;
|
|
230
|
+
try {
|
|
231
|
+
query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
|
|
232
|
+
} catch {
|
|
233
|
+
/* skip malformed pairs */
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return query;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Advertise `image` in the `inputModalities` reported by the llm service for
|
|
241
|
+
* every model, so the host admission gate lets pasted images through on
|
|
242
|
+
* text-only models. The bridge turns those images into text hints anyway, so
|
|
243
|
+
* this is pure admission: it never changes what the adapter actually streams
|
|
244
|
+
* (the adapter's own stream validation reads the model's real `input` from
|
|
245
|
+
* the provider config, untouched here).
|
|
246
|
+
*
|
|
247
|
+
* The wrap is installed on the shared llm service instance, so it must be
|
|
248
|
+
* idempotent across HMR re-applies and restored on dispose. `marker` is a
|
|
249
|
+
* per-instance record proving this plugin already owns the wrap; the
|
|
250
|
+
* `installed` flag distinguishes "we wrapped it" from "the original was
|
|
251
|
+
* replaced by something else" so dispose only restores what we replaced.
|
|
252
|
+
*
|
|
253
|
+
* Exported for unit testing; the test passes a fake llm service.
|
|
254
|
+
*/
|
|
255
|
+
const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
|
|
256
|
+
function installAutoImageAdmission(llm, logger) {
|
|
257
|
+
if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
|
|
258
|
+
logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
|
|
259
|
+
return () => {};
|
|
260
|
+
}
|
|
261
|
+
if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
|
|
262
|
+
const original = llm.resolveModelInfo.bind(llm);
|
|
263
|
+
const wrapped = async (provider, model, signal) => {
|
|
264
|
+
const info = await original(provider, model, signal);
|
|
265
|
+
if (!info) return info;
|
|
266
|
+
const mods = info.inputModalities;
|
|
267
|
+
if (Array.isArray(mods) && mods.includes("image")) return info;
|
|
268
|
+
return { ...info, inputModalities: [...(mods ?? []), "image"] };
|
|
269
|
+
};
|
|
270
|
+
let installed = false;
|
|
271
|
+
llm[LLM_RESOLVE_WRAP_MARK] = true;
|
|
272
|
+
llm.resolveModelInfo = wrapped;
|
|
273
|
+
installed = true;
|
|
274
|
+
logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
|
|
275
|
+
return () => {
|
|
276
|
+
if (installed) {
|
|
277
|
+
if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
|
|
278
|
+
installed = false;
|
|
279
|
+
}
|
|
280
|
+
delete llm[LLM_RESOLVE_WRAP_MARK];
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Register the loopback route that serves bridged images to the same-origin
|
|
286
|
+
* page (the preview thumbnails). Read-only and tightly scoped:
|
|
287
|
+
* - only files inside the bridge export directory (no traversal);
|
|
288
|
+
* - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
|
|
289
|
+
* - Host restricted to the local machine;
|
|
290
|
+
* - hard 20MB cap per file.
|
|
291
|
+
*/
|
|
292
|
+
function registerBridgePreviewRoute(ctx, exportDir, logger) {
|
|
293
|
+
const webServer = ctx.get("webServer");
|
|
294
|
+
if (webServer === undefined) {
|
|
295
|
+
logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
const bridgeDir = resolvePath(exportDir);
|
|
299
|
+
ctx.effect(() => webServer.register({
|
|
300
|
+
kind: "exact",
|
|
301
|
+
path: BRIDGE_PREVIEW_ROUTE,
|
|
302
|
+
async handler(req, res) {
|
|
303
|
+
try {
|
|
304
|
+
const raw = String(req.url ?? "");
|
|
305
|
+
const query = parseQuery(raw);
|
|
306
|
+
const p = query.p;
|
|
307
|
+
if (typeof p !== "string" || p.length === 0) {
|
|
308
|
+
res.writeHead(400);
|
|
309
|
+
res.end("bad request");
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const lower = p.toLowerCase();
|
|
313
|
+
const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
|
|
314
|
+
if (mediaType === undefined) {
|
|
315
|
+
res.writeHead(400);
|
|
316
|
+
res.end("not an image path");
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const host = String(req.headers?.host ?? "");
|
|
320
|
+
if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
|
|
321
|
+
res.writeHead(403);
|
|
322
|
+
res.end("forbidden");
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
const target = resolvePath(p);
|
|
326
|
+
if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
|
|
327
|
+
res.writeHead(403);
|
|
328
|
+
res.end("forbidden");
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
const info = await stat(target);
|
|
332
|
+
if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
|
|
333
|
+
res.writeHead(404);
|
|
334
|
+
res.end("not found");
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
const bytes = await readFile(target);
|
|
338
|
+
res.writeHead(200, {
|
|
339
|
+
"Content-Type": mediaType,
|
|
340
|
+
"Cache-Control": "private, max-age=60",
|
|
341
|
+
});
|
|
342
|
+
res.end(bytes);
|
|
343
|
+
} catch {
|
|
344
|
+
try {
|
|
345
|
+
res.writeHead(404);
|
|
346
|
+
res.end("not found");
|
|
347
|
+
} catch {
|
|
348
|
+
/* response already sent */
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
},
|
|
352
|
+
}), "dsh-tool-vision: bridge preview route");
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Whether the session's current model may receive image blocks directly.
|
|
357
|
+
* Uses the last logged request header first, then the agent's own options,
|
|
358
|
+
* and consults ONLY the `multimodalModels` whitelist — never the model's
|
|
359
|
+
* declared `inputModalities`, because profiles routinely declare
|
|
360
|
+
* `input: [text, image]` on text-only models to pass the harness's prompt
|
|
361
|
+
* admission check (that declaration says nothing about whether the upstream
|
|
362
|
+
* endpoint really accepts `image_url` parts).
|
|
363
|
+
* Returns true when bridging is disabled (nothing would be bridged anyway).
|
|
364
|
+
*/
|
|
365
|
+
async function currentModelAcceptsImage(agent, config) {
|
|
366
|
+
if (!config.bridgeTextOnly) return true;
|
|
367
|
+
const header = agent?.session?.requestHeader?.();
|
|
368
|
+
const model = header?.config?.model ?? agent?.options?.model;
|
|
369
|
+
if (!model) return false;
|
|
370
|
+
return config.multimodalModels.includes(model);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Lazily bridge image blocks that are already part of the session log
|
|
375
|
+
* (pasted before the plugin was active, or before a restart). Each affected
|
|
376
|
+
* event is rewritten once with a surface `replace`, which swaps the durable
|
|
377
|
+
* derivation (and the transcript) to the text hint. Events that are no
|
|
378
|
+
* longer on the surface (already shadowed) are skipped and remembered.
|
|
379
|
+
* `repaired` tracks per-session state: a `Set` of handled seqs plus a
|
|
380
|
+
* monotonic scan cursor.
|
|
381
|
+
*/
|
|
382
|
+
async function repairLoggedImages(ctx, session, exportDir, repaired) {
|
|
383
|
+
const events = session.events;
|
|
384
|
+
for (let index = repaired.cursor; index < events.length; index += 1) {
|
|
385
|
+
const event = events[index];
|
|
386
|
+
if (event.type !== "user/message" || repaired.set.has(event.seq)) {
|
|
387
|
+
repaired.set.add(event.seq);
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
const content = event.data?.content;
|
|
391
|
+
if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
|
|
392
|
+
repaired.set.add(event.seq);
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
|
|
396
|
+
try {
|
|
397
|
+
session.append("user/message", bridged, {
|
|
398
|
+
surfaceOp: { op: "replace", start: event.seq, end: event.seq },
|
|
399
|
+
sourceEventSeqs: [event.seq],
|
|
400
|
+
});
|
|
401
|
+
ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
|
|
402
|
+
} catch (error) {
|
|
403
|
+
ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
|
|
404
|
+
}
|
|
405
|
+
repaired.set.add(event.seq);
|
|
406
|
+
}
|
|
407
|
+
repaired.cursor = events.length;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Install the pre-step bridge at the root level. Agent-scoped waterfalls
|
|
412
|
+
* admit untagged (root) listeners, so one listener serves every agent —
|
|
413
|
+
* including sessions resumed after a server restart, which never re-fire
|
|
414
|
+
* `session/created` for per-agent attachments. Runs before every proposed
|
|
415
|
+
* step: new pasted images are bridged into the durable log, and stuck
|
|
416
|
+
* logged images are repaired, before the model request is derived from it.
|
|
417
|
+
*/
|
|
418
|
+
function attachPreStepBridge(ctx, getConfig, exportDir) {
|
|
419
|
+
const repairedBySession = new Map();
|
|
420
|
+
ctx.on("agent/pre-step", async (payload, next) => {
|
|
421
|
+
const decision = await next();
|
|
422
|
+
if (!decision || decision.kind !== "enter") return decision;
|
|
423
|
+
const agent = payload?.agent;
|
|
424
|
+
if (!agent?.session) return decision;
|
|
425
|
+
try {
|
|
426
|
+
const acceptsImage = await currentModelAcceptsImage(agent, getConfig());
|
|
427
|
+
if (!acceptsImage) {
|
|
428
|
+
let repaired = repairedBySession.get(agent.session.id);
|
|
429
|
+
if (!repaired) {
|
|
430
|
+
repaired = { set: new Set(), cursor: 0 };
|
|
431
|
+
repairedBySession.set(agent.session.id, repaired);
|
|
432
|
+
}
|
|
433
|
+
await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
|
|
434
|
+
ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
if (acceptsImage) return decision;
|
|
438
|
+
const messages = await bridgeMessages(decision.messages, ctx, exportDir);
|
|
439
|
+
if (messages.every((message, index) => message === decision.messages[index])) return decision;
|
|
440
|
+
return { ...decision, messages };
|
|
441
|
+
} catch (error) {
|
|
442
|
+
ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
|
|
443
|
+
return decision;
|
|
444
|
+
}
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function resolveApiKey(config) {
|
|
449
|
+
if (config.apiKey) return config.apiKey;
|
|
450
|
+
if (config.apiKeyEnv) {
|
|
451
|
+
const fromEnv = process.env[config.apiKeyEnv];
|
|
452
|
+
if (fromEnv) return fromEnv;
|
|
453
|
+
}
|
|
454
|
+
return process.env.OPENAI_API_KEY ?? "";
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
|
|
458
|
+
async function toImageUrl(target, cwd, config) {
|
|
459
|
+
if (/^https?:\/\//i.test(target)) return { url: target, note: target };
|
|
460
|
+
const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
|
|
461
|
+
const info = await stat(abs).catch(() => null);
|
|
462
|
+
if (!info) throw new Error(`image not found: ${abs}`);
|
|
463
|
+
if (info.size > config.maxImageBytes) {
|
|
464
|
+
throw new Error(
|
|
465
|
+
`image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
|
|
469
|
+
if (!mime) {
|
|
470
|
+
throw new Error(
|
|
471
|
+
`unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
|
|
472
|
+
);
|
|
473
|
+
}
|
|
474
|
+
const data = await readFile(abs);
|
|
475
|
+
return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/** One OpenAI-compatible chat/completions call with an image_url content part. */
|
|
479
|
+
async function callVision(config, imageUrl, question, detail, signal) {
|
|
480
|
+
const key = resolveApiKey(config);
|
|
481
|
+
if (!key) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
`vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
|
|
487
|
+
const endpoint = new URL("chat/completions", base);
|
|
488
|
+
const controller = new AbortController();
|
|
489
|
+
const timer = setTimeout(
|
|
490
|
+
() => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
|
|
491
|
+
config.timeoutMs,
|
|
492
|
+
);
|
|
493
|
+
const onSignalAbort = () => controller.abort(signal?.reason);
|
|
494
|
+
if (signal) {
|
|
495
|
+
if (signal.aborted) controller.abort();
|
|
496
|
+
else signal.addEventListener("abort", onSignalAbort, { once: true });
|
|
497
|
+
}
|
|
498
|
+
const content = [
|
|
499
|
+
{ type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
|
|
500
|
+
{ type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
|
|
501
|
+
];
|
|
502
|
+
try {
|
|
503
|
+
const response = await fetch(endpoint, {
|
|
504
|
+
method: "POST",
|
|
505
|
+
headers: {
|
|
506
|
+
"Content-Type": "application/json",
|
|
507
|
+
Authorization: `Bearer ${key}`,
|
|
508
|
+
},
|
|
509
|
+
body: JSON.stringify({
|
|
510
|
+
model: config.model,
|
|
511
|
+
messages: [{ role: "user", content }],
|
|
512
|
+
max_tokens: config.maxTokens,
|
|
513
|
+
}),
|
|
514
|
+
signal: controller.signal,
|
|
515
|
+
});
|
|
516
|
+
const body = await response.json().catch(() => null);
|
|
517
|
+
if (!response.ok) {
|
|
518
|
+
const detailText = body?.error?.message ?? response.statusText;
|
|
519
|
+
throw new Error(
|
|
520
|
+
`vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
// Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
|
|
524
|
+
// `reasoning_content` first; when the final `content` is empty or was cut
|
|
525
|
+
// off by max_tokens, fall back to the reasoning text so the answer is
|
|
526
|
+
// still useful.
|
|
527
|
+
const message = body?.choices?.[0]?.message;
|
|
528
|
+
let answer = message?.content ?? "";
|
|
529
|
+
if (!answer.trim()) answer = message?.reasoning_content ?? "";
|
|
530
|
+
if (typeof answer !== "string" || !answer.trim()) {
|
|
531
|
+
throw new Error("vision endpoint returned an empty response");
|
|
532
|
+
}
|
|
533
|
+
return answer.trim();
|
|
534
|
+
} finally {
|
|
535
|
+
clearTimeout(timer);
|
|
536
|
+
signal?.removeEventListener("abort", onSignalAbort);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function apply(ctx, config) {
|
|
541
|
+
// ── settings-backed configuration ─────────────────────────────────────────
|
|
542
|
+
// The composition entry stays the `base` layer; a registered `tool-vision`
|
|
543
|
+
// settings section (Web UI section, settings.yaml) overlays it live, so
|
|
544
|
+
// edits hot-apply without a restart. `installSettingsSection` hands
|
|
545
|
+
// `setSource` a GETTER (`() => scope.get()`), not the config object — keep
|
|
546
|
+
// it and call it at use time, or `getConfig()` would return a function and
|
|
547
|
+
// every `cfg.*` read would be undefined (apiKey included).
|
|
548
|
+
let current = config;
|
|
549
|
+
let sourceGetter = null;
|
|
550
|
+
const getConfig = () => (sourceGetter ? sourceGetter() : current);
|
|
551
|
+
installSettingsSection(ctx, NS, Config, config, {
|
|
552
|
+
setSource: (getter) => {
|
|
553
|
+
sourceGetter = getter;
|
|
554
|
+
},
|
|
555
|
+
onChange: () => {},
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
// ── image bridge: pasted images become inspect_image hints on text-only models ──
|
|
559
|
+
if (getConfig().bridgeTextOnly) {
|
|
560
|
+
const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
|
|
561
|
+
mkdir(exportDir, { recursive: true }).catch(() => {});
|
|
562
|
+
// Root-level listener: agent-scoped waterfalls admit untagged listeners,
|
|
563
|
+
// so one registration serves every agent (new and resumed alike) and the
|
|
564
|
+
// agent is read from the fused payload.
|
|
565
|
+
attachPreStepBridge(ctx, getConfig, exportDir);
|
|
566
|
+
|
|
567
|
+
// ── automatic image admission: let pasted images through on text-only models ──
|
|
568
|
+
// The host gate refuses images unless the model declares `image` input;
|
|
569
|
+
// the bridge handles them anyway, so report image support for all models.
|
|
570
|
+
// The wrap is installed on the shared llm service instance (idempotent,
|
|
571
|
+
// restored on dispose/HMR).
|
|
572
|
+
if (getConfig().bridgeAutoImage) {
|
|
573
|
+
const unwrap = installAutoImageAdmission(ctx.get("llm"), ctx.logger);
|
|
574
|
+
ctx.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// ── bridge image preview: same-origin thumbnails for bridged images ──
|
|
578
|
+
if (getConfig().bridgePreview) {
|
|
579
|
+
registerBridgePreviewRoute(ctx, exportDir, ctx.logger);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
ctx.tools.register(defineTool({
|
|
584
|
+
name: "inspect_image",
|
|
585
|
+
description: getConfig().description,
|
|
586
|
+
parameters: {
|
|
587
|
+
path: {
|
|
588
|
+
type: "string",
|
|
589
|
+
required: true,
|
|
590
|
+
description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
|
|
591
|
+
},
|
|
592
|
+
question: {
|
|
593
|
+
type: "string",
|
|
594
|
+
description: "Optional specific question about the image. Omit for a general detailed description.",
|
|
595
|
+
},
|
|
596
|
+
detail: {
|
|
597
|
+
type: "string",
|
|
598
|
+
enum: ["auto", "low", "high"],
|
|
599
|
+
description: "Optional image resolution hint for the vision API (auto by default).",
|
|
600
|
+
},
|
|
601
|
+
},
|
|
602
|
+
output: {
|
|
603
|
+
schema: { type: "string" },
|
|
604
|
+
render: (_args, value) => [{ type: "text", text: value }],
|
|
605
|
+
},
|
|
606
|
+
async execute(args, exec) {
|
|
607
|
+
const cfg = getConfig();
|
|
608
|
+
const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
|
|
609
|
+
const { url, note } = await toImageUrl(args.path, cwd, cfg);
|
|
610
|
+
try {
|
|
611
|
+
const answer = await callVision(cfg, url, args.question, args.detail, exec.signal);
|
|
612
|
+
return note === url ? answer : `${answer}\n\n(image: ${note})`;
|
|
613
|
+
} catch (error) {
|
|
614
|
+
const raw = error && error.message ? String(error.message) : String(error);
|
|
615
|
+
if (CONTENT_FILTER_RE.test(raw)) {
|
|
616
|
+
throw new Error(
|
|
617
|
+
"inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
|
|
618
|
+
"这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
throw error;
|
|
622
|
+
}
|
|
623
|
+
},
|
|
624
|
+
}));
|
|
625
|
+
|
|
626
|
+
// ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
|
|
627
|
+
// 14 vision_* tools driven by the SAME configured endpoint as inspect_image
|
|
628
|
+
// (baseURL/apiKey/model). No provider chain, no local models, no extra
|
|
629
|
+
// settings: everything comes from the existing tool-vision configuration.
|
|
630
|
+
registerVisionTools(ctx, getConfig);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
export {
|
|
634
|
+
BRIDGE_MARKER,
|
|
635
|
+
BRIDGE_PREVIEW_MAX_BYTES,
|
|
636
|
+
BRIDGE_PREVIEW_MEDIA,
|
|
637
|
+
BRIDGE_PREVIEW_ROUTE,
|
|
638
|
+
Config,
|
|
639
|
+
DEFAULT_DESCRIPTION,
|
|
640
|
+
EXT_BY_MEDIA,
|
|
641
|
+
apply,
|
|
642
|
+
attachPreStepBridge,
|
|
643
|
+
bridgeMessages,
|
|
644
|
+
currentModelAcceptsImage,
|
|
645
|
+
deepFreeze,
|
|
646
|
+
exportImage,
|
|
647
|
+
hasImageBlock,
|
|
648
|
+
inject,
|
|
649
|
+
installAutoImageAdmission,
|
|
650
|
+
name,
|
|
651
|
+
parseQuery,
|
|
652
|
+
registerBridgePreviewRoute,
|
|
653
|
+
repairLoggedImages,
|
|
654
|
+
};
|