dsh-tool-vision 0.6.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,654 +1,654 @@
1
- /**
2
- * dsh-tool-vision — external vision model for DeepSeek Harness.
3
- *
4
- * Two capabilities:
5
- *
6
- * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
- * any OpenAI-compatible chat/completions endpoint that supports
8
- * `image_url` content parts, and returns the vision model's text answer.
9
- *
10
- * 2. Image bridge — pasted images are bridged to text hints before they
11
- * enter a text-only model's request:
12
- *
13
- * - New images are bridged on the `agent/pre-step` waterfall (the only
14
- * seam where the harness lets a plugin replace the messages that enter
15
- * a step — they become the durable `user/message` log, so the
16
- * `llm/stream` request-reconstruction invariant stays satisfied).
17
- * - Images already logged before the plugin was installed (or before a
18
- * server restart) are repaired lazily with a surface `replace`, one
19
- * event at a time, on the first pre-step of the session.
20
- *
21
- * The bridged hint points at an exported local copy of the image, which
22
- * the agent hands to `inspect_image`. Models listed in
23
- * `multimodalModels` (or whose resolved `inputModalities` include
24
- * "image") receive image blocks directly and are never bridged.
25
- *
26
- * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
27
- * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
28
- * inline thumbnails for bridged pasted images:
29
- *
30
- * - `bridgeMessages` stamps every bridged hint with an invisible marker
31
- * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
32
- * precisely instead of pattern-matching free text.
33
- * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
34
- * exported images to the same-origin page; the client inserts a
35
- * thumbnail above the hint text and opens a lightbox on click.
36
- * - Pure display layer: persisted messages, the transcript, the
37
- * model-facing text and the `inspect_image` chain are untouched.
38
- */
39
- import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
40
- import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
41
- import os from "node:os";
42
- import z from "@deepseek-ai/schemastery";
43
- import { defineTool } from "@deepseek-ai/dsh-tools";
44
- import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
45
- import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
46
-
47
- /** Cordis plugin name. */
48
- const name = "tool-vision";
49
- /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
50
- const inject = ["tools", "llm", "attachments", "webServer"];
51
- /** Settings namespace owned by this plugin (Web UI settings section). */
52
- const NS = settingsNamespace("tool-vision");
53
-
54
- /**
55
- * Invisible prefix stamped onto every bridged hint text block. The browser
56
- * half uses it to recognize bridge text precisely (no free-text regex over
57
- * user messages), and the model-facing hint stays intact otherwise.
58
- */
59
- const BRIDGE_MARKER = "\u200b[bridge]";
60
-
61
- /** Loopback route serving bridged images to the same-origin page. */
62
- const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
63
- /** Hard cap for a single served image (defense in depth; export is bounded). */
64
- const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
65
- /** Extensions the preview route serves (svg/ico intentionally excluded). */
66
- const BRIDGE_PREVIEW_MEDIA = {
67
- ".png": "image/png",
68
- ".jpg": "image/jpeg",
69
- ".jpeg": "image/jpeg",
70
- ".webp": "image/webp",
71
- ".gif": "image/gif",
72
- ".avif": "image/avif",
73
- ".bmp": "image/bmp",
74
- };
75
-
76
- const DEFAULT_DESCRIPTION =
77
- "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
78
- "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
79
- "optionally with a specific question. Returns the vision model's textual description or answer. " +
80
- "Use this whenever you need to read, describe, or extract information from image content, " +
81
- "since the main model is text-only.";
82
-
83
- /** Runtime schema for the tool-vision row. */
84
- const Config = z.object({
85
- /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
86
- baseURL: z.string().default("https://api.openai.com/v1"),
87
- /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
88
- apiKey: z.string().default("").role("secret"),
89
- /** Environment variable holding the API key. */
90
- apiKeyEnv: z.string().default("VISION_API_KEY"),
91
- /** Vision model id served by the endpoint. */
92
- model: z.string().default("gpt-4o-mini"),
93
- /** Max output tokens for the vision call. */
94
- maxTokens: z.number().default(4096),
95
- /** Per-request timeout in milliseconds. */
96
- timeoutMs: z.number().default(60000),
97
- /** Largest local image accepted, in bytes. */
98
- maxImageBytes: z.number().default(10 * 1024 * 1024),
99
- /** Tool description shown to the model; overrides the default. */
100
- description: z.string().default(DEFAULT_DESCRIPTION),
101
- /** Bridge pasted images to text hints on models that cannot see images. */
102
- bridgeTextOnly: z.boolean().default(true),
103
- /** Export directory for bridged images; empty = system temp. */
104
- bridgeExportDir: z.string().default(""),
105
- /** Model ids that receive image blocks directly (never bridged). */
106
- multimodalModels: z.array(z.string()).default([]),
107
- /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
108
- bridgePreview: z.boolean().default(true),
109
- /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
110
- bridgePreviewScanIntervalMs: z.number().default(2000),
111
- /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
112
- bridgePreviewHideHint: z.boolean().default(true),
113
- /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
114
- desktopScreenshot: z.boolean().default(false),
115
- /**
116
- * Advertise image input capability for every model while the bridge is on.
117
- * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
118
- * pasted images unless the current model declares `image` in its
119
- * `inputModalities`. The bridge handles those images anyway (they become
120
- * text hints the agent inspects through `inspect_image`), so this wraps the
121
- * llm service's `resolveModelInfo` to report image support for text-only
122
- * models too — users can paste images on any model without hand-editing
123
- * provider model configs.
124
- */
125
- bridgeAutoImage: z.boolean().default(true),
126
- });
127
-
128
- const MIME_BY_EXT = {
129
- ".png": "image/png",
130
- ".jpg": "image/jpeg",
131
- ".jpeg": "image/jpeg",
132
- ".gif": "image/gif",
133
- ".webp": "image/webp",
134
- ".bmp": "image/bmp",
135
- ".avif": "image/avif",
136
- ".svg": "image/svg+xml",
137
- ".ico": "image/x-icon",
138
- };
139
-
140
- const EXT_BY_MEDIA = {
141
- "image/png": ".png",
142
- "image/jpeg": ".jpg",
143
- "image/webp": ".webp",
144
- "image/gif": ".gif",
145
- };
146
-
147
- /** True when any message carries an image content block. */
148
- function hasImageBlock(messages) {
149
- return (messages ?? []).some((m) =>
150
- Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
151
- );
152
- }
153
-
154
- /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
155
- function deepFreeze(value) {
156
- if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
157
- if (Array.isArray(value)) {
158
- for (const item of value) deepFreeze(item);
159
- return Object.freeze(value);
160
- }
161
- for (const key of Object.keys(value)) deepFreeze(value[key]);
162
- return Object.freeze(value);
163
- }
164
-
165
- /** Export one attachment to disk; returns the file path (cached per process). */
166
- const exportedPaths = new Map();
167
- async function exportImage(attachment, ctx, dir) {
168
- const cached = exportedPaths.get(attachment.attachmentId);
169
- if (cached) return cached;
170
- const { data } = await ctx.attachments.readImage(attachment);
171
- const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
172
- const safeName = attachment.name
173
- ? attachment.name
174
- .replace(/\.[^.]+$/, "")
175
- .replace(/[^\w\-]+/g, "_")
176
- .replace(/^_+|_+$/g, "")
177
- .slice(0, 40)
178
- : "";
179
- const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
180
- const path = join(dir, `${base}${ext}`);
181
- await writeFile(path, data);
182
- exportedPaths.set(attachment.attachmentId, path);
183
- return path;
184
- }
185
-
186
- /**
187
- * Replace image content blocks with text hints pointing at exported files.
188
- * Non-image messages are returned as-is (same reference); bridged messages
189
- * are fresh, deep-frozen objects with the original identity and source.
190
- * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
191
- * prefix so the browser half can recognize it precisely.
192
- * Exported for unit testing; `ctx` only needs `attachments`.
193
- */
194
- async function bridgeMessages(messages, ctx, dir) {
195
- const next = [];
196
- for (const message of messages) {
197
- const content = message?.content;
198
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
199
- next.push(message);
200
- continue;
201
- }
202
- const blocks = [];
203
- for (const block of content) {
204
- if (block?.type !== "image") {
205
- blocks.push(block);
206
- continue;
207
- }
208
- const path = await exportImage(block.attachment, ctx, dir);
209
- const name = block.attachment.name ? ` (${block.attachment.name})` : "";
210
- blocks.push({
211
- type: "text",
212
- text:
213
- `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
214
- `Inspect it with the inspect_image tool to see its content.]`,
215
- });
216
- }
217
- next.push(deepFreeze({ ...message, content: blocks }));
218
- }
219
- return next;
220
- }
221
-
222
- /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
223
- function parseQuery(rawUrl) {
224
- const query = {};
225
- const at = rawUrl.indexOf("?");
226
- if (at === -1) return query;
227
- for (const pair of rawUrl.slice(at + 1).split("&")) {
228
- const eq = pair.indexOf("=");
229
- if (eq === -1) continue;
230
- try {
231
- query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
232
- } catch {
233
- /* skip malformed pairs */
234
- }
235
- }
236
- return query;
237
- }
238
-
239
- /**
240
- * Advertise `image` in the `inputModalities` reported by the llm service for
241
- * every model, so the host admission gate lets pasted images through on
242
- * text-only models. The bridge turns those images into text hints anyway, so
243
- * this is pure admission: it never changes what the adapter actually streams
244
- * (the adapter's own stream validation reads the model's real `input` from
245
- * the provider config, untouched here).
246
- *
247
- * The wrap is installed on the shared llm service instance, so it must be
248
- * idempotent across HMR re-applies and restored on dispose. `marker` is a
249
- * per-instance record proving this plugin already owns the wrap; the
250
- * `installed` flag distinguishes "we wrapped it" from "the original was
251
- * replaced by something else" so dispose only restores what we replaced.
252
- *
253
- * Exported for unit testing; the test passes a fake llm service.
254
- */
255
- const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
256
- function installAutoImageAdmission(llm, logger) {
257
- if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
258
- logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
259
- return () => {};
260
- }
261
- if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
262
- const original = llm.resolveModelInfo.bind(llm);
263
- const wrapped = async (provider, model, signal) => {
264
- const info = await original(provider, model, signal);
265
- if (!info) return info;
266
- const mods = info.inputModalities;
267
- if (Array.isArray(mods) && mods.includes("image")) return info;
268
- return { ...info, inputModalities: [...(mods ?? []), "image"] };
269
- };
270
- let installed = false;
271
- llm[LLM_RESOLVE_WRAP_MARK] = true;
272
- llm.resolveModelInfo = wrapped;
273
- installed = true;
274
- logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
275
- return () => {
276
- if (installed) {
277
- if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
278
- installed = false;
279
- }
280
- delete llm[LLM_RESOLVE_WRAP_MARK];
281
- };
282
- }
283
-
284
- /**
285
- * Register the loopback route that serves bridged images to the same-origin
286
- * page (the preview thumbnails). Read-only and tightly scoped:
287
- * - only files inside the bridge export directory (no traversal);
288
- * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
289
- * - Host restricted to the local machine;
290
- * - hard 20MB cap per file.
291
- */
292
- function registerBridgePreviewRoute(ctx, exportDir, logger) {
293
- const webServer = ctx.get("webServer");
294
- if (webServer === undefined) {
295
- logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
296
- return;
297
- }
298
- const bridgeDir = resolvePath(exportDir);
299
- ctx.effect(() => webServer.register({
300
- kind: "exact",
301
- path: BRIDGE_PREVIEW_ROUTE,
302
- async handler(req, res) {
303
- try {
304
- const raw = String(req.url ?? "");
305
- const query = parseQuery(raw);
306
- const p = query.p;
307
- if (typeof p !== "string" || p.length === 0) {
308
- res.writeHead(400);
309
- res.end("bad request");
310
- return;
311
- }
312
- const lower = p.toLowerCase();
313
- const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
314
- if (mediaType === undefined) {
315
- res.writeHead(400);
316
- res.end("not an image path");
317
- return;
318
- }
319
- const host = String(req.headers?.host ?? "");
320
- if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
321
- res.writeHead(403);
322
- res.end("forbidden");
323
- return;
324
- }
325
- const target = resolvePath(p);
326
- if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
327
- res.writeHead(403);
328
- res.end("forbidden");
329
- return;
330
- }
331
- const info = await stat(target);
332
- if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
333
- res.writeHead(404);
334
- res.end("not found");
335
- return;
336
- }
337
- const bytes = await readFile(target);
338
- res.writeHead(200, {
339
- "Content-Type": mediaType,
340
- "Cache-Control": "private, max-age=60",
341
- });
342
- res.end(bytes);
343
- } catch {
344
- try {
345
- res.writeHead(404);
346
- res.end("not found");
347
- } catch {
348
- /* response already sent */
349
- }
350
- }
351
- },
352
- }), "dsh-tool-vision: bridge preview route");
353
- }
354
-
355
- /**
356
- * Whether the session's current model may receive image blocks directly.
357
- * Uses the last logged request header first, then the agent's own options,
358
- * and consults ONLY the `multimodalModels` whitelist — never the model's
359
- * declared `inputModalities`, because profiles routinely declare
360
- * `input: [text, image]` on text-only models to pass the harness's prompt
361
- * admission check (that declaration says nothing about whether the upstream
362
- * endpoint really accepts `image_url` parts).
363
- * Returns true when bridging is disabled (nothing would be bridged anyway).
364
- */
365
- async function currentModelAcceptsImage(agent, config) {
366
- if (!config.bridgeTextOnly) return true;
367
- const header = agent?.session?.requestHeader?.();
368
- const model = header?.config?.model ?? agent?.options?.model;
369
- if (!model) return false;
370
- return config.multimodalModels.includes(model);
371
- }
372
-
373
- /**
374
- * Lazily bridge image blocks that are already part of the session log
375
- * (pasted before the plugin was active, or before a restart). Each affected
376
- * event is rewritten once with a surface `replace`, which swaps the durable
377
- * derivation (and the transcript) to the text hint. Events that are no
378
- * longer on the surface (already shadowed) are skipped and remembered.
379
- * `repaired` tracks per-session state: a `Set` of handled seqs plus a
380
- * monotonic scan cursor.
381
- */
382
- async function repairLoggedImages(ctx, session, exportDir, repaired) {
383
- const events = session.events;
384
- for (let index = repaired.cursor; index < events.length; index += 1) {
385
- const event = events[index];
386
- if (event.type !== "user/message" || repaired.set.has(event.seq)) {
387
- repaired.set.add(event.seq);
388
- continue;
389
- }
390
- const content = event.data?.content;
391
- if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
392
- repaired.set.add(event.seq);
393
- continue;
394
- }
395
- const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
396
- try {
397
- session.append("user/message", bridged, {
398
- surfaceOp: { op: "replace", start: event.seq, end: event.seq },
399
- sourceEventSeqs: [event.seq],
400
- });
401
- ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
402
- } catch (error) {
403
- ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
404
- }
405
- repaired.set.add(event.seq);
406
- }
407
- repaired.cursor = events.length;
408
- }
409
-
410
- /**
411
- * Install the pre-step bridge at the root level. Agent-scoped waterfalls
412
- * admit untagged (root) listeners, so one listener serves every agent —
413
- * including sessions resumed after a server restart, which never re-fire
414
- * `session/created` for per-agent attachments. Runs before every proposed
415
- * step: new pasted images are bridged into the durable log, and stuck
416
- * logged images are repaired, before the model request is derived from it.
417
- */
418
- function attachPreStepBridge(ctx, getConfig, exportDir) {
419
- const repairedBySession = new Map();
420
- ctx.on("agent/pre-step", async (payload, next) => {
421
- const decision = await next();
422
- if (!decision || decision.kind !== "enter") return decision;
423
- const agent = payload?.agent;
424
- if (!agent?.session) return decision;
425
- try {
426
- const acceptsImage = await currentModelAcceptsImage(agent, getConfig());
427
- if (!acceptsImage) {
428
- let repaired = repairedBySession.get(agent.session.id);
429
- if (!repaired) {
430
- repaired = { set: new Set(), cursor: 0 };
431
- repairedBySession.set(agent.session.id, repaired);
432
- }
433
- await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
434
- ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
435
- });
436
- }
437
- if (acceptsImage) return decision;
438
- const messages = await bridgeMessages(decision.messages, ctx, exportDir);
439
- if (messages.every((message, index) => message === decision.messages[index])) return decision;
440
- return { ...decision, messages };
441
- } catch (error) {
442
- ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
443
- return decision;
444
- }
445
- });
446
- }
447
-
448
- function resolveApiKey(config) {
449
- if (config.apiKey) return config.apiKey;
450
- if (config.apiKeyEnv) {
451
- const fromEnv = process.env[config.apiKeyEnv];
452
- if (fromEnv) return fromEnv;
453
- }
454
- return process.env.OPENAI_API_KEY ?? "";
455
- }
456
-
457
- /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
458
- async function toImageUrl(target, cwd, config) {
459
- if (/^https?:\/\//i.test(target)) return { url: target, note: target };
460
- const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
461
- const info = await stat(abs).catch(() => null);
462
- if (!info) throw new Error(`image not found: ${abs}`);
463
- if (info.size > config.maxImageBytes) {
464
- throw new Error(
465
- `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
466
- );
467
- }
468
- const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
469
- if (!mime) {
470
- throw new Error(
471
- `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
472
- );
473
- }
474
- const data = await readFile(abs);
475
- return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
476
- }
477
-
478
- /** One OpenAI-compatible chat/completions call with an image_url content part. */
479
- async function callVision(config, imageUrl, question, detail, signal) {
480
- const key = resolveApiKey(config);
481
- if (!key) {
482
- throw new Error(
483
- `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
484
- );
485
- }
486
- const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
487
- const endpoint = new URL("chat/completions", base);
488
- const controller = new AbortController();
489
- const timer = setTimeout(
490
- () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
491
- config.timeoutMs,
492
- );
493
- const onSignalAbort = () => controller.abort(signal?.reason);
494
- if (signal) {
495
- if (signal.aborted) controller.abort();
496
- else signal.addEventListener("abort", onSignalAbort, { once: true });
497
- }
498
- const content = [
499
- { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
500
- { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
501
- ];
502
- try {
503
- const response = await fetch(endpoint, {
504
- method: "POST",
505
- headers: {
506
- "Content-Type": "application/json",
507
- Authorization: `Bearer ${key}`,
508
- },
509
- body: JSON.stringify({
510
- model: config.model,
511
- messages: [{ role: "user", content }],
512
- max_tokens: config.maxTokens,
513
- }),
514
- signal: controller.signal,
515
- });
516
- const body = await response.json().catch(() => null);
517
- if (!response.ok) {
518
- const detailText = body?.error?.message ?? response.statusText;
519
- throw new Error(
520
- `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
521
- );
522
- }
523
- // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
524
- // `reasoning_content` first; when the final `content` is empty or was cut
525
- // off by max_tokens, fall back to the reasoning text so the answer is
526
- // still useful.
527
- const message = body?.choices?.[0]?.message;
528
- let answer = message?.content ?? "";
529
- if (!answer.trim()) answer = message?.reasoning_content ?? "";
530
- if (typeof answer !== "string" || !answer.trim()) {
531
- throw new Error("vision endpoint returned an empty response");
532
- }
533
- return answer.trim();
534
- } finally {
535
- clearTimeout(timer);
536
- signal?.removeEventListener("abort", onSignalAbort);
537
- }
538
- }
539
-
540
- function apply(ctx, config) {
541
- // ── settings-backed configuration ─────────────────────────────────────────
542
- // The composition entry stays the `base` layer; a registered `tool-vision`
543
- // settings section (Web UI section, settings.yaml) overlays it live, so
544
- // edits hot-apply without a restart. `installSettingsSection` hands
545
- // `setSource` a GETTER (`() => scope.get()`), not the config object — keep
546
- // it and call it at use time, or `getConfig()` would return a function and
547
- // every `cfg.*` read would be undefined (apiKey included).
548
- let current = config;
549
- let sourceGetter = null;
550
- const getConfig = () => (sourceGetter ? sourceGetter() : current);
551
- installSettingsSection(ctx, NS, Config, config, {
552
- setSource: (getter) => {
553
- sourceGetter = getter;
554
- },
555
- onChange: () => {},
556
- });
557
-
558
- // ── image bridge: pasted images become inspect_image hints on text-only models ──
559
- if (getConfig().bridgeTextOnly) {
560
- const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
561
- mkdir(exportDir, { recursive: true }).catch(() => {});
562
- // Root-level listener: agent-scoped waterfalls admit untagged listeners,
563
- // so one registration serves every agent (new and resumed alike) and the
564
- // agent is read from the fused payload.
565
- attachPreStepBridge(ctx, getConfig, exportDir);
566
-
567
- // ── automatic image admission: let pasted images through on text-only models ──
568
- // The host gate refuses images unless the model declares `image` input;
569
- // the bridge handles them anyway, so report image support for all models.
570
- // The wrap is installed on the shared llm service instance (idempotent,
571
- // restored on dispose/HMR).
572
- if (getConfig().bridgeAutoImage) {
573
- const unwrap = installAutoImageAdmission(ctx.get("llm"), ctx.logger);
574
- ctx.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
575
- }
576
-
577
- // ── bridge image preview: same-origin thumbnails for bridged images ──
578
- if (getConfig().bridgePreview) {
579
- registerBridgePreviewRoute(ctx, exportDir, ctx.logger);
580
- }
581
- }
582
-
583
- ctx.tools.register(defineTool({
584
- name: "inspect_image",
585
- description: getConfig().description,
586
- parameters: {
587
- path: {
588
- type: "string",
589
- required: true,
590
- description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
591
- },
592
- question: {
593
- type: "string",
594
- description: "Optional specific question about the image. Omit for a general detailed description.",
595
- },
596
- detail: {
597
- type: "string",
598
- enum: ["auto", "low", "high"],
599
- description: "Optional image resolution hint for the vision API (auto by default).",
600
- },
601
- },
602
- output: {
603
- schema: { type: "string" },
604
- render: (_args, value) => [{ type: "text", text: value }],
605
- },
606
- async execute(args, exec) {
607
- const cfg = getConfig();
608
- const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
609
- const { url, note } = await toImageUrl(args.path, cwd, cfg);
610
- try {
611
- const answer = await callVision(cfg, url, args.question, args.detail, exec.signal);
612
- return note === url ? answer : `${answer}\n\n(image: ${note})`;
613
- } catch (error) {
614
- const raw = error && error.message ? String(error.message) : String(error);
615
- if (CONTENT_FILTER_RE.test(raw)) {
616
- throw new Error(
617
- "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
618
- "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
619
- );
620
- }
621
- throw error;
622
- }
623
- },
624
- }));
625
-
626
- // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
627
- // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
628
- // (baseURL/apiKey/model). No provider chain, no local models, no extra
629
- // settings: everything comes from the existing tool-vision configuration.
630
- registerVisionTools(ctx, getConfig);
631
- }
632
-
633
- export {
634
- BRIDGE_MARKER,
635
- BRIDGE_PREVIEW_MAX_BYTES,
636
- BRIDGE_PREVIEW_MEDIA,
637
- BRIDGE_PREVIEW_ROUTE,
638
- Config,
639
- DEFAULT_DESCRIPTION,
640
- EXT_BY_MEDIA,
641
- apply,
642
- attachPreStepBridge,
643
- bridgeMessages,
644
- currentModelAcceptsImage,
645
- deepFreeze,
646
- exportImage,
647
- hasImageBlock,
648
- inject,
649
- installAutoImageAdmission,
650
- name,
651
- parseQuery,
652
- registerBridgePreviewRoute,
653
- repairLoggedImages,
654
- };
1
+ /**
2
+ * dsh-tool-vision — external vision model for DeepSeek Harness.
3
+ *
4
+ * Two capabilities:
5
+ *
6
+ * 1. `inspect_image` tool — sends an image (local file, or http(s) URL) to
7
+ * any OpenAI-compatible chat/completions endpoint that supports
8
+ * `image_url` content parts, and returns the vision model's text answer.
9
+ *
10
+ * 2. Image bridge — pasted images are bridged to text hints before they
11
+ * enter a text-only model's request:
12
+ *
13
+ * - New images are bridged on the `agent/pre-step` waterfall (the only
14
+ * seam where the harness lets a plugin replace the messages that enter
15
+ * a step — they become the durable `user/message` log, so the
16
+ * `llm/stream` request-reconstruction invariant stays satisfied).
17
+ * - Images already logged before the plugin was installed (or before a
18
+ * server restart) are repaired lazily with a surface `replace`, one
19
+ * event at a time, on the first pre-step of the session.
20
+ *
21
+ * The bridged hint points at an exported local copy of the image, which
22
+ * the agent hands to `inspect_image`. Models listed in
23
+ * `multimodalModels` (or whose resolved `inputModalities` include
24
+ * "image") receive image blocks directly and are never bridged.
25
+ *
26
+ * 3. Bridge image preview (v0.4.0, contributed by xing666173 from
27
+ * dsh-bridge-preview, MIT © 2026 xing666173) — the browser half renders
28
+ * inline thumbnails for bridged pasted images:
29
+ *
30
+ * - `bridgeMessages` stamps every bridged hint with an invisible marker
31
+ * (`BRIDGE_MARKER`), so the client can identify bridge text blocks
32
+ * precisely instead of pattern-matching free text.
33
+ * - A loopback route (`/plugins/dsh-tool-vision/image`) serves the
34
+ * exported images to the same-origin page; the client inserts a
35
+ * thumbnail above the hint text and opens a lightbox on click.
36
+ * - Pure display layer: persisted messages, the transcript, the
37
+ * model-facing text and the `inspect_image` chain are untouched.
38
+ */
39
+ import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
40
+ import { extname, isAbsolute, join, resolve as resolvePath, sep } from "node:path";
41
+ import os from "node:os";
42
+ import z from "@deepseek-ai/schemastery";
43
+ import { defineTool } from "@deepseek-ai/dsh-tools";
44
+ import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
45
+ import { registerVisionTools, CONTENT_FILTER_RE } from "./lib/vision-tools.js";
46
+
47
+ /** Cordis plugin name. */
48
+ const name = "tool-vision";
49
+ /** The tool registry, the llm seam (model capability lookup), the attachment store, and the host web server. */
50
+ const inject = ["tools", "llm", "attachments", "webServer"];
51
+ /** Settings namespace owned by this plugin (Web UI settings section). */
52
+ const NS = settingsNamespace("tool-vision");
53
+
54
+ /**
55
+ * Invisible prefix stamped onto every bridged hint text block. The browser
56
+ * half uses it to recognize bridge text precisely (no free-text regex over
57
+ * user messages), and the model-facing hint stays intact otherwise.
58
+ */
59
+ const BRIDGE_MARKER = "\u200b[bridge]";
60
+
61
+ /** Loopback route serving bridged images to the same-origin page. */
62
+ const BRIDGE_PREVIEW_ROUTE = "/plugins/dsh-tool-vision/image";
63
+ /** Hard cap for a single served image (defense in depth; export is bounded). */
64
+ const BRIDGE_PREVIEW_MAX_BYTES = 20 * 1024 * 1024;
65
+ /** Extensions the preview route serves (svg/ico intentionally excluded). */
66
+ const BRIDGE_PREVIEW_MEDIA = {
67
+ ".png": "image/png",
68
+ ".jpg": "image/jpeg",
69
+ ".jpeg": "image/jpeg",
70
+ ".webp": "image/webp",
71
+ ".gif": "image/gif",
72
+ ".avif": "image/avif",
73
+ ".bmp": "image/bmp",
74
+ };
75
+
76
+ const DEFAULT_DESCRIPTION =
77
+ "Analyze an image using an external vision-capable model through an OpenAI-compatible API. " +
78
+ "Provide the path to a local image file (absolute, or relative to the current workspace) or an http(s) URL, " +
79
+ "optionally with a specific question. Returns the vision model's textual description or answer. " +
80
+ "Use this whenever you need to read, describe, or extract information from image content, " +
81
+ "since the main model is text-only.";
82
+
83
+ /** Runtime schema for the tool-vision row. */
84
+ const Config = z.object({
85
+ /** Base URL of an OpenAI-compatible API, e.g. https://api.openai.com/v1 or https://dashscope.aliyuncs.com/compatible-mode/v1 */
86
+ baseURL: z.string().default("https://api.openai.com/v1"),
87
+ /** API key; takes precedence over apiKeyEnv. Rendered as a write-only secret in the Web UI. */
88
+ apiKey: z.string().default("").role("secret"),
89
+ /** Environment variable holding the API key. */
90
+ apiKeyEnv: z.string().default("VISION_API_KEY"),
91
+ /** Vision model id served by the endpoint. */
92
+ model: z.string().default("gpt-4o-mini"),
93
+ /** Max output tokens for the vision call. */
94
+ maxTokens: z.number().default(4096),
95
+ /** Per-request timeout in milliseconds. */
96
+ timeoutMs: z.number().default(60000),
97
+ /** Largest local image accepted, in bytes. */
98
+ maxImageBytes: z.number().default(10 * 1024 * 1024),
99
+ /** Tool description shown to the model; overrides the default. */
100
+ description: z.string().default(DEFAULT_DESCRIPTION),
101
+ /** Bridge pasted images to text hints on models that cannot see images. */
102
+ bridgeTextOnly: z.boolean().default(true),
103
+ /** Export directory for bridged images; empty = system temp. */
104
+ bridgeExportDir: z.string().default(""),
105
+ /** Model ids that receive image blocks directly (never bridged). */
106
+ multimodalModels: z.array(z.string()).default([]),
107
+ /** Inline preview for bridged images: thumbnail above the hint text in the user bubble (click to zoom). */
108
+ bridgePreview: z.boolean().default(true),
109
+ /** Fallback scan interval for the preview scanner in ms; 0 disables the periodic fallback. */
110
+ bridgePreviewScanIntervalMs: z.number().default(2000),
111
+ /** Hide the bridged hint text once the preview image has loaded (kept on failure — never "no image AND no text"). */
112
+ bridgePreviewHideHint: z.boolean().default(true),
113
+ /** Privacy gate for vision_screenshot: desktop capture is only registered when explicitly enabled. */
114
+ desktopScreenshot: z.boolean().default(false),
115
+ /**
116
+ * Advertise image input capability for every model while the bridge is on.
117
+ * The host admission gate (host-apiproxy `prompt`/`selectModel`) refuses
118
+ * pasted images unless the current model declares `image` in its
119
+ * `inputModalities`. The bridge handles those images anyway (they become
120
+ * text hints the agent inspects through `inspect_image`), so this wraps the
121
+ * llm service's `resolveModelInfo` to report image support for text-only
122
+ * models too — users can paste images on any model without hand-editing
123
+ * provider model configs.
124
+ */
125
+ bridgeAutoImage: z.boolean().default(true),
126
+ });
127
+
128
+ const MIME_BY_EXT = {
129
+ ".png": "image/png",
130
+ ".jpg": "image/jpeg",
131
+ ".jpeg": "image/jpeg",
132
+ ".gif": "image/gif",
133
+ ".webp": "image/webp",
134
+ ".bmp": "image/bmp",
135
+ ".avif": "image/avif",
136
+ ".svg": "image/svg+xml",
137
+ ".ico": "image/x-icon",
138
+ };
139
+
140
+ const EXT_BY_MEDIA = {
141
+ "image/png": ".png",
142
+ "image/jpeg": ".jpg",
143
+ "image/webp": ".webp",
144
+ "image/gif": ".gif",
145
+ };
146
+
147
+ /** True when any message carries an image content block. */
148
+ function hasImageBlock(messages) {
149
+ return (messages ?? []).some((m) =>
150
+ Array.isArray(m?.content) && m.content.some((b) => b?.type === "image"),
151
+ );
152
+ }
153
+
154
+ /** Deep-freeze an acyclic JSON-safe value in place (the harness freezes every durable message). */
155
+ function deepFreeze(value) {
156
+ if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
157
+ if (Array.isArray(value)) {
158
+ for (const item of value) deepFreeze(item);
159
+ return Object.freeze(value);
160
+ }
161
+ for (const key of Object.keys(value)) deepFreeze(value[key]);
162
+ return Object.freeze(value);
163
+ }
164
+
165
+ /** Export one attachment to disk; returns the file path (cached per process). */
166
+ const exportedPaths = new Map();
167
+ async function exportImage(attachment, ctx, dir) {
168
+ const cached = exportedPaths.get(attachment.attachmentId);
169
+ if (cached) return cached;
170
+ const { data } = await ctx.attachments.readImage(attachment);
171
+ const ext = EXT_BY_MEDIA[attachment.mediaType] ?? ".img";
172
+ const safeName = attachment.name
173
+ ? attachment.name
174
+ .replace(/\.[^.]+$/, "")
175
+ .replace(/[^\w\-]+/g, "_")
176
+ .replace(/^_+|_+$/g, "")
177
+ .slice(0, 40)
178
+ : "";
179
+ const base = (safeName ? `${safeName}_` : "") + attachment.attachmentId.slice(0, 12);
180
+ const path = join(dir, `${base}${ext}`);
181
+ await writeFile(path, data);
182
+ exportedPaths.set(attachment.attachmentId, path);
183
+ return path;
184
+ }
185
+
186
+ /**
187
+ * Replace image content blocks with text hints pointing at exported files.
188
+ * Non-image messages are returned as-is (same reference); bridged messages
189
+ * are fresh, deep-frozen objects with the original identity and source.
190
+ * Each bridged hint is stamped with the invisible {@link BRIDGE_MARKER}
191
+ * prefix so the browser half can recognize it precisely.
192
+ * Exported for unit testing; `ctx` only needs `attachments`.
193
+ */
194
+ async function bridgeMessages(messages, ctx, dir) {
195
+ const next = [];
196
+ for (const message of messages) {
197
+ const content = message?.content;
198
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
199
+ next.push(message);
200
+ continue;
201
+ }
202
+ const blocks = [];
203
+ for (const block of content) {
204
+ if (block?.type !== "image") {
205
+ blocks.push(block);
206
+ continue;
207
+ }
208
+ const path = await exportImage(block.attachment, ctx, dir);
209
+ const name = block.attachment.name ? ` (${block.attachment.name})` : "";
210
+ blocks.push({
211
+ type: "text",
212
+ text:
213
+ `${BRIDGE_MARKER}[User sent an image${name}, exported to: ${path}. ` +
214
+ `Inspect it with the inspect_image tool to see its content.]`,
215
+ });
216
+ }
217
+ next.push(deepFreeze({ ...message, content: blocks }));
218
+ }
219
+ return next;
220
+ }
221
+
222
+ /** Parse `?a=b&c=d` from a raw request URL (percent-decoded). */
223
+ function parseQuery(rawUrl) {
224
+ const query = {};
225
+ const at = rawUrl.indexOf("?");
226
+ if (at === -1) return query;
227
+ for (const pair of rawUrl.slice(at + 1).split("&")) {
228
+ const eq = pair.indexOf("=");
229
+ if (eq === -1) continue;
230
+ try {
231
+ query[pair.slice(0, eq)] = decodeURIComponent(pair.slice(eq + 1).replace(/\+/g, " "));
232
+ } catch {
233
+ /* skip malformed pairs */
234
+ }
235
+ }
236
+ return query;
237
+ }
238
+
239
+ /**
240
+ * Advertise `image` in the `inputModalities` reported by the llm service for
241
+ * every model, so the host admission gate lets pasted images through on
242
+ * text-only models. The bridge turns those images into text hints anyway, so
243
+ * this is pure admission: it never changes what the adapter actually streams
244
+ * (the adapter's own stream validation reads the model's real `input` from
245
+ * the provider config, untouched here).
246
+ *
247
+ * The wrap is installed on the shared llm service instance, so it must be
248
+ * idempotent across HMR re-applies and restored on dispose. `marker` is a
249
+ * per-instance record proving this plugin already owns the wrap; the
250
+ * `installed` flag distinguishes "we wrapped it" from "the original was
251
+ * replaced by something else" so dispose only restores what we replaced.
252
+ *
253
+ * Exported for unit testing; the test passes a fake llm service.
254
+ */
255
+ const LLM_RESOLVE_WRAP_MARK = Symbol("dsh-tool-vision.resolveModelInfo.wrapped");
256
+ function installAutoImageAdmission(llm, logger) {
257
+ if (llm === undefined || llm === null || typeof llm.resolveModelInfo !== "function") {
258
+ logger?.warn?.("[tool-vision] llm service unavailable; automatic image admission not installed");
259
+ return () => {};
260
+ }
261
+ if (llm[LLM_RESOLVE_WRAP_MARK]) return () => {}; // already wrapped by us (HMR re-apply)
262
+ const original = llm.resolveModelInfo.bind(llm);
263
+ const wrapped = async (provider, model, signal) => {
264
+ const info = await original(provider, model, signal);
265
+ if (!info) return info;
266
+ const mods = info.inputModalities;
267
+ if (Array.isArray(mods) && mods.includes("image")) return info;
268
+ return { ...info, inputModalities: [...(mods ?? []), "image"] };
269
+ };
270
+ let installed = false;
271
+ llm[LLM_RESOLVE_WRAP_MARK] = true;
272
+ llm.resolveModelInfo = wrapped;
273
+ installed = true;
274
+ logger?.debug?.("[tool-vision] automatic image admission installed (resolveModelInfo wrapped)");
275
+ return () => {
276
+ if (installed) {
277
+ if (llm.resolveModelInfo === wrapped) llm.resolveModelInfo = original;
278
+ installed = false;
279
+ }
280
+ delete llm[LLM_RESOLVE_WRAP_MARK];
281
+ };
282
+ }
283
+
284
+ /**
285
+ * Register the loopback route that serves bridged images to the same-origin
286
+ * page (the preview thumbnails). Read-only and tightly scoped:
287
+ * - only files inside the bridge export directory (no traversal);
288
+ * - only image extensions from {@link BRIDGE_PREVIEW_MEDIA};
289
+ * - Host restricted to the local machine;
290
+ * - hard 20MB cap per file.
291
+ */
292
+ function registerBridgePreviewRoute(ctx, exportDir, logger) {
293
+ const webServer = ctx.get("webServer");
294
+ if (webServer === undefined) {
295
+ logger?.warn?.("[tool-vision] webServer unavailable; bridge preview route not registered");
296
+ return;
297
+ }
298
+ const bridgeDir = resolvePath(exportDir);
299
+ ctx.effect(() => webServer.register({
300
+ kind: "exact",
301
+ path: BRIDGE_PREVIEW_ROUTE,
302
+ async handler(req, res) {
303
+ try {
304
+ const raw = String(req.url ?? "");
305
+ const query = parseQuery(raw);
306
+ const p = query.p;
307
+ if (typeof p !== "string" || p.length === 0) {
308
+ res.writeHead(400);
309
+ res.end("bad request");
310
+ return;
311
+ }
312
+ const lower = p.toLowerCase();
313
+ const mediaType = BRIDGE_PREVIEW_MEDIA[extname(lower)];
314
+ if (mediaType === undefined) {
315
+ res.writeHead(400);
316
+ res.end("not an image path");
317
+ return;
318
+ }
319
+ const host = String(req.headers?.host ?? "");
320
+ if (host !== "" && !/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(host)) {
321
+ res.writeHead(403);
322
+ res.end("forbidden");
323
+ return;
324
+ }
325
+ const target = resolvePath(p);
326
+ if (target !== bridgeDir && !target.startsWith(bridgeDir + sep)) {
327
+ res.writeHead(403);
328
+ res.end("forbidden");
329
+ return;
330
+ }
331
+ const info = await stat(target);
332
+ if (!info.isFile() || info.size > BRIDGE_PREVIEW_MAX_BYTES) {
333
+ res.writeHead(404);
334
+ res.end("not found");
335
+ return;
336
+ }
337
+ const bytes = await readFile(target);
338
+ res.writeHead(200, {
339
+ "Content-Type": mediaType,
340
+ "Cache-Control": "private, max-age=60",
341
+ });
342
+ res.end(bytes);
343
+ } catch {
344
+ try {
345
+ res.writeHead(404);
346
+ res.end("not found");
347
+ } catch {
348
+ /* response already sent */
349
+ }
350
+ }
351
+ },
352
+ }), "dsh-tool-vision: bridge preview route");
353
+ }
354
+
355
+ /**
356
+ * Whether the session's current model may receive image blocks directly.
357
+ * Uses the last logged request header first, then the agent's own options,
358
+ * and consults ONLY the `multimodalModels` whitelist — never the model's
359
+ * declared `inputModalities`, because profiles routinely declare
360
+ * `input: [text, image]` on text-only models to pass the harness's prompt
361
+ * admission check (that declaration says nothing about whether the upstream
362
+ * endpoint really accepts `image_url` parts).
363
+ * Returns true when bridging is disabled (nothing would be bridged anyway).
364
+ */
365
+ async function currentModelAcceptsImage(agent, config) {
366
+ if (!config.bridgeTextOnly) return true;
367
+ const header = agent?.session?.requestHeader?.();
368
+ const model = header?.config?.model ?? agent?.options?.model;
369
+ if (!model) return false;
370
+ return config.multimodalModels.includes(model);
371
+ }
372
+
373
+ /**
374
+ * Lazily bridge image blocks that are already part of the session log
375
+ * (pasted before the plugin was active, or before a restart). Each affected
376
+ * event is rewritten once with a surface `replace`, which swaps the durable
377
+ * derivation (and the transcript) to the text hint. Events that are no
378
+ * longer on the surface (already shadowed) are skipped and remembered.
379
+ * `repaired` tracks per-session state: a `Set` of handled seqs plus a
380
+ * monotonic scan cursor.
381
+ */
382
+ async function repairLoggedImages(ctx, session, exportDir, repaired) {
383
+ const events = session.events;
384
+ for (let index = repaired.cursor; index < events.length; index += 1) {
385
+ const event = events[index];
386
+ if (event.type !== "user/message" || repaired.set.has(event.seq)) {
387
+ repaired.set.add(event.seq);
388
+ continue;
389
+ }
390
+ const content = event.data?.content;
391
+ if (!Array.isArray(content) || !content.some((b) => b?.type === "image")) {
392
+ repaired.set.add(event.seq);
393
+ continue;
394
+ }
395
+ const [bridged] = await bridgeMessages([event.data], ctx, exportDir);
396
+ try {
397
+ session.append("user/message", bridged, {
398
+ surfaceOp: { op: "replace", start: event.seq, end: event.seq },
399
+ sourceEventSeqs: [event.seq],
400
+ });
401
+ ctx.logger.info(`[tool-vision] bridged logged image at seq ${event.seq} (${session.id})`);
402
+ } catch (error) {
403
+ ctx.logger.debug(`[tool-vision] skip repair of seq ${event.seq}: ${String(error)}`);
404
+ }
405
+ repaired.set.add(event.seq);
406
+ }
407
+ repaired.cursor = events.length;
408
+ }
409
+
410
+ /**
411
+ * Install the pre-step bridge at the root level. Agent-scoped waterfalls
412
+ * admit untagged (root) listeners, so one listener serves every agent —
413
+ * including sessions resumed after a server restart, which never re-fire
414
+ * `session/created` for per-agent attachments. Runs before every proposed
415
+ * step: new pasted images are bridged into the durable log, and stuck
416
+ * logged images are repaired, before the model request is derived from it.
417
+ */
418
+ function attachPreStepBridge(ctx, getConfig, exportDir) {
419
+ const repairedBySession = new Map();
420
+ ctx.on("agent/pre-step", async (payload, next) => {
421
+ const decision = await next();
422
+ if (!decision || decision.kind !== "enter") return decision;
423
+ const agent = payload?.agent;
424
+ if (!agent?.session) return decision;
425
+ try {
426
+ const acceptsImage = await currentModelAcceptsImage(agent, getConfig());
427
+ if (!acceptsImage) {
428
+ let repaired = repairedBySession.get(agent.session.id);
429
+ if (!repaired) {
430
+ repaired = { set: new Set(), cursor: 0 };
431
+ repairedBySession.set(agent.session.id, repaired);
432
+ }
433
+ await repairLoggedImages(ctx, agent.session, exportDir, repaired).catch((error) => {
434
+ ctx.logger.warn(`[tool-vision] logged-image repair failed: ${String(error)}`);
435
+ });
436
+ }
437
+ if (acceptsImage) return decision;
438
+ const messages = await bridgeMessages(decision.messages, ctx, exportDir);
439
+ if (messages.every((message, index) => message === decision.messages[index])) return decision;
440
+ return { ...decision, messages };
441
+ } catch (error) {
442
+ ctx.logger.warn(`[tool-vision] pre-step bridge failed: ${String(error)}`);
443
+ return decision;
444
+ }
445
+ });
446
+ }
447
+
448
+ function resolveApiKey(config) {
449
+ if (config.apiKey) return config.apiKey;
450
+ if (config.apiKeyEnv) {
451
+ const fromEnv = process.env[config.apiKeyEnv];
452
+ if (fromEnv) return fromEnv;
453
+ }
454
+ return process.env.OPENAI_API_KEY ?? "";
455
+ }
456
+
457
+ /** Turn a tool argument into an image_url payload: local file -> data URL, http(s) -> as-is. */
458
+ async function toImageUrl(target, cwd, config) {
459
+ if (/^https?:\/\//i.test(target)) return { url: target, note: target };
460
+ const abs = isAbsolute(target) ? target : resolvePath(cwd, target);
461
+ const info = await stat(abs).catch(() => null);
462
+ if (!info) throw new Error(`image not found: ${abs}`);
463
+ if (info.size > config.maxImageBytes) {
464
+ throw new Error(
465
+ `image too large: ${abs} (${info.size} bytes, limit ${config.maxImageBytes})`,
466
+ );
467
+ }
468
+ const mime = MIME_BY_EXT[extname(abs).toLowerCase()];
469
+ if (!mime) {
470
+ throw new Error(
471
+ `unsupported image extension: ${abs} (supported: ${Object.keys(MIME_BY_EXT).join(", ")})`,
472
+ );
473
+ }
474
+ const data = await readFile(abs);
475
+ return { url: `data:${mime};base64,${data.toString("base64")}`, note: abs };
476
+ }
477
+
478
+ /** One OpenAI-compatible chat/completions call with an image_url content part. */
479
+ async function callVision(config, imageUrl, question, detail, signal) {
480
+ const key = resolveApiKey(config);
481
+ if (!key) {
482
+ throw new Error(
483
+ `vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable`,
484
+ );
485
+ }
486
+ const base = config.baseURL.endsWith("/") ? config.baseURL : `${config.baseURL}/`;
487
+ const endpoint = new URL("chat/completions", base);
488
+ const controller = new AbortController();
489
+ const timer = setTimeout(
490
+ () => controller.abort(new Error(`vision request timed out after ${config.timeoutMs}ms`)),
491
+ config.timeoutMs,
492
+ );
493
+ const onSignalAbort = () => controller.abort(signal?.reason);
494
+ if (signal) {
495
+ if (signal.aborted) controller.abort();
496
+ else signal.addEventListener("abort", onSignalAbort, { once: true });
497
+ }
498
+ const content = [
499
+ { type: "text", text: question || "Describe this image in detail, including all key visual elements, text, and context you can see." },
500
+ { type: "image_url", image_url: detail ? { url: imageUrl, detail } : { url: imageUrl } },
501
+ ];
502
+ try {
503
+ const response = await fetch(endpoint, {
504
+ method: "POST",
505
+ headers: {
506
+ "Content-Type": "application/json",
507
+ Authorization: `Bearer ${key}`,
508
+ },
509
+ body: JSON.stringify({
510
+ model: config.model,
511
+ messages: [{ role: "user", content }],
512
+ max_tokens: config.maxTokens,
513
+ }),
514
+ signal: controller.signal,
515
+ });
516
+ const body = await response.json().catch(() => null);
517
+ if (!response.ok) {
518
+ const detailText = body?.error?.message ?? response.statusText;
519
+ throw new Error(
520
+ `vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
521
+ );
522
+ }
523
+ // Reasoning models (mimo-v2.5, deepseek-r1, ...) spend the token budget on
524
+ // `reasoning_content` first; when the final `content` is empty or was cut
525
+ // off by max_tokens, fall back to the reasoning text so the answer is
526
+ // still useful.
527
+ const message = body?.choices?.[0]?.message;
528
+ let answer = message?.content ?? "";
529
+ if (!answer.trim()) answer = message?.reasoning_content ?? "";
530
+ if (typeof answer !== "string" || !answer.trim()) {
531
+ throw new Error("vision endpoint returned an empty response");
532
+ }
533
+ return answer.trim();
534
+ } finally {
535
+ clearTimeout(timer);
536
+ signal?.removeEventListener("abort", onSignalAbort);
537
+ }
538
+ }
539
+
540
+ function apply(ctx, config) {
541
+ // ── settings-backed configuration ─────────────────────────────────────────
542
+ // The composition entry stays the `base` layer; a registered `tool-vision`
543
+ // settings section (Web UI section, settings.yaml) overlays it live, so
544
+ // edits hot-apply without a restart. `installSettingsSection` hands
545
+ // `setSource` a GETTER (`() => scope.get()`), not the config object — keep
546
+ // it and call it at use time, or `getConfig()` would return a function and
547
+ // every `cfg.*` read would be undefined (apiKey included).
548
+ let current = config;
549
+ let sourceGetter = null;
550
+ const getConfig = () => (sourceGetter ? sourceGetter() : current);
551
+ installSettingsSection(ctx, NS, Config, config, {
552
+ setSource: (getter) => {
553
+ sourceGetter = getter;
554
+ },
555
+ onChange: () => {},
556
+ });
557
+
558
+ // ── image bridge: pasted images become inspect_image hints on text-only models ──
559
+ if (getConfig().bridgeTextOnly) {
560
+ const exportDir = getConfig().bridgeExportDir || join(os.tmpdir(), "dsh-vision-bridge");
561
+ mkdir(exportDir, { recursive: true }).catch(() => {});
562
+ // Root-level listener: agent-scoped waterfalls admit untagged listeners,
563
+ // so one registration serves every agent (new and resumed alike) and the
564
+ // agent is read from the fused payload.
565
+ attachPreStepBridge(ctx, getConfig, exportDir);
566
+
567
+ // ── automatic image admission: let pasted images through on text-only models ──
568
+ // The host gate refuses images unless the model declares `image` input;
569
+ // the bridge handles them anyway, so report image support for all models.
570
+ // The wrap is installed on the shared llm service instance (idempotent,
571
+ // restored on dispose/HMR).
572
+ if (getConfig().bridgeAutoImage) {
573
+ const unwrap = installAutoImageAdmission(ctx.get("llm"), ctx.logger);
574
+ ctx.effect(() => unwrap, "dsh-tool-vision: automatic image admission");
575
+ }
576
+
577
+ // ── bridge image preview: same-origin thumbnails for bridged images ──
578
+ if (getConfig().bridgePreview) {
579
+ registerBridgePreviewRoute(ctx, exportDir, ctx.logger);
580
+ }
581
+ }
582
+
583
+ ctx.tools.register(defineTool({
584
+ name: "inspect_image",
585
+ description: getConfig().description,
586
+ parameters: {
587
+ path: {
588
+ type: "string",
589
+ required: true,
590
+ description: "Path to the image file (absolute, or relative to the current workspace) or an http(s) URL.",
591
+ },
592
+ question: {
593
+ type: "string",
594
+ description: "Optional specific question about the image. Omit for a general detailed description.",
595
+ },
596
+ detail: {
597
+ type: "string",
598
+ enum: ["auto", "low", "high"],
599
+ description: "Optional image resolution hint for the vision API (auto by default).",
600
+ },
601
+ },
602
+ output: {
603
+ schema: { type: "string" },
604
+ render: (_args, value) => [{ type: "text", text: value }],
605
+ },
606
+ async execute(args, exec) {
607
+ const cfg = getConfig();
608
+ const cwd = exec.agent?.session?.header?.cwd ?? process.cwd();
609
+ const { url, note } = await toImageUrl(args.path, cwd, cfg);
610
+ try {
611
+ const answer = await callVision(cfg, url, args.question, args.detail, exec.signal);
612
+ return note === url ? answer : `${answer}\n\n(image: ${note})`;
613
+ } catch (error) {
614
+ const raw = error && error.message ? String(error.message) : String(error);
615
+ if (CONTENT_FILTER_RE.test(raw)) {
616
+ throw new Error(
617
+ "inspect_image: 图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。" +
618
+ "这不是网络或配置问题,请换一张图片或调整图片内容后再试。",
619
+ );
620
+ }
621
+ throw error;
622
+ }
623
+ },
624
+ }));
625
+
626
+ // ── pixel-level vision tools (ported from dsh-vision-router) ─────────────
627
+ // 14 vision_* tools driven by the SAME configured endpoint as inspect_image
628
+ // (baseURL/apiKey/model). No provider chain, no local models, no extra
629
+ // settings: everything comes from the existing tool-vision configuration.
630
+ registerVisionTools(ctx, getConfig);
631
+ }
632
+
633
+ export {
634
+ BRIDGE_MARKER,
635
+ BRIDGE_PREVIEW_MAX_BYTES,
636
+ BRIDGE_PREVIEW_MEDIA,
637
+ BRIDGE_PREVIEW_ROUTE,
638
+ Config,
639
+ DEFAULT_DESCRIPTION,
640
+ EXT_BY_MEDIA,
641
+ apply,
642
+ attachPreStepBridge,
643
+ bridgeMessages,
644
+ currentModelAcceptsImage,
645
+ deepFreeze,
646
+ exportImage,
647
+ hasImageBlock,
648
+ inject,
649
+ installAutoImageAdmission,
650
+ name,
651
+ parseQuery,
652
+ registerBridgePreviewRoute,
653
+ repairLoggedImages,
654
+ };