dsh-tool-vision 0.9.3 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2485 +1,2485 @@
1
- // dsh-tool-vision — pixel-level vision tools (ported from dsh-vision-router).
2
- //
3
- // The 14 vision_* tools share ONE configured OpenAI-compatible endpoint
4
- // (the plugin's own baseURL/apiKey/model settings — the same GLM endpoint
5
- // inspect_image uses). No provider chain, no local models, no fallbacks:
6
- // the router's tool *semantics* (prompts, parameter contracts, pixel
7
- // post-processing) are kept; its routing/resilience machinery is not.
8
- //
9
- // Everything here is unit-testable without a harness: the only injected
10
- // dependencies are `ctx` (attachments/fs services), `getConfig` (live
11
- // settings) and `exec` (tool execution context).
12
-
13
- import { createHash } from 'node:crypto'
14
- import { join, dirname, isAbsolute, relative as relativePath, sep, win32 as winPath } from 'node:path'
15
- import { existsSync } from 'node:fs'
16
- import { mkdir as mkdirP, writeFile as writeFileP, readFile as readFileP, rm as rmP, stat as statP } from 'node:fs/promises'
17
- import { Worker } from 'node:worker_threads'
18
- import { execFile } from 'node:child_process'
19
- import { promisify } from 'node:util'
20
- import { pathToFileURL } from 'node:url'
21
- import { createRequire } from 'node:module'
22
- import { sessionHeaders } from './session-header.js'
23
-
24
- // ── image formats ──────────────────────────────────────────────────────────
25
-
26
- export const IMAGE_EXTENSIONS = {
27
- png: 'image/png',
28
- jpg: 'image/jpeg',
29
- jpeg: 'image/jpeg',
30
- webp: 'image/webp',
31
- gif: 'image/gif',
32
- bmp: 'image/bmp',
33
- avif: 'image/avif',
34
- svg: 'image/svg+xml',
35
- ico: 'image/x-icon',
36
- }
37
-
38
- const EXT_BY_MEDIA = {
39
- 'image/png': '.png',
40
- 'image/jpeg': '.jpg',
41
- 'image/webp': '.webp',
42
- 'image/gif': '.gif',
43
- 'image/bmp': '.bmp',
44
- 'image/avif': '.avif',
45
- 'image/svg+xml': '.svg',
46
- 'image/x-icon': '.ico',
47
- }
48
-
49
- export function mediaTypeOf(path) {
50
- const match = String(path).toLowerCase().match(/\.([a-z0-9]+)$/)
51
- return match ? IMAGE_EXTENSIONS[match[1]] : undefined
52
- }
53
-
54
- /**
55
- * Detect the image format from magic bytes instead of the file extension.
56
- * Attachments are stored as content-addressed files WITHOUT an extension,
57
- * so extension-based detection rejects them; the pixel tools must sniff.
58
- */
59
- export function sniffMediaType(bytes) {
60
- if (!bytes || bytes.length < 12) return undefined
61
- const head = (offset, count) => {
62
- const parts = []
63
- for (let i = offset; i < offset + count; i++) parts.push(bytes[i].toString(16).padStart(2, '0'))
64
- return parts.join('')
65
- }
66
- if (head(0, 8) === '89504e470d0a1a0a') return 'image/png'
67
- if (head(0, 3) === 'ffd8ff') return 'image/jpeg'
68
- const riff = head(0, 4)
69
- const webp = head(8, 4)
70
- if (riff === '52494646' && webp === '57454250') return 'image/webp'
71
- if (riff === '47494638') return 'image/gif' // GIF87a / GIF89a
72
- return undefined
73
- }
74
-
75
- // ── input resolution ───────────────────────────────────────────────────────
76
-
77
- export function basenameOf(path) {
78
- const parts = String(path).split('/')
79
- return parts[parts.length - 1] || undefined
80
- }
81
-
82
- /**
83
- * True when the string is a durable attachment id such as "sha256:<hex>" —
84
- * the form the harness uses for uploaded images.
85
- */
86
- export function isAttachmentIdInput(input) {
87
- return typeof input === 'string' && /^[a-z0-9]+:[0-9a-f]{32,}$/i.test(input.trim())
88
- }
89
-
90
- /** Cross-platform absolute-path test (posix drive-relative forms included). */
91
- export function isAbsolutePath(input) {
92
- const value = String(input ?? '')
93
- return isAbsolute(value) || winPath.isAbsolute(value)
94
- }
95
-
96
- /** Largest local image accepted by the pixel tools, in bytes. */
97
- export const MAX_IMAGE_BYTES = 20 * 1024 * 1024
98
-
99
- /** Resolve one tool argument (local path or attachment id) to image bytes. */
100
- export async function readImageBytes(ctx, exec, imagePath) {
101
- const input = String(imagePath ?? '')
102
- let bytes
103
- let storedMediaType
104
- if (isAttachmentIdInput(input)) {
105
- const attachments = ctx.get('attachments')
106
- if (attachments === undefined) {
107
- throw new Error('tool-vision: the attachment service is not available in this deployment')
108
- }
109
- const session = exec && exec.agent && exec.agent.session
110
- const ref = lookupAttachment(session, input.trim())
111
- if (ref === undefined) {
112
- throw new Error(
113
- `tool-vision: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
114
- )
115
- }
116
- let stored
117
- try {
118
- stored = await attachments.readImage(ref)
119
- } catch (error) {
120
- throw new Error(
121
- `tool-vision: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
122
- )
123
- }
124
- bytes = stored.data
125
- if (bytes && bytes.length > MAX_IMAGE_BYTES) {
126
- throw new Error(
127
- `tool-vision: image too large: ${input} (${bytes.length} bytes, limit ${MAX_IMAGE_BYTES})`,
128
- )
129
- }
130
- if (stored.ref && typeof stored.ref.mediaType === 'string') {
131
- storedMediaType = stored.ref.mediaType
132
- }
133
- } else {
134
- const fsService = ctx.get('fs')
135
- if (fsService === undefined) throw new Error('tool-vision: the fs service is not available')
136
- const resolved = await fsService.resolve(input)
137
- const target = toRealPath(fsService, resolved)
138
- // Second line of defense beyond the fs sandbox: a relative input must
139
- // resolve inside the session workspace (refuse ../ escapes).
140
- if (!isAbsolutePath(input)) {
141
- const root = workspaceOf(exec)
142
- const relative = relativePath(root, target)
143
- if (relative === '..' || relative.startsWith('..' + sep) || isAbsolutePath(relative)) {
144
- throw new Error(`tool-vision: path escapes the workspace: ${input}`)
145
- }
146
- }
147
- // Pre-check size BEFORE reading: a huge file must not be buffered whole
148
- // (readFileP has no limit and would OOM the process).
149
- const info = await statP(target).catch(() => null)
150
- if (info !== null && info.size > MAX_IMAGE_BYTES) {
151
- throw new Error(
152
- `tool-vision: image too large: ${input} (${info.size} bytes, limit ${MAX_IMAGE_BYTES})`,
153
- )
154
- }
155
- bytes = await readFileP(target)
156
- if (bytes.length > MAX_IMAGE_BYTES) {
157
- throw new Error(`tool-vision: image too large: ${input} (${bytes.length} bytes, limit ${MAX_IMAGE_BYTES})`)
158
- }
159
- }
160
- const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
161
- if (mediaType === undefined) {
162
- throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
163
- }
164
- return { bytes, mediaType }
165
- }
166
-
167
- /** Find an uploaded attachment ref in the session's recorded upload index. */
168
- export function lookupAttachment(session, attachmentId) {
169
- if (!session) return undefined
170
- const recorded = session.recordedUploads
171
- if (recorded) {
172
- const ref = recorded.get ? recorded.get(attachmentId) : undefined
173
- if (ref) return ref
174
- }
175
- const events = session.events ?? []
176
- for (const event of events) {
177
- const data = event && event.data
178
- if (!data) continue
179
- for (const block of Array.isArray(data.content) ? data.content : []) {
180
- if (block && block.type === 'image' && block.attachment && block.attachment.attachmentId === attachmentId) {
181
- return block.attachment
182
- }
183
- }
184
- const message = data.message
185
- if (message && Array.isArray(message.content)) {
186
- for (const block of message.content) {
187
- if (block && block.type === 'image' && block.attachment && block.attachment.attachmentId === attachmentId) {
188
- return block.attachment
189
- }
190
- }
191
- }
192
- }
193
- return undefined
194
- }
195
-
196
- /** Normalize an fs.resolve() result (string or { targetKey, displayPath }) to a real path. */
197
- export function toRealPath(fsService, resolved) {
198
- if (typeof resolved === 'string') return resolved
199
- if (typeof fsService?.processPath === 'function') {
200
- const p = fsService.processPath(resolved)
201
- if (typeof p === 'string' && p !== '') return p
202
- }
203
- const key = resolved?.targetKey
204
- return typeof key === 'string' && key !== '' ? key : String(resolved ?? '')
205
- }
206
-
207
- // ── sharp (lazy) ───────────────────────────────────────────────────────────
208
-
209
- let sharpPromise = null
210
-
211
- /** Lazily load sharp; resolves null when the host cannot provide it. */
212
- export function loadSharp() {
213
- if (!sharpPromise) {
214
- sharpPromise = import('sharp')
215
- .then((mod) => mod.default ?? mod)
216
- .catch(() => null)
217
- }
218
- return sharpPromise
219
- }
220
-
221
- /**
222
- * Downscale an image so its pixel count stays within maxPixels (router
223
- * default 4MP). Returns the original buffer when the image is already small
224
- * enough or when sharp is unavailable — pixel tools degrade gracefully
225
- * instead of failing.
226
- */
227
- export async function downscaleImage(bytes, maxPixels) {
228
- if (!bytes || bytes.length === 0) return bytes
229
- const sharp = await loadSharp()
230
- if (!sharp) return bytes
231
- try {
232
- const image = sharp(bytes, { failOn: 'none' })
233
- const meta = await image.metadata()
234
- const width = Number(meta.width) || 0
235
- const height = Number(meta.height) || 0
236
- if (width <= 0 || height <= 0 || width * height <= maxPixels) return bytes
237
- const scale = Math.sqrt(maxPixels / (width * height))
238
- const resized = await image
239
- .resize({
240
- width: Math.max(1, Math.floor(width * scale)),
241
- height: Math.max(1, Math.floor(height * scale)),
242
- fit: 'inside',
243
- })
244
- .toBuffer()
245
- return resized && resized.length > 0 ? resized : bytes
246
- } catch {
247
- return bytes
248
- }
249
- }
250
-
251
- /** Image dimensions via sharp (0/0 when unavailable). */
252
- export async function imageDims(bytes) {
253
- const sharp = await loadSharp()
254
- if (!sharp) return { width: 0, height: 0 }
255
- try {
256
- const meta = await sharp(bytes, { failOn: 'none' }).metadata()
257
- return { width: meta.width ?? 0, height: meta.height ?? 0 }
258
- } catch {
259
- return { width: 0, height: 0 }
260
- }
261
- }
262
-
263
- // ── content-hash cache ─────────────────────────────────────────────────────
264
-
265
- export function sha256Of(bytes) {
266
- return createHash('sha256').update(bytes).digest('hex')
267
- }
268
-
269
- /** Small LRU+TTL cache for repeated image+question calls (router semantics). */
270
- export function createCache(maxEntries, ttlMs) {
271
- const map = new Map()
272
- return {
273
- get(key) {
274
- const entry = map.get(key)
275
- if (!entry) return undefined
276
- if (entry.expires <= Date.now()) {
277
- map.delete(key)
278
- return undefined
279
- }
280
- map.delete(key)
281
- map.set(key, entry)
282
- return entry.value
283
- },
284
- set(key, value) {
285
- if (map.size >= maxEntries) {
286
- const oldest = map.keys().next().value
287
- if (oldest !== undefined) map.delete(oldest)
288
- }
289
- map.set(key, { value, expires: Date.now() + ttlMs })
290
- },
291
- size() {
292
- return map.size
293
- },
294
- clear() {
295
- map.clear()
296
- },
297
- }
298
- }
299
-
300
- // ── artifacts (tool outputs land in the session workspace) ─────────────────
301
-
302
- export function workspaceOf(exec) {
303
- const session = exec && exec.agent && exec.agent.session
304
- const cwd = session && session.header && session.header.cwd
305
- return typeof cwd === 'string' && cwd !== '' ? cwd : process.cwd()
306
- }
307
-
308
- /** Write a tool artifact under <workspace>/.dsh-tool-vision/ and return its path.
309
- * Recursive: long-OCR writes stem/chunk-*.png subdirectories. */
310
- export async function writeArtifact(exec, relPath, data) {
311
- const root = workspaceOf(exec)
312
- const target = join(root, '.dsh-tool-vision', relPath)
313
- await mkdirP(dirname(target), { recursive: true })
314
- await writeFileP(target, data)
315
- return target
316
- }
317
-
318
- /** Build an artifact filename stem from an input image reference. */
319
- export function artifactStemOf(imagePath, suffix) {
320
- const base = String(basenameOf(imagePath) ?? 'image')
321
- .replace(/\.(png|jpe?g|webp|gif)$/i, '')
322
- .replace(/[^a-zA-Z0-9._-]/g, '-')
323
- .slice(0, 32)
324
- const fingerprint = createHash('sha256').update(String(imagePath)).digest('hex').slice(0, 8)
325
- return `${base || 'image'}-${fingerprint}-${suffix}`
326
- }
327
-
328
- // ── JSON & prompt helpers ──────────────────────────────────────────────────
329
-
330
- const MAX_EXTRACT_JSON_CHARS = 200_000
331
-
332
- /** Extract the first balanced JSON value ({...} or [...]) from model output. */
333
- export function extractJson(text) {
334
- const source = String(text ?? '')
335
- const bounded = source.length > MAX_EXTRACT_JSON_CHARS
336
- ? source.slice(0, MAX_EXTRACT_JSON_CHARS)
337
- : source
338
- const fenced = bounded.match(/```(?:json)?\s*([\s\S]*?)```/i)
339
- const candidate = fenced ? fenced[1] : bounded
340
- const start = candidate.search(/[[{]/)
341
- if (start === -1) return undefined
342
- const stack = []
343
- let inString = false
344
- let escaped = false
345
- for (let index = start; index < candidate.length; index++) {
346
- const char = candidate[index]
347
- if (inString) {
348
- if (escaped) escaped = false
349
- else if (char === '\\') escaped = true
350
- else if (char === '"') inString = false
351
- continue
352
- }
353
- if (char === '"') {
354
- inString = true
355
- continue
356
- }
357
- if (char === '{') stack.push('}')
358
- else if (char === '[') stack.push(']')
359
- else if (char === '}' || char === ']') {
360
- if (stack.length === 0 || stack.pop() !== char) return undefined
361
- if (stack.length === 0) {
362
- try {
363
- const value = JSON.parse(candidate.slice(start, index + 1))
364
- return typeof value === 'object' && value !== null ? value : undefined
365
- } catch {
366
- return undefined
367
- }
368
- }
369
- }
370
- }
371
- return undefined
372
- }
373
-
374
- /** Legacy alias for callers of the earlier draft API. */
375
- export function parseModelJson(text) {
376
- return extractJson(text)
377
- }
378
-
379
- /** Parse "x1,y1,x2,y2" (string) or {x1,y1,x2,y2} (object) into a valid pixel box. */
380
- export function parseBox(value) {
381
- let box
382
- if (typeof value === 'string') {
383
- const parts = value.split(',').map((part) => Number(part.trim()))
384
- if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return undefined
385
- box = { x1: parts[0], y1: parts[1], x2: parts[2], y2: parts[3] }
386
- } else if (value && typeof value === 'object') {
387
- box = { x1: value.x1, y1: value.y1, x2: value.x2, y2: value.y2 }
388
- } else {
389
- return undefined
390
- }
391
- const { x1, y1, x2, y2 } = box
392
- if (![x1, y1, x2, y2].every((n) => Number.isInteger(n))) return undefined
393
- if (x1 < 0 || y1 < 0 || x2 <= x1 || y2 <= y1) return undefined
394
- return { x1, y1, x2, y2 }
395
- }
396
-
397
- /** Fixed JSON contract for vision_detect. */
398
- export function visionDetectInstruction(target, width, height) {
399
- return (
400
- `The image is ${width}x${height} pixels. Find every "${String(target).slice(0, 300)}" in it. ` +
401
- 'Return ONE JSON object and nothing else, shaped EXACTLY as:\n' +
402
- '{"elements":[{"label":"<short element name>","box":{"x1":0,"y1":0,"x2":0,"y2":0}},...]}\n' +
403
- '- "elements" is a numbered list (array order = element number) of every match, from top-left to bottom-right in reading order;\n' +
404
- '- every box is the tight bounding box in ORIGINAL image pixels, integers, 0 <= x1 < x2 <= ' +
405
- `${width}, 0 <= y1 < y2 <= ${height}` +
406
- ';\n- if nothing matches, return {"elements":[]}.'
407
- )
408
- }
409
-
410
- /** Structured JSON contract for vision_describe's json:true mode. */
411
- export function describeStructuredInstruction(question) {
412
- return (
413
- `Look at the image and answer the question: 「${String(question).slice(0, 1500)}」. ` +
414
- 'Return ONE JSON object and nothing else, shaped EXACTLY as:\n' +
415
- '{"summary":"<1-2 sentence answer to the question>",' +
416
- '"layout":[{"region":"<e.g. top-left / header / center>","content":"<what is there>"}],' +
417
- '"entities":[{"type":"<button|input|text|image|link|icon|other>","label":"<name or text>"}],' +
418
- '"text":"<the full text visible in the image, transcribed in reading order, as faithful as possible>"}\n' +
419
- '- "layout" lists the main regions in reading order (top-to-bottom, left-to-right);\n' +
420
- '- "entities" lists notable elements; use only the listed type values;\n' +
421
- '- "text" is the verbatim transcription; write "" when the image contains no text.'
422
- )
423
- }
424
-
425
- /** Shared vision_describe prompt. */
426
- export function visionDescribePrompt(question, wantJson = false) {
427
- const raw = String(question ?? '').trim()
428
- const text = raw === ''
429
- ? 'Describe the image accurately and answer based only on visible content.'
430
- : raw
431
- return wantJson ? text + '\n\n' + describeStructuredInstruction(text) : text
432
- }
433
-
434
- /** Normalize a vision_detect answer into the canonical shape, clamping boxes. */
435
- export function normalizeDetectResult(parsed, width, height) {
436
- if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.elements)) return undefined
437
- const clamp = (value, min, max) => Math.max(min, Math.min(value, max))
438
- const elements = []
439
- for (const item of parsed.elements) {
440
- if (!item || typeof item !== 'object' || !item.box || typeof item.box !== 'object') continue
441
- const x1 = Math.round(Number(item.box.x1))
442
- const y1 = Math.round(Number(item.box.y1))
443
- const x2 = Math.round(Number(item.box.x2))
444
- const y2 = Math.round(Number(item.box.y2))
445
- if (![x1, y1, x2, y2].every(Number.isFinite)) continue
446
- const box = {
447
- x1: clamp(x1, 0, width - 1),
448
- y1: clamp(y1, 0, height - 1),
449
- x2: clamp(x2, 1, width),
450
- y2: clamp(y2, 1, height),
451
- }
452
- if (box.x2 <= box.x1 || box.y2 <= box.y1) continue
453
- elements.push({
454
- number: elements.length + 1,
455
- label: typeof item.label === 'string' && item.label.trim() !== '' ? item.label.trim() : `element ${elements.length + 1}`,
456
- box,
457
- })
458
- }
459
- return { width, height, elements }
460
- }
461
-
462
- /** Normalize a structured vision_describe answer with defaults for missing fields. */
463
- export function normalizeDescribeResult(parsed) {
464
- if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined
465
- const layout = Array.isArray(parsed.layout) ? parsed.layout.filter((r) => r && typeof r === 'object' && typeof r.region === 'string' && typeof r.content === 'string') : []
466
- const entities = Array.isArray(parsed.entities)
467
- ? parsed.entities
468
- .filter((e) => e && typeof e === 'object' && typeof e.type === 'string' && typeof e.label === 'string')
469
- .map((e) => ({ type: e.type, label: e.label }))
470
- : []
471
- return {
472
- summary: typeof parsed.summary === 'string' ? parsed.summary : '',
473
- layout,
474
- entities,
475
- text: typeof parsed.text === 'string' ? parsed.text : '',
476
- }
477
- }
478
-
479
- // ── pixel operations ───────────────────────────────────────────────────────
480
-
481
- /** Per-pixel RGBA comparison; returns ratio, mask and worst 8x8 cells. */
482
- export function computePixelDiff(bufferA, bufferB, threshold = 16, width = 0, height = 0) {
483
- const length = Math.min(bufferA.length, bufferB.length)
484
- const pixels = Math.floor(length / 4)
485
- let differing = 0
486
- const mask = new Uint8Array(pixels)
487
- for (let i = 0; i < pixels; i++) {
488
- const o = i * 4
489
- const d =
490
- Math.max(
491
- Math.abs(bufferA[o] - bufferB[o]),
492
- Math.abs(bufferA[o + 1] - bufferB[o + 1]),
493
- Math.abs(bufferA[o + 2] - bufferB[o + 2]),
494
- ) - threshold
495
- if (d > 0) {
496
- differing += 1
497
- mask[i] = 1
498
- }
499
- }
500
- const ratio = pixels === 0 ? 0 : differing / pixels
501
- const cells = []
502
- if (width > 0 && height > 0) {
503
- const cols = 8
504
- const rows = 8
505
- const cw = Math.ceil(width / cols)
506
- const ch = Math.ceil(height / rows)
507
- for (let cy = 0; cy < rows; cy++) {
508
- for (let cx = 0; cx < cols; cx++) {
509
- let hit = 0
510
- let total = 0
511
- for (let y = cy * ch; y < Math.min((cy + 1) * ch, height); y++) {
512
- for (let x = cx * cw; x < Math.min((cx + 1) * cw, width); x++) {
513
- total += 1
514
- if (mask[y * width + x]) hit += 1
515
- }
516
- }
517
- if (total > 0 && hit > 0) {
518
- cells.push({
519
- x1: cx * cw,
520
- y1: cy * ch,
521
- x2: Math.min((cx + 1) * cw, width),
522
- y2: Math.min((cy + 1) * ch, height),
523
- ratio: hit / total,
524
- differing: hit,
525
- total,
526
- })
527
- }
528
- }
529
- }
530
- cells.sort((a, b) => b.ratio - a.ratio)
531
- }
532
- return { differing, total: pixels, ratio, mask, cells }
533
- }
534
-
535
- function asBuffer(chunk) {
536
- if (Buffer.isBuffer(chunk)) return chunk
537
- if (chunk instanceof Uint8Array) return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
538
- return Buffer.from(chunk ?? [])
539
- }
540
-
541
- class AsyncByteReader {
542
- constructor(iterable) {
543
- if (!iterable || typeof iterable[Symbol.asyncIterator] !== 'function') {
544
- throw new TypeError('pixel diff stream must be async iterable')
545
- }
546
- this.iterator = iterable[Symbol.asyncIterator]()
547
- this.buffer = Buffer.alloc(0)
548
- this.done = false
549
- }
550
-
551
- async fill(minBytes) {
552
- while (!this.done && this.buffer.length < minBytes) {
553
- const next = await this.iterator.next()
554
- if (next.done) {
555
- this.done = true
556
- break
557
- }
558
- const chunk = asBuffer(next.value)
559
- if (chunk.length === 0) continue
560
- this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk])
561
- }
562
- }
563
-
564
- consume(bytes) {
565
- const out = this.buffer.subarray(0, bytes)
566
- this.buffer = this.buffer.subarray(bytes)
567
- return out
568
- }
569
- }
570
-
571
- function cellBounds(index, width, height, cols, rows) {
572
- const cx = index % cols
573
- const cy = Math.floor(index / cols)
574
- const cw = Math.ceil(width / cols)
575
- const ch = Math.ceil(height / rows)
576
- const x1 = cx * cw
577
- const y1 = cy * ch
578
- const x2 = Math.min((cx + 1) * cw, width)
579
- const y2 = Math.min((cy + 1) * ch, height)
580
- return { x1, y1, x2, y2, total: Math.max(0, x2 - x1) * Math.max(0, y2 - y1) }
581
- }
582
-
583
- /**
584
- * Exact RGB pixel comparison over two RGBA async streams. The function never
585
- * materializes either full frame; it holds only the current upstream chunks
586
- * plus a bounded region counter grid.
587
- */
588
- export async function compareRgbaStreams(
589
- streamA,
590
- streamB,
591
- { width, height, threshold = 16, cols = 8, rows = 8 } = {},
592
- ) {
593
- const w = Math.max(1, Math.floor(Number(width) || 1))
594
- const h = Math.max(1, Math.floor(Number(height) || 1))
595
- const limit = w * h
596
- const gate = Number.isFinite(Number(threshold)) && Number(threshold) >= 0
597
- ? Number(threshold)
598
- : 16
599
- const gridCols = Math.min(w, Math.max(1, Math.floor(Number(cols) || 8)))
600
- const gridRows = Math.min(h, Math.max(1, Math.floor(Number(rows) || 8)))
601
- const cellDiff = new Uint32Array(gridCols * gridRows)
602
- const readerA = new AsyncByteReader(streamA)
603
- const readerB = new AsyncByteReader(streamB)
604
- let pixels = 0
605
- let differing = 0
606
-
607
- while (pixels < limit) {
608
- await Promise.all([readerA.fill(4), readerB.fill(4)])
609
- if (readerA.buffer.length < 4 || readerB.buffer.length < 4) break
610
- const remainingBytes = (limit - pixels) * 4
611
- const usable = Math.min(
612
- remainingBytes,
613
- Math.floor(Math.min(readerA.buffer.length, readerB.buffer.length) / 4) * 4,
614
- )
615
- if (usable <= 0) break
616
- const a = readerA.consume(usable)
617
- const b = readerB.consume(usable)
618
- const chunkPixels = usable / 4
619
- for (let i = 0; i < chunkPixels; i++) {
620
- const o = i * 4
621
- const different =
622
- Math.max(
623
- Math.abs(a[o] - b[o]),
624
- Math.abs(a[o + 1] - b[o + 1]),
625
- Math.abs(a[o + 2] - b[o + 2]),
626
- ) > gate
627
- if (!different) continue
628
- differing += 1
629
- const absolute = pixels + i
630
- const x = absolute % w
631
- const y = Math.floor(absolute / w)
632
- const cx = Math.min(gridCols - 1, Math.floor((x * gridCols) / w))
633
- const cy = Math.min(gridRows - 1, Math.floor((y * gridRows) / h))
634
- cellDiff[cy * gridCols + cx] += 1
635
- }
636
- pixels += chunkPixels
637
- }
638
-
639
- if (pixels !== limit) {
640
- throw new Error(`pixel diff stream ended early (${pixels}/${limit} pixels)`)
641
- }
642
- await Promise.all([readerA.fill(1), readerB.fill(1)])
643
- if (readerA.buffer.length > 0 || readerB.buffer.length > 0) {
644
- throw new Error('pixel diff stream produced more RGBA bytes than the declared dimensions')
645
- }
646
-
647
- const cells = []
648
- for (let index = 0; index < cellDiff.length; index++) {
649
- const hit = cellDiff[index]
650
- if (hit === 0) continue
651
- const bounds = cellBounds(index, w, h, gridCols, gridRows)
652
- if (bounds.total <= 0) continue
653
- cells.push({
654
- x1: bounds.x1,
655
- y1: bounds.y1,
656
- x2: bounds.x2,
657
- y2: bounds.y2,
658
- ratio: hit / bounds.total,
659
- differing: hit,
660
- total: bounds.total,
661
- })
662
- }
663
- cells.sort((a, b) => b.ratio - a.ratio)
664
- return { differing, total: limit, ratio: limit === 0 ? 0 : differing / limit, mask: null, cells }
665
- }
666
-
667
- /** Render a diff heatmap: grayscale base, red where the mask marks a differing pixel. */
668
- export function renderDiffHeatmap(originalRaw, mask, width, height) { const out = Buffer.alloc(width * height * 4)
669
- for (let i = 0; i < width * height; i++) {
670
- const o = i * 4
671
- const gray = Math.round(
672
- 0.299 * originalRaw[o] + 0.587 * originalRaw[o + 1] + 0.114 * originalRaw[o + 2],
673
- )
674
- if (mask[i]) {
675
- out[o] = 255
676
- out[o + 1] = 0
677
- out[o + 2] = 0
678
- out[o + 3] = 255
679
- } else {
680
- out[o] = gray
681
- out[o + 1] = gray
682
- out[o + 2] = gray
683
- out[o + 3] = 255
684
- }
685
- }
686
- return out
687
- }
688
-
689
- /** Dominant colors via bin quantization of an RGBA raw buffer. */
690
- export function quantizeColors(raw, topN = 8, bins = 32) {
691
- const step = 256 / bins
692
- const counts = new Map()
693
- const pixels = Math.floor(raw.length / 4)
694
- for (let i = 0; i < pixels; i++) {
695
- const o = i * 4
696
- if (raw[o + 3] < 128) continue
697
- const r = Math.floor(raw[o] / step) * step
698
- const g = Math.floor(raw[o + 1] / step) * step
699
- const b = Math.floor(raw[o + 2] / step) * step
700
- const key = `${r},${g},${b}`
701
- counts.set(key, (counts.get(key) ?? 0) + 1)
702
- }
703
- return [...counts.entries()]
704
- .sort((a, b) => b[1] - a[1])
705
- .slice(0, topN)
706
- .map(([key, count]) => {
707
- const [r, g, b] = key.split(',').map(Number)
708
- const hex = '#' + [r, g, b].map((v) => v.toString(16).padStart(2, '0')).join('')
709
- return { hex, count, share: pixels === 0 ? 0 : count / pixels }
710
- })
711
- }
712
-
713
- /** SVG overlay string drawing one red pixel box on a width x height canvas. */
714
- export function boxToSvg(box, width, height) {
715
- return Buffer.from(
716
- `<svg width="${width}" height="${height}">` +
717
- `<rect x="${box.x1}" y="${box.y1}" width="${box.x2 - box.x1}" height="${box.y2 - box.y1}" ` +
718
- `fill="none" stroke="#ff2d55" stroke-width="${Math.max(2, Math.round(Math.max(width, height) / 400))}"/></svg>`,
719
- )
720
- }
721
-
722
- /** SVG overlay drawing numbered red boxes (detect inventory). */
723
- export function boxesToSvg(boxes, width, height) {
724
- const stroke = Math.max(2, Math.round(Math.max(width, height) / 400))
725
- const labelR = Math.max(10, stroke * 4)
726
- const parts = [`<svg width="${width}" height="${height}">`]
727
- for (let i = 0; i < boxes.length; i++) {
728
- const box = boxes[i]
729
- parts.push(
730
- `<rect x="${box.x1}" y="${box.y1}" width="${box.x2 - box.x1}" height="${box.y2 - box.y1}" ` +
731
- `fill="none" stroke="#ff2d55" stroke-width="${stroke}"/>`,
732
- )
733
- const cx = Math.max(labelR, Math.min(box.x1, width - labelR))
734
- const cy = Math.max(labelR, Math.min(box.y1, height - labelR))
735
- parts.push(
736
- `<circle cx="${cx}" cy="${cy}" r="${labelR}" fill="#ff2d55"/>` +
737
- `<text x="${cx}" y="${cy + labelR * 0.36}" text-anchor="middle" ` +
738
- `font-family="sans-serif" font-size="${Math.round(labelR * 1.2)}" fill="#ffffff" ` +
739
- `font-weight="bold">${i + 1}</text>`,
740
- )
741
- }
742
- parts.push('</svg>')
743
- return Buffer.from(parts.join(''))
744
- }
745
-
746
- export function scaledDimensions(width, height, maxPixels) {
747
- const w = Math.max(0, Math.floor(Number(width) || 0))
748
- const h = Math.max(0, Math.floor(Number(height) || 0))
749
- const limit = Math.max(1, Math.floor(Number(maxPixels) || 1))
750
- if (w <= 0 || h <= 0) return { width: w, height: h, scale: 1 }
751
- const pixels = w * h
752
- if (pixels <= limit) return { width: w, height: h, scale: 1 }
753
- const scale = Math.sqrt(limit / pixels)
754
- return {
755
- width: Math.max(1, Math.floor(w * scale)),
756
- height: Math.max(1, Math.floor(h * scale)),
757
- scale,
758
- }
759
- }
760
-
761
- export function scaleBox(box, fromWidth, fromHeight, toWidth, toHeight) {
762
- const sx = fromWidth > 0 ? toWidth / fromWidth : 1
763
- const sy = fromHeight > 0 ? toHeight / fromHeight : 1
764
- return {
765
- x1: Math.max(0, Math.round(box.x1 * sx)),
766
- y1: Math.max(0, Math.round(box.y1 * sy)),
767
- x2: Math.min(toWidth, Math.max(1, Math.round(box.x2 * sx))),
768
- y2: Math.min(toHeight, Math.max(1, Math.round(box.y2 * sy))),
769
- }
770
- }
771
-
772
- /** Draw one red pixel box onto an image buffer via sharp. */
773
- export async function annotateBoxBuffer(bytes, box) {
774
- const sharp = await loadSharp()
775
- if (!sharp) return bytes
776
- const meta = await sharp(bytes, { failOn: 'none' }).metadata()
777
- const width = meta.width ?? box.x2
778
- const height = meta.height ?? box.y2
779
- const preview = scaledDimensions(width, height, 4_000_000)
780
- const displayBox = preview.scale === 1
781
- ? box
782
- : scaleBox(box, width, height, preview.width, preview.height)
783
- let image = sharp(bytes, { failOn: 'none' })
784
- if (preview.scale !== 1) image = image.resize(preview.width, preview.height, { fit: 'fill' })
785
- return image
786
- .composite([{ input: boxToSvg(displayBox, preview.width, preview.height), top: 0, left: 0 }])
787
- .png()
788
- .toBuffer()
789
- }
790
-
791
- /** Draw numbered boxes for a detected-element inventory onto an image buffer. */
792
- export async function annotateBoxesBuffer(bytes, boxes) {
793
- const sharp = await loadSharp()
794
- if (!sharp) return bytes
795
- const meta = await sharp(bytes, { failOn: 'none' }).metadata()
796
- const width = meta.width ?? 0
797
- const height = meta.height ?? 0
798
- if (width <= 0 || height <= 0 || boxes.length === 0) return bytes
799
- const preview = scaledDimensions(width, height, 4_000_000)
800
- const displayBoxes = preview.scale === 1
801
- ? boxes
802
- : boxes.map((box) => scaleBox(box, width, height, preview.width, preview.height))
803
- let image = sharp(bytes, { failOn: 'none' })
804
- if (preview.scale !== 1) image = image.resize(preview.width, preview.height, { fit: 'fill' })
805
- return image
806
- .composite([{ input: boxesToSvg(displayBoxes, preview.width, preview.height), top: 0, left: 0 }])
807
- .png()
808
- .toBuffer()
809
- }
810
-
811
- /** Remove a solid-ish background via border flood fill; background becomes transparent. */
812
- export function floodFillBackground(raw, width, height, tolerance = 40) {
813
- const total = width * height
814
- const out = Buffer.from(raw)
815
- const marked = new Uint8Array(total)
816
- let r = 0
817
- let g = 0
818
- let b = 0
819
- const corners = [0, width - 1, (height - 1) * width, total - 1]
820
- for (const c of corners) {
821
- const o = c * 4
822
- r += raw[o]
823
- g += raw[o + 1]
824
- b += raw[o + 2]
825
- }
826
- r /= 4
827
- g /= 4
828
- b /= 4
829
- const queue = []
830
- let head = 0
831
- const push = (x, y) => {
832
- const i = y * width + x
833
- if (marked[i]) return
834
- const o = i * 4
835
- const d = Math.max(Math.abs(raw[o] - r), Math.abs(raw[o + 1] - g), Math.abs(raw[o + 2] - b))
836
- if (d > tolerance) return
837
- marked[i] = 1
838
- queue.push(i)
839
- }
840
- for (let x = 0; x < width; x++) {
841
- push(x, 0)
842
- push(x, height - 1)
843
- }
844
- for (let y = 0; y < height; y++) {
845
- push(0, y)
846
- push(width - 1, y)
847
- }
848
- while (head < queue.length) {
849
- const i = queue[head++]
850
- const x = i % width
851
- const y = (i - x) / width
852
- if (x > 0) push(x - 1, y)
853
- if (x < width - 1) push(x + 1, y)
854
- if (y > 0) push(x, y - 1)
855
- if (y < height - 1) push(x, y + 1)
856
- }
857
- for (let i = 0; i < total; i++) {
858
- if (marked[i]) out[i * 4 + 3] = 0
859
- }
860
- return out
861
- }
862
-
863
- /** Cover a long/large image with bounded tiles in reading order. */
864
- export function boundedOcrTiles(
865
- width,
866
- height,
867
- { chunkHeight = 1200, overlap = 120, maxTilePixels = 4_000_000 } = {},
868
- ) {
869
- const w = Math.max(1, Math.floor(Number(width) || 1))
870
- const h = Math.max(1, Math.floor(Number(height) || 1))
871
- const requestedHeight = Math.max(1, Math.min(h, Math.floor(Number(chunkHeight) || 1200)))
872
- const pixelLimit = Math.max(1, Math.floor(Number(maxTilePixels) || 4_000_000))
873
- const tileWidth = Math.min(w, Math.max(1, Math.floor(pixelLimit / requestedHeight)))
874
- const tileHeight = Math.min(requestedHeight, Math.max(1, Math.floor(pixelLimit / tileWidth)))
875
- const verticalOverlap = Math.min(
876
- Math.max(0, Math.floor(Number(overlap) || 0)),
877
- Math.max(0, Math.floor(tileHeight / 2)),
878
- )
879
- const verticalStep = Math.max(1, tileHeight - verticalOverlap)
880
- const tiles = []
881
- for (let top = 0; top < h; top += verticalStep) {
882
- const bottom = Math.min(h, top + tileHeight)
883
- for (let left = 0; left < w; left += tileWidth) {
884
- const right = Math.min(w, left + tileWidth)
885
- tiles.push({ left, right, top, bottom })
886
- }
887
- if (bottom >= h) break
888
- }
889
- return tiles
890
- }
891
-
892
- // ── potrace (worker-thread, hard timeout) ──────────────────────────────────
893
-
894
- function runPotraceWorker(workerData, source, timeoutMs) {
895
- return new Promise((resolve, reject) => {
896
- let settled = false
897
- let worker
898
- const finish = (error, svg) => {
899
- if (settled) return
900
- settled = true
901
- clearTimeout(timer)
902
- void worker?.terminate()
903
- if (error) reject(error)
904
- else resolve(svg)
905
- }
906
- const timer = setTimeout(() => {
907
- if (settled) return
908
- settled = true
909
- void worker?.terminate()
910
- reject(
911
- new Error(
912
- 'potrace timed out — the image is too large or too complex; crop it to the target region first',
913
- ),
914
- )
915
- }, timeoutMs)
916
- try {
917
- worker = new Worker(source, { eval: true, workerData })
918
- worker.once('message', (message) => {
919
- if (message && message.error) finish(new Error(message.error))
920
- else finish(undefined, message && message.svg)
921
- })
922
- worker.once('error', (error) => finish(error))
923
- worker.once('exit', (code) => {
924
- if (code !== 0 && !settled) finish(new Error(`potrace worker exited with code ${code}`))
925
- })
926
- } catch (error) {
927
- finish(error)
928
- }
929
- })
930
- }
931
-
932
- /** Vectorize an image buffer into an SVG string via potrace posterization. */
933
- export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000) {
934
- const potraceUrl = pathToFileURL(createRequire(import.meta.url).resolve('potrace')).href
935
- const source = `
936
- import('node:worker_threads').then(({ parentPort, workerData }) => {
937
- import(workerData.potraceUrl).then((mod) => {
938
- const potrace = mod.default ?? mod
939
- potrace.posterize(Buffer.from(workerData.bytes), {
940
- steps: workerData.steps,
941
- fillStrategy: workerData.fillStrategy,
942
- }, (error, svg) => {
943
- parentPort.postMessage(error ? { error: String((error && error.message) || error) } : { svg })
944
- })
945
- }).catch((error) => {
946
- parentPort.postMessage({ error: String((error && error.message) || error) })
947
- })
948
- })
949
- `
950
- return runPotraceWorker({ potraceUrl, bytes, steps, fillStrategy }, source, timeoutMs)
951
- }
952
-
953
- /**
954
- * Color-preserving vectorization: quantize into the top colors, trace one
955
- * 1-bit mask per color with potrace, emit a colored SVG (one path per color).
956
- */
957
- export function posterizeSvgColor(data, info, palette, timeoutMs = 60000) {
958
- const sharpUrl = pathToFileURL(createRequire(import.meta.url).resolve('sharp')).href
959
- const potraceUrl = pathToFileURL(createRequire(import.meta.url).resolve('potrace')).href
960
- const source = `
961
- import('node:worker_threads').then(({ parentPort, workerData }) => {
962
- Promise.all([import(workerData.sharpUrl), import(workerData.potraceUrl)]).then(([sharpMod, potraceMod]) => {
963
- const sharp = sharpMod.default ?? sharpMod
964
- const potrace = potraceMod.default ?? potraceMod
965
- const { width, height, palette } = workerData
966
- const raw = Buffer.from(workerData.raw)
967
- const hexRgb = (hex) => {
968
- const n = parseInt(hex.slice(1), 16)
969
- return [(n >> 16) & 255, (n >> 8) & 255, n & 255]
970
- }
971
- const paletteRgb = palette.map((p) => hexRgb(p.hex))
972
- const pixels = width * height
973
- const masks = palette.map(() => Buffer.alloc(pixels))
974
- for (let p = 0; p < pixels; p++) {
975
- const o = p * 4
976
- if (raw[o + 3] < 128) continue
977
- let best = 0
978
- let bestD = Infinity
979
- for (let c = 0; c < paletteRgb.length; c++) {
980
- const dr = raw[o] - paletteRgb[c][0]
981
- const dg = raw[o + 1] - paletteRgb[c][1]
982
- const db = raw[o + 2] - paletteRgb[c][2]
983
- const d = dr * dr + dg * dg + db * db
984
- if (d < bestD) { bestD = d; best = c }
985
- }
986
- masks[best][p] = 1
987
- }
988
- const paths = []
989
- let pending = palette.length
990
- const maybeDone = () => {
991
- if (pending > 0) return
992
- const pathSvg = paths.map((p) => '<path fill="' + p.hex + '" d="' + p.d + '"/>').join('')
993
- parentPort.postMessage({
994
- ok: true,
995
- svg: '<svg xmlns="http://www.w3.org/2000/svg" width="' + width + '" height="' + height +
996
- '" viewBox="0 0 ' + width + ' ' + height + '"><rect width="' + width + '" height="' + height +
997
- '" fill="#ffffff"/>' + pathSvg + '</svg>',
998
- })
999
- }
1000
- if (pending === 0) { maybeDone(); return }
1001
- palette.forEach((entry, index) => {
1002
- const gray = Buffer.alloc(pixels)
1003
- const mask = masks[index]
1004
- for (let p = 0; p < pixels; p++) gray[p] = mask[p] ? 0 : 255
1005
- sharp(gray, { raw: { width, height, channels: 1 } })
1006
- .png()
1007
- .toBuffer()
1008
- .then((pngBuf) => {
1009
- potrace.trace(pngBuf, (err, svg) => {
1010
- pending -= 1
1011
- if (!err && svg) {
1012
- const found = [...svg.matchAll(/d="([^"]+)"/g)].map((m) => m[1])
1013
- for (const d of found) paths.push({ hex: entry.hex, d })
1014
- }
1015
- maybeDone()
1016
- })
1017
- })
1018
- .catch(() => {
1019
- pending -= 1
1020
- maybeDone()
1021
- })
1022
- })
1023
- }).catch((error) => {
1024
- parentPort.postMessage({ error: String((error && error.message) || error) })
1025
- })
1026
- })
1027
- `
1028
- return runPotraceWorker(
1029
- { sharpUrl, potraceUrl, width: info.width, height: info.height, palette, raw: data },
1030
- source,
1031
- timeoutMs,
1032
- )
1033
- }
1034
-
1035
- // ── browser (HTML screenshot) ──────────────────────────────────────────────
1036
-
1037
- /** Cross-platform Chrome/Chromium/Edge discovery. */
1038
- export function chromiumCandidates(env = {}, platform = typeof process !== 'undefined' ? process.platform : '') {
1039
- const out = []
1040
- const add = (value) => {
1041
- if (typeof value === 'string' && value !== '' && !out.includes(value)) out.push(value)
1042
- }
1043
- add(env.CHROME_PATH)
1044
- add(env.PUPPETEER_EXECUTABLE_PATH)
1045
-
1046
- if (platform === 'win32') {
1047
- const pf = env.PROGRAMFILES
1048
- const pfx86 = env['PROGRAMFILES(X86)']
1049
- const local = env.LOCALAPPDATA
1050
- if (pf) {
1051
- add(winPath.join(pf, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1052
- add(winPath.join(pf, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1053
- }
1054
- if (pfx86) {
1055
- add(winPath.join(pfx86, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1056
- add(winPath.join(pfx86, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1057
- }
1058
- if (local) {
1059
- add(winPath.join(local, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1060
- add(winPath.join(local, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1061
- add(winPath.join(local, 'Chromium', 'Application', 'chrome.exe'))
1062
- }
1063
- } else if (platform === 'darwin') {
1064
- add('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
1065
- add('/Applications/Chromium.app/Contents/MacOS/Chromium')
1066
- add('/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge')
1067
- } else {
1068
- add('/usr/bin/google-chrome')
1069
- add('/usr/bin/google-chrome-stable')
1070
- add('/usr/bin/chromium')
1071
- add('/usr/bin/chromium-browser')
1072
- add('/usr/bin/microsoft-edge')
1073
- add('/usr/bin/microsoft-edge-stable')
1074
- }
1075
- return out
1076
- }
1077
-
1078
- /** Wake lazy/revealed content before a full-page capture. */
1079
- export async function wakePageForFullCapture(page, viewportHeight) {
1080
- const step = Number.isInteger(viewportHeight) && viewportHeight > 0 ? viewportHeight : 720
1081
- await page.evaluate(() => {
1082
- document.documentElement.style.scrollBehavior = 'auto'
1083
- })
1084
- const total = await page.evaluate(() =>
1085
- Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0),
1086
- )
1087
- for (let y = 0; y < total; y += step) {
1088
- await page.evaluate((yy) => window.scrollTo(0, yy), y)
1089
- await new Promise((resolve) => setTimeout(resolve, 60))
1090
- }
1091
- await page.evaluate(() => window.scrollTo(0, 0))
1092
- await new Promise((resolve) => setTimeout(resolve, 800))
1093
- }
1094
-
1095
- /** Full scrollable page height (CSS px). */
1096
- export async function fullPageHeightOf(page) {
1097
- return await page.evaluate(() =>
1098
- Math.max(
1099
- document.documentElement.scrollHeight,
1100
- document.body ? document.body.scrollHeight : 0,
1101
- window.innerHeight,
1102
- ),
1103
- )
1104
- }
1105
-
1106
- // ── model call (single configured endpoint) ────────────────────────────────
1107
-
1108
- /** Endpoint content-safety rejection markers (GLM: "系统检测到输入或生成内容
1109
- * 可能包含不安全或敏感内容…"; OpenAI-style: content_filter/policy/safety…). */
1110
- export const CONTENT_FILTER_RE =
1111
- /敏感|不安全|内容审核|内容安全|政策|content.?filter|safety|inappropriate|flagged|拒绝.?内容|违反.*(规范|政策|安全)|violat/i
1112
-
1113
- export function resolveApiKey(config) {
1114
- if (config.apiKey) return config.apiKey
1115
- if (config.apiKeyEnv) {
1116
- const fromEnv = process.env[config.apiKeyEnv]
1117
- if (fromEnv) return fromEnv
1118
- }
1119
- return process.env.OPENAI_API_KEY ?? ''
1120
- }
1121
-
1122
- /** Turn image bytes into a data URL for the vision call.
1123
- * Defensive: attachment-service reads return Uint8Array (whose
1124
- * toString('base64') is a comma-joined number list, not base64), so always
1125
- * wrap in a real Buffer first. */
1126
- export function bytesToDataUrl(bytes, mediaType) {
1127
- const mime = mediaType ?? 'image/png'
1128
- const buf = Buffer.isBuffer(bytes)
1129
- ? bytes
1130
- : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
1131
- return `data:${mime};base64,${buf.toString('base64')}`
1132
- }
1133
-
1134
- /**
1135
- * One OpenAI-compatible chat/completions call with image content parts.
1136
- * This is the ONLY model-call path the ported tools use: the plugin's own
1137
- * baseURL/apiKey/model settings drive every vision_* tool, exactly like
1138
- * inspect_image.
1139
- */
1140
- export async function callVisionModel(getConfig, prompt, imageUrls, options = {}) {
1141
- const cfg = getConfig()
1142
- const key = resolveApiKey(cfg)
1143
- if (!key) {
1144
- throw new Error(
1145
- 'tool-vision: vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable',
1146
- )
1147
- }
1148
- const sessionHeader = sessionHeaders(cfg, options.exec)
1149
- const base = cfg.baseURL.endsWith('/') ? cfg.baseURL : `${cfg.baseURL}/`
1150
- const endpoint = new URL('chat/completions', base)
1151
- const controller = new AbortController()
1152
- const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : cfg.timeoutMs
1153
- const timer = setTimeout(
1154
- () => controller.abort(new Error(`tool-vision: vision request timed out after ${timeoutMs}ms`)),
1155
- timeoutMs,
1156
- )
1157
- const signal = options.signal
1158
- const onSignalAbort = () => controller.abort(signal?.reason)
1159
- if (signal) {
1160
- if (signal.aborted) controller.abort()
1161
- else signal.addEventListener('abort', onSignalAbort, { once: true })
1162
- }
1163
- const content = [
1164
- { type: 'text', text: prompt },
1165
- ...imageUrls.map((url) => ({ type: 'image_url', image_url: { url } })),
1166
- ]
1167
- const bodyOf = () => JSON.stringify({
1168
- model: cfg.model,
1169
- messages: [{ role: 'user', content }],
1170
- max_tokens: Number.isFinite(options.maxTokens) ? options.maxTokens : cfg.maxTokens,
1171
- })
1172
- const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
1173
- const retryAfterOf = (response) => {
1174
- const header = response?.headers?.get?.('retry-after')
1175
- if (header) {
1176
- const seconds = Number(header)
1177
- if (Number.isFinite(seconds) && seconds > 0) return Math.min(seconds * 1000, 15000)
1178
- }
1179
- return 0
1180
- }
1181
- const MAX_TRANSIENT_RETRIES = 3
1182
- try {
1183
- let lastResponse = null
1184
- let lastBody = null
1185
- for (let attempt = 0; ; attempt++) {
1186
- const response = await fetch(endpoint, {
1187
- method: 'POST',
1188
- headers: {
1189
- 'Content-Type': 'application/json',
1190
- Authorization: `Bearer ${key}`,
1191
- ...sessionHeader,
1192
- },
1193
- body: bodyOf(),
1194
- signal: controller.signal,
1195
- })
1196
- lastResponse = response
1197
- lastBody = await response.json().catch(() => null)
1198
- // Transient failures (rate limit / 5xx) retry with backoff; everything
1199
- // else (4xx auth/format/content-filter) is terminal.
1200
- const retriable =
1201
- response.status === 429 ||
1202
- response.status === 408 ||
1203
- response.status === 500 ||
1204
- response.status === 502 ||
1205
- response.status === 503 ||
1206
- response.status === 504
1207
- if (!response.ok && retriable && attempt < MAX_TRANSIENT_RETRIES) {
1208
- const waitMs = retryAfterOf(response) || 800 * (attempt + 1)
1209
- await sleep(waitMs)
1210
- continue
1211
- }
1212
- break
1213
- }
1214
- const response = lastResponse
1215
- const body = lastBody
1216
- if (!response.ok) {
1217
- const detailText = body?.error?.message ?? response.statusText
1218
- throw new Error(
1219
- `tool-vision: vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
1220
- )
1221
- }
1222
- const message = body?.choices?.[0]?.message
1223
- let answer = message?.content ?? ''
1224
- if (!answer.trim()) answer = message?.reasoning_content ?? ''
1225
- if (typeof answer !== 'string' || !answer.trim()) {
1226
- throw new Error('tool-vision: vision endpoint returned an empty response')
1227
- }
1228
- const trimmed = answer.trim()
1229
- // max_tokens truncation: glm-4v-flash caps at 1024; a truncated JSON or
1230
- // transcript must not masquerade as a complete answer.
1231
- const finishReason = body?.choices?.[0]?.finish_reason
1232
- if (finishReason === 'length') {
1233
- return `${trimmed}\n\n[注意:模型输出因 max_tokens 上限被截断(finish_reason=length);以上内容不完整。]`
1234
- }
1235
- return trimmed
1236
- } finally {
1237
- clearTimeout(timer)
1238
- signal?.removeEventListener('abort', onSignalAbort)
1239
- }
1240
- }
1241
-
1242
- // ── shared tool plumbing ───────────────────────────────────────────────────
1243
-
1244
- export const stringOutput = {
1245
- schema: { type: 'string' },
1246
- render: (_args, value) => [{ type: 'text', text: value }],
1247
- }
1248
-
1249
- /**
1250
- * Build the 14 vision tools bound to this plugin instance.
1251
- * @param {object} ctx - plugin context (attachments/fs services via ctx.get)
1252
- * @param {() => object} getConfig - live settings getter
1253
- */
1254
- export function registerVisionTools(ctx, getConfig) {
1255
- const logger = ctx.logger
1256
-
1257
- /** Single-endpoint model call wrapper with the router's ok:false contract.
1258
- * `images` is a single Buffer, or an array of Buffers / data-URL strings. */
1259
- const answerVisionForTool = async (images, mediaType, instruction, options = {}) => {
1260
- try {
1261
- // Uniform pixel budget for EVERY model call: ground/detect/ocr/etc.
1262
- // must not ship raw 20MB images (base64 ~27MB -> endpoint 413/timeout).
1263
- // Downscale only Buffer inputs; pre-made data URLs pass through.
1264
- const bounded = Array.isArray(images)
1265
- ? await Promise.all(images.map(async (item) => (typeof item === 'string' ? item : bytesToDataUrl(await downscaleImage(item, 4_000_000), mediaType ?? 'image/png'))))
1266
- : [bytesToDataUrl(await downscaleImage(images, 4_000_000), mediaType)]
1267
- const text = await callVisionModel(getConfig, instruction, bounded, {
1268
- signal: options.signal ?? (options.exec && options.exec.signal),
1269
- exec: options.exec,
1270
- })
1271
- return { ok: true, text }
1272
- } catch (error) {
1273
- const raw = error && error.message ? String(error.message) : String(error)
1274
- // Endpoint content-safety rejections are NOT backend outages: the image
1275
- // itself was refused. Surface them distinctly so the model tells the
1276
- // user to pick a different image instead of reporting "vision down".
1277
- if (CONTENT_FILTER_RE.test(raw)) {
1278
- return {
1279
- ok: false,
1280
- code: 'VISION_CONTENT_FILTERED',
1281
- retryable: false,
1282
- contentFiltered: true,
1283
- message:
1284
- '图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。' +
1285
- '这不是网络或配置问题,请换一张图片或调整图片内容后再试。',
1286
- }
1287
- }
1288
- return {
1289
- ok: false,
1290
- code: 'VISION_BACKEND_UNAVAILABLE',
1291
- retryable: false,
1292
- message: raw,
1293
- }
1294
- }
1295
- }
1296
-
1297
- const saveArtifact = (exec, relPath, data) => writeArtifact(exec, relPath, data)
1298
- const artifactStem = (imagePath, suffix) => artifactStemOf(imagePath, suffix)
1299
-
1300
- const tools = []
1301
-
1302
- // ── vision_describe ──────────────────────────────────────────────────────
1303
- tools.push({
1304
- name: 'vision_describe',
1305
- description:
1306
- 'Look at images with the configured vision model and answer a focused question about them. ' +
1307
- 'For text-only sessions this is the bridge that provides image understanding. ' +
1308
- 'Supports comparing multiple images (e.g. a design mock vs an implementation screenshot). Provide ' +
1309
- '`paths` (absolute local image file paths, png/jpeg/webp/gif) and/or ' +
1310
- '`attachmentIds` (ids of images the user uploaded in this conversation), 1-4 images in ' +
1311
- 'total. `question` is the question to answer; be specific. Set `json: true` to require a ' +
1312
- 'single valid JSON object as the answer. ' +
1313
- 'FAILURE SEMANTICS: if the result is JSON with ok:false, the vision backend is unavailable. ' +
1314
- 'Do NOT call vision_describe again with a reworded question — rephrasing cannot fix an auth, ' +
1315
- 'rate-limit or outage problem. Answer from the information you already have and continue the ' +
1316
- 'text task, telling the user vision is temporarily unavailable.',
1317
- parameters: {
1318
- type: 'object',
1319
- properties: {
1320
- paths: {
1321
- type: 'array',
1322
- items: { type: 'string' },
1323
- description:
1324
- 'Absolute local image file paths and/or attachment ids (e.g. "sha256:...") of uploaded images, 1-4 images',
1325
- },
1326
- attachmentIds: {
1327
- type: 'array',
1328
- items: { type: 'string' },
1329
- description: 'Attachment ids of images uploaded earlier in this conversation',
1330
- },
1331
- question: {
1332
- type: 'string',
1333
- description:
1334
- 'The question for the vision model, e.g. "compare the two images and list the differences"',
1335
- },
1336
- json: {
1337
- type: 'boolean',
1338
- description: 'Require the answer to be a single valid JSON object',
1339
- },
1340
- },
1341
- required: ['question'],
1342
- additionalProperties: false,
1343
- },
1344
- output: stringOutput,
1345
- async execute(args, exec) {
1346
- const attachments = ctx.get('attachments')
1347
- if (attachments === undefined) {
1348
- throw new Error('vision_describe: the durable attachment service is not available in this deployment')
1349
- }
1350
- const blocks = []
1351
- const contentIds = []
1352
- const paths = Array.isArray(args.paths) ? args.paths : []
1353
- const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
1354
- if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
1355
- throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
1356
- }
1357
-
1358
- for (const path of paths) {
1359
- let bytes
1360
- let mediaType
1361
- try {
1362
- ;({ bytes, mediaType } = await readImageBytes(ctx, exec, path))
1363
- } catch (error) {
1364
- throw new Error(
1365
- `vision_describe: failed to read ${path} (${error && error.message ? error.message : String(error)})`,
1366
- )
1367
- }
1368
- bytes = await downscaleImage(bytes, 4_000_000)
1369
- let ref
1370
- try {
1371
- ref = await attachments.saveImage({ data: bytes, mediaType })
1372
- } catch (error) {
1373
- throw new Error(
1374
- `vision_describe: image ${path} was rejected (${error && error.message ? error.message : String(error)})`,
1375
- )
1376
- }
1377
- contentIds.push(String(ref.attachmentId))
1378
- blocks.push({ type: 'image', attachment: ref })
1379
- }
1380
-
1381
- for (const id of attachmentIds) {
1382
- const session = exec && exec.agent && exec.agent.session
1383
- const ref = lookupAttachment(session, String(id))
1384
- if (ref === undefined) {
1385
- throw new Error(
1386
- `vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
1387
- )
1388
- }
1389
- let stored
1390
- try {
1391
- stored = await attachments.readImage(ref)
1392
- } catch (error) {
1393
- throw new Error(
1394
- `vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
1395
- )
1396
- }
1397
- if (stored.data && stored.data.length > 0) {
1398
- const resized = await downscaleImage(stored.data, 4_000_000)
1399
- if (resized !== stored.data) {
1400
- try {
1401
- const resizedRef = await attachments.saveImage({
1402
- data: resized,
1403
- mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
1404
- ...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
1405
- })
1406
- stored = { ref: resizedRef, data: resized }
1407
- } catch {
1408
- stored = { ...stored, data: resized }
1409
- }
1410
- }
1411
- }
1412
- contentIds.push(String(ref.attachmentId))
1413
- blocks.push({ type: 'image', attachment: stored.ref })
1414
- }
1415
-
1416
- const question = String(args.question ?? '')
1417
- const wantJson = args.json === true
1418
- const promptText = visionDescribePrompt(question, wantJson)
1419
- // Cache key carries the endpoint/model identity: switching the vision
1420
- // model must never serve stale answers from the previous model.
1421
- const cfg = getConfig()
1422
- const key = `${cfg.baseURL ?? ''}|${cfg.model ?? ''}|${contentIds.join('|')}|${wantJson ? 'json' : 'text'}|${question}`
1423
- const cache = visionCache
1424
- if (cacheEnabled()) {
1425
- const hit = cache.get(key)
1426
- if (hit !== undefined) return hit
1427
- }
1428
-
1429
- // Call with all images as data URLs.
1430
- const imageUrls = []
1431
- for (const block of blocks) {
1432
- const stored = await attachments.readImage(block.attachment)
1433
- imageUrls.push(bytesToDataUrl(stored.data, stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png'))
1434
- }
1435
- const vision = await answerVisionForTool(imageUrls, 'image/png', promptText, { exec })
1436
- if (vision.ok === false) return JSON.stringify(vision)
1437
-
1438
- let text = vision.text
1439
- if (wantJson) {
1440
- for (let attempt = 0; attempt < 2; attempt++) {
1441
- const parsed = extractJson(text)
1442
- if (parsed !== undefined) {
1443
- const compact = JSON.stringify(normalizeDescribeResult(parsed) ?? parsed)
1444
- if (cacheEnabled()) cache.set(key, compact)
1445
- return compact
1446
- }
1447
- if (attempt === 0) {
1448
- const retry = await answerVisionForTool(imageUrls, 'image/png', promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.', { exec })
1449
- if (retry.ok === false) return JSON.stringify(retry)
1450
- text = retry.text
1451
- }
1452
- }
1453
- // Failure/fallback output must NOT be cached: it would poison the
1454
- // cache for the whole TTL with an error string masquerading as an
1455
- // answer.
1456
- return `vision_describe: the model did not produce valid JSON. Raw output:\n${text.slice(0, 2000)}`
1457
- }
1458
- if (text !== '') {
1459
- if (cacheEnabled()) cache.set(key, text)
1460
- return text
1461
- }
1462
- // Empty output is a failure signal, not a cacheable answer.
1463
- return '(the vision model returned empty content)'
1464
- },
1465
- })
1466
-
1467
- // ── vision_materialize ───────────────────────────────────────────────────
1468
- tools.push({
1469
- name: 'vision_materialize',
1470
- description:
1471
- 'Copy an uploaded image attachment (sha256:...) or readable local image into the session workspace and return a real filesystem path. ' +
1472
- 'This tool performs NO vision model/network call. Use it after vision_describe returns ok:false when a local OCR/parser accepts only file_path. ' +
1473
- 'Never guess the attachment store path or search for a same-named file.',
1474
- parameters: {
1475
- type: 'object',
1476
- properties: {
1477
- image: { type: 'string', description: 'Uploaded image attachment id (recommended, e.g. sha256:...) or a readable local image path' },
1478
- },
1479
- required: ['image'],
1480
- additionalProperties: false,
1481
- },
1482
- output: stringOutput,
1483
- async execute(args, exec) {
1484
- const source = String(args.image ?? '')
1485
- const { bytes, mediaType } = await readImageBytes(ctx, exec, source)
1486
- const extension = mediaType === 'image/jpeg' ? '.jpg' : (EXT_BY_MEDIA[mediaType] ?? '.png')
1487
- const target = await saveArtifact(exec, `${artifactStem(source, 'materialized')}${extension}`, bytes)
1488
- return JSON.stringify({ path: target, bytes: bytes.length, mediaType })
1489
- },
1490
- })
1491
-
1492
- // ── vision_ground ────────────────────────────────────────────────────────
1493
- tools.push({
1494
- name: 'vision_ground',
1495
- description:
1496
- 'Locate a target in an image and return its ORIGINAL-pixel bounding box (x1/y1/x2/y2), ' +
1497
- 'optionally producing an annotated PNG artifact. Pair with vision_crop and vision_pixel_diff ' +
1498
- 'for a verify-able pixel loop (reference -> implementation -> screenshot -> metrics). ' +
1499
- 'If the result is JSON with ok:false, the vision backend is unavailable — do not retry with ' +
1500
- 'reworded instructions this turn.',
1501
- parameters: {
1502
- type: 'object',
1503
- properties: {
1504
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1505
- target: { type: 'string', description: 'What to locate, e.g. "the send button"' },
1506
- annotate: { type: 'boolean', description: 'Also write an annotated PNG with the box drawn (default true)' },
1507
- },
1508
- required: ['image', 'target'],
1509
- additionalProperties: false,
1510
- },
1511
- output: stringOutput,
1512
- async execute(args, exec) {
1513
- const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1514
- const { width, height } = await imageDims(bytes)
1515
- if (width <= 0 || height <= 0) throw new Error('vision_ground: could not read image dimensions')
1516
- const instruction =
1517
- `Target to locate: "${String(args.target).slice(0, 500)}". ` +
1518
- `The image is ${width}x${height} pixels. Return ONE JSON object with integer fields ` +
1519
- `{"x1":...,"y1":...,"x2":...,"y2":...} — the tight bounding box of that target in ` +
1520
- `ORIGINAL image pixels (0 <= x1 < x2 <= ${width}, 0 <= y1 < y2 <= ${height}). ` +
1521
- `Output only the JSON object.`
1522
- const vision = await answerVisionForTool(bytes, mediaType, instruction, { exec })
1523
- if (vision.ok === false) return JSON.stringify(vision)
1524
- const parsed = extractJson(vision.text)
1525
- const box = parsed !== undefined ? parseBox(parsed) : undefined
1526
- if (box === undefined) {
1527
- throw new Error(`vision_ground: the vision model did not return a valid box. Raw output: ${vision.text.slice(0, 500)}`)
1528
- }
1529
- let clamped = {
1530
- x1: Math.max(0, Math.min(box.x1, width - 1)),
1531
- y1: Math.max(0, Math.min(box.y1, height - 1)),
1532
- x2: Math.max(1, Math.min(box.x2, width)),
1533
- y2: Math.max(1, Math.min(box.y2, height)),
1534
- }
1535
- if (clamped.x2 - clamped.x1 < 2 || clamped.y2 - clamped.y1 < 2) {
1536
- const retry = await answerVisionForTool(
1537
- bytes,
1538
- mediaType,
1539
- `Your previous box ${JSON.stringify(clamped)} was a degenerate sliver, not the target. ` +
1540
- `Return ONE JSON object with the FULL tight bounding box of the target in ORIGINAL ` +
1541
- `image pixels (0 <= x1 < x2 <= ${width}, 0 <= y1 < y2 <= ${height}). Output only the JSON object.`,
1542
- { exec },
1543
- )
1544
- if (retry.ok === false) return JSON.stringify(retry)
1545
- const retryParsed = extractJson(retry.text)
1546
- const retryBox = retryParsed !== undefined ? parseBox(retryParsed) : undefined
1547
- if (retryBox === undefined) {
1548
- throw new Error(
1549
- `vision_ground: the vision model returned a degenerate box (${clamped.x1},${clamped.y1},${clamped.x2},${clamped.y2}) ` +
1550
- `and the retry returned no valid box. Raw output: ${retry.text.slice(0, 500)}`,
1551
- )
1552
- }
1553
- clamped = {
1554
- x1: Math.max(0, Math.min(retryBox.x1, width - 1)),
1555
- y1: Math.max(0, Math.min(retryBox.y1, height - 1)),
1556
- x2: Math.max(1, Math.min(retryBox.x2, width)),
1557
- y2: Math.max(1, Math.min(retryBox.y2, height)),
1558
- }
1559
- if (clamped.x2 - clamped.x1 < 2 || clamped.y2 - clamped.y1 < 2) {
1560
- throw new Error(
1561
- `vision_ground: the vision model returned only degenerate boxes for a ${width}x${height} image. ` +
1562
- `Last raw output: ${retry.text.slice(0, 500)}`,
1563
- )
1564
- }
1565
- }
1566
- const result = { ...clamped, width, height }
1567
- if (args.annotate !== false) {
1568
- const annotated = await annotateBoxBuffer(bytes, clamped)
1569
- result.annotatedPath = await saveArtifact(exec, `${artifactStem(args.image, 'ground')}.png`, annotated)
1570
- }
1571
- return JSON.stringify(result)
1572
- },
1573
- })
1574
-
1575
- // ── vision_detect ────────────────────────────────────────────────────────
1576
- tools.push({
1577
- name: 'vision_detect',
1578
- description:
1579
- 'Find every element of a kind in an image (buttons, inputs, links, icons…) and return a ' +
1580
- 'numbered inventory with ORIGINAL-pixel boxes, optionally annotated on the image. The model ' +
1581
- 'can then reference "element #3" in follow-up vision_crop / vision_describe calls. ' +
1582
- 'If the result is JSON with ok:false, the vision backend is unavailable — do not retry with ' +
1583
- 'reworded instructions this turn.',
1584
- parameters: {
1585
- type: 'object',
1586
- properties: {
1587
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1588
- target: {
1589
- type: 'string',
1590
- description: 'What kind of elements to list, e.g. "buttons", "input fields", "navigation links" (default: interactive elements)',
1591
- },
1592
- annotate: {
1593
- type: 'boolean',
1594
- description: 'Also write an annotated PNG with numbered boxes (default true)',
1595
- },
1596
- },
1597
- required: ['image'],
1598
- additionalProperties: false,
1599
- },
1600
- output: stringOutput,
1601
- async execute(args, exec) {
1602
- const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1603
- const { width, height } = await imageDims(bytes)
1604
- if (width <= 0 || height <= 0) throw new Error('vision_detect: could not read image dimensions')
1605
- const target = typeof args.target === 'string' && args.target.trim() !== '' ? args.target : 'interactive elements'
1606
- const vision = await answerVisionForTool(bytes, mediaType, visionDetectInstruction(target, width, height), { exec })
1607
- if (vision.ok === false) return JSON.stringify(vision)
1608
- let text = vision.text
1609
- let parsed = extractJson(text)
1610
- if (parsed === undefined) {
1611
- const retry = await answerVisionForTool(
1612
- bytes,
1613
- mediaType,
1614
- visionDetectInstruction(target, width, height) +
1615
- '\nYour previous answer was not valid JSON. Respond with ONLY the JSON object, no prose, no fences.',
1616
- { exec },
1617
- )
1618
- if (retry.ok === false) return JSON.stringify(retry)
1619
- parsed = extractJson(retry.text)
1620
- text = retry.text
1621
- }
1622
- const result = normalizeDetectResult(parsed, width, height)
1623
- if (result === undefined) {
1624
- throw new Error(`vision_detect: the vision model did not return a valid inventory. Raw output: ${text.slice(0, 500)}`)
1625
- }
1626
- if (args.annotate !== false && result.elements.length > 0) {
1627
- const annotated = await annotateBoxesBuffer(bytes, result.elements.map((e) => e.box))
1628
- result.annotatedPath = await saveArtifact(exec, `${artifactStem(args.image, 'detect')}.png`, annotated)
1629
- }
1630
- return JSON.stringify(result)
1631
- },
1632
- })
1633
-
1634
- // ── vision_crop ──────────────────────────────────────────────────────────
1635
- tools.push({
1636
- name: 'vision_crop',
1637
- description:
1638
- 'Crop a pixel region (x1,y1,x2,y2 in ORIGINAL pixels) out of an image and write the ' +
1639
- 'result as a PNG artifact for a closer look. Very large regions are rendered as a bounded ' +
1640
- 'preview; crop a smaller ORIGINAL-pixel region when tiny details must be preserved.',
1641
- parameters: {
1642
- type: 'object',
1643
- properties: {
1644
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1645
- region: {
1646
- type: 'string',
1647
- description: 'Pixel box "x1,y1,x2,y2" in original image coordinates',
1648
- },
1649
- },
1650
- required: ['image', 'region'],
1651
- additionalProperties: false,
1652
- },
1653
- output: stringOutput,
1654
- async execute(args, exec) {
1655
- const { bytes } = await readImageBytes(ctx, exec, args.image)
1656
- const { width, height } = await imageDims(bytes)
1657
- const box = parseBox(args.region)
1658
- if (box === undefined) {
1659
- throw new Error(`vision_crop: invalid region "${args.region}" (expect "x1,y1,x2,y2" integers)`)
1660
- }
1661
- if (box.x2 > width || box.y2 > height) {
1662
- throw new Error(`vision_crop: region exceeds image bounds (${width}x${height})`)
1663
- }
1664
- const sharp = await loadSharp()
1665
- if (!sharp) throw new Error('vision_crop: the sharp image library is unavailable')
1666
- const sourceWidth = box.x2 - box.x1
1667
- const sourceHeight = box.y2 - box.y1
1668
- const preview = scaledDimensions(sourceWidth, sourceHeight, 4_000_000)
1669
- let pipeline = sharp(bytes, { failOn: 'none' }).extract({
1670
- left: box.x1,
1671
- top: box.y1,
1672
- width: sourceWidth,
1673
- height: sourceHeight,
1674
- })
1675
- if (preview.scale !== 1) {
1676
- pipeline = pipeline.resize(preview.width, preview.height, { fit: 'fill' })
1677
- }
1678
- const cropped = await pipeline.png().toBuffer()
1679
- const target = await saveArtifact(
1680
- exec,
1681
- `${artifactStem(args.image, `crop-${box.x1}-${box.y1}-${box.x2}-${box.y2}`)}.png`,
1682
- cropped,
1683
- )
1684
- const meta = await sharp(cropped).metadata()
1685
- return JSON.stringify({
1686
- path: target,
1687
- width: meta.width ?? preview.width,
1688
- height: meta.height ?? preview.height,
1689
- bytes: cropped.length,
1690
- ...(preview.scale !== 1
1691
- ? {
1692
- preview: true,
1693
- sourceRegion: box,
1694
- sourceWidth,
1695
- sourceHeight,
1696
- scale: preview.scale,
1697
- advice: 'This was a bounded preview of a large crop. Use vision_crop again with a smaller ORIGINAL-pixel region for tiny details.',
1698
- }
1699
- : {}),
1700
- })
1701
- },
1702
- })
1703
-
1704
- // ── vision_present ───────────────────────────────────────────────────────
1705
- const visionPresentOutput = {
1706
- schema: {
1707
- type: 'object',
1708
- properties: {
1709
- path: { type: 'string' },
1710
- label: { type: 'string' },
1711
- width: { type: 'number' },
1712
- height: { type: 'number' },
1713
- bytes: { type: 'number' },
1714
- safePresentation: { type: 'boolean' },
1715
- attachment: {
1716
- type: 'object',
1717
- properties: {
1718
- attachmentId: { type: 'string' },
1719
- mediaType: { type: 'string' },
1720
- bytes: { type: 'number' },
1721
- width: { type: 'number' },
1722
- height: { type: 'number' },
1723
- name: { type: 'string' },
1724
- },
1725
- required: ['attachmentId', 'mediaType', 'bytes', 'width', 'height'],
1726
- additionalProperties: false,
1727
- },
1728
- },
1729
- required: ['path', 'label', 'width', 'height', 'bytes', 'safePresentation', 'attachment'],
1730
- additionalProperties: false,
1731
- },
1732
- render: (_args, value) => {
1733
- const attachment = value.attachment
1734
- return [
1735
- {
1736
- type: 'text',
1737
- text: JSON.stringify({
1738
- path: value.path,
1739
- label: value.label,
1740
- width: value.width,
1741
- height: value.height,
1742
- bytes: value.bytes,
1743
- safePresentation: true,
1744
- attachmentId: String(attachment.attachmentId),
1745
- }),
1746
- },
1747
- { type: 'image', attachment },
1748
- ]
1749
- },
1750
- }
1751
- tools.push({
1752
- name: 'vision_present',
1753
- description:
1754
- 'Present a generated local image directly to the user with the host-native image preview. ' +
1755
- 'MANDATORY PRESENTATION RULE: when you generate, edit, screenshot, or export an image and want the user to see it, ' +
1756
- 'you MUST call vision_present. read_image is only for model-side inspection; NEVER use read_image to present or send ' +
1757
- 'an image to the user. The image is retained in the session UI but sanitized out of later text-only model requests.',
1758
- parameters: {
1759
- type: 'object',
1760
- properties: {
1761
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1762
- label: { type: 'string', description: 'Optional short user-facing label for the image' },
1763
- },
1764
- required: ['image'],
1765
- additionalProperties: false,
1766
- },
1767
- output: visionPresentOutput,
1768
- async execute(args, exec) {
1769
- const attachments = ctx.get('attachments')
1770
- if (attachments === undefined) {
1771
- throw new Error('vision_present: the durable attachment service is not available in this deployment')
1772
- }
1773
- const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1774
- const label =
1775
- typeof args.label === 'string' && args.label.trim() !== '' ? args.label.trim().slice(0, 200) : 'image'
1776
- const extension =
1777
- mediaType === 'image/jpeg' ? 'jpg' :
1778
- mediaType === 'image/webp' ? 'webp' :
1779
- mediaType === 'image/gif' ? 'gif' : 'png'
1780
- const target = await saveArtifact(exec, `${artifactStem(args.image, 'present')}.${extension}`, bytes)
1781
- let attachment
1782
- try {
1783
- attachment = await attachments.saveImage({
1784
- data: bytes,
1785
- mediaType,
1786
- name: label,
1787
- })
1788
- } catch (error) {
1789
- throw new Error(
1790
- `vision_present: failed to publish the image attachment (${error && error.message ? error.message : String(error)})`,
1791
- )
1792
- }
1793
- return {
1794
- path: target,
1795
- label,
1796
- width: attachment.width,
1797
- height: attachment.height,
1798
- bytes: attachment.bytes,
1799
- safePresentation: true,
1800
- attachment,
1801
- }
1802
- },
1803
- })
1804
-
1805
- // ── vision_pixel_diff ────────────────────────────────────────────────────
1806
- tools.push({
1807
- name: 'vision_pixel_diff',
1808
- description:
1809
- 'Compare two images pixel by pixel (sharp-based, no Python): returns the differing-pixel ' +
1810
- 'ratio, the worst 8x8-grid regions as original-pixel boxes, and writes a red heatmap PNG ' +
1811
- 'plus a JSON report as artifacts. Use it to verify an implementation against a reference.',
1812
- parameters: {
1813
- type: 'object',
1814
- properties: {
1815
- original: { type: 'string', description: 'Reference image path or attachment id (e.g. "sha256:...")' },
1816
- rebuilt: { type: 'string', description: 'Candidate image path or attachment id (e.g. "sha256:..."); resized to the original size before comparing' },
1817
- threshold: { type: 'number', description: 'Per-channel difference threshold, default 16' },
1818
- },
1819
- required: ['original', 'rebuilt'],
1820
- additionalProperties: false,
1821
- },
1822
- output: stringOutput,
1823
- async execute(args, exec) {
1824
- const { bytes: originalBytes } = await readImageBytes(ctx, exec, args.original)
1825
- const { bytes: rebuiltBytes } = await readImageBytes(ctx, exec, args.rebuilt)
1826
- const sharp = await loadSharp()
1827
- if (!sharp) throw new Error('vision_pixel_diff: the sharp image library is unavailable')
1828
- const meta = await sharp(originalBytes, { failOn: 'none' }).metadata()
1829
- const width = meta.width ?? 0
1830
- const height = meta.height ?? 0
1831
- if (width <= 0 || height <= 0) throw new Error('vision_pixel_diff: could not read original dimensions')
1832
- const threshold = Number.isFinite(args.threshold) && args.threshold >= 0 ? Math.round(args.threshold) : 16
1833
- const pixels = width * height
1834
- let diff
1835
- let heatmapPng
1836
- let heatmapPreview = false
1837
- let heatmapWidth = width
1838
- let heatmapHeight = height
1839
- if (pixels <= 4_000_000) {
1840
- const originalRaw = await sharp(originalBytes, { failOn: 'none' })
1841
- .ensureAlpha()
1842
- .raw()
1843
- .toBuffer({ resolveWithObject: true })
1844
- const rebuiltRaw = await sharp(rebuiltBytes, { failOn: 'none' })
1845
- .resize(width, height, { fit: 'fill' })
1846
- .ensureAlpha()
1847
- .raw()
1848
- .toBuffer({ resolveWithObject: true })
1849
- diff = computePixelDiff(originalRaw.data, rebuiltRaw.data, threshold, width, height)
1850
- const heatmap = renderDiffHeatmap(originalRaw.data, diff.mask, width, height)
1851
- heatmapPng = await sharp(heatmap, { raw: { width, height, channels: 4 } })
1852
- .png()
1853
- .toBuffer()
1854
- } else {
1855
- // Exact large-image metrics from streaming RGBA; heatmap bounded to 4MP.
1856
- const originalStream = sharp(originalBytes, { failOn: 'none' }).ensureAlpha().raw()
1857
- const rebuiltStream = sharp(rebuiltBytes, { failOn: 'none' })
1858
- .resize(width, height, { fit: 'fill' })
1859
- .ensureAlpha()
1860
- .raw()
1861
- diff = await compareRgbaStreams(originalStream, rebuiltStream, { width, height, threshold })
1862
- const preview = scaledDimensions(width, height, 4_000_000)
1863
- heatmapWidth = preview.width
1864
- heatmapHeight = preview.height
1865
- heatmapPreview = true
1866
- const originalPreview = await sharp(originalBytes, { failOn: 'none' })
1867
- .resize(preview.width, preview.height, { fit: 'fill' })
1868
- .ensureAlpha()
1869
- .raw()
1870
- .toBuffer({ resolveWithObject: true })
1871
- const rebuiltPreview = await sharp(rebuiltBytes, { failOn: 'none' })
1872
- .resize(preview.width, preview.height, { fit: 'fill' })
1873
- .ensureAlpha()
1874
- .raw()
1875
- .toBuffer({ resolveWithObject: true })
1876
- const previewDiff = computePixelDiff(
1877
- originalPreview.data,
1878
- rebuiltPreview.data,
1879
- threshold,
1880
- preview.width,
1881
- preview.height,
1882
- )
1883
- const heatmap = renderDiffHeatmap(originalPreview.data, previewDiff.mask, preview.width, preview.height)
1884
- heatmapPng = await sharp(heatmap, {
1885
- raw: { width: preview.width, height: preview.height, channels: 4 },
1886
- }).png().toBuffer()
1887
- }
1888
- const worst = diff.cells.slice(0, 5).map((cell) => ({
1889
- x1: cell.x1,
1890
- y1: cell.y1,
1891
- x2: cell.x2,
1892
- y2: cell.y2,
1893
- ratio: Number(cell.ratio.toFixed(4)),
1894
- differing: cell.differing,
1895
- }))
1896
- const heatmapPath = await saveArtifact(exec, `${artifactStem(args.original, 'diff-heatmap')}.png`, heatmapPng)
1897
- const report = {
1898
- width,
1899
- height,
1900
- differingPixels: diff.differing,
1901
- totalPixels: diff.total,
1902
- differingRatio: Number(diff.ratio.toFixed(6)),
1903
- threshold,
1904
- worstRegions: worst,
1905
- heatmapPath,
1906
- heatmapWidth,
1907
- heatmapHeight,
1908
- heatmapPreview,
1909
- }
1910
- const reportPath = await saveArtifact(exec, `${artifactStem(args.original, 'diff-report')}.json`, Buffer.from(JSON.stringify(report, null, 2)))
1911
- return JSON.stringify({ ...report, reportPath })
1912
- },
1913
- })
1914
-
1915
- // ── vision_colors ────────────────────────────────────────────────────────
1916
- tools.push({
1917
- name: 'vision_colors',
1918
- description:
1919
- 'Extract the dominant colors of an image (sharp-based quantization) with their share of ' +
1920
- 'pixels, e.g. to match a palette when rebuilding a UI.',
1921
- parameters: {
1922
- type: 'object',
1923
- properties: {
1924
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1925
- top: { type: 'number', description: 'How many colors to return, default 8' },
1926
- },
1927
- required: ['image'],
1928
- additionalProperties: false,
1929
- },
1930
- output: stringOutput,
1931
- async execute(args, exec) {
1932
- const { bytes } = await readImageBytes(ctx, exec, args.image)
1933
- const top = Number.isInteger(args.top) && args.top > 0 ? args.top : 8
1934
- const sharp = await loadSharp()
1935
- if (!sharp) throw new Error('vision_colors: the sharp image library is unavailable')
1936
- const raw = await sharp(bytes, { failOn: 'none' })
1937
- .resize(64, 64, { fit: 'inside' })
1938
- .ensureAlpha()
1939
- .raw()
1940
- .toBuffer({ resolveWithObject: true })
1941
- const colors = quantizeColors(raw.data, Math.min(top, 32))
1942
- return JSON.stringify(colors)
1943
- },
1944
- })
1945
-
1946
- // ── vision_ocr ───────────────────────────────────────────────────────────
1947
- tools.push({
1948
- name: 'vision_ocr',
1949
- description:
1950
- 'Transcribe TEXT from an image with the vision model. ' +
1951
- 'Returns the text and which engine produced it. ' +
1952
- 'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
1953
- 'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
1954
- '"这是什么东西" questions are answered by vision_describe, not OCR). If vision_describe returns ' +
1955
- 'ok:false with a backend-unavailable code, calling vision_ocr instead will fail the same way — ' +
1956
- 'do not chain these tools as retries of each other. ' +
1957
- 'ACCURACY: OCR transcribes characters verbatim and is systematically unreliable for confusable ' +
1958
- 'glyphs (1/l, 0/O), spacing and line breaks; prefer vision_describe / vision_detect for semantic ' +
1959
- 'understanding and use OCR only when exact verbatim text is required (executable code, exact ' +
1960
- 'quotation, forms/contracts, table digits, CAPTCHAs). Treat OCR output as evidence to verify, ' +
1961
- 'never as ground truth.',
1962
- parameters: {
1963
- type: 'object',
1964
- properties: {
1965
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1966
- engine: {
1967
- type: 'string',
1968
- description: '"auto" (default) or "vision": transcribe with the configured vision model. "tesseract" is not available in this build.',
1969
- },
1970
- },
1971
- required: ['image'],
1972
- additionalProperties: false,
1973
- },
1974
- output: stringOutput,
1975
- async execute(args, exec) {
1976
- const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1977
- const engine = args.engine === 'vision' ? 'vision' : 'auto'
1978
- if (args.engine === 'tesseract') {
1979
- throw new Error('vision_ocr: the local tesseract engine is not available in this build; use engine "auto" or "vision"')
1980
- }
1981
- const instruction =
1982
- '请原样转述图中的所有文字,保持阅读顺序(从上到下、从左到右)与段落结构,不要添加解释。只输出文字本身。'
1983
- const vision = await answerVisionForTool(bytes, mediaType, instruction, { exec })
1984
- if (vision.ok === false) {
1985
- return JSON.stringify({ engine: 'none', ...vision, text: '' })
1986
- }
1987
- return JSON.stringify({ engine, text: vision.text })
1988
- },
1989
- })
1990
-
1991
- // ── vision_long_screenshot_ocr ───────────────────────────────────────────
1992
- tools.push({
1993
- name: 'vision_long_screenshot_ocr',
1994
- description:
1995
- 'Transcribe a LONG screenshot (chat logs, long documents) into ordered Markdown. ' +
1996
- 'Splits the image into overlapping horizontal chunks, OCRs each chunk with the vision ' +
1997
- 'model, and stitches the text in reading order. Writes chunk PNGs, the Markdown, and a ' +
1998
- 'manifest into the workspace artifacts directory.',
1999
- parameters: {
2000
- type: 'object',
2001
- properties: {
2002
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2003
- chunkHeight: { type: 'number', description: 'Chunk height in pixels, default 1200' },
2004
- overlap: { type: 'number', description: 'Overlap between adjacent chunks in pixels, default 120' },
2005
- engine: { type: 'string', description: '"auto" (default) or "vision": transcribe with the configured vision model.' },
2006
- },
2007
- required: ['image'],
2008
- additionalProperties: false,
2009
- },
2010
- output: stringOutput,
2011
- async execute(args, exec) {
2012
- const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
2013
- const sharp = await loadSharp()
2014
- if (!sharp) throw new Error('vision_long_screenshot_ocr: the sharp image library is unavailable')
2015
- const meta = await sharp(bytes, { failOn: 'none' }).metadata()
2016
- const width = meta.width ?? 0
2017
- const height = meta.height ?? 0
2018
- if (width <= 0 || height <= 0) {
2019
- throw new Error('vision_long_screenshot_ocr: could not read image dimensions')
2020
- }
2021
- const chunkHeight =
2022
- Number.isInteger(args.chunkHeight) && args.chunkHeight >= 400
2023
- ? Math.min(args.chunkHeight, 2000)
2024
- : 1200
2025
- const overlap =
2026
- Number.isInteger(args.overlap) && args.overlap >= 0
2027
- ? Math.min(args.overlap, Math.floor(chunkHeight / 2))
2028
- : 120
2029
- if (args.engine === 'tesseract') {
2030
- throw new Error('vision_long_screenshot_ocr: the local tesseract engine is not available in this build; use engine "auto" or "vision"')
2031
- }
2032
- const windows = boundedOcrTiles(width, height, {
2033
- chunkHeight,
2034
- overlap,
2035
- maxTilePixels: 4_000_000,
2036
- })
2037
- // Hard bounds: a tall screenshot must never turn into an unbounded,
2038
- // uncancellable model-call loop (router's 30s budget was dropped here,
2039
- // so re-introduce explicit limits).
2040
- const MAX_OCR_CHUNKS = 40
2041
- const OCR_TOTAL_BUDGET_MS = 120_000
2042
- const ocrStartedAt = Date.now()
2043
- const signal = exec && exec.signal
2044
- const stem = artifactStem(args.image, 'ocr')
2045
- const results = []
2046
- let visionFailed = false
2047
- for (let i = 0; i < windows.length; i++) {
2048
- if (signal?.aborted) break
2049
- if (i >= MAX_OCR_CHUNKS) {
2050
- results.push({
2051
- chunk: i + 1,
2052
- left: windows[i].left,
2053
- right: windows[i].right,
2054
- top: windows[i].top,
2055
- bottom: windows[i].bottom,
2056
- engine: 'skipped',
2057
- chars: 0,
2058
- text: '',
2059
- error: 'chunk limit reached',
2060
- })
2061
- continue
2062
- }
2063
- if (Date.now() - ocrStartedAt > OCR_TOTAL_BUDGET_MS) {
2064
- results.push({
2065
- chunk: i + 1,
2066
- left: windows[i].left,
2067
- right: windows[i].right,
2068
- top: windows[i].top,
2069
- bottom: windows[i].bottom,
2070
- engine: 'skipped',
2071
- chars: 0,
2072
- text: '',
2073
- error: 'OCR total budget exhausted',
2074
- })
2075
- continue
2076
- }
2077
- const { left, right, top, bottom } = windows[i]
2078
- const tileWidth = right - left
2079
- const tileHeight = bottom - top
2080
- const chunk = await sharp(bytes, { failOn: 'none' })
2081
- .extract({ left, top, width: tileWidth, height: tileHeight })
2082
- .png()
2083
- .toBuffer()
2084
- const chunkRel = `chunk-${String(i + 1).padStart(2, '0')}.png`
2085
- await writeArtifact(exec, join(stem, chunkRel), chunk)
2086
- let text = ''
2087
- let used = 'none'
2088
- // Upload JPEG without an alpha channel: some vision backends degrade
2089
- // on RGBA PNGs and hallucinate token-fragment text.
2090
- const visionBytes = await sharp(chunk, { failOn: 'none' })
2091
- .removeAlpha()
2092
- .jpeg({ quality: 92 })
2093
- .toBuffer()
2094
- const instruction =
2095
- '请原样转述这张长截图分片中的所有文字,保持阅读顺序(从上到下、从左到右),' +
2096
- '不要添加解释,只输出文字本身。如果画面中没有可见文字,只输出 EMPTY,不要编造内容。'
2097
- if (!visionFailed) {
2098
- const visionResult = await answerVisionForTool(visionBytes, 'image/jpeg', instruction, { exec })
2099
- if (visionResult.ok === false) {
2100
- // Backend failure: stop burning calls on the remaining chunks.
2101
- visionFailed = true
2102
- used = 'failed'
2103
- } else {
2104
- text = visionResult.text.trim()
2105
- if (text.length > 12000) {
2106
- const retry = await answerVisionForTool(
2107
- visionBytes,
2108
- 'image/jpeg',
2109
- '重新转写这张图片中的真实文字,保持阅读顺序。只输出图中肉眼可见的文字,' +
2110
- '禁止编造、禁止重复;总输出不超过 3000 字。没有任何文字就只输出 EMPTY。',
2111
- { exec },
2112
- )
2113
- if (retry.ok === false) {
2114
- visionFailed = true
2115
- used = 'failed'
2116
- text = ''
2117
- } else {
2118
- const retryText = retry.text.trim()
2119
- if (retryText !== '') text = retryText
2120
- used = 'vision'
2121
- }
2122
- } else {
2123
- used = 'vision'
2124
- }
2125
- if (text === 'EMPTY') text = ''
2126
- }
2127
- }
2128
- results.push({ chunk: i + 1, left, right, top, bottom, engine: used, chars: text.length, text })
2129
- }
2130
- const joined = results.map((r) => r.text).filter((t) => t !== '').join('\n\n')
2131
- const engines = {}
2132
- for (const r of results) engines[r.engine] = (engines[r.engine] ?? 0) + 1
2133
- const manifest = {
2134
- source: args.image,
2135
- width,
2136
- height,
2137
- chunkHeight,
2138
- overlap,
2139
- chunks: results.length,
2140
- engines,
2141
- perChunk: results.map(({ text, ...rest }) => rest),
2142
- }
2143
- const manifestPath = await writeArtifact(exec, join(stem, 'manifest.json'), JSON.stringify(manifest, null, 2))
2144
- const mdPath = await writeArtifact(exec, join(stem, 'ocr.md'), joined)
2145
- return JSON.stringify({
2146
- text: joined,
2147
- chunks: results.length,
2148
- engines,
2149
- markdownPath: mdPath,
2150
- manifestPath,
2151
- artifactsDir: dirname(mdPath),
2152
- })
2153
- },
2154
- })
2155
-
2156
- // ── vision_trace ─────────────────────────────────────────────────────────
2157
- tools.push({
2158
- name: 'vision_trace',
2159
- description:
2160
- 'Vectorize an image (icon/logo) into an SVG via a local potrace pipeline (no Python). ' +
2161
- 'Default: COLOR-preserving vectorization — one path per dominant color with fill="#rrggbb". ' +
2162
- 'Set color=false for the layered grayscale posterization, where `steps` (1-16, default 4) ' +
2163
- 'controls levels. Writes the SVG as an artifact.',
2164
- parameters: {
2165
- type: 'object',
2166
- properties: {
2167
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2168
- steps: { type: 'number', description: 'Posterization steps, 1-16, default 4 (only when color=false)' },
2169
- color: { type: 'boolean', description: 'Preserve original colors (default true)' },
2170
- colors: { type: 'number', description: 'Number of dominant colors in color mode, 1-16, default 8' },
2171
- },
2172
- required: ['image'],
2173
- additionalProperties: false,
2174
- },
2175
- output: stringOutput,
2176
- async execute(args, exec) {
2177
- const { bytes } = await readImageBytes(ctx, exec, args.image)
2178
- const steps = Number.isInteger(args.steps) && args.steps > 0 ? Math.min(args.steps, 16) : 4
2179
- const colorMode = args.color !== false
2180
- // Trace-specific pixel budget: vectorization gains nothing beyond ~1MP.
2181
- let traceBytes = bytes
2182
- if (bytes && bytes.length > 0) {
2183
- traceBytes = await downscaleImage(bytes, 1_000_000)
2184
- }
2185
- let svg
2186
- let colorCount = 0
2187
- try {
2188
- if (colorMode) {
2189
- const sharp = await loadSharp()
2190
- if (!sharp) throw new Error('vision_trace: the sharp image library is unavailable')
2191
- const colors = Number.isInteger(args.colors) && args.colors > 0 ? Math.min(args.colors, 16) : 8
2192
- const raw = await sharp(traceBytes, { failOn: 'none' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true })
2193
- const palette = quantizeColors(raw.data, colors)
2194
- colorCount = palette.length
2195
- svg = await posterizeSvgColor(raw.data, raw.info, palette, getConfig().timeoutMs || 60000)
2196
- } else {
2197
- svg = await posterizeSvg(traceBytes, steps, 'dominant', getConfig().timeoutMs || 60000)
2198
- }
2199
- } catch (error) {
2200
- throw new Error(
2201
- `vision_trace: potrace failed (${error && error.message ? error.message : String(error)}). ` +
2202
- 'If potrace is missing, install the plugin dependencies in the profile (pnpm add potrace).',
2203
- )
2204
- }
2205
- const target = await saveArtifact(
2206
- exec,
2207
- `${artifactStem(args.image, colorMode ? 'trace-color' : `trace-${steps}`)}.svg`,
2208
- Buffer.from(svg),
2209
- )
2210
- return JSON.stringify({ path: target, bytes: Buffer.byteLength(svg), ...(colorMode ? { colors: colorCount } : {}) })
2211
- },
2212
- })
2213
-
2214
- // ── vision_extract_foreground ────────────────────────────────────────────
2215
- tools.push({
2216
- name: 'vision_extract_foreground',
2217
- description:
2218
- 'Remove a solid-ish background (border flood fill with color tolerance, no Python) and ' +
2219
- 'write the cutout as a transparent PNG artifact. Best for logos on uniform backgrounds.',
2220
- parameters: {
2221
- type: 'object',
2222
- properties: {
2223
- image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2224
- tolerance: { type: 'number', description: 'Max per-channel color distance from the background, default 40' },
2225
- },
2226
- required: ['image'],
2227
- additionalProperties: false,
2228
- },
2229
- output: stringOutput,
2230
- async execute(args, exec) {
2231
- const { bytes } = await readImageBytes(ctx, exec, args.image)
2232
- let fgBytes = bytes
2233
- if (bytes && bytes.length > 0) {
2234
- fgBytes = await downscaleImage(bytes, 4_000_000)
2235
- }
2236
- const tolerance = Number.isFinite(args.tolerance) && args.tolerance >= 0 ? Math.round(args.tolerance) : 40
2237
- const sharp = await loadSharp()
2238
- if (!sharp) throw new Error('vision_extract_foreground: the sharp image library is unavailable')
2239
- const { data, info } = await sharp(fgBytes, { failOn: 'none' })
2240
- .ensureAlpha()
2241
- .raw()
2242
- .toBuffer({ resolveWithObject: true })
2243
- const cutout = floodFillBackground(data, info.width, info.height, tolerance)
2244
- const png = await sharp(cutout, {
2245
- raw: { width: info.width, height: info.height, channels: 4 },
2246
- })
2247
- .png()
2248
- .toBuffer()
2249
- const target = await saveArtifact(exec, `${artifactStem(args.image, 'fg')}.png`, png)
2250
- return JSON.stringify({ path: target, width: info.width, height: info.height, bytes: png.length })
2251
- },
2252
- })
2253
-
2254
- // ── vision_html_screenshot ───────────────────────────────────────────────
2255
- tools.push({
2256
- name: 'vision_html_screenshot',
2257
- description:
2258
- 'Render a local .html/.htm file in the system Chrome (headless, network disabled by ' +
2259
- 'default) and save a PNG screenshot as an artifact — the verify step of the ' +
2260
- 'reference -> implementation -> screenshot -> pixel-diff loop. With fullPage: true the ' +
2261
- 'page keeps the requested viewport but the whole scrollable height is captured and the ' +
2262
- 'result JSON reports pageHeight (CSS px).',
2263
- parameters: {
2264
- type: 'object',
2265
- properties: {
2266
- source: { type: 'string', description: 'Local .html or .htm file path' },
2267
- width: { type: 'number', description: 'Viewport width, default 1200' },
2268
- height: { type: 'number', description: 'Viewport height, default 720' },
2269
- fullPage: {
2270
- type: 'boolean',
2271
- description:
2272
- 'Capture the complete scrollable page height at the requested viewport width ' +
2273
- 'instead of just the viewport (default false). Lazy-loaded images and ' +
2274
- 'scroll-triggered reveals are woken first; the result JSON then includes ' +
2275
- 'pageHeight (CSS px).',
2276
- },
2277
- },
2278
- required: ['source'],
2279
- additionalProperties: false,
2280
- },
2281
- output: stringOutput,
2282
- async execute(args, exec) {
2283
- const source = String(args.source ?? '')
2284
- if (!/\.(html?|htm)$/i.test(source)) {
2285
- throw new Error('vision_html_screenshot: source must be a local .html/.htm file')
2286
- }
2287
- const fsService = ctx.get('fs')
2288
- if (fsService === undefined) {
2289
- throw new Error('vision_html_screenshot: the fs service is not available')
2290
- }
2291
- const resolved = await fsService.resolve(source)
2292
- const targetPath = toRealPath(fsService, resolved)
2293
- if (!existsSync(targetPath)) {
2294
- throw new Error(`vision_html_screenshot: file not found: ${source}`)
2295
- }
2296
- let puppeteer
2297
- try {
2298
- puppeteer = await import('puppeteer-core')
2299
- } catch {
2300
- throw new Error(
2301
- 'vision_html_screenshot: puppeteer-core is not installed. Install the plugin dependencies in the profile (pnpm add puppeteer-core).',
2302
- )
2303
- }
2304
- const candidates = chromiumCandidates(
2305
- typeof process !== 'undefined' && process.env ? process.env : {},
2306
- typeof process !== 'undefined' ? process.platform : '',
2307
- )
2308
- const executablePath = candidates.find((p) => existsSync(p))
2309
- if (executablePath === undefined) {
2310
- throw new Error(
2311
- 'vision_html_screenshot: no Chrome/Chromium/Edge found; install one or set CHROME_PATH / PUPPETEER_EXECUTABLE_PATH',
2312
- )
2313
- }
2314
- const width = Number.isInteger(args.width) && args.width > 0 ? args.width : 1200
2315
- const height = Number.isInteger(args.height) && args.height > 0 ? args.height : 720
2316
- const fullPage = args.fullPage === true
2317
- const launchArgs = ['--no-sandbox', '--disable-gpu', '--hide-scrollbars', '--incognito']
2318
- if (fullPage) {
2319
- launchArgs.push('--blink-settings=imagesLazyLoadingEnabled=false')
2320
- }
2321
- const browser = await puppeteer.default.launch({
2322
- executablePath,
2323
- headless: true,
2324
- args: launchArgs,
2325
- })
2326
- try {
2327
- const page = await browser.newPage()
2328
- await page.setViewport({ width, height })
2329
- // Honor the "network disabled by default" contract: the page may only
2330
- // load file:/data:/blob: subresources; everything else is aborted so
2331
- // a malicious local HTML cannot exfiltrate data or pull remote assets.
2332
- await page.setRequestInterception(true)
2333
- page.on('request', (request) => {
2334
- const url = request.url() ?? ''
2335
- if (/^(file|data|blob):/i.test(url)) request.continue()
2336
- else request.abort()
2337
- })
2338
- await page.goto(pathToFileURL(targetPath).href, { waitUntil: 'networkidle0', timeout: 30000 })
2339
- let pageHeight
2340
- if (fullPage) {
2341
- await wakePageForFullCapture(page, height)
2342
- pageHeight = await fullPageHeightOf(page)
2343
- }
2344
- const png = fullPage
2345
- ? await page.screenshot({ type: 'png', fullPage: true })
2346
- : await page.screenshot({ type: 'png' })
2347
- const stem = fullPage ? `shot-${width}x${height}-fullpage` : `shot-${width}x${height}`
2348
- const target = await saveArtifact(exec, `${artifactStem(source, stem)}.png`, png)
2349
- const result = { path: target, width, height, bytes: png.length }
2350
- if (fullPage) {
2351
- result.pageHeight = pageHeight
2352
- }
2353
- return JSON.stringify(result)
2354
- } finally {
2355
- await browser.close()
2356
- }
2357
- },
2358
- })
2359
-
2360
- // ── vision_screenshot (privacy-gated: only registered when desktopScreenshot is enabled) ──
2361
- if (getConfig().desktopScreenshot === true) {
2362
- tools.push({
2363
- name: 'vision_screenshot',
2364
- description:
2365
- 'Capture the user\'s desktop screen as a PNG artifact (the virtual screen on Windows; the main display on macOS; the root display on Linux). ' +
2366
- 'Windows: PowerShell CopyFromScreen; macOS: screencapture; Linux: ImageMagick import (falls back to scrot; either command must be installed). ' +
2367
- 'Use it when you need to see what is on the user\'s screen right now — e.g. their current GUI, an app, or a page outside this browser. ' +
2368
- 'Optional identify=true also runs recognition on the capture with the configured vision model and returns the description alongside the path.',
2369
- parameters: {
2370
- type: 'object',
2371
- properties: {
2372
- identify: {
2373
- type: 'boolean',
2374
- description:
2375
- 'Also recognize the captured screen with the configured vision model and return the description text with the path. Default false.',
2376
- },
2377
- },
2378
- additionalProperties: false,
2379
- },
2380
- output: stringOutput,
2381
- async execute(args, exec) {
2382
- const tmp = join(
2383
- (typeof process !== 'undefined' && process.env && process.env.TMPDIR) || process.env.TEMP || '/tmp',
2384
- `vision-screenshot-${Date.now()}-${Math.floor(Math.random() * 1e9)}.png`,
2385
- )
2386
- const platform = process.platform
2387
- try {
2388
- if (platform === 'win32') {
2389
- const script = [
2390
- 'Add-Type -AssemblyName System.Windows.Forms,System.Drawing',
2391
- '$b=[System.Windows.Forms.SystemInformation]::VirtualScreen',
2392
- '$bmp=New-Object System.Drawing.Bitmap($b.Width,$b.Height)',
2393
- '$g=[System.Drawing.Graphics]::FromImage($bmp)',
2394
- '$g.CopyFromScreen($b.X,$b.Y,0,0,$bmp.Size)',
2395
- `$bmp.Save('${tmp.replace(/'/g, "''")}')`,
2396
- '$g.Dispose();$bmp.Dispose()',
2397
- ].join('; ')
2398
- await promisify(execFile)('powershell.exe', ['-NoProfile', '-STA', '-Command', script], {
2399
- timeout: 60000,
2400
- windowsHide: true,
2401
- })
2402
- } else if (platform === 'darwin') {
2403
- await promisify(execFile)('screencapture', ['-x', '-m', tmp], {
2404
- timeout: 60000,
2405
- windowsHide: true,
2406
- })
2407
- } else {
2408
- try {
2409
- await promisify(execFile)('import', ['-window', 'root', tmp], { timeout: 60000 })
2410
- } catch {
2411
- await promisify(execFile)('scrot', [tmp], { timeout: 60000 })
2412
- }
2413
- }
2414
- if (!existsSync(tmp)) {
2415
- throw new Error(
2416
- `vision_screenshot: no output produced on ${platform} (is a screen available?)`,
2417
- )
2418
- }
2419
- const data = await readFileP(tmp)
2420
- const target = await saveArtifact(exec, `screenshot-${Date.now()}.png`, data)
2421
- const result = { path: target, bytes: data.length }
2422
- if (args.identify === true) {
2423
- let identifyBytes = data
2424
- try {
2425
- const sharp = await loadSharp()
2426
- if (sharp) {
2427
- const downscaled = await sharp(data, { failOn: 'none' })
2428
- .resize({ width: 1280, height: 1280, fit: 'inside', withoutEnlargement: true })
2429
- .png()
2430
- .toBuffer()
2431
- if (downscaled.length > 0 && downscaled.length < data.length) {
2432
- identifyBytes = downscaled
2433
- }
2434
- }
2435
- } catch {
2436
- /* keep the original capture */
2437
- }
2438
- const vision = await answerVisionForTool(
2439
- identifyBytes,
2440
- 'image/png',
2441
- 'Describe what is currently visible on the user\'s screen, in detail and in reading order. Mention the app/window names you can see.',
2442
- { exec },
2443
- )
2444
- if (vision.ok === false) {
2445
- result.identify = { ok: false, ...vision, text: '' }
2446
- } else {
2447
- result.identify = { ok: true, text: vision.text }
2448
- }
2449
- }
2450
- try {
2451
- await rmP(tmp, { force: true })
2452
- } catch {
2453
- /* best-effort cleanup */
2454
- }
2455
- return JSON.stringify(result)
2456
- } catch (error) {
2457
- throw new Error(
2458
- `vision_screenshot: failed (${error && error.message ? error.message : String(error)})`,
2459
- )
2460
- }
2461
- },
2462
- })
2463
- }
2464
-
2465
- for (const def of tools) {
2466
- ctx.tools.register(def)
2467
- }
2468
- logger?.info?.('[tool-vision] registered %d pixel-level vision tools', tools.length)
2469
- return tools.length
2470
- }
2471
-
2472
- // ── cache instance shared across tools ─────────────────────────────────────
2473
-
2474
- const visionCache = createCache(200, 3600 * 1000)
2475
- let cacheDisabled = false
2476
-
2477
- function cacheEnabled() {
2478
- return !cacheDisabled
2479
- }
2480
-
2481
- /** Test hook: reset the shared cache. */
2482
- export function resetVisionCacheForTests() {
2483
- visionCache.clear()
2484
- cacheDisabled = false
2485
- }
1
+ // dsh-tool-vision — pixel-level vision tools (ported from dsh-vision-router).
2
+ //
3
+ // The 14 vision_* tools share ONE configured OpenAI-compatible endpoint
4
+ // (the plugin's own baseURL/apiKey/model settings — the same GLM endpoint
5
+ // inspect_image uses). No provider chain, no local models, no fallbacks:
6
+ // the router's tool *semantics* (prompts, parameter contracts, pixel
7
+ // post-processing) are kept; its routing/resilience machinery is not.
8
+ //
9
+ // Everything here is unit-testable without a harness: the only injected
10
+ // dependencies are `ctx` (attachments/fs services), `getConfig` (live
11
+ // settings) and `exec` (tool execution context).
12
+
13
+ import { createHash } from 'node:crypto'
14
+ import { join, dirname, isAbsolute, relative as relativePath, sep, win32 as winPath } from 'node:path'
15
+ import { existsSync } from 'node:fs'
16
+ import { mkdir as mkdirP, writeFile as writeFileP, readFile as readFileP, rm as rmP, stat as statP } from 'node:fs/promises'
17
+ import { Worker } from 'node:worker_threads'
18
+ import { execFile } from 'node:child_process'
19
+ import { promisify } from 'node:util'
20
+ import { pathToFileURL } from 'node:url'
21
+ import { createRequire } from 'node:module'
22
+ import { sessionHeaders } from './session-header.js'
23
+
24
+ // ── image formats ──────────────────────────────────────────────────────────
25
+
26
+ export const IMAGE_EXTENSIONS = {
27
+ png: 'image/png',
28
+ jpg: 'image/jpeg',
29
+ jpeg: 'image/jpeg',
30
+ webp: 'image/webp',
31
+ gif: 'image/gif',
32
+ bmp: 'image/bmp',
33
+ avif: 'image/avif',
34
+ svg: 'image/svg+xml',
35
+ ico: 'image/x-icon',
36
+ }
37
+
38
+ const EXT_BY_MEDIA = {
39
+ 'image/png': '.png',
40
+ 'image/jpeg': '.jpg',
41
+ 'image/webp': '.webp',
42
+ 'image/gif': '.gif',
43
+ 'image/bmp': '.bmp',
44
+ 'image/avif': '.avif',
45
+ 'image/svg+xml': '.svg',
46
+ 'image/x-icon': '.ico',
47
+ }
48
+
49
+ export function mediaTypeOf(path) {
50
+ const match = String(path).toLowerCase().match(/\.([a-z0-9]+)$/)
51
+ return match ? IMAGE_EXTENSIONS[match[1]] : undefined
52
+ }
53
+
54
+ /**
55
+ * Detect the image format from magic bytes instead of the file extension.
56
+ * Attachments are stored as content-addressed files WITHOUT an extension,
57
+ * so extension-based detection rejects them; the pixel tools must sniff.
58
+ */
59
+ export function sniffMediaType(bytes) {
60
+ if (!bytes || bytes.length < 12) return undefined
61
+ const head = (offset, count) => {
62
+ const parts = []
63
+ for (let i = offset; i < offset + count; i++) parts.push(bytes[i].toString(16).padStart(2, '0'))
64
+ return parts.join('')
65
+ }
66
+ if (head(0, 8) === '89504e470d0a1a0a') return 'image/png'
67
+ if (head(0, 3) === 'ffd8ff') return 'image/jpeg'
68
+ const riff = head(0, 4)
69
+ const webp = head(8, 4)
70
+ if (riff === '52494646' && webp === '57454250') return 'image/webp'
71
+ if (riff === '47494638') return 'image/gif' // GIF87a / GIF89a
72
+ return undefined
73
+ }
74
+
75
+ // ── input resolution ───────────────────────────────────────────────────────
76
+
77
+ export function basenameOf(path) {
78
+ const parts = String(path).split('/')
79
+ return parts[parts.length - 1] || undefined
80
+ }
81
+
82
+ /**
83
+ * True when the string is a durable attachment id such as "sha256:<hex>" —
84
+ * the form the harness uses for uploaded images.
85
+ */
86
+ export function isAttachmentIdInput(input) {
87
+ return typeof input === 'string' && /^[a-z0-9]+:[0-9a-f]{32,}$/i.test(input.trim())
88
+ }
89
+
90
+ /** Cross-platform absolute-path test (posix drive-relative forms included). */
91
+ export function isAbsolutePath(input) {
92
+ const value = String(input ?? '')
93
+ return isAbsolute(value) || winPath.isAbsolute(value)
94
+ }
95
+
96
+ /** Largest local image accepted by the pixel tools, in bytes. */
97
+ export const MAX_IMAGE_BYTES = 20 * 1024 * 1024
98
+
99
+ /** Resolve one tool argument (local path or attachment id) to image bytes. */
100
+ export async function readImageBytes(ctx, exec, imagePath) {
101
+ const input = String(imagePath ?? '')
102
+ let bytes
103
+ let storedMediaType
104
+ if (isAttachmentIdInput(input)) {
105
+ const attachments = ctx.get('attachments')
106
+ if (attachments === undefined) {
107
+ throw new Error('tool-vision: the attachment service is not available in this deployment')
108
+ }
109
+ const session = exec && exec.agent && exec.agent.session
110
+ const ref = lookupAttachment(session, input.trim())
111
+ if (ref === undefined) {
112
+ throw new Error(
113
+ `tool-vision: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
114
+ )
115
+ }
116
+ let stored
117
+ try {
118
+ stored = await attachments.readImage(ref)
119
+ } catch (error) {
120
+ throw new Error(
121
+ `tool-vision: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
122
+ )
123
+ }
124
+ bytes = stored.data
125
+ if (bytes && bytes.length > MAX_IMAGE_BYTES) {
126
+ throw new Error(
127
+ `tool-vision: image too large: ${input} (${bytes.length} bytes, limit ${MAX_IMAGE_BYTES})`,
128
+ )
129
+ }
130
+ if (stored.ref && typeof stored.ref.mediaType === 'string') {
131
+ storedMediaType = stored.ref.mediaType
132
+ }
133
+ } else {
134
+ const fsService = ctx.get('fs')
135
+ if (fsService === undefined) throw new Error('tool-vision: the fs service is not available')
136
+ const resolved = await fsService.resolve(input)
137
+ const target = toRealPath(fsService, resolved)
138
+ // Second line of defense beyond the fs sandbox: a relative input must
139
+ // resolve inside the session workspace (refuse ../ escapes).
140
+ if (!isAbsolutePath(input)) {
141
+ const root = workspaceOf(exec)
142
+ const relative = relativePath(root, target)
143
+ if (relative === '..' || relative.startsWith('..' + sep) || isAbsolutePath(relative)) {
144
+ throw new Error(`tool-vision: path escapes the workspace: ${input}`)
145
+ }
146
+ }
147
+ // Pre-check size BEFORE reading: a huge file must not be buffered whole
148
+ // (readFileP has no limit and would OOM the process).
149
+ const info = await statP(target).catch(() => null)
150
+ if (info !== null && info.size > MAX_IMAGE_BYTES) {
151
+ throw new Error(
152
+ `tool-vision: image too large: ${input} (${info.size} bytes, limit ${MAX_IMAGE_BYTES})`,
153
+ )
154
+ }
155
+ bytes = await readFileP(target)
156
+ if (bytes.length > MAX_IMAGE_BYTES) {
157
+ throw new Error(`tool-vision: image too large: ${input} (${bytes.length} bytes, limit ${MAX_IMAGE_BYTES})`)
158
+ }
159
+ }
160
+ const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
161
+ if (mediaType === undefined) {
162
+ throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
163
+ }
164
+ return { bytes, mediaType }
165
+ }
166
+
167
+ /** Find an uploaded attachment ref in the session's recorded upload index. */
168
+ export function lookupAttachment(session, attachmentId) {
169
+ if (!session) return undefined
170
+ const recorded = session.recordedUploads
171
+ if (recorded) {
172
+ const ref = recorded.get ? recorded.get(attachmentId) : undefined
173
+ if (ref) return ref
174
+ }
175
+ const events = session.events ?? []
176
+ for (const event of events) {
177
+ const data = event && event.data
178
+ if (!data) continue
179
+ for (const block of Array.isArray(data.content) ? data.content : []) {
180
+ if (block && block.type === 'image' && block.attachment && block.attachment.attachmentId === attachmentId) {
181
+ return block.attachment
182
+ }
183
+ }
184
+ const message = data.message
185
+ if (message && Array.isArray(message.content)) {
186
+ for (const block of message.content) {
187
+ if (block && block.type === 'image' && block.attachment && block.attachment.attachmentId === attachmentId) {
188
+ return block.attachment
189
+ }
190
+ }
191
+ }
192
+ }
193
+ return undefined
194
+ }
195
+
196
+ /** Normalize an fs.resolve() result (string or { targetKey, displayPath }) to a real path. */
197
+ export function toRealPath(fsService, resolved) {
198
+ if (typeof resolved === 'string') return resolved
199
+ if (typeof fsService?.processPath === 'function') {
200
+ const p = fsService.processPath(resolved)
201
+ if (typeof p === 'string' && p !== '') return p
202
+ }
203
+ const key = resolved?.targetKey
204
+ return typeof key === 'string' && key !== '' ? key : String(resolved ?? '')
205
+ }
206
+
207
+ // ── sharp (lazy) ───────────────────────────────────────────────────────────
208
+
209
+ let sharpPromise = null
210
+
211
+ /** Lazily load sharp; resolves null when the host cannot provide it. */
212
+ export function loadSharp() {
213
+ if (!sharpPromise) {
214
+ sharpPromise = import('sharp')
215
+ .then((mod) => mod.default ?? mod)
216
+ .catch(() => null)
217
+ }
218
+ return sharpPromise
219
+ }
220
+
221
+ /**
222
+ * Downscale an image so its pixel count stays within maxPixels (router
223
+ * default 4MP). Returns the original buffer when the image is already small
224
+ * enough or when sharp is unavailable — pixel tools degrade gracefully
225
+ * instead of failing.
226
+ */
227
+ export async function downscaleImage(bytes, maxPixels) {
228
+ if (!bytes || bytes.length === 0) return bytes
229
+ const sharp = await loadSharp()
230
+ if (!sharp) return bytes
231
+ try {
232
+ const image = sharp(bytes, { failOn: 'none' })
233
+ const meta = await image.metadata()
234
+ const width = Number(meta.width) || 0
235
+ const height = Number(meta.height) || 0
236
+ if (width <= 0 || height <= 0 || width * height <= maxPixels) return bytes
237
+ const scale = Math.sqrt(maxPixels / (width * height))
238
+ const resized = await image
239
+ .resize({
240
+ width: Math.max(1, Math.floor(width * scale)),
241
+ height: Math.max(1, Math.floor(height * scale)),
242
+ fit: 'inside',
243
+ })
244
+ .toBuffer()
245
+ return resized && resized.length > 0 ? resized : bytes
246
+ } catch {
247
+ return bytes
248
+ }
249
+ }
250
+
251
+ /** Image dimensions via sharp (0/0 when unavailable). */
252
+ export async function imageDims(bytes) {
253
+ const sharp = await loadSharp()
254
+ if (!sharp) return { width: 0, height: 0 }
255
+ try {
256
+ const meta = await sharp(bytes, { failOn: 'none' }).metadata()
257
+ return { width: meta.width ?? 0, height: meta.height ?? 0 }
258
+ } catch {
259
+ return { width: 0, height: 0 }
260
+ }
261
+ }
262
+
263
+ // ── content-hash cache ─────────────────────────────────────────────────────
264
+
265
+ export function sha256Of(bytes) {
266
+ return createHash('sha256').update(bytes).digest('hex')
267
+ }
268
+
269
+ /** Small LRU+TTL cache for repeated image+question calls (router semantics). */
270
+ export function createCache(maxEntries, ttlMs) {
271
+ const map = new Map()
272
+ return {
273
+ get(key) {
274
+ const entry = map.get(key)
275
+ if (!entry) return undefined
276
+ if (entry.expires <= Date.now()) {
277
+ map.delete(key)
278
+ return undefined
279
+ }
280
+ map.delete(key)
281
+ map.set(key, entry)
282
+ return entry.value
283
+ },
284
+ set(key, value) {
285
+ if (map.size >= maxEntries) {
286
+ const oldest = map.keys().next().value
287
+ if (oldest !== undefined) map.delete(oldest)
288
+ }
289
+ map.set(key, { value, expires: Date.now() + ttlMs })
290
+ },
291
+ size() {
292
+ return map.size
293
+ },
294
+ clear() {
295
+ map.clear()
296
+ },
297
+ }
298
+ }
299
+
300
+ // ── artifacts (tool outputs land in the session workspace) ─────────────────
301
+
302
+ export function workspaceOf(exec) {
303
+ const session = exec && exec.agent && exec.agent.session
304
+ const cwd = session && session.header && session.header.cwd
305
+ return typeof cwd === 'string' && cwd !== '' ? cwd : process.cwd()
306
+ }
307
+
308
+ /** Write a tool artifact under <workspace>/.dsh-tool-vision/ and return its path.
309
+ * Recursive: long-OCR writes stem/chunk-*.png subdirectories. */
310
+ export async function writeArtifact(exec, relPath, data) {
311
+ const root = workspaceOf(exec)
312
+ const target = join(root, '.dsh-tool-vision', relPath)
313
+ await mkdirP(dirname(target), { recursive: true })
314
+ await writeFileP(target, data)
315
+ return target
316
+ }
317
+
318
+ /** Build an artifact filename stem from an input image reference. */
319
+ export function artifactStemOf(imagePath, suffix) {
320
+ const base = String(basenameOf(imagePath) ?? 'image')
321
+ .replace(/\.(png|jpe?g|webp|gif)$/i, '')
322
+ .replace(/[^a-zA-Z0-9._-]/g, '-')
323
+ .slice(0, 32)
324
+ const fingerprint = createHash('sha256').update(String(imagePath)).digest('hex').slice(0, 8)
325
+ return `${base || 'image'}-${fingerprint}-${suffix}`
326
+ }
327
+
328
+ // ── JSON & prompt helpers ──────────────────────────────────────────────────
329
+
330
+ const MAX_EXTRACT_JSON_CHARS = 200_000
331
+
332
+ /** Extract the first balanced JSON value ({...} or [...]) from model output. */
333
+ export function extractJson(text) {
334
+ const source = String(text ?? '')
335
+ const bounded = source.length > MAX_EXTRACT_JSON_CHARS
336
+ ? source.slice(0, MAX_EXTRACT_JSON_CHARS)
337
+ : source
338
+ const fenced = bounded.match(/```(?:json)?\s*([\s\S]*?)```/i)
339
+ const candidate = fenced ? fenced[1] : bounded
340
+ const start = candidate.search(/[[{]/)
341
+ if (start === -1) return undefined
342
+ const stack = []
343
+ let inString = false
344
+ let escaped = false
345
+ for (let index = start; index < candidate.length; index++) {
346
+ const char = candidate[index]
347
+ if (inString) {
348
+ if (escaped) escaped = false
349
+ else if (char === '\\') escaped = true
350
+ else if (char === '"') inString = false
351
+ continue
352
+ }
353
+ if (char === '"') {
354
+ inString = true
355
+ continue
356
+ }
357
+ if (char === '{') stack.push('}')
358
+ else if (char === '[') stack.push(']')
359
+ else if (char === '}' || char === ']') {
360
+ if (stack.length === 0 || stack.pop() !== char) return undefined
361
+ if (stack.length === 0) {
362
+ try {
363
+ const value = JSON.parse(candidate.slice(start, index + 1))
364
+ return typeof value === 'object' && value !== null ? value : undefined
365
+ } catch {
366
+ return undefined
367
+ }
368
+ }
369
+ }
370
+ }
371
+ return undefined
372
+ }
373
+
374
+ /** Legacy alias for callers of the earlier draft API. */
375
+ export function parseModelJson(text) {
376
+ return extractJson(text)
377
+ }
378
+
379
+ /** Parse "x1,y1,x2,y2" (string) or {x1,y1,x2,y2} (object) into a valid pixel box. */
380
+ export function parseBox(value) {
381
+ let box
382
+ if (typeof value === 'string') {
383
+ const parts = value.split(',').map((part) => Number(part.trim()))
384
+ if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return undefined
385
+ box = { x1: parts[0], y1: parts[1], x2: parts[2], y2: parts[3] }
386
+ } else if (value && typeof value === 'object') {
387
+ box = { x1: value.x1, y1: value.y1, x2: value.x2, y2: value.y2 }
388
+ } else {
389
+ return undefined
390
+ }
391
+ const { x1, y1, x2, y2 } = box
392
+ if (![x1, y1, x2, y2].every((n) => Number.isInteger(n))) return undefined
393
+ if (x1 < 0 || y1 < 0 || x2 <= x1 || y2 <= y1) return undefined
394
+ return { x1, y1, x2, y2 }
395
+ }
396
+
397
+ /** Fixed JSON contract for vision_detect. */
398
+ export function visionDetectInstruction(target, width, height) {
399
+ return (
400
+ `The image is ${width}x${height} pixels. Find every "${String(target).slice(0, 300)}" in it. ` +
401
+ 'Return ONE JSON object and nothing else, shaped EXACTLY as:\n' +
402
+ '{"elements":[{"label":"<short element name>","box":{"x1":0,"y1":0,"x2":0,"y2":0}},...]}\n' +
403
+ '- "elements" is a numbered list (array order = element number) of every match, from top-left to bottom-right in reading order;\n' +
404
+ '- every box is the tight bounding box in ORIGINAL image pixels, integers, 0 <= x1 < x2 <= ' +
405
+ `${width}, 0 <= y1 < y2 <= ${height}` +
406
+ ';\n- if nothing matches, return {"elements":[]}.'
407
+ )
408
+ }
409
+
410
+ /** Structured JSON contract for vision_describe's json:true mode. */
411
+ export function describeStructuredInstruction(question) {
412
+ return (
413
+ `Look at the image and answer the question: 「${String(question).slice(0, 1500)}」. ` +
414
+ 'Return ONE JSON object and nothing else, shaped EXACTLY as:\n' +
415
+ '{"summary":"<1-2 sentence answer to the question>",' +
416
+ '"layout":[{"region":"<e.g. top-left / header / center>","content":"<what is there>"}],' +
417
+ '"entities":[{"type":"<button|input|text|image|link|icon|other>","label":"<name or text>"}],' +
418
+ '"text":"<the full text visible in the image, transcribed in reading order, as faithful as possible>"}\n' +
419
+ '- "layout" lists the main regions in reading order (top-to-bottom, left-to-right);\n' +
420
+ '- "entities" lists notable elements; use only the listed type values;\n' +
421
+ '- "text" is the verbatim transcription; write "" when the image contains no text.'
422
+ )
423
+ }
424
+
425
+ /** Shared vision_describe prompt. */
426
+ export function visionDescribePrompt(question, wantJson = false) {
427
+ const raw = String(question ?? '').trim()
428
+ const text = raw === ''
429
+ ? 'Describe the image accurately and answer based only on visible content.'
430
+ : raw
431
+ return wantJson ? text + '\n\n' + describeStructuredInstruction(text) : text
432
+ }
433
+
434
+ /** Normalize a vision_detect answer into the canonical shape, clamping boxes. */
435
+ export function normalizeDetectResult(parsed, width, height) {
436
+ if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.elements)) return undefined
437
+ const clamp = (value, min, max) => Math.max(min, Math.min(value, max))
438
+ const elements = []
439
+ for (const item of parsed.elements) {
440
+ if (!item || typeof item !== 'object' || !item.box || typeof item.box !== 'object') continue
441
+ const x1 = Math.round(Number(item.box.x1))
442
+ const y1 = Math.round(Number(item.box.y1))
443
+ const x2 = Math.round(Number(item.box.x2))
444
+ const y2 = Math.round(Number(item.box.y2))
445
+ if (![x1, y1, x2, y2].every(Number.isFinite)) continue
446
+ const box = {
447
+ x1: clamp(x1, 0, width - 1),
448
+ y1: clamp(y1, 0, height - 1),
449
+ x2: clamp(x2, 1, width),
450
+ y2: clamp(y2, 1, height),
451
+ }
452
+ if (box.x2 <= box.x1 || box.y2 <= box.y1) continue
453
+ elements.push({
454
+ number: elements.length + 1,
455
+ label: typeof item.label === 'string' && item.label.trim() !== '' ? item.label.trim() : `element ${elements.length + 1}`,
456
+ box,
457
+ })
458
+ }
459
+ return { width, height, elements }
460
+ }
461
+
462
+ /** Normalize a structured vision_describe answer with defaults for missing fields. */
463
+ export function normalizeDescribeResult(parsed) {
464
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined
465
+ const layout = Array.isArray(parsed.layout) ? parsed.layout.filter((r) => r && typeof r === 'object' && typeof r.region === 'string' && typeof r.content === 'string') : []
466
+ const entities = Array.isArray(parsed.entities)
467
+ ? parsed.entities
468
+ .filter((e) => e && typeof e === 'object' && typeof e.type === 'string' && typeof e.label === 'string')
469
+ .map((e) => ({ type: e.type, label: e.label }))
470
+ : []
471
+ return {
472
+ summary: typeof parsed.summary === 'string' ? parsed.summary : '',
473
+ layout,
474
+ entities,
475
+ text: typeof parsed.text === 'string' ? parsed.text : '',
476
+ }
477
+ }
478
+
479
+ // ── pixel operations ───────────────────────────────────────────────────────
480
+
481
+ /** Per-pixel RGBA comparison; returns ratio, mask and worst 8x8 cells. */
482
+ export function computePixelDiff(bufferA, bufferB, threshold = 16, width = 0, height = 0) {
483
+ const length = Math.min(bufferA.length, bufferB.length)
484
+ const pixels = Math.floor(length / 4)
485
+ let differing = 0
486
+ const mask = new Uint8Array(pixels)
487
+ for (let i = 0; i < pixels; i++) {
488
+ const o = i * 4
489
+ const d =
490
+ Math.max(
491
+ Math.abs(bufferA[o] - bufferB[o]),
492
+ Math.abs(bufferA[o + 1] - bufferB[o + 1]),
493
+ Math.abs(bufferA[o + 2] - bufferB[o + 2]),
494
+ ) - threshold
495
+ if (d > 0) {
496
+ differing += 1
497
+ mask[i] = 1
498
+ }
499
+ }
500
+ const ratio = pixels === 0 ? 0 : differing / pixels
501
+ const cells = []
502
+ if (width > 0 && height > 0) {
503
+ const cols = 8
504
+ const rows = 8
505
+ const cw = Math.ceil(width / cols)
506
+ const ch = Math.ceil(height / rows)
507
+ for (let cy = 0; cy < rows; cy++) {
508
+ for (let cx = 0; cx < cols; cx++) {
509
+ let hit = 0
510
+ let total = 0
511
+ for (let y = cy * ch; y < Math.min((cy + 1) * ch, height); y++) {
512
+ for (let x = cx * cw; x < Math.min((cx + 1) * cw, width); x++) {
513
+ total += 1
514
+ if (mask[y * width + x]) hit += 1
515
+ }
516
+ }
517
+ if (total > 0 && hit > 0) {
518
+ cells.push({
519
+ x1: cx * cw,
520
+ y1: cy * ch,
521
+ x2: Math.min((cx + 1) * cw, width),
522
+ y2: Math.min((cy + 1) * ch, height),
523
+ ratio: hit / total,
524
+ differing: hit,
525
+ total,
526
+ })
527
+ }
528
+ }
529
+ }
530
+ cells.sort((a, b) => b.ratio - a.ratio)
531
+ }
532
+ return { differing, total: pixels, ratio, mask, cells }
533
+ }
534
+
535
+ function asBuffer(chunk) {
536
+ if (Buffer.isBuffer(chunk)) return chunk
537
+ if (chunk instanceof Uint8Array) return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
538
+ return Buffer.from(chunk ?? [])
539
+ }
540
+
541
+ class AsyncByteReader {
542
+ constructor(iterable) {
543
+ if (!iterable || typeof iterable[Symbol.asyncIterator] !== 'function') {
544
+ throw new TypeError('pixel diff stream must be async iterable')
545
+ }
546
+ this.iterator = iterable[Symbol.asyncIterator]()
547
+ this.buffer = Buffer.alloc(0)
548
+ this.done = false
549
+ }
550
+
551
+ async fill(minBytes) {
552
+ while (!this.done && this.buffer.length < minBytes) {
553
+ const next = await this.iterator.next()
554
+ if (next.done) {
555
+ this.done = true
556
+ break
557
+ }
558
+ const chunk = asBuffer(next.value)
559
+ if (chunk.length === 0) continue
560
+ this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk])
561
+ }
562
+ }
563
+
564
+ consume(bytes) {
565
+ const out = this.buffer.subarray(0, bytes)
566
+ this.buffer = this.buffer.subarray(bytes)
567
+ return out
568
+ }
569
+ }
570
+
571
+ function cellBounds(index, width, height, cols, rows) {
572
+ const cx = index % cols
573
+ const cy = Math.floor(index / cols)
574
+ const cw = Math.ceil(width / cols)
575
+ const ch = Math.ceil(height / rows)
576
+ const x1 = cx * cw
577
+ const y1 = cy * ch
578
+ const x2 = Math.min((cx + 1) * cw, width)
579
+ const y2 = Math.min((cy + 1) * ch, height)
580
+ return { x1, y1, x2, y2, total: Math.max(0, x2 - x1) * Math.max(0, y2 - y1) }
581
+ }
582
+
583
+ /**
584
+ * Exact RGB pixel comparison over two RGBA async streams. The function never
585
+ * materializes either full frame; it holds only the current upstream chunks
586
+ * plus a bounded region counter grid.
587
+ */
588
+ export async function compareRgbaStreams(
589
+ streamA,
590
+ streamB,
591
+ { width, height, threshold = 16, cols = 8, rows = 8 } = {},
592
+ ) {
593
+ const w = Math.max(1, Math.floor(Number(width) || 1))
594
+ const h = Math.max(1, Math.floor(Number(height) || 1))
595
+ const limit = w * h
596
+ const gate = Number.isFinite(Number(threshold)) && Number(threshold) >= 0
597
+ ? Number(threshold)
598
+ : 16
599
+ const gridCols = Math.min(w, Math.max(1, Math.floor(Number(cols) || 8)))
600
+ const gridRows = Math.min(h, Math.max(1, Math.floor(Number(rows) || 8)))
601
+ const cellDiff = new Uint32Array(gridCols * gridRows)
602
+ const readerA = new AsyncByteReader(streamA)
603
+ const readerB = new AsyncByteReader(streamB)
604
+ let pixels = 0
605
+ let differing = 0
606
+
607
+ while (pixels < limit) {
608
+ await Promise.all([readerA.fill(4), readerB.fill(4)])
609
+ if (readerA.buffer.length < 4 || readerB.buffer.length < 4) break
610
+ const remainingBytes = (limit - pixels) * 4
611
+ const usable = Math.min(
612
+ remainingBytes,
613
+ Math.floor(Math.min(readerA.buffer.length, readerB.buffer.length) / 4) * 4,
614
+ )
615
+ if (usable <= 0) break
616
+ const a = readerA.consume(usable)
617
+ const b = readerB.consume(usable)
618
+ const chunkPixels = usable / 4
619
+ for (let i = 0; i < chunkPixels; i++) {
620
+ const o = i * 4
621
+ const different =
622
+ Math.max(
623
+ Math.abs(a[o] - b[o]),
624
+ Math.abs(a[o + 1] - b[o + 1]),
625
+ Math.abs(a[o + 2] - b[o + 2]),
626
+ ) > gate
627
+ if (!different) continue
628
+ differing += 1
629
+ const absolute = pixels + i
630
+ const x = absolute % w
631
+ const y = Math.floor(absolute / w)
632
+ const cx = Math.min(gridCols - 1, Math.floor((x * gridCols) / w))
633
+ const cy = Math.min(gridRows - 1, Math.floor((y * gridRows) / h))
634
+ cellDiff[cy * gridCols + cx] += 1
635
+ }
636
+ pixels += chunkPixels
637
+ }
638
+
639
+ if (pixels !== limit) {
640
+ throw new Error(`pixel diff stream ended early (${pixels}/${limit} pixels)`)
641
+ }
642
+ await Promise.all([readerA.fill(1), readerB.fill(1)])
643
+ if (readerA.buffer.length > 0 || readerB.buffer.length > 0) {
644
+ throw new Error('pixel diff stream produced more RGBA bytes than the declared dimensions')
645
+ }
646
+
647
+ const cells = []
648
+ for (let index = 0; index < cellDiff.length; index++) {
649
+ const hit = cellDiff[index]
650
+ if (hit === 0) continue
651
+ const bounds = cellBounds(index, w, h, gridCols, gridRows)
652
+ if (bounds.total <= 0) continue
653
+ cells.push({
654
+ x1: bounds.x1,
655
+ y1: bounds.y1,
656
+ x2: bounds.x2,
657
+ y2: bounds.y2,
658
+ ratio: hit / bounds.total,
659
+ differing: hit,
660
+ total: bounds.total,
661
+ })
662
+ }
663
+ cells.sort((a, b) => b.ratio - a.ratio)
664
+ return { differing, total: limit, ratio: limit === 0 ? 0 : differing / limit, mask: null, cells }
665
+ }
666
+
667
+ /** Render a diff heatmap: grayscale base, red where the mask marks a differing pixel. */
668
+ export function renderDiffHeatmap(originalRaw, mask, width, height) { const out = Buffer.alloc(width * height * 4)
669
+ for (let i = 0; i < width * height; i++) {
670
+ const o = i * 4
671
+ const gray = Math.round(
672
+ 0.299 * originalRaw[o] + 0.587 * originalRaw[o + 1] + 0.114 * originalRaw[o + 2],
673
+ )
674
+ if (mask[i]) {
675
+ out[o] = 255
676
+ out[o + 1] = 0
677
+ out[o + 2] = 0
678
+ out[o + 3] = 255
679
+ } else {
680
+ out[o] = gray
681
+ out[o + 1] = gray
682
+ out[o + 2] = gray
683
+ out[o + 3] = 255
684
+ }
685
+ }
686
+ return out
687
+ }
688
+
689
+ /** Dominant colors via bin quantization of an RGBA raw buffer. */
690
+ export function quantizeColors(raw, topN = 8, bins = 32) {
691
+ const step = 256 / bins
692
+ const counts = new Map()
693
+ const pixels = Math.floor(raw.length / 4)
694
+ for (let i = 0; i < pixels; i++) {
695
+ const o = i * 4
696
+ if (raw[o + 3] < 128) continue
697
+ const r = Math.floor(raw[o] / step) * step
698
+ const g = Math.floor(raw[o + 1] / step) * step
699
+ const b = Math.floor(raw[o + 2] / step) * step
700
+ const key = `${r},${g},${b}`
701
+ counts.set(key, (counts.get(key) ?? 0) + 1)
702
+ }
703
+ return [...counts.entries()]
704
+ .sort((a, b) => b[1] - a[1])
705
+ .slice(0, topN)
706
+ .map(([key, count]) => {
707
+ const [r, g, b] = key.split(',').map(Number)
708
+ const hex = '#' + [r, g, b].map((v) => v.toString(16).padStart(2, '0')).join('')
709
+ return { hex, count, share: pixels === 0 ? 0 : count / pixels }
710
+ })
711
+ }
712
+
713
+ /** SVG overlay string drawing one red pixel box on a width x height canvas. */
714
+ export function boxToSvg(box, width, height) {
715
+ return Buffer.from(
716
+ `<svg width="${width}" height="${height}">` +
717
+ `<rect x="${box.x1}" y="${box.y1}" width="${box.x2 - box.x1}" height="${box.y2 - box.y1}" ` +
718
+ `fill="none" stroke="#ff2d55" stroke-width="${Math.max(2, Math.round(Math.max(width, height) / 400))}"/></svg>`,
719
+ )
720
+ }
721
+
722
+ /** SVG overlay drawing numbered red boxes (detect inventory). */
723
+ export function boxesToSvg(boxes, width, height) {
724
+ const stroke = Math.max(2, Math.round(Math.max(width, height) / 400))
725
+ const labelR = Math.max(10, stroke * 4)
726
+ const parts = [`<svg width="${width}" height="${height}">`]
727
+ for (let i = 0; i < boxes.length; i++) {
728
+ const box = boxes[i]
729
+ parts.push(
730
+ `<rect x="${box.x1}" y="${box.y1}" width="${box.x2 - box.x1}" height="${box.y2 - box.y1}" ` +
731
+ `fill="none" stroke="#ff2d55" stroke-width="${stroke}"/>`,
732
+ )
733
+ const cx = Math.max(labelR, Math.min(box.x1, width - labelR))
734
+ const cy = Math.max(labelR, Math.min(box.y1, height - labelR))
735
+ parts.push(
736
+ `<circle cx="${cx}" cy="${cy}" r="${labelR}" fill="#ff2d55"/>` +
737
+ `<text x="${cx}" y="${cy + labelR * 0.36}" text-anchor="middle" ` +
738
+ `font-family="sans-serif" font-size="${Math.round(labelR * 1.2)}" fill="#ffffff" ` +
739
+ `font-weight="bold">${i + 1}</text>`,
740
+ )
741
+ }
742
+ parts.push('</svg>')
743
+ return Buffer.from(parts.join(''))
744
+ }
745
+
746
+ export function scaledDimensions(width, height, maxPixels) {
747
+ const w = Math.max(0, Math.floor(Number(width) || 0))
748
+ const h = Math.max(0, Math.floor(Number(height) || 0))
749
+ const limit = Math.max(1, Math.floor(Number(maxPixels) || 1))
750
+ if (w <= 0 || h <= 0) return { width: w, height: h, scale: 1 }
751
+ const pixels = w * h
752
+ if (pixels <= limit) return { width: w, height: h, scale: 1 }
753
+ const scale = Math.sqrt(limit / pixels)
754
+ return {
755
+ width: Math.max(1, Math.floor(w * scale)),
756
+ height: Math.max(1, Math.floor(h * scale)),
757
+ scale,
758
+ }
759
+ }
760
+
761
+ export function scaleBox(box, fromWidth, fromHeight, toWidth, toHeight) {
762
+ const sx = fromWidth > 0 ? toWidth / fromWidth : 1
763
+ const sy = fromHeight > 0 ? toHeight / fromHeight : 1
764
+ return {
765
+ x1: Math.max(0, Math.round(box.x1 * sx)),
766
+ y1: Math.max(0, Math.round(box.y1 * sy)),
767
+ x2: Math.min(toWidth, Math.max(1, Math.round(box.x2 * sx))),
768
+ y2: Math.min(toHeight, Math.max(1, Math.round(box.y2 * sy))),
769
+ }
770
+ }
771
+
772
+ /** Draw one red pixel box onto an image buffer via sharp. */
773
+ export async function annotateBoxBuffer(bytes, box) {
774
+ const sharp = await loadSharp()
775
+ if (!sharp) return bytes
776
+ const meta = await sharp(bytes, { failOn: 'none' }).metadata()
777
+ const width = meta.width ?? box.x2
778
+ const height = meta.height ?? box.y2
779
+ const preview = scaledDimensions(width, height, 4_000_000)
780
+ const displayBox = preview.scale === 1
781
+ ? box
782
+ : scaleBox(box, width, height, preview.width, preview.height)
783
+ let image = sharp(bytes, { failOn: 'none' })
784
+ if (preview.scale !== 1) image = image.resize(preview.width, preview.height, { fit: 'fill' })
785
+ return image
786
+ .composite([{ input: boxToSvg(displayBox, preview.width, preview.height), top: 0, left: 0 }])
787
+ .png()
788
+ .toBuffer()
789
+ }
790
+
791
+ /** Draw numbered boxes for a detected-element inventory onto an image buffer. */
792
+ export async function annotateBoxesBuffer(bytes, boxes) {
793
+ const sharp = await loadSharp()
794
+ if (!sharp) return bytes
795
+ const meta = await sharp(bytes, { failOn: 'none' }).metadata()
796
+ const width = meta.width ?? 0
797
+ const height = meta.height ?? 0
798
+ if (width <= 0 || height <= 0 || boxes.length === 0) return bytes
799
+ const preview = scaledDimensions(width, height, 4_000_000)
800
+ const displayBoxes = preview.scale === 1
801
+ ? boxes
802
+ : boxes.map((box) => scaleBox(box, width, height, preview.width, preview.height))
803
+ let image = sharp(bytes, { failOn: 'none' })
804
+ if (preview.scale !== 1) image = image.resize(preview.width, preview.height, { fit: 'fill' })
805
+ return image
806
+ .composite([{ input: boxesToSvg(displayBoxes, preview.width, preview.height), top: 0, left: 0 }])
807
+ .png()
808
+ .toBuffer()
809
+ }
810
+
811
+ /** Remove a solid-ish background via border flood fill; background becomes transparent. */
812
+ export function floodFillBackground(raw, width, height, tolerance = 40) {
813
+ const total = width * height
814
+ const out = Buffer.from(raw)
815
+ const marked = new Uint8Array(total)
816
+ let r = 0
817
+ let g = 0
818
+ let b = 0
819
+ const corners = [0, width - 1, (height - 1) * width, total - 1]
820
+ for (const c of corners) {
821
+ const o = c * 4
822
+ r += raw[o]
823
+ g += raw[o + 1]
824
+ b += raw[o + 2]
825
+ }
826
+ r /= 4
827
+ g /= 4
828
+ b /= 4
829
+ const queue = []
830
+ let head = 0
831
+ const push = (x, y) => {
832
+ const i = y * width + x
833
+ if (marked[i]) return
834
+ const o = i * 4
835
+ const d = Math.max(Math.abs(raw[o] - r), Math.abs(raw[o + 1] - g), Math.abs(raw[o + 2] - b))
836
+ if (d > tolerance) return
837
+ marked[i] = 1
838
+ queue.push(i)
839
+ }
840
+ for (let x = 0; x < width; x++) {
841
+ push(x, 0)
842
+ push(x, height - 1)
843
+ }
844
+ for (let y = 0; y < height; y++) {
845
+ push(0, y)
846
+ push(width - 1, y)
847
+ }
848
+ while (head < queue.length) {
849
+ const i = queue[head++]
850
+ const x = i % width
851
+ const y = (i - x) / width
852
+ if (x > 0) push(x - 1, y)
853
+ if (x < width - 1) push(x + 1, y)
854
+ if (y > 0) push(x, y - 1)
855
+ if (y < height - 1) push(x, y + 1)
856
+ }
857
+ for (let i = 0; i < total; i++) {
858
+ if (marked[i]) out[i * 4 + 3] = 0
859
+ }
860
+ return out
861
+ }
862
+
863
+ /** Cover a long/large image with bounded tiles in reading order. */
864
+ export function boundedOcrTiles(
865
+ width,
866
+ height,
867
+ { chunkHeight = 1200, overlap = 120, maxTilePixels = 4_000_000 } = {},
868
+ ) {
869
+ const w = Math.max(1, Math.floor(Number(width) || 1))
870
+ const h = Math.max(1, Math.floor(Number(height) || 1))
871
+ const requestedHeight = Math.max(1, Math.min(h, Math.floor(Number(chunkHeight) || 1200)))
872
+ const pixelLimit = Math.max(1, Math.floor(Number(maxTilePixels) || 4_000_000))
873
+ const tileWidth = Math.min(w, Math.max(1, Math.floor(pixelLimit / requestedHeight)))
874
+ const tileHeight = Math.min(requestedHeight, Math.max(1, Math.floor(pixelLimit / tileWidth)))
875
+ const verticalOverlap = Math.min(
876
+ Math.max(0, Math.floor(Number(overlap) || 0)),
877
+ Math.max(0, Math.floor(tileHeight / 2)),
878
+ )
879
+ const verticalStep = Math.max(1, tileHeight - verticalOverlap)
880
+ const tiles = []
881
+ for (let top = 0; top < h; top += verticalStep) {
882
+ const bottom = Math.min(h, top + tileHeight)
883
+ for (let left = 0; left < w; left += tileWidth) {
884
+ const right = Math.min(w, left + tileWidth)
885
+ tiles.push({ left, right, top, bottom })
886
+ }
887
+ if (bottom >= h) break
888
+ }
889
+ return tiles
890
+ }
891
+
892
+ // ── potrace (worker-thread, hard timeout) ──────────────────────────────────
893
+
894
+ function runPotraceWorker(workerData, source, timeoutMs) {
895
+ return new Promise((resolve, reject) => {
896
+ let settled = false
897
+ let worker
898
+ const finish = (error, svg) => {
899
+ if (settled) return
900
+ settled = true
901
+ clearTimeout(timer)
902
+ void worker?.terminate()
903
+ if (error) reject(error)
904
+ else resolve(svg)
905
+ }
906
+ const timer = setTimeout(() => {
907
+ if (settled) return
908
+ settled = true
909
+ void worker?.terminate()
910
+ reject(
911
+ new Error(
912
+ 'potrace timed out — the image is too large or too complex; crop it to the target region first',
913
+ ),
914
+ )
915
+ }, timeoutMs)
916
+ try {
917
+ worker = new Worker(source, { eval: true, workerData })
918
+ worker.once('message', (message) => {
919
+ if (message && message.error) finish(new Error(message.error))
920
+ else finish(undefined, message && message.svg)
921
+ })
922
+ worker.once('error', (error) => finish(error))
923
+ worker.once('exit', (code) => {
924
+ if (code !== 0 && !settled) finish(new Error(`potrace worker exited with code ${code}`))
925
+ })
926
+ } catch (error) {
927
+ finish(error)
928
+ }
929
+ })
930
+ }
931
+
932
+ /** Vectorize an image buffer into an SVG string via potrace posterization. */
933
+ export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000) {
934
+ const potraceUrl = pathToFileURL(createRequire(import.meta.url).resolve('potrace')).href
935
+ const source = `
936
+ import('node:worker_threads').then(({ parentPort, workerData }) => {
937
+ import(workerData.potraceUrl).then((mod) => {
938
+ const potrace = mod.default ?? mod
939
+ potrace.posterize(Buffer.from(workerData.bytes), {
940
+ steps: workerData.steps,
941
+ fillStrategy: workerData.fillStrategy,
942
+ }, (error, svg) => {
943
+ parentPort.postMessage(error ? { error: String((error && error.message) || error) } : { svg })
944
+ })
945
+ }).catch((error) => {
946
+ parentPort.postMessage({ error: String((error && error.message) || error) })
947
+ })
948
+ })
949
+ `
950
+ return runPotraceWorker({ potraceUrl, bytes, steps, fillStrategy }, source, timeoutMs)
951
+ }
952
+
953
+ /**
954
+ * Color-preserving vectorization: quantize into the top colors, trace one
955
+ * 1-bit mask per color with potrace, emit a colored SVG (one path per color).
956
+ */
957
+ export function posterizeSvgColor(data, info, palette, timeoutMs = 60000) {
958
+ const sharpUrl = pathToFileURL(createRequire(import.meta.url).resolve('sharp')).href
959
+ const potraceUrl = pathToFileURL(createRequire(import.meta.url).resolve('potrace')).href
960
+ const source = `
961
+ import('node:worker_threads').then(({ parentPort, workerData }) => {
962
+ Promise.all([import(workerData.sharpUrl), import(workerData.potraceUrl)]).then(([sharpMod, potraceMod]) => {
963
+ const sharp = sharpMod.default ?? sharpMod
964
+ const potrace = potraceMod.default ?? potraceMod
965
+ const { width, height, palette } = workerData
966
+ const raw = Buffer.from(workerData.raw)
967
+ const hexRgb = (hex) => {
968
+ const n = parseInt(hex.slice(1), 16)
969
+ return [(n >> 16) & 255, (n >> 8) & 255, n & 255]
970
+ }
971
+ const paletteRgb = palette.map((p) => hexRgb(p.hex))
972
+ const pixels = width * height
973
+ const masks = palette.map(() => Buffer.alloc(pixels))
974
+ for (let p = 0; p < pixels; p++) {
975
+ const o = p * 4
976
+ if (raw[o + 3] < 128) continue
977
+ let best = 0
978
+ let bestD = Infinity
979
+ for (let c = 0; c < paletteRgb.length; c++) {
980
+ const dr = raw[o] - paletteRgb[c][0]
981
+ const dg = raw[o + 1] - paletteRgb[c][1]
982
+ const db = raw[o + 2] - paletteRgb[c][2]
983
+ const d = dr * dr + dg * dg + db * db
984
+ if (d < bestD) { bestD = d; best = c }
985
+ }
986
+ masks[best][p] = 1
987
+ }
988
+ const paths = []
989
+ let pending = palette.length
990
+ const maybeDone = () => {
991
+ if (pending > 0) return
992
+ const pathSvg = paths.map((p) => '<path fill="' + p.hex + '" d="' + p.d + '"/>').join('')
993
+ parentPort.postMessage({
994
+ ok: true,
995
+ svg: '<svg xmlns="http://www.w3.org/2000/svg" width="' + width + '" height="' + height +
996
+ '" viewBox="0 0 ' + width + ' ' + height + '"><rect width="' + width + '" height="' + height +
997
+ '" fill="#ffffff"/>' + pathSvg + '</svg>',
998
+ })
999
+ }
1000
+ if (pending === 0) { maybeDone(); return }
1001
+ palette.forEach((entry, index) => {
1002
+ const gray = Buffer.alloc(pixels)
1003
+ const mask = masks[index]
1004
+ for (let p = 0; p < pixels; p++) gray[p] = mask[p] ? 0 : 255
1005
+ sharp(gray, { raw: { width, height, channels: 1 } })
1006
+ .png()
1007
+ .toBuffer()
1008
+ .then((pngBuf) => {
1009
+ potrace.trace(pngBuf, (err, svg) => {
1010
+ pending -= 1
1011
+ if (!err && svg) {
1012
+ const found = [...svg.matchAll(/d="([^"]+)"/g)].map((m) => m[1])
1013
+ for (const d of found) paths.push({ hex: entry.hex, d })
1014
+ }
1015
+ maybeDone()
1016
+ })
1017
+ })
1018
+ .catch(() => {
1019
+ pending -= 1
1020
+ maybeDone()
1021
+ })
1022
+ })
1023
+ }).catch((error) => {
1024
+ parentPort.postMessage({ error: String((error && error.message) || error) })
1025
+ })
1026
+ })
1027
+ `
1028
+ return runPotraceWorker(
1029
+ { sharpUrl, potraceUrl, width: info.width, height: info.height, palette, raw: data },
1030
+ source,
1031
+ timeoutMs,
1032
+ )
1033
+ }
1034
+
1035
+ // ── browser (HTML screenshot) ──────────────────────────────────────────────
1036
+
1037
+ /** Cross-platform Chrome/Chromium/Edge discovery. */
1038
+ export function chromiumCandidates(env = {}, platform = typeof process !== 'undefined' ? process.platform : '') {
1039
+ const out = []
1040
+ const add = (value) => {
1041
+ if (typeof value === 'string' && value !== '' && !out.includes(value)) out.push(value)
1042
+ }
1043
+ add(env.CHROME_PATH)
1044
+ add(env.PUPPETEER_EXECUTABLE_PATH)
1045
+
1046
+ if (platform === 'win32') {
1047
+ const pf = env.PROGRAMFILES
1048
+ const pfx86 = env['PROGRAMFILES(X86)']
1049
+ const local = env.LOCALAPPDATA
1050
+ if (pf) {
1051
+ add(winPath.join(pf, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1052
+ add(winPath.join(pf, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1053
+ }
1054
+ if (pfx86) {
1055
+ add(winPath.join(pfx86, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1056
+ add(winPath.join(pfx86, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1057
+ }
1058
+ if (local) {
1059
+ add(winPath.join(local, 'Google', 'Chrome', 'Application', 'chrome.exe'))
1060
+ add(winPath.join(local, 'Microsoft', 'Edge', 'Application', 'msedge.exe'))
1061
+ add(winPath.join(local, 'Chromium', 'Application', 'chrome.exe'))
1062
+ }
1063
+ } else if (platform === 'darwin') {
1064
+ add('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
1065
+ add('/Applications/Chromium.app/Contents/MacOS/Chromium')
1066
+ add('/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge')
1067
+ } else {
1068
+ add('/usr/bin/google-chrome')
1069
+ add('/usr/bin/google-chrome-stable')
1070
+ add('/usr/bin/chromium')
1071
+ add('/usr/bin/chromium-browser')
1072
+ add('/usr/bin/microsoft-edge')
1073
+ add('/usr/bin/microsoft-edge-stable')
1074
+ }
1075
+ return out
1076
+ }
1077
+
1078
+ /** Wake lazy/revealed content before a full-page capture. */
1079
+ export async function wakePageForFullCapture(page, viewportHeight) {
1080
+ const step = Number.isInteger(viewportHeight) && viewportHeight > 0 ? viewportHeight : 720
1081
+ await page.evaluate(() => {
1082
+ document.documentElement.style.scrollBehavior = 'auto'
1083
+ })
1084
+ const total = await page.evaluate(() =>
1085
+ Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0),
1086
+ )
1087
+ for (let y = 0; y < total; y += step) {
1088
+ await page.evaluate((yy) => window.scrollTo(0, yy), y)
1089
+ await new Promise((resolve) => setTimeout(resolve, 60))
1090
+ }
1091
+ await page.evaluate(() => window.scrollTo(0, 0))
1092
+ await new Promise((resolve) => setTimeout(resolve, 800))
1093
+ }
1094
+
1095
+ /** Full scrollable page height (CSS px). */
1096
+ export async function fullPageHeightOf(page) {
1097
+ return await page.evaluate(() =>
1098
+ Math.max(
1099
+ document.documentElement.scrollHeight,
1100
+ document.body ? document.body.scrollHeight : 0,
1101
+ window.innerHeight,
1102
+ ),
1103
+ )
1104
+ }
1105
+
1106
+ // ── model call (single configured endpoint) ────────────────────────────────
1107
+
1108
+ /** Endpoint content-safety rejection markers (GLM: "系统检测到输入或生成内容
1109
+ * 可能包含不安全或敏感内容…"; OpenAI-style: content_filter/policy/safety…). */
1110
+ export const CONTENT_FILTER_RE =
1111
+ /敏感|不安全|内容审核|内容安全|政策|content.?filter|safety|inappropriate|flagged|拒绝.?内容|违反.*(规范|政策|安全)|violat/i
1112
+
1113
+ export function resolveApiKey(config) {
1114
+ if (config.apiKey) return config.apiKey
1115
+ if (config.apiKeyEnv) {
1116
+ const fromEnv = process.env[config.apiKeyEnv]
1117
+ if (fromEnv) return fromEnv
1118
+ }
1119
+ return process.env.OPENAI_API_KEY ?? ''
1120
+ }
1121
+
1122
+ /** Turn image bytes into a data URL for the vision call.
1123
+ * Defensive: attachment-service reads return Uint8Array (whose
1124
+ * toString('base64') is a comma-joined number list, not base64), so always
1125
+ * wrap in a real Buffer first. */
1126
+ export function bytesToDataUrl(bytes, mediaType) {
1127
+ const mime = mediaType ?? 'image/png'
1128
+ const buf = Buffer.isBuffer(bytes)
1129
+ ? bytes
1130
+ : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
1131
+ return `data:${mime};base64,${buf.toString('base64')}`
1132
+ }
1133
+
1134
+ /**
1135
+ * One OpenAI-compatible chat/completions call with image content parts.
1136
+ * This is the ONLY model-call path the ported tools use: the plugin's own
1137
+ * baseURL/apiKey/model settings drive every vision_* tool, exactly like
1138
+ * inspect_image.
1139
+ */
1140
+ export async function callVisionModel(getConfig, prompt, imageUrls, options = {}) {
1141
+ const cfg = getConfig()
1142
+ const key = resolveApiKey(cfg)
1143
+ if (!key) {
1144
+ throw new Error(
1145
+ 'tool-vision: vision API key missing: set the plugin config (apiKey / apiKeyEnv) or the OPENAI_API_KEY environment variable',
1146
+ )
1147
+ }
1148
+ const sessionHeader = sessionHeaders(cfg, options.exec)
1149
+ const base = cfg.baseURL.endsWith('/') ? cfg.baseURL : `${cfg.baseURL}/`
1150
+ const endpoint = new URL('chat/completions', base)
1151
+ const controller = new AbortController()
1152
+ const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : cfg.timeoutMs
1153
+ const timer = setTimeout(
1154
+ () => controller.abort(new Error(`tool-vision: vision request timed out after ${timeoutMs}ms`)),
1155
+ timeoutMs,
1156
+ )
1157
+ const signal = options.signal
1158
+ const onSignalAbort = () => controller.abort(signal?.reason)
1159
+ if (signal) {
1160
+ if (signal.aborted) controller.abort()
1161
+ else signal.addEventListener('abort', onSignalAbort, { once: true })
1162
+ }
1163
+ const content = [
1164
+ { type: 'text', text: prompt },
1165
+ ...imageUrls.map((url) => ({ type: 'image_url', image_url: { url } })),
1166
+ ]
1167
+ const bodyOf = () => JSON.stringify({
1168
+ model: cfg.model,
1169
+ messages: [{ role: 'user', content }],
1170
+ max_tokens: Number.isFinite(options.maxTokens) ? options.maxTokens : cfg.maxTokens,
1171
+ })
1172
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
1173
+ const retryAfterOf = (response) => {
1174
+ const header = response?.headers?.get?.('retry-after')
1175
+ if (header) {
1176
+ const seconds = Number(header)
1177
+ if (Number.isFinite(seconds) && seconds > 0) return Math.min(seconds * 1000, 15000)
1178
+ }
1179
+ return 0
1180
+ }
1181
+ const MAX_TRANSIENT_RETRIES = 3
1182
+ try {
1183
+ let lastResponse = null
1184
+ let lastBody = null
1185
+ for (let attempt = 0; ; attempt++) {
1186
+ const response = await fetch(endpoint, {
1187
+ method: 'POST',
1188
+ headers: {
1189
+ 'Content-Type': 'application/json',
1190
+ Authorization: `Bearer ${key}`,
1191
+ ...sessionHeader,
1192
+ },
1193
+ body: bodyOf(),
1194
+ signal: controller.signal,
1195
+ })
1196
+ lastResponse = response
1197
+ lastBody = await response.json().catch(() => null)
1198
+ // Transient failures (rate limit / 5xx) retry with backoff; everything
1199
+ // else (4xx auth/format/content-filter) is terminal.
1200
+ const retriable =
1201
+ response.status === 429 ||
1202
+ response.status === 408 ||
1203
+ response.status === 500 ||
1204
+ response.status === 502 ||
1205
+ response.status === 503 ||
1206
+ response.status === 504
1207
+ if (!response.ok && retriable && attempt < MAX_TRANSIENT_RETRIES) {
1208
+ const waitMs = retryAfterOf(response) || 800 * (attempt + 1)
1209
+ await sleep(waitMs)
1210
+ continue
1211
+ }
1212
+ break
1213
+ }
1214
+ const response = lastResponse
1215
+ const body = lastBody
1216
+ if (!response.ok) {
1217
+ const detailText = body?.error?.message ?? response.statusText
1218
+ throw new Error(
1219
+ `tool-vision: vision endpoint returned ${response.status}: ${detailText} (endpoint ${endpoint})`,
1220
+ )
1221
+ }
1222
+ const message = body?.choices?.[0]?.message
1223
+ let answer = message?.content ?? ''
1224
+ if (!answer.trim()) answer = message?.reasoning_content ?? ''
1225
+ if (typeof answer !== 'string' || !answer.trim()) {
1226
+ throw new Error('tool-vision: vision endpoint returned an empty response')
1227
+ }
1228
+ const trimmed = answer.trim()
1229
+ // max_tokens truncation: glm-4v-flash caps at 1024; a truncated JSON or
1230
+ // transcript must not masquerade as a complete answer.
1231
+ const finishReason = body?.choices?.[0]?.finish_reason
1232
+ if (finishReason === 'length') {
1233
+ return `${trimmed}\n\n[注意:模型输出因 max_tokens 上限被截断(finish_reason=length);以上内容不完整。]`
1234
+ }
1235
+ return trimmed
1236
+ } finally {
1237
+ clearTimeout(timer)
1238
+ signal?.removeEventListener('abort', onSignalAbort)
1239
+ }
1240
+ }
1241
+
1242
+ // ── shared tool plumbing ───────────────────────────────────────────────────
1243
+
1244
+ export const stringOutput = {
1245
+ schema: { type: 'string' },
1246
+ render: (_args, value) => [{ type: 'text', text: value }],
1247
+ }
1248
+
1249
+ /**
1250
+ * Build the 14 vision tools bound to this plugin instance.
1251
+ * @param {object} ctx - plugin context (attachments/fs services via ctx.get)
1252
+ * @param {() => object} getConfig - live settings getter
1253
+ */
1254
+ export function registerVisionTools(ctx, getConfig) {
1255
+ const logger = ctx.logger
1256
+
1257
+ /** Single-endpoint model call wrapper with the router's ok:false contract.
1258
+ * `images` is a single Buffer, or an array of Buffers / data-URL strings. */
1259
+ const answerVisionForTool = async (images, mediaType, instruction, options = {}) => {
1260
+ try {
1261
+ // Uniform pixel budget for EVERY model call: ground/detect/ocr/etc.
1262
+ // must not ship raw 20MB images (base64 ~27MB -> endpoint 413/timeout).
1263
+ // Downscale only Buffer inputs; pre-made data URLs pass through.
1264
+ const bounded = Array.isArray(images)
1265
+ ? await Promise.all(images.map(async (item) => (typeof item === 'string' ? item : bytesToDataUrl(await downscaleImage(item, 4_000_000), mediaType ?? 'image/png'))))
1266
+ : [bytesToDataUrl(await downscaleImage(images, 4_000_000), mediaType)]
1267
+ const text = await callVisionModel(getConfig, instruction, bounded, {
1268
+ signal: options.signal ?? (options.exec && options.exec.signal),
1269
+ exec: options.exec,
1270
+ })
1271
+ return { ok: true, text }
1272
+ } catch (error) {
1273
+ const raw = error && error.message ? String(error.message) : String(error)
1274
+ // Endpoint content-safety rejections are NOT backend outages: the image
1275
+ // itself was refused. Surface them distinctly so the model tells the
1276
+ // user to pick a different image instead of reporting "vision down".
1277
+ if (CONTENT_FILTER_RE.test(raw)) {
1278
+ return {
1279
+ ok: false,
1280
+ code: 'VISION_CONTENT_FILTERED',
1281
+ retryable: false,
1282
+ contentFiltered: true,
1283
+ message:
1284
+ '图片被视觉端点的内容安全策略拒绝(检测到敏感或不安全内容)。' +
1285
+ '这不是网络或配置问题,请换一张图片或调整图片内容后再试。',
1286
+ }
1287
+ }
1288
+ return {
1289
+ ok: false,
1290
+ code: 'VISION_BACKEND_UNAVAILABLE',
1291
+ retryable: false,
1292
+ message: raw,
1293
+ }
1294
+ }
1295
+ }
1296
+
1297
+ const saveArtifact = (exec, relPath, data) => writeArtifact(exec, relPath, data)
1298
+ const artifactStem = (imagePath, suffix) => artifactStemOf(imagePath, suffix)
1299
+
1300
+ const tools = []
1301
+
1302
+ // ── vision_describe ──────────────────────────────────────────────────────
1303
+ tools.push({
1304
+ name: 'vision_describe',
1305
+ description:
1306
+ 'Look at images with the configured vision model and answer a focused question about them. ' +
1307
+ 'For text-only sessions this is the bridge that provides image understanding. ' +
1308
+ 'Supports comparing multiple images (e.g. a design mock vs an implementation screenshot). Provide ' +
1309
+ '`paths` (absolute local image file paths, png/jpeg/webp/gif) and/or ' +
1310
+ '`attachmentIds` (ids of images the user uploaded in this conversation), 1-4 images in ' +
1311
+ 'total. `question` is the question to answer; be specific. Set `json: true` to require a ' +
1312
+ 'single valid JSON object as the answer. ' +
1313
+ 'FAILURE SEMANTICS: if the result is JSON with ok:false, the vision backend is unavailable. ' +
1314
+ 'Do NOT call vision_describe again with a reworded question — rephrasing cannot fix an auth, ' +
1315
+ 'rate-limit or outage problem. Answer from the information you already have and continue the ' +
1316
+ 'text task, telling the user vision is temporarily unavailable.',
1317
+ parameters: {
1318
+ type: 'object',
1319
+ properties: {
1320
+ paths: {
1321
+ type: 'array',
1322
+ items: { type: 'string' },
1323
+ description:
1324
+ 'Absolute local image file paths and/or attachment ids (e.g. "sha256:...") of uploaded images, 1-4 images',
1325
+ },
1326
+ attachmentIds: {
1327
+ type: 'array',
1328
+ items: { type: 'string' },
1329
+ description: 'Attachment ids of images uploaded earlier in this conversation',
1330
+ },
1331
+ question: {
1332
+ type: 'string',
1333
+ description:
1334
+ 'The question for the vision model, e.g. "compare the two images and list the differences"',
1335
+ },
1336
+ json: {
1337
+ type: 'boolean',
1338
+ description: 'Require the answer to be a single valid JSON object',
1339
+ },
1340
+ },
1341
+ required: ['question'],
1342
+ additionalProperties: false,
1343
+ },
1344
+ output: stringOutput,
1345
+ async execute(args, exec) {
1346
+ const attachments = ctx.get('attachments')
1347
+ if (attachments === undefined) {
1348
+ throw new Error('vision_describe: the durable attachment service is not available in this deployment')
1349
+ }
1350
+ const blocks = []
1351
+ const contentIds = []
1352
+ const paths = Array.isArray(args.paths) ? args.paths : []
1353
+ const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
1354
+ if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
1355
+ throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
1356
+ }
1357
+
1358
+ for (const path of paths) {
1359
+ let bytes
1360
+ let mediaType
1361
+ try {
1362
+ ;({ bytes, mediaType } = await readImageBytes(ctx, exec, path))
1363
+ } catch (error) {
1364
+ throw new Error(
1365
+ `vision_describe: failed to read ${path} (${error && error.message ? error.message : String(error)})`,
1366
+ )
1367
+ }
1368
+ bytes = await downscaleImage(bytes, 4_000_000)
1369
+ let ref
1370
+ try {
1371
+ ref = await attachments.saveImage({ data: bytes, mediaType })
1372
+ } catch (error) {
1373
+ throw new Error(
1374
+ `vision_describe: image ${path} was rejected (${error && error.message ? error.message : String(error)})`,
1375
+ )
1376
+ }
1377
+ contentIds.push(String(ref.attachmentId))
1378
+ blocks.push({ type: 'image', attachment: ref })
1379
+ }
1380
+
1381
+ for (const id of attachmentIds) {
1382
+ const session = exec && exec.agent && exec.agent.session
1383
+ const ref = lookupAttachment(session, String(id))
1384
+ if (ref === undefined) {
1385
+ throw new Error(
1386
+ `vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
1387
+ )
1388
+ }
1389
+ let stored
1390
+ try {
1391
+ stored = await attachments.readImage(ref)
1392
+ } catch (error) {
1393
+ throw new Error(
1394
+ `vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
1395
+ )
1396
+ }
1397
+ if (stored.data && stored.data.length > 0) {
1398
+ const resized = await downscaleImage(stored.data, 4_000_000)
1399
+ if (resized !== stored.data) {
1400
+ try {
1401
+ const resizedRef = await attachments.saveImage({
1402
+ data: resized,
1403
+ mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
1404
+ ...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
1405
+ })
1406
+ stored = { ref: resizedRef, data: resized }
1407
+ } catch {
1408
+ stored = { ...stored, data: resized }
1409
+ }
1410
+ }
1411
+ }
1412
+ contentIds.push(String(ref.attachmentId))
1413
+ blocks.push({ type: 'image', attachment: stored.ref })
1414
+ }
1415
+
1416
+ const question = String(args.question ?? '')
1417
+ const wantJson = args.json === true
1418
+ const promptText = visionDescribePrompt(question, wantJson)
1419
+ // Cache key carries the endpoint/model identity: switching the vision
1420
+ // model must never serve stale answers from the previous model.
1421
+ const cfg = getConfig()
1422
+ const key = `${cfg.baseURL ?? ''}|${cfg.model ?? ''}|${contentIds.join('|')}|${wantJson ? 'json' : 'text'}|${question}`
1423
+ const cache = visionCache
1424
+ if (cacheEnabled()) {
1425
+ const hit = cache.get(key)
1426
+ if (hit !== undefined) return hit
1427
+ }
1428
+
1429
+ // Call with all images as data URLs.
1430
+ const imageUrls = []
1431
+ for (const block of blocks) {
1432
+ const stored = await attachments.readImage(block.attachment)
1433
+ imageUrls.push(bytesToDataUrl(stored.data, stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png'))
1434
+ }
1435
+ const vision = await answerVisionForTool(imageUrls, 'image/png', promptText, { exec })
1436
+ if (vision.ok === false) return JSON.stringify(vision)
1437
+
1438
+ let text = vision.text
1439
+ if (wantJson) {
1440
+ for (let attempt = 0; attempt < 2; attempt++) {
1441
+ const parsed = extractJson(text)
1442
+ if (parsed !== undefined) {
1443
+ const compact = JSON.stringify(normalizeDescribeResult(parsed) ?? parsed)
1444
+ if (cacheEnabled()) cache.set(key, compact)
1445
+ return compact
1446
+ }
1447
+ if (attempt === 0) {
1448
+ const retry = await answerVisionForTool(imageUrls, 'image/png', promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.', { exec })
1449
+ if (retry.ok === false) return JSON.stringify(retry)
1450
+ text = retry.text
1451
+ }
1452
+ }
1453
+ // Failure/fallback output must NOT be cached: it would poison the
1454
+ // cache for the whole TTL with an error string masquerading as an
1455
+ // answer.
1456
+ return `vision_describe: the model did not produce valid JSON. Raw output:\n${text.slice(0, 2000)}`
1457
+ }
1458
+ if (text !== '') {
1459
+ if (cacheEnabled()) cache.set(key, text)
1460
+ return text
1461
+ }
1462
+ // Empty output is a failure signal, not a cacheable answer.
1463
+ return '(the vision model returned empty content)'
1464
+ },
1465
+ })
1466
+
1467
+ // ── vision_materialize ───────────────────────────────────────────────────
1468
+ tools.push({
1469
+ name: 'vision_materialize',
1470
+ description:
1471
+ 'Copy an uploaded image attachment (sha256:...) or readable local image into the session workspace and return a real filesystem path. ' +
1472
+ 'This tool performs NO vision model/network call. Use it after vision_describe returns ok:false when a local OCR/parser accepts only file_path. ' +
1473
+ 'Never guess the attachment store path or search for a same-named file.',
1474
+ parameters: {
1475
+ type: 'object',
1476
+ properties: {
1477
+ image: { type: 'string', description: 'Uploaded image attachment id (recommended, e.g. sha256:...) or a readable local image path' },
1478
+ },
1479
+ required: ['image'],
1480
+ additionalProperties: false,
1481
+ },
1482
+ output: stringOutput,
1483
+ async execute(args, exec) {
1484
+ const source = String(args.image ?? '')
1485
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, source)
1486
+ const extension = mediaType === 'image/jpeg' ? '.jpg' : (EXT_BY_MEDIA[mediaType] ?? '.png')
1487
+ const target = await saveArtifact(exec, `${artifactStem(source, 'materialized')}${extension}`, bytes)
1488
+ return JSON.stringify({ path: target, bytes: bytes.length, mediaType })
1489
+ },
1490
+ })
1491
+
1492
+ // ── vision_ground ────────────────────────────────────────────────────────
1493
+ tools.push({
1494
+ name: 'vision_ground',
1495
+ description:
1496
+ 'Locate a target in an image and return its ORIGINAL-pixel bounding box (x1/y1/x2/y2), ' +
1497
+ 'optionally producing an annotated PNG artifact. Pair with vision_crop and vision_pixel_diff ' +
1498
+ 'for a verify-able pixel loop (reference -> implementation -> screenshot -> metrics). ' +
1499
+ 'If the result is JSON with ok:false, the vision backend is unavailable — do not retry with ' +
1500
+ 'reworded instructions this turn.',
1501
+ parameters: {
1502
+ type: 'object',
1503
+ properties: {
1504
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1505
+ target: { type: 'string', description: 'What to locate, e.g. "the send button"' },
1506
+ annotate: { type: 'boolean', description: 'Also write an annotated PNG with the box drawn (default true)' },
1507
+ },
1508
+ required: ['image', 'target'],
1509
+ additionalProperties: false,
1510
+ },
1511
+ output: stringOutput,
1512
+ async execute(args, exec) {
1513
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1514
+ const { width, height } = await imageDims(bytes)
1515
+ if (width <= 0 || height <= 0) throw new Error('vision_ground: could not read image dimensions')
1516
+ const instruction =
1517
+ `Target to locate: "${String(args.target).slice(0, 500)}". ` +
1518
+ `The image is ${width}x${height} pixels. Return ONE JSON object with integer fields ` +
1519
+ `{"x1":...,"y1":...,"x2":...,"y2":...} — the tight bounding box of that target in ` +
1520
+ `ORIGINAL image pixels (0 <= x1 < x2 <= ${width}, 0 <= y1 < y2 <= ${height}). ` +
1521
+ `Output only the JSON object.`
1522
+ const vision = await answerVisionForTool(bytes, mediaType, instruction, { exec })
1523
+ if (vision.ok === false) return JSON.stringify(vision)
1524
+ const parsed = extractJson(vision.text)
1525
+ const box = parsed !== undefined ? parseBox(parsed) : undefined
1526
+ if (box === undefined) {
1527
+ throw new Error(`vision_ground: the vision model did not return a valid box. Raw output: ${vision.text.slice(0, 500)}`)
1528
+ }
1529
+ let clamped = {
1530
+ x1: Math.max(0, Math.min(box.x1, width - 1)),
1531
+ y1: Math.max(0, Math.min(box.y1, height - 1)),
1532
+ x2: Math.max(1, Math.min(box.x2, width)),
1533
+ y2: Math.max(1, Math.min(box.y2, height)),
1534
+ }
1535
+ if (clamped.x2 - clamped.x1 < 2 || clamped.y2 - clamped.y1 < 2) {
1536
+ const retry = await answerVisionForTool(
1537
+ bytes,
1538
+ mediaType,
1539
+ `Your previous box ${JSON.stringify(clamped)} was a degenerate sliver, not the target. ` +
1540
+ `Return ONE JSON object with the FULL tight bounding box of the target in ORIGINAL ` +
1541
+ `image pixels (0 <= x1 < x2 <= ${width}, 0 <= y1 < y2 <= ${height}). Output only the JSON object.`,
1542
+ { exec },
1543
+ )
1544
+ if (retry.ok === false) return JSON.stringify(retry)
1545
+ const retryParsed = extractJson(retry.text)
1546
+ const retryBox = retryParsed !== undefined ? parseBox(retryParsed) : undefined
1547
+ if (retryBox === undefined) {
1548
+ throw new Error(
1549
+ `vision_ground: the vision model returned a degenerate box (${clamped.x1},${clamped.y1},${clamped.x2},${clamped.y2}) ` +
1550
+ `and the retry returned no valid box. Raw output: ${retry.text.slice(0, 500)}`,
1551
+ )
1552
+ }
1553
+ clamped = {
1554
+ x1: Math.max(0, Math.min(retryBox.x1, width - 1)),
1555
+ y1: Math.max(0, Math.min(retryBox.y1, height - 1)),
1556
+ x2: Math.max(1, Math.min(retryBox.x2, width)),
1557
+ y2: Math.max(1, Math.min(retryBox.y2, height)),
1558
+ }
1559
+ if (clamped.x2 - clamped.x1 < 2 || clamped.y2 - clamped.y1 < 2) {
1560
+ throw new Error(
1561
+ `vision_ground: the vision model returned only degenerate boxes for a ${width}x${height} image. ` +
1562
+ `Last raw output: ${retry.text.slice(0, 500)}`,
1563
+ )
1564
+ }
1565
+ }
1566
+ const result = { ...clamped, width, height }
1567
+ if (args.annotate !== false) {
1568
+ const annotated = await annotateBoxBuffer(bytes, clamped)
1569
+ result.annotatedPath = await saveArtifact(exec, `${artifactStem(args.image, 'ground')}.png`, annotated)
1570
+ }
1571
+ return JSON.stringify(result)
1572
+ },
1573
+ })
1574
+
1575
+ // ── vision_detect ────────────────────────────────────────────────────────
1576
+ tools.push({
1577
+ name: 'vision_detect',
1578
+ description:
1579
+ 'Find every element of a kind in an image (buttons, inputs, links, icons…) and return a ' +
1580
+ 'numbered inventory with ORIGINAL-pixel boxes, optionally annotated on the image. The model ' +
1581
+ 'can then reference "element #3" in follow-up vision_crop / vision_describe calls. ' +
1582
+ 'If the result is JSON with ok:false, the vision backend is unavailable — do not retry with ' +
1583
+ 'reworded instructions this turn.',
1584
+ parameters: {
1585
+ type: 'object',
1586
+ properties: {
1587
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1588
+ target: {
1589
+ type: 'string',
1590
+ description: 'What kind of elements to list, e.g. "buttons", "input fields", "navigation links" (default: interactive elements)',
1591
+ },
1592
+ annotate: {
1593
+ type: 'boolean',
1594
+ description: 'Also write an annotated PNG with numbered boxes (default true)',
1595
+ },
1596
+ },
1597
+ required: ['image'],
1598
+ additionalProperties: false,
1599
+ },
1600
+ output: stringOutput,
1601
+ async execute(args, exec) {
1602
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1603
+ const { width, height } = await imageDims(bytes)
1604
+ if (width <= 0 || height <= 0) throw new Error('vision_detect: could not read image dimensions')
1605
+ const target = typeof args.target === 'string' && args.target.trim() !== '' ? args.target : 'interactive elements'
1606
+ const vision = await answerVisionForTool(bytes, mediaType, visionDetectInstruction(target, width, height), { exec })
1607
+ if (vision.ok === false) return JSON.stringify(vision)
1608
+ let text = vision.text
1609
+ let parsed = extractJson(text)
1610
+ if (parsed === undefined) {
1611
+ const retry = await answerVisionForTool(
1612
+ bytes,
1613
+ mediaType,
1614
+ visionDetectInstruction(target, width, height) +
1615
+ '\nYour previous answer was not valid JSON. Respond with ONLY the JSON object, no prose, no fences.',
1616
+ { exec },
1617
+ )
1618
+ if (retry.ok === false) return JSON.stringify(retry)
1619
+ parsed = extractJson(retry.text)
1620
+ text = retry.text
1621
+ }
1622
+ const result = normalizeDetectResult(parsed, width, height)
1623
+ if (result === undefined) {
1624
+ throw new Error(`vision_detect: the vision model did not return a valid inventory. Raw output: ${text.slice(0, 500)}`)
1625
+ }
1626
+ if (args.annotate !== false && result.elements.length > 0) {
1627
+ const annotated = await annotateBoxesBuffer(bytes, result.elements.map((e) => e.box))
1628
+ result.annotatedPath = await saveArtifact(exec, `${artifactStem(args.image, 'detect')}.png`, annotated)
1629
+ }
1630
+ return JSON.stringify(result)
1631
+ },
1632
+ })
1633
+
1634
+ // ── vision_crop ──────────────────────────────────────────────────────────
1635
+ tools.push({
1636
+ name: 'vision_crop',
1637
+ description:
1638
+ 'Crop a pixel region (x1,y1,x2,y2 in ORIGINAL pixels) out of an image and write the ' +
1639
+ 'result as a PNG artifact for a closer look. Very large regions are rendered as a bounded ' +
1640
+ 'preview; crop a smaller ORIGINAL-pixel region when tiny details must be preserved.',
1641
+ parameters: {
1642
+ type: 'object',
1643
+ properties: {
1644
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1645
+ region: {
1646
+ type: 'string',
1647
+ description: 'Pixel box "x1,y1,x2,y2" in original image coordinates',
1648
+ },
1649
+ },
1650
+ required: ['image', 'region'],
1651
+ additionalProperties: false,
1652
+ },
1653
+ output: stringOutput,
1654
+ async execute(args, exec) {
1655
+ const { bytes } = await readImageBytes(ctx, exec, args.image)
1656
+ const { width, height } = await imageDims(bytes)
1657
+ const box = parseBox(args.region)
1658
+ if (box === undefined) {
1659
+ throw new Error(`vision_crop: invalid region "${args.region}" (expect "x1,y1,x2,y2" integers)`)
1660
+ }
1661
+ if (box.x2 > width || box.y2 > height) {
1662
+ throw new Error(`vision_crop: region exceeds image bounds (${width}x${height})`)
1663
+ }
1664
+ const sharp = await loadSharp()
1665
+ if (!sharp) throw new Error('vision_crop: the sharp image library is unavailable')
1666
+ const sourceWidth = box.x2 - box.x1
1667
+ const sourceHeight = box.y2 - box.y1
1668
+ const preview = scaledDimensions(sourceWidth, sourceHeight, 4_000_000)
1669
+ let pipeline = sharp(bytes, { failOn: 'none' }).extract({
1670
+ left: box.x1,
1671
+ top: box.y1,
1672
+ width: sourceWidth,
1673
+ height: sourceHeight,
1674
+ })
1675
+ if (preview.scale !== 1) {
1676
+ pipeline = pipeline.resize(preview.width, preview.height, { fit: 'fill' })
1677
+ }
1678
+ const cropped = await pipeline.png().toBuffer()
1679
+ const target = await saveArtifact(
1680
+ exec,
1681
+ `${artifactStem(args.image, `crop-${box.x1}-${box.y1}-${box.x2}-${box.y2}`)}.png`,
1682
+ cropped,
1683
+ )
1684
+ const meta = await sharp(cropped).metadata()
1685
+ return JSON.stringify({
1686
+ path: target,
1687
+ width: meta.width ?? preview.width,
1688
+ height: meta.height ?? preview.height,
1689
+ bytes: cropped.length,
1690
+ ...(preview.scale !== 1
1691
+ ? {
1692
+ preview: true,
1693
+ sourceRegion: box,
1694
+ sourceWidth,
1695
+ sourceHeight,
1696
+ scale: preview.scale,
1697
+ advice: 'This was a bounded preview of a large crop. Use vision_crop again with a smaller ORIGINAL-pixel region for tiny details.',
1698
+ }
1699
+ : {}),
1700
+ })
1701
+ },
1702
+ })
1703
+
1704
+ // ── vision_present ───────────────────────────────────────────────────────
1705
+ const visionPresentOutput = {
1706
+ schema: {
1707
+ type: 'object',
1708
+ properties: {
1709
+ path: { type: 'string' },
1710
+ label: { type: 'string' },
1711
+ width: { type: 'number' },
1712
+ height: { type: 'number' },
1713
+ bytes: { type: 'number' },
1714
+ safePresentation: { type: 'boolean' },
1715
+ attachment: {
1716
+ type: 'object',
1717
+ properties: {
1718
+ attachmentId: { type: 'string' },
1719
+ mediaType: { type: 'string' },
1720
+ bytes: { type: 'number' },
1721
+ width: { type: 'number' },
1722
+ height: { type: 'number' },
1723
+ name: { type: 'string' },
1724
+ },
1725
+ required: ['attachmentId', 'mediaType', 'bytes', 'width', 'height'],
1726
+ additionalProperties: false,
1727
+ },
1728
+ },
1729
+ required: ['path', 'label', 'width', 'height', 'bytes', 'safePresentation', 'attachment'],
1730
+ additionalProperties: false,
1731
+ },
1732
+ render: (_args, value) => {
1733
+ const attachment = value.attachment
1734
+ return [
1735
+ {
1736
+ type: 'text',
1737
+ text: JSON.stringify({
1738
+ path: value.path,
1739
+ label: value.label,
1740
+ width: value.width,
1741
+ height: value.height,
1742
+ bytes: value.bytes,
1743
+ safePresentation: true,
1744
+ attachmentId: String(attachment.attachmentId),
1745
+ }),
1746
+ },
1747
+ { type: 'image', attachment },
1748
+ ]
1749
+ },
1750
+ }
1751
+ tools.push({
1752
+ name: 'vision_present',
1753
+ description:
1754
+ 'Present a generated local image directly to the user with the host-native image preview. ' +
1755
+ 'MANDATORY PRESENTATION RULE: when you generate, edit, screenshot, or export an image and want the user to see it, ' +
1756
+ 'you MUST call vision_present. read_image is only for model-side inspection; NEVER use read_image to present or send ' +
1757
+ 'an image to the user. The image is retained in the session UI but sanitized out of later text-only model requests.',
1758
+ parameters: {
1759
+ type: 'object',
1760
+ properties: {
1761
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1762
+ label: { type: 'string', description: 'Optional short user-facing label for the image' },
1763
+ },
1764
+ required: ['image'],
1765
+ additionalProperties: false,
1766
+ },
1767
+ output: visionPresentOutput,
1768
+ async execute(args, exec) {
1769
+ const attachments = ctx.get('attachments')
1770
+ if (attachments === undefined) {
1771
+ throw new Error('vision_present: the durable attachment service is not available in this deployment')
1772
+ }
1773
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1774
+ const label =
1775
+ typeof args.label === 'string' && args.label.trim() !== '' ? args.label.trim().slice(0, 200) : 'image'
1776
+ const extension =
1777
+ mediaType === 'image/jpeg' ? 'jpg' :
1778
+ mediaType === 'image/webp' ? 'webp' :
1779
+ mediaType === 'image/gif' ? 'gif' : 'png'
1780
+ const target = await saveArtifact(exec, `${artifactStem(args.image, 'present')}.${extension}`, bytes)
1781
+ let attachment
1782
+ try {
1783
+ attachment = await attachments.saveImage({
1784
+ data: bytes,
1785
+ mediaType,
1786
+ name: label,
1787
+ })
1788
+ } catch (error) {
1789
+ throw new Error(
1790
+ `vision_present: failed to publish the image attachment (${error && error.message ? error.message : String(error)})`,
1791
+ )
1792
+ }
1793
+ return {
1794
+ path: target,
1795
+ label,
1796
+ width: attachment.width,
1797
+ height: attachment.height,
1798
+ bytes: attachment.bytes,
1799
+ safePresentation: true,
1800
+ attachment,
1801
+ }
1802
+ },
1803
+ })
1804
+
1805
+ // ── vision_pixel_diff ────────────────────────────────────────────────────
1806
+ tools.push({
1807
+ name: 'vision_pixel_diff',
1808
+ description:
1809
+ 'Compare two images pixel by pixel (sharp-based, no Python): returns the differing-pixel ' +
1810
+ 'ratio, the worst 8x8-grid regions as original-pixel boxes, and writes a red heatmap PNG ' +
1811
+ 'plus a JSON report as artifacts. Use it to verify an implementation against a reference.',
1812
+ parameters: {
1813
+ type: 'object',
1814
+ properties: {
1815
+ original: { type: 'string', description: 'Reference image path or attachment id (e.g. "sha256:...")' },
1816
+ rebuilt: { type: 'string', description: 'Candidate image path or attachment id (e.g. "sha256:..."); resized to the original size before comparing' },
1817
+ threshold: { type: 'number', description: 'Per-channel difference threshold, default 16' },
1818
+ },
1819
+ required: ['original', 'rebuilt'],
1820
+ additionalProperties: false,
1821
+ },
1822
+ output: stringOutput,
1823
+ async execute(args, exec) {
1824
+ const { bytes: originalBytes } = await readImageBytes(ctx, exec, args.original)
1825
+ const { bytes: rebuiltBytes } = await readImageBytes(ctx, exec, args.rebuilt)
1826
+ const sharp = await loadSharp()
1827
+ if (!sharp) throw new Error('vision_pixel_diff: the sharp image library is unavailable')
1828
+ const meta = await sharp(originalBytes, { failOn: 'none' }).metadata()
1829
+ const width = meta.width ?? 0
1830
+ const height = meta.height ?? 0
1831
+ if (width <= 0 || height <= 0) throw new Error('vision_pixel_diff: could not read original dimensions')
1832
+ const threshold = Number.isFinite(args.threshold) && args.threshold >= 0 ? Math.round(args.threshold) : 16
1833
+ const pixels = width * height
1834
+ let diff
1835
+ let heatmapPng
1836
+ let heatmapPreview = false
1837
+ let heatmapWidth = width
1838
+ let heatmapHeight = height
1839
+ if (pixels <= 4_000_000) {
1840
+ const originalRaw = await sharp(originalBytes, { failOn: 'none' })
1841
+ .ensureAlpha()
1842
+ .raw()
1843
+ .toBuffer({ resolveWithObject: true })
1844
+ const rebuiltRaw = await sharp(rebuiltBytes, { failOn: 'none' })
1845
+ .resize(width, height, { fit: 'fill' })
1846
+ .ensureAlpha()
1847
+ .raw()
1848
+ .toBuffer({ resolveWithObject: true })
1849
+ diff = computePixelDiff(originalRaw.data, rebuiltRaw.data, threshold, width, height)
1850
+ const heatmap = renderDiffHeatmap(originalRaw.data, diff.mask, width, height)
1851
+ heatmapPng = await sharp(heatmap, { raw: { width, height, channels: 4 } })
1852
+ .png()
1853
+ .toBuffer()
1854
+ } else {
1855
+ // Exact large-image metrics from streaming RGBA; heatmap bounded to 4MP.
1856
+ const originalStream = sharp(originalBytes, { failOn: 'none' }).ensureAlpha().raw()
1857
+ const rebuiltStream = sharp(rebuiltBytes, { failOn: 'none' })
1858
+ .resize(width, height, { fit: 'fill' })
1859
+ .ensureAlpha()
1860
+ .raw()
1861
+ diff = await compareRgbaStreams(originalStream, rebuiltStream, { width, height, threshold })
1862
+ const preview = scaledDimensions(width, height, 4_000_000)
1863
+ heatmapWidth = preview.width
1864
+ heatmapHeight = preview.height
1865
+ heatmapPreview = true
1866
+ const originalPreview = await sharp(originalBytes, { failOn: 'none' })
1867
+ .resize(preview.width, preview.height, { fit: 'fill' })
1868
+ .ensureAlpha()
1869
+ .raw()
1870
+ .toBuffer({ resolveWithObject: true })
1871
+ const rebuiltPreview = await sharp(rebuiltBytes, { failOn: 'none' })
1872
+ .resize(preview.width, preview.height, { fit: 'fill' })
1873
+ .ensureAlpha()
1874
+ .raw()
1875
+ .toBuffer({ resolveWithObject: true })
1876
+ const previewDiff = computePixelDiff(
1877
+ originalPreview.data,
1878
+ rebuiltPreview.data,
1879
+ threshold,
1880
+ preview.width,
1881
+ preview.height,
1882
+ )
1883
+ const heatmap = renderDiffHeatmap(originalPreview.data, previewDiff.mask, preview.width, preview.height)
1884
+ heatmapPng = await sharp(heatmap, {
1885
+ raw: { width: preview.width, height: preview.height, channels: 4 },
1886
+ }).png().toBuffer()
1887
+ }
1888
+ const worst = diff.cells.slice(0, 5).map((cell) => ({
1889
+ x1: cell.x1,
1890
+ y1: cell.y1,
1891
+ x2: cell.x2,
1892
+ y2: cell.y2,
1893
+ ratio: Number(cell.ratio.toFixed(4)),
1894
+ differing: cell.differing,
1895
+ }))
1896
+ const heatmapPath = await saveArtifact(exec, `${artifactStem(args.original, 'diff-heatmap')}.png`, heatmapPng)
1897
+ const report = {
1898
+ width,
1899
+ height,
1900
+ differingPixels: diff.differing,
1901
+ totalPixels: diff.total,
1902
+ differingRatio: Number(diff.ratio.toFixed(6)),
1903
+ threshold,
1904
+ worstRegions: worst,
1905
+ heatmapPath,
1906
+ heatmapWidth,
1907
+ heatmapHeight,
1908
+ heatmapPreview,
1909
+ }
1910
+ const reportPath = await saveArtifact(exec, `${artifactStem(args.original, 'diff-report')}.json`, Buffer.from(JSON.stringify(report, null, 2)))
1911
+ return JSON.stringify({ ...report, reportPath })
1912
+ },
1913
+ })
1914
+
1915
+ // ── vision_colors ────────────────────────────────────────────────────────
1916
+ tools.push({
1917
+ name: 'vision_colors',
1918
+ description:
1919
+ 'Extract the dominant colors of an image (sharp-based quantization) with their share of ' +
1920
+ 'pixels, e.g. to match a palette when rebuilding a UI.',
1921
+ parameters: {
1922
+ type: 'object',
1923
+ properties: {
1924
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1925
+ top: { type: 'number', description: 'How many colors to return, default 8' },
1926
+ },
1927
+ required: ['image'],
1928
+ additionalProperties: false,
1929
+ },
1930
+ output: stringOutput,
1931
+ async execute(args, exec) {
1932
+ const { bytes } = await readImageBytes(ctx, exec, args.image)
1933
+ const top = Number.isInteger(args.top) && args.top > 0 ? args.top : 8
1934
+ const sharp = await loadSharp()
1935
+ if (!sharp) throw new Error('vision_colors: the sharp image library is unavailable')
1936
+ const raw = await sharp(bytes, { failOn: 'none' })
1937
+ .resize(64, 64, { fit: 'inside' })
1938
+ .ensureAlpha()
1939
+ .raw()
1940
+ .toBuffer({ resolveWithObject: true })
1941
+ const colors = quantizeColors(raw.data, Math.min(top, 32))
1942
+ return JSON.stringify(colors)
1943
+ },
1944
+ })
1945
+
1946
+ // ── vision_ocr ───────────────────────────────────────────────────────────
1947
+ tools.push({
1948
+ name: 'vision_ocr',
1949
+ description:
1950
+ 'Transcribe TEXT from an image with the vision model. ' +
1951
+ 'Returns the text and which engine produced it. ' +
1952
+ 'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
1953
+ 'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
1954
+ '"这是什么东西" questions are answered by vision_describe, not OCR). If vision_describe returns ' +
1955
+ 'ok:false with a backend-unavailable code, calling vision_ocr instead will fail the same way — ' +
1956
+ 'do not chain these tools as retries of each other. ' +
1957
+ 'ACCURACY: OCR transcribes characters verbatim and is systematically unreliable for confusable ' +
1958
+ 'glyphs (1/l, 0/O), spacing and line breaks; prefer vision_describe / vision_detect for semantic ' +
1959
+ 'understanding and use OCR only when exact verbatim text is required (executable code, exact ' +
1960
+ 'quotation, forms/contracts, table digits, CAPTCHAs). Treat OCR output as evidence to verify, ' +
1961
+ 'never as ground truth.',
1962
+ parameters: {
1963
+ type: 'object',
1964
+ properties: {
1965
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
1966
+ engine: {
1967
+ type: 'string',
1968
+ description: '"auto" (default) or "vision": transcribe with the configured vision model. "tesseract" is not available in this build.',
1969
+ },
1970
+ },
1971
+ required: ['image'],
1972
+ additionalProperties: false,
1973
+ },
1974
+ output: stringOutput,
1975
+ async execute(args, exec) {
1976
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
1977
+ const engine = args.engine === 'vision' ? 'vision' : 'auto'
1978
+ if (args.engine === 'tesseract') {
1979
+ throw new Error('vision_ocr: the local tesseract engine is not available in this build; use engine "auto" or "vision"')
1980
+ }
1981
+ const instruction =
1982
+ '请原样转述图中的所有文字,保持阅读顺序(从上到下、从左到右)与段落结构,不要添加解释。只输出文字本身。'
1983
+ const vision = await answerVisionForTool(bytes, mediaType, instruction, { exec })
1984
+ if (vision.ok === false) {
1985
+ return JSON.stringify({ engine: 'none', ...vision, text: '' })
1986
+ }
1987
+ return JSON.stringify({ engine, text: vision.text })
1988
+ },
1989
+ })
1990
+
1991
+ // ── vision_long_screenshot_ocr ───────────────────────────────────────────
1992
+ tools.push({
1993
+ name: 'vision_long_screenshot_ocr',
1994
+ description:
1995
+ 'Transcribe a LONG screenshot (chat logs, long documents) into ordered Markdown. ' +
1996
+ 'Splits the image into overlapping horizontal chunks, OCRs each chunk with the vision ' +
1997
+ 'model, and stitches the text in reading order. Writes chunk PNGs, the Markdown, and a ' +
1998
+ 'manifest into the workspace artifacts directory.',
1999
+ parameters: {
2000
+ type: 'object',
2001
+ properties: {
2002
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2003
+ chunkHeight: { type: 'number', description: 'Chunk height in pixels, default 1200' },
2004
+ overlap: { type: 'number', description: 'Overlap between adjacent chunks in pixels, default 120' },
2005
+ engine: { type: 'string', description: '"auto" (default) or "vision": transcribe with the configured vision model.' },
2006
+ },
2007
+ required: ['image'],
2008
+ additionalProperties: false,
2009
+ },
2010
+ output: stringOutput,
2011
+ async execute(args, exec) {
2012
+ const { bytes, mediaType } = await readImageBytes(ctx, exec, args.image)
2013
+ const sharp = await loadSharp()
2014
+ if (!sharp) throw new Error('vision_long_screenshot_ocr: the sharp image library is unavailable')
2015
+ const meta = await sharp(bytes, { failOn: 'none' }).metadata()
2016
+ const width = meta.width ?? 0
2017
+ const height = meta.height ?? 0
2018
+ if (width <= 0 || height <= 0) {
2019
+ throw new Error('vision_long_screenshot_ocr: could not read image dimensions')
2020
+ }
2021
+ const chunkHeight =
2022
+ Number.isInteger(args.chunkHeight) && args.chunkHeight >= 400
2023
+ ? Math.min(args.chunkHeight, 2000)
2024
+ : 1200
2025
+ const overlap =
2026
+ Number.isInteger(args.overlap) && args.overlap >= 0
2027
+ ? Math.min(args.overlap, Math.floor(chunkHeight / 2))
2028
+ : 120
2029
+ if (args.engine === 'tesseract') {
2030
+ throw new Error('vision_long_screenshot_ocr: the local tesseract engine is not available in this build; use engine "auto" or "vision"')
2031
+ }
2032
+ const windows = boundedOcrTiles(width, height, {
2033
+ chunkHeight,
2034
+ overlap,
2035
+ maxTilePixels: 4_000_000,
2036
+ })
2037
+ // Hard bounds: a tall screenshot must never turn into an unbounded,
2038
+ // uncancellable model-call loop (router's 30s budget was dropped here,
2039
+ // so re-introduce explicit limits).
2040
+ const MAX_OCR_CHUNKS = 40
2041
+ const OCR_TOTAL_BUDGET_MS = 120_000
2042
+ const ocrStartedAt = Date.now()
2043
+ const signal = exec && exec.signal
2044
+ const stem = artifactStem(args.image, 'ocr')
2045
+ const results = []
2046
+ let visionFailed = false
2047
+ for (let i = 0; i < windows.length; i++) {
2048
+ if (signal?.aborted) break
2049
+ if (i >= MAX_OCR_CHUNKS) {
2050
+ results.push({
2051
+ chunk: i + 1,
2052
+ left: windows[i].left,
2053
+ right: windows[i].right,
2054
+ top: windows[i].top,
2055
+ bottom: windows[i].bottom,
2056
+ engine: 'skipped',
2057
+ chars: 0,
2058
+ text: '',
2059
+ error: 'chunk limit reached',
2060
+ })
2061
+ continue
2062
+ }
2063
+ if (Date.now() - ocrStartedAt > OCR_TOTAL_BUDGET_MS) {
2064
+ results.push({
2065
+ chunk: i + 1,
2066
+ left: windows[i].left,
2067
+ right: windows[i].right,
2068
+ top: windows[i].top,
2069
+ bottom: windows[i].bottom,
2070
+ engine: 'skipped',
2071
+ chars: 0,
2072
+ text: '',
2073
+ error: 'OCR total budget exhausted',
2074
+ })
2075
+ continue
2076
+ }
2077
+ const { left, right, top, bottom } = windows[i]
2078
+ const tileWidth = right - left
2079
+ const tileHeight = bottom - top
2080
+ const chunk = await sharp(bytes, { failOn: 'none' })
2081
+ .extract({ left, top, width: tileWidth, height: tileHeight })
2082
+ .png()
2083
+ .toBuffer()
2084
+ const chunkRel = `chunk-${String(i + 1).padStart(2, '0')}.png`
2085
+ await writeArtifact(exec, join(stem, chunkRel), chunk)
2086
+ let text = ''
2087
+ let used = 'none'
2088
+ // Upload JPEG without an alpha channel: some vision backends degrade
2089
+ // on RGBA PNGs and hallucinate token-fragment text.
2090
+ const visionBytes = await sharp(chunk, { failOn: 'none' })
2091
+ .removeAlpha()
2092
+ .jpeg({ quality: 92 })
2093
+ .toBuffer()
2094
+ const instruction =
2095
+ '请原样转述这张长截图分片中的所有文字,保持阅读顺序(从上到下、从左到右),' +
2096
+ '不要添加解释,只输出文字本身。如果画面中没有可见文字,只输出 EMPTY,不要编造内容。'
2097
+ if (!visionFailed) {
2098
+ const visionResult = await answerVisionForTool(visionBytes, 'image/jpeg', instruction, { exec })
2099
+ if (visionResult.ok === false) {
2100
+ // Backend failure: stop burning calls on the remaining chunks.
2101
+ visionFailed = true
2102
+ used = 'failed'
2103
+ } else {
2104
+ text = visionResult.text.trim()
2105
+ if (text.length > 12000) {
2106
+ const retry = await answerVisionForTool(
2107
+ visionBytes,
2108
+ 'image/jpeg',
2109
+ '重新转写这张图片中的真实文字,保持阅读顺序。只输出图中肉眼可见的文字,' +
2110
+ '禁止编造、禁止重复;总输出不超过 3000 字。没有任何文字就只输出 EMPTY。',
2111
+ { exec },
2112
+ )
2113
+ if (retry.ok === false) {
2114
+ visionFailed = true
2115
+ used = 'failed'
2116
+ text = ''
2117
+ } else {
2118
+ const retryText = retry.text.trim()
2119
+ if (retryText !== '') text = retryText
2120
+ used = 'vision'
2121
+ }
2122
+ } else {
2123
+ used = 'vision'
2124
+ }
2125
+ if (text === 'EMPTY') text = ''
2126
+ }
2127
+ }
2128
+ results.push({ chunk: i + 1, left, right, top, bottom, engine: used, chars: text.length, text })
2129
+ }
2130
+ const joined = results.map((r) => r.text).filter((t) => t !== '').join('\n\n')
2131
+ const engines = {}
2132
+ for (const r of results) engines[r.engine] = (engines[r.engine] ?? 0) + 1
2133
+ const manifest = {
2134
+ source: args.image,
2135
+ width,
2136
+ height,
2137
+ chunkHeight,
2138
+ overlap,
2139
+ chunks: results.length,
2140
+ engines,
2141
+ perChunk: results.map(({ text, ...rest }) => rest),
2142
+ }
2143
+ const manifestPath = await writeArtifact(exec, join(stem, 'manifest.json'), JSON.stringify(manifest, null, 2))
2144
+ const mdPath = await writeArtifact(exec, join(stem, 'ocr.md'), joined)
2145
+ return JSON.stringify({
2146
+ text: joined,
2147
+ chunks: results.length,
2148
+ engines,
2149
+ markdownPath: mdPath,
2150
+ manifestPath,
2151
+ artifactsDir: dirname(mdPath),
2152
+ })
2153
+ },
2154
+ })
2155
+
2156
+ // ── vision_trace ─────────────────────────────────────────────────────────
2157
+ tools.push({
2158
+ name: 'vision_trace',
2159
+ description:
2160
+ 'Vectorize an image (icon/logo) into an SVG via a local potrace pipeline (no Python). ' +
2161
+ 'Default: COLOR-preserving vectorization — one path per dominant color with fill="#rrggbb". ' +
2162
+ 'Set color=false for the layered grayscale posterization, where `steps` (1-16, default 4) ' +
2163
+ 'controls levels. Writes the SVG as an artifact.',
2164
+ parameters: {
2165
+ type: 'object',
2166
+ properties: {
2167
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2168
+ steps: { type: 'number', description: 'Posterization steps, 1-16, default 4 (only when color=false)' },
2169
+ color: { type: 'boolean', description: 'Preserve original colors (default true)' },
2170
+ colors: { type: 'number', description: 'Number of dominant colors in color mode, 1-16, default 8' },
2171
+ },
2172
+ required: ['image'],
2173
+ additionalProperties: false,
2174
+ },
2175
+ output: stringOutput,
2176
+ async execute(args, exec) {
2177
+ const { bytes } = await readImageBytes(ctx, exec, args.image)
2178
+ const steps = Number.isInteger(args.steps) && args.steps > 0 ? Math.min(args.steps, 16) : 4
2179
+ const colorMode = args.color !== false
2180
+ // Trace-specific pixel budget: vectorization gains nothing beyond ~1MP.
2181
+ let traceBytes = bytes
2182
+ if (bytes && bytes.length > 0) {
2183
+ traceBytes = await downscaleImage(bytes, 1_000_000)
2184
+ }
2185
+ let svg
2186
+ let colorCount = 0
2187
+ try {
2188
+ if (colorMode) {
2189
+ const sharp = await loadSharp()
2190
+ if (!sharp) throw new Error('vision_trace: the sharp image library is unavailable')
2191
+ const colors = Number.isInteger(args.colors) && args.colors > 0 ? Math.min(args.colors, 16) : 8
2192
+ const raw = await sharp(traceBytes, { failOn: 'none' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true })
2193
+ const palette = quantizeColors(raw.data, colors)
2194
+ colorCount = palette.length
2195
+ svg = await posterizeSvgColor(raw.data, raw.info, palette, getConfig().timeoutMs || 60000)
2196
+ } else {
2197
+ svg = await posterizeSvg(traceBytes, steps, 'dominant', getConfig().timeoutMs || 60000)
2198
+ }
2199
+ } catch (error) {
2200
+ throw new Error(
2201
+ `vision_trace: potrace failed (${error && error.message ? error.message : String(error)}). ` +
2202
+ 'If potrace is missing, install the plugin dependencies in the profile (pnpm add potrace).',
2203
+ )
2204
+ }
2205
+ const target = await saveArtifact(
2206
+ exec,
2207
+ `${artifactStem(args.image, colorMode ? 'trace-color' : `trace-${steps}`)}.svg`,
2208
+ Buffer.from(svg),
2209
+ )
2210
+ return JSON.stringify({ path: target, bytes: Buffer.byteLength(svg), ...(colorMode ? { colors: colorCount } : {}) })
2211
+ },
2212
+ })
2213
+
2214
+ // ── vision_extract_foreground ────────────────────────────────────────────
2215
+ tools.push({
2216
+ name: 'vision_extract_foreground',
2217
+ description:
2218
+ 'Remove a solid-ish background (border flood fill with color tolerance, no Python) and ' +
2219
+ 'write the cutout as a transparent PNG artifact. Best for logos on uniform backgrounds.',
2220
+ parameters: {
2221
+ type: 'object',
2222
+ properties: {
2223
+ image: { type: 'string', description: 'Local image path (png/jpeg/webp/gif), workspace-relative or absolute; or the attachment id (e.g. "sha256:...") of an image uploaded in this conversation' },
2224
+ tolerance: { type: 'number', description: 'Max per-channel color distance from the background, default 40' },
2225
+ },
2226
+ required: ['image'],
2227
+ additionalProperties: false,
2228
+ },
2229
+ output: stringOutput,
2230
+ async execute(args, exec) {
2231
+ const { bytes } = await readImageBytes(ctx, exec, args.image)
2232
+ let fgBytes = bytes
2233
+ if (bytes && bytes.length > 0) {
2234
+ fgBytes = await downscaleImage(bytes, 4_000_000)
2235
+ }
2236
+ const tolerance = Number.isFinite(args.tolerance) && args.tolerance >= 0 ? Math.round(args.tolerance) : 40
2237
+ const sharp = await loadSharp()
2238
+ if (!sharp) throw new Error('vision_extract_foreground: the sharp image library is unavailable')
2239
+ const { data, info } = await sharp(fgBytes, { failOn: 'none' })
2240
+ .ensureAlpha()
2241
+ .raw()
2242
+ .toBuffer({ resolveWithObject: true })
2243
+ const cutout = floodFillBackground(data, info.width, info.height, tolerance)
2244
+ const png = await sharp(cutout, {
2245
+ raw: { width: info.width, height: info.height, channels: 4 },
2246
+ })
2247
+ .png()
2248
+ .toBuffer()
2249
+ const target = await saveArtifact(exec, `${artifactStem(args.image, 'fg')}.png`, png)
2250
+ return JSON.stringify({ path: target, width: info.width, height: info.height, bytes: png.length })
2251
+ },
2252
+ })
2253
+
2254
+ // ── vision_html_screenshot ───────────────────────────────────────────────
2255
+ tools.push({
2256
+ name: 'vision_html_screenshot',
2257
+ description:
2258
+ 'Render a local .html/.htm file in the system Chrome (headless, network disabled by ' +
2259
+ 'default) and save a PNG screenshot as an artifact — the verify step of the ' +
2260
+ 'reference -> implementation -> screenshot -> pixel-diff loop. With fullPage: true the ' +
2261
+ 'page keeps the requested viewport but the whole scrollable height is captured and the ' +
2262
+ 'result JSON reports pageHeight (CSS px).',
2263
+ parameters: {
2264
+ type: 'object',
2265
+ properties: {
2266
+ source: { type: 'string', description: 'Local .html or .htm file path' },
2267
+ width: { type: 'number', description: 'Viewport width, default 1200' },
2268
+ height: { type: 'number', description: 'Viewport height, default 720' },
2269
+ fullPage: {
2270
+ type: 'boolean',
2271
+ description:
2272
+ 'Capture the complete scrollable page height at the requested viewport width ' +
2273
+ 'instead of just the viewport (default false). Lazy-loaded images and ' +
2274
+ 'scroll-triggered reveals are woken first; the result JSON then includes ' +
2275
+ 'pageHeight (CSS px).',
2276
+ },
2277
+ },
2278
+ required: ['source'],
2279
+ additionalProperties: false,
2280
+ },
2281
+ output: stringOutput,
2282
+ async execute(args, exec) {
2283
+ const source = String(args.source ?? '')
2284
+ if (!/\.(html?|htm)$/i.test(source)) {
2285
+ throw new Error('vision_html_screenshot: source must be a local .html/.htm file')
2286
+ }
2287
+ const fsService = ctx.get('fs')
2288
+ if (fsService === undefined) {
2289
+ throw new Error('vision_html_screenshot: the fs service is not available')
2290
+ }
2291
+ const resolved = await fsService.resolve(source)
2292
+ const targetPath = toRealPath(fsService, resolved)
2293
+ if (!existsSync(targetPath)) {
2294
+ throw new Error(`vision_html_screenshot: file not found: ${source}`)
2295
+ }
2296
+ let puppeteer
2297
+ try {
2298
+ puppeteer = await import('puppeteer-core')
2299
+ } catch {
2300
+ throw new Error(
2301
+ 'vision_html_screenshot: puppeteer-core is not installed. Install the plugin dependencies in the profile (pnpm add puppeteer-core).',
2302
+ )
2303
+ }
2304
+ const candidates = chromiumCandidates(
2305
+ typeof process !== 'undefined' && process.env ? process.env : {},
2306
+ typeof process !== 'undefined' ? process.platform : '',
2307
+ )
2308
+ const executablePath = candidates.find((p) => existsSync(p))
2309
+ if (executablePath === undefined) {
2310
+ throw new Error(
2311
+ 'vision_html_screenshot: no Chrome/Chromium/Edge found; install one or set CHROME_PATH / PUPPETEER_EXECUTABLE_PATH',
2312
+ )
2313
+ }
2314
+ const width = Number.isInteger(args.width) && args.width > 0 ? args.width : 1200
2315
+ const height = Number.isInteger(args.height) && args.height > 0 ? args.height : 720
2316
+ const fullPage = args.fullPage === true
2317
+ const launchArgs = ['--no-sandbox', '--disable-gpu', '--hide-scrollbars', '--incognito']
2318
+ if (fullPage) {
2319
+ launchArgs.push('--blink-settings=imagesLazyLoadingEnabled=false')
2320
+ }
2321
+ const browser = await puppeteer.default.launch({
2322
+ executablePath,
2323
+ headless: true,
2324
+ args: launchArgs,
2325
+ })
2326
+ try {
2327
+ const page = await browser.newPage()
2328
+ await page.setViewport({ width, height })
2329
+ // Honor the "network disabled by default" contract: the page may only
2330
+ // load file:/data:/blob: subresources; everything else is aborted so
2331
+ // a malicious local HTML cannot exfiltrate data or pull remote assets.
2332
+ await page.setRequestInterception(true)
2333
+ page.on('request', (request) => {
2334
+ const url = request.url() ?? ''
2335
+ if (/^(file|data|blob):/i.test(url)) request.continue()
2336
+ else request.abort()
2337
+ })
2338
+ await page.goto(pathToFileURL(targetPath).href, { waitUntil: 'networkidle0', timeout: 30000 })
2339
+ let pageHeight
2340
+ if (fullPage) {
2341
+ await wakePageForFullCapture(page, height)
2342
+ pageHeight = await fullPageHeightOf(page)
2343
+ }
2344
+ const png = fullPage
2345
+ ? await page.screenshot({ type: 'png', fullPage: true })
2346
+ : await page.screenshot({ type: 'png' })
2347
+ const stem = fullPage ? `shot-${width}x${height}-fullpage` : `shot-${width}x${height}`
2348
+ const target = await saveArtifact(exec, `${artifactStem(source, stem)}.png`, png)
2349
+ const result = { path: target, width, height, bytes: png.length }
2350
+ if (fullPage) {
2351
+ result.pageHeight = pageHeight
2352
+ }
2353
+ return JSON.stringify(result)
2354
+ } finally {
2355
+ await browser.close()
2356
+ }
2357
+ },
2358
+ })
2359
+
2360
+ // ── vision_screenshot (privacy-gated: only registered when desktopScreenshot is enabled) ──
2361
+ if (getConfig().desktopScreenshot === true) {
2362
+ tools.push({
2363
+ name: 'vision_screenshot',
2364
+ description:
2365
+ 'Capture the user\'s desktop screen as a PNG artifact (the virtual screen on Windows; the main display on macOS; the root display on Linux). ' +
2366
+ 'Windows: PowerShell CopyFromScreen; macOS: screencapture; Linux: ImageMagick import (falls back to scrot; either command must be installed). ' +
2367
+ 'Use it when you need to see what is on the user\'s screen right now — e.g. their current GUI, an app, or a page outside this browser. ' +
2368
+ 'Optional identify=true also runs recognition on the capture with the configured vision model and returns the description alongside the path.',
2369
+ parameters: {
2370
+ type: 'object',
2371
+ properties: {
2372
+ identify: {
2373
+ type: 'boolean',
2374
+ description:
2375
+ 'Also recognize the captured screen with the configured vision model and return the description text with the path. Default false.',
2376
+ },
2377
+ },
2378
+ additionalProperties: false,
2379
+ },
2380
+ output: stringOutput,
2381
+ async execute(args, exec) {
2382
+ const tmp = join(
2383
+ (typeof process !== 'undefined' && process.env && process.env.TMPDIR) || process.env.TEMP || '/tmp',
2384
+ `vision-screenshot-${Date.now()}-${Math.floor(Math.random() * 1e9)}.png`,
2385
+ )
2386
+ const platform = process.platform
2387
+ try {
2388
+ if (platform === 'win32') {
2389
+ const script = [
2390
+ 'Add-Type -AssemblyName System.Windows.Forms,System.Drawing',
2391
+ '$b=[System.Windows.Forms.SystemInformation]::VirtualScreen',
2392
+ '$bmp=New-Object System.Drawing.Bitmap($b.Width,$b.Height)',
2393
+ '$g=[System.Drawing.Graphics]::FromImage($bmp)',
2394
+ '$g.CopyFromScreen($b.X,$b.Y,0,0,$bmp.Size)',
2395
+ `$bmp.Save('${tmp.replace(/'/g, "''")}')`,
2396
+ '$g.Dispose();$bmp.Dispose()',
2397
+ ].join('; ')
2398
+ await promisify(execFile)('powershell.exe', ['-NoProfile', '-STA', '-Command', script], {
2399
+ timeout: 60000,
2400
+ windowsHide: true,
2401
+ })
2402
+ } else if (platform === 'darwin') {
2403
+ await promisify(execFile)('screencapture', ['-x', '-m', tmp], {
2404
+ timeout: 60000,
2405
+ windowsHide: true,
2406
+ })
2407
+ } else {
2408
+ try {
2409
+ await promisify(execFile)('import', ['-window', 'root', tmp], { timeout: 60000 })
2410
+ } catch {
2411
+ await promisify(execFile)('scrot', [tmp], { timeout: 60000 })
2412
+ }
2413
+ }
2414
+ if (!existsSync(tmp)) {
2415
+ throw new Error(
2416
+ `vision_screenshot: no output produced on ${platform} (is a screen available?)`,
2417
+ )
2418
+ }
2419
+ const data = await readFileP(tmp)
2420
+ const target = await saveArtifact(exec, `screenshot-${Date.now()}.png`, data)
2421
+ const result = { path: target, bytes: data.length }
2422
+ if (args.identify === true) {
2423
+ let identifyBytes = data
2424
+ try {
2425
+ const sharp = await loadSharp()
2426
+ if (sharp) {
2427
+ const downscaled = await sharp(data, { failOn: 'none' })
2428
+ .resize({ width: 1280, height: 1280, fit: 'inside', withoutEnlargement: true })
2429
+ .png()
2430
+ .toBuffer()
2431
+ if (downscaled.length > 0 && downscaled.length < data.length) {
2432
+ identifyBytes = downscaled
2433
+ }
2434
+ }
2435
+ } catch {
2436
+ /* keep the original capture */
2437
+ }
2438
+ const vision = await answerVisionForTool(
2439
+ identifyBytes,
2440
+ 'image/png',
2441
+ 'Describe what is currently visible on the user\'s screen, in detail and in reading order. Mention the app/window names you can see.',
2442
+ { exec },
2443
+ )
2444
+ if (vision.ok === false) {
2445
+ result.identify = { ok: false, ...vision, text: '' }
2446
+ } else {
2447
+ result.identify = { ok: true, text: vision.text }
2448
+ }
2449
+ }
2450
+ try {
2451
+ await rmP(tmp, { force: true })
2452
+ } catch {
2453
+ /* best-effort cleanup */
2454
+ }
2455
+ return JSON.stringify(result)
2456
+ } catch (error) {
2457
+ throw new Error(
2458
+ `vision_screenshot: failed (${error && error.message ? error.message : String(error)})`,
2459
+ )
2460
+ }
2461
+ },
2462
+ })
2463
+ }
2464
+
2465
+ for (const def of tools) {
2466
+ ctx.tools.register(def)
2467
+ }
2468
+ logger?.info?.('[tool-vision] registered %d pixel-level vision tools', tools.length)
2469
+ return tools.length
2470
+ }
2471
+
2472
+ // ── cache instance shared across tools ─────────────────────────────────────
2473
+
2474
+ const visionCache = createCache(200, 3600 * 1000)
2475
+ let cacheDisabled = false
2476
+
2477
+ function cacheEnabled() {
2478
+ return !cacheDisabled
2479
+ }
2480
+
2481
+ /** Test hook: reset the shared cache. */
2482
+ export function resetVisionCacheForTests() {
2483
+ visionCache.clear()
2484
+ cacheDisabled = false
2485
+ }