@mengruo/dsh-vision-toolkit 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/LICENSE +21 -0
  2. package/README.i18n.yaml +6 -0
  3. package/README.md +356 -0
  4. package/README.zh.md +354 -0
  5. package/assets/community-group-qr.png +0 -0
  6. package/assets/dsh-conversation-artifact.png +0 -0
  7. package/assets/dsh-conversation-image-qa-top.png +0 -0
  8. package/assets/dsh-conversation-image-qa.png +0 -0
  9. package/assets/dsh-conversation-pixel-diff.png +0 -0
  10. package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
  11. package/assets/dsh-conversation-screenshot-debugging.png +0 -0
  12. package/assets/dsh-conversation-tool-call.png +0 -0
  13. package/assets/dsh-conversation-vision-trace.png +0 -0
  14. package/assets/dsh-view-example.png +0 -0
  15. package/assets/hero-v2.png +0 -0
  16. package/assets/logo_aihubmix.png +0 -0
  17. package/assets/python-bootstrap.json +48 -0
  18. package/assets/skill/SKILL.md +329 -0
  19. package/assets/skill/UPSTREAM.json +71 -0
  20. package/assets/skill/references/gui.md +88 -0
  21. package/assets/skill/references/long-screenshot-ocr.md +77 -0
  22. package/assets/skill/references/restore-graphic.md +84 -0
  23. package/assets/skill/references/restore-structure.md +45 -0
  24. package/assets/skill/references/restore-ui.md +202 -0
  25. package/assets/social-preview.png +0 -0
  26. package/assets/upstream/README.md +18 -0
  27. package/assets/upstream/focus-hint-comparison-1.webp +0 -0
  28. package/assets/upstream/focus-hint-comparison-2.webp +0 -0
  29. package/assets/upstream/infographic-reference.webp +0 -0
  30. package/assets/upstream/infographic-result.webp +0 -0
  31. package/assets/upstream/ui-fast-restore-reference.webp +0 -0
  32. package/assets/upstream/ui-fast-restore-result.webp +0 -0
  33. package/assets/upstream/ui-result.webp +0 -0
  34. package/assets/upstream/ui-sketch.webp +0 -0
  35. package/assets/vision-model-test.png +0 -0
  36. package/assets/vision-settings.png +0 -0
  37. package/assets/wechat-reward.png +0 -0
  38. package/cordis.patch.yml +6 -0
  39. package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
  40. package/docs/aihubmix-gemini-vision.md +138 -0
  41. package/docs/aihubmix-gemini-vision.zh.md +138 -0
  42. package/docs/assets/aihubmix-api-keys.png +0 -0
  43. package/docs/assets/aihubmix-create-key.png +0 -0
  44. package/docs/assets/aihubmix-free-vision-model.png +0 -0
  45. package/docs/assets/aihubmix-home.png +0 -0
  46. package/docs/assets/aihubmix-sign-up.png +0 -0
  47. package/docs/assets/groq-console-home.png +0 -0
  48. package/docs/assets/groq-console-keys-login.png +0 -0
  49. package/docs/assets/groq-docs-quickstart.png +0 -0
  50. package/docs/assets/groq-docs-qwen3.6.png +0 -0
  51. package/docs/assets/groq-docs-vision.png +0 -0
  52. package/docs/assets/vision-settings.png +0 -0
  53. package/docs/dsh-desktop-install.md +67 -0
  54. package/docs/dsh-desktop-install.zh.md +67 -0
  55. package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
  56. package/docs/groq-qwen3.6-vision.md +236 -0
  57. package/docs/groq-qwen3.6-vision.zh.md +236 -0
  58. package/docs/python-runtime.i18n.yaml +6 -0
  59. package/docs/python-runtime.md +89 -0
  60. package/docs/python-runtime.zh.md +89 -0
  61. package/docs/requirements-traceability/README.i18n.yaml +6 -0
  62. package/docs/requirements-traceability/README.md +75 -0
  63. package/docs/requirements-traceability/README.zh.md +75 -0
  64. package/examples/ui-restoration/README.i18n.yaml +6 -0
  65. package/examples/ui-restoration/README.md +70 -0
  66. package/examples/ui-restoration/README.zh.md +70 -0
  67. package/examples/ui-restoration/assets/final-heatmap.png +0 -0
  68. package/examples/ui-restoration/assets/final-report.json +83 -0
  69. package/examples/ui-restoration/assets/implementation.png +0 -0
  70. package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
  71. package/examples/ui-restoration/assets/initial-report.json +83 -0
  72. package/examples/ui-restoration/assets/initial.png +0 -0
  73. package/examples/ui-restoration/assets/metrics.json +12 -0
  74. package/examples/ui-restoration/assets/reference.png +0 -0
  75. package/examples/ui-restoration/implementation.html +94 -0
  76. package/examples/ui-restoration/initial.html +57 -0
  77. package/lib/artifact-access.js +369 -0
  78. package/lib/artifact-access.js.map +1 -0
  79. package/lib/artifacts.js +56 -0
  80. package/lib/artifacts.js.map +1 -0
  81. package/lib/client.js +2099 -0
  82. package/lib/client.js.map +1 -0
  83. package/lib/config.js +172 -0
  84. package/lib/config.js.map +1 -0
  85. package/lib/defaults.js +6 -0
  86. package/lib/defaults.js.map +1 -0
  87. package/lib/errors.js +56 -0
  88. package/lib/errors.js.map +1 -0
  89. package/lib/evidence-cache.js +350 -0
  90. package/lib/evidence-cache.js.map +1 -0
  91. package/lib/exposure.js +247 -0
  92. package/lib/exposure.js.map +1 -0
  93. package/lib/image-input-variants.js +882 -0
  94. package/lib/image-input-variants.js.map +1 -0
  95. package/lib/index.js +105 -0
  96. package/lib/index.js.map +1 -0
  97. package/lib/paste-images.js +217 -0
  98. package/lib/paste-images.js.map +1 -0
  99. package/lib/paths.js +354 -0
  100. package/lib/paths.js.map +1 -0
  101. package/lib/plugin-update.js +1003 -0
  102. package/lib/plugin-update.js.map +1 -0
  103. package/lib/runtime-install.js +964 -0
  104. package/lib/runtime-install.js.map +1 -0
  105. package/lib/runtime-manager.js +133 -0
  106. package/lib/runtime-manager.js.map +1 -0
  107. package/lib/runtime.js +1678 -0
  108. package/lib/runtime.js.map +1 -0
  109. package/lib/skill.js +25 -0
  110. package/lib/skill.js.map +1 -0
  111. package/lib/tools.js +549 -0
  112. package/lib/tools.js.map +1 -0
  113. package/lib/types/artifact-access.d.ts +61 -0
  114. package/lib/types/artifact-access.d.ts.map +1 -0
  115. package/lib/types/artifacts.d.ts +42 -0
  116. package/lib/types/artifacts.d.ts.map +1 -0
  117. package/lib/types/client/display-config.d.ts +24 -0
  118. package/lib/types/client/display-config.d.ts.map +1 -0
  119. package/lib/types/client/index.d.ts +350 -0
  120. package/lib/types/client/index.d.ts.map +1 -0
  121. package/lib/types/client/model-variants-hider.d.ts +40 -0
  122. package/lib/types/client/model-variants-hider.d.ts.map +1 -0
  123. package/lib/types/client/paste-images.d.ts +126 -0
  124. package/lib/types/client/paste-images.d.ts.map +1 -0
  125. package/lib/types/config.d.ts +123 -0
  126. package/lib/types/config.d.ts.map +1 -0
  127. package/lib/types/defaults.d.ts +6 -0
  128. package/lib/types/defaults.d.ts.map +1 -0
  129. package/lib/types/errors.d.ts +35 -0
  130. package/lib/types/errors.d.ts.map +1 -0
  131. package/lib/types/evidence-cache.d.ts +91 -0
  132. package/lib/types/evidence-cache.d.ts.map +1 -0
  133. package/lib/types/exposure.d.ts +50 -0
  134. package/lib/types/exposure.d.ts.map +1 -0
  135. package/lib/types/image-input-variants.d.ts +144 -0
  136. package/lib/types/image-input-variants.d.ts.map +1 -0
  137. package/lib/types/index.d.ts +19 -0
  138. package/lib/types/index.d.ts.map +1 -0
  139. package/lib/types/paste-images.d.ts +73 -0
  140. package/lib/types/paste-images.d.ts.map +1 -0
  141. package/lib/types/paths.d.ts +117 -0
  142. package/lib/types/paths.d.ts.map +1 -0
  143. package/lib/types/plugin-update.d.ts +119 -0
  144. package/lib/types/plugin-update.d.ts.map +1 -0
  145. package/lib/types/runtime-install.d.ts +92 -0
  146. package/lib/types/runtime-install.d.ts.map +1 -0
  147. package/lib/types/runtime-manager.d.ts +60 -0
  148. package/lib/types/runtime-manager.d.ts.map +1 -0
  149. package/lib/types/runtime.d.ts +412 -0
  150. package/lib/types/runtime.d.ts.map +1 -0
  151. package/lib/types/skill.d.ts +14 -0
  152. package/lib/types/skill.d.ts.map +1 -0
  153. package/lib/types/tools.d.ts +35 -0
  154. package/lib/types/tools.d.ts.map +1 -0
  155. package/lib/types/upstream.d.ts +233 -0
  156. package/lib/types/upstream.d.ts.map +1 -0
  157. package/lib/types/version.d.ts +15 -0
  158. package/lib/types/version.d.ts.map +1 -0
  159. package/lib/types/web-request.d.ts +11 -0
  160. package/lib/types/web-request.d.ts.map +1 -0
  161. package/lib/types/web.d.ts +118 -0
  162. package/lib/types/web.d.ts.map +1 -0
  163. package/lib/upstream.js +860 -0
  164. package/lib/upstream.js.map +1 -0
  165. package/lib/version.js +18 -0
  166. package/lib/version.js.map +1 -0
  167. package/lib/web-request.js +29 -0
  168. package/lib/web-request.js.map +1 -0
  169. package/lib/web.js +462 -0
  170. package/lib/web.js.map +1 -0
  171. package/package.json +149 -0
  172. package/patches/vision-tools-dsh.patch +970 -0
  173. package/runtime/requirements.lock +3 -0
  174. package/src/artifact-access.ts +386 -0
  175. package/src/artifacts.ts +85 -0
  176. package/src/client/display-config.ts +62 -0
  177. package/src/client/index.tsx +1580 -0
  178. package/src/client/model-variants-hider.ts +159 -0
  179. package/src/client/paste-images.tsx +747 -0
  180. package/src/config.ts +290 -0
  181. package/src/defaults.ts +5 -0
  182. package/src/errors.ts +62 -0
  183. package/src/evidence-cache.ts +418 -0
  184. package/src/exposure.ts +265 -0
  185. package/src/image-input-variants.ts +996 -0
  186. package/src/index.ts +141 -0
  187. package/src/paste-images.ts +286 -0
  188. package/src/paths.ts +383 -0
  189. package/src/plugin-update.ts +1143 -0
  190. package/src/runtime-install.ts +1161 -0
  191. package/src/runtime-manager.ts +174 -0
  192. package/src/runtime.ts +2189 -0
  193. package/src/skill.ts +35 -0
  194. package/src/tools.ts +691 -0
  195. package/src/upstream.ts +1078 -0
  196. package/src/version.ts +37 -0
  197. package/src/web-request.ts +27 -0
  198. package/src/web.ts +594 -0
  199. package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
  200. package/vendor/agent-vision-toolkit/LICENSE +21 -0
  201. package/vendor/agent-vision-toolkit/README.md +405 -0
  202. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
  203. package/vendor/agent-vision-toolkit/bin/crop +90 -0
  204. package/vendor/agent-vision-toolkit/bin/detect +13 -0
  205. package/vendor/agent-vision-toolkit/bin/glance +93 -0
  206. package/vendor/agent-vision-toolkit/bin/ground +13 -0
  207. package/vendor/agent-vision-toolkit/bin/trace +129 -0
  208. package/vendor/agent-vision-toolkit/detect.py +60 -0
  209. package/vendor/agent-vision-toolkit/ground.py +267 -0
  210. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
  211. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
  212. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
  213. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
  214. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
  215. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
  216. package/vendor/agent-vision-toolkit/vision_client.py +310 -0
package/src/runtime.ts ADDED
@@ -0,0 +1,2189 @@
1
+ /**
2
+ * Vision Toolkit runtime: structured requests in, structured results out.
3
+ * One operation-wide deadline reaches every subprocess; image decoding,
4
+ * byte/pixel limits, session-scoped concurrency, credential resolution, safe
5
+ * output staging, and diagnostic logging stay below the model-facing tools.
6
+ * @module dsh-vision-toolkit/runtime
7
+ */
8
+
9
+ import { createHash, randomUUID } from 'node:crypto'
10
+ import { lstat, mkdir, readFile, readdir, realpath, rename, rm, stat, writeFile } from 'node:fs/promises'
11
+ import { basename, extname, join } from 'node:path'
12
+ import { fileURLToPath } from 'node:url'
13
+ import type { Context } from '@deepseek-ai/cordis'
14
+ import type { ResolvedCredential } from '@deepseek-ai/dsh-credentials'
15
+ import { SaxesParser } from 'saxes'
16
+ import { describeArtifact, type ArtifactDescriptor } from './artifacts.ts'
17
+ import { isBuiltInFreeVisionProvider, type ResolvedVisionToolkitConfig } from './config.ts'
18
+ import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
19
+ import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
20
+ import { VisionToolkitError } from './errors.ts'
21
+ import {
22
+ assertDistinctOutput,
23
+ commitStagedDirectory,
24
+ commitStagedOutput,
25
+ createPathPolicy,
26
+ createStagedDirectory,
27
+ createStagedOutput,
28
+ isWithin,
29
+ resolveHtmlFile,
30
+ resolveInputFile,
31
+ resolveOutputDirectory,
32
+ resolveOutputFile,
33
+ seedStagedDirectory,
34
+ type PathPolicy,
35
+ } from './paths.ts'
36
+ import {
37
+ parseCropOutput,
38
+ parseDominantColorsOutput,
39
+ parseExtractForegroundOutput,
40
+ parseHtmlScreenshotOutput,
41
+ parseLocationOutput,
42
+ parsePixelDiffOutput,
43
+ parseTraceOutput,
44
+ UpstreamAdapter,
45
+ type CompressedImageInfo,
46
+ type DominantColorsOutput,
47
+ type LocatedElement,
48
+ type UpstreamEnvironment,
49
+ type UpstreamRunResult,
50
+ type UpstreamTool,
51
+ type UpstreamVersionInfo,
52
+ } from './upstream.ts'
53
+ import { PLUGIN_VERSION } from './version.ts'
54
+
55
+ const SVG_NAMESPACE = 'http://www.w3.org/2000/svg'
56
+ const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-test.png', import.meta.url))
57
+ const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.'
58
+
59
+ /** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
60
+ const COMPRESSED_IMAGE_CACHE_VERSION = 'v2'
61
+ /** Cache keys carry 64-bit digests so Windows paths stay below MAX_PATH; the full file sha256 is computed on read and compared against this prefix. */
62
+ const COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH = 16
63
+ const COMPRESSED_IMAGE_CACHE_MAX_ENTRIES = 200
64
+ const COMPRESSED_IMAGE_CACHE_MAX_BYTES = 512 * 1024 * 1024
65
+ const COMPRESSED_IMAGE_CACHE_STALE_PARTIAL_MS = 60 * 60 * 1000
66
+
67
+ function svgDocumentPathCount(svg: string): number | undefined {
68
+ const parser = new SaxesParser({ xmlns: true })
69
+ let depth = 0
70
+ let invalid = false
71
+ let pathCount = 0
72
+ let rootSeen = false
73
+ let rootClosed = false
74
+ parser.on('doctype', () => { invalid = true })
75
+ parser.on('error', () => { invalid = true })
76
+ parser.on('opentag', (tag) => {
77
+ if (depth === 0) {
78
+ if (rootSeen || tag.local !== 'svg' || tag.uri !== SVG_NAMESPACE) invalid = true
79
+ rootSeen = true
80
+ }
81
+ if (tag.local === 'path' && tag.uri === SVG_NAMESPACE) pathCount += 1
82
+ depth += 1
83
+ })
84
+ parser.on('closetag', () => {
85
+ depth -= 1
86
+ if (depth === 0) rootClosed = true
87
+ if (depth < 0) invalid = true
88
+ })
89
+ try {
90
+ parser.write(svg).close()
91
+ } catch {
92
+ return undefined
93
+ }
94
+ return invalid || !rootSeen || !rootClosed || depth !== 0 ? undefined : pathCount
95
+ }
96
+
97
+ /** Per-invocation cancellation and timeout facts. */
98
+ export interface Deadline {
99
+ signal: AbortSignal
100
+ /** True when the deadline timer fired. */
101
+ timedOut: boolean
102
+ /** True when the caller signal fired first. */
103
+ cancelled: boolean
104
+ /** Clear the timer and caller listener. */
105
+ cleanup(): void
106
+ }
107
+
108
+ /** Combine a caller abort signal with one hard operation timeout. */
109
+ export function createDeadline(signal: AbortSignal, timeoutMs: number): Deadline {
110
+ const controller = new AbortController()
111
+ const state = { timedOut: false, cancelled: false }
112
+ const onCallerAbort = (): void => {
113
+ if (controller.signal.aborted) return
114
+ state.cancelled = true
115
+ controller.abort()
116
+ }
117
+ if (signal.aborted) {
118
+ state.cancelled = true
119
+ controller.abort()
120
+ } else {
121
+ signal.addEventListener('abort', onCallerAbort, { once: true })
122
+ }
123
+ const timer = setTimeout(() => {
124
+ if (controller.signal.aborted) return
125
+ state.timedOut = true
126
+ controller.abort()
127
+ }, timeoutMs)
128
+ return {
129
+ signal: controller.signal,
130
+ get timedOut(): boolean { return state.timedOut },
131
+ get cancelled(): boolean { return state.cancelled },
132
+ cleanup(): void {
133
+ clearTimeout(timer)
134
+ signal.removeEventListener('abort', onCallerAbort)
135
+ },
136
+ }
137
+ }
138
+
139
+ /** FIFO bounded concurrency gate whose queued callers remain cancellable. */
140
+ export class Semaphore {
141
+ private active = 0
142
+ private readonly waiters: Array<{
143
+ resolve: () => void
144
+ reject: (error: unknown) => void
145
+ signal: AbortSignal
146
+ permits: number
147
+ onAbort: () => void
148
+ }> = []
149
+
150
+ constructor(private readonly limit: number) {}
151
+
152
+ /** Whether no active or queued caller still owns this gate. */
153
+ get idle(): boolean {
154
+ return this.active === 0 && this.waiters.length === 0
155
+ }
156
+
157
+ /** Acquire one slot, aborting while queued when `signal` fires. */
158
+ async acquire(signal: AbortSignal, permits = 1): Promise<void> {
159
+ if (signal.aborted) throw new VisionToolkitError('cancelled', 'vision-toolkit: cancelled before execution')
160
+ if (!Number.isInteger(permits) || permits < 1 || permits > this.limit) {
161
+ throw new VisionToolkitError('input', `concurrency permits must be between 1 and ${this.limit}`)
162
+ }
163
+ if (this.waiters.length === 0 && this.active + permits <= this.limit) {
164
+ this.active += permits
165
+ return
166
+ }
167
+ return new Promise<void>((resolveAcquire, reject) => {
168
+ const entry = {
169
+ resolve: resolveAcquire,
170
+ reject,
171
+ signal,
172
+ permits,
173
+ onAbort: () => {},
174
+ }
175
+ entry.onAbort = (): void => {
176
+ const index = this.waiters.indexOf(entry)
177
+ if (index >= 0) this.waiters.splice(index, 1)
178
+ reject(new VisionToolkitError('cancelled', 'vision-toolkit: cancelled while waiting for a concurrency slot'))
179
+ }
180
+ this.waiters.push(entry)
181
+ signal.addEventListener('abort', entry.onAbort, { once: true })
182
+ })
183
+ }
184
+
185
+ /** Release owned permits and wake FIFO waiters whose full weight now fits. */
186
+ release(permits = 1): void {
187
+ this.active = Math.max(0, this.active - permits)
188
+ while (this.waiters.length > 0) {
189
+ const next = this.waiters[0]
190
+ if (next === undefined || this.active + next.permits > this.limit) break
191
+ this.waiters.shift()
192
+ next.signal.removeEventListener('abort', next.onAbort)
193
+ this.active += next.permits
194
+ next.resolve()
195
+ }
196
+ }
197
+ }
198
+
199
+ /** Validated image metadata retained in structured results and diagnostics. */
200
+ export interface ImageInfo {
201
+ path: string
202
+ bytes: number
203
+ width: number
204
+ height: number
205
+ format: string
206
+ /** Original user-facing image path before any automatic compression. */
207
+ originalPath: string
208
+ }
209
+
210
+ /** Structured input for one glance call. */
211
+ export interface GlanceRequest {
212
+ images: string[]
213
+ query?: string
214
+ ocr?: boolean
215
+ region?: string
216
+ }
217
+
218
+ /** Structured glance result. */
219
+ export interface GlanceResult {
220
+ images: ImageInfo[]
221
+ mode: 'describe' | 'qa' | 'ocr'
222
+ answer: string
223
+ truncated: boolean
224
+ }
225
+
226
+ /** Structured input for ground/detect. */
227
+ export interface LocateRequest {
228
+ image: string
229
+ target: string
230
+ region?: string
231
+ }
232
+
233
+ /** One located element with an upstream or caller label. */
234
+ export interface LocateMatch {
235
+ label: string
236
+ box: { x1: number; y1: number; x2: number; y2: number }
237
+ }
238
+
239
+ /** Structured ground result. */
240
+ export interface GroundResult {
241
+ target: string
242
+ image: ImageInfo
243
+ imageWidth: number
244
+ imageHeight: number
245
+ matches: LocateMatch[]
246
+ preview?: ArtifactDescriptor
247
+ }
248
+
249
+ /** Structured detect result. */
250
+ export interface DetectResult {
251
+ category: string
252
+ image: ImageInfo
253
+ imageWidth: number
254
+ imageHeight: number
255
+ elements: Array<{ index: number; label: string; box: { x1: number; y1: number; x2: number; y2: number } }>
256
+ preview?: ArtifactDescriptor
257
+ }
258
+
259
+ /** Structured crop request. */
260
+ export interface CropRequest {
261
+ image: string
262
+ region: string
263
+ scale?: number
264
+ output?: string
265
+ }
266
+
267
+ /** Structured crop result. */
268
+ export interface CropResult {
269
+ imageWidth: number
270
+ imageHeight: number
271
+ region: { x1: number; y1: number; x2: number; y2: number }
272
+ outputPath: string
273
+ mimeType: 'image/png' | 'image/jpeg'
274
+ width: number
275
+ height: number
276
+ clamped: boolean
277
+ artifact: ArtifactDescriptor
278
+ note?: string
279
+ }
280
+
281
+ /** Structured trace request supported by the pinned upstream snapshot. */
282
+ export interface TraceRequest {
283
+ image: string
284
+ region?: string
285
+ scale?: number
286
+ color?: boolean
287
+ polygon?: boolean
288
+ output?: string
289
+ }
290
+
291
+ /** Structured trace result. */
292
+ export interface TraceResult {
293
+ imageWidth: number
294
+ imageHeight: number
295
+ outputPath: string
296
+ mimeType: 'image/svg+xml'
297
+ geometry: {
298
+ status: 'generated' | 'empty'
299
+ pathCount: number
300
+ tracedScale: number
301
+ bytes: number
302
+ }
303
+ artifact: ArtifactDescriptor
304
+ warning?: string
305
+ }
306
+
307
+ /** Structured input for local image comparison. */
308
+ export interface PixelDiffRequest {
309
+ original: string
310
+ rebuilt: string
311
+ grid?: number
312
+ top?: number
313
+ runName?: string
314
+ }
315
+
316
+ /** Structured local pixel comparison plus formally delivered files. */
317
+ export interface PixelDiffResult {
318
+ original: ImageInfo
319
+ rebuilt: ImageInfo
320
+ scaled: boolean
321
+ rebuiltOriginalSize?: { width: number; height: number }
322
+ overallDifferencePct: number
323
+ worstRegions: Array<{ index: number; differencePct: number; box: { x1: number; y1: number; x2: number; y2: number } }>
324
+ heatmap: ArtifactDescriptor
325
+ report: ArtifactDescriptor
326
+ }
327
+
328
+ /** Structured input for the pinned long-screenshot OCR pipeline. */
329
+ export interface LongScreenshotOcrRequest {
330
+ image: string
331
+ mode?: 'general' | 'chat'
332
+ output?: string
333
+ runName?: string
334
+ targetHeight?: number
335
+ minHeight?: number
336
+ maxHeight?: number
337
+ overlap?: number
338
+ prompt?: string
339
+ jobs?: number
340
+ chunkTimeoutSeconds?: number
341
+ splitOnly?: boolean
342
+ resume?: boolean
343
+ }
344
+
345
+ /** One long-OCR chunk and the files retained for audit or reuse. */
346
+ export interface LongScreenshotChunk {
347
+ index: number
348
+ coreTop: number
349
+ coreBottom: number
350
+ cropTop: number
351
+ cropBottom: number
352
+ image: ArtifactDescriptor
353
+ ocr?: ArtifactDescriptor
354
+ reused?: boolean
355
+ }
356
+
357
+ /** Long-screenshot split/OCR result with every durable deliverable. */
358
+ export interface LongScreenshotOcrResult {
359
+ source: ImageInfo
360
+ mode: 'general' | 'chat'
361
+ splitOnly: boolean
362
+ complete: boolean
363
+ chunkCount: number
364
+ runDirectory: string
365
+ output?: ArtifactDescriptor
366
+ manifest: ArtifactDescriptor
367
+ audit?: ArtifactDescriptor
368
+ chunks: LongScreenshotChunk[]
369
+ }
370
+
371
+ /** Structured input for transparent foreground extraction. */
372
+ export interface ExtractForegroundRequest {
373
+ image: string
374
+ region?: string
375
+ boxes?: string
376
+ mode?: 'color' | 'dark'
377
+ discRadius?: number
378
+ saturation?: number
379
+ darkThreshold?: number
380
+ excludeColor?: string
381
+ excludeTolerance?: number
382
+ padding?: number
383
+ keepWhites?: boolean
384
+ output?: string
385
+ }
386
+
387
+ /** Transparent foreground file plus the pinned script's component metrics. */
388
+ export interface ExtractForegroundResult {
389
+ source: ImageInfo
390
+ box: { x1: number; y1: number; x2: number; y2: number }
391
+ foregroundPixels: number
392
+ keptComponents: number
393
+ totalComponents: number
394
+ largestComponentPct: number
395
+ width: number
396
+ height: number
397
+ artifact: ArtifactDescriptor
398
+ autoSummary?: string
399
+ }
400
+
401
+ /** Structured input for palette extraction or candidate scoring. */
402
+ export interface DominantColorsRequest {
403
+ image: string
404
+ region?: string
405
+ candidates?: string[]
406
+ top?: number
407
+ quantize?: number
408
+ maxPixels?: number
409
+ mergeTolerance?: number
410
+ candidateTolerance?: number
411
+ }
412
+
413
+ /** Stable dominant-colour result enriched with source image facts. */
414
+ export interface DominantColorsResult {
415
+ image: ImageInfo
416
+ analysis: DominantColorsOutput
417
+ }
418
+
419
+ /** Structured input for rendering an authorized local HTML document. */
420
+ export interface HtmlScreenshotRequest {
421
+ source: string
422
+ width?: number
423
+ height?: number
424
+ scale?: number
425
+ waitMs?: number
426
+ fullPage?: boolean
427
+ output?: string
428
+ }
429
+
430
+ /** Browser-rendered PNG plus viewport and source facts. */
431
+ export interface HtmlScreenshotResult {
432
+ sourcePath: string
433
+ sourceBytes: number
434
+ viewport: { width: number; height: number; scale: number }
435
+ width: number
436
+ height: number
437
+ /** Full document height in CSS pixels; present only for full-page captures. */
438
+ pageHeight?: number
439
+ artifact: ArtifactDescriptor
440
+ }
441
+
442
+ /** Optional preview controls shared by ground and detect. */
443
+ export interface LocatePreviewRequest extends LocateRequest {
444
+ preview?: boolean
445
+ previewOutput?: string
446
+ }
447
+
448
+ /** One named health-check state. */
449
+ export interface HealthCheck {
450
+ status: 'ok' | 'warning' | 'error' | 'not_tested'
451
+ detail: string
452
+ }
453
+
454
+ /** Runtime, dependency, storage, credential, and optional service health. */
455
+ export interface VisionToolkitHealthResult {
456
+ pluginVersion: string
457
+ upstream: UpstreamVersionInfo
458
+ checks: {
459
+ python: HealthCheck
460
+ dependencies: HealthCheck
461
+ chrome: HealthCheck
462
+ credential: HealthCheck
463
+ artifactDirectory: HealthCheck
464
+ tempDirectory: HealthCheck
465
+ service: HealthCheck
466
+ model: HealthCheck
467
+ }
468
+ healthy: boolean
469
+ connectionTested: boolean
470
+ modelTested: boolean
471
+ }
472
+
473
+ /** Shared per-call execution options. */
474
+ export interface ToolCallOptions {
475
+ signal: AbortSignal
476
+ timeoutMs?: number
477
+ workspace: string
478
+ /** Session identity for the per-session concurrency cap. */
479
+ sessionId?: string
480
+ /** Live Session object whose lifetime bounds the one-entry glance cache. */
481
+ sessionScope?: object
482
+ }
483
+
484
+ /** One immutable vision-service snapshot used to generate cache-keyed evidence. */
485
+ export interface CapturedEvidenceRuntime {
486
+ readonly evidenceFingerprint: string
487
+ glance(request: GlanceRequest, options: ToolCallOptions): Promise<GlanceResult>
488
+ }
489
+
490
+ interface GlanceCacheEntry {
491
+ key: string
492
+ result: GlanceResult
493
+ }
494
+
495
+ interface OperationMetrics {
496
+ startedAt: number
497
+ queueMs: number
498
+ upstreamMs: number
499
+ imageBytes: number
500
+ imagePixels: number
501
+ imageCount: number
502
+ cacheHits: number
503
+ usedVisionService: boolean
504
+ }
505
+
506
+ interface OperationContext {
507
+ signal: AbortSignal
508
+ metrics: OperationMetrics
509
+ }
510
+
511
+ const REGION_PATTERN = /^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$/
512
+ const MAX_TIMEOUT_MS = 600_000
513
+ const FORMAT_BY_EXTENSION = new Map([
514
+ ['.png', 'png'],
515
+ ['.jpg', 'jpeg'],
516
+ ['.jpeg', 'jpeg'],
517
+ ['.gif', 'gif'],
518
+ ['.webp', 'webp'],
519
+ ])
520
+ const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
521
+
522
+ function integerInRange(value: number | undefined, fallback: number, minimum: number, maximum: number, name: string): number {
523
+ const resolved = value ?? fallback
524
+ if (!Number.isInteger(resolved) || resolved < minimum || resolved > maximum) {
525
+ throw new VisionToolkitError('input', `${name} must be an integer between ${minimum} and ${maximum}`)
526
+ }
527
+ return resolved
528
+ }
529
+
530
+ function finiteInRange(value: number | undefined, minimum: number, maximum: number, name: string): number | undefined {
531
+ if (value === undefined) return undefined
532
+ if (!Number.isFinite(value) || value < minimum || value > maximum) {
533
+ throw new VisionToolkitError('input', `${name} must be between ${minimum} and ${maximum}`)
534
+ }
535
+ return value
536
+ }
537
+
538
+ function assertBoxWithin(
539
+ box: { x1: number; y1: number; x2: number; y2: number },
540
+ width: number,
541
+ height: number,
542
+ source: string,
543
+ ): void {
544
+ if (
545
+ ![box.x1, box.y1, box.x2, box.y2].every(Number.isInteger)
546
+ || box.x1 < 0
547
+ || box.y1 < 0
548
+ || box.x2 <= box.x1
549
+ || box.y2 <= box.y1
550
+ || box.x2 > width
551
+ || box.y2 > height
552
+ ) {
553
+ throw new VisionToolkitError('output', `${source} returned an out-of-range box for ${width}x${height}`)
554
+ }
555
+ }
556
+
557
+ function safeGeneratedName(name: unknown, source: string): string {
558
+ if (typeof name !== 'string' || name.length === 0 || basename(name) !== name || name === '.' || name === '..') {
559
+ throw new VisionToolkitError('output', `${source} returned an unsafe generated filename`)
560
+ }
561
+ return name
562
+ }
563
+
564
+ interface ParsedLongOcrChunk {
565
+ index: number
566
+ image: string
567
+ imageSha256: string
568
+ coreTop: number
569
+ coreBottom: number
570
+ cropTop: number
571
+ cropBottom: number
572
+ ocr?: string
573
+ ocrReused?: boolean
574
+ }
575
+
576
+ interface ParsedLongOcrManifest {
577
+ mode: 'general' | 'chat'
578
+ complete: boolean
579
+ chunks: ParsedLongOcrChunk[]
580
+ raw: Record<string, unknown>
581
+ }
582
+
583
+ function objectRecord(value: unknown, source: string): Record<string, unknown> {
584
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
585
+ throw new VisionToolkitError('output', `${source} must be a JSON object`)
586
+ }
587
+ return value as Record<string, unknown>
588
+ }
589
+
590
+ function manifestInteger(record: Record<string, unknown>, key: string, source: string): number {
591
+ const value = record[key]
592
+ if (typeof value !== 'number' || !Number.isInteger(value)) {
593
+ throw new VisionToolkitError('output', `${source}.${key} must be an integer`)
594
+ }
595
+ return value
596
+ }
597
+
598
+ function parseLongOcrManifest(
599
+ text: string,
600
+ expected: { source: string; output: string; width: number; height: number; mode: 'general' | 'chat'; splitOnly: boolean },
601
+ ): ParsedLongOcrManifest {
602
+ let parsed: unknown
603
+ try {
604
+ parsed = JSON.parse(text)
605
+ } catch (error) {
606
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest is not valid JSON', { cause: error })
607
+ }
608
+ const manifest = objectRecord(parsed, 'long_screenshot_ocr manifest')
609
+ if (manifest.schema_version !== 1) throw new VisionToolkitError('output', 'long_screenshot_ocr: unsupported manifest schema')
610
+ if (manifest.input !== expected.source) throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest source does not match the requested image')
611
+ if (manifestInteger(manifest, 'image_width', 'long_screenshot_ocr manifest') !== expected.width
612
+ || manifestInteger(manifest, 'image_height', 'long_screenshot_ocr manifest') !== expected.height) {
613
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest dimensions do not match the source image')
614
+ }
615
+ if (manifest.mode !== expected.mode) throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest mode does not match the request')
616
+ const complete = manifest.complete
617
+ if (typeof complete !== 'boolean' || complete === expected.splitOnly) {
618
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest completion state is inconsistent')
619
+ }
620
+ if (!Array.isArray(manifest.chunks) || manifest.chunks.length === 0) {
621
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest contains no chunks')
622
+ }
623
+ if (manifest.output !== (expected.splitOnly ? null : expected.output)) {
624
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest output path is inconsistent')
625
+ }
626
+ const chunks = manifest.chunks.map((value, position): ParsedLongOcrChunk => {
627
+ const record = objectRecord(value, `long_screenshot_ocr manifest.chunks[${position}]`)
628
+ const index = manifestInteger(record, 'index', `long_screenshot_ocr manifest.chunks[${position}]`)
629
+ if (index !== position + 1) throw new VisionToolkitError('output', 'long_screenshot_ocr: chunk indexes are not contiguous')
630
+ const image = safeGeneratedName(record.image, 'long_screenshot_ocr')
631
+ const imageSha256 = record.image_sha256
632
+ if (typeof imageSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(imageSha256)) {
633
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: chunk image hash is invalid')
634
+ }
635
+ const coreTop = manifestInteger(record, 'core_top', `long_screenshot_ocr manifest.chunks[${position}]`)
636
+ const coreBottom = manifestInteger(record, 'core_bottom', `long_screenshot_ocr manifest.chunks[${position}]`)
637
+ const cropTop = manifestInteger(record, 'crop_top', `long_screenshot_ocr manifest.chunks[${position}]`)
638
+ const cropBottom = manifestInteger(record, 'crop_bottom', `long_screenshot_ocr manifest.chunks[${position}]`)
639
+ if (
640
+ coreTop < 0
641
+ || coreBottom <= coreTop
642
+ || cropTop < 0
643
+ || cropBottom <= cropTop
644
+ || cropTop > coreTop
645
+ || cropBottom < coreBottom
646
+ || cropBottom > expected.height
647
+ ) {
648
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest contains an invalid chunk range')
649
+ }
650
+ const ocr = record.ocr === undefined ? undefined : safeGeneratedName(record.ocr, 'long_screenshot_ocr')
651
+ const ocrReused = record.ocr_reused
652
+ if (ocrReused !== undefined && typeof ocrReused !== 'boolean') {
653
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: ocr_reused must be boolean')
654
+ }
655
+ if (complete && ocr === undefined) throw new VisionToolkitError('output', 'long_screenshot_ocr: complete manifest is missing an OCR sidecar')
656
+ return {
657
+ index,
658
+ image,
659
+ imageSha256,
660
+ coreTop,
661
+ coreBottom,
662
+ cropTop,
663
+ cropBottom,
664
+ ...(ocr === undefined ? {} : { ocr }),
665
+ ...(ocrReused === undefined ? {} : { ocrReused }),
666
+ }
667
+ })
668
+ return { mode: expected.mode, complete, chunks, raw: manifest }
669
+ }
670
+
671
+ /** Parse a non-empty four-integer pixel box. */
672
+ export function parseRegion(region: string): { x1: number; y1: number; x2: number; y2: number } {
673
+ const match = REGION_PATTERN.exec(region)
674
+ if (match === null) {
675
+ throw new VisionToolkitError('input', 'region must be four integers: X1,Y1,X2,Y2 (pixels)')
676
+ }
677
+ const box = {
678
+ x1: Number(match[1]),
679
+ y1: Number(match[2]),
680
+ x2: Number(match[3]),
681
+ y2: Number(match[4]),
682
+ }
683
+ if (box.x2 <= box.x1 || box.y2 <= box.y1) {
684
+ throw new VisionToolkitError('input', 'region must have x2 > x1 and y2 > y1')
685
+ }
686
+ return box
687
+ }
688
+
689
+ /** Runtime facade used by every native tool. */
690
+ export class VisionToolkitRuntime {
691
+ private readonly semaphores = new Map<string, Semaphore>()
692
+ private readonly glanceCache = new WeakMap<object, GlanceCacheEntry>()
693
+ private readonly adapter: UpstreamAdapter
694
+
695
+ constructor(
696
+ private readonly ctx: Context,
697
+ private readonly config: ResolvedVisionToolkitConfig,
698
+ adapter?: UpstreamAdapter,
699
+ ) {
700
+ this.adapter = adapter ?? new UpstreamAdapter(ctx, config)
701
+ }
702
+
703
+ /** Pinned and prepared upstream identity. */
704
+ get upstreamVersion(): UpstreamVersionInfo {
705
+ return this.adapter.versionInfo
706
+ }
707
+
708
+ /** Stable identity for persisted image descriptions produced by this runtime. */
709
+ get evidenceFingerprint(): string {
710
+ return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
711
+ }
712
+
713
+ /** Capture the credential and provider identity used by one evidence conversion. */
714
+ async captureEvidenceRuntime(): Promise<CapturedEvidenceRuntime> {
715
+ const env = await this.resolveVisionEnv()
716
+ const evidenceFingerprint = evidenceRuntimeFingerprint(
717
+ this.config,
718
+ createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
719
+ env.VISION_SSL_VERIFY,
720
+ )
721
+ return Object.freeze({
722
+ evidenceFingerprint,
723
+ glance: (request: GlanceRequest, options: ToolCallOptions) => this.glanceWithEnv(request, options, env),
724
+ })
725
+ }
726
+
727
+ private timeout(options: ToolCallOptions): number {
728
+ const value = options.timeoutMs ?? this.config.timeoutMs
729
+ if (!Number.isInteger(value) || value < 1000 || value > MAX_TIMEOUT_MS) {
730
+ throw new VisionToolkitError('input', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`)
731
+ }
732
+ return value
733
+ }
734
+
735
+ private operationError(
736
+ tool: string,
737
+ error: unknown,
738
+ deadline: Deadline,
739
+ phase: 'queue' | 'execution' = 'execution',
740
+ ): VisionToolkitError {
741
+ if (deadline.cancelled) {
742
+ return new VisionToolkitError(
743
+ 'cancelled',
744
+ phase === 'queue' ? `${tool}: cancelled while waiting for a concurrency slot` : `${tool}: cancelled`,
745
+ )
746
+ }
747
+ if (deadline.timedOut) {
748
+ return new VisionToolkitError(
749
+ 'timeout',
750
+ phase === 'queue' ? `${tool}: timed out while waiting for a concurrency slot` : `${tool}: timed out`,
751
+ )
752
+ }
753
+ if (error instanceof VisionToolkitError) return error
754
+ return new VisionToolkitError('runtime', `${tool}: execution failed`, { cause: error })
755
+ }
756
+
757
+ private semaphore(options: ToolCallOptions): { key: string; value: Semaphore } {
758
+ const key = options.sessionId ?? `workspace:${options.workspace}`
759
+ const value = this.semaphores.get(key) ?? new Semaphore(this.config.concurrency)
760
+ this.semaphores.set(key, value)
761
+ return { key, value }
762
+ }
763
+
764
+ private async runOperation<T>(
765
+ tool: string,
766
+ options: ToolCallOptions,
767
+ action: (operation: OperationContext) => Promise<T>,
768
+ permits = 1,
769
+ ): Promise<T> {
770
+ const timeoutMs = this.timeout(options)
771
+ const semaphore = this.semaphore(options)
772
+ const metrics: OperationMetrics = {
773
+ startedAt: Date.now(),
774
+ queueMs: 0,
775
+ upstreamMs: 0,
776
+ imageBytes: 0,
777
+ imagePixels: 0,
778
+ imageCount: 0,
779
+ cacheHits: 0,
780
+ usedVisionService: false,
781
+ }
782
+ let acquired = false
783
+ const queueDeadline = createDeadline(options.signal, timeoutMs)
784
+ try {
785
+ await semaphore.value.acquire(queueDeadline.signal, permits)
786
+ acquired = true
787
+ if (queueDeadline.signal.aborted) throw this.operationError(tool, undefined, queueDeadline, 'queue')
788
+ metrics.queueMs = Date.now() - metrics.startedAt
789
+ } catch (error) {
790
+ metrics.queueMs = Date.now() - metrics.startedAt
791
+ const classified = this.operationError(tool, error, queueDeadline, 'queue')
792
+ if (acquired) {
793
+ semaphore.value.release(permits)
794
+ acquired = false
795
+ }
796
+ this.ctx.logger.warn(
797
+ 'dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d',
798
+ tool,
799
+ classified.code,
800
+ Date.now() - metrics.startedAt,
801
+ metrics.queueMs,
802
+ metrics.upstreamMs,
803
+ metrics.imageCount,
804
+ metrics.imageBytes,
805
+ metrics.imagePixels,
806
+ metrics.cacheHits,
807
+ )
808
+ throw classified
809
+ } finally {
810
+ queueDeadline.cleanup()
811
+ if (!acquired && semaphore.value.idle) this.semaphores.delete(semaphore.key)
812
+ }
813
+
814
+ const executionDeadline = createDeadline(options.signal, timeoutMs)
815
+ try {
816
+ if (executionDeadline.signal.aborted) throw this.operationError(tool, undefined, executionDeadline)
817
+ const value = await action({ signal: executionDeadline.signal, metrics })
818
+ if (executionDeadline.signal.aborted) throw this.operationError(tool, undefined, executionDeadline)
819
+ this.ctx.logger.info(
820
+ 'dsh-vision-toolkit tool=%s outcome=ok totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d model=%s',
821
+ tool,
822
+ Date.now() - metrics.startedAt,
823
+ metrics.queueMs,
824
+ metrics.upstreamMs,
825
+ metrics.imageCount,
826
+ metrics.imageBytes,
827
+ metrics.imagePixels,
828
+ metrics.cacheHits,
829
+ metrics.usedVisionService ? this.config.provider.model : 'local',
830
+ )
831
+ return value
832
+ } catch (error) {
833
+ const classified = this.operationError(tool, error, executionDeadline)
834
+ this.ctx.logger.warn(
835
+ 'dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d',
836
+ tool,
837
+ classified.code,
838
+ Date.now() - metrics.startedAt,
839
+ metrics.queueMs,
840
+ metrics.upstreamMs,
841
+ metrics.imageCount,
842
+ metrics.imageBytes,
843
+ metrics.imagePixels,
844
+ metrics.cacheHits,
845
+ )
846
+ throw classified
847
+ } finally {
848
+ if (acquired) semaphore.value.release(permits)
849
+ executionDeadline.cleanup()
850
+ if (semaphore.value.idle) this.semaphores.delete(semaphore.key)
851
+ }
852
+ }
853
+
854
+ /** Resolve the configured credential at the remote-operation boundary. */
855
+ async resolveVisionEnv(): Promise<UpstreamEnvironment> {
856
+ const resolved: ResolvedCredential | undefined = isBuiltInFreeVisionProvider(this.config.provider)
857
+ ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
858
+ : await this.ctx.credentials.resolve(this.config.provider.credential)
859
+ if (resolved === undefined) {
860
+ throw new VisionToolkitError(
861
+ 'config',
862
+ `credential ${this.config.provider.credential} is not configured; set it through DSH credentials`,
863
+ )
864
+ }
865
+ return this.visionEnv(resolved)
866
+ }
867
+
868
+ private visionEnv(resolved: ResolvedCredential): UpstreamEnvironment {
869
+ const sslVerify = process.env.VISION_SSL_VERIFY?.trim()
870
+ return {
871
+ VISION_API_KEY: resolved.value,
872
+ VISION_BASE_URL: this.config.provider.baseUrl,
873
+ VISION_MODEL: this.config.provider.model,
874
+ VISION_API_PROTOCOL: this.config.provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
875
+ VISION_ANTHROPIC_THINKING: this.config.provider.anthropicThinking,
876
+ ...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
877
+ VISION_USER_AGENT: this.config.provider.userAgent,
878
+ LANG: this.config.language,
879
+ }
880
+ }
881
+
882
+ private pathPolicy(workspace: string): Promise<PathPolicy> {
883
+ return createPathPolicy(workspace, this.config.allowedDirs)
884
+ }
885
+
886
+ private async compressedImageRoot(policy: PathPolicy): Promise<string> {
887
+ const root = join(policy.workspace, '.dsh-vision-toolkit', 'tmp', 'compressed-images')
888
+ let current = policy.workspace
889
+ for (const segment of ['.dsh-vision-toolkit', 'tmp', 'compressed-images']) {
890
+ current = join(current, segment)
891
+ try {
892
+ await mkdir(current, { mode: 0o700 })
893
+ } catch (error) {
894
+ if (!(error instanceof Error && 'code' in error && error.code === 'EEXIST')) throw error
895
+ }
896
+ const info = await lstat(current)
897
+ if (info.isSymbolicLink() || !info.isDirectory()) {
898
+ throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`)
899
+ }
900
+ if (!isWithin(policy.workspace, current)) {
901
+ throw new VisionToolkitError('path', `compressed-image cache path escaped the workspace: ${current}`)
902
+ }
903
+ }
904
+ const canonical = await realpath(root)
905
+ if (!isWithin(policy.workspace, canonical)) {
906
+ throw new VisionToolkitError('path', 'compressed-image cache resolved outside the workspace')
907
+ }
908
+ return canonical
909
+ }
910
+
911
+ private async readCacheCandidate(
912
+ root: string,
913
+ name: string,
914
+ expectedOutDigestPrefix: string,
915
+ maxBytes: number,
916
+ maxPixels: number,
917
+ operation: OperationContext,
918
+ ): Promise<{ path: string; bytes: number; width: number; height: number; format: string } | undefined> {
919
+ const candidate = join(root, name)
920
+ let info
921
+ try {
922
+ info = await lstat(candidate)
923
+ } catch {
924
+ return undefined
925
+ }
926
+ if (!info.isFile() || info.size < 1 || info.size > maxBytes) return undefined
927
+ let real: string
928
+ try {
929
+ real = await realpath(candidate)
930
+ } catch {
931
+ return undefined
932
+ }
933
+ if (!isWithin(root, real)) return undefined
934
+ let bytes: Buffer
935
+ try {
936
+ bytes = await readFile(real, { signal: operation.signal })
937
+ } catch {
938
+ return undefined
939
+ }
940
+ const digest = createHash('sha256').update(bytes).digest('hex')
941
+ if (bytes.length !== info.size || !digest.startsWith(expectedOutDigestPrefix)) return undefined
942
+ let probed: { width: number; height: number; format: string } | undefined
943
+ try {
944
+ probed = await this.adapter.probeImageSize(real, { signal: operation.signal })
945
+ } catch {
946
+ probed = undefined
947
+ }
948
+ const extension = extname(real).toLowerCase()
949
+ if (
950
+ probed === undefined
951
+ || FORMAT_BY_EXTENSION.get(extension) !== probed.format
952
+ || probed.width * probed.height > maxPixels
953
+ ) {
954
+ return undefined
955
+ }
956
+ return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format }
957
+ }
958
+
959
+ private cacheEntryOutDigest(entry: string, prefix: string): string | undefined {
960
+ const tail = entry.slice(prefix.length + 1)
961
+ return /^[0-9a-f]{16}-/u.test(tail) ? tail.slice(0, 16) : undefined
962
+ }
963
+
964
+ private async pruneCompressedCache(root: string): Promise<void> {
965
+ let entries: string[]
966
+ try {
967
+ entries = await readdir(root)
968
+ } catch {
969
+ return
970
+ }
971
+ const stalePartials: string[] = []
972
+ const candidates: Array<{ name: string; size: number; mtime: number; removable: boolean }> = []
973
+ for (const name of entries) {
974
+ if (name.startsWith('.')) {
975
+ if (name.endsWith('.partial')) {
976
+ const info = await lstat(join(root, name)).catch(() => undefined)
977
+ if (info !== undefined && Date.now() - info.mtimeMs > COMPRESSED_IMAGE_CACHE_STALE_PARTIAL_MS) {
978
+ stalePartials.push(name)
979
+ }
980
+ }
981
+ continue
982
+ }
983
+ let info
984
+ try {
985
+ info = await lstat(join(root, name))
986
+ } catch {
987
+ continue
988
+ }
989
+ candidates.push({
990
+ name,
991
+ size: info.isFile() ? info.size : 0,
992
+ mtime: info.mtimeMs,
993
+ removable: !info.isFile() || !name.startsWith(`${COMPRESSED_IMAGE_CACHE_VERSION}-`),
994
+ })
995
+ }
996
+ candidates.sort((a, b) => a.mtime - b.mtime)
997
+ let totalBytes = 0
998
+ let kept = 0
999
+ const remove: string[] = []
1000
+ for (const candidate of candidates) {
1001
+ if (
1002
+ candidate.removable
1003
+ || totalBytes + candidate.size > COMPRESSED_IMAGE_CACHE_MAX_BYTES
1004
+ || kept >= COMPRESSED_IMAGE_CACHE_MAX_ENTRIES
1005
+ ) {
1006
+ remove.push(candidate.name)
1007
+ } else {
1008
+ totalBytes += candidate.size
1009
+ kept += 1
1010
+ }
1011
+ }
1012
+ await Promise.all([...stalePartials, ...remove].map(name => rm(join(root, name), { force: true }).catch(() => {})))
1013
+ }
1014
+
1015
+ private async autoCompressImage(
1016
+ image: { path: string; bytes: number },
1017
+ policy: PathPolicy,
1018
+ operation: OperationContext,
1019
+ ): Promise<ImageInfo> {
1020
+ let bytes: Buffer
1021
+ try {
1022
+ bytes = await readFile(image.path, { signal: operation.signal })
1023
+ } catch (error) {
1024
+ throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`, { cause: error })
1025
+ }
1026
+ if (bytes.length !== image.bytes) {
1027
+ throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`)
1028
+ }
1029
+ const digest = createHash('sha256').update(bytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
1030
+ const root = await this.compressedImageRoot(policy)
1031
+ await this.pruneCompressedCache(root)
1032
+ const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${this.config.maxImageBytes}-p${this.config.maxImagePixels}`
1033
+ for (const entry of await readdir(root)) {
1034
+ if (!entry.startsWith(`${prefix}-`) || entry.startsWith('.')) continue
1035
+ const outDigestPrefix = this.cacheEntryOutDigest(entry, prefix)
1036
+ if (outDigestPrefix === undefined) {
1037
+ await rm(join(root, entry), { force: true }).catch(() => {})
1038
+ continue
1039
+ }
1040
+ const cached = await this.readCacheCandidate(
1041
+ root,
1042
+ entry,
1043
+ outDigestPrefix,
1044
+ this.config.maxImageBytes,
1045
+ this.config.maxImagePixels,
1046
+ operation,
1047
+ )
1048
+ if (cached !== undefined) {
1049
+ return { ...cached, originalPath: image.path }
1050
+ }
1051
+ await rm(join(root, entry), { force: true }).catch(() => {})
1052
+ }
1053
+ const staged = join(root, `.${prefix}-${randomUUID()}.partial`)
1054
+ let compressed: CompressedImageInfo
1055
+ try {
1056
+ compressed = await this.adapter.compressImage(
1057
+ image.path,
1058
+ staged,
1059
+ this.config.maxImageBytes,
1060
+ this.config.maxImagePixels,
1061
+ { signal: operation.signal },
1062
+ )
1063
+ } catch (error) {
1064
+ await rm(staged, { force: true }).catch(() => {})
1065
+ throw error
1066
+ }
1067
+ const extension = compressed.format === 'jpeg' ? 'jpg' : compressed.format
1068
+ const stagedBytes = await readFile(staged, { signal: operation.signal })
1069
+ const outDigest = createHash('sha256').update(stagedBytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
1070
+ const finalName = `${prefix}-${outDigest}-${compressed.width}x${compressed.height}.${extension}`
1071
+ const finalPath = join(root, finalName)
1072
+ const existing = await this.readCacheCandidate(
1073
+ root,
1074
+ finalName,
1075
+ outDigest,
1076
+ this.config.maxImageBytes,
1077
+ this.config.maxImagePixels,
1078
+ operation,
1079
+ )
1080
+ if (existing !== undefined) {
1081
+ await rm(staged, { force: true }).catch(() => {})
1082
+ return { ...existing, originalPath: image.path }
1083
+ }
1084
+ await rm(finalPath, { force: true }).catch(() => {})
1085
+ try {
1086
+ await rename(staged, finalPath)
1087
+ } catch (error) {
1088
+ await rm(staged, { force: true }).catch(() => {})
1089
+ throw new VisionToolkitError('path', `cannot commit compressed image cache entry: ${finalPath}`, { cause: error })
1090
+ }
1091
+ await this.pruneCompressedCache(root)
1092
+ return {
1093
+ path: finalPath,
1094
+ bytes: compressed.bytes,
1095
+ width: compressed.width,
1096
+ height: compressed.height,
1097
+ format: compressed.format,
1098
+ originalPath: image.path,
1099
+ }
1100
+ }
1101
+
1102
+ private async validateImage(raw: string, policy: PathPolicy, operation: OperationContext): Promise<ImageInfo> {
1103
+ const image = await resolveInputFile(raw, policy)
1104
+ const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal })
1105
+ const pixels = decoded.width * decoded.height
1106
+ if (!Number.isSafeInteger(pixels) || pixels < 1) {
1107
+ throw new VisionToolkitError('input', `image dimensions are invalid: ${decoded.width}x${decoded.height}`)
1108
+ }
1109
+ const extension = extname(image.path).toLowerCase()
1110
+ const expected = FORMAT_BY_EXTENSION.get(extension)
1111
+ if (expected !== decoded.format) {
1112
+ throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`)
1113
+ }
1114
+ if (image.bytes <= this.config.maxImageBytes && pixels <= this.config.maxImagePixels) {
1115
+ return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, originalPath: image.path }
1116
+ }
1117
+ return this.autoCompressImage(image, policy, operation)
1118
+ }
1119
+
1120
+ private accountImage(image: ImageInfo, operation: OperationContext): void {
1121
+ operation.metrics.imageCount += 1
1122
+ operation.metrics.imageBytes += image.bytes
1123
+ operation.metrics.imagePixels += image.width * image.height
1124
+ }
1125
+
1126
+ private async glanceCacheKey(
1127
+ request: GlanceRequest,
1128
+ images: readonly ImageInfo[],
1129
+ env: UpstreamEnvironment,
1130
+ signal: AbortSignal,
1131
+ ): Promise<string> {
1132
+ const imageFingerprints = await Promise.all(images.map(async (image) => {
1133
+ let bytes: Buffer
1134
+ try {
1135
+ bytes = await readFile(image.path, { signal })
1136
+ } catch (error) {
1137
+ throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`, { cause: error })
1138
+ }
1139
+ if (bytes.length !== image.bytes) {
1140
+ throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`)
1141
+ }
1142
+ return {
1143
+ path: image.path,
1144
+ sha256: createHash('sha256').update(bytes).digest('hex'),
1145
+ }
1146
+ }))
1147
+ return JSON.stringify({
1148
+ images: imageFingerprints,
1149
+ query: request.query ?? null,
1150
+ ocr: request.ocr === true,
1151
+ region: request.region ?? null,
1152
+ provider: {
1153
+ baseUrl: env.VISION_BASE_URL,
1154
+ model: env.VISION_MODEL,
1155
+ protocol: env.VISION_API_PROTOCOL,
1156
+ anthropicThinking: env.VISION_ANTHROPIC_THINKING,
1157
+ sslVerify: env.VISION_SSL_VERIFY ?? null,
1158
+ userAgent: env.VISION_USER_AGENT,
1159
+ language: env.LANG,
1160
+ credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
1161
+ },
1162
+ })
1163
+ }
1164
+
1165
+ private async runUpstream(
1166
+ tool: UpstreamTool,
1167
+ args: readonly string[],
1168
+ operation: OperationContext,
1169
+ env?: UpstreamEnvironment,
1170
+ ): Promise<UpstreamRunResult> {
1171
+ const started = Date.now()
1172
+ if (env !== undefined) operation.metrics.usedVisionService = true
1173
+ const result = await this.adapter.run(tool, args, {
1174
+ signal: operation.signal,
1175
+ ...(env === undefined ? {} : { env }),
1176
+ })
1177
+ operation.metrics.upstreamMs += Date.now() - started
1178
+ if (result.outcome.exitCode !== 0) {
1179
+ throw this.adapter.classifyFailure(tool, result, {
1180
+ timedOut: false,
1181
+ cancelled: operation.signal.aborted,
1182
+ ...(env === undefined ? {} : { secrets: [env.VISION_API_KEY] }),
1183
+ })
1184
+ }
1185
+ if (result.stdoutTruncated || result.stderrTruncated) {
1186
+ throw new VisionToolkitError('output', `${tool}: upstream output exceeded the capture limit`)
1187
+ }
1188
+ return result
1189
+ }
1190
+
1191
+ private async probeGeneratedImage(
1192
+ path: string,
1193
+ operation: OperationContext,
1194
+ source: string,
1195
+ ): Promise<{ width: number; height: number; format: string; mode: string }> {
1196
+ try {
1197
+ return await this.adapter.probeImageSize(path, { signal: operation.signal })
1198
+ } catch (error) {
1199
+ if (operation.signal.aborted) throw error
1200
+ throw new VisionToolkitError('output', `${source}: generated image is missing, corrupt, or unsupported`, { cause: error })
1201
+ }
1202
+ }
1203
+
1204
+ private async annotateLocations(
1205
+ tool: 'vision_ground' | 'vision_detect',
1206
+ image: ImageInfo,
1207
+ elements: readonly LocatedElement[],
1208
+ output: string | undefined,
1209
+ policy: PathPolicy,
1210
+ operation: OperationContext,
1211
+ ): Promise<ArtifactDescriptor> {
1212
+ const extension = extname(image.originalPath).toLowerCase()
1213
+ const stem = basename(image.originalPath, extension)
1214
+ const suffix = tool === 'vision_ground' ? 'ground' : 'detect'
1215
+ const finalPath = resolveOutputFile(output, policy, `${stem}.${suffix}.preview.png`, ['.png'])
1216
+ assertDistinctOutput(image.path, finalPath)
1217
+ assertDistinctOutput(image.originalPath, finalPath)
1218
+ const staged = createStagedOutput(policy, '.png')
1219
+ try {
1220
+ const started = Date.now()
1221
+ await this.adapter.renderAnnotatedPreview(image.path, staged, elements, { signal: operation.signal })
1222
+ operation.metrics.upstreamMs += Date.now() - started
1223
+ const preview = await this.probeGeneratedImage(staged, operation, tool)
1224
+ if (preview.format !== 'png' || preview.width !== image.width || preview.height !== image.height) {
1225
+ throw new VisionToolkitError('output', `${tool}: annotation preview dimensions or format do not match the source image`)
1226
+ }
1227
+ await commitStagedOutput(staged, finalPath, policy)
1228
+ return describeArtifact(finalPath, policy, {
1229
+ mimeType: 'image/png',
1230
+ kind: 'image',
1231
+ description: tool === 'vision_ground' ? 'Grounding bounding-box preview' : 'Detected-element bounding-box preview',
1232
+ sourceTool: tool,
1233
+ previewIntent: 'image',
1234
+ })
1235
+ } finally {
1236
+ await rm(staged, { force: true }).catch(() => {})
1237
+ }
1238
+ }
1239
+
1240
+ /** glance: describe, targeted QA, OCR, or multi-image comparison. */
1241
+ async glance(request: GlanceRequest, options: ToolCallOptions): Promise<GlanceResult> {
1242
+ return this.glanceWithEnv(request, options)
1243
+ }
1244
+
1245
+ private async glanceWithEnv(
1246
+ request: GlanceRequest,
1247
+ options: ToolCallOptions,
1248
+ capturedEnv?: UpstreamEnvironment,
1249
+ ): Promise<GlanceResult> {
1250
+ return this.runOperation('vision_glance', options, async (operation) => {
1251
+ if (request.images.length === 0) throw new VisionToolkitError('input', 'glance requires at least one image')
1252
+ if (request.query !== undefined && request.ocr === true) {
1253
+ throw new VisionToolkitError('input', 'glance: query and ocr are mutually exclusive')
1254
+ }
1255
+ if (request.region !== undefined && request.images.length > 1) {
1256
+ throw new VisionToolkitError('input', 'glance: region works with exactly one image')
1257
+ }
1258
+ if (request.region !== undefined) parseRegion(request.region)
1259
+ const policy = await this.pathPolicy(options.workspace)
1260
+ const images: ImageInfo[] = []
1261
+ const seen = new Set<string>()
1262
+ for (const raw of request.images) {
1263
+ const image = await this.validateImage(raw, policy, operation)
1264
+ if (seen.has(image.path)) {
1265
+ operation.metrics.cacheHits += 1
1266
+ continue
1267
+ }
1268
+ seen.add(image.path)
1269
+ this.accountImage(image, operation)
1270
+ images.push(image)
1271
+ }
1272
+ const env = capturedEnv ?? await this.resolveVisionEnv()
1273
+ const cacheKey = options.sessionScope === undefined
1274
+ ? undefined
1275
+ : await this.glanceCacheKey(request, images, env, operation.signal)
1276
+ if (options.sessionScope !== undefined && cacheKey !== undefined) {
1277
+ const cached = this.glanceCache.get(options.sessionScope)
1278
+ if (cached?.key === cacheKey) {
1279
+ operation.metrics.cacheHits += 1
1280
+ return cached.result
1281
+ }
1282
+ }
1283
+ const result = await this.runUpstream('glance', [
1284
+ ...images.map(image => image.path),
1285
+ ...(request.region !== undefined ? ['--region', request.region] : []),
1286
+ ...(request.ocr === true ? ['--ocr'] : []),
1287
+ ...(request.query !== undefined ? ['-q', request.query] : []),
1288
+ ], operation, env)
1289
+ const answer = result.stdout.trim()
1290
+ if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
1291
+ const value: GlanceResult = {
1292
+ images,
1293
+ mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
1294
+ answer,
1295
+ truncated: false,
1296
+ }
1297
+ if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
1298
+ this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
1299
+ }
1300
+ return value
1301
+ })
1302
+ }
1303
+
1304
+ private validateLocations(elements: LocatedElement[], width: number, height: number): void {
1305
+ for (const element of elements) {
1306
+ const { x1, y1, x2, y2 } = element.box
1307
+ if (
1308
+ ![x1, y1, x2, y2].every(Number.isInteger)
1309
+ || x1 < 0
1310
+ || y1 < 0
1311
+ || x2 <= x1
1312
+ || y2 <= y1
1313
+ || x2 > width
1314
+ || y2 > height
1315
+ ) {
1316
+ throw new VisionToolkitError('output', `upstream returned an out-of-range box for ${width}x${height}`)
1317
+ }
1318
+ }
1319
+ }
1320
+
1321
+ private async locate(
1322
+ request: LocateRequest,
1323
+ options: ToolCallOptions,
1324
+ operation: OperationContext,
1325
+ tool: 'ground' | 'detect',
1326
+ ): Promise<{ image: ImageInfo; elements: LocatedElement[] }> {
1327
+ if (request.target.trim().length === 0) throw new VisionToolkitError('input', 'target must not be empty')
1328
+ if (request.region !== undefined) parseRegion(request.region)
1329
+ const policy = await this.pathPolicy(options.workspace)
1330
+ const image = await this.validateImage(request.image, policy, operation)
1331
+ this.accountImage(image, operation)
1332
+ const env = await this.resolveVisionEnv()
1333
+ const result = await this.runUpstream(tool, [
1334
+ image.path,
1335
+ request.target,
1336
+ ...(request.region !== undefined ? ['--region', request.region] : []),
1337
+ ], operation, env)
1338
+ const elements = parseLocationOutput(result.stdout)
1339
+ this.validateLocations(elements, image.width, image.height)
1340
+ return { image, elements }
1341
+ }
1342
+
1343
+ /** ground: locate one named target and return pixel boxes. */
1344
+ async ground(request: LocatePreviewRequest, options: ToolCallOptions): Promise<GroundResult> {
1345
+ return this.runOperation('vision_ground', options, async (operation) => {
1346
+ const { image, elements } = await this.locate(request, options, operation, 'ground')
1347
+ const labeled = elements.map(element => ({ label: element.label ?? request.target, box: element.box }))
1348
+ const preview = request.preview === true
1349
+ ? await this.annotateLocations('vision_ground', image, labeled, request.previewOutput, await this.pathPolicy(options.workspace), operation)
1350
+ : undefined
1351
+ return {
1352
+ target: request.target,
1353
+ image,
1354
+ imageWidth: image.width,
1355
+ imageHeight: image.height,
1356
+ matches: labeled,
1357
+ ...(preview === undefined ? {} : { preview }),
1358
+ }
1359
+ })
1360
+ }
1361
+
1362
+ /** detect: inventory every instance of a kind. */
1363
+ async detect(request: LocatePreviewRequest, options: ToolCallOptions): Promise<DetectResult> {
1364
+ return this.runOperation('vision_detect', options, async (operation) => {
1365
+ const { image, elements } = await this.locate(request, options, operation, 'detect')
1366
+ const labeled = elements.map(element => ({ label: element.label ?? request.target, box: element.box }))
1367
+ const preview = request.preview === true
1368
+ ? await this.annotateLocations('vision_detect', image, labeled, request.previewOutput, await this.pathPolicy(options.workspace), operation)
1369
+ : undefined
1370
+ return {
1371
+ category: request.target,
1372
+ image,
1373
+ imageWidth: image.width,
1374
+ imageHeight: image.height,
1375
+ elements: labeled.map((element, index) => ({
1376
+ index: index + 1,
1377
+ label: element.label,
1378
+ box: element.box,
1379
+ })),
1380
+ ...(preview === undefined ? {} : { preview }),
1381
+ }
1382
+ })
1383
+ }
1384
+
1385
+ /** crop: cut a pixel box into its own image file without requiring a credential. */
1386
+ async crop(request: CropRequest, options: ToolCallOptions): Promise<CropResult> {
1387
+ return this.runOperation('vision_crop', options, async (operation) => {
1388
+ const region = parseRegion(request.region)
1389
+ if (request.scale !== undefined && (!Number.isInteger(request.scale) || request.scale < 1 || request.scale > 8)) {
1390
+ throw new VisionToolkitError('input', 'crop: scale must be an integer between 1 and 8')
1391
+ }
1392
+ const policy = await this.pathPolicy(options.workspace)
1393
+ const image = await this.validateImage(request.image, policy, operation)
1394
+ this.accountImage(image, operation)
1395
+ const sourceExtension = extname(image.originalPath).toLowerCase()
1396
+ const stem = basename(image.originalPath, sourceExtension)
1397
+ const finalPath = resolveOutputFile(
1398
+ request.output,
1399
+ policy,
1400
+ request.scale !== undefined && request.scale > 1 ? `${stem}.crop@${request.scale}x.png` : `${stem}.crop.png`,
1401
+ ['.png', '.jpg', '.jpeg'],
1402
+ )
1403
+ assertDistinctOutput(image.path, finalPath)
1404
+ assertDistinctOutput(image.originalPath, finalPath)
1405
+ const outputExtension = extname(finalPath).toLowerCase()
1406
+ const staged = createStagedOutput(policy, outputExtension)
1407
+ try {
1408
+ const result = await this.runUpstream('crop', [
1409
+ image.path,
1410
+ '--region',
1411
+ request.region,
1412
+ '-o',
1413
+ staged,
1414
+ ...(request.scale !== undefined ? ['--scale', String(request.scale)] : []),
1415
+ ], operation)
1416
+ const parsed = parseCropOutput(result.stdout, result.stderr)
1417
+ const generated = await this.probeGeneratedImage(staged, operation, 'crop')
1418
+ const expectedFormat = outputExtension === '.png' ? 'png' : 'jpeg'
1419
+ if (
1420
+ generated.format !== expectedFormat
1421
+ || generated.width !== parsed.width
1422
+ || generated.height !== parsed.height
1423
+ ) {
1424
+ throw new VisionToolkitError('output', 'crop: generated image does not match the upstream summary')
1425
+ }
1426
+ await commitStagedOutput(staged, finalPath, policy)
1427
+ const mimeType = outputExtension === '.png' ? 'image/png' : 'image/jpeg'
1428
+ const artifact = await describeArtifact(finalPath, policy, {
1429
+ mimeType,
1430
+ kind: 'image',
1431
+ description: 'Cropped image region',
1432
+ sourceTool: 'vision_crop',
1433
+ previewIntent: 'image',
1434
+ })
1435
+ return {
1436
+ imageWidth: image.width,
1437
+ imageHeight: image.height,
1438
+ region,
1439
+ outputPath: finalPath,
1440
+ mimeType,
1441
+ width: parsed.width,
1442
+ height: parsed.height,
1443
+ clamped: parsed.clamped,
1444
+ artifact,
1445
+ ...(parsed.note === undefined ? {} : { note: parsed.note }),
1446
+ }
1447
+ } finally {
1448
+ await rm(staged, { force: true }).catch(() => {})
1449
+ }
1450
+ })
1451
+ }
1452
+
1453
+ /** trace: recover an SVG through the pinned upstream vtracer pipeline. */
1454
+ async trace(request: TraceRequest, options: ToolCallOptions): Promise<TraceResult> {
1455
+ return this.runOperation('vision_trace', options, async (operation) => {
1456
+ if (request.region !== undefined) parseRegion(request.region)
1457
+ if (request.scale !== undefined && (!Number.isInteger(request.scale) || request.scale < 1 || request.scale > 16)) {
1458
+ throw new VisionToolkitError('input', 'trace: scale must be an integer between 1 and 16')
1459
+ }
1460
+ const policy = await this.pathPolicy(options.workspace)
1461
+ const image = await this.validateImage(request.image, policy, operation)
1462
+ this.accountImage(image, operation)
1463
+ const extension = extname(image.originalPath).toLowerCase()
1464
+ const stem = basename(image.originalPath, extension)
1465
+ const finalPath = resolveOutputFile(request.output, policy, `${stem}.svg`, ['.svg'])
1466
+ assertDistinctOutput(image.path, finalPath)
1467
+ assertDistinctOutput(image.originalPath, finalPath)
1468
+ const staged = createStagedOutput(policy, '.svg')
1469
+ try {
1470
+ const result = await this.runUpstream('trace', [
1471
+ image.path,
1472
+ ...(request.region !== undefined ? ['--region', request.region] : []),
1473
+ ...(request.scale !== undefined ? ['--scale', String(request.scale)] : []),
1474
+ ...(request.polygon === true ? ['--polygon'] : []),
1475
+ ...(request.color === true ? ['--color'] : []),
1476
+ '-o',
1477
+ staged,
1478
+ ], operation)
1479
+ const parsed = parseTraceOutput(result.stdout)
1480
+ const svg = await readFile(staged, 'utf8').catch(() => '')
1481
+ const actualPathCount = svgDocumentPathCount(svg)
1482
+ if (actualPathCount === undefined) {
1483
+ throw new VisionToolkitError('output', 'trace: output SVG is not a parseable document')
1484
+ }
1485
+ if (actualPathCount !== parsed.pathCount) {
1486
+ throw new VisionToolkitError('output', 'trace: reported path count does not match the generated SVG')
1487
+ }
1488
+ const svgInfo = await stat(staged)
1489
+ if (svgInfo.size !== parsed.bytes) {
1490
+ throw new VisionToolkitError('output', 'trace: reported byte count does not match the generated SVG')
1491
+ }
1492
+ await commitStagedOutput(staged, finalPath, policy)
1493
+ const artifact = await describeArtifact(finalPath, policy, {
1494
+ mimeType: 'image/svg+xml',
1495
+ kind: 'svg',
1496
+ description: 'Traced vector geometry',
1497
+ sourceTool: 'vision_trace',
1498
+ previewIntent: 'svg',
1499
+ })
1500
+ const warning = result.stderr.trim()
1501
+ return {
1502
+ imageWidth: image.width,
1503
+ imageHeight: image.height,
1504
+ outputPath: finalPath,
1505
+ mimeType: 'image/svg+xml',
1506
+ geometry: {
1507
+ status: parsed.pathCount === 0 ? 'empty' : 'generated',
1508
+ pathCount: parsed.pathCount,
1509
+ tracedScale: parsed.tracedScale,
1510
+ bytes: parsed.bytes,
1511
+ },
1512
+ artifact,
1513
+ ...(warning.length === 0 ? {} : { warning: warning.split(/\r?\n/).slice(-1)[0] ?? warning }),
1514
+ }
1515
+ } finally {
1516
+ await rm(staged, { force: true }).catch(() => {})
1517
+ }
1518
+ })
1519
+ }
1520
+
1521
+ /** pixel_diff: compare real pixels, rank error regions, and deliver a heatmap plus JSON report. */
1522
+ async pixelDiff(request: PixelDiffRequest, options: ToolCallOptions): Promise<PixelDiffResult> {
1523
+ return this.runOperation('vision_pixel_diff', options, async (operation) => {
1524
+ const grid = integerInRange(request.grid, 6, 1, 32, 'pixel_diff.grid')
1525
+ const top = integerInRange(request.top, 5, 1, grid * grid, 'pixel_diff.top')
1526
+ const policy = await this.pathPolicy(options.workspace)
1527
+ const original = await this.validateImage(request.original, policy, operation)
1528
+ const rebuilt = await this.validateImage(request.rebuilt, policy, operation)
1529
+ this.accountImage(original, operation)
1530
+ this.accountImage(rebuilt, operation)
1531
+ const originalStem = basename(original.originalPath, extname(original.originalPath))
1532
+ const rebuiltStem = basename(rebuilt.originalPath, extname(rebuilt.originalPath))
1533
+ const finalDirectory = resolveOutputDirectory(
1534
+ request.runName,
1535
+ policy,
1536
+ `${originalStem}-vs-${rebuiltStem}.pixel-diff`,
1537
+ )
1538
+ if (
1539
+ isWithin(finalDirectory, original.path)
1540
+ || isWithin(finalDirectory, rebuilt.path)
1541
+ || isWithin(finalDirectory, original.originalPath)
1542
+ || isWithin(finalDirectory, rebuilt.originalPath)
1543
+ ) {
1544
+ throw new VisionToolkitError('input', 'pixel_diff artifact directory would replace an input image')
1545
+ }
1546
+ const stagedDirectory = await createStagedDirectory(policy)
1547
+ const stagedHeatmap = join(stagedDirectory, 'heatmap.png')
1548
+ const stagedReport = join(stagedDirectory, 'report.json')
1549
+ try {
1550
+ const result = await this.runUpstream('pixel_diff', [
1551
+ original.path,
1552
+ rebuilt.path,
1553
+ '--grid',
1554
+ String(grid),
1555
+ '--top',
1556
+ String(top),
1557
+ '-o',
1558
+ stagedHeatmap,
1559
+ ], operation)
1560
+ const parsed = parsePixelDiffOutput(result.stdout)
1561
+ if (parsed.heatmapPath !== stagedHeatmap) {
1562
+ throw new VisionToolkitError('output', 'pixel_diff: upstream reported an unexpected heatmap path')
1563
+ }
1564
+ if (!Number.isFinite(parsed.overallDifferencePct) || parsed.overallDifferencePct < 0 || parsed.overallDifferencePct > 100) {
1565
+ throw new VisionToolkitError('output', 'pixel_diff: overall difference is outside 0-100%')
1566
+ }
1567
+ if (
1568
+ parsed.scaled !== (original.width !== rebuilt.width || original.height !== rebuilt.height)
1569
+ || (parsed.scaledToSize !== undefined
1570
+ && (parsed.scaledToSize.width !== original.width || parsed.scaledToSize.height !== original.height))
1571
+ || parsed.worstRegions.length > top
1572
+ || parsed.worstRegions.some((region, index) => region.index !== index + 1)
1573
+ ) {
1574
+ throw new VisionToolkitError('output', 'pixel_diff: scaling or ranked-region metadata is inconsistent')
1575
+ }
1576
+ for (const region of parsed.worstRegions) assertBoxWithin(region.box, original.width, original.height, 'pixel_diff')
1577
+ const heatmapInfo = await this.probeGeneratedImage(stagedHeatmap, operation, 'pixel_diff')
1578
+ if (heatmapInfo.format !== 'png' || heatmapInfo.width !== original.width || heatmapInfo.height !== original.height) {
1579
+ throw new VisionToolkitError('output', 'pixel_diff: heatmap dimensions or format do not match the reference image')
1580
+ }
1581
+ const reportPayload = {
1582
+ schemaVersion: 1,
1583
+ sourceTool: 'vision_pixel_diff',
1584
+ original,
1585
+ rebuilt,
1586
+ scaled: parsed.scaled,
1587
+ ...(parsed.rebuiltOriginalSize === undefined ? {} : { rebuiltOriginalSize: parsed.rebuiltOriginalSize }),
1588
+ overallDifferencePct: parsed.overallDifferencePct,
1589
+ grid,
1590
+ worstRegions: parsed.worstRegions,
1591
+ }
1592
+ await writeFile(stagedReport, `${JSON.stringify(reportPayload, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' })
1593
+ await commitStagedDirectory(stagedDirectory, finalDirectory, policy)
1594
+ const heatmapPath = join(finalDirectory, 'heatmap.png')
1595
+ const reportPath = join(finalDirectory, 'report.json')
1596
+ const [heatmap, report] = await Promise.all([
1597
+ describeArtifact(heatmapPath, policy, {
1598
+ mimeType: 'image/png',
1599
+ kind: 'image',
1600
+ description: 'Pixel-difference heatmap',
1601
+ sourceTool: 'vision_pixel_diff',
1602
+ previewIntent: 'image',
1603
+ }),
1604
+ describeArtifact(reportPath, policy, {
1605
+ mimeType: 'application/json',
1606
+ kind: 'json',
1607
+ description: 'Structured pixel-difference report',
1608
+ sourceTool: 'vision_pixel_diff',
1609
+ previewIntent: 'text',
1610
+ }),
1611
+ ])
1612
+ return {
1613
+ original,
1614
+ rebuilt,
1615
+ scaled: parsed.scaled,
1616
+ ...(parsed.rebuiltOriginalSize === undefined ? {} : { rebuiltOriginalSize: parsed.rebuiltOriginalSize }),
1617
+ overallDifferencePct: parsed.overallDifferencePct,
1618
+ worstRegions: parsed.worstRegions,
1619
+ heatmap,
1620
+ report,
1621
+ }
1622
+ } finally {
1623
+ await rm(stagedDirectory, { recursive: true, force: true }).catch(() => {})
1624
+ }
1625
+ })
1626
+ }
1627
+
1628
+ /** long_screenshot_ocr: split safely, optionally OCR, and atomically deliver the complete audit run. */
1629
+ async longScreenshotOcr(request: LongScreenshotOcrRequest, options: ToolCallOptions): Promise<LongScreenshotOcrResult> {
1630
+ const jobs = integerInRange(request.jobs, Math.min(2, this.config.concurrency), 1, this.config.concurrency, 'long_screenshot_ocr.jobs')
1631
+ const splitOnly = request.splitOnly === true
1632
+ const permits = splitOnly ? 1 : jobs
1633
+ return this.runOperation('vision_long_screenshot_ocr', options, async (operation) => {
1634
+ const mode = request.mode ?? 'general'
1635
+ if (mode !== 'general' && mode !== 'chat') throw new VisionToolkitError('input', 'long_screenshot_ocr.mode must be general or chat')
1636
+ const targetHeight = request.targetHeight === undefined
1637
+ ? undefined
1638
+ : integerInRange(request.targetHeight, request.targetHeight, 64, 100000, 'long_screenshot_ocr.targetHeight')
1639
+ const minHeight = request.minHeight === undefined
1640
+ ? undefined
1641
+ : integerInRange(request.minHeight, request.minHeight, 64, 100000, 'long_screenshot_ocr.minHeight')
1642
+ const maxHeight = request.maxHeight === undefined
1643
+ ? undefined
1644
+ : integerInRange(request.maxHeight, request.maxHeight, 64, 100000, 'long_screenshot_ocr.maxHeight')
1645
+ if (minHeight !== undefined && maxHeight !== undefined && minHeight > maxHeight) {
1646
+ throw new VisionToolkitError('input', 'long_screenshot_ocr.minHeight must not exceed maxHeight')
1647
+ }
1648
+ if (targetHeight !== undefined && minHeight !== undefined && targetHeight < minHeight) {
1649
+ throw new VisionToolkitError('input', 'long_screenshot_ocr.targetHeight must not be below minHeight')
1650
+ }
1651
+ if (targetHeight !== undefined && maxHeight !== undefined && targetHeight > maxHeight) {
1652
+ throw new VisionToolkitError('input', 'long_screenshot_ocr.targetHeight must not exceed maxHeight')
1653
+ }
1654
+ const overlap = request.overlap === undefined
1655
+ ? undefined
1656
+ : integerInRange(request.overlap, request.overlap, 0, 10000, 'long_screenshot_ocr.overlap')
1657
+ const chunkTimeoutSeconds = finiteInRange(
1658
+ request.chunkTimeoutSeconds ?? Math.min(180, Math.max(1, Math.ceil(this.timeout(options) / 1000))),
1659
+ 1,
1660
+ 600,
1661
+ 'long_screenshot_ocr.chunkTimeoutSeconds',
1662
+ )
1663
+ if (chunkTimeoutSeconds === undefined) throw new VisionToolkitError('input', 'long_screenshot_ocr chunk timeout is required')
1664
+ if (request.prompt !== undefined && request.prompt.trim().length === 0) {
1665
+ throw new VisionToolkitError('input', 'long_screenshot_ocr.prompt must not be empty when provided')
1666
+ }
1667
+ const policy = await this.pathPolicy(options.workspace)
1668
+ const image = await this.validateImage(request.image, policy, operation)
1669
+ this.accountImage(image, operation)
1670
+ const stem = basename(image.originalPath, extname(image.originalPath))
1671
+ const finalDirectory = resolveOutputDirectory(request.runName, policy, `${stem}.long-ocr`)
1672
+ if (isWithin(finalDirectory, image.path) || isWithin(finalDirectory, image.originalPath)) {
1673
+ throw new VisionToolkitError('input', 'long_screenshot_ocr artifact directory would replace the input image')
1674
+ }
1675
+ const stagedDirectory = await createStagedDirectory(policy)
1676
+ try {
1677
+ if (request.resume === true) await seedStagedDirectory(finalDirectory, stagedDirectory, policy)
1678
+ const stagedPolicy = { ...policy, outputDir: stagedDirectory }
1679
+ const stagedOutput = resolveOutputFile(request.output, stagedPolicy, `${stem}.ocr.md`, ['.md', '.markdown'])
1680
+ const finalOutput = join(finalDirectory, basename(stagedOutput))
1681
+ const stagedChunks = join(stagedDirectory, 'chunks')
1682
+ const stagedManifest = join(stagedChunks, 'manifest.json')
1683
+ const result = await this.runUpstream('long_screenshot_ocr', [
1684
+ image.path,
1685
+ '--mode',
1686
+ mode,
1687
+ '-o',
1688
+ stagedOutput,
1689
+ '--chunks-dir',
1690
+ stagedChunks,
1691
+ ...(targetHeight === undefined ? [] : ['--target-height', String(targetHeight)]),
1692
+ ...(minHeight === undefined ? [] : ['--min-height', String(minHeight)]),
1693
+ ...(maxHeight === undefined ? [] : ['--max-height', String(maxHeight)]),
1694
+ ...(overlap === undefined ? [] : ['--overlap', String(overlap)]),
1695
+ ...(request.prompt === undefined ? [] : ['--prompt', request.prompt]),
1696
+ '--jobs',
1697
+ String(jobs),
1698
+ '--timeout',
1699
+ String(chunkTimeoutSeconds),
1700
+ ...(splitOnly ? ['--split-only'] : []),
1701
+ ...(request.resume === true ? ['--resume'] : []),
1702
+ ], operation, splitOnly ? undefined : await this.resolveVisionEnv())
1703
+ const reported = result.stdout.trim()
1704
+ const expectedReported = splitOnly ? stagedManifest : stagedOutput
1705
+ if (reported !== expectedReported) {
1706
+ throw new VisionToolkitError('output', 'long_screenshot_ocr: upstream reported an unexpected output path')
1707
+ }
1708
+ const parsedManifest = parseLongOcrManifest(await readFile(stagedManifest, 'utf8'), {
1709
+ source: image.path,
1710
+ output: stagedOutput,
1711
+ width: image.width,
1712
+ height: image.height,
1713
+ mode,
1714
+ splitOnly,
1715
+ })
1716
+ for (const chunk of parsedManifest.chunks) {
1717
+ const chunkBytes = await readFile(join(stagedChunks, chunk.image))
1718
+ if (createHash('sha256').update(chunkBytes).digest('hex') !== chunk.imageSha256) {
1719
+ throw new VisionToolkitError('output', `long_screenshot_ocr: chunk ${chunk.index} hash does not match the manifest`)
1720
+ }
1721
+ }
1722
+ parsedManifest.raw.output = splitOnly ? null : finalOutput
1723
+ await writeFile(stagedManifest, `${JSON.stringify(parsedManifest.raw, null, 2)}\n`, 'utf8')
1724
+ await commitStagedDirectory(stagedDirectory, finalDirectory, policy)
1725
+ const finalChunks = join(finalDirectory, 'chunks')
1726
+ const manifest = await describeArtifact(join(finalChunks, 'manifest.json'), policy, {
1727
+ mimeType: 'application/json',
1728
+ kind: 'json',
1729
+ description: 'Long-screenshot split and merge manifest',
1730
+ sourceTool: 'vision_long_screenshot_ocr',
1731
+ previewIntent: 'text',
1732
+ })
1733
+ const output = splitOnly
1734
+ ? undefined
1735
+ : await describeArtifact(finalOutput, policy, {
1736
+ mimeType: 'text/markdown',
1737
+ kind: 'markdown',
1738
+ description: 'Merged long-screenshot OCR transcript',
1739
+ sourceTool: 'vision_long_screenshot_ocr',
1740
+ previewIntent: 'text',
1741
+ })
1742
+ const audit = splitOnly
1743
+ ? undefined
1744
+ : await describeArtifact(join(finalChunks, 'ocr_audit.md'), policy, {
1745
+ mimeType: 'text/markdown',
1746
+ kind: 'markdown',
1747
+ description: 'Long-screenshot OCR boundary audit',
1748
+ sourceTool: 'vision_long_screenshot_ocr',
1749
+ previewIntent: 'text',
1750
+ })
1751
+ const chunks: LongScreenshotChunk[] = []
1752
+ for (const chunk of parsedManifest.chunks) {
1753
+ const chunkImage = await describeArtifact(join(finalChunks, chunk.image), policy, {
1754
+ mimeType: 'image/png',
1755
+ kind: 'image',
1756
+ description: `Long-screenshot OCR chunk ${chunk.index}`,
1757
+ sourceTool: 'vision_long_screenshot_ocr',
1758
+ previewIntent: 'image',
1759
+ })
1760
+ const ocr = chunk.ocr === undefined
1761
+ ? undefined
1762
+ : await describeArtifact(join(finalChunks, chunk.ocr), policy, {
1763
+ mimeType: mode === 'chat' ? 'application/json' : 'text/markdown',
1764
+ kind: mode === 'chat' ? 'json' : 'markdown',
1765
+ description: `OCR sidecar for chunk ${chunk.index}`,
1766
+ sourceTool: 'vision_long_screenshot_ocr',
1767
+ previewIntent: 'text',
1768
+ })
1769
+ chunks.push({
1770
+ index: chunk.index,
1771
+ coreTop: chunk.coreTop,
1772
+ coreBottom: chunk.coreBottom,
1773
+ cropTop: chunk.cropTop,
1774
+ cropBottom: chunk.cropBottom,
1775
+ image: chunkImage,
1776
+ ...(ocr === undefined ? {} : { ocr }),
1777
+ ...(chunk.ocrReused === undefined ? {} : { reused: chunk.ocrReused }),
1778
+ })
1779
+ }
1780
+ return {
1781
+ source: image,
1782
+ mode,
1783
+ splitOnly,
1784
+ complete: parsedManifest.complete,
1785
+ chunkCount: chunks.length,
1786
+ runDirectory: finalDirectory,
1787
+ ...(output === undefined ? {} : { output }),
1788
+ manifest,
1789
+ ...(audit === undefined ? {} : { audit }),
1790
+ chunks,
1791
+ }
1792
+ } finally {
1793
+ await rm(stagedDirectory, { recursive: true, force: true }).catch(() => {})
1794
+ }
1795
+ }, permits)
1796
+ }
1797
+
1798
+ /** extract_foreground: preserve the pinned component selection and deliver an RGBA PNG. */
1799
+ async extractForeground(request: ExtractForegroundRequest, options: ToolCallOptions): Promise<ExtractForegroundResult> {
1800
+ return this.runOperation('vision_extract_foreground', options, async (operation) => {
1801
+ const policy = await this.pathPolicy(options.workspace)
1802
+ const image = await this.validateImage(request.image, policy, operation)
1803
+ this.accountImage(image, operation)
1804
+ const region = request.region === undefined ? undefined : parseRegion(request.region)
1805
+ if (region !== undefined) assertBoxWithin(region, image.width, image.height, 'extract_foreground.region')
1806
+ const boxes = request.boxes === undefined ? undefined : parseRegion(request.boxes)
1807
+ if (boxes !== undefined) assertBoxWithin(boxes, image.width, image.height, 'extract_foreground.boxes')
1808
+ const discRadius = finiteInRange(request.discRadius, 1, Math.max(image.width, image.height) * 4, 'extract_foreground.discRadius')
1809
+ const saturation = integerInRange(request.saturation, 12, 0, 255, 'extract_foreground.saturation')
1810
+ const darkThreshold = integerInRange(request.darkThreshold, 215, 0, 255, 'extract_foreground.darkThreshold')
1811
+ const excludeTolerance = finiteInRange(request.excludeTolerance ?? 24, 0, 442, 'extract_foreground.excludeTolerance')
1812
+ const padding = integerInRange(request.padding, 3, 0, 4096, 'extract_foreground.padding')
1813
+ const excludeColor = request.excludeColor === undefined
1814
+ ? undefined
1815
+ : `#${request.excludeColor.trim().replace(/^#/, '').toUpperCase()}`
1816
+ if (excludeColor !== undefined && !HEX_COLOR_PATTERN.test(excludeColor)) {
1817
+ throw new VisionToolkitError('input', 'extract_foreground.excludeColor must be #RRGGBB')
1818
+ }
1819
+ const extension = extname(image.originalPath).toLowerCase()
1820
+ const stem = basename(image.originalPath, extension)
1821
+ const finalPath = resolveOutputFile(request.output, policy, `${stem}.foreground.png`, ['.png'])
1822
+ assertDistinctOutput(image.path, finalPath)
1823
+ assertDistinctOutput(image.originalPath, finalPath)
1824
+ const staged = createStagedOutput(policy, '.png')
1825
+ try {
1826
+ const result = await this.runUpstream('extract_foreground', [
1827
+ image.path,
1828
+ ...(region === undefined ? [] : ['--region', `${region.x1},${region.y1},${region.x2},${region.y2}`]),
1829
+ ...(boxes === undefined ? [] : ['--boxes', `${boxes.x1},${boxes.y1},${boxes.x2},${boxes.y2}`]),
1830
+ '--mode',
1831
+ request.mode ?? 'color',
1832
+ '--sat',
1833
+ String(saturation),
1834
+ '--dark',
1835
+ String(darkThreshold),
1836
+ '--exclude-tol',
1837
+ String(excludeTolerance),
1838
+ '--pad',
1839
+ String(padding),
1840
+ ...(discRadius === undefined ? [] : ['--disc-radius', String(discRadius)]),
1841
+ ...(excludeColor === undefined ? [] : ['--exclude-color', excludeColor]),
1842
+ ...(request.keepWhites === false ? ['--no-keep-whites'] : []),
1843
+ '-o',
1844
+ staged,
1845
+ ], operation)
1846
+ const parsed = parseExtractForegroundOutput(result.stdout)
1847
+ if (parsed.outputPath !== staged) throw new VisionToolkitError('output', 'extract_foreground: upstream reported an unexpected output path')
1848
+ assertBoxWithin(parsed.box, image.width, image.height, 'extract_foreground')
1849
+ if (
1850
+ parsed.foregroundPixels <= 0
1851
+ || parsed.keptComponents <= 0
1852
+ || parsed.totalComponents < parsed.keptComponents
1853
+ || parsed.largestComponentPct < 0
1854
+ || parsed.largestComponentPct > 100
1855
+ ) {
1856
+ throw new VisionToolkitError('output', 'extract_foreground: component metrics are invalid')
1857
+ }
1858
+ const generated = await this.probeGeneratedImage(staged, operation, 'extract_foreground')
1859
+ if (
1860
+ generated.format !== 'png'
1861
+ || (generated.mode !== 'RGBA' && generated.mode !== 'LA')
1862
+ || generated.width !== parsed.width
1863
+ || generated.height !== parsed.height
1864
+ ) {
1865
+ throw new VisionToolkitError('output', 'extract_foreground: output is not the reported transparent PNG')
1866
+ }
1867
+ await commitStagedOutput(staged, finalPath, policy)
1868
+ const artifact = await describeArtifact(finalPath, policy, {
1869
+ mimeType: 'image/png',
1870
+ kind: 'image',
1871
+ description: 'Extracted transparent foreground',
1872
+ sourceTool: 'vision_extract_foreground',
1873
+ previewIntent: 'image',
1874
+ })
1875
+ return {
1876
+ source: image,
1877
+ box: parsed.box,
1878
+ foregroundPixels: parsed.foregroundPixels,
1879
+ keptComponents: parsed.keptComponents,
1880
+ totalComponents: parsed.totalComponents,
1881
+ largestComponentPct: parsed.largestComponentPct,
1882
+ width: parsed.width,
1883
+ height: parsed.height,
1884
+ artifact,
1885
+ ...(parsed.autoSummary === undefined ? {} : { autoSummary: parsed.autoSummary }),
1886
+ }
1887
+ } finally {
1888
+ await rm(staged, { force: true }).catch(() => {})
1889
+ }
1890
+ })
1891
+ }
1892
+
1893
+ /** dominant_colors: expose palette clusters or candidate scores as structure, never stdout prose. */
1894
+ async dominantColors(request: DominantColorsRequest, options: ToolCallOptions): Promise<DominantColorsResult> {
1895
+ return this.runOperation('vision_dominant_colors', options, async (operation) => {
1896
+ const top = integerInRange(request.top, 5, 1, 64, 'dominant_colors.top')
1897
+ const quantize = integerInRange(request.quantize, 16, 2, 256, 'dominant_colors.quantize')
1898
+ const maxPixels = integerInRange(request.maxPixels, 96, 8, 4096, 'dominant_colors.maxPixels')
1899
+ const mergeTolerance = integerInRange(request.mergeTolerance, 8, 0, 255, 'dominant_colors.mergeTolerance')
1900
+ const candidateTolerance = integerInRange(request.candidateTolerance, 16, 0, 255, 'dominant_colors.candidateTolerance')
1901
+ const policy = await this.pathPolicy(options.workspace)
1902
+ const image = await this.validateImage(request.image, policy, operation)
1903
+ this.accountImage(image, operation)
1904
+ const region = request.region === undefined ? undefined : parseRegion(request.region)
1905
+ if (region !== undefined) assertBoxWithin(region, image.width, image.height, 'dominant_colors.region')
1906
+ const candidates = request.candidates?.map(value => `#${value.trim().replace(/^#/, '').toUpperCase()}`)
1907
+ if (candidates !== undefined) {
1908
+ if (candidates.length === 0 || candidates.length > 32) {
1909
+ throw new VisionToolkitError('input', 'dominant_colors.candidates must contain between 1 and 32 colors')
1910
+ }
1911
+ if (candidates.some(candidate => !HEX_COLOR_PATTERN.test(candidate))) {
1912
+ throw new VisionToolkitError('input', 'dominant_colors.candidates must contain only #RRGGBB colors')
1913
+ }
1914
+ if (new Set(candidates).size !== candidates.length) {
1915
+ throw new VisionToolkitError('input', 'dominant_colors.candidates must not contain duplicates')
1916
+ }
1917
+ }
1918
+ const result = await this.runUpstream('dominant_colors', [
1919
+ image.path,
1920
+ ...(region === undefined ? [] : ['--region', `${region.x1},${region.y1},${region.x2},${region.y2}`]),
1921
+ ...(candidates === undefined ? [] : ['--candidates', candidates.join(',')]),
1922
+ '--top',
1923
+ String(top),
1924
+ '--quantize',
1925
+ String(quantize),
1926
+ '--max-pixels',
1927
+ String(maxPixels),
1928
+ '--merge-tol',
1929
+ String(mergeTolerance),
1930
+ '--tol',
1931
+ String(candidateTolerance),
1932
+ ], operation)
1933
+ const analysis = parseDominantColorsOutput(result.stdout)
1934
+ assertBoxWithin(analysis.region, image.width, image.height, 'dominant_colors')
1935
+ if (analysis.width !== analysis.region.x2 - analysis.region.x1 || analysis.height !== analysis.region.y2 - analysis.region.y1) {
1936
+ throw new VisionToolkitError('output', 'dominant_colors: reported region dimensions are inconsistent')
1937
+ }
1938
+ if (candidates !== undefined) {
1939
+ if (analysis.mode !== 'candidates') throw new VisionToolkitError('output', 'dominant_colors: expected candidate mode output')
1940
+ if (analysis.candidates.map(candidate => candidate.color).join(',') !== candidates.join(',')) {
1941
+ throw new VisionToolkitError('output', 'dominant_colors: candidate rows do not match the request')
1942
+ }
1943
+ } else if (analysis.mode !== 'palette') {
1944
+ throw new VisionToolkitError('output', 'dominant_colors: expected palette mode output')
1945
+ }
1946
+ return { image, analysis }
1947
+ })
1948
+ }
1949
+
1950
+ /** html_screenshot: render only a path-fenced local HTML file in the pinned Chrome adapter. */
1951
+ async htmlScreenshot(request: HtmlScreenshotRequest, options: ToolCallOptions): Promise<HtmlScreenshotResult> {
1952
+ return this.runOperation('vision_html_screenshot', options, async (operation) => {
1953
+ const width = integerInRange(request.width, 1280, 1, 8192, 'html_screenshot.width')
1954
+ const height = integerInRange(request.height, 800, 1, 8192, 'html_screenshot.height')
1955
+ const scale = integerInRange(request.scale, 1, 1, 4, 'html_screenshot.scale')
1956
+ const waitMs = integerInRange(request.waitMs, 0, 0, 120000, 'html_screenshot.waitMs')
1957
+ const fullPage = request.fullPage === true
1958
+ const outputPixels = width * height * scale * scale
1959
+ if (!Number.isSafeInteger(outputPixels) || outputPixels > this.config.maxImagePixels) {
1960
+ throw new VisionToolkitError('capacity', `HTML screenshot would create ${outputPixels} pixels, exceeding maxImagePixels ${this.config.maxImagePixels}`)
1961
+ }
1962
+ const policy = await this.pathPolicy(options.workspace)
1963
+ const source = await resolveHtmlFile(request.source, policy)
1964
+ if (source.bytes > this.config.maxImageBytes) {
1965
+ throw new VisionToolkitError('capacity', `HTML source is ${source.bytes} bytes, exceeding maxImageBytes ${this.config.maxImageBytes}`)
1966
+ }
1967
+ const stem = basename(source.path, extname(source.path))
1968
+ const finalPath = resolveOutputFile(request.output, policy, `${stem}.screenshot.png`, ['.png'])
1969
+ assertDistinctOutput(source.path, finalPath)
1970
+ const staged = createStagedOutput(policy, '.png')
1971
+ try {
1972
+ const result = await this.runUpstream('html_screenshot', [
1973
+ source.path,
1974
+ '-o',
1975
+ staged,
1976
+ '--width',
1977
+ String(width),
1978
+ '--height',
1979
+ String(height),
1980
+ '--scale',
1981
+ String(scale),
1982
+ '--wait-ms',
1983
+ String(waitMs),
1984
+ ...(fullPage ? ['--full-page', '--max-pixels', String(this.config.maxImagePixels)] : []),
1985
+ ], operation)
1986
+ const parsed = parseHtmlScreenshotOutput(result.stdout)
1987
+ const expectedWidth = width * scale
1988
+ if (parsed.outputPath !== staged || parsed.width !== expectedWidth) {
1989
+ throw new VisionToolkitError('output', 'html_screenshot: upstream summary does not match the requested output')
1990
+ }
1991
+ if (fullPage !== (parsed.pageHeight !== undefined)) {
1992
+ throw new VisionToolkitError('output', 'html_screenshot: upstream full-page metadata does not match the request')
1993
+ }
1994
+ const pageHeight = parsed.pageHeight
1995
+ const expectedHeight = pageHeight === undefined ? height * scale : pageHeight * scale
1996
+ if (pageHeight !== undefined && (!Number.isInteger(pageHeight) || pageHeight <= 0)) {
1997
+ throw new VisionToolkitError('output', 'html_screenshot: upstream reported an invalid page height')
1998
+ }
1999
+ const outputPixels = expectedWidth * expectedHeight
2000
+ if (!Number.isSafeInteger(outputPixels) || outputPixels > this.config.maxImagePixels) {
2001
+ throw new VisionToolkitError('capacity', `HTML screenshot would create ${outputPixels} pixels, exceeding maxImagePixels ${this.config.maxImagePixels}`)
2002
+ }
2003
+ if (parsed.height !== expectedHeight) {
2004
+ throw new VisionToolkitError('output', 'html_screenshot: upstream summary does not match the requested output')
2005
+ }
2006
+ const generated = await this.probeGeneratedImage(staged, operation, 'html_screenshot')
2007
+ if (generated.format !== 'png' || generated.width !== expectedWidth || generated.height !== expectedHeight) {
2008
+ throw new VisionToolkitError('output', 'html_screenshot: generated PNG dimensions do not match the reported output')
2009
+ }
2010
+ await commitStagedOutput(staged, finalPath, policy)
2011
+ const artifact = await describeArtifact(finalPath, policy, {
2012
+ mimeType: 'image/png',
2013
+ kind: 'image',
2014
+ description: 'Headless browser screenshot of local HTML',
2015
+ sourceTool: 'vision_html_screenshot',
2016
+ previewIntent: 'image',
2017
+ })
2018
+ return {
2019
+ sourcePath: source.path,
2020
+ sourceBytes: source.bytes,
2021
+ viewport: { width, height, scale },
2022
+ width: expectedWidth,
2023
+ height: expectedHeight,
2024
+ ...(pageHeight === undefined ? {} : { pageHeight }),
2025
+ artifact,
2026
+ }
2027
+ } finally {
2028
+ await rm(staged, { force: true }).catch(() => {})
2029
+ }
2030
+ })
2031
+ }
2032
+
2033
+ private async writableDirectoryCheck(path: string, label: string): Promise<HealthCheck> {
2034
+ const probe = join(path, `.vision-toolkit-health-${randomUUID()}`)
2035
+ try {
2036
+ await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' })
2037
+ await rm(probe, { force: true })
2038
+ return { status: 'ok', detail: `${label} is writable: ${path}` }
2039
+ } catch {
2040
+ await rm(probe, { force: true }).catch(() => {})
2041
+ return { status: 'error', detail: `${label} is not writable: ${path}` }
2042
+ }
2043
+ }
2044
+
2045
+ /** Health: inspect local readiness, optionally probe `/models`, and explicitly test one real multimodal request. */
2046
+ async health(testConnection: boolean, options: ToolCallOptions, testModel = false): Promise<VisionToolkitHealthResult> {
2047
+ return this.runOperation('vision_toolkit_health', options, async (operation) => {
2048
+ const info = this.upstreamVersion
2049
+ const python: HealthCheck = { status: 'ok', detail: `${info.pythonVersion} via ${info.python}` }
2050
+ const dependencies: HealthCheck = {
2051
+ status: 'ok',
2052
+ detail: Object.entries(info.dependencies).map(([name, version]) => `${name}=${version}`).join(', '),
2053
+ }
2054
+ let chrome: HealthCheck
2055
+ try {
2056
+ const started = Date.now()
2057
+ const chromePath = await this.adapter.findChrome({ signal: operation.signal })
2058
+ operation.metrics.upstreamMs += Date.now() - started
2059
+ chrome = chromePath === undefined
2060
+ ? { status: 'warning', detail: 'Chrome/Chromium/Edge was not found; vision_html_screenshot is unavailable' }
2061
+ : { status: 'ok', detail: chromePath }
2062
+ } catch {
2063
+ if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled')
2064
+ chrome = { status: 'error', detail: 'Chrome availability probe failed' }
2065
+ }
2066
+ let resolvedCredential: ResolvedCredential | undefined
2067
+ let credential: HealthCheck
2068
+ try {
2069
+ resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
2070
+ ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
2071
+ : await this.ctx.credentials.resolve(this.config.provider.credential)
2072
+ credential = resolvedCredential === undefined
2073
+ ? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
2074
+ : { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` }
2075
+ } catch {
2076
+ credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` }
2077
+ }
2078
+ let artifactDirectory: HealthCheck
2079
+ try {
2080
+ // allowedDirs are session input roots; they do not affect output readiness.
2081
+ const policy = await createPathPolicy(options.workspace, [])
2082
+ artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory')
2083
+ } catch {
2084
+ artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' }
2085
+ }
2086
+ const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory')
2087
+ let service: HealthCheck = {
2088
+ status: 'not_tested',
2089
+ detail: 'Connection was not tested; pass testConnection=true to query the configured /models endpoint',
2090
+ }
2091
+ let model: HealthCheck = {
2092
+ status: 'not_tested',
2093
+ detail: 'Vision model was not tested; run an explicit model test to send the bundled diagnostic image',
2094
+ }
2095
+ if (testConnection) {
2096
+ if (resolvedCredential === undefined) {
2097
+ service = { status: 'error', detail: 'Connection test skipped because the configured credential is unavailable' }
2098
+ } else {
2099
+ operation.metrics.usedVisionService = true
2100
+ const endpoint = `${this.config.provider.baseUrl}/models`
2101
+ try {
2102
+ const started = Date.now()
2103
+ const headers: Record<string, string> = {
2104
+ Accept: 'application/json',
2105
+ 'User-Agent': this.config.provider.userAgent,
2106
+ }
2107
+ if (this.config.provider.protocol === 'anthropic') {
2108
+ headers['x-api-key'] = resolvedCredential.value
2109
+ headers['anthropic-version'] = '2023-06-01'
2110
+ } else {
2111
+ headers.Authorization = `Bearer ${resolvedCredential.value}`
2112
+ }
2113
+ const response = await fetch(endpoint, {
2114
+ method: 'GET',
2115
+ headers,
2116
+ signal: operation.signal,
2117
+ })
2118
+ operation.metrics.upstreamMs += Date.now() - started
2119
+ await response.body?.cancel().catch(() => {})
2120
+ if (response.ok) {
2121
+ service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` }
2122
+ } else if (response.status === 401) {
2123
+ service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` }
2124
+ } else if (response.status === 403) {
2125
+ // Some providers (e.g. Groq preview/account restrictions) block GET /models
2126
+ // while real multimodal requests still work. Treat 403 as a warning so the
2127
+ // explicit vision-model test, not the model list endpoint, decides access.
2128
+ service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` }
2129
+ } else if (response.status === 404 || response.status === 405) {
2130
+ service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` }
2131
+ } else if (response.status === 429) {
2132
+ service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' }
2133
+ } else {
2134
+ service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` }
2135
+ }
2136
+ } catch {
2137
+ if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled')
2138
+ service = { status: 'error', detail: `Service could not be reached at ${endpoint}` }
2139
+ }
2140
+ }
2141
+ }
2142
+ if (testModel) {
2143
+ if (resolvedCredential === undefined) {
2144
+ model = { status: 'error', detail: 'Vision model test skipped because the configured credential is unavailable' }
2145
+ } else {
2146
+ try {
2147
+ const result = await this.runUpstream(
2148
+ 'glance',
2149
+ [VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT],
2150
+ operation,
2151
+ this.visionEnv(resolvedCredential),
2152
+ )
2153
+ if (result.stdout.trim().length === 0) {
2154
+ throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
2155
+ }
2156
+ model = {
2157
+ status: 'ok',
2158
+ detail: `Vision model ${this.config.provider.model} completed a multimodal request`,
2159
+ }
2160
+ } catch (error) {
2161
+ if (operation.signal.aborted) throw error
2162
+ const detail = error instanceof Error ? error.message : String(error)
2163
+ model = { status: 'error', detail: `Vision model test failed: ${detail.slice(0, 600)}` }
2164
+ }
2165
+ }
2166
+ }
2167
+ const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model }
2168
+ const healthy = Object.values(checks).every(check => check.status !== 'error')
2169
+ return {
2170
+ pluginVersion: PLUGIN_VERSION,
2171
+ upstream: info,
2172
+ checks,
2173
+ healthy,
2174
+ connectionTested: testConnection,
2175
+ modelTested: testModel,
2176
+ }
2177
+ })
2178
+ }
2179
+
2180
+ /** Report the packaged upstream snapshot version. */
2181
+ checkoutVersion(): Promise<string> {
2182
+ return this.adapter.readCheckoutVersion()
2183
+ }
2184
+
2185
+ /** Prepared Python command. */
2186
+ python(): string {
2187
+ return this.adapter.versionInfo.python
2188
+ }
2189
+ }