@mengruo/dsh-vision-toolkit 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +356 -0
- package/README.zh.md +354 -0
- package/assets/community-group-qr.png +0 -0
- package/assets/dsh-conversation-artifact.png +0 -0
- package/assets/dsh-conversation-image-qa-top.png +0 -0
- package/assets/dsh-conversation-image-qa.png +0 -0
- package/assets/dsh-conversation-pixel-diff.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging.png +0 -0
- package/assets/dsh-conversation-tool-call.png +0 -0
- package/assets/dsh-conversation-vision-trace.png +0 -0
- package/assets/dsh-view-example.png +0 -0
- package/assets/hero-v2.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/python-bootstrap.json +48 -0
- package/assets/skill/SKILL.md +329 -0
- package/assets/skill/UPSTREAM.json +71 -0
- package/assets/skill/references/gui.md +88 -0
- package/assets/skill/references/long-screenshot-ocr.md +77 -0
- package/assets/skill/references/restore-graphic.md +84 -0
- package/assets/skill/references/restore-structure.md +45 -0
- package/assets/skill/references/restore-ui.md +202 -0
- package/assets/social-preview.png +0 -0
- package/assets/upstream/README.md +18 -0
- package/assets/upstream/focus-hint-comparison-1.webp +0 -0
- package/assets/upstream/focus-hint-comparison-2.webp +0 -0
- package/assets/upstream/infographic-reference.webp +0 -0
- package/assets/upstream/infographic-result.webp +0 -0
- package/assets/upstream/ui-fast-restore-reference.webp +0 -0
- package/assets/upstream/ui-fast-restore-result.webp +0 -0
- package/assets/upstream/ui-result.webp +0 -0
- package/assets/upstream/ui-sketch.webp +0 -0
- package/assets/vision-model-test.png +0 -0
- package/assets/vision-settings.png +0 -0
- package/assets/wechat-reward.png +0 -0
- package/cordis.patch.yml +6 -0
- package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
- package/docs/aihubmix-gemini-vision.md +138 -0
- package/docs/aihubmix-gemini-vision.zh.md +138 -0
- package/docs/assets/aihubmix-api-keys.png +0 -0
- package/docs/assets/aihubmix-create-key.png +0 -0
- package/docs/assets/aihubmix-free-vision-model.png +0 -0
- package/docs/assets/aihubmix-home.png +0 -0
- package/docs/assets/aihubmix-sign-up.png +0 -0
- package/docs/assets/groq-console-home.png +0 -0
- package/docs/assets/groq-console-keys-login.png +0 -0
- package/docs/assets/groq-docs-quickstart.png +0 -0
- package/docs/assets/groq-docs-qwen3.6.png +0 -0
- package/docs/assets/groq-docs-vision.png +0 -0
- package/docs/assets/vision-settings.png +0 -0
- package/docs/dsh-desktop-install.md +67 -0
- package/docs/dsh-desktop-install.zh.md +67 -0
- package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
- package/docs/groq-qwen3.6-vision.md +236 -0
- package/docs/groq-qwen3.6-vision.zh.md +236 -0
- package/docs/python-runtime.i18n.yaml +6 -0
- package/docs/python-runtime.md +89 -0
- package/docs/python-runtime.zh.md +89 -0
- package/docs/requirements-traceability/README.i18n.yaml +6 -0
- package/docs/requirements-traceability/README.md +75 -0
- package/docs/requirements-traceability/README.zh.md +75 -0
- package/examples/ui-restoration/README.i18n.yaml +6 -0
- package/examples/ui-restoration/README.md +70 -0
- package/examples/ui-restoration/README.zh.md +70 -0
- package/examples/ui-restoration/assets/final-heatmap.png +0 -0
- package/examples/ui-restoration/assets/final-report.json +83 -0
- package/examples/ui-restoration/assets/implementation.png +0 -0
- package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
- package/examples/ui-restoration/assets/initial-report.json +83 -0
- package/examples/ui-restoration/assets/initial.png +0 -0
- package/examples/ui-restoration/assets/metrics.json +12 -0
- package/examples/ui-restoration/assets/reference.png +0 -0
- package/examples/ui-restoration/implementation.html +94 -0
- package/examples/ui-restoration/initial.html +57 -0
- package/lib/artifact-access.js +369 -0
- package/lib/artifact-access.js.map +1 -0
- package/lib/artifacts.js +56 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/client.js +2099 -0
- package/lib/client.js.map +1 -0
- package/lib/config.js +172 -0
- package/lib/config.js.map +1 -0
- package/lib/defaults.js +6 -0
- package/lib/defaults.js.map +1 -0
- package/lib/errors.js +56 -0
- package/lib/errors.js.map +1 -0
- package/lib/evidence-cache.js +350 -0
- package/lib/evidence-cache.js.map +1 -0
- package/lib/exposure.js +247 -0
- package/lib/exposure.js.map +1 -0
- package/lib/image-input-variants.js +882 -0
- package/lib/image-input-variants.js.map +1 -0
- package/lib/index.js +105 -0
- package/lib/index.js.map +1 -0
- package/lib/paste-images.js +217 -0
- package/lib/paste-images.js.map +1 -0
- package/lib/paths.js +354 -0
- package/lib/paths.js.map +1 -0
- package/lib/plugin-update.js +1003 -0
- package/lib/plugin-update.js.map +1 -0
- package/lib/runtime-install.js +964 -0
- package/lib/runtime-install.js.map +1 -0
- package/lib/runtime-manager.js +133 -0
- package/lib/runtime-manager.js.map +1 -0
- package/lib/runtime.js +1678 -0
- package/lib/runtime.js.map +1 -0
- package/lib/skill.js +25 -0
- package/lib/skill.js.map +1 -0
- package/lib/tools.js +549 -0
- package/lib/tools.js.map +1 -0
- package/lib/types/artifact-access.d.ts +61 -0
- package/lib/types/artifact-access.d.ts.map +1 -0
- package/lib/types/artifacts.d.ts +42 -0
- package/lib/types/artifacts.d.ts.map +1 -0
- package/lib/types/client/display-config.d.ts +24 -0
- package/lib/types/client/display-config.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +350 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/lib/types/client/model-variants-hider.d.ts +40 -0
- package/lib/types/client/model-variants-hider.d.ts.map +1 -0
- package/lib/types/client/paste-images.d.ts +126 -0
- package/lib/types/client/paste-images.d.ts.map +1 -0
- package/lib/types/config.d.ts +123 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/defaults.d.ts +6 -0
- package/lib/types/defaults.d.ts.map +1 -0
- package/lib/types/errors.d.ts +35 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/evidence-cache.d.ts +91 -0
- package/lib/types/evidence-cache.d.ts.map +1 -0
- package/lib/types/exposure.d.ts +50 -0
- package/lib/types/exposure.d.ts.map +1 -0
- package/lib/types/image-input-variants.d.ts +144 -0
- package/lib/types/image-input-variants.d.ts.map +1 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +73 -0
- package/lib/types/paste-images.d.ts.map +1 -0
- package/lib/types/paths.d.ts +117 -0
- package/lib/types/paths.d.ts.map +1 -0
- package/lib/types/plugin-update.d.ts +119 -0
- package/lib/types/plugin-update.d.ts.map +1 -0
- package/lib/types/runtime-install.d.ts +92 -0
- package/lib/types/runtime-install.d.ts.map +1 -0
- package/lib/types/runtime-manager.d.ts +60 -0
- package/lib/types/runtime-manager.d.ts.map +1 -0
- package/lib/types/runtime.d.ts +412 -0
- package/lib/types/runtime.d.ts.map +1 -0
- package/lib/types/skill.d.ts +14 -0
- package/lib/types/skill.d.ts.map +1 -0
- package/lib/types/tools.d.ts +35 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +233 -0
- package/lib/types/upstream.d.ts.map +1 -0
- package/lib/types/version.d.ts +15 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/web-request.d.ts +11 -0
- package/lib/types/web-request.d.ts.map +1 -0
- package/lib/types/web.d.ts +118 -0
- package/lib/types/web.d.ts.map +1 -0
- package/lib/upstream.js +860 -0
- package/lib/upstream.js.map +1 -0
- package/lib/version.js +18 -0
- package/lib/version.js.map +1 -0
- package/lib/web-request.js +29 -0
- package/lib/web-request.js.map +1 -0
- package/lib/web.js +462 -0
- package/lib/web.js.map +1 -0
- package/package.json +149 -0
- package/patches/vision-tools-dsh.patch +970 -0
- package/runtime/requirements.lock +3 -0
- package/src/artifact-access.ts +386 -0
- package/src/artifacts.ts +85 -0
- package/src/client/display-config.ts +62 -0
- package/src/client/index.tsx +1580 -0
- package/src/client/model-variants-hider.ts +159 -0
- package/src/client/paste-images.tsx +747 -0
- package/src/config.ts +290 -0
- package/src/defaults.ts +5 -0
- package/src/errors.ts +62 -0
- package/src/evidence-cache.ts +418 -0
- package/src/exposure.ts +265 -0
- package/src/image-input-variants.ts +996 -0
- package/src/index.ts +141 -0
- package/src/paste-images.ts +286 -0
- package/src/paths.ts +383 -0
- package/src/plugin-update.ts +1143 -0
- package/src/runtime-install.ts +1161 -0
- package/src/runtime-manager.ts +174 -0
- package/src/runtime.ts +2189 -0
- package/src/skill.ts +35 -0
- package/src/tools.ts +691 -0
- package/src/upstream.ts +1078 -0
- package/src/version.ts +37 -0
- package/src/web-request.ts +27 -0
- package/src/web.ts +594 -0
- package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
- package/vendor/agent-vision-toolkit/LICENSE +21 -0
- package/vendor/agent-vision-toolkit/README.md +405 -0
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
- package/vendor/agent-vision-toolkit/bin/crop +90 -0
- package/vendor/agent-vision-toolkit/bin/detect +13 -0
- package/vendor/agent-vision-toolkit/bin/glance +93 -0
- package/vendor/agent-vision-toolkit/bin/ground +13 -0
- package/vendor/agent-vision-toolkit/bin/trace +129 -0
- package/vendor/agent-vision-toolkit/detect.py +60 -0
- package/vendor/agent-vision-toolkit/ground.py +267 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
- package/vendor/agent-vision-toolkit/vision_client.py +310 -0
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,2189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision Toolkit runtime: structured requests in, structured results out.
|
|
3
|
+
* One operation-wide deadline reaches every subprocess; image decoding,
|
|
4
|
+
* byte/pixel limits, session-scoped concurrency, credential resolution, safe
|
|
5
|
+
* output staging, and diagnostic logging stay below the model-facing tools.
|
|
6
|
+
* @module dsh-vision-toolkit/runtime
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
10
|
+
import { lstat, mkdir, readFile, readdir, realpath, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
11
|
+
import { basename, extname, join } from 'node:path'
|
|
12
|
+
import { fileURLToPath } from 'node:url'
|
|
13
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
14
|
+
import type { ResolvedCredential } from '@deepseek-ai/dsh-credentials'
|
|
15
|
+
import { SaxesParser } from 'saxes'
|
|
16
|
+
import { describeArtifact, type ArtifactDescriptor } from './artifacts.ts'
|
|
17
|
+
import { isBuiltInFreeVisionProvider, type ResolvedVisionToolkitConfig } from './config.ts'
|
|
18
|
+
import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
|
|
19
|
+
import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
|
|
20
|
+
import { VisionToolkitError } from './errors.ts'
|
|
21
|
+
import {
|
|
22
|
+
assertDistinctOutput,
|
|
23
|
+
commitStagedDirectory,
|
|
24
|
+
commitStagedOutput,
|
|
25
|
+
createPathPolicy,
|
|
26
|
+
createStagedDirectory,
|
|
27
|
+
createStagedOutput,
|
|
28
|
+
isWithin,
|
|
29
|
+
resolveHtmlFile,
|
|
30
|
+
resolveInputFile,
|
|
31
|
+
resolveOutputDirectory,
|
|
32
|
+
resolveOutputFile,
|
|
33
|
+
seedStagedDirectory,
|
|
34
|
+
type PathPolicy,
|
|
35
|
+
} from './paths.ts'
|
|
36
|
+
import {
|
|
37
|
+
parseCropOutput,
|
|
38
|
+
parseDominantColorsOutput,
|
|
39
|
+
parseExtractForegroundOutput,
|
|
40
|
+
parseHtmlScreenshotOutput,
|
|
41
|
+
parseLocationOutput,
|
|
42
|
+
parsePixelDiffOutput,
|
|
43
|
+
parseTraceOutput,
|
|
44
|
+
UpstreamAdapter,
|
|
45
|
+
type CompressedImageInfo,
|
|
46
|
+
type DominantColorsOutput,
|
|
47
|
+
type LocatedElement,
|
|
48
|
+
type UpstreamEnvironment,
|
|
49
|
+
type UpstreamRunResult,
|
|
50
|
+
type UpstreamTool,
|
|
51
|
+
type UpstreamVersionInfo,
|
|
52
|
+
} from './upstream.ts'
|
|
53
|
+
import { PLUGIN_VERSION } from './version.ts'
|
|
54
|
+
|
|
55
|
+
const SVG_NAMESPACE = 'http://www.w3.org/2000/svg'
|
|
56
|
+
const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-test.png', import.meta.url))
|
|
57
|
+
const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.'
|
|
58
|
+
|
|
59
|
+
/** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
|
|
60
|
+
const COMPRESSED_IMAGE_CACHE_VERSION = 'v2'
|
|
61
|
+
/** Cache keys carry 64-bit digests so Windows paths stay below MAX_PATH; the full file sha256 is computed on read and compared against this prefix. */
|
|
62
|
+
const COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH = 16
|
|
63
|
+
const COMPRESSED_IMAGE_CACHE_MAX_ENTRIES = 200
|
|
64
|
+
const COMPRESSED_IMAGE_CACHE_MAX_BYTES = 512 * 1024 * 1024
|
|
65
|
+
const COMPRESSED_IMAGE_CACHE_STALE_PARTIAL_MS = 60 * 60 * 1000
|
|
66
|
+
|
|
67
|
+
function svgDocumentPathCount(svg: string): number | undefined {
|
|
68
|
+
const parser = new SaxesParser({ xmlns: true })
|
|
69
|
+
let depth = 0
|
|
70
|
+
let invalid = false
|
|
71
|
+
let pathCount = 0
|
|
72
|
+
let rootSeen = false
|
|
73
|
+
let rootClosed = false
|
|
74
|
+
parser.on('doctype', () => { invalid = true })
|
|
75
|
+
parser.on('error', () => { invalid = true })
|
|
76
|
+
parser.on('opentag', (tag) => {
|
|
77
|
+
if (depth === 0) {
|
|
78
|
+
if (rootSeen || tag.local !== 'svg' || tag.uri !== SVG_NAMESPACE) invalid = true
|
|
79
|
+
rootSeen = true
|
|
80
|
+
}
|
|
81
|
+
if (tag.local === 'path' && tag.uri === SVG_NAMESPACE) pathCount += 1
|
|
82
|
+
depth += 1
|
|
83
|
+
})
|
|
84
|
+
parser.on('closetag', () => {
|
|
85
|
+
depth -= 1
|
|
86
|
+
if (depth === 0) rootClosed = true
|
|
87
|
+
if (depth < 0) invalid = true
|
|
88
|
+
})
|
|
89
|
+
try {
|
|
90
|
+
parser.write(svg).close()
|
|
91
|
+
} catch {
|
|
92
|
+
return undefined
|
|
93
|
+
}
|
|
94
|
+
return invalid || !rootSeen || !rootClosed || depth !== 0 ? undefined : pathCount
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Per-invocation cancellation and timeout facts. */
|
|
98
|
+
export interface Deadline {
|
|
99
|
+
signal: AbortSignal
|
|
100
|
+
/** True when the deadline timer fired. */
|
|
101
|
+
timedOut: boolean
|
|
102
|
+
/** True when the caller signal fired first. */
|
|
103
|
+
cancelled: boolean
|
|
104
|
+
/** Clear the timer and caller listener. */
|
|
105
|
+
cleanup(): void
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Combine a caller abort signal with one hard operation timeout. */
|
|
109
|
+
export function createDeadline(signal: AbortSignal, timeoutMs: number): Deadline {
|
|
110
|
+
const controller = new AbortController()
|
|
111
|
+
const state = { timedOut: false, cancelled: false }
|
|
112
|
+
const onCallerAbort = (): void => {
|
|
113
|
+
if (controller.signal.aborted) return
|
|
114
|
+
state.cancelled = true
|
|
115
|
+
controller.abort()
|
|
116
|
+
}
|
|
117
|
+
if (signal.aborted) {
|
|
118
|
+
state.cancelled = true
|
|
119
|
+
controller.abort()
|
|
120
|
+
} else {
|
|
121
|
+
signal.addEventListener('abort', onCallerAbort, { once: true })
|
|
122
|
+
}
|
|
123
|
+
const timer = setTimeout(() => {
|
|
124
|
+
if (controller.signal.aborted) return
|
|
125
|
+
state.timedOut = true
|
|
126
|
+
controller.abort()
|
|
127
|
+
}, timeoutMs)
|
|
128
|
+
return {
|
|
129
|
+
signal: controller.signal,
|
|
130
|
+
get timedOut(): boolean { return state.timedOut },
|
|
131
|
+
get cancelled(): boolean { return state.cancelled },
|
|
132
|
+
cleanup(): void {
|
|
133
|
+
clearTimeout(timer)
|
|
134
|
+
signal.removeEventListener('abort', onCallerAbort)
|
|
135
|
+
},
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** FIFO bounded concurrency gate whose queued callers remain cancellable. */
|
|
140
|
+
export class Semaphore {
|
|
141
|
+
private active = 0
|
|
142
|
+
private readonly waiters: Array<{
|
|
143
|
+
resolve: () => void
|
|
144
|
+
reject: (error: unknown) => void
|
|
145
|
+
signal: AbortSignal
|
|
146
|
+
permits: number
|
|
147
|
+
onAbort: () => void
|
|
148
|
+
}> = []
|
|
149
|
+
|
|
150
|
+
constructor(private readonly limit: number) {}
|
|
151
|
+
|
|
152
|
+
/** Whether no active or queued caller still owns this gate. */
|
|
153
|
+
get idle(): boolean {
|
|
154
|
+
return this.active === 0 && this.waiters.length === 0
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Acquire one slot, aborting while queued when `signal` fires. */
|
|
158
|
+
async acquire(signal: AbortSignal, permits = 1): Promise<void> {
|
|
159
|
+
if (signal.aborted) throw new VisionToolkitError('cancelled', 'vision-toolkit: cancelled before execution')
|
|
160
|
+
if (!Number.isInteger(permits) || permits < 1 || permits > this.limit) {
|
|
161
|
+
throw new VisionToolkitError('input', `concurrency permits must be between 1 and ${this.limit}`)
|
|
162
|
+
}
|
|
163
|
+
if (this.waiters.length === 0 && this.active + permits <= this.limit) {
|
|
164
|
+
this.active += permits
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
return new Promise<void>((resolveAcquire, reject) => {
|
|
168
|
+
const entry = {
|
|
169
|
+
resolve: resolveAcquire,
|
|
170
|
+
reject,
|
|
171
|
+
signal,
|
|
172
|
+
permits,
|
|
173
|
+
onAbort: () => {},
|
|
174
|
+
}
|
|
175
|
+
entry.onAbort = (): void => {
|
|
176
|
+
const index = this.waiters.indexOf(entry)
|
|
177
|
+
if (index >= 0) this.waiters.splice(index, 1)
|
|
178
|
+
reject(new VisionToolkitError('cancelled', 'vision-toolkit: cancelled while waiting for a concurrency slot'))
|
|
179
|
+
}
|
|
180
|
+
this.waiters.push(entry)
|
|
181
|
+
signal.addEventListener('abort', entry.onAbort, { once: true })
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Release owned permits and wake FIFO waiters whose full weight now fits. */
|
|
186
|
+
release(permits = 1): void {
|
|
187
|
+
this.active = Math.max(0, this.active - permits)
|
|
188
|
+
while (this.waiters.length > 0) {
|
|
189
|
+
const next = this.waiters[0]
|
|
190
|
+
if (next === undefined || this.active + next.permits > this.limit) break
|
|
191
|
+
this.waiters.shift()
|
|
192
|
+
next.signal.removeEventListener('abort', next.onAbort)
|
|
193
|
+
this.active += next.permits
|
|
194
|
+
next.resolve()
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Validated image metadata retained in structured results and diagnostics. */
|
|
200
|
+
export interface ImageInfo {
|
|
201
|
+
path: string
|
|
202
|
+
bytes: number
|
|
203
|
+
width: number
|
|
204
|
+
height: number
|
|
205
|
+
format: string
|
|
206
|
+
/** Original user-facing image path before any automatic compression. */
|
|
207
|
+
originalPath: string
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Structured input for one glance call. */
|
|
211
|
+
export interface GlanceRequest {
|
|
212
|
+
images: string[]
|
|
213
|
+
query?: string
|
|
214
|
+
ocr?: boolean
|
|
215
|
+
region?: string
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Structured glance result. */
|
|
219
|
+
export interface GlanceResult {
|
|
220
|
+
images: ImageInfo[]
|
|
221
|
+
mode: 'describe' | 'qa' | 'ocr'
|
|
222
|
+
answer: string
|
|
223
|
+
truncated: boolean
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Structured input for ground/detect. */
|
|
227
|
+
export interface LocateRequest {
|
|
228
|
+
image: string
|
|
229
|
+
target: string
|
|
230
|
+
region?: string
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** One located element with an upstream or caller label. */
|
|
234
|
+
export interface LocateMatch {
|
|
235
|
+
label: string
|
|
236
|
+
box: { x1: number; y1: number; x2: number; y2: number }
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Structured ground result. */
|
|
240
|
+
export interface GroundResult {
|
|
241
|
+
target: string
|
|
242
|
+
image: ImageInfo
|
|
243
|
+
imageWidth: number
|
|
244
|
+
imageHeight: number
|
|
245
|
+
matches: LocateMatch[]
|
|
246
|
+
preview?: ArtifactDescriptor
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Structured detect result. */
|
|
250
|
+
export interface DetectResult {
|
|
251
|
+
category: string
|
|
252
|
+
image: ImageInfo
|
|
253
|
+
imageWidth: number
|
|
254
|
+
imageHeight: number
|
|
255
|
+
elements: Array<{ index: number; label: string; box: { x1: number; y1: number; x2: number; y2: number } }>
|
|
256
|
+
preview?: ArtifactDescriptor
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Structured crop request. */
|
|
260
|
+
export interface CropRequest {
|
|
261
|
+
image: string
|
|
262
|
+
region: string
|
|
263
|
+
scale?: number
|
|
264
|
+
output?: string
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Structured crop result. */
|
|
268
|
+
export interface CropResult {
|
|
269
|
+
imageWidth: number
|
|
270
|
+
imageHeight: number
|
|
271
|
+
region: { x1: number; y1: number; x2: number; y2: number }
|
|
272
|
+
outputPath: string
|
|
273
|
+
mimeType: 'image/png' | 'image/jpeg'
|
|
274
|
+
width: number
|
|
275
|
+
height: number
|
|
276
|
+
clamped: boolean
|
|
277
|
+
artifact: ArtifactDescriptor
|
|
278
|
+
note?: string
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Structured trace request supported by the pinned upstream snapshot. */
|
|
282
|
+
export interface TraceRequest {
|
|
283
|
+
image: string
|
|
284
|
+
region?: string
|
|
285
|
+
scale?: number
|
|
286
|
+
color?: boolean
|
|
287
|
+
polygon?: boolean
|
|
288
|
+
output?: string
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Structured trace result. */
|
|
292
|
+
export interface TraceResult {
|
|
293
|
+
imageWidth: number
|
|
294
|
+
imageHeight: number
|
|
295
|
+
outputPath: string
|
|
296
|
+
mimeType: 'image/svg+xml'
|
|
297
|
+
geometry: {
|
|
298
|
+
status: 'generated' | 'empty'
|
|
299
|
+
pathCount: number
|
|
300
|
+
tracedScale: number
|
|
301
|
+
bytes: number
|
|
302
|
+
}
|
|
303
|
+
artifact: ArtifactDescriptor
|
|
304
|
+
warning?: string
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** Structured input for local image comparison. */
|
|
308
|
+
export interface PixelDiffRequest {
|
|
309
|
+
original: string
|
|
310
|
+
rebuilt: string
|
|
311
|
+
grid?: number
|
|
312
|
+
top?: number
|
|
313
|
+
runName?: string
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** Structured local pixel comparison plus formally delivered files. */
|
|
317
|
+
export interface PixelDiffResult {
|
|
318
|
+
original: ImageInfo
|
|
319
|
+
rebuilt: ImageInfo
|
|
320
|
+
scaled: boolean
|
|
321
|
+
rebuiltOriginalSize?: { width: number; height: number }
|
|
322
|
+
overallDifferencePct: number
|
|
323
|
+
worstRegions: Array<{ index: number; differencePct: number; box: { x1: number; y1: number; x2: number; y2: number } }>
|
|
324
|
+
heatmap: ArtifactDescriptor
|
|
325
|
+
report: ArtifactDescriptor
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Structured input for the pinned long-screenshot OCR pipeline. */
|
|
329
|
+
export interface LongScreenshotOcrRequest {
|
|
330
|
+
image: string
|
|
331
|
+
mode?: 'general' | 'chat'
|
|
332
|
+
output?: string
|
|
333
|
+
runName?: string
|
|
334
|
+
targetHeight?: number
|
|
335
|
+
minHeight?: number
|
|
336
|
+
maxHeight?: number
|
|
337
|
+
overlap?: number
|
|
338
|
+
prompt?: string
|
|
339
|
+
jobs?: number
|
|
340
|
+
chunkTimeoutSeconds?: number
|
|
341
|
+
splitOnly?: boolean
|
|
342
|
+
resume?: boolean
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** One long-OCR chunk and the files retained for audit or reuse. */
|
|
346
|
+
export interface LongScreenshotChunk {
|
|
347
|
+
index: number
|
|
348
|
+
coreTop: number
|
|
349
|
+
coreBottom: number
|
|
350
|
+
cropTop: number
|
|
351
|
+
cropBottom: number
|
|
352
|
+
image: ArtifactDescriptor
|
|
353
|
+
ocr?: ArtifactDescriptor
|
|
354
|
+
reused?: boolean
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/** Long-screenshot split/OCR result with every durable deliverable. */
|
|
358
|
+
export interface LongScreenshotOcrResult {
|
|
359
|
+
source: ImageInfo
|
|
360
|
+
mode: 'general' | 'chat'
|
|
361
|
+
splitOnly: boolean
|
|
362
|
+
complete: boolean
|
|
363
|
+
chunkCount: number
|
|
364
|
+
runDirectory: string
|
|
365
|
+
output?: ArtifactDescriptor
|
|
366
|
+
manifest: ArtifactDescriptor
|
|
367
|
+
audit?: ArtifactDescriptor
|
|
368
|
+
chunks: LongScreenshotChunk[]
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Structured input for transparent foreground extraction. */
|
|
372
|
+
export interface ExtractForegroundRequest {
|
|
373
|
+
image: string
|
|
374
|
+
region?: string
|
|
375
|
+
boxes?: string
|
|
376
|
+
mode?: 'color' | 'dark'
|
|
377
|
+
discRadius?: number
|
|
378
|
+
saturation?: number
|
|
379
|
+
darkThreshold?: number
|
|
380
|
+
excludeColor?: string
|
|
381
|
+
excludeTolerance?: number
|
|
382
|
+
padding?: number
|
|
383
|
+
keepWhites?: boolean
|
|
384
|
+
output?: string
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Transparent foreground file plus the pinned script's component metrics. */
|
|
388
|
+
export interface ExtractForegroundResult {
|
|
389
|
+
source: ImageInfo
|
|
390
|
+
box: { x1: number; y1: number; x2: number; y2: number }
|
|
391
|
+
foregroundPixels: number
|
|
392
|
+
keptComponents: number
|
|
393
|
+
totalComponents: number
|
|
394
|
+
largestComponentPct: number
|
|
395
|
+
width: number
|
|
396
|
+
height: number
|
|
397
|
+
artifact: ArtifactDescriptor
|
|
398
|
+
autoSummary?: string
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Structured input for palette extraction or candidate scoring. */
|
|
402
|
+
export interface DominantColorsRequest {
|
|
403
|
+
image: string
|
|
404
|
+
region?: string
|
|
405
|
+
candidates?: string[]
|
|
406
|
+
top?: number
|
|
407
|
+
quantize?: number
|
|
408
|
+
maxPixels?: number
|
|
409
|
+
mergeTolerance?: number
|
|
410
|
+
candidateTolerance?: number
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Stable dominant-colour result enriched with source image facts. */
|
|
414
|
+
export interface DominantColorsResult {
|
|
415
|
+
image: ImageInfo
|
|
416
|
+
analysis: DominantColorsOutput
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/** Structured input for rendering an authorized local HTML document. */
|
|
420
|
+
export interface HtmlScreenshotRequest {
|
|
421
|
+
source: string
|
|
422
|
+
width?: number
|
|
423
|
+
height?: number
|
|
424
|
+
scale?: number
|
|
425
|
+
waitMs?: number
|
|
426
|
+
fullPage?: boolean
|
|
427
|
+
output?: string
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** Browser-rendered PNG plus viewport and source facts. */
|
|
431
|
+
export interface HtmlScreenshotResult {
|
|
432
|
+
sourcePath: string
|
|
433
|
+
sourceBytes: number
|
|
434
|
+
viewport: { width: number; height: number; scale: number }
|
|
435
|
+
width: number
|
|
436
|
+
height: number
|
|
437
|
+
/** Full document height in CSS pixels; present only for full-page captures. */
|
|
438
|
+
pageHeight?: number
|
|
439
|
+
artifact: ArtifactDescriptor
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** Optional preview controls shared by ground and detect. */
|
|
443
|
+
export interface LocatePreviewRequest extends LocateRequest {
|
|
444
|
+
preview?: boolean
|
|
445
|
+
previewOutput?: string
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/** One named health-check state. */
|
|
449
|
+
export interface HealthCheck {
|
|
450
|
+
status: 'ok' | 'warning' | 'error' | 'not_tested'
|
|
451
|
+
detail: string
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** Runtime, dependency, storage, credential, and optional service health. */
|
|
455
|
+
export interface VisionToolkitHealthResult {
|
|
456
|
+
pluginVersion: string
|
|
457
|
+
upstream: UpstreamVersionInfo
|
|
458
|
+
checks: {
|
|
459
|
+
python: HealthCheck
|
|
460
|
+
dependencies: HealthCheck
|
|
461
|
+
chrome: HealthCheck
|
|
462
|
+
credential: HealthCheck
|
|
463
|
+
artifactDirectory: HealthCheck
|
|
464
|
+
tempDirectory: HealthCheck
|
|
465
|
+
service: HealthCheck
|
|
466
|
+
model: HealthCheck
|
|
467
|
+
}
|
|
468
|
+
healthy: boolean
|
|
469
|
+
connectionTested: boolean
|
|
470
|
+
modelTested: boolean
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Shared per-call execution options. */
|
|
474
|
+
export interface ToolCallOptions {
|
|
475
|
+
signal: AbortSignal
|
|
476
|
+
timeoutMs?: number
|
|
477
|
+
workspace: string
|
|
478
|
+
/** Session identity for the per-session concurrency cap. */
|
|
479
|
+
sessionId?: string
|
|
480
|
+
/** Live Session object whose lifetime bounds the one-entry glance cache. */
|
|
481
|
+
sessionScope?: object
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/** One immutable vision-service snapshot used to generate cache-keyed evidence. */
|
|
485
|
+
export interface CapturedEvidenceRuntime {
|
|
486
|
+
readonly evidenceFingerprint: string
|
|
487
|
+
glance(request: GlanceRequest, options: ToolCallOptions): Promise<GlanceResult>
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
interface GlanceCacheEntry {
|
|
491
|
+
key: string
|
|
492
|
+
result: GlanceResult
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
interface OperationMetrics {
|
|
496
|
+
startedAt: number
|
|
497
|
+
queueMs: number
|
|
498
|
+
upstreamMs: number
|
|
499
|
+
imageBytes: number
|
|
500
|
+
imagePixels: number
|
|
501
|
+
imageCount: number
|
|
502
|
+
cacheHits: number
|
|
503
|
+
usedVisionService: boolean
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
interface OperationContext {
|
|
507
|
+
signal: AbortSignal
|
|
508
|
+
metrics: OperationMetrics
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
const REGION_PATTERN = /^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$/
|
|
512
|
+
const MAX_TIMEOUT_MS = 600_000
|
|
513
|
+
const FORMAT_BY_EXTENSION = new Map([
|
|
514
|
+
['.png', 'png'],
|
|
515
|
+
['.jpg', 'jpeg'],
|
|
516
|
+
['.jpeg', 'jpeg'],
|
|
517
|
+
['.gif', 'gif'],
|
|
518
|
+
['.webp', 'webp'],
|
|
519
|
+
])
|
|
520
|
+
const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
|
|
521
|
+
|
|
522
|
+
function integerInRange(value: number | undefined, fallback: number, minimum: number, maximum: number, name: string): number {
|
|
523
|
+
const resolved = value ?? fallback
|
|
524
|
+
if (!Number.isInteger(resolved) || resolved < minimum || resolved > maximum) {
|
|
525
|
+
throw new VisionToolkitError('input', `${name} must be an integer between ${minimum} and ${maximum}`)
|
|
526
|
+
}
|
|
527
|
+
return resolved
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function finiteInRange(value: number | undefined, minimum: number, maximum: number, name: string): number | undefined {
|
|
531
|
+
if (value === undefined) return undefined
|
|
532
|
+
if (!Number.isFinite(value) || value < minimum || value > maximum) {
|
|
533
|
+
throw new VisionToolkitError('input', `${name} must be between ${minimum} and ${maximum}`)
|
|
534
|
+
}
|
|
535
|
+
return value
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function assertBoxWithin(
|
|
539
|
+
box: { x1: number; y1: number; x2: number; y2: number },
|
|
540
|
+
width: number,
|
|
541
|
+
height: number,
|
|
542
|
+
source: string,
|
|
543
|
+
): void {
|
|
544
|
+
if (
|
|
545
|
+
![box.x1, box.y1, box.x2, box.y2].every(Number.isInteger)
|
|
546
|
+
|| box.x1 < 0
|
|
547
|
+
|| box.y1 < 0
|
|
548
|
+
|| box.x2 <= box.x1
|
|
549
|
+
|| box.y2 <= box.y1
|
|
550
|
+
|| box.x2 > width
|
|
551
|
+
|| box.y2 > height
|
|
552
|
+
) {
|
|
553
|
+
throw new VisionToolkitError('output', `${source} returned an out-of-range box for ${width}x${height}`)
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function safeGeneratedName(name: unknown, source: string): string {
|
|
558
|
+
if (typeof name !== 'string' || name.length === 0 || basename(name) !== name || name === '.' || name === '..') {
|
|
559
|
+
throw new VisionToolkitError('output', `${source} returned an unsafe generated filename`)
|
|
560
|
+
}
|
|
561
|
+
return name
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
interface ParsedLongOcrChunk {
|
|
565
|
+
index: number
|
|
566
|
+
image: string
|
|
567
|
+
imageSha256: string
|
|
568
|
+
coreTop: number
|
|
569
|
+
coreBottom: number
|
|
570
|
+
cropTop: number
|
|
571
|
+
cropBottom: number
|
|
572
|
+
ocr?: string
|
|
573
|
+
ocrReused?: boolean
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
interface ParsedLongOcrManifest {
|
|
577
|
+
mode: 'general' | 'chat'
|
|
578
|
+
complete: boolean
|
|
579
|
+
chunks: ParsedLongOcrChunk[]
|
|
580
|
+
raw: Record<string, unknown>
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
function objectRecord(value: unknown, source: string): Record<string, unknown> {
|
|
584
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
585
|
+
throw new VisionToolkitError('output', `${source} must be a JSON object`)
|
|
586
|
+
}
|
|
587
|
+
return value as Record<string, unknown>
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
function manifestInteger(record: Record<string, unknown>, key: string, source: string): number {
|
|
591
|
+
const value = record[key]
|
|
592
|
+
if (typeof value !== 'number' || !Number.isInteger(value)) {
|
|
593
|
+
throw new VisionToolkitError('output', `${source}.${key} must be an integer`)
|
|
594
|
+
}
|
|
595
|
+
return value
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function parseLongOcrManifest(
|
|
599
|
+
text: string,
|
|
600
|
+
expected: { source: string; output: string; width: number; height: number; mode: 'general' | 'chat'; splitOnly: boolean },
|
|
601
|
+
): ParsedLongOcrManifest {
|
|
602
|
+
let parsed: unknown
|
|
603
|
+
try {
|
|
604
|
+
parsed = JSON.parse(text)
|
|
605
|
+
} catch (error) {
|
|
606
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest is not valid JSON', { cause: error })
|
|
607
|
+
}
|
|
608
|
+
const manifest = objectRecord(parsed, 'long_screenshot_ocr manifest')
|
|
609
|
+
if (manifest.schema_version !== 1) throw new VisionToolkitError('output', 'long_screenshot_ocr: unsupported manifest schema')
|
|
610
|
+
if (manifest.input !== expected.source) throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest source does not match the requested image')
|
|
611
|
+
if (manifestInteger(manifest, 'image_width', 'long_screenshot_ocr manifest') !== expected.width
|
|
612
|
+
|| manifestInteger(manifest, 'image_height', 'long_screenshot_ocr manifest') !== expected.height) {
|
|
613
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest dimensions do not match the source image')
|
|
614
|
+
}
|
|
615
|
+
if (manifest.mode !== expected.mode) throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest mode does not match the request')
|
|
616
|
+
const complete = manifest.complete
|
|
617
|
+
if (typeof complete !== 'boolean' || complete === expected.splitOnly) {
|
|
618
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest completion state is inconsistent')
|
|
619
|
+
}
|
|
620
|
+
if (!Array.isArray(manifest.chunks) || manifest.chunks.length === 0) {
|
|
621
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest contains no chunks')
|
|
622
|
+
}
|
|
623
|
+
if (manifest.output !== (expected.splitOnly ? null : expected.output)) {
|
|
624
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest output path is inconsistent')
|
|
625
|
+
}
|
|
626
|
+
const chunks = manifest.chunks.map((value, position): ParsedLongOcrChunk => {
|
|
627
|
+
const record = objectRecord(value, `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
628
|
+
const index = manifestInteger(record, 'index', `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
629
|
+
if (index !== position + 1) throw new VisionToolkitError('output', 'long_screenshot_ocr: chunk indexes are not contiguous')
|
|
630
|
+
const image = safeGeneratedName(record.image, 'long_screenshot_ocr')
|
|
631
|
+
const imageSha256 = record.image_sha256
|
|
632
|
+
if (typeof imageSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(imageSha256)) {
|
|
633
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: chunk image hash is invalid')
|
|
634
|
+
}
|
|
635
|
+
const coreTop = manifestInteger(record, 'core_top', `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
636
|
+
const coreBottom = manifestInteger(record, 'core_bottom', `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
637
|
+
const cropTop = manifestInteger(record, 'crop_top', `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
638
|
+
const cropBottom = manifestInteger(record, 'crop_bottom', `long_screenshot_ocr manifest.chunks[${position}]`)
|
|
639
|
+
if (
|
|
640
|
+
coreTop < 0
|
|
641
|
+
|| coreBottom <= coreTop
|
|
642
|
+
|| cropTop < 0
|
|
643
|
+
|| cropBottom <= cropTop
|
|
644
|
+
|| cropTop > coreTop
|
|
645
|
+
|| cropBottom < coreBottom
|
|
646
|
+
|| cropBottom > expected.height
|
|
647
|
+
) {
|
|
648
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: manifest contains an invalid chunk range')
|
|
649
|
+
}
|
|
650
|
+
const ocr = record.ocr === undefined ? undefined : safeGeneratedName(record.ocr, 'long_screenshot_ocr')
|
|
651
|
+
const ocrReused = record.ocr_reused
|
|
652
|
+
if (ocrReused !== undefined && typeof ocrReused !== 'boolean') {
|
|
653
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: ocr_reused must be boolean')
|
|
654
|
+
}
|
|
655
|
+
if (complete && ocr === undefined) throw new VisionToolkitError('output', 'long_screenshot_ocr: complete manifest is missing an OCR sidecar')
|
|
656
|
+
return {
|
|
657
|
+
index,
|
|
658
|
+
image,
|
|
659
|
+
imageSha256,
|
|
660
|
+
coreTop,
|
|
661
|
+
coreBottom,
|
|
662
|
+
cropTop,
|
|
663
|
+
cropBottom,
|
|
664
|
+
...(ocr === undefined ? {} : { ocr }),
|
|
665
|
+
...(ocrReused === undefined ? {} : { ocrReused }),
|
|
666
|
+
}
|
|
667
|
+
})
|
|
668
|
+
return { mode: expected.mode, complete, chunks, raw: manifest }
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** Parse a non-empty four-integer pixel box. */
|
|
672
|
+
export function parseRegion(region: string): { x1: number; y1: number; x2: number; y2: number } {
|
|
673
|
+
const match = REGION_PATTERN.exec(region)
|
|
674
|
+
if (match === null) {
|
|
675
|
+
throw new VisionToolkitError('input', 'region must be four integers: X1,Y1,X2,Y2 (pixels)')
|
|
676
|
+
}
|
|
677
|
+
const box = {
|
|
678
|
+
x1: Number(match[1]),
|
|
679
|
+
y1: Number(match[2]),
|
|
680
|
+
x2: Number(match[3]),
|
|
681
|
+
y2: Number(match[4]),
|
|
682
|
+
}
|
|
683
|
+
if (box.x2 <= box.x1 || box.y2 <= box.y1) {
|
|
684
|
+
throw new VisionToolkitError('input', 'region must have x2 > x1 and y2 > y1')
|
|
685
|
+
}
|
|
686
|
+
return box
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/** Runtime facade used by every native tool. */
|
|
690
|
+
export class VisionToolkitRuntime {
|
|
691
|
+
private readonly semaphores = new Map<string, Semaphore>()
|
|
692
|
+
private readonly glanceCache = new WeakMap<object, GlanceCacheEntry>()
|
|
693
|
+
private readonly adapter: UpstreamAdapter
|
|
694
|
+
|
|
695
|
+
constructor(
|
|
696
|
+
private readonly ctx: Context,
|
|
697
|
+
private readonly config: ResolvedVisionToolkitConfig,
|
|
698
|
+
adapter?: UpstreamAdapter,
|
|
699
|
+
) {
|
|
700
|
+
this.adapter = adapter ?? new UpstreamAdapter(ctx, config)
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/** Pinned and prepared upstream identity. */
|
|
704
|
+
get upstreamVersion(): UpstreamVersionInfo {
|
|
705
|
+
return this.adapter.versionInfo
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/** Stable identity for persisted image descriptions produced by this runtime. */
|
|
709
|
+
get evidenceFingerprint(): string {
|
|
710
|
+
return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
/** Capture the credential and provider identity used by one evidence conversion. */
|
|
714
|
+
async captureEvidenceRuntime(): Promise<CapturedEvidenceRuntime> {
|
|
715
|
+
const env = await this.resolveVisionEnv()
|
|
716
|
+
const evidenceFingerprint = evidenceRuntimeFingerprint(
|
|
717
|
+
this.config,
|
|
718
|
+
createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
|
|
719
|
+
env.VISION_SSL_VERIFY,
|
|
720
|
+
)
|
|
721
|
+
return Object.freeze({
|
|
722
|
+
evidenceFingerprint,
|
|
723
|
+
glance: (request: GlanceRequest, options: ToolCallOptions) => this.glanceWithEnv(request, options, env),
|
|
724
|
+
})
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
private timeout(options: ToolCallOptions): number {
|
|
728
|
+
const value = options.timeoutMs ?? this.config.timeoutMs
|
|
729
|
+
if (!Number.isInteger(value) || value < 1000 || value > MAX_TIMEOUT_MS) {
|
|
730
|
+
throw new VisionToolkitError('input', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`)
|
|
731
|
+
}
|
|
732
|
+
return value
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
private operationError(
|
|
736
|
+
tool: string,
|
|
737
|
+
error: unknown,
|
|
738
|
+
deadline: Deadline,
|
|
739
|
+
phase: 'queue' | 'execution' = 'execution',
|
|
740
|
+
): VisionToolkitError {
|
|
741
|
+
if (deadline.cancelled) {
|
|
742
|
+
return new VisionToolkitError(
|
|
743
|
+
'cancelled',
|
|
744
|
+
phase === 'queue' ? `${tool}: cancelled while waiting for a concurrency slot` : `${tool}: cancelled`,
|
|
745
|
+
)
|
|
746
|
+
}
|
|
747
|
+
if (deadline.timedOut) {
|
|
748
|
+
return new VisionToolkitError(
|
|
749
|
+
'timeout',
|
|
750
|
+
phase === 'queue' ? `${tool}: timed out while waiting for a concurrency slot` : `${tool}: timed out`,
|
|
751
|
+
)
|
|
752
|
+
}
|
|
753
|
+
if (error instanceof VisionToolkitError) return error
|
|
754
|
+
return new VisionToolkitError('runtime', `${tool}: execution failed`, { cause: error })
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
private semaphore(options: ToolCallOptions): { key: string; value: Semaphore } {
|
|
758
|
+
const key = options.sessionId ?? `workspace:${options.workspace}`
|
|
759
|
+
const value = this.semaphores.get(key) ?? new Semaphore(this.config.concurrency)
|
|
760
|
+
this.semaphores.set(key, value)
|
|
761
|
+
return { key, value }
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
private async runOperation<T>(
|
|
765
|
+
tool: string,
|
|
766
|
+
options: ToolCallOptions,
|
|
767
|
+
action: (operation: OperationContext) => Promise<T>,
|
|
768
|
+
permits = 1,
|
|
769
|
+
): Promise<T> {
|
|
770
|
+
const timeoutMs = this.timeout(options)
|
|
771
|
+
const semaphore = this.semaphore(options)
|
|
772
|
+
const metrics: OperationMetrics = {
|
|
773
|
+
startedAt: Date.now(),
|
|
774
|
+
queueMs: 0,
|
|
775
|
+
upstreamMs: 0,
|
|
776
|
+
imageBytes: 0,
|
|
777
|
+
imagePixels: 0,
|
|
778
|
+
imageCount: 0,
|
|
779
|
+
cacheHits: 0,
|
|
780
|
+
usedVisionService: false,
|
|
781
|
+
}
|
|
782
|
+
let acquired = false
|
|
783
|
+
const queueDeadline = createDeadline(options.signal, timeoutMs)
|
|
784
|
+
try {
|
|
785
|
+
await semaphore.value.acquire(queueDeadline.signal, permits)
|
|
786
|
+
acquired = true
|
|
787
|
+
if (queueDeadline.signal.aborted) throw this.operationError(tool, undefined, queueDeadline, 'queue')
|
|
788
|
+
metrics.queueMs = Date.now() - metrics.startedAt
|
|
789
|
+
} catch (error) {
|
|
790
|
+
metrics.queueMs = Date.now() - metrics.startedAt
|
|
791
|
+
const classified = this.operationError(tool, error, queueDeadline, 'queue')
|
|
792
|
+
if (acquired) {
|
|
793
|
+
semaphore.value.release(permits)
|
|
794
|
+
acquired = false
|
|
795
|
+
}
|
|
796
|
+
this.ctx.logger.warn(
|
|
797
|
+
'dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d',
|
|
798
|
+
tool,
|
|
799
|
+
classified.code,
|
|
800
|
+
Date.now() - metrics.startedAt,
|
|
801
|
+
metrics.queueMs,
|
|
802
|
+
metrics.upstreamMs,
|
|
803
|
+
metrics.imageCount,
|
|
804
|
+
metrics.imageBytes,
|
|
805
|
+
metrics.imagePixels,
|
|
806
|
+
metrics.cacheHits,
|
|
807
|
+
)
|
|
808
|
+
throw classified
|
|
809
|
+
} finally {
|
|
810
|
+
queueDeadline.cleanup()
|
|
811
|
+
if (!acquired && semaphore.value.idle) this.semaphores.delete(semaphore.key)
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
const executionDeadline = createDeadline(options.signal, timeoutMs)
|
|
815
|
+
try {
|
|
816
|
+
if (executionDeadline.signal.aborted) throw this.operationError(tool, undefined, executionDeadline)
|
|
817
|
+
const value = await action({ signal: executionDeadline.signal, metrics })
|
|
818
|
+
if (executionDeadline.signal.aborted) throw this.operationError(tool, undefined, executionDeadline)
|
|
819
|
+
this.ctx.logger.info(
|
|
820
|
+
'dsh-vision-toolkit tool=%s outcome=ok totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d model=%s',
|
|
821
|
+
tool,
|
|
822
|
+
Date.now() - metrics.startedAt,
|
|
823
|
+
metrics.queueMs,
|
|
824
|
+
metrics.upstreamMs,
|
|
825
|
+
metrics.imageCount,
|
|
826
|
+
metrics.imageBytes,
|
|
827
|
+
metrics.imagePixels,
|
|
828
|
+
metrics.cacheHits,
|
|
829
|
+
metrics.usedVisionService ? this.config.provider.model : 'local',
|
|
830
|
+
)
|
|
831
|
+
return value
|
|
832
|
+
} catch (error) {
|
|
833
|
+
const classified = this.operationError(tool, error, executionDeadline)
|
|
834
|
+
this.ctx.logger.warn(
|
|
835
|
+
'dsh-vision-toolkit tool=%s outcome=error category=%s totalMs=%d queueMs=%d upstreamMs=%d images=%d imageBytes=%d imagePixels=%d cacheHits=%d',
|
|
836
|
+
tool,
|
|
837
|
+
classified.code,
|
|
838
|
+
Date.now() - metrics.startedAt,
|
|
839
|
+
metrics.queueMs,
|
|
840
|
+
metrics.upstreamMs,
|
|
841
|
+
metrics.imageCount,
|
|
842
|
+
metrics.imageBytes,
|
|
843
|
+
metrics.imagePixels,
|
|
844
|
+
metrics.cacheHits,
|
|
845
|
+
)
|
|
846
|
+
throw classified
|
|
847
|
+
} finally {
|
|
848
|
+
if (acquired) semaphore.value.release(permits)
|
|
849
|
+
executionDeadline.cleanup()
|
|
850
|
+
if (semaphore.value.idle) this.semaphores.delete(semaphore.key)
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/** Resolve the configured credential at the remote-operation boundary. */
|
|
855
|
+
async resolveVisionEnv(): Promise<UpstreamEnvironment> {
|
|
856
|
+
const resolved: ResolvedCredential | undefined = isBuiltInFreeVisionProvider(this.config.provider)
|
|
857
|
+
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
858
|
+
: await this.ctx.credentials.resolve(this.config.provider.credential)
|
|
859
|
+
if (resolved === undefined) {
|
|
860
|
+
throw new VisionToolkitError(
|
|
861
|
+
'config',
|
|
862
|
+
`credential ${this.config.provider.credential} is not configured; set it through DSH credentials`,
|
|
863
|
+
)
|
|
864
|
+
}
|
|
865
|
+
return this.visionEnv(resolved)
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
private visionEnv(resolved: ResolvedCredential): UpstreamEnvironment {
|
|
869
|
+
const sslVerify = process.env.VISION_SSL_VERIFY?.trim()
|
|
870
|
+
return {
|
|
871
|
+
VISION_API_KEY: resolved.value,
|
|
872
|
+
VISION_BASE_URL: this.config.provider.baseUrl,
|
|
873
|
+
VISION_MODEL: this.config.provider.model,
|
|
874
|
+
VISION_API_PROTOCOL: this.config.provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
|
|
875
|
+
VISION_ANTHROPIC_THINKING: this.config.provider.anthropicThinking,
|
|
876
|
+
...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
|
|
877
|
+
VISION_USER_AGENT: this.config.provider.userAgent,
|
|
878
|
+
LANG: this.config.language,
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
private pathPolicy(workspace: string): Promise<PathPolicy> {
|
|
883
|
+
return createPathPolicy(workspace, this.config.allowedDirs)
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
private async compressedImageRoot(policy: PathPolicy): Promise<string> {
|
|
887
|
+
const root = join(policy.workspace, '.dsh-vision-toolkit', 'tmp', 'compressed-images')
|
|
888
|
+
let current = policy.workspace
|
|
889
|
+
for (const segment of ['.dsh-vision-toolkit', 'tmp', 'compressed-images']) {
|
|
890
|
+
current = join(current, segment)
|
|
891
|
+
try {
|
|
892
|
+
await mkdir(current, { mode: 0o700 })
|
|
893
|
+
} catch (error) {
|
|
894
|
+
if (!(error instanceof Error && 'code' in error && error.code === 'EEXIST')) throw error
|
|
895
|
+
}
|
|
896
|
+
const info = await lstat(current)
|
|
897
|
+
if (info.isSymbolicLink() || !info.isDirectory()) {
|
|
898
|
+
throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`)
|
|
899
|
+
}
|
|
900
|
+
if (!isWithin(policy.workspace, current)) {
|
|
901
|
+
throw new VisionToolkitError('path', `compressed-image cache path escaped the workspace: ${current}`)
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
const canonical = await realpath(root)
|
|
905
|
+
if (!isWithin(policy.workspace, canonical)) {
|
|
906
|
+
throw new VisionToolkitError('path', 'compressed-image cache resolved outside the workspace')
|
|
907
|
+
}
|
|
908
|
+
return canonical
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
private async readCacheCandidate(
|
|
912
|
+
root: string,
|
|
913
|
+
name: string,
|
|
914
|
+
expectedOutDigestPrefix: string,
|
|
915
|
+
maxBytes: number,
|
|
916
|
+
maxPixels: number,
|
|
917
|
+
operation: OperationContext,
|
|
918
|
+
): Promise<{ path: string; bytes: number; width: number; height: number; format: string } | undefined> {
|
|
919
|
+
const candidate = join(root, name)
|
|
920
|
+
let info
|
|
921
|
+
try {
|
|
922
|
+
info = await lstat(candidate)
|
|
923
|
+
} catch {
|
|
924
|
+
return undefined
|
|
925
|
+
}
|
|
926
|
+
if (!info.isFile() || info.size < 1 || info.size > maxBytes) return undefined
|
|
927
|
+
let real: string
|
|
928
|
+
try {
|
|
929
|
+
real = await realpath(candidate)
|
|
930
|
+
} catch {
|
|
931
|
+
return undefined
|
|
932
|
+
}
|
|
933
|
+
if (!isWithin(root, real)) return undefined
|
|
934
|
+
let bytes: Buffer
|
|
935
|
+
try {
|
|
936
|
+
bytes = await readFile(real, { signal: operation.signal })
|
|
937
|
+
} catch {
|
|
938
|
+
return undefined
|
|
939
|
+
}
|
|
940
|
+
const digest = createHash('sha256').update(bytes).digest('hex')
|
|
941
|
+
if (bytes.length !== info.size || !digest.startsWith(expectedOutDigestPrefix)) return undefined
|
|
942
|
+
let probed: { width: number; height: number; format: string } | undefined
|
|
943
|
+
try {
|
|
944
|
+
probed = await this.adapter.probeImageSize(real, { signal: operation.signal })
|
|
945
|
+
} catch {
|
|
946
|
+
probed = undefined
|
|
947
|
+
}
|
|
948
|
+
const extension = extname(real).toLowerCase()
|
|
949
|
+
if (
|
|
950
|
+
probed === undefined
|
|
951
|
+
|| FORMAT_BY_EXTENSION.get(extension) !== probed.format
|
|
952
|
+
|| probed.width * probed.height > maxPixels
|
|
953
|
+
) {
|
|
954
|
+
return undefined
|
|
955
|
+
}
|
|
956
|
+
return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format }
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
private cacheEntryOutDigest(entry: string, prefix: string): string | undefined {
|
|
960
|
+
const tail = entry.slice(prefix.length + 1)
|
|
961
|
+
return /^[0-9a-f]{16}-/u.test(tail) ? tail.slice(0, 16) : undefined
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
private async pruneCompressedCache(root: string): Promise<void> {
|
|
965
|
+
let entries: string[]
|
|
966
|
+
try {
|
|
967
|
+
entries = await readdir(root)
|
|
968
|
+
} catch {
|
|
969
|
+
return
|
|
970
|
+
}
|
|
971
|
+
const stalePartials: string[] = []
|
|
972
|
+
const candidates: Array<{ name: string; size: number; mtime: number; removable: boolean }> = []
|
|
973
|
+
for (const name of entries) {
|
|
974
|
+
if (name.startsWith('.')) {
|
|
975
|
+
if (name.endsWith('.partial')) {
|
|
976
|
+
const info = await lstat(join(root, name)).catch(() => undefined)
|
|
977
|
+
if (info !== undefined && Date.now() - info.mtimeMs > COMPRESSED_IMAGE_CACHE_STALE_PARTIAL_MS) {
|
|
978
|
+
stalePartials.push(name)
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
continue
|
|
982
|
+
}
|
|
983
|
+
let info
|
|
984
|
+
try {
|
|
985
|
+
info = await lstat(join(root, name))
|
|
986
|
+
} catch {
|
|
987
|
+
continue
|
|
988
|
+
}
|
|
989
|
+
candidates.push({
|
|
990
|
+
name,
|
|
991
|
+
size: info.isFile() ? info.size : 0,
|
|
992
|
+
mtime: info.mtimeMs,
|
|
993
|
+
removable: !info.isFile() || !name.startsWith(`${COMPRESSED_IMAGE_CACHE_VERSION}-`),
|
|
994
|
+
})
|
|
995
|
+
}
|
|
996
|
+
candidates.sort((a, b) => a.mtime - b.mtime)
|
|
997
|
+
let totalBytes = 0
|
|
998
|
+
let kept = 0
|
|
999
|
+
const remove: string[] = []
|
|
1000
|
+
for (const candidate of candidates) {
|
|
1001
|
+
if (
|
|
1002
|
+
candidate.removable
|
|
1003
|
+
|| totalBytes + candidate.size > COMPRESSED_IMAGE_CACHE_MAX_BYTES
|
|
1004
|
+
|| kept >= COMPRESSED_IMAGE_CACHE_MAX_ENTRIES
|
|
1005
|
+
) {
|
|
1006
|
+
remove.push(candidate.name)
|
|
1007
|
+
} else {
|
|
1008
|
+
totalBytes += candidate.size
|
|
1009
|
+
kept += 1
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
await Promise.all([...stalePartials, ...remove].map(name => rm(join(root, name), { force: true }).catch(() => {})))
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
private async autoCompressImage(
|
|
1016
|
+
image: { path: string; bytes: number },
|
|
1017
|
+
policy: PathPolicy,
|
|
1018
|
+
operation: OperationContext,
|
|
1019
|
+
): Promise<ImageInfo> {
|
|
1020
|
+
let bytes: Buffer
|
|
1021
|
+
try {
|
|
1022
|
+
bytes = await readFile(image.path, { signal: operation.signal })
|
|
1023
|
+
} catch (error) {
|
|
1024
|
+
throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`, { cause: error })
|
|
1025
|
+
}
|
|
1026
|
+
if (bytes.length !== image.bytes) {
|
|
1027
|
+
throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`)
|
|
1028
|
+
}
|
|
1029
|
+
const digest = createHash('sha256').update(bytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
|
|
1030
|
+
const root = await this.compressedImageRoot(policy)
|
|
1031
|
+
await this.pruneCompressedCache(root)
|
|
1032
|
+
const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${this.config.maxImageBytes}-p${this.config.maxImagePixels}`
|
|
1033
|
+
for (const entry of await readdir(root)) {
|
|
1034
|
+
if (!entry.startsWith(`${prefix}-`) || entry.startsWith('.')) continue
|
|
1035
|
+
const outDigestPrefix = this.cacheEntryOutDigest(entry, prefix)
|
|
1036
|
+
if (outDigestPrefix === undefined) {
|
|
1037
|
+
await rm(join(root, entry), { force: true }).catch(() => {})
|
|
1038
|
+
continue
|
|
1039
|
+
}
|
|
1040
|
+
const cached = await this.readCacheCandidate(
|
|
1041
|
+
root,
|
|
1042
|
+
entry,
|
|
1043
|
+
outDigestPrefix,
|
|
1044
|
+
this.config.maxImageBytes,
|
|
1045
|
+
this.config.maxImagePixels,
|
|
1046
|
+
operation,
|
|
1047
|
+
)
|
|
1048
|
+
if (cached !== undefined) {
|
|
1049
|
+
return { ...cached, originalPath: image.path }
|
|
1050
|
+
}
|
|
1051
|
+
await rm(join(root, entry), { force: true }).catch(() => {})
|
|
1052
|
+
}
|
|
1053
|
+
const staged = join(root, `.${prefix}-${randomUUID()}.partial`)
|
|
1054
|
+
let compressed: CompressedImageInfo
|
|
1055
|
+
try {
|
|
1056
|
+
compressed = await this.adapter.compressImage(
|
|
1057
|
+
image.path,
|
|
1058
|
+
staged,
|
|
1059
|
+
this.config.maxImageBytes,
|
|
1060
|
+
this.config.maxImagePixels,
|
|
1061
|
+
{ signal: operation.signal },
|
|
1062
|
+
)
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1065
|
+
throw error
|
|
1066
|
+
}
|
|
1067
|
+
const extension = compressed.format === 'jpeg' ? 'jpg' : compressed.format
|
|
1068
|
+
const stagedBytes = await readFile(staged, { signal: operation.signal })
|
|
1069
|
+
const outDigest = createHash('sha256').update(stagedBytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
|
|
1070
|
+
const finalName = `${prefix}-${outDigest}-${compressed.width}x${compressed.height}.${extension}`
|
|
1071
|
+
const finalPath = join(root, finalName)
|
|
1072
|
+
const existing = await this.readCacheCandidate(
|
|
1073
|
+
root,
|
|
1074
|
+
finalName,
|
|
1075
|
+
outDigest,
|
|
1076
|
+
this.config.maxImageBytes,
|
|
1077
|
+
this.config.maxImagePixels,
|
|
1078
|
+
operation,
|
|
1079
|
+
)
|
|
1080
|
+
if (existing !== undefined) {
|
|
1081
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1082
|
+
return { ...existing, originalPath: image.path }
|
|
1083
|
+
}
|
|
1084
|
+
await rm(finalPath, { force: true }).catch(() => {})
|
|
1085
|
+
try {
|
|
1086
|
+
await rename(staged, finalPath)
|
|
1087
|
+
} catch (error) {
|
|
1088
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1089
|
+
throw new VisionToolkitError('path', `cannot commit compressed image cache entry: ${finalPath}`, { cause: error })
|
|
1090
|
+
}
|
|
1091
|
+
await this.pruneCompressedCache(root)
|
|
1092
|
+
return {
|
|
1093
|
+
path: finalPath,
|
|
1094
|
+
bytes: compressed.bytes,
|
|
1095
|
+
width: compressed.width,
|
|
1096
|
+
height: compressed.height,
|
|
1097
|
+
format: compressed.format,
|
|
1098
|
+
originalPath: image.path,
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
private async validateImage(raw: string, policy: PathPolicy, operation: OperationContext): Promise<ImageInfo> {
|
|
1103
|
+
const image = await resolveInputFile(raw, policy)
|
|
1104
|
+
const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal })
|
|
1105
|
+
const pixels = decoded.width * decoded.height
|
|
1106
|
+
if (!Number.isSafeInteger(pixels) || pixels < 1) {
|
|
1107
|
+
throw new VisionToolkitError('input', `image dimensions are invalid: ${decoded.width}x${decoded.height}`)
|
|
1108
|
+
}
|
|
1109
|
+
const extension = extname(image.path).toLowerCase()
|
|
1110
|
+
const expected = FORMAT_BY_EXTENSION.get(extension)
|
|
1111
|
+
if (expected !== decoded.format) {
|
|
1112
|
+
throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`)
|
|
1113
|
+
}
|
|
1114
|
+
if (image.bytes <= this.config.maxImageBytes && pixels <= this.config.maxImagePixels) {
|
|
1115
|
+
return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, originalPath: image.path }
|
|
1116
|
+
}
|
|
1117
|
+
return this.autoCompressImage(image, policy, operation)
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
private accountImage(image: ImageInfo, operation: OperationContext): void {
|
|
1121
|
+
operation.metrics.imageCount += 1
|
|
1122
|
+
operation.metrics.imageBytes += image.bytes
|
|
1123
|
+
operation.metrics.imagePixels += image.width * image.height
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
private async glanceCacheKey(
|
|
1127
|
+
request: GlanceRequest,
|
|
1128
|
+
images: readonly ImageInfo[],
|
|
1129
|
+
env: UpstreamEnvironment,
|
|
1130
|
+
signal: AbortSignal,
|
|
1131
|
+
): Promise<string> {
|
|
1132
|
+
const imageFingerprints = await Promise.all(images.map(async (image) => {
|
|
1133
|
+
let bytes: Buffer
|
|
1134
|
+
try {
|
|
1135
|
+
bytes = await readFile(image.path, { signal })
|
|
1136
|
+
} catch (error) {
|
|
1137
|
+
throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`, { cause: error })
|
|
1138
|
+
}
|
|
1139
|
+
if (bytes.length !== image.bytes) {
|
|
1140
|
+
throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`)
|
|
1141
|
+
}
|
|
1142
|
+
return {
|
|
1143
|
+
path: image.path,
|
|
1144
|
+
sha256: createHash('sha256').update(bytes).digest('hex'),
|
|
1145
|
+
}
|
|
1146
|
+
}))
|
|
1147
|
+
return JSON.stringify({
|
|
1148
|
+
images: imageFingerprints,
|
|
1149
|
+
query: request.query ?? null,
|
|
1150
|
+
ocr: request.ocr === true,
|
|
1151
|
+
region: request.region ?? null,
|
|
1152
|
+
provider: {
|
|
1153
|
+
baseUrl: env.VISION_BASE_URL,
|
|
1154
|
+
model: env.VISION_MODEL,
|
|
1155
|
+
protocol: env.VISION_API_PROTOCOL,
|
|
1156
|
+
anthropicThinking: env.VISION_ANTHROPIC_THINKING,
|
|
1157
|
+
sslVerify: env.VISION_SSL_VERIFY ?? null,
|
|
1158
|
+
userAgent: env.VISION_USER_AGENT,
|
|
1159
|
+
language: env.LANG,
|
|
1160
|
+
credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
|
|
1161
|
+
},
|
|
1162
|
+
})
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
private async runUpstream(
|
|
1166
|
+
tool: UpstreamTool,
|
|
1167
|
+
args: readonly string[],
|
|
1168
|
+
operation: OperationContext,
|
|
1169
|
+
env?: UpstreamEnvironment,
|
|
1170
|
+
): Promise<UpstreamRunResult> {
|
|
1171
|
+
const started = Date.now()
|
|
1172
|
+
if (env !== undefined) operation.metrics.usedVisionService = true
|
|
1173
|
+
const result = await this.adapter.run(tool, args, {
|
|
1174
|
+
signal: operation.signal,
|
|
1175
|
+
...(env === undefined ? {} : { env }),
|
|
1176
|
+
})
|
|
1177
|
+
operation.metrics.upstreamMs += Date.now() - started
|
|
1178
|
+
if (result.outcome.exitCode !== 0) {
|
|
1179
|
+
throw this.adapter.classifyFailure(tool, result, {
|
|
1180
|
+
timedOut: false,
|
|
1181
|
+
cancelled: operation.signal.aborted,
|
|
1182
|
+
...(env === undefined ? {} : { secrets: [env.VISION_API_KEY] }),
|
|
1183
|
+
})
|
|
1184
|
+
}
|
|
1185
|
+
if (result.stdoutTruncated || result.stderrTruncated) {
|
|
1186
|
+
throw new VisionToolkitError('output', `${tool}: upstream output exceeded the capture limit`)
|
|
1187
|
+
}
|
|
1188
|
+
return result
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
private async probeGeneratedImage(
|
|
1192
|
+
path: string,
|
|
1193
|
+
operation: OperationContext,
|
|
1194
|
+
source: string,
|
|
1195
|
+
): Promise<{ width: number; height: number; format: string; mode: string }> {
|
|
1196
|
+
try {
|
|
1197
|
+
return await this.adapter.probeImageSize(path, { signal: operation.signal })
|
|
1198
|
+
} catch (error) {
|
|
1199
|
+
if (operation.signal.aborted) throw error
|
|
1200
|
+
throw new VisionToolkitError('output', `${source}: generated image is missing, corrupt, or unsupported`, { cause: error })
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
private async annotateLocations(
|
|
1205
|
+
tool: 'vision_ground' | 'vision_detect',
|
|
1206
|
+
image: ImageInfo,
|
|
1207
|
+
elements: readonly LocatedElement[],
|
|
1208
|
+
output: string | undefined,
|
|
1209
|
+
policy: PathPolicy,
|
|
1210
|
+
operation: OperationContext,
|
|
1211
|
+
): Promise<ArtifactDescriptor> {
|
|
1212
|
+
const extension = extname(image.originalPath).toLowerCase()
|
|
1213
|
+
const stem = basename(image.originalPath, extension)
|
|
1214
|
+
const suffix = tool === 'vision_ground' ? 'ground' : 'detect'
|
|
1215
|
+
const finalPath = resolveOutputFile(output, policy, `${stem}.${suffix}.preview.png`, ['.png'])
|
|
1216
|
+
assertDistinctOutput(image.path, finalPath)
|
|
1217
|
+
assertDistinctOutput(image.originalPath, finalPath)
|
|
1218
|
+
const staged = createStagedOutput(policy, '.png')
|
|
1219
|
+
try {
|
|
1220
|
+
const started = Date.now()
|
|
1221
|
+
await this.adapter.renderAnnotatedPreview(image.path, staged, elements, { signal: operation.signal })
|
|
1222
|
+
operation.metrics.upstreamMs += Date.now() - started
|
|
1223
|
+
const preview = await this.probeGeneratedImage(staged, operation, tool)
|
|
1224
|
+
if (preview.format !== 'png' || preview.width !== image.width || preview.height !== image.height) {
|
|
1225
|
+
throw new VisionToolkitError('output', `${tool}: annotation preview dimensions or format do not match the source image`)
|
|
1226
|
+
}
|
|
1227
|
+
await commitStagedOutput(staged, finalPath, policy)
|
|
1228
|
+
return describeArtifact(finalPath, policy, {
|
|
1229
|
+
mimeType: 'image/png',
|
|
1230
|
+
kind: 'image',
|
|
1231
|
+
description: tool === 'vision_ground' ? 'Grounding bounding-box preview' : 'Detected-element bounding-box preview',
|
|
1232
|
+
sourceTool: tool,
|
|
1233
|
+
previewIntent: 'image',
|
|
1234
|
+
})
|
|
1235
|
+
} finally {
|
|
1236
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/** glance: describe, targeted QA, OCR, or multi-image comparison. */
|
|
1241
|
+
async glance(request: GlanceRequest, options: ToolCallOptions): Promise<GlanceResult> {
|
|
1242
|
+
return this.glanceWithEnv(request, options)
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
private async glanceWithEnv(
|
|
1246
|
+
request: GlanceRequest,
|
|
1247
|
+
options: ToolCallOptions,
|
|
1248
|
+
capturedEnv?: UpstreamEnvironment,
|
|
1249
|
+
): Promise<GlanceResult> {
|
|
1250
|
+
return this.runOperation('vision_glance', options, async (operation) => {
|
|
1251
|
+
if (request.images.length === 0) throw new VisionToolkitError('input', 'glance requires at least one image')
|
|
1252
|
+
if (request.query !== undefined && request.ocr === true) {
|
|
1253
|
+
throw new VisionToolkitError('input', 'glance: query and ocr are mutually exclusive')
|
|
1254
|
+
}
|
|
1255
|
+
if (request.region !== undefined && request.images.length > 1) {
|
|
1256
|
+
throw new VisionToolkitError('input', 'glance: region works with exactly one image')
|
|
1257
|
+
}
|
|
1258
|
+
if (request.region !== undefined) parseRegion(request.region)
|
|
1259
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1260
|
+
const images: ImageInfo[] = []
|
|
1261
|
+
const seen = new Set<string>()
|
|
1262
|
+
for (const raw of request.images) {
|
|
1263
|
+
const image = await this.validateImage(raw, policy, operation)
|
|
1264
|
+
if (seen.has(image.path)) {
|
|
1265
|
+
operation.metrics.cacheHits += 1
|
|
1266
|
+
continue
|
|
1267
|
+
}
|
|
1268
|
+
seen.add(image.path)
|
|
1269
|
+
this.accountImage(image, operation)
|
|
1270
|
+
images.push(image)
|
|
1271
|
+
}
|
|
1272
|
+
const env = capturedEnv ?? await this.resolveVisionEnv()
|
|
1273
|
+
const cacheKey = options.sessionScope === undefined
|
|
1274
|
+
? undefined
|
|
1275
|
+
: await this.glanceCacheKey(request, images, env, operation.signal)
|
|
1276
|
+
if (options.sessionScope !== undefined && cacheKey !== undefined) {
|
|
1277
|
+
const cached = this.glanceCache.get(options.sessionScope)
|
|
1278
|
+
if (cached?.key === cacheKey) {
|
|
1279
|
+
operation.metrics.cacheHits += 1
|
|
1280
|
+
return cached.result
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
const result = await this.runUpstream('glance', [
|
|
1284
|
+
...images.map(image => image.path),
|
|
1285
|
+
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
1286
|
+
...(request.ocr === true ? ['--ocr'] : []),
|
|
1287
|
+
...(request.query !== undefined ? ['-q', request.query] : []),
|
|
1288
|
+
], operation, env)
|
|
1289
|
+
const answer = result.stdout.trim()
|
|
1290
|
+
if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
|
|
1291
|
+
const value: GlanceResult = {
|
|
1292
|
+
images,
|
|
1293
|
+
mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
|
|
1294
|
+
answer,
|
|
1295
|
+
truncated: false,
|
|
1296
|
+
}
|
|
1297
|
+
if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
|
|
1298
|
+
this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
|
|
1299
|
+
}
|
|
1300
|
+
return value
|
|
1301
|
+
})
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
private validateLocations(elements: LocatedElement[], width: number, height: number): void {
|
|
1305
|
+
for (const element of elements) {
|
|
1306
|
+
const { x1, y1, x2, y2 } = element.box
|
|
1307
|
+
if (
|
|
1308
|
+
![x1, y1, x2, y2].every(Number.isInteger)
|
|
1309
|
+
|| x1 < 0
|
|
1310
|
+
|| y1 < 0
|
|
1311
|
+
|| x2 <= x1
|
|
1312
|
+
|| y2 <= y1
|
|
1313
|
+
|| x2 > width
|
|
1314
|
+
|| y2 > height
|
|
1315
|
+
) {
|
|
1316
|
+
throw new VisionToolkitError('output', `upstream returned an out-of-range box for ${width}x${height}`)
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
private async locate(
|
|
1322
|
+
request: LocateRequest,
|
|
1323
|
+
options: ToolCallOptions,
|
|
1324
|
+
operation: OperationContext,
|
|
1325
|
+
tool: 'ground' | 'detect',
|
|
1326
|
+
): Promise<{ image: ImageInfo; elements: LocatedElement[] }> {
|
|
1327
|
+
if (request.target.trim().length === 0) throw new VisionToolkitError('input', 'target must not be empty')
|
|
1328
|
+
if (request.region !== undefined) parseRegion(request.region)
|
|
1329
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1330
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1331
|
+
this.accountImage(image, operation)
|
|
1332
|
+
const env = await this.resolveVisionEnv()
|
|
1333
|
+
const result = await this.runUpstream(tool, [
|
|
1334
|
+
image.path,
|
|
1335
|
+
request.target,
|
|
1336
|
+
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
1337
|
+
], operation, env)
|
|
1338
|
+
const elements = parseLocationOutput(result.stdout)
|
|
1339
|
+
this.validateLocations(elements, image.width, image.height)
|
|
1340
|
+
return { image, elements }
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
/** ground: locate one named target and return pixel boxes. */
|
|
1344
|
+
async ground(request: LocatePreviewRequest, options: ToolCallOptions): Promise<GroundResult> {
|
|
1345
|
+
return this.runOperation('vision_ground', options, async (operation) => {
|
|
1346
|
+
const { image, elements } = await this.locate(request, options, operation, 'ground')
|
|
1347
|
+
const labeled = elements.map(element => ({ label: element.label ?? request.target, box: element.box }))
|
|
1348
|
+
const preview = request.preview === true
|
|
1349
|
+
? await this.annotateLocations('vision_ground', image, labeled, request.previewOutput, await this.pathPolicy(options.workspace), operation)
|
|
1350
|
+
: undefined
|
|
1351
|
+
return {
|
|
1352
|
+
target: request.target,
|
|
1353
|
+
image,
|
|
1354
|
+
imageWidth: image.width,
|
|
1355
|
+
imageHeight: image.height,
|
|
1356
|
+
matches: labeled,
|
|
1357
|
+
...(preview === undefined ? {} : { preview }),
|
|
1358
|
+
}
|
|
1359
|
+
})
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
/** detect: inventory every instance of a kind. */
|
|
1363
|
+
async detect(request: LocatePreviewRequest, options: ToolCallOptions): Promise<DetectResult> {
|
|
1364
|
+
return this.runOperation('vision_detect', options, async (operation) => {
|
|
1365
|
+
const { image, elements } = await this.locate(request, options, operation, 'detect')
|
|
1366
|
+
const labeled = elements.map(element => ({ label: element.label ?? request.target, box: element.box }))
|
|
1367
|
+
const preview = request.preview === true
|
|
1368
|
+
? await this.annotateLocations('vision_detect', image, labeled, request.previewOutput, await this.pathPolicy(options.workspace), operation)
|
|
1369
|
+
: undefined
|
|
1370
|
+
return {
|
|
1371
|
+
category: request.target,
|
|
1372
|
+
image,
|
|
1373
|
+
imageWidth: image.width,
|
|
1374
|
+
imageHeight: image.height,
|
|
1375
|
+
elements: labeled.map((element, index) => ({
|
|
1376
|
+
index: index + 1,
|
|
1377
|
+
label: element.label,
|
|
1378
|
+
box: element.box,
|
|
1379
|
+
})),
|
|
1380
|
+
...(preview === undefined ? {} : { preview }),
|
|
1381
|
+
}
|
|
1382
|
+
})
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
/** crop: cut a pixel box into its own image file without requiring a credential. */
|
|
1386
|
+
async crop(request: CropRequest, options: ToolCallOptions): Promise<CropResult> {
|
|
1387
|
+
return this.runOperation('vision_crop', options, async (operation) => {
|
|
1388
|
+
const region = parseRegion(request.region)
|
|
1389
|
+
if (request.scale !== undefined && (!Number.isInteger(request.scale) || request.scale < 1 || request.scale > 8)) {
|
|
1390
|
+
throw new VisionToolkitError('input', 'crop: scale must be an integer between 1 and 8')
|
|
1391
|
+
}
|
|
1392
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1393
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1394
|
+
this.accountImage(image, operation)
|
|
1395
|
+
const sourceExtension = extname(image.originalPath).toLowerCase()
|
|
1396
|
+
const stem = basename(image.originalPath, sourceExtension)
|
|
1397
|
+
const finalPath = resolveOutputFile(
|
|
1398
|
+
request.output,
|
|
1399
|
+
policy,
|
|
1400
|
+
request.scale !== undefined && request.scale > 1 ? `${stem}.crop@${request.scale}x.png` : `${stem}.crop.png`,
|
|
1401
|
+
['.png', '.jpg', '.jpeg'],
|
|
1402
|
+
)
|
|
1403
|
+
assertDistinctOutput(image.path, finalPath)
|
|
1404
|
+
assertDistinctOutput(image.originalPath, finalPath)
|
|
1405
|
+
const outputExtension = extname(finalPath).toLowerCase()
|
|
1406
|
+
const staged = createStagedOutput(policy, outputExtension)
|
|
1407
|
+
try {
|
|
1408
|
+
const result = await this.runUpstream('crop', [
|
|
1409
|
+
image.path,
|
|
1410
|
+
'--region',
|
|
1411
|
+
request.region,
|
|
1412
|
+
'-o',
|
|
1413
|
+
staged,
|
|
1414
|
+
...(request.scale !== undefined ? ['--scale', String(request.scale)] : []),
|
|
1415
|
+
], operation)
|
|
1416
|
+
const parsed = parseCropOutput(result.stdout, result.stderr)
|
|
1417
|
+
const generated = await this.probeGeneratedImage(staged, operation, 'crop')
|
|
1418
|
+
const expectedFormat = outputExtension === '.png' ? 'png' : 'jpeg'
|
|
1419
|
+
if (
|
|
1420
|
+
generated.format !== expectedFormat
|
|
1421
|
+
|| generated.width !== parsed.width
|
|
1422
|
+
|| generated.height !== parsed.height
|
|
1423
|
+
) {
|
|
1424
|
+
throw new VisionToolkitError('output', 'crop: generated image does not match the upstream summary')
|
|
1425
|
+
}
|
|
1426
|
+
await commitStagedOutput(staged, finalPath, policy)
|
|
1427
|
+
const mimeType = outputExtension === '.png' ? 'image/png' : 'image/jpeg'
|
|
1428
|
+
const artifact = await describeArtifact(finalPath, policy, {
|
|
1429
|
+
mimeType,
|
|
1430
|
+
kind: 'image',
|
|
1431
|
+
description: 'Cropped image region',
|
|
1432
|
+
sourceTool: 'vision_crop',
|
|
1433
|
+
previewIntent: 'image',
|
|
1434
|
+
})
|
|
1435
|
+
return {
|
|
1436
|
+
imageWidth: image.width,
|
|
1437
|
+
imageHeight: image.height,
|
|
1438
|
+
region,
|
|
1439
|
+
outputPath: finalPath,
|
|
1440
|
+
mimeType,
|
|
1441
|
+
width: parsed.width,
|
|
1442
|
+
height: parsed.height,
|
|
1443
|
+
clamped: parsed.clamped,
|
|
1444
|
+
artifact,
|
|
1445
|
+
...(parsed.note === undefined ? {} : { note: parsed.note }),
|
|
1446
|
+
}
|
|
1447
|
+
} finally {
|
|
1448
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1449
|
+
}
|
|
1450
|
+
})
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
/** trace: recover an SVG through the pinned upstream vtracer pipeline. */
|
|
1454
|
+
async trace(request: TraceRequest, options: ToolCallOptions): Promise<TraceResult> {
|
|
1455
|
+
return this.runOperation('vision_trace', options, async (operation) => {
|
|
1456
|
+
if (request.region !== undefined) parseRegion(request.region)
|
|
1457
|
+
if (request.scale !== undefined && (!Number.isInteger(request.scale) || request.scale < 1 || request.scale > 16)) {
|
|
1458
|
+
throw new VisionToolkitError('input', 'trace: scale must be an integer between 1 and 16')
|
|
1459
|
+
}
|
|
1460
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1461
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1462
|
+
this.accountImage(image, operation)
|
|
1463
|
+
const extension = extname(image.originalPath).toLowerCase()
|
|
1464
|
+
const stem = basename(image.originalPath, extension)
|
|
1465
|
+
const finalPath = resolveOutputFile(request.output, policy, `${stem}.svg`, ['.svg'])
|
|
1466
|
+
assertDistinctOutput(image.path, finalPath)
|
|
1467
|
+
assertDistinctOutput(image.originalPath, finalPath)
|
|
1468
|
+
const staged = createStagedOutput(policy, '.svg')
|
|
1469
|
+
try {
|
|
1470
|
+
const result = await this.runUpstream('trace', [
|
|
1471
|
+
image.path,
|
|
1472
|
+
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
1473
|
+
...(request.scale !== undefined ? ['--scale', String(request.scale)] : []),
|
|
1474
|
+
...(request.polygon === true ? ['--polygon'] : []),
|
|
1475
|
+
...(request.color === true ? ['--color'] : []),
|
|
1476
|
+
'-o',
|
|
1477
|
+
staged,
|
|
1478
|
+
], operation)
|
|
1479
|
+
const parsed = parseTraceOutput(result.stdout)
|
|
1480
|
+
const svg = await readFile(staged, 'utf8').catch(() => '')
|
|
1481
|
+
const actualPathCount = svgDocumentPathCount(svg)
|
|
1482
|
+
if (actualPathCount === undefined) {
|
|
1483
|
+
throw new VisionToolkitError('output', 'trace: output SVG is not a parseable document')
|
|
1484
|
+
}
|
|
1485
|
+
if (actualPathCount !== parsed.pathCount) {
|
|
1486
|
+
throw new VisionToolkitError('output', 'trace: reported path count does not match the generated SVG')
|
|
1487
|
+
}
|
|
1488
|
+
const svgInfo = await stat(staged)
|
|
1489
|
+
if (svgInfo.size !== parsed.bytes) {
|
|
1490
|
+
throw new VisionToolkitError('output', 'trace: reported byte count does not match the generated SVG')
|
|
1491
|
+
}
|
|
1492
|
+
await commitStagedOutput(staged, finalPath, policy)
|
|
1493
|
+
const artifact = await describeArtifact(finalPath, policy, {
|
|
1494
|
+
mimeType: 'image/svg+xml',
|
|
1495
|
+
kind: 'svg',
|
|
1496
|
+
description: 'Traced vector geometry',
|
|
1497
|
+
sourceTool: 'vision_trace',
|
|
1498
|
+
previewIntent: 'svg',
|
|
1499
|
+
})
|
|
1500
|
+
const warning = result.stderr.trim()
|
|
1501
|
+
return {
|
|
1502
|
+
imageWidth: image.width,
|
|
1503
|
+
imageHeight: image.height,
|
|
1504
|
+
outputPath: finalPath,
|
|
1505
|
+
mimeType: 'image/svg+xml',
|
|
1506
|
+
geometry: {
|
|
1507
|
+
status: parsed.pathCount === 0 ? 'empty' : 'generated',
|
|
1508
|
+
pathCount: parsed.pathCount,
|
|
1509
|
+
tracedScale: parsed.tracedScale,
|
|
1510
|
+
bytes: parsed.bytes,
|
|
1511
|
+
},
|
|
1512
|
+
artifact,
|
|
1513
|
+
...(warning.length === 0 ? {} : { warning: warning.split(/\r?\n/).slice(-1)[0] ?? warning }),
|
|
1514
|
+
}
|
|
1515
|
+
} finally {
|
|
1516
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1517
|
+
}
|
|
1518
|
+
})
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
/** pixel_diff: compare real pixels, rank error regions, and deliver a heatmap plus JSON report. */
|
|
1522
|
+
async pixelDiff(request: PixelDiffRequest, options: ToolCallOptions): Promise<PixelDiffResult> {
|
|
1523
|
+
return this.runOperation('vision_pixel_diff', options, async (operation) => {
|
|
1524
|
+
const grid = integerInRange(request.grid, 6, 1, 32, 'pixel_diff.grid')
|
|
1525
|
+
const top = integerInRange(request.top, 5, 1, grid * grid, 'pixel_diff.top')
|
|
1526
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1527
|
+
const original = await this.validateImage(request.original, policy, operation)
|
|
1528
|
+
const rebuilt = await this.validateImage(request.rebuilt, policy, operation)
|
|
1529
|
+
this.accountImage(original, operation)
|
|
1530
|
+
this.accountImage(rebuilt, operation)
|
|
1531
|
+
const originalStem = basename(original.originalPath, extname(original.originalPath))
|
|
1532
|
+
const rebuiltStem = basename(rebuilt.originalPath, extname(rebuilt.originalPath))
|
|
1533
|
+
const finalDirectory = resolveOutputDirectory(
|
|
1534
|
+
request.runName,
|
|
1535
|
+
policy,
|
|
1536
|
+
`${originalStem}-vs-${rebuiltStem}.pixel-diff`,
|
|
1537
|
+
)
|
|
1538
|
+
if (
|
|
1539
|
+
isWithin(finalDirectory, original.path)
|
|
1540
|
+
|| isWithin(finalDirectory, rebuilt.path)
|
|
1541
|
+
|| isWithin(finalDirectory, original.originalPath)
|
|
1542
|
+
|| isWithin(finalDirectory, rebuilt.originalPath)
|
|
1543
|
+
) {
|
|
1544
|
+
throw new VisionToolkitError('input', 'pixel_diff artifact directory would replace an input image')
|
|
1545
|
+
}
|
|
1546
|
+
const stagedDirectory = await createStagedDirectory(policy)
|
|
1547
|
+
const stagedHeatmap = join(stagedDirectory, 'heatmap.png')
|
|
1548
|
+
const stagedReport = join(stagedDirectory, 'report.json')
|
|
1549
|
+
try {
|
|
1550
|
+
const result = await this.runUpstream('pixel_diff', [
|
|
1551
|
+
original.path,
|
|
1552
|
+
rebuilt.path,
|
|
1553
|
+
'--grid',
|
|
1554
|
+
String(grid),
|
|
1555
|
+
'--top',
|
|
1556
|
+
String(top),
|
|
1557
|
+
'-o',
|
|
1558
|
+
stagedHeatmap,
|
|
1559
|
+
], operation)
|
|
1560
|
+
const parsed = parsePixelDiffOutput(result.stdout)
|
|
1561
|
+
if (parsed.heatmapPath !== stagedHeatmap) {
|
|
1562
|
+
throw new VisionToolkitError('output', 'pixel_diff: upstream reported an unexpected heatmap path')
|
|
1563
|
+
}
|
|
1564
|
+
if (!Number.isFinite(parsed.overallDifferencePct) || parsed.overallDifferencePct < 0 || parsed.overallDifferencePct > 100) {
|
|
1565
|
+
throw new VisionToolkitError('output', 'pixel_diff: overall difference is outside 0-100%')
|
|
1566
|
+
}
|
|
1567
|
+
if (
|
|
1568
|
+
parsed.scaled !== (original.width !== rebuilt.width || original.height !== rebuilt.height)
|
|
1569
|
+
|| (parsed.scaledToSize !== undefined
|
|
1570
|
+
&& (parsed.scaledToSize.width !== original.width || parsed.scaledToSize.height !== original.height))
|
|
1571
|
+
|| parsed.worstRegions.length > top
|
|
1572
|
+
|| parsed.worstRegions.some((region, index) => region.index !== index + 1)
|
|
1573
|
+
) {
|
|
1574
|
+
throw new VisionToolkitError('output', 'pixel_diff: scaling or ranked-region metadata is inconsistent')
|
|
1575
|
+
}
|
|
1576
|
+
for (const region of parsed.worstRegions) assertBoxWithin(region.box, original.width, original.height, 'pixel_diff')
|
|
1577
|
+
const heatmapInfo = await this.probeGeneratedImage(stagedHeatmap, operation, 'pixel_diff')
|
|
1578
|
+
if (heatmapInfo.format !== 'png' || heatmapInfo.width !== original.width || heatmapInfo.height !== original.height) {
|
|
1579
|
+
throw new VisionToolkitError('output', 'pixel_diff: heatmap dimensions or format do not match the reference image')
|
|
1580
|
+
}
|
|
1581
|
+
const reportPayload = {
|
|
1582
|
+
schemaVersion: 1,
|
|
1583
|
+
sourceTool: 'vision_pixel_diff',
|
|
1584
|
+
original,
|
|
1585
|
+
rebuilt,
|
|
1586
|
+
scaled: parsed.scaled,
|
|
1587
|
+
...(parsed.rebuiltOriginalSize === undefined ? {} : { rebuiltOriginalSize: parsed.rebuiltOriginalSize }),
|
|
1588
|
+
overallDifferencePct: parsed.overallDifferencePct,
|
|
1589
|
+
grid,
|
|
1590
|
+
worstRegions: parsed.worstRegions,
|
|
1591
|
+
}
|
|
1592
|
+
await writeFile(stagedReport, `${JSON.stringify(reportPayload, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' })
|
|
1593
|
+
await commitStagedDirectory(stagedDirectory, finalDirectory, policy)
|
|
1594
|
+
const heatmapPath = join(finalDirectory, 'heatmap.png')
|
|
1595
|
+
const reportPath = join(finalDirectory, 'report.json')
|
|
1596
|
+
const [heatmap, report] = await Promise.all([
|
|
1597
|
+
describeArtifact(heatmapPath, policy, {
|
|
1598
|
+
mimeType: 'image/png',
|
|
1599
|
+
kind: 'image',
|
|
1600
|
+
description: 'Pixel-difference heatmap',
|
|
1601
|
+
sourceTool: 'vision_pixel_diff',
|
|
1602
|
+
previewIntent: 'image',
|
|
1603
|
+
}),
|
|
1604
|
+
describeArtifact(reportPath, policy, {
|
|
1605
|
+
mimeType: 'application/json',
|
|
1606
|
+
kind: 'json',
|
|
1607
|
+
description: 'Structured pixel-difference report',
|
|
1608
|
+
sourceTool: 'vision_pixel_diff',
|
|
1609
|
+
previewIntent: 'text',
|
|
1610
|
+
}),
|
|
1611
|
+
])
|
|
1612
|
+
return {
|
|
1613
|
+
original,
|
|
1614
|
+
rebuilt,
|
|
1615
|
+
scaled: parsed.scaled,
|
|
1616
|
+
...(parsed.rebuiltOriginalSize === undefined ? {} : { rebuiltOriginalSize: parsed.rebuiltOriginalSize }),
|
|
1617
|
+
overallDifferencePct: parsed.overallDifferencePct,
|
|
1618
|
+
worstRegions: parsed.worstRegions,
|
|
1619
|
+
heatmap,
|
|
1620
|
+
report,
|
|
1621
|
+
}
|
|
1622
|
+
} finally {
|
|
1623
|
+
await rm(stagedDirectory, { recursive: true, force: true }).catch(() => {})
|
|
1624
|
+
}
|
|
1625
|
+
})
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
/** long_screenshot_ocr: split safely, optionally OCR, and atomically deliver the complete audit run. */
|
|
1629
|
+
async longScreenshotOcr(request: LongScreenshotOcrRequest, options: ToolCallOptions): Promise<LongScreenshotOcrResult> {
|
|
1630
|
+
const jobs = integerInRange(request.jobs, Math.min(2, this.config.concurrency), 1, this.config.concurrency, 'long_screenshot_ocr.jobs')
|
|
1631
|
+
const splitOnly = request.splitOnly === true
|
|
1632
|
+
const permits = splitOnly ? 1 : jobs
|
|
1633
|
+
return this.runOperation('vision_long_screenshot_ocr', options, async (operation) => {
|
|
1634
|
+
const mode = request.mode ?? 'general'
|
|
1635
|
+
if (mode !== 'general' && mode !== 'chat') throw new VisionToolkitError('input', 'long_screenshot_ocr.mode must be general or chat')
|
|
1636
|
+
const targetHeight = request.targetHeight === undefined
|
|
1637
|
+
? undefined
|
|
1638
|
+
: integerInRange(request.targetHeight, request.targetHeight, 64, 100000, 'long_screenshot_ocr.targetHeight')
|
|
1639
|
+
const minHeight = request.minHeight === undefined
|
|
1640
|
+
? undefined
|
|
1641
|
+
: integerInRange(request.minHeight, request.minHeight, 64, 100000, 'long_screenshot_ocr.minHeight')
|
|
1642
|
+
const maxHeight = request.maxHeight === undefined
|
|
1643
|
+
? undefined
|
|
1644
|
+
: integerInRange(request.maxHeight, request.maxHeight, 64, 100000, 'long_screenshot_ocr.maxHeight')
|
|
1645
|
+
if (minHeight !== undefined && maxHeight !== undefined && minHeight > maxHeight) {
|
|
1646
|
+
throw new VisionToolkitError('input', 'long_screenshot_ocr.minHeight must not exceed maxHeight')
|
|
1647
|
+
}
|
|
1648
|
+
if (targetHeight !== undefined && minHeight !== undefined && targetHeight < minHeight) {
|
|
1649
|
+
throw new VisionToolkitError('input', 'long_screenshot_ocr.targetHeight must not be below minHeight')
|
|
1650
|
+
}
|
|
1651
|
+
if (targetHeight !== undefined && maxHeight !== undefined && targetHeight > maxHeight) {
|
|
1652
|
+
throw new VisionToolkitError('input', 'long_screenshot_ocr.targetHeight must not exceed maxHeight')
|
|
1653
|
+
}
|
|
1654
|
+
const overlap = request.overlap === undefined
|
|
1655
|
+
? undefined
|
|
1656
|
+
: integerInRange(request.overlap, request.overlap, 0, 10000, 'long_screenshot_ocr.overlap')
|
|
1657
|
+
const chunkTimeoutSeconds = finiteInRange(
|
|
1658
|
+
request.chunkTimeoutSeconds ?? Math.min(180, Math.max(1, Math.ceil(this.timeout(options) / 1000))),
|
|
1659
|
+
1,
|
|
1660
|
+
600,
|
|
1661
|
+
'long_screenshot_ocr.chunkTimeoutSeconds',
|
|
1662
|
+
)
|
|
1663
|
+
if (chunkTimeoutSeconds === undefined) throw new VisionToolkitError('input', 'long_screenshot_ocr chunk timeout is required')
|
|
1664
|
+
if (request.prompt !== undefined && request.prompt.trim().length === 0) {
|
|
1665
|
+
throw new VisionToolkitError('input', 'long_screenshot_ocr.prompt must not be empty when provided')
|
|
1666
|
+
}
|
|
1667
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1668
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1669
|
+
this.accountImage(image, operation)
|
|
1670
|
+
const stem = basename(image.originalPath, extname(image.originalPath))
|
|
1671
|
+
const finalDirectory = resolveOutputDirectory(request.runName, policy, `${stem}.long-ocr`)
|
|
1672
|
+
if (isWithin(finalDirectory, image.path) || isWithin(finalDirectory, image.originalPath)) {
|
|
1673
|
+
throw new VisionToolkitError('input', 'long_screenshot_ocr artifact directory would replace the input image')
|
|
1674
|
+
}
|
|
1675
|
+
const stagedDirectory = await createStagedDirectory(policy)
|
|
1676
|
+
try {
|
|
1677
|
+
if (request.resume === true) await seedStagedDirectory(finalDirectory, stagedDirectory, policy)
|
|
1678
|
+
const stagedPolicy = { ...policy, outputDir: stagedDirectory }
|
|
1679
|
+
const stagedOutput = resolveOutputFile(request.output, stagedPolicy, `${stem}.ocr.md`, ['.md', '.markdown'])
|
|
1680
|
+
const finalOutput = join(finalDirectory, basename(stagedOutput))
|
|
1681
|
+
const stagedChunks = join(stagedDirectory, 'chunks')
|
|
1682
|
+
const stagedManifest = join(stagedChunks, 'manifest.json')
|
|
1683
|
+
const result = await this.runUpstream('long_screenshot_ocr', [
|
|
1684
|
+
image.path,
|
|
1685
|
+
'--mode',
|
|
1686
|
+
mode,
|
|
1687
|
+
'-o',
|
|
1688
|
+
stagedOutput,
|
|
1689
|
+
'--chunks-dir',
|
|
1690
|
+
stagedChunks,
|
|
1691
|
+
...(targetHeight === undefined ? [] : ['--target-height', String(targetHeight)]),
|
|
1692
|
+
...(minHeight === undefined ? [] : ['--min-height', String(minHeight)]),
|
|
1693
|
+
...(maxHeight === undefined ? [] : ['--max-height', String(maxHeight)]),
|
|
1694
|
+
...(overlap === undefined ? [] : ['--overlap', String(overlap)]),
|
|
1695
|
+
...(request.prompt === undefined ? [] : ['--prompt', request.prompt]),
|
|
1696
|
+
'--jobs',
|
|
1697
|
+
String(jobs),
|
|
1698
|
+
'--timeout',
|
|
1699
|
+
String(chunkTimeoutSeconds),
|
|
1700
|
+
...(splitOnly ? ['--split-only'] : []),
|
|
1701
|
+
...(request.resume === true ? ['--resume'] : []),
|
|
1702
|
+
], operation, splitOnly ? undefined : await this.resolveVisionEnv())
|
|
1703
|
+
const reported = result.stdout.trim()
|
|
1704
|
+
const expectedReported = splitOnly ? stagedManifest : stagedOutput
|
|
1705
|
+
if (reported !== expectedReported) {
|
|
1706
|
+
throw new VisionToolkitError('output', 'long_screenshot_ocr: upstream reported an unexpected output path')
|
|
1707
|
+
}
|
|
1708
|
+
const parsedManifest = parseLongOcrManifest(await readFile(stagedManifest, 'utf8'), {
|
|
1709
|
+
source: image.path,
|
|
1710
|
+
output: stagedOutput,
|
|
1711
|
+
width: image.width,
|
|
1712
|
+
height: image.height,
|
|
1713
|
+
mode,
|
|
1714
|
+
splitOnly,
|
|
1715
|
+
})
|
|
1716
|
+
for (const chunk of parsedManifest.chunks) {
|
|
1717
|
+
const chunkBytes = await readFile(join(stagedChunks, chunk.image))
|
|
1718
|
+
if (createHash('sha256').update(chunkBytes).digest('hex') !== chunk.imageSha256) {
|
|
1719
|
+
throw new VisionToolkitError('output', `long_screenshot_ocr: chunk ${chunk.index} hash does not match the manifest`)
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
parsedManifest.raw.output = splitOnly ? null : finalOutput
|
|
1723
|
+
await writeFile(stagedManifest, `${JSON.stringify(parsedManifest.raw, null, 2)}\n`, 'utf8')
|
|
1724
|
+
await commitStagedDirectory(stagedDirectory, finalDirectory, policy)
|
|
1725
|
+
const finalChunks = join(finalDirectory, 'chunks')
|
|
1726
|
+
const manifest = await describeArtifact(join(finalChunks, 'manifest.json'), policy, {
|
|
1727
|
+
mimeType: 'application/json',
|
|
1728
|
+
kind: 'json',
|
|
1729
|
+
description: 'Long-screenshot split and merge manifest',
|
|
1730
|
+
sourceTool: 'vision_long_screenshot_ocr',
|
|
1731
|
+
previewIntent: 'text',
|
|
1732
|
+
})
|
|
1733
|
+
const output = splitOnly
|
|
1734
|
+
? undefined
|
|
1735
|
+
: await describeArtifact(finalOutput, policy, {
|
|
1736
|
+
mimeType: 'text/markdown',
|
|
1737
|
+
kind: 'markdown',
|
|
1738
|
+
description: 'Merged long-screenshot OCR transcript',
|
|
1739
|
+
sourceTool: 'vision_long_screenshot_ocr',
|
|
1740
|
+
previewIntent: 'text',
|
|
1741
|
+
})
|
|
1742
|
+
const audit = splitOnly
|
|
1743
|
+
? undefined
|
|
1744
|
+
: await describeArtifact(join(finalChunks, 'ocr_audit.md'), policy, {
|
|
1745
|
+
mimeType: 'text/markdown',
|
|
1746
|
+
kind: 'markdown',
|
|
1747
|
+
description: 'Long-screenshot OCR boundary audit',
|
|
1748
|
+
sourceTool: 'vision_long_screenshot_ocr',
|
|
1749
|
+
previewIntent: 'text',
|
|
1750
|
+
})
|
|
1751
|
+
const chunks: LongScreenshotChunk[] = []
|
|
1752
|
+
for (const chunk of parsedManifest.chunks) {
|
|
1753
|
+
const chunkImage = await describeArtifact(join(finalChunks, chunk.image), policy, {
|
|
1754
|
+
mimeType: 'image/png',
|
|
1755
|
+
kind: 'image',
|
|
1756
|
+
description: `Long-screenshot OCR chunk ${chunk.index}`,
|
|
1757
|
+
sourceTool: 'vision_long_screenshot_ocr',
|
|
1758
|
+
previewIntent: 'image',
|
|
1759
|
+
})
|
|
1760
|
+
const ocr = chunk.ocr === undefined
|
|
1761
|
+
? undefined
|
|
1762
|
+
: await describeArtifact(join(finalChunks, chunk.ocr), policy, {
|
|
1763
|
+
mimeType: mode === 'chat' ? 'application/json' : 'text/markdown',
|
|
1764
|
+
kind: mode === 'chat' ? 'json' : 'markdown',
|
|
1765
|
+
description: `OCR sidecar for chunk ${chunk.index}`,
|
|
1766
|
+
sourceTool: 'vision_long_screenshot_ocr',
|
|
1767
|
+
previewIntent: 'text',
|
|
1768
|
+
})
|
|
1769
|
+
chunks.push({
|
|
1770
|
+
index: chunk.index,
|
|
1771
|
+
coreTop: chunk.coreTop,
|
|
1772
|
+
coreBottom: chunk.coreBottom,
|
|
1773
|
+
cropTop: chunk.cropTop,
|
|
1774
|
+
cropBottom: chunk.cropBottom,
|
|
1775
|
+
image: chunkImage,
|
|
1776
|
+
...(ocr === undefined ? {} : { ocr }),
|
|
1777
|
+
...(chunk.ocrReused === undefined ? {} : { reused: chunk.ocrReused }),
|
|
1778
|
+
})
|
|
1779
|
+
}
|
|
1780
|
+
return {
|
|
1781
|
+
source: image,
|
|
1782
|
+
mode,
|
|
1783
|
+
splitOnly,
|
|
1784
|
+
complete: parsedManifest.complete,
|
|
1785
|
+
chunkCount: chunks.length,
|
|
1786
|
+
runDirectory: finalDirectory,
|
|
1787
|
+
...(output === undefined ? {} : { output }),
|
|
1788
|
+
manifest,
|
|
1789
|
+
...(audit === undefined ? {} : { audit }),
|
|
1790
|
+
chunks,
|
|
1791
|
+
}
|
|
1792
|
+
} finally {
|
|
1793
|
+
await rm(stagedDirectory, { recursive: true, force: true }).catch(() => {})
|
|
1794
|
+
}
|
|
1795
|
+
}, permits)
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
/** extract_foreground: preserve the pinned component selection and deliver an RGBA PNG. */
|
|
1799
|
+
async extractForeground(request: ExtractForegroundRequest, options: ToolCallOptions): Promise<ExtractForegroundResult> {
|
|
1800
|
+
return this.runOperation('vision_extract_foreground', options, async (operation) => {
|
|
1801
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1802
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1803
|
+
this.accountImage(image, operation)
|
|
1804
|
+
const region = request.region === undefined ? undefined : parseRegion(request.region)
|
|
1805
|
+
if (region !== undefined) assertBoxWithin(region, image.width, image.height, 'extract_foreground.region')
|
|
1806
|
+
const boxes = request.boxes === undefined ? undefined : parseRegion(request.boxes)
|
|
1807
|
+
if (boxes !== undefined) assertBoxWithin(boxes, image.width, image.height, 'extract_foreground.boxes')
|
|
1808
|
+
const discRadius = finiteInRange(request.discRadius, 1, Math.max(image.width, image.height) * 4, 'extract_foreground.discRadius')
|
|
1809
|
+
const saturation = integerInRange(request.saturation, 12, 0, 255, 'extract_foreground.saturation')
|
|
1810
|
+
const darkThreshold = integerInRange(request.darkThreshold, 215, 0, 255, 'extract_foreground.darkThreshold')
|
|
1811
|
+
const excludeTolerance = finiteInRange(request.excludeTolerance ?? 24, 0, 442, 'extract_foreground.excludeTolerance')
|
|
1812
|
+
const padding = integerInRange(request.padding, 3, 0, 4096, 'extract_foreground.padding')
|
|
1813
|
+
const excludeColor = request.excludeColor === undefined
|
|
1814
|
+
? undefined
|
|
1815
|
+
: `#${request.excludeColor.trim().replace(/^#/, '').toUpperCase()}`
|
|
1816
|
+
if (excludeColor !== undefined && !HEX_COLOR_PATTERN.test(excludeColor)) {
|
|
1817
|
+
throw new VisionToolkitError('input', 'extract_foreground.excludeColor must be #RRGGBB')
|
|
1818
|
+
}
|
|
1819
|
+
const extension = extname(image.originalPath).toLowerCase()
|
|
1820
|
+
const stem = basename(image.originalPath, extension)
|
|
1821
|
+
const finalPath = resolveOutputFile(request.output, policy, `${stem}.foreground.png`, ['.png'])
|
|
1822
|
+
assertDistinctOutput(image.path, finalPath)
|
|
1823
|
+
assertDistinctOutput(image.originalPath, finalPath)
|
|
1824
|
+
const staged = createStagedOutput(policy, '.png')
|
|
1825
|
+
try {
|
|
1826
|
+
const result = await this.runUpstream('extract_foreground', [
|
|
1827
|
+
image.path,
|
|
1828
|
+
...(region === undefined ? [] : ['--region', `${region.x1},${region.y1},${region.x2},${region.y2}`]),
|
|
1829
|
+
...(boxes === undefined ? [] : ['--boxes', `${boxes.x1},${boxes.y1},${boxes.x2},${boxes.y2}`]),
|
|
1830
|
+
'--mode',
|
|
1831
|
+
request.mode ?? 'color',
|
|
1832
|
+
'--sat',
|
|
1833
|
+
String(saturation),
|
|
1834
|
+
'--dark',
|
|
1835
|
+
String(darkThreshold),
|
|
1836
|
+
'--exclude-tol',
|
|
1837
|
+
String(excludeTolerance),
|
|
1838
|
+
'--pad',
|
|
1839
|
+
String(padding),
|
|
1840
|
+
...(discRadius === undefined ? [] : ['--disc-radius', String(discRadius)]),
|
|
1841
|
+
...(excludeColor === undefined ? [] : ['--exclude-color', excludeColor]),
|
|
1842
|
+
...(request.keepWhites === false ? ['--no-keep-whites'] : []),
|
|
1843
|
+
'-o',
|
|
1844
|
+
staged,
|
|
1845
|
+
], operation)
|
|
1846
|
+
const parsed = parseExtractForegroundOutput(result.stdout)
|
|
1847
|
+
if (parsed.outputPath !== staged) throw new VisionToolkitError('output', 'extract_foreground: upstream reported an unexpected output path')
|
|
1848
|
+
assertBoxWithin(parsed.box, image.width, image.height, 'extract_foreground')
|
|
1849
|
+
if (
|
|
1850
|
+
parsed.foregroundPixels <= 0
|
|
1851
|
+
|| parsed.keptComponents <= 0
|
|
1852
|
+
|| parsed.totalComponents < parsed.keptComponents
|
|
1853
|
+
|| parsed.largestComponentPct < 0
|
|
1854
|
+
|| parsed.largestComponentPct > 100
|
|
1855
|
+
) {
|
|
1856
|
+
throw new VisionToolkitError('output', 'extract_foreground: component metrics are invalid')
|
|
1857
|
+
}
|
|
1858
|
+
const generated = await this.probeGeneratedImage(staged, operation, 'extract_foreground')
|
|
1859
|
+
if (
|
|
1860
|
+
generated.format !== 'png'
|
|
1861
|
+
|| (generated.mode !== 'RGBA' && generated.mode !== 'LA')
|
|
1862
|
+
|| generated.width !== parsed.width
|
|
1863
|
+
|| generated.height !== parsed.height
|
|
1864
|
+
) {
|
|
1865
|
+
throw new VisionToolkitError('output', 'extract_foreground: output is not the reported transparent PNG')
|
|
1866
|
+
}
|
|
1867
|
+
await commitStagedOutput(staged, finalPath, policy)
|
|
1868
|
+
const artifact = await describeArtifact(finalPath, policy, {
|
|
1869
|
+
mimeType: 'image/png',
|
|
1870
|
+
kind: 'image',
|
|
1871
|
+
description: 'Extracted transparent foreground',
|
|
1872
|
+
sourceTool: 'vision_extract_foreground',
|
|
1873
|
+
previewIntent: 'image',
|
|
1874
|
+
})
|
|
1875
|
+
return {
|
|
1876
|
+
source: image,
|
|
1877
|
+
box: parsed.box,
|
|
1878
|
+
foregroundPixels: parsed.foregroundPixels,
|
|
1879
|
+
keptComponents: parsed.keptComponents,
|
|
1880
|
+
totalComponents: parsed.totalComponents,
|
|
1881
|
+
largestComponentPct: parsed.largestComponentPct,
|
|
1882
|
+
width: parsed.width,
|
|
1883
|
+
height: parsed.height,
|
|
1884
|
+
artifact,
|
|
1885
|
+
...(parsed.autoSummary === undefined ? {} : { autoSummary: parsed.autoSummary }),
|
|
1886
|
+
}
|
|
1887
|
+
} finally {
|
|
1888
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
1889
|
+
}
|
|
1890
|
+
})
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
/** dominant_colors: expose palette clusters or candidate scores as structure, never stdout prose. */
|
|
1894
|
+
async dominantColors(request: DominantColorsRequest, options: ToolCallOptions): Promise<DominantColorsResult> {
|
|
1895
|
+
return this.runOperation('vision_dominant_colors', options, async (operation) => {
|
|
1896
|
+
const top = integerInRange(request.top, 5, 1, 64, 'dominant_colors.top')
|
|
1897
|
+
const quantize = integerInRange(request.quantize, 16, 2, 256, 'dominant_colors.quantize')
|
|
1898
|
+
const maxPixels = integerInRange(request.maxPixels, 96, 8, 4096, 'dominant_colors.maxPixels')
|
|
1899
|
+
const mergeTolerance = integerInRange(request.mergeTolerance, 8, 0, 255, 'dominant_colors.mergeTolerance')
|
|
1900
|
+
const candidateTolerance = integerInRange(request.candidateTolerance, 16, 0, 255, 'dominant_colors.candidateTolerance')
|
|
1901
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1902
|
+
const image = await this.validateImage(request.image, policy, operation)
|
|
1903
|
+
this.accountImage(image, operation)
|
|
1904
|
+
const region = request.region === undefined ? undefined : parseRegion(request.region)
|
|
1905
|
+
if (region !== undefined) assertBoxWithin(region, image.width, image.height, 'dominant_colors.region')
|
|
1906
|
+
const candidates = request.candidates?.map(value => `#${value.trim().replace(/^#/, '').toUpperCase()}`)
|
|
1907
|
+
if (candidates !== undefined) {
|
|
1908
|
+
if (candidates.length === 0 || candidates.length > 32) {
|
|
1909
|
+
throw new VisionToolkitError('input', 'dominant_colors.candidates must contain between 1 and 32 colors')
|
|
1910
|
+
}
|
|
1911
|
+
if (candidates.some(candidate => !HEX_COLOR_PATTERN.test(candidate))) {
|
|
1912
|
+
throw new VisionToolkitError('input', 'dominant_colors.candidates must contain only #RRGGBB colors')
|
|
1913
|
+
}
|
|
1914
|
+
if (new Set(candidates).size !== candidates.length) {
|
|
1915
|
+
throw new VisionToolkitError('input', 'dominant_colors.candidates must not contain duplicates')
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
const result = await this.runUpstream('dominant_colors', [
|
|
1919
|
+
image.path,
|
|
1920
|
+
...(region === undefined ? [] : ['--region', `${region.x1},${region.y1},${region.x2},${region.y2}`]),
|
|
1921
|
+
...(candidates === undefined ? [] : ['--candidates', candidates.join(',')]),
|
|
1922
|
+
'--top',
|
|
1923
|
+
String(top),
|
|
1924
|
+
'--quantize',
|
|
1925
|
+
String(quantize),
|
|
1926
|
+
'--max-pixels',
|
|
1927
|
+
String(maxPixels),
|
|
1928
|
+
'--merge-tol',
|
|
1929
|
+
String(mergeTolerance),
|
|
1930
|
+
'--tol',
|
|
1931
|
+
String(candidateTolerance),
|
|
1932
|
+
], operation)
|
|
1933
|
+
const analysis = parseDominantColorsOutput(result.stdout)
|
|
1934
|
+
assertBoxWithin(analysis.region, image.width, image.height, 'dominant_colors')
|
|
1935
|
+
if (analysis.width !== analysis.region.x2 - analysis.region.x1 || analysis.height !== analysis.region.y2 - analysis.region.y1) {
|
|
1936
|
+
throw new VisionToolkitError('output', 'dominant_colors: reported region dimensions are inconsistent')
|
|
1937
|
+
}
|
|
1938
|
+
if (candidates !== undefined) {
|
|
1939
|
+
if (analysis.mode !== 'candidates') throw new VisionToolkitError('output', 'dominant_colors: expected candidate mode output')
|
|
1940
|
+
if (analysis.candidates.map(candidate => candidate.color).join(',') !== candidates.join(',')) {
|
|
1941
|
+
throw new VisionToolkitError('output', 'dominant_colors: candidate rows do not match the request')
|
|
1942
|
+
}
|
|
1943
|
+
} else if (analysis.mode !== 'palette') {
|
|
1944
|
+
throw new VisionToolkitError('output', 'dominant_colors: expected palette mode output')
|
|
1945
|
+
}
|
|
1946
|
+
return { image, analysis }
|
|
1947
|
+
})
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
/** html_screenshot: render only a path-fenced local HTML file in the pinned Chrome adapter. */
|
|
1951
|
+
async htmlScreenshot(request: HtmlScreenshotRequest, options: ToolCallOptions): Promise<HtmlScreenshotResult> {
|
|
1952
|
+
return this.runOperation('vision_html_screenshot', options, async (operation) => {
|
|
1953
|
+
const width = integerInRange(request.width, 1280, 1, 8192, 'html_screenshot.width')
|
|
1954
|
+
const height = integerInRange(request.height, 800, 1, 8192, 'html_screenshot.height')
|
|
1955
|
+
const scale = integerInRange(request.scale, 1, 1, 4, 'html_screenshot.scale')
|
|
1956
|
+
const waitMs = integerInRange(request.waitMs, 0, 0, 120000, 'html_screenshot.waitMs')
|
|
1957
|
+
const fullPage = request.fullPage === true
|
|
1958
|
+
const outputPixels = width * height * scale * scale
|
|
1959
|
+
if (!Number.isSafeInteger(outputPixels) || outputPixels > this.config.maxImagePixels) {
|
|
1960
|
+
throw new VisionToolkitError('capacity', `HTML screenshot would create ${outputPixels} pixels, exceeding maxImagePixels ${this.config.maxImagePixels}`)
|
|
1961
|
+
}
|
|
1962
|
+
const policy = await this.pathPolicy(options.workspace)
|
|
1963
|
+
const source = await resolveHtmlFile(request.source, policy)
|
|
1964
|
+
if (source.bytes > this.config.maxImageBytes) {
|
|
1965
|
+
throw new VisionToolkitError('capacity', `HTML source is ${source.bytes} bytes, exceeding maxImageBytes ${this.config.maxImageBytes}`)
|
|
1966
|
+
}
|
|
1967
|
+
const stem = basename(source.path, extname(source.path))
|
|
1968
|
+
const finalPath = resolveOutputFile(request.output, policy, `${stem}.screenshot.png`, ['.png'])
|
|
1969
|
+
assertDistinctOutput(source.path, finalPath)
|
|
1970
|
+
const staged = createStagedOutput(policy, '.png')
|
|
1971
|
+
try {
|
|
1972
|
+
const result = await this.runUpstream('html_screenshot', [
|
|
1973
|
+
source.path,
|
|
1974
|
+
'-o',
|
|
1975
|
+
staged,
|
|
1976
|
+
'--width',
|
|
1977
|
+
String(width),
|
|
1978
|
+
'--height',
|
|
1979
|
+
String(height),
|
|
1980
|
+
'--scale',
|
|
1981
|
+
String(scale),
|
|
1982
|
+
'--wait-ms',
|
|
1983
|
+
String(waitMs),
|
|
1984
|
+
...(fullPage ? ['--full-page', '--max-pixels', String(this.config.maxImagePixels)] : []),
|
|
1985
|
+
], operation)
|
|
1986
|
+
const parsed = parseHtmlScreenshotOutput(result.stdout)
|
|
1987
|
+
const expectedWidth = width * scale
|
|
1988
|
+
if (parsed.outputPath !== staged || parsed.width !== expectedWidth) {
|
|
1989
|
+
throw new VisionToolkitError('output', 'html_screenshot: upstream summary does not match the requested output')
|
|
1990
|
+
}
|
|
1991
|
+
if (fullPage !== (parsed.pageHeight !== undefined)) {
|
|
1992
|
+
throw new VisionToolkitError('output', 'html_screenshot: upstream full-page metadata does not match the request')
|
|
1993
|
+
}
|
|
1994
|
+
const pageHeight = parsed.pageHeight
|
|
1995
|
+
const expectedHeight = pageHeight === undefined ? height * scale : pageHeight * scale
|
|
1996
|
+
if (pageHeight !== undefined && (!Number.isInteger(pageHeight) || pageHeight <= 0)) {
|
|
1997
|
+
throw new VisionToolkitError('output', 'html_screenshot: upstream reported an invalid page height')
|
|
1998
|
+
}
|
|
1999
|
+
const outputPixels = expectedWidth * expectedHeight
|
|
2000
|
+
if (!Number.isSafeInteger(outputPixels) || outputPixels > this.config.maxImagePixels) {
|
|
2001
|
+
throw new VisionToolkitError('capacity', `HTML screenshot would create ${outputPixels} pixels, exceeding maxImagePixels ${this.config.maxImagePixels}`)
|
|
2002
|
+
}
|
|
2003
|
+
if (parsed.height !== expectedHeight) {
|
|
2004
|
+
throw new VisionToolkitError('output', 'html_screenshot: upstream summary does not match the requested output')
|
|
2005
|
+
}
|
|
2006
|
+
const generated = await this.probeGeneratedImage(staged, operation, 'html_screenshot')
|
|
2007
|
+
if (generated.format !== 'png' || generated.width !== expectedWidth || generated.height !== expectedHeight) {
|
|
2008
|
+
throw new VisionToolkitError('output', 'html_screenshot: generated PNG dimensions do not match the reported output')
|
|
2009
|
+
}
|
|
2010
|
+
await commitStagedOutput(staged, finalPath, policy)
|
|
2011
|
+
const artifact = await describeArtifact(finalPath, policy, {
|
|
2012
|
+
mimeType: 'image/png',
|
|
2013
|
+
kind: 'image',
|
|
2014
|
+
description: 'Headless browser screenshot of local HTML',
|
|
2015
|
+
sourceTool: 'vision_html_screenshot',
|
|
2016
|
+
previewIntent: 'image',
|
|
2017
|
+
})
|
|
2018
|
+
return {
|
|
2019
|
+
sourcePath: source.path,
|
|
2020
|
+
sourceBytes: source.bytes,
|
|
2021
|
+
viewport: { width, height, scale },
|
|
2022
|
+
width: expectedWidth,
|
|
2023
|
+
height: expectedHeight,
|
|
2024
|
+
...(pageHeight === undefined ? {} : { pageHeight }),
|
|
2025
|
+
artifact,
|
|
2026
|
+
}
|
|
2027
|
+
} finally {
|
|
2028
|
+
await rm(staged, { force: true }).catch(() => {})
|
|
2029
|
+
}
|
|
2030
|
+
})
|
|
2031
|
+
}
|
|
2032
|
+
|
|
2033
|
+
private async writableDirectoryCheck(path: string, label: string): Promise<HealthCheck> {
|
|
2034
|
+
const probe = join(path, `.vision-toolkit-health-${randomUUID()}`)
|
|
2035
|
+
try {
|
|
2036
|
+
await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' })
|
|
2037
|
+
await rm(probe, { force: true })
|
|
2038
|
+
return { status: 'ok', detail: `${label} is writable: ${path}` }
|
|
2039
|
+
} catch {
|
|
2040
|
+
await rm(probe, { force: true }).catch(() => {})
|
|
2041
|
+
return { status: 'error', detail: `${label} is not writable: ${path}` }
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
/** Health: inspect local readiness, optionally probe `/models`, and explicitly test one real multimodal request. */
|
|
2046
|
+
async health(testConnection: boolean, options: ToolCallOptions, testModel = false): Promise<VisionToolkitHealthResult> {
|
|
2047
|
+
return this.runOperation('vision_toolkit_health', options, async (operation) => {
|
|
2048
|
+
const info = this.upstreamVersion
|
|
2049
|
+
const python: HealthCheck = { status: 'ok', detail: `${info.pythonVersion} via ${info.python}` }
|
|
2050
|
+
const dependencies: HealthCheck = {
|
|
2051
|
+
status: 'ok',
|
|
2052
|
+
detail: Object.entries(info.dependencies).map(([name, version]) => `${name}=${version}`).join(', '),
|
|
2053
|
+
}
|
|
2054
|
+
let chrome: HealthCheck
|
|
2055
|
+
try {
|
|
2056
|
+
const started = Date.now()
|
|
2057
|
+
const chromePath = await this.adapter.findChrome({ signal: operation.signal })
|
|
2058
|
+
operation.metrics.upstreamMs += Date.now() - started
|
|
2059
|
+
chrome = chromePath === undefined
|
|
2060
|
+
? { status: 'warning', detail: 'Chrome/Chromium/Edge was not found; vision_html_screenshot is unavailable' }
|
|
2061
|
+
: { status: 'ok', detail: chromePath }
|
|
2062
|
+
} catch {
|
|
2063
|
+
if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled')
|
|
2064
|
+
chrome = { status: 'error', detail: 'Chrome availability probe failed' }
|
|
2065
|
+
}
|
|
2066
|
+
let resolvedCredential: ResolvedCredential | undefined
|
|
2067
|
+
let credential: HealthCheck
|
|
2068
|
+
try {
|
|
2069
|
+
resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
|
|
2070
|
+
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
2071
|
+
: await this.ctx.credentials.resolve(this.config.provider.credential)
|
|
2072
|
+
credential = resolvedCredential === undefined
|
|
2073
|
+
? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
|
|
2074
|
+
: { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` }
|
|
2075
|
+
} catch {
|
|
2076
|
+
credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` }
|
|
2077
|
+
}
|
|
2078
|
+
let artifactDirectory: HealthCheck
|
|
2079
|
+
try {
|
|
2080
|
+
// allowedDirs are session input roots; they do not affect output readiness.
|
|
2081
|
+
const policy = await createPathPolicy(options.workspace, [])
|
|
2082
|
+
artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory')
|
|
2083
|
+
} catch {
|
|
2084
|
+
artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' }
|
|
2085
|
+
}
|
|
2086
|
+
const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory')
|
|
2087
|
+
let service: HealthCheck = {
|
|
2088
|
+
status: 'not_tested',
|
|
2089
|
+
detail: 'Connection was not tested; pass testConnection=true to query the configured /models endpoint',
|
|
2090
|
+
}
|
|
2091
|
+
let model: HealthCheck = {
|
|
2092
|
+
status: 'not_tested',
|
|
2093
|
+
detail: 'Vision model was not tested; run an explicit model test to send the bundled diagnostic image',
|
|
2094
|
+
}
|
|
2095
|
+
if (testConnection) {
|
|
2096
|
+
if (resolvedCredential === undefined) {
|
|
2097
|
+
service = { status: 'error', detail: 'Connection test skipped because the configured credential is unavailable' }
|
|
2098
|
+
} else {
|
|
2099
|
+
operation.metrics.usedVisionService = true
|
|
2100
|
+
const endpoint = `${this.config.provider.baseUrl}/models`
|
|
2101
|
+
try {
|
|
2102
|
+
const started = Date.now()
|
|
2103
|
+
const headers: Record<string, string> = {
|
|
2104
|
+
Accept: 'application/json',
|
|
2105
|
+
'User-Agent': this.config.provider.userAgent,
|
|
2106
|
+
}
|
|
2107
|
+
if (this.config.provider.protocol === 'anthropic') {
|
|
2108
|
+
headers['x-api-key'] = resolvedCredential.value
|
|
2109
|
+
headers['anthropic-version'] = '2023-06-01'
|
|
2110
|
+
} else {
|
|
2111
|
+
headers.Authorization = `Bearer ${resolvedCredential.value}`
|
|
2112
|
+
}
|
|
2113
|
+
const response = await fetch(endpoint, {
|
|
2114
|
+
method: 'GET',
|
|
2115
|
+
headers,
|
|
2116
|
+
signal: operation.signal,
|
|
2117
|
+
})
|
|
2118
|
+
operation.metrics.upstreamMs += Date.now() - started
|
|
2119
|
+
await response.body?.cancel().catch(() => {})
|
|
2120
|
+
if (response.ok) {
|
|
2121
|
+
service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` }
|
|
2122
|
+
} else if (response.status === 401) {
|
|
2123
|
+
service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` }
|
|
2124
|
+
} else if (response.status === 403) {
|
|
2125
|
+
// Some providers (e.g. Groq preview/account restrictions) block GET /models
|
|
2126
|
+
// while real multimodal requests still work. Treat 403 as a warning so the
|
|
2127
|
+
// explicit vision-model test, not the model list endpoint, decides access.
|
|
2128
|
+
service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` }
|
|
2129
|
+
} else if (response.status === 404 || response.status === 405) {
|
|
2130
|
+
service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` }
|
|
2131
|
+
} else if (response.status === 429) {
|
|
2132
|
+
service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' }
|
|
2133
|
+
} else {
|
|
2134
|
+
service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` }
|
|
2135
|
+
}
|
|
2136
|
+
} catch {
|
|
2137
|
+
if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled')
|
|
2138
|
+
service = { status: 'error', detail: `Service could not be reached at ${endpoint}` }
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
if (testModel) {
|
|
2143
|
+
if (resolvedCredential === undefined) {
|
|
2144
|
+
model = { status: 'error', detail: 'Vision model test skipped because the configured credential is unavailable' }
|
|
2145
|
+
} else {
|
|
2146
|
+
try {
|
|
2147
|
+
const result = await this.runUpstream(
|
|
2148
|
+
'glance',
|
|
2149
|
+
[VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT],
|
|
2150
|
+
operation,
|
|
2151
|
+
this.visionEnv(resolvedCredential),
|
|
2152
|
+
)
|
|
2153
|
+
if (result.stdout.trim().length === 0) {
|
|
2154
|
+
throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
|
|
2155
|
+
}
|
|
2156
|
+
model = {
|
|
2157
|
+
status: 'ok',
|
|
2158
|
+
detail: `Vision model ${this.config.provider.model} completed a multimodal request`,
|
|
2159
|
+
}
|
|
2160
|
+
} catch (error) {
|
|
2161
|
+
if (operation.signal.aborted) throw error
|
|
2162
|
+
const detail = error instanceof Error ? error.message : String(error)
|
|
2163
|
+
model = { status: 'error', detail: `Vision model test failed: ${detail.slice(0, 600)}` }
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model }
|
|
2168
|
+
const healthy = Object.values(checks).every(check => check.status !== 'error')
|
|
2169
|
+
return {
|
|
2170
|
+
pluginVersion: PLUGIN_VERSION,
|
|
2171
|
+
upstream: info,
|
|
2172
|
+
checks,
|
|
2173
|
+
healthy,
|
|
2174
|
+
connectionTested: testConnection,
|
|
2175
|
+
modelTested: testModel,
|
|
2176
|
+
}
|
|
2177
|
+
})
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
/** Report the packaged upstream snapshot version. */
|
|
2181
|
+
checkoutVersion(): Promise<string> {
|
|
2182
|
+
return this.adapter.readCheckoutVersion()
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
/** Prepared Python command. */
|
|
2186
|
+
python(): string {
|
|
2187
|
+
return this.adapter.versionInfo.python
|
|
2188
|
+
}
|
|
2189
|
+
}
|