pantheon-opencode 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -3
- package/bin/pantheon-init.mjs +79 -0
- package/commands/pantheon-vision.md +67 -0
- package/opencode.json +13 -9
- package/package.json +12 -5
- package/scripts/_pantheon_paths.py +0 -2
- package/scripts/doctor.mjs +4 -2
- package/scripts/install/health-check.mjs +101 -1
- package/scripts/install/model-picker.mjs +118 -0
- package/scripts/install/opencode.mjs +102 -19
- package/scripts/install/venv.mjs +25 -2
- package/scripts/install-mcp.mjs +98 -13
- package/scripts/mcp_persistence_server.py +1 -1
- package/scripts/secret-scan.mjs +88 -0
- package/scripts/uninstall.mjs +3 -3
- package/scripts/validate-routing.mjs +15 -0
- package/src/agents/hermes.md +2 -2
- package/src/agents/mnemosyne.md +1 -1
- package/src/agents/zeus.md +1 -1
- package/src/mcp/_pantheon_paths.py +0 -2
- package/src/mcp/install-mcp.mjs +6 -913
- package/src/mcp/mcp_persistence_server.py +2 -1
- package/src/mcp/pantheon_vision_server.py +495 -0
- package/src/mcp/requirements-vision.txt +3 -0
- package/src/pantheon/auto-wake.ts +91 -0
- package/src/pantheon/background-job-board.ts +568 -0
- package/src/pantheon/file-persistence.ts +95 -0
- package/src/pantheon/presets.d.mts +108 -0
- package/src/pantheon/presets.mjs +554 -0
- package/src/pantheon/vision.ts +1437 -0
- package/src/plugin.ts +69 -25
- package/src/plugins/tui/package.json +3 -3
- package/src/plugins/tui/src/index.tsx +866 -121
- package/src/routing.yml +148 -0
- package/src/skills/context-compression/SKILL.md +1 -1
- package/src/skills/loop-engineering/SKILL.md +1 -1
- package/src/skills/orchestration-workflow/SKILL.md +11 -11
- package/scripts/scrub_secrets.py +0 -281
- package/src/plugins/tui/dist/tui.d.ts +0 -8
- package/src/plugins/tui/dist/tui.d.ts.map +0 -1
- package/src/plugins/tui/dist/tui.js +0 -609
- package/src/plugins/tui/dist/tui.js.map +0 -1
- package/src/plugins/tui/dist/tui.tsx +0 -444
|
@@ -0,0 +1,1437 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
+
import { existsSync } from 'node:fs'
|
|
3
|
+
import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { homedir, tmpdir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import type { Hooks, PluginInput } from '@opencode-ai/plugin'
|
|
8
|
+
import type { FilePart, Part, TextPart, UserMessage } from '@opencode-ai/sdk'
|
|
9
|
+
import type { ResolvedPreset } from './presets.mjs'
|
|
10
|
+
import { hasVision, resolveActivePreset, visionBrokenOnGateway } from './presets.mjs'
|
|
11
|
+
|
|
12
|
+
// ─── Constants ────────────────────────────────────────────────────────────
|
|
13
|
+
|
|
14
|
+
const CONFIG_FILENAME = 'opencode-vision.json'
|
|
15
|
+
const TEMP_DIR_NAME = 'pantheon-vision'
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Temp images younger than this survive a `session.idle` fired between
|
|
19
|
+
* auto-continue turns; abandoned files are swept by the next idle.
|
|
20
|
+
*/
|
|
21
|
+
export const TEMP_FILE_GRACE_MS = 120_000
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Global cap on temp images in `/tmp/pantheon-vision`. When the LRU registry
|
|
25
|
+
* (see `enforceTempFileCap`) exceeds this, the least-recently-used files older
|
|
26
|
+
* than TEMP_FILE_GRACE_MS are unlinked — never a young file that an active
|
|
27
|
+
* turn may still reference.
|
|
28
|
+
*/
|
|
29
|
+
export const TEMP_MAX_FILES = 200
|
|
30
|
+
|
|
31
|
+
// Tool the model is instructed to call. Configurable via:
|
|
32
|
+
// 1. env PANTHEON_VISION_TOOL
|
|
33
|
+
// 2. config file imageAnalysisTool (project > user)
|
|
34
|
+
// 3. dynamic detection of an available MCP vision tool
|
|
35
|
+
// 4. the canonical Pantheon Vision MCP tool
|
|
36
|
+
//
|
|
37
|
+
// OpenCode 1.18.11 generates MCP tool IDs as sanitize(clientName) + '_' +
|
|
38
|
+
// sanitize(name): the `pantheon-vision` MCP (FastMCP name) yields
|
|
39
|
+
// `pantheon_vision_vision_describe` (underscores) — there is no `mcp__` prefix
|
|
40
|
+
// in that version. The legacy `mcp__pantheon-vision__vision_describe` form is
|
|
41
|
+
// still accepted when explicitly configured or detected.
|
|
42
|
+
const DEFAULT_IMAGE_ANALYSIS_TOOL = 'pantheon_vision_vision_describe'
|
|
43
|
+
|
|
44
|
+
/** True for IDs referencing the Pantheon Vision MCP — hyphen OR underscore server form. */
|
|
45
|
+
const isPantheonVisionServerTool = (id: string): boolean => /[pP]antheon[-_]vision/.test(id)
|
|
46
|
+
|
|
47
|
+
/** True for the describe/ocr/analyze tool suffix (sanitized MCP tool name). */
|
|
48
|
+
const isPantheonVisionActionTool = (id: string): boolean =>
|
|
49
|
+
/vision_(describe|ocr|analyze)/i.test(id)
|
|
50
|
+
|
|
51
|
+
// Enabled for ALL models by default ("paste and ask" universal). The config
|
|
52
|
+
// file can restrict this with wildcard patterns.
|
|
53
|
+
const DEFAULT_MODEL_PATTERNS: readonly string[] = ['*']
|
|
54
|
+
|
|
55
|
+
// Only PNG/JPEG/WebP are intercepted (data: URLs need a known extension).
|
|
56
|
+
const SUPPORTED_MIME_TYPES = new Set([
|
|
57
|
+
'image/png',
|
|
58
|
+
'image/jpeg',
|
|
59
|
+
'image/jpg',
|
|
60
|
+
'image/webp',
|
|
61
|
+
'image/gif',
|
|
62
|
+
])
|
|
63
|
+
const MIME_TO_EXTENSION: Record<string, string> = {
|
|
64
|
+
'image/png': 'png',
|
|
65
|
+
'image/jpeg': 'jpg',
|
|
66
|
+
'image/jpg': 'jpg',
|
|
67
|
+
'image/webp': 'webp',
|
|
68
|
+
'image/gif': 'gif',
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const PROMPT_TEMPLATE_VARIABLES = [
|
|
72
|
+
'{imageList}',
|
|
73
|
+
'{imageCount}',
|
|
74
|
+
'{toolName}',
|
|
75
|
+
'{userText}',
|
|
76
|
+
] as const
|
|
77
|
+
|
|
78
|
+
// ─── Native vision (provider-direct, OpenAI-compatible) ─────────────────────
|
|
79
|
+
// Preferred path: when an API key is available the plugin calls the multimodal
|
|
80
|
+
// model DIRECTLY via the opencode Zen OpenAI-compatible endpoint, so any user
|
|
81
|
+
// with a provider key gets vision WITHOUT an MCP tool. The MCP tool pattern
|
|
82
|
+
// (imageAnalysisTool) remains the fallback for missing keys / provider errors.
|
|
83
|
+
//
|
|
84
|
+
// MVP scope: only opencode-go / opencode providers (the user's stack).
|
|
85
|
+
// anthropic/openai native vision is phase 2 — those presets fall back to the
|
|
86
|
+
// MCP tool pattern.
|
|
87
|
+
|
|
88
|
+
const DEFAULT_VISION_MODEL = 'opencode-go/mimo-v2.5'
|
|
89
|
+
const ZEN_BASE_URL = 'https://opencode.ai/zen/v1'
|
|
90
|
+
const ZEN_GO_BASE_URL = 'https://opencode.ai/zen/go/v1'
|
|
91
|
+
const NATIVE_PROMPT =
|
|
92
|
+
'Descreva fielmente esta imagem em detalhes (texto visível, cores, layout, objetos).'
|
|
93
|
+
const NATIVE_TIMEOUT_MS = 20_000
|
|
94
|
+
const NATIVE_MAX_TOKENS = 500
|
|
95
|
+
const MAX_NATIVE_IMAGE_BYTES = 25 * 1024 * 1024
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Descriptions produced by the native path are cached per session, keyed by
|
|
99
|
+
* the user prompt + the content hashes of the attached images (see
|
|
100
|
+
* `buildDescriptionCacheKey`). TTL bounds staleness: an identical re-ask
|
|
101
|
+
* within the window reuses the stored text without another gateway call;
|
|
102
|
+
* after expiry the multimodal model is called again.
|
|
103
|
+
*/
|
|
104
|
+
export const DESCRIPTION_CACHE_TTL_MS = 30 * 60 * 1000
|
|
105
|
+
|
|
106
|
+
// ─── Types ─────────────────────────────────────────────────────────────────
|
|
107
|
+
|
|
108
|
+
type VisionConfig = {
|
|
109
|
+
models?: string[]
|
|
110
|
+
imageAnalysisTool?: string
|
|
111
|
+
promptTemplate?: string
|
|
112
|
+
mode?: VisionMode
|
|
113
|
+
visionModel?: string
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export type ModelInfo = { providerID: string; modelID: string }
|
|
117
|
+
|
|
118
|
+
export type NativeVisionTarget = { modelID: string; baseURL: string; apiKey: string }
|
|
119
|
+
|
|
120
|
+
type NativeVisionEndpoint = { modelID: string; baseURL: string }
|
|
121
|
+
|
|
122
|
+
export type VisionMode = 'native' | 'tool' | 'auto'
|
|
123
|
+
|
|
124
|
+
/** Return whether OpenCode's current model is known to accept image input. */
|
|
125
|
+
export function modelAcceptsImages(model: ModelInfo | undefined): boolean {
|
|
126
|
+
if (!model) return true
|
|
127
|
+
try {
|
|
128
|
+
const id = `${model.providerID}/${model.modelID}`
|
|
129
|
+
// Provider/gateway-aware: a model can be vision-capable per models.dev
|
|
130
|
+
// yet BROKEN on its runtime gateway (qwen3.7-plus on opencode-go returns
|
|
131
|
+
// HTTP 500 on image turns). Such models are treated as text-only so the
|
|
132
|
+
// image is intercepted and routed to the multimodal fallback instead of
|
|
133
|
+
// being sent directly to the gateway.
|
|
134
|
+
return hasVision(id) && !visionBrokenOnGateway(id, model.providerID)
|
|
135
|
+
} catch {
|
|
136
|
+
// Unknown providers/models should retain OpenCode's native behavior.
|
|
137
|
+
return true
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
type SavedImage = {
|
|
142
|
+
path: string
|
|
143
|
+
partId: string
|
|
144
|
+
temporary: boolean
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
type NativeImagePayload = { url: string }
|
|
148
|
+
|
|
149
|
+
// ─── Config loading (project > user) ──────────────────────────────────────
|
|
150
|
+
|
|
151
|
+
// ─── OpenCode auth store (fallback credential source) ────────────────────
|
|
152
|
+
// `opencode auth login` stores provider tokens in <data dir>/auth.json.
|
|
153
|
+
// The vision plugin reads that store so connected users don't need to export
|
|
154
|
+
// any env var. Mirrors the usage-bar pattern (src/plugins/tui) but stays
|
|
155
|
+
// local to this module — no TUI imports.
|
|
156
|
+
|
|
157
|
+
function defaultAuthStorePath(): string {
|
|
158
|
+
return join(homedir(), '.local', 'share', 'opencode', 'auth.json')
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Read a provider token from opencode's auth store (`opencode auth login`).
|
|
163
|
+
* `statePath` is the opencode data dir (default ~/.local/share/opencode); the
|
|
164
|
+
* auth.json inside it maps provider IDs to `{ access, expires, ... }`.
|
|
165
|
+
* Returns the first non-expired `access` token across `providerIDs`, or null
|
|
166
|
+
* when the file is missing/corrupted or no entry has a usable token — callers
|
|
167
|
+
* fall back to the MCP tool pattern. The token is never logged.
|
|
168
|
+
*/
|
|
169
|
+
export async function readOpencodeAuthToken(
|
|
170
|
+
providerIDs: readonly string[],
|
|
171
|
+
statePath?: string,
|
|
172
|
+
): Promise<string | null> {
|
|
173
|
+
const authPath = statePath ? join(statePath, 'auth.json') : defaultAuthStorePath()
|
|
174
|
+
let raw: string
|
|
175
|
+
try {
|
|
176
|
+
raw = await readFile(authPath, 'utf8')
|
|
177
|
+
} catch {
|
|
178
|
+
return null // missing/unreadable → no crash
|
|
179
|
+
}
|
|
180
|
+
let auth: Record<string, { access?: unknown; expires?: unknown } | undefined>
|
|
181
|
+
try {
|
|
182
|
+
auth = JSON.parse(raw) as typeof auth
|
|
183
|
+
} catch {
|
|
184
|
+
return null // corrupted → no crash
|
|
185
|
+
}
|
|
186
|
+
for (const id of providerIDs) {
|
|
187
|
+
const entry = auth[id]
|
|
188
|
+
const access = typeof entry?.access === 'string' ? entry.access.trim() : ''
|
|
189
|
+
if (access === '') continue
|
|
190
|
+
const expires = entry?.expires
|
|
191
|
+
if (typeof expires === 'number' && expires > 0 && Date.now() >= expires) continue
|
|
192
|
+
return access
|
|
193
|
+
}
|
|
194
|
+
return null
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function userConfigPath(): string {
|
|
198
|
+
const xdg = process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config')
|
|
199
|
+
return join(xdg, 'opencode', CONFIG_FILENAME)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async function readConfigFile(configPath: string): Promise<VisionConfig | null> {
|
|
203
|
+
if (!existsSync(configPath)) return null
|
|
204
|
+
try {
|
|
205
|
+
const parsed = JSON.parse(await readFile(configPath, 'utf8')) as Record<string, unknown>
|
|
206
|
+
const config: VisionConfig = {}
|
|
207
|
+
if (Array.isArray(parsed.models)) {
|
|
208
|
+
const models = parsed.models.filter((m): m is string => typeof m === 'string')
|
|
209
|
+
if (models.length > 0) config.models = models
|
|
210
|
+
}
|
|
211
|
+
if (typeof parsed.imageAnalysisTool === 'string' && parsed.imageAnalysisTool.trim() !== '') {
|
|
212
|
+
config.imageAnalysisTool = parsed.imageAnalysisTool
|
|
213
|
+
}
|
|
214
|
+
const template = parsed.promptTemplate
|
|
215
|
+
if (
|
|
216
|
+
typeof template === 'string' &&
|
|
217
|
+
PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))
|
|
218
|
+
) {
|
|
219
|
+
config.promptTemplate = template
|
|
220
|
+
}
|
|
221
|
+
if (parsed.mode === 'native' || parsed.mode === 'tool' || parsed.mode === 'auto') {
|
|
222
|
+
config.mode = parsed.mode
|
|
223
|
+
}
|
|
224
|
+
if (typeof parsed.visionModel === 'string' && parsed.visionModel.trim() !== '') {
|
|
225
|
+
config.visionModel = parsed.visionModel.trim()
|
|
226
|
+
}
|
|
227
|
+
return Object.keys(config).length > 0 ? config : null
|
|
228
|
+
} catch {
|
|
229
|
+
return null
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export async function loadVisionConfig(directory: string): Promise<VisionConfig> {
|
|
234
|
+
const project = await readConfigFile(join(directory, '.opencode', CONFIG_FILENAME))
|
|
235
|
+
const user = await readConfigFile(userConfigPath())
|
|
236
|
+
const merged: VisionConfig = {}
|
|
237
|
+
if (project?.models) merged.models = project.models
|
|
238
|
+
else if (user?.models) merged.models = user.models
|
|
239
|
+
if (project?.imageAnalysisTool) merged.imageAnalysisTool = project.imageAnalysisTool
|
|
240
|
+
else if (user?.imageAnalysisTool) merged.imageAnalysisTool = user.imageAnalysisTool
|
|
241
|
+
if (project?.promptTemplate) merged.promptTemplate = project.promptTemplate
|
|
242
|
+
else if (user?.promptTemplate) merged.promptTemplate = user.promptTemplate
|
|
243
|
+
if (project?.mode) merged.mode = project.mode
|
|
244
|
+
else if (user?.mode) merged.mode = user.mode
|
|
245
|
+
if (project?.visionModel) merged.visionModel = project.visionModel
|
|
246
|
+
else if (user?.visionModel) merged.visionModel = user.visionModel
|
|
247
|
+
return merged
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ─── Active-preset resolution (native vision model) ────────────────────────
|
|
251
|
+
// Same candidate order as the plugin config hook, so the native vision model
|
|
252
|
+
// matches the active preset (resolved.vision?.model) exactly.
|
|
253
|
+
|
|
254
|
+
export function activePresetCandidates(): string[] {
|
|
255
|
+
const home = process.env.HOME ?? ''
|
|
256
|
+
const xdg = process.env.XDG_CONFIG_HOME ?? `${home}/.config`
|
|
257
|
+
return [
|
|
258
|
+
`${process.cwd()}/.pantheon/active-preset.json`,
|
|
259
|
+
`${xdg}/opencode/.pantheon/active-preset.json`,
|
|
260
|
+
`${home}/.opencode/.pantheon/active-preset.json`,
|
|
261
|
+
]
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Resolve the vision mode: env PANTHEON_VISION_MODE, then the config file's
|
|
266
|
+
* `mode` field, then default `auto`. `auto` = try native vision first (when a
|
|
267
|
+
* key exists), fall back to the MCP tool pattern. `tool` forces the legacy
|
|
268
|
+
* tool pattern. `native` tries native first and still falls back on failure
|
|
269
|
+
* (never break a user turn).
|
|
270
|
+
*/
|
|
271
|
+
export function getVisionMode(
|
|
272
|
+
env: Record<string, string | undefined> = process.env,
|
|
273
|
+
configMode?: string,
|
|
274
|
+
): VisionMode {
|
|
275
|
+
const envMode = env.PANTHEON_VISION_MODE?.trim().toLowerCase()
|
|
276
|
+
if (envMode === 'native' || envMode === 'tool') return envMode
|
|
277
|
+
const cfgMode = configMode?.trim().toLowerCase()
|
|
278
|
+
if (cfgMode === 'native' || cfgMode === 'tool') return cfgMode
|
|
279
|
+
return 'auto'
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ─── Wildcard model matching ───────────────────────────────────────────────
|
|
283
|
+
// Patterns: `*`, `prefix*`, `*suffix`, `*contains*`, `provider/*`, `*/model`.
|
|
284
|
+
|
|
285
|
+
export function matchesWildcardPattern(pattern: string, value: string): boolean {
|
|
286
|
+
const p = pattern.toLowerCase()
|
|
287
|
+
const v = value.toLowerCase()
|
|
288
|
+
if (p === '*') return true
|
|
289
|
+
if (p.startsWith('*') && p.endsWith('*') && p.length > 2) return v.includes(p.slice(1, -1))
|
|
290
|
+
if (p.endsWith('*')) return v.startsWith(p.slice(0, -1))
|
|
291
|
+
if (p.startsWith('*')) return v.endsWith(p.slice(1))
|
|
292
|
+
return v === p
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
export function matchesModelPattern(pattern: string, model: ModelInfo): boolean {
|
|
296
|
+
if (pattern === '*') return true
|
|
297
|
+
const slash = pattern.indexOf('/')
|
|
298
|
+
if (slash === -1) {
|
|
299
|
+
return (
|
|
300
|
+
matchesWildcardPattern(pattern, model.providerID) ||
|
|
301
|
+
matchesWildcardPattern(pattern, model.modelID)
|
|
302
|
+
)
|
|
303
|
+
}
|
|
304
|
+
const provider = pattern.slice(0, slash)
|
|
305
|
+
const modelPattern = pattern.slice(slash + 1)
|
|
306
|
+
return (
|
|
307
|
+
matchesWildcardPattern(provider, model.providerID) &&
|
|
308
|
+
matchesWildcardPattern(modelPattern, model.modelID)
|
|
309
|
+
)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function modelMatchesAnyPattern(
|
|
313
|
+
model: ModelInfo | undefined,
|
|
314
|
+
patterns: readonly string[],
|
|
315
|
+
): boolean {
|
|
316
|
+
if (!model) return false
|
|
317
|
+
return patterns.some((pattern) => matchesModelPattern(pattern, model))
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// ─── Type guards ───────────────────────────────────────────────────────────
|
|
321
|
+
|
|
322
|
+
export function isImageFilePart(part: unknown): part is FilePart {
|
|
323
|
+
if (!part || typeof part !== 'object') return false
|
|
324
|
+
const value = part as { type?: unknown; mime?: unknown }
|
|
325
|
+
return (
|
|
326
|
+
value.type === 'file' &&
|
|
327
|
+
typeof value.mime === 'string' &&
|
|
328
|
+
value.mime.toLowerCase().startsWith('image/')
|
|
329
|
+
)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function isSupportedImageFilePart(part: unknown): part is FilePart {
|
|
333
|
+
if (!isImageFilePart(part)) return false
|
|
334
|
+
return SUPPORTED_MIME_TYPES.has(part.mime.toLowerCase())
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function isTextPart(part: Part): part is TextPart {
|
|
338
|
+
return part.type === 'text'
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function syntheticPartId(): string {
|
|
342
|
+
return `prt${randomUUID()}`
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// ─── Image target resolution ───────────────────────────────────────────────
|
|
346
|
+
// file:// → local path used directly; data: → decoded + saved to temp;
|
|
347
|
+
// http(s):// → URL passed through unchanged.
|
|
348
|
+
|
|
349
|
+
function getExtensionForMime(mime: string): string {
|
|
350
|
+
return MIME_TO_EXTENSION[mime.toLowerCase()] ?? 'png'
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Same set as the paste-and-ask interception and the Pantheon Vision MCP.
|
|
354
|
+
const EXTENSION_TO_MIME: Record<string, string> = {
|
|
355
|
+
png: 'image/png',
|
|
356
|
+
jpg: 'image/jpeg',
|
|
357
|
+
jpeg: 'image/jpeg',
|
|
358
|
+
webp: 'image/webp',
|
|
359
|
+
gif: 'image/gif',
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Guess a MIME type from a file extension (.png/.jpg/.jpeg/.webp/.gif).
|
|
364
|
+
* Returns null for unknown extensions — callers surface a friendly error.
|
|
365
|
+
*/
|
|
366
|
+
export function getMimeForPath(path: string): string | null {
|
|
367
|
+
const match = /\.([a-z0-9]+)$/i.exec(path)
|
|
368
|
+
if (!match) return null
|
|
369
|
+
const extension = (match[1] ?? '').toLowerCase()
|
|
370
|
+
return EXTENSION_TO_MIME[extension] ?? null
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// ─── Temp-image LRU (global disk cap) ─────────────────────────────────────
|
|
374
|
+
// Content-hash filenames mean identical re-pasted images share one file; the
|
|
375
|
+
// LRU registry keeps the total on disk bounded. Insertion order = recency:
|
|
376
|
+
// touching a path deletes + re-sets it so the oldest entries sit at the front.
|
|
377
|
+
|
|
378
|
+
/** LRU registry: temp image path → last touch time. Exposed for tests. */
|
|
379
|
+
export const tempFileLRU = new Map<string, number>()
|
|
380
|
+
|
|
381
|
+
/** Mark a temp file as recently used (recency = map insertion order). */
|
|
382
|
+
export function touchTempFile(path: string, now: number = Date.now()): void {
|
|
383
|
+
tempFileLRU.delete(path)
|
|
384
|
+
tempFileLRU.set(path, now)
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Drop a temp file from the registry (after it has been unlinked). */
|
|
388
|
+
export function forgetTempFile(path: string): void {
|
|
389
|
+
tempFileLRU.delete(path)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Enforce the global temp-file cap: when the registry exceeds `maxFiles`,
|
|
394
|
+
* unlink the least-recently-used files — but NEVER a file younger than
|
|
395
|
+
* `graceMs`, because an active auto-continue turn may still reference it.
|
|
396
|
+
* The temp directory itself is never removed (other sessions/processes may
|
|
397
|
+
* share it). Returns how many files were evicted.
|
|
398
|
+
*/
|
|
399
|
+
export async function enforceTempFileCap(
|
|
400
|
+
maxFiles: number = TEMP_MAX_FILES,
|
|
401
|
+
graceMs: number = TEMP_FILE_GRACE_MS,
|
|
402
|
+
now: number = Date.now(),
|
|
403
|
+
): Promise<number> {
|
|
404
|
+
let evicted = 0
|
|
405
|
+
for (const [path, touchedAt] of tempFileLRU) {
|
|
406
|
+
if (tempFileLRU.size <= maxFiles) break
|
|
407
|
+
if (now - touchedAt <= graceMs) continue
|
|
408
|
+
tempFileLRU.delete(path)
|
|
409
|
+
try {
|
|
410
|
+
await unlink(path)
|
|
411
|
+
} catch {
|
|
412
|
+
// Best effort.
|
|
413
|
+
}
|
|
414
|
+
evicted += 1
|
|
415
|
+
}
|
|
416
|
+
return evicted
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
export async function saveDataUrlImage(dataUrl: string, mime: string): Promise<string | null> {
|
|
420
|
+
const match = /^data:([^;,]+);base64,(.+)$/s.exec(dataUrl)
|
|
421
|
+
if (!match) return null
|
|
422
|
+
const data = Buffer.from(match[2] ?? '', 'base64')
|
|
423
|
+
if (data.length === 0) return null
|
|
424
|
+
const dir = join(tmpdir(), TEMP_DIR_NAME)
|
|
425
|
+
// Screenshots may hold sensitive content: force private modes even under a
|
|
426
|
+
// permissive umask. mkdir/writeFile honor the umask, so chmod is applied
|
|
427
|
+
// AFTER creation to guarantee 0700/0600 regardless of the environment.
|
|
428
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
429
|
+
await chmod(dir, 0o700)
|
|
430
|
+
// Content-hash filename: identical re-pasted images reuse the same file
|
|
431
|
+
// (dedup — less disk, and it stabilizes the per-image description cache).
|
|
432
|
+
const filePath = join(dir, `${hashContent(data)}.${getExtensionForMime(mime)}`)
|
|
433
|
+
if (!existsSync(filePath)) await writeFile(filePath, data, { mode: 0o600 })
|
|
434
|
+
await chmod(filePath, 0o600)
|
|
435
|
+
touchTempFile(filePath)
|
|
436
|
+
await enforceTempFileCap()
|
|
437
|
+
return filePath
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function resolveImageTarget(filePart: FilePart): Promise<SavedImage | null> {
|
|
441
|
+
const url = filePart.url
|
|
442
|
+
if (!url) return null
|
|
443
|
+
if (url.startsWith('file://')) {
|
|
444
|
+
// fileURLToPath decodes percent-encoding (file:///tmp/my%20screen.png →
|
|
445
|
+
// /tmp/my screen.png) and normalizes Windows drive-letter URLs
|
|
446
|
+
// (file:///C:/... → C:\... on Windows). A raw slice('file://') would
|
|
447
|
+
// keep the literal %20 in the path, so reading the file would fail.
|
|
448
|
+
try {
|
|
449
|
+
return { path: fileURLToPath(url), partId: filePart.id, temporary: false }
|
|
450
|
+
} catch {
|
|
451
|
+
return null // malformed file URL → unresolvable
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (url.startsWith('data:')) {
|
|
455
|
+
const path = await saveDataUrlImage(url, filePart.mime)
|
|
456
|
+
if (!path) return null
|
|
457
|
+
return { path, partId: filePart.id, temporary: true }
|
|
458
|
+
}
|
|
459
|
+
if (url.startsWith('http://') || url.startsWith('https://')) {
|
|
460
|
+
return { path: url, partId: filePart.id, temporary: false }
|
|
461
|
+
}
|
|
462
|
+
return null
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// ─── Native vision (provider-direct) ───────────────────────────────────────
|
|
466
|
+
|
|
467
|
+
/** First non-empty (trimmed) value across the priority list, else `fallback`. */
|
|
468
|
+
function firstNonEmpty(fallback: string, ...values: Array<string | undefined>): string {
|
|
469
|
+
for (const value of values) {
|
|
470
|
+
const trimmed = value?.trim()
|
|
471
|
+
if (trimmed) return trimmed
|
|
472
|
+
}
|
|
473
|
+
return fallback
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Resolve the native vision endpoint: model ID + Zen base URL (no key I/O).
|
|
478
|
+
*
|
|
479
|
+
* Model precedence: env PANTHEON_VISION_MODEL > config file `visionModel` >
|
|
480
|
+
* active preset `vision.model` > DEFAULT_VISION_MODEL. Returns null when the
|
|
481
|
+
* model's provider is not opencode-go/opencode (anthropic/openai are phase 2).
|
|
482
|
+
*/
|
|
483
|
+
function resolveNativeVisionEndpoint(
|
|
484
|
+
preset: Pick<ResolvedPreset, 'vision'> | null | undefined,
|
|
485
|
+
env: Record<string, string | undefined>,
|
|
486
|
+
configVisionModel?: string,
|
|
487
|
+
): NativeVisionEndpoint | null {
|
|
488
|
+
const resolved = preset ?? resolveActivePreset({ env, candidates: activePresetCandidates() })
|
|
489
|
+
const modelID = firstNonEmpty(
|
|
490
|
+
DEFAULT_VISION_MODEL,
|
|
491
|
+
env.PANTHEON_VISION_MODEL,
|
|
492
|
+
configVisionModel,
|
|
493
|
+
resolved?.vision?.model,
|
|
494
|
+
)
|
|
495
|
+
const providerID = modelID.slice(0, modelID.indexOf('/'))
|
|
496
|
+
let baseURL: string | null = null
|
|
497
|
+
if (providerID.startsWith('opencode-go')) baseURL = ZEN_GO_BASE_URL
|
|
498
|
+
else if (providerID === 'opencode') baseURL = ZEN_BASE_URL
|
|
499
|
+
if (!baseURL) return null
|
|
500
|
+
return { modelID, baseURL }
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Resolve the native vision target: model + Zen endpoint + API key.
|
|
505
|
+
*
|
|
506
|
+
* Key comes from env only (PANTHEON_OPENCODE_API_KEY then OPENCODE_API_KEY).
|
|
507
|
+
* Returns null when the model's provider is not opencode-go/opencode or when
|
|
508
|
+
* no env key is set — callers fall back to the MCP tool. For the auth-store
|
|
509
|
+
* fallback (`opencode auth login`) use `resolveNativeVisionTarget`.
|
|
510
|
+
*/
|
|
511
|
+
export function resolveNativeVisionConfig(
|
|
512
|
+
preset: Pick<ResolvedPreset, 'vision'> | null | undefined = undefined,
|
|
513
|
+
env: Record<string, string | undefined> = process.env,
|
|
514
|
+
configVisionModel?: string,
|
|
515
|
+
): NativeVisionTarget | null {
|
|
516
|
+
const endpoint = resolveNativeVisionEndpoint(preset, env, configVisionModel)
|
|
517
|
+
if (!endpoint) return null
|
|
518
|
+
const apiKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
|
|
519
|
+
if (!apiKey || apiKey.trim() === '') return null
|
|
520
|
+
return { ...endpoint, apiKey }
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Full native-vision target resolution for the chat hook (async): model +
|
|
525
|
+
* endpoint via `resolveNativeVisionConfig`, then the API key from env
|
|
526
|
+
* (PANTHEON_OPENCODE_API_KEY ?? OPENCODE_API_KEY) falling back to the opencode
|
|
527
|
+
* auth store (`opencode auth login`) — 'opencode-go' entry first, then
|
|
528
|
+
* 'opencode' — so connected users don't need to export anything. Returns null
|
|
529
|
+
* when the provider isn't opencode-go/opencode or no usable key exists —
|
|
530
|
+
* callers fall back to the MCP tool.
|
|
531
|
+
*/
|
|
532
|
+
export async function resolveNativeVisionTarget(
|
|
533
|
+
input: {
|
|
534
|
+
preset?: Pick<ResolvedPreset, 'vision'> | null
|
|
535
|
+
env?: Record<string, string | undefined>
|
|
536
|
+
config?: VisionConfig | null
|
|
537
|
+
authStatePath?: string
|
|
538
|
+
} = {},
|
|
539
|
+
): Promise<NativeVisionTarget | null> {
|
|
540
|
+
const env = input.env ?? process.env
|
|
541
|
+
const endpoint = resolveNativeVisionEndpoint(input.preset, env, input.config?.visionModel)
|
|
542
|
+
if (!endpoint) return null
|
|
543
|
+
const envKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
|
|
544
|
+
if (envKey && envKey.trim() !== '') return { ...endpoint, apiKey: envKey }
|
|
545
|
+
const authToken = await readOpencodeAuthToken(['opencode-go', 'opencode'], input.authStatePath)
|
|
546
|
+
if (!authToken) return null
|
|
547
|
+
return { ...endpoint, apiKey: authToken }
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* Resolve a single image source into an OpenAI-compatible image_url payload:
|
|
552
|
+
* - http(s) URL → passed through unchanged
|
|
553
|
+
* - data: URI → validated + size-capped, used as-is
|
|
554
|
+
* - local path → bytes read, converted to a data URI with the given MIME
|
|
555
|
+
* Returns null when the source is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
|
|
556
|
+
* callers fall back to the MCP tool pattern (or surface a friendly error).
|
|
557
|
+
*/
|
|
558
|
+
export async function resolveImageSourcePayload(
|
|
559
|
+
source: string,
|
|
560
|
+
mime: string,
|
|
561
|
+
): Promise<NativeImagePayload | null> {
|
|
562
|
+
if (source.startsWith('http://') || source.startsWith('https://')) {
|
|
563
|
+
return { url: source }
|
|
564
|
+
}
|
|
565
|
+
if (source.startsWith('data:')) {
|
|
566
|
+
const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/.exec(source)
|
|
567
|
+
if (!match) return null
|
|
568
|
+
const approxBytes = Math.ceil(((match[2]?.length ?? 0) * 3) / 4)
|
|
569
|
+
if (approxBytes > MAX_NATIVE_IMAGE_BYTES) return null
|
|
570
|
+
return { url: source }
|
|
571
|
+
}
|
|
572
|
+
// Local path (from a file:// FilePart or a standalone tool path).
|
|
573
|
+
try {
|
|
574
|
+
const data = await readFile(source)
|
|
575
|
+
if (data.length === 0 || data.length > MAX_NATIVE_IMAGE_BYTES) return null
|
|
576
|
+
return { url: `data:${mime.toLowerCase()};base64,${data.toString('base64')}` }
|
|
577
|
+
} catch {
|
|
578
|
+
return null
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Build OpenAI-compatible image_url payloads for the native vision call.
|
|
584
|
+
* - data: URI → used as-is (validated + size-capped).
|
|
585
|
+
* - file:// path → read bytes, converted to a data URI.
|
|
586
|
+
* - http(s) URL → passed through unchanged.
|
|
587
|
+
* Returns null when any image is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
|
|
588
|
+
* the caller falls back to the MCP tool pattern.
|
|
589
|
+
*/
|
|
590
|
+
async function buildNativeImagePayloads(
|
|
591
|
+
resolved: ReadonlyArray<{ filePart: FilePart; target: SavedImage }>,
|
|
592
|
+
): Promise<NativeImagePayload[] | null> {
|
|
593
|
+
const payloads: NativeImagePayload[] = []
|
|
594
|
+
for (const { filePart, target } of resolved) {
|
|
595
|
+
const payload = await resolveImageSourcePayload(target.path, filePart.mime)
|
|
596
|
+
if (!payload) return null
|
|
597
|
+
payloads.push(payload)
|
|
598
|
+
}
|
|
599
|
+
return payloads
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Strip the provider prefix from a model ID before sending it to the Zen
|
|
604
|
+
* gateway. The endpoint URL already encodes the provider (see
|
|
605
|
+
* `resolveNativeVisionEndpoint`), and a qualified model ID such as
|
|
606
|
+
* `opencode-go/mimo-v2.5` is rejected by the gateway with a 401
|
|
607
|
+
* ("Model ... is not supported"). Models without a `/` are returned unchanged.
|
|
608
|
+
*/
|
|
609
|
+
export function stripProviderPrefix(modelID: string): string {
|
|
610
|
+
const slash = modelID.lastIndexOf('/')
|
|
611
|
+
return slash === -1 ? modelID : modelID.slice(slash + 1)
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/**
|
|
615
|
+
* Call the multimodal model directly (OpenAI-compatible /chat/completions on
|
|
616
|
+
* the opencode Zen endpoint). Returns the raw description text, or null on any
|
|
617
|
+
* failure (HTTP error, network error, timeout, empty content) — the caller
|
|
618
|
+
* falls back to the MCP tool pattern. The image never reaches the main
|
|
619
|
+
* provider: it is sent only here and replaced by the returned text.
|
|
620
|
+
*
|
|
621
|
+
* `prompt` overrides the default describe prompt (e.g. OCR extraction);
|
|
622
|
+
* `maxTokens` caps the completion (OCR needs more headroom); `signal` links
|
|
623
|
+
* an external abort (e.g. the plugin-tool ToolContext.abort) to the request.
|
|
624
|
+
*/
|
|
625
|
+
export async function describeImagesNative(
|
|
626
|
+
payloads: readonly NativeImagePayload[],
|
|
627
|
+
target: NativeVisionTarget,
|
|
628
|
+
prompt: string = NATIVE_PROMPT,
|
|
629
|
+
maxTokens: number = NATIVE_MAX_TOKENS,
|
|
630
|
+
signal?: AbortSignal,
|
|
631
|
+
): Promise<string | null> {
|
|
632
|
+
const controller = new AbortController()
|
|
633
|
+
const timer = setTimeout(() => controller.abort(), NATIVE_TIMEOUT_MS)
|
|
634
|
+
const onExternalAbort = () => controller.abort()
|
|
635
|
+
if (signal) {
|
|
636
|
+
if (signal.aborted) controller.abort()
|
|
637
|
+
else signal.addEventListener('abort', onExternalAbort, { once: true })
|
|
638
|
+
}
|
|
639
|
+
try {
|
|
640
|
+
const response = await fetch(`${target.baseURL}/chat/completions`, {
|
|
641
|
+
method: 'POST',
|
|
642
|
+
headers: {
|
|
643
|
+
'Content-Type': 'application/json',
|
|
644
|
+
Authorization: `Bearer ${target.apiKey}`,
|
|
645
|
+
},
|
|
646
|
+
body: JSON.stringify({
|
|
647
|
+
model: stripProviderPrefix(target.modelID),
|
|
648
|
+
messages: [
|
|
649
|
+
{
|
|
650
|
+
role: 'user',
|
|
651
|
+
content: [
|
|
652
|
+
{ type: 'text', text: prompt },
|
|
653
|
+
...payloads.map((payload) => ({
|
|
654
|
+
type: 'image_url',
|
|
655
|
+
image_url: { url: payload.url },
|
|
656
|
+
})),
|
|
657
|
+
],
|
|
658
|
+
},
|
|
659
|
+
],
|
|
660
|
+
max_tokens: maxTokens,
|
|
661
|
+
}),
|
|
662
|
+
signal: controller.signal,
|
|
663
|
+
})
|
|
664
|
+
if (!response.ok) return null
|
|
665
|
+
const data = (await response.json()) as {
|
|
666
|
+
choices?: Array<{ message?: { content?: string } }>
|
|
667
|
+
}
|
|
668
|
+
const content = data.choices?.[0]?.message?.content
|
|
669
|
+
if (typeof content !== 'string' || content.trim() === '') return null
|
|
670
|
+
return content.trim()
|
|
671
|
+
} catch {
|
|
672
|
+
return null
|
|
673
|
+
} finally {
|
|
674
|
+
clearTimeout(timer)
|
|
675
|
+
if (signal) signal.removeEventListener('abort', onExternalAbort)
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// ─── Session description cache ─────────────────────────────────────────────
|
|
680
|
+
// The native path stores the gateway's text answer per session, keyed by the
|
|
681
|
+
// user prompt + the sorted content hashes of every image payload. An identical
|
|
682
|
+
// re-ask within the TTL reuses the stored text and never contacts the gateway.
|
|
683
|
+
// Only textual descriptions are cached — never image bytes — and a cached
|
|
684
|
+
// description is injected as text (it is never re-sent as an image payload).
|
|
685
|
+
|
|
686
|
+
export type DescriptionCacheEntry = { text: string; storedAt: number }
|
|
687
|
+
|
|
688
|
+
/** sha256 hex of arbitrary content (prompt, data-URI, mime+bytes). */
|
|
689
|
+
export function hashContent(content: string | Buffer): string {
|
|
690
|
+
return createHash('sha256').update(content).digest('hex')
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Composite cache key: hash of the user prompt + the sorted content hashes of
|
|
695
|
+
* every image payload. The prompt is part of the key so a NEW question about
|
|
696
|
+
* the same image never reuses a stale description; sorting makes the key
|
|
697
|
+
* order-insensitive (the same image set in any order maps to one key). For
|
|
698
|
+
* data-URI/local payloads the URL encodes both mime and bytes, so hashing it
|
|
699
|
+
* covers "mime+bytes"; remote URLs hash the URL itself.
|
|
700
|
+
*/
|
|
701
|
+
export function buildDescriptionCacheKey(
|
|
702
|
+
prompt: string,
|
|
703
|
+
payloads: readonly { url: string }[],
|
|
704
|
+
): string {
|
|
705
|
+
const promptHash = hashContent(prompt)
|
|
706
|
+
const imageHashes = payloads.map((payload) => hashContent(payload.url)).sort()
|
|
707
|
+
return [promptHash, ...imageHashes].join(':')
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* Look up a cached description. Returns null on miss or when the entry is
|
|
712
|
+
* older than `ttlMs` — callers then hit the gateway and store the result.
|
|
713
|
+
*/
|
|
714
|
+
export function getCachedDescription(
|
|
715
|
+
cache: ReadonlyMap<string, DescriptionCacheEntry>,
|
|
716
|
+
key: string,
|
|
717
|
+
now: number = Date.now(),
|
|
718
|
+
ttlMs: number = DESCRIPTION_CACHE_TTL_MS,
|
|
719
|
+
): string | null {
|
|
720
|
+
const entry = cache.get(key)
|
|
721
|
+
if (!entry) return null
|
|
722
|
+
if (now - entry.storedAt >= ttlMs) return null
|
|
723
|
+
return entry.text
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/** Store a description with the current timestamp (overwrites on re-ask). */
|
|
727
|
+
export function setCachedDescription(
|
|
728
|
+
cache: Map<string, DescriptionCacheEntry>,
|
|
729
|
+
key: string,
|
|
730
|
+
text: string,
|
|
731
|
+
now: number = Date.now(),
|
|
732
|
+
): void {
|
|
733
|
+
cache.set(key, { text, storedAt: now })
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// ─── Prompt generation ─────────────────────────────────────────────────────
|
|
737
|
+
|
|
738
|
+
export function generateInjectionPrompt(
|
|
739
|
+
images: Array<{ path: string }>,
|
|
740
|
+
userText: string,
|
|
741
|
+
toolName: string,
|
|
742
|
+
template?: string,
|
|
743
|
+
): string {
|
|
744
|
+
if (!Array.isArray(images)) {
|
|
745
|
+
throw new TypeError('Vision image resolution must return an array')
|
|
746
|
+
}
|
|
747
|
+
const imageList = images.map((image, index) => `- Image ${index + 1}: ${image.path}`).join('\n')
|
|
748
|
+
if (template && PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))) {
|
|
749
|
+
return template
|
|
750
|
+
.replaceAll('{imageList}', imageList)
|
|
751
|
+
.replaceAll('{imageCount}', String(images.length))
|
|
752
|
+
.replaceAll('{toolName}', toolName)
|
|
753
|
+
.replaceAll('{userText}', userText)
|
|
754
|
+
}
|
|
755
|
+
const plural = images.length > 1
|
|
756
|
+
const heading = plural
|
|
757
|
+
? `The user has shared ${images.length} images. The images are available at:`
|
|
758
|
+
: 'The user has shared an image. The image is available at:'
|
|
759
|
+
const analyze = plural ? 'each image' : 'this image'
|
|
760
|
+
return [
|
|
761
|
+
heading,
|
|
762
|
+
imageList,
|
|
763
|
+
'',
|
|
764
|
+
`Use the \`${toolName}\` tool to analyze ${analyze}.`,
|
|
765
|
+
'',
|
|
766
|
+
`User's request: ${userText || '(analyze the image)'}`,
|
|
767
|
+
].join('\n')
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/**
|
|
771
|
+
* Injection text for the native-vision path: the description returned by the
|
|
772
|
+
* multimodal model, without any tool instruction — the main model already has
|
|
773
|
+
* the visual content in text form.
|
|
774
|
+
*/
|
|
775
|
+
export function generateNativeInjection(
|
|
776
|
+
description: string,
|
|
777
|
+
userText: string,
|
|
778
|
+
modelID: string,
|
|
779
|
+
): string {
|
|
780
|
+
return [
|
|
781
|
+
`The user shared an image. Vision description (native, ${modelID}):`,
|
|
782
|
+
description,
|
|
783
|
+
'',
|
|
784
|
+
`User's request: ${userText || '(analyze the image)'}`,
|
|
785
|
+
].join('\n')
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
// ─── Intent-calibrated native prompts + structured responses ───────────────
|
|
789
|
+
// The native vision path calibrates its prompt to the user's actual request
|
|
790
|
+
// (compare / ocr / reconstruct / bugs / describe) and asks the multimodal
|
|
791
|
+
// model for a structured, parseable answer: one `<item id="N"><description>…`
|
|
792
|
+
// block per image plus an optional `<context>` block for cross-image analysis.
|
|
793
|
+
// The parser maps each item back to its image in the text injected into the
|
|
794
|
+
// main model; any parse failure falls back to treating the whole gateway
|
|
795
|
+
// answer as a single description — the turn never breaks.
|
|
796
|
+
|
|
797
|
+
export type VisionIntent = 'compare' | 'ocr' | 'reconstruct' | 'bugs' | 'describe'
|
|
798
|
+
|
|
799
|
+
const INTENT_PATTERNS: Record<Exclude<VisionIntent, 'describe'>, RegExp> = {
|
|
800
|
+
compare:
|
|
801
|
+
/\b(compare|comparar|comparação|diferença|diferenças|difference|differences|qual é melhor|qual e melhor|which is better|versus|vs)\b/i,
|
|
802
|
+
ocr: /\b(texto|ler|read|ocr|extrair|extraia|extract|transcrever|transcreva|transcribe)\b/i,
|
|
803
|
+
reconstruct:
|
|
804
|
+
/\b(código|codigo|code|implementar|implemente|reconstruir|reconstrua|html|css|component|ui|tela|screen)\b/i,
|
|
805
|
+
bugs: /\b(bug|erro|problema|issue|debug|não funciona|nao funciona|not working)\b/i,
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/**
|
|
809
|
+
* Classify the user's request into a vision intent, in priority order:
|
|
810
|
+
* `compare` (requires 2+ images) → `ocr` → `reconstruct` → `bugs` → `describe`.
|
|
811
|
+
* Term matching is case-insensitive, word-boundary based, PT + EN.
|
|
812
|
+
*/
|
|
813
|
+
export function detectVisionIntent(userText: string, imageCount: number): VisionIntent {
|
|
814
|
+
if (imageCount >= 2 && INTENT_PATTERNS.compare.test(userText)) return 'compare'
|
|
815
|
+
if (INTENT_PATTERNS.ocr.test(userText)) return 'ocr'
|
|
816
|
+
if (INTENT_PATTERNS.reconstruct.test(userText)) return 'reconstruct'
|
|
817
|
+
if (INTENT_PATTERNS.bugs.test(userText)) return 'bugs'
|
|
818
|
+
return 'describe'
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
/** Escape `&`, `<`, `>` as XML entities (& first) — for template interpolation. */
|
|
822
|
+
export function escapeXml(text: string): string {
|
|
823
|
+
return text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>')
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/** Reverse of `escapeXml` — applied to parsed `<description>`/`<context>` bodies. */
|
|
827
|
+
function unescapeXml(text: string): string {
|
|
828
|
+
return text.replaceAll('<', '<').replaceAll('>', '>').replaceAll('&', '&')
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
const INTENT_TASK_INSTRUCTIONS: Record<VisionIntent, string> = {
|
|
832
|
+
describe:
|
|
833
|
+
'Descreva cada imagem fielmente e em detalhes: texto visível verbatim, cores, layout, objetos.',
|
|
834
|
+
compare:
|
|
835
|
+
'Descreva cada imagem no seu próprio bloco <item>; depois coloque uma comparação direta e exaustiva no <context> (diferenças por região/elemento), terminando com um veredito claro.',
|
|
836
|
+
ocr: 'Transcreva literalmente todo o texto visível em cada imagem, preservando ordem e estrutura. A transcrição vai dentro do <description> da imagem correspondente.',
|
|
837
|
+
reconstruct:
|
|
838
|
+
'Descreva cada imagem com detalhe de reconstrução para reprodução em código: layout completo (regiões, alinhamento, espaçamento), todo texto visível verbatim, paleta de cores exata em hex, fontes/tamanhos/pesos por elemento, estrutura de componentes e estados.',
|
|
839
|
+
bugs: 'Descreva o sintoma visual observado em cada imagem: o que está errado, onde, esperado vs real, com referências de região precisas e contexto suficiente para localizar e reproduzir.',
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
const INTENT_CONTEXT_INSTRUCTIONS: Record<VisionIntent, string> = {
|
|
843
|
+
compare:
|
|
844
|
+
'Depois dos itens, adicione UM bloco <context> com a análise cruzada das imagens (comparação, veredito).',
|
|
845
|
+
describe: '',
|
|
846
|
+
ocr: '',
|
|
847
|
+
reconstruct: '',
|
|
848
|
+
bugs: '',
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Build the calibrated prompt for the native vision gateway call. The model is
|
|
853
|
+
* instructed to reply with N `<item id="N"><description>…</description></item>`
|
|
854
|
+
* blocks (N = imageCount) and, for `compare`, one extra `<context>` block.
|
|
855
|
+
* The user's text is XML-escaped when interpolated so it cannot break the
|
|
856
|
+
* format instructions. `intent` is auto-detected from `userText` +
|
|
857
|
+
* `imageCount` when omitted.
|
|
858
|
+
*/
|
|
859
|
+
export function buildNativeVisionPrompt(
|
|
860
|
+
userText: string,
|
|
861
|
+
imageCount: number,
|
|
862
|
+
intent: VisionIntent = detectVisionIntent(userText, imageCount),
|
|
863
|
+
): string {
|
|
864
|
+
const count = Math.max(1, imageCount)
|
|
865
|
+
const request =
|
|
866
|
+
escapeXml(userText.trim()) || '(sem pedido explícito — descreva as imagens com precisão)'
|
|
867
|
+
const contextInstruction = INTENT_CONTEXT_INSTRUCTIONS[intent]
|
|
868
|
+
const contextLine = contextInstruction ? ` ${contextInstruction}` : ''
|
|
869
|
+
return [
|
|
870
|
+
'You are a multimodal analysis specialist embedded inside a coding assistant. The coding',
|
|
871
|
+
'assistant CANNOT perceive the images in this turn — your output is its ONLY view of them.',
|
|
872
|
+
'It will act on exactly what you write.',
|
|
873
|
+
'',
|
|
874
|
+
"READ THE USER REQUEST FIRST: infer the user's goal and calibrate the depth of each",
|
|
875
|
+
'description to it.',
|
|
876
|
+
'',
|
|
877
|
+
INTENT_TASK_INSTRUCTIONS[intent],
|
|
878
|
+
'',
|
|
879
|
+
'OUTPUT FORMAT — binding, no exceptions:',
|
|
880
|
+
`Responda com UM bloco <item id="N"> para cada uma das ${count} imagens anexadas`,
|
|
881
|
+
`(N = 1..${count}), em ordem. Cada bloco contém um <description>.${contextLine}`,
|
|
882
|
+
'',
|
|
883
|
+
'<item id="1">',
|
|
884
|
+
'<description>...apenas fatos sobre a imagem 1...</description>',
|
|
885
|
+
'</item>',
|
|
886
|
+
'<item id="2">',
|
|
887
|
+
'<description>...apenas fatos sobre a imagem 2...</description>',
|
|
888
|
+
'</item>',
|
|
889
|
+
'',
|
|
890
|
+
'RULES',
|
|
891
|
+
'- O primeiro caractere da sua resposta deve ser "<". Sem preâmbulo, sem "Aqui está",',
|
|
892
|
+
' sem observações finais, sem wrap em blocos de código.',
|
|
893
|
+
'- ESCAPING — obrigatório: dentro dos corpos de <description> e <context>, escape TODO',
|
|
894
|
+
' "&", "<" e ">" literais como "&", "<", ">". Isso vale para qualquer texto',
|
|
895
|
+
' tag-like reproduzido verbatim (HTML, XML, erros com colchetes angulares). NÃO escape',
|
|
896
|
+
' as tags estruturais <item>, <description>, <context>.',
|
|
897
|
+
'- Nunca invente detalhes que não consegue perceber. Se um texto estiver ilegível,',
|
|
898
|
+
' escreva "ilegível". Coloque análises cruzadas (comparação, veredito) no <context>,',
|
|
899
|
+
' nunca dentro de um <description>.',
|
|
900
|
+
'',
|
|
901
|
+
`User's request: ${request}`,
|
|
902
|
+
].join('\n')
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
export type StructuredVisionItem = { id: number; description: string }
|
|
906
|
+
export type StructuredVisionResponse = { items: StructuredVisionItem[]; context: string | null }
|
|
907
|
+
|
|
908
|
+
const ITEM_BLOCK_RE = /<item\b[^>]*>([\s\S]*?)<\/item>/gi
|
|
909
|
+
const OPEN_TAG_RE = /<item\b[^>]*>/i
|
|
910
|
+
const DESCRIPTION_BODY_RE = /<description>([\s\S]*?)<\/description>/i
|
|
911
|
+
const CONTEXT_BODY_RE = /<context>([\s\S]*?)<\/context>/i
|
|
912
|
+
const ITEM_ID_ATTR_RE = /id\s*=\s*["']?(\d+)["']?/i
|
|
913
|
+
const BARE_DESCRIPTION_RE = /<description>([\s\S]*?)<\/description>/gi
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* Parse the gateway's structured answer into per-image `<item>` descriptions
|
|
917
|
+
* plus an optional `<context>` block. Resistant to formatting variations
|
|
918
|
+
* (whitespace, quotes, tag case, missing id → sequential ids, markdown fences
|
|
919
|
+
* around the XML). XML entities in the bodies are unescaped for the main
|
|
920
|
+
* model. Returns null when no `<item>`/`<description>` block is found —
|
|
921
|
+
* callers then treat the whole answer as a single description (the
|
|
922
|
+
* pre-structured behavior, no extra gateway call).
|
|
923
|
+
*/
|
|
924
|
+
export function parseStructuredVisionResponse(text: string): StructuredVisionResponse | null {
|
|
925
|
+
const items: StructuredVisionItem[] = []
|
|
926
|
+
let match = ITEM_BLOCK_RE.exec(text)
|
|
927
|
+
while (match !== null) {
|
|
928
|
+
const desc = DESCRIPTION_BODY_RE.exec(match[1] ?? '')
|
|
929
|
+
if (desc) {
|
|
930
|
+
const openTag = OPEN_TAG_RE.exec(match[0] ?? '')?.[0] ?? ''
|
|
931
|
+
const rawId = ITEM_ID_ATTR_RE.exec(openTag)?.[1]
|
|
932
|
+
const description = unescapeXml((desc[1] ?? '').trim())
|
|
933
|
+
items.push({
|
|
934
|
+
id: rawId !== undefined ? Number(rawId) : items.length + 1,
|
|
935
|
+
description,
|
|
936
|
+
})
|
|
937
|
+
}
|
|
938
|
+
match = ITEM_BLOCK_RE.exec(text)
|
|
939
|
+
}
|
|
940
|
+
if (items.length === 0) {
|
|
941
|
+
let bareMatch = BARE_DESCRIPTION_RE.exec(text)
|
|
942
|
+
while (bareMatch !== null) {
|
|
943
|
+
items.push({
|
|
944
|
+
id: items.length + 1,
|
|
945
|
+
description: unescapeXml((bareMatch[1] ?? '').trim()),
|
|
946
|
+
})
|
|
947
|
+
bareMatch = BARE_DESCRIPTION_RE.exec(text)
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
const contextMatch = CONTEXT_BODY_RE.exec(text)
|
|
951
|
+
const context = contextMatch ? unescapeXml((contextMatch[1] ?? '').trim()) : null
|
|
952
|
+
return items.length > 0 ? { items, context } : null
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Build the text injected into the main model from a parsed structured
|
|
957
|
+
* response: each `<item id>` is mapped to its image (by id, falling back to
|
|
958
|
+
* position when an id is missing), followed by the optional cross-image
|
|
959
|
+
* `<context>` block. The fallback single-description path stays in
|
|
960
|
+
* `generateNativeInjection`.
|
|
961
|
+
*/
|
|
962
|
+
export function buildStructuredInjection(
|
|
963
|
+
items: readonly StructuredVisionItem[],
|
|
964
|
+
context: string | null,
|
|
965
|
+
userText: string,
|
|
966
|
+
modelID: string,
|
|
967
|
+
imageCount: number,
|
|
968
|
+
): string {
|
|
969
|
+
const count = Math.max(1, imageCount)
|
|
970
|
+
const byId = new Map(items.map((item) => [item.id, item.description]))
|
|
971
|
+
const header =
|
|
972
|
+
count > 1
|
|
973
|
+
? `The user shared ${count} images. Vision descriptions (native, ${modelID}):`
|
|
974
|
+
: `The user shared an image. Vision description (native, ${modelID}):`
|
|
975
|
+
const lines = [header]
|
|
976
|
+
for (let i = 1; i <= count; i += 1) {
|
|
977
|
+
const description = byId.get(i) ?? items[i - 1]?.description
|
|
978
|
+
lines.push(`- Image ${i}: ${description ?? '(no description provided)'}`)
|
|
979
|
+
}
|
|
980
|
+
if (context) {
|
|
981
|
+
lines.push('', 'Cross-image context:', context)
|
|
982
|
+
}
|
|
983
|
+
lines.push('', `User's request: ${userText || '(analyze the image)'}`)
|
|
984
|
+
return lines.join('\n')
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// ─── Tool name resolution ──────────────────────────────────────────────────
|
|
988
|
+
// env PANTHEON_VISION_TOOL > config imageAnalysisTool > dynamic detection
|
|
989
|
+
// (prefer the canonical Pantheon MCP) > canonical Pantheon MCP default —
|
|
990
|
+
// but ONLY when real evidence says the MCP is available (P2-3).
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Real evidence that the pantheon-vision MCP is available at runtime:
|
|
994
|
+
* - its tools appear in client.tool.ids() (direct), OR
|
|
995
|
+
* - client.mcp.status() lists it with status 'connected' (registration).
|
|
996
|
+
* With --no-mcp / an uninstalled MCP there is NO such evidence, and
|
|
997
|
+
* defaulting to the canonical tool would make the model call a tool that
|
|
998
|
+
* does not exist — callers must return null instead ('Vision fallback
|
|
999
|
+
* unavailable').
|
|
1000
|
+
*/
|
|
1001
|
+
async function hasPantheonVisionMcp(
|
|
1002
|
+
client: PluginInput['client'],
|
|
1003
|
+
directory: string,
|
|
1004
|
+
): Promise<boolean> {
|
|
1005
|
+
try {
|
|
1006
|
+
const idsResponse = await client.tool?.ids?.({ query: { directory } })
|
|
1007
|
+
const ids = idsResponse?.data
|
|
1008
|
+
if (Array.isArray(ids) && ids.some((id) => isPantheonVisionServerTool(id))) return true
|
|
1009
|
+
} catch {
|
|
1010
|
+
// Fall through to the MCP status map.
|
|
1011
|
+
}
|
|
1012
|
+
try {
|
|
1013
|
+
const mcpResponse = await client.mcp?.status?.({ query: { directory } })
|
|
1014
|
+
const statuses = mcpResponse?.data
|
|
1015
|
+
if (statuses && typeof statuses === 'object') {
|
|
1016
|
+
for (const [name, status] of Object.entries(statuses)) {
|
|
1017
|
+
if (!/[pP]antheon[-_]vision/.test(name)) continue
|
|
1018
|
+
if ((status as { status?: string } | undefined)?.status === 'connected') return true
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
} catch {
|
|
1022
|
+
// Fall through — no evidence.
|
|
1023
|
+
}
|
|
1024
|
+
return false
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
async function resolveImageAnalysisTool(
|
|
1028
|
+
client: PluginInput['client'],
|
|
1029
|
+
directory: string,
|
|
1030
|
+
configPromise: Promise<VisionConfig>,
|
|
1031
|
+
): Promise<string | null> {
|
|
1032
|
+
const envTool = process.env.PANTHEON_VISION_TOOL?.trim()
|
|
1033
|
+
if (envTool) return envTool
|
|
1034
|
+
const config = await configPromise
|
|
1035
|
+
if (config.imageAnalysisTool) return config.imageAnalysisTool
|
|
1036
|
+
try {
|
|
1037
|
+
const response = await client.tool?.ids?.({ query: { directory } })
|
|
1038
|
+
const ids = response?.data
|
|
1039
|
+
if (Array.isArray(ids)) {
|
|
1040
|
+
// Exact canonical ID first (fast path), then the underscore form and the
|
|
1041
|
+
// legacy mcp__ hyphen form (both match the pantheon[-_]vision + action
|
|
1042
|
+
// regexes), then the loose describe fallback for older ID shapes.
|
|
1043
|
+
const match =
|
|
1044
|
+
ids.find((id) => id === DEFAULT_IMAGE_ANALYSIS_TOOL) ??
|
|
1045
|
+
ids.find((id) => isPantheonVisionServerTool(id) && isPantheonVisionActionTool(id)) ??
|
|
1046
|
+
ids.find((id) => isPantheonVisionServerTool(id) && /describe/i.test(id))
|
|
1047
|
+
if (match) return match
|
|
1048
|
+
}
|
|
1049
|
+
} catch {
|
|
1050
|
+
// Detection failed (network/MCP error). Returning null keeps the explicit
|
|
1051
|
+
// 'Vision fallback unavailable' safe-failure message in the turn instead
|
|
1052
|
+
// of guessing at a tool we could not verify — the turn never breaks
|
|
1053
|
+
// (interception is best-effort). Returning DEFAULT here too would hide
|
|
1054
|
+
// real detection failures behind a tool name the model might not have.
|
|
1055
|
+
return null
|
|
1056
|
+
}
|
|
1057
|
+
// OpenCode 1.18.11 keeps MCP tools in the MCP service, NOT in
|
|
1058
|
+
// client.tool.ids() — an empty/absent list is the NORMAL runtime state, so
|
|
1059
|
+
// fall back to the canonical default ONLY when there is independent
|
|
1060
|
+
// evidence the pantheon-vision MCP is installed and connected. Without it
|
|
1061
|
+
// (--no-mcp, uninstalled, disabled) instructing the canonical tool would
|
|
1062
|
+
// fail the turn with a phantom tool — return null instead.
|
|
1063
|
+
const available = await hasPantheonVisionMcp(client, directory)
|
|
1064
|
+
return available ? DEFAULT_IMAGE_ANALYSIS_TOOL : null
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// ─── Temp image lifecycle ──────────────────────────────────────────────────
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* Age-guarded, REFCOUNTED cleanup of a session's temp images.
|
|
1071
|
+
*
|
|
1072
|
+
* A `session.idle` can fire between auto-continue turns — not just at real
|
|
1073
|
+
* session end — so a temp file referenced by the next turn (seconds old) must
|
|
1074
|
+
* survive. Only paths older than `graceMs` are unlinked; younger paths stay
|
|
1075
|
+
* registered and are swept by the next idle. `server.instance.disposed` and
|
|
1076
|
+
* `session.deleted` pass `graceMs = 0` to unlink everything while the session
|
|
1077
|
+
* is gone / process is dying.
|
|
1078
|
+
*
|
|
1079
|
+
* Content-hash filenames dedup identical images ACROSS sessions: two sessions
|
|
1080
|
+
* can reference the SAME path. `tempFileRefs` (path → set of sessions) makes
|
|
1081
|
+
* the unlink decision reference-aware — the file is only unlinked when the
|
|
1082
|
+
* LAST referencing session releases it; otherwise the other session's MCP
|
|
1083
|
+
* tool call would hit file-not-found (P2-4). When `tempFileRefs` is omitted
|
|
1084
|
+
* (legacy callers) the unconditional behavior is kept.
|
|
1085
|
+
*
|
|
1086
|
+
* The temp directory itself is never removed: it may hold files from other
|
|
1087
|
+
* sessions or processes. The session's description cache is always dropped
|
|
1088
|
+
* (cache is per-session state).
|
|
1089
|
+
*/
|
|
1090
|
+
export async function cleanupSessionTempImages(
|
|
1091
|
+
sessionTempFiles: Map<string, Map<string, number>>,
|
|
1092
|
+
sessionVisionText: Map<string, Map<string, string>>,
|
|
1093
|
+
nativeVisionSessions: Set<string>,
|
|
1094
|
+
sessionID: string,
|
|
1095
|
+
graceMs: number = TEMP_FILE_GRACE_MS,
|
|
1096
|
+
sessionDescriptionCache?: Map<string, Map<string, DescriptionCacheEntry>>,
|
|
1097
|
+
tempFileRefs?: Map<string, Set<string>>,
|
|
1098
|
+
): Promise<void> {
|
|
1099
|
+
const files = sessionTempFiles.get(sessionID)
|
|
1100
|
+
if (files) {
|
|
1101
|
+
const now = Date.now()
|
|
1102
|
+
for (const [path, createdAt] of files) {
|
|
1103
|
+
// graceMs === 0 means "unlink everything now" (session gone / process
|
|
1104
|
+
// dying); a positive grace skips files younger than the threshold so the
|
|
1105
|
+
// next auto-continue turn can still read them.
|
|
1106
|
+
if (graceMs > 0 && now - createdAt <= graceMs) continue
|
|
1107
|
+
const refs = tempFileRefs?.get(path)
|
|
1108
|
+
if (refs && tempFileRefs) {
|
|
1109
|
+
refs.delete(sessionID)
|
|
1110
|
+
// Another session still references this content-hash path: keep the
|
|
1111
|
+
// file on disk and drop this session's tracking of it.
|
|
1112
|
+
if (refs.size > 0) {
|
|
1113
|
+
files.delete(path)
|
|
1114
|
+
continue
|
|
1115
|
+
}
|
|
1116
|
+
tempFileRefs.delete(path)
|
|
1117
|
+
}
|
|
1118
|
+
try {
|
|
1119
|
+
await unlink(path)
|
|
1120
|
+
} catch {
|
|
1121
|
+
// Best effort.
|
|
1122
|
+
}
|
|
1123
|
+
forgetTempFile(path)
|
|
1124
|
+
files.delete(path)
|
|
1125
|
+
}
|
|
1126
|
+
if (files.size === 0) sessionTempFiles.delete(sessionID)
|
|
1127
|
+
}
|
|
1128
|
+
sessionVisionText.delete(sessionID)
|
|
1129
|
+
nativeVisionSessions.delete(sessionID)
|
|
1130
|
+
sessionDescriptionCache?.delete(sessionID)
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
type VisionHistoryMessage = {
|
|
1134
|
+
info?: { sessionID?: string; id?: string }
|
|
1135
|
+
sessionID?: string
|
|
1136
|
+
id?: string
|
|
1137
|
+
parts?: unknown
|
|
1138
|
+
content?: unknown
|
|
1139
|
+
[key: string]: unknown
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
function syntheticHistoryPart(message: VisionHistoryMessage, text: string): TextPart {
|
|
1143
|
+
return {
|
|
1144
|
+
id: syntheticPartId(),
|
|
1145
|
+
sessionID: message.info?.sessionID ?? '',
|
|
1146
|
+
messageID: message.info?.id ?? '',
|
|
1147
|
+
type: 'text',
|
|
1148
|
+
text,
|
|
1149
|
+
synthetic: true,
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
/**
|
|
1154
|
+
* Remove image-bearing parts before a provider serializer can turn them into
|
|
1155
|
+
* image_url content. This hook is deliberately synchronous in spirit: it only
|
|
1156
|
+
* reuses text cached by chat.message and never calls a vision tool/model.
|
|
1157
|
+
*/
|
|
1158
|
+
export function sanitizeVisionHistory(
|
|
1159
|
+
output: { messages?: unknown },
|
|
1160
|
+
sessionVisionText: ReadonlyMap<string, ReadonlyMap<string, string>>,
|
|
1161
|
+
nativeVisionSessions: ReadonlySet<string> = new Set(),
|
|
1162
|
+
): void {
|
|
1163
|
+
if (!Array.isArray(output.messages)) return
|
|
1164
|
+
|
|
1165
|
+
for (const message of output.messages) {
|
|
1166
|
+
if (!message || typeof message !== 'object') continue
|
|
1167
|
+
const historyMessage = message as VisionHistoryMessage
|
|
1168
|
+
const sessionID = historyMessage.info?.sessionID ?? historyMessage.sessionID ?? ''
|
|
1169
|
+
if (nativeVisionSessions.has(sessionID)) continue
|
|
1170
|
+
const descriptions = sessionVisionText.get(sessionID)
|
|
1171
|
+
const sanitize = (value: unknown, inParts: boolean): unknown => {
|
|
1172
|
+
if (Array.isArray(value)) return value.map((item) => sanitize(item, inParts))
|
|
1173
|
+
if (!value || typeof value !== 'object') return value
|
|
1174
|
+
|
|
1175
|
+
const raw = value as { type?: unknown; id?: unknown; mime?: unknown }
|
|
1176
|
+
const isImageUrl = raw.type === 'image_url'
|
|
1177
|
+
const isImageFile =
|
|
1178
|
+
raw.type === 'file' &&
|
|
1179
|
+
typeof raw.mime === 'string' &&
|
|
1180
|
+
raw.mime.toLowerCase().startsWith('image/')
|
|
1181
|
+
if (isImageUrl || isImageFile) {
|
|
1182
|
+
const partID = typeof raw.id === 'string' ? raw.id : ''
|
|
1183
|
+
const text =
|
|
1184
|
+
descriptions?.get(partID) ??
|
|
1185
|
+
'Imagem removida do histórico antes do envio ao provedor textual.'
|
|
1186
|
+
return inParts ? syntheticHistoryPart(historyMessage, text) : { type: 'text', text }
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
const copy: Record<string, unknown> = {}
|
|
1190
|
+
for (const [key, child] of Object.entries(value)) {
|
|
1191
|
+
copy[key] = sanitize(child, key === 'parts')
|
|
1192
|
+
}
|
|
1193
|
+
return copy
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
const sanitized = sanitize(historyMessage, false) as VisionHistoryMessage
|
|
1197
|
+
for (const [key, value] of Object.entries(sanitized)) {
|
|
1198
|
+
historyMessage[key] = value
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// ─── Handler factory ───────────────────────────────────────────────────────
|
|
1204
|
+
|
|
1205
|
+
export function createVisionHandler(input: PluginInput) {
|
|
1206
|
+
const { client, directory } = input
|
|
1207
|
+
const sessionTempFiles = new Map<string, Map<string, number>>()
|
|
1208
|
+
// Content-hash dedup makes identical images share one path ACROSS sessions;
|
|
1209
|
+
// this map (path → sessions holding it) makes cleanup reference-aware so a
|
|
1210
|
+
// file is only unlinked when the LAST session releases it (P2-4).
|
|
1211
|
+
const tempFileRefs = new Map<string, Set<string>>()
|
|
1212
|
+
const sessionVisionText = new Map<string, Map<string, string>>()
|
|
1213
|
+
const nativeVisionSessions = new Set<string>()
|
|
1214
|
+
const sessionDescriptionCache = new Map<string, Map<string, DescriptionCacheEntry>>()
|
|
1215
|
+
let configPromise: Promise<VisionConfig> | null = null
|
|
1216
|
+
let toolNamePromise: Promise<string | null> | null = null
|
|
1217
|
+
|
|
1218
|
+
const getConfig = (): Promise<VisionConfig> => {
|
|
1219
|
+
configPromise ??= loadVisionConfig(directory)
|
|
1220
|
+
return configPromise
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
const getToolName = (): Promise<string | null> => {
|
|
1224
|
+
toolNamePromise ??= resolveImageAnalysisTool(client, directory, getConfig())
|
|
1225
|
+
return toolNamePromise
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
const chatMessage: NonNullable<Hooks['chat.message']> = async (hookInput, output) => {
|
|
1229
|
+
try {
|
|
1230
|
+
const message: UserMessage | undefined = output?.message
|
|
1231
|
+
if (!message || message.role !== 'user') return
|
|
1232
|
+
const model = hookInput?.model ?? message.model
|
|
1233
|
+
if (modelAcceptsImages(model)) {
|
|
1234
|
+
nativeVisionSessions.add(hookInput.sessionID)
|
|
1235
|
+
return
|
|
1236
|
+
}
|
|
1237
|
+
nativeVisionSessions.delete(hookInput.sessionID)
|
|
1238
|
+
const config = await getConfig()
|
|
1239
|
+
const patterns =
|
|
1240
|
+
config.models && config.models.length > 0 ? config.models : DEFAULT_MODEL_PATTERNS
|
|
1241
|
+
if (!modelMatchesAnyPattern(model, patterns)) {
|
|
1242
|
+
// Pattern configuration may disable enrichment, but it must never
|
|
1243
|
+
// disable the text-only provider safety boundary.
|
|
1244
|
+
const currentTurn = { messages: [{ ...message, parts: output.parts }] }
|
|
1245
|
+
sanitizeVisionHistory(currentTurn, sessionVisionText)
|
|
1246
|
+
const sanitized = currentTurn.messages as VisionHistoryMessage[]
|
|
1247
|
+
output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
|
|
1248
|
+
return
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
const parts: Part[] = Array.isArray(output.parts) ? output.parts : []
|
|
1252
|
+
const images = parts.filter(isImageFilePart)
|
|
1253
|
+
if (images.length === 0) {
|
|
1254
|
+
// Some runtimes hand the current turn to plugins already serialized as
|
|
1255
|
+
// provider content. Do not let that shape bypass the final sanitizer.
|
|
1256
|
+
const currentTurn = { messages: [{ ...message, parts: output.parts }] }
|
|
1257
|
+
sanitizeVisionHistory(currentTurn, sessionVisionText)
|
|
1258
|
+
const sanitized = currentTurn.messages as VisionHistoryMessage[]
|
|
1259
|
+
output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
|
|
1260
|
+
return
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
const userText = parts
|
|
1264
|
+
.filter(isTextPart)
|
|
1265
|
+
.map((part) => part.text)
|
|
1266
|
+
.join('\n')
|
|
1267
|
+
.trim()
|
|
1268
|
+
|
|
1269
|
+
const resolved: Array<{ filePart: FilePart; target: SavedImage }> = []
|
|
1270
|
+
for (const image of images.filter(isSupportedImageFilePart)) {
|
|
1271
|
+
const target = await resolveImageTarget(image)
|
|
1272
|
+
if (target) resolved.push({ filePart: image, target })
|
|
1273
|
+
}
|
|
1274
|
+
const saved = resolved.map((item) => item.target)
|
|
1275
|
+
|
|
1276
|
+
const tempPaths = saved.filter((item) => item.temporary).map((item) => item.path)
|
|
1277
|
+
if (tempPaths.length > 0) {
|
|
1278
|
+
const createdAt = Date.now()
|
|
1279
|
+
const existing = sessionTempFiles.get(hookInput.sessionID) ?? new Map<string, number>()
|
|
1280
|
+
for (const path of tempPaths) {
|
|
1281
|
+
existing.set(path, createdAt)
|
|
1282
|
+
// Track the session as a holder of this (content-hash) path so
|
|
1283
|
+
// cleanup only unlinks when the LAST holder releases it.
|
|
1284
|
+
let refs = tempFileRefs.get(path)
|
|
1285
|
+
if (!refs) {
|
|
1286
|
+
refs = new Set()
|
|
1287
|
+
tempFileRefs.set(path, refs)
|
|
1288
|
+
}
|
|
1289
|
+
refs.add(hookInput.sessionID)
|
|
1290
|
+
}
|
|
1291
|
+
sessionTempFiles.set(hookInput.sessionID, existing)
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// Native-first: call the multimodal model directly when a key exists
|
|
1295
|
+
// (mode auto/native). Any failure — no key, unsupported provider,
|
|
1296
|
+
// oversized/invalid image, HTTP/network error — falls back to the tool.
|
|
1297
|
+
// The prompt is calibrated to the user's intent (compare/ocr/
|
|
1298
|
+
// reconstruct/bugs/describe) and asks for a structured answer
|
|
1299
|
+
// (<item>/<context>), which is parsed and mapped back to each image.
|
|
1300
|
+
// Descriptions are cached per session: the key is the calibrated prompt
|
|
1301
|
+
// + the content hashes of the images, so an identical re-ask within the
|
|
1302
|
+
// TTL reuses the stored (possibly structured) text without another
|
|
1303
|
+
// gateway call, while a new question about the same image misses and is
|
|
1304
|
+
// answered freshly.
|
|
1305
|
+
let injection: string | null = null
|
|
1306
|
+
if (getVisionMode(process.env, config.mode) !== 'tool') {
|
|
1307
|
+
try {
|
|
1308
|
+
const target = await resolveNativeVisionTarget({ config })
|
|
1309
|
+
if (target) {
|
|
1310
|
+
const payloads = await buildNativeImagePayloads(resolved)
|
|
1311
|
+
if (payloads) {
|
|
1312
|
+
let sessionCache = sessionDescriptionCache.get(hookInput.sessionID)
|
|
1313
|
+
if (!sessionCache) {
|
|
1314
|
+
sessionCache = new Map()
|
|
1315
|
+
sessionDescriptionCache.set(hookInput.sessionID, sessionCache)
|
|
1316
|
+
}
|
|
1317
|
+
const intent = detectVisionIntent(userText, payloads.length)
|
|
1318
|
+
const prompt = buildNativeVisionPrompt(userText, payloads.length, intent)
|
|
1319
|
+
const cacheKey = buildDescriptionCacheKey(prompt, payloads)
|
|
1320
|
+
let rawText = getCachedDescription(sessionCache, cacheKey)
|
|
1321
|
+
if (rawText === null) {
|
|
1322
|
+
rawText = await describeImagesNative(payloads, target, prompt)
|
|
1323
|
+
if (rawText !== null) setCachedDescription(sessionCache, cacheKey, rawText)
|
|
1324
|
+
}
|
|
1325
|
+
if (rawText !== null) {
|
|
1326
|
+
const structured = parseStructuredVisionResponse(rawText)
|
|
1327
|
+
injection = structured
|
|
1328
|
+
? buildStructuredInjection(
|
|
1329
|
+
structured.items,
|
|
1330
|
+
structured.context,
|
|
1331
|
+
userText,
|
|
1332
|
+
target.modelID,
|
|
1333
|
+
payloads.length,
|
|
1334
|
+
)
|
|
1335
|
+
: generateNativeInjection(rawText, userText, target.modelID)
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
} catch {
|
|
1340
|
+
// Native vision is best-effort; fall back to the tool pattern.
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
if (injection === null && saved.length > 0) {
|
|
1345
|
+
const toolName = await getToolName()
|
|
1346
|
+
injection = toolName
|
|
1347
|
+
? generateInjectionPrompt(saved, userText, toolName, config.promptTemplate)
|
|
1348
|
+
: 'Vision fallback unavailable: pantheon-vision MCP is unavailable. The image was removed safely; please configure the MCP server and retry.'
|
|
1349
|
+
}
|
|
1350
|
+
if (injection === null) {
|
|
1351
|
+
injection = 'A imagem compartilhada foi removida antes do envio ao provedor textual.'
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
const descriptions = sessionVisionText.get(hookInput.sessionID) ?? new Map<string, string>()
|
|
1355
|
+
for (const image of images) descriptions.set(image.id, injection)
|
|
1356
|
+
sessionVisionText.set(hookInput.sessionID, descriptions)
|
|
1357
|
+
|
|
1358
|
+
const injectionPart: TextPart = {
|
|
1359
|
+
id: syntheticPartId(),
|
|
1360
|
+
sessionID: message.sessionID,
|
|
1361
|
+
messageID: message.id,
|
|
1362
|
+
type: 'text',
|
|
1363
|
+
text: injection,
|
|
1364
|
+
synthetic: true,
|
|
1365
|
+
}
|
|
1366
|
+
const kept = parts.filter((part) => !isImageFilePart(part))
|
|
1367
|
+
// Mutate in place: drop image parts, lead with the vision text
|
|
1368
|
+
// (native description or tool instruction).
|
|
1369
|
+
output.parts.splice(0, output.parts.length, injectionPart, ...kept)
|
|
1370
|
+
sanitizeVisionHistory({ messages: [{ ...message, parts: output.parts }] }, sessionVisionText)
|
|
1371
|
+
} catch {
|
|
1372
|
+
// Vision interception is best-effort; never fail a user turn.
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
const event: NonNullable<Hooks['event']> = async ({ event: ev }) => {
|
|
1377
|
+
if (ev.type === 'session.idle') {
|
|
1378
|
+
// An idle can fire between auto-continue turns: temp files younger than
|
|
1379
|
+
// the grace period survive, but the per-session description cache is
|
|
1380
|
+
// dropped (it is pure optimization, safe to reset on every idle).
|
|
1381
|
+
const sessionID = ev.properties?.sessionID
|
|
1382
|
+
if (sessionID)
|
|
1383
|
+
await cleanupSessionTempImages(
|
|
1384
|
+
sessionTempFiles,
|
|
1385
|
+
sessionVisionText,
|
|
1386
|
+
nativeVisionSessions,
|
|
1387
|
+
sessionID,
|
|
1388
|
+
TEMP_FILE_GRACE_MS,
|
|
1389
|
+
sessionDescriptionCache,
|
|
1390
|
+
tempFileRefs,
|
|
1391
|
+
)
|
|
1392
|
+
} else if (ev.type === 'session.deleted') {
|
|
1393
|
+
// The session is gone for good: no active turn can reference its temp
|
|
1394
|
+
// images, so unlink them all immediately and drop its cached state.
|
|
1395
|
+
// Shared paths survive while another session still references them.
|
|
1396
|
+
const sessionID = ev.properties.info.id
|
|
1397
|
+
await cleanupSessionTempImages(
|
|
1398
|
+
sessionTempFiles,
|
|
1399
|
+
sessionVisionText,
|
|
1400
|
+
nativeVisionSessions,
|
|
1401
|
+
sessionID,
|
|
1402
|
+
0,
|
|
1403
|
+
sessionDescriptionCache,
|
|
1404
|
+
tempFileRefs,
|
|
1405
|
+
)
|
|
1406
|
+
} else if (ev.type === 'server.instance.disposed') {
|
|
1407
|
+
const sessions = [...sessionTempFiles.keys()]
|
|
1408
|
+
for (const sessionID of sessions) {
|
|
1409
|
+
await cleanupSessionTempImages(
|
|
1410
|
+
sessionTempFiles,
|
|
1411
|
+
sessionVisionText,
|
|
1412
|
+
nativeVisionSessions,
|
|
1413
|
+
sessionID,
|
|
1414
|
+
0, // The process is dying: unlink everything immediately.
|
|
1415
|
+
sessionDescriptionCache,
|
|
1416
|
+
tempFileRefs,
|
|
1417
|
+
)
|
|
1418
|
+
}
|
|
1419
|
+
sessionVisionText.clear()
|
|
1420
|
+
sessionDescriptionCache.clear()
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
const messagesTransform: NonNullable<Hooks['experimental.chat.messages.transform']> = async (
|
|
1425
|
+
_hookInput,
|
|
1426
|
+
output,
|
|
1427
|
+
) => {
|
|
1428
|
+
try {
|
|
1429
|
+
sanitizeVisionHistory(output, sessionVisionText, nativeVisionSessions)
|
|
1430
|
+
} catch {
|
|
1431
|
+
// History sanitization is best effort; never break runtimes that expose
|
|
1432
|
+
// the experimental hook with a partial output shape.
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
return { chatMessage, messagesTransform, event }
|
|
1437
|
+
}
|