pantheon-opencode 1.1.3-beta.0 → 1.2.0-beta.9.7bef03e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +74 -1
  2. package/bin/pantheon-init.mjs +79 -0
  3. package/commands/pantheon-vision.md +67 -0
  4. package/opencode.json +12 -41
  5. package/package.json +5 -1
  6. package/scripts/doctor.mjs +4 -2
  7. package/scripts/install/health-check.mjs +101 -1
  8. package/scripts/install/model-picker.mjs +118 -0
  9. package/scripts/install/opencode.mjs +98 -17
  10. package/scripts/install-mcp.mjs +98 -13
  11. package/scripts/mcp_persistence_server.py +331 -1
  12. package/scripts/secret-scan.mjs +76 -0
  13. package/scripts/validate-routing.mjs +15 -0
  14. package/src/agents/aphrodite.md +7 -0
  15. package/src/agents/demeter.md +7 -0
  16. package/src/agents/hephaestus.md +7 -0
  17. package/src/agents/hermes.md +7 -0
  18. package/src/agents/zeus.md +31 -18
  19. package/src/instructions/agent-return-format.instructions.md +1 -0
  20. package/src/instructions/zeus-anti-stall.instructions.md +43 -5
  21. package/src/instructions/zeus-timeout-retry.instructions.md +47 -10
  22. package/src/mcp/install-mcp.mjs +6 -913
  23. package/src/mcp/mcp_persistence_server.py +332 -0
  24. package/src/mcp/pantheon_vision_server.py +495 -0
  25. package/src/mcp/requirements-vision.txt +3 -0
  26. package/src/pantheon/auto-wake.ts +91 -0
  27. package/src/pantheon/background-job-board.ts +568 -0
  28. package/src/pantheon/file-persistence.ts +95 -0
  29. package/src/pantheon/presets.d.mts +99 -0
  30. package/src/pantheon/presets.mjs +491 -0
  31. package/src/pantheon/vision.ts +1340 -0
  32. package/src/plugin.ts +40 -25
  33. package/src/plugins/tui/dist/tui.d.ts.map +1 -1
  34. package/src/plugins/tui/dist/tui.js +806 -222
  35. package/src/plugins/tui/dist/tui.tsx +866 -121
  36. package/src/plugins/tui/package.json +2 -2
  37. package/src/plugins/tui/src/index.tsx +866 -121
  38. package/src/routing.yml +179 -0
@@ -0,0 +1,1340 @@
1
+ import { createHash, randomUUID } from 'node:crypto'
2
+ import { existsSync } from 'node:fs'
3
+ import { mkdir, readFile, unlink, writeFile } from 'node:fs/promises'
4
+ import { homedir, tmpdir } from 'node:os'
5
+ import { join } from 'node:path'
6
+ import type { Hooks, PluginInput } from '@opencode-ai/plugin'
7
+ import type { FilePart, Part, TextPart, UserMessage } from '@opencode-ai/sdk'
8
+ import type { ResolvedPreset } from './presets.mjs'
9
+ import { hasVision, resolveActivePreset } from './presets.mjs'
10
+
11
+ // ─── Constants ────────────────────────────────────────────────────────────
12
+
13
+ const CONFIG_FILENAME = 'opencode-vision.json'
14
+ const TEMP_DIR_NAME = 'pantheon-vision'
15
+
16
+ /**
17
+ * Temp images younger than this survive a `session.idle` fired between
18
+ * auto-continue turns; abandoned files are swept by the next idle.
19
+ */
20
+ export const TEMP_FILE_GRACE_MS = 120_000
21
+
22
+ /**
23
+ * Global cap on temp images in `/tmp/pantheon-vision`. When the LRU registry
24
+ * (see `enforceTempFileCap`) exceeds this, the least-recently-used files older
25
+ * than TEMP_FILE_GRACE_MS are unlinked — never a young file that an active
26
+ * turn may still reference.
27
+ */
28
+ export const TEMP_MAX_FILES = 200
29
+
30
+ // Tool the model is instructed to call. Configurable via:
31
+ // 1. env PANTHEON_VISION_TOOL
32
+ // 2. config file imageAnalysisTool (project > user)
33
+ // 3. dynamic detection of an available MCP vision tool
34
+ // 4. the canonical Pantheon Vision MCP tool
35
+ //
36
+ // OpenCode 1.18.11 generates MCP tool IDs as sanitize(clientName) + '_' +
37
+ // sanitize(name): the `pantheon-vision` MCP (FastMCP name) yields
38
+ // `pantheon_vision_vision_describe` (underscores) — there is no `mcp__` prefix
39
+ // in that version. The legacy `mcp__pantheon-vision__vision_describe` form is
40
+ // still accepted when explicitly configured or detected.
41
+ const DEFAULT_IMAGE_ANALYSIS_TOOL = 'pantheon_vision_vision_describe'
42
+
43
+ /** True for IDs referencing the Pantheon Vision MCP — hyphen OR underscore server form. */
44
+ const isPantheonVisionServerTool = (id: string): boolean => /[pP]antheon[-_]vision/.test(id)
45
+
46
+ /** True for the describe/ocr/analyze tool suffix (sanitized MCP tool name). */
47
+ const isPantheonVisionActionTool = (id: string): boolean =>
48
+ /vision_(describe|ocr|analyze)/i.test(id)
49
+
50
+ // Enabled for ALL models by default ("paste and ask" universal). The config
51
+ // file can restrict this with wildcard patterns.
52
+ const DEFAULT_MODEL_PATTERNS: readonly string[] = ['*']
53
+
54
+ // Only PNG/JPEG/WebP are intercepted (data: URLs need a known extension).
55
+ const SUPPORTED_MIME_TYPES = new Set([
56
+ 'image/png',
57
+ 'image/jpeg',
58
+ 'image/jpg',
59
+ 'image/webp',
60
+ 'image/gif',
61
+ ])
62
+ const MIME_TO_EXTENSION: Record<string, string> = {
63
+ 'image/png': 'png',
64
+ 'image/jpeg': 'jpg',
65
+ 'image/jpg': 'jpg',
66
+ 'image/webp': 'webp',
67
+ 'image/gif': 'gif',
68
+ }
69
+
70
+ const PROMPT_TEMPLATE_VARIABLES = [
71
+ '{imageList}',
72
+ '{imageCount}',
73
+ '{toolName}',
74
+ '{userText}',
75
+ ] as const
76
+
77
+ // ─── Native vision (provider-direct, OpenAI-compatible) ─────────────────────
78
+ // Preferred path: when an API key is available the plugin calls the multimodal
79
+ // model DIRECTLY via the opencode Zen OpenAI-compatible endpoint, so any user
80
+ // with a provider key gets vision WITHOUT an MCP tool. The MCP tool pattern
81
+ // (imageAnalysisTool) remains the fallback for missing keys / provider errors.
82
+ //
83
+ // MVP scope: only opencode-go / opencode providers (the user's stack).
84
+ // anthropic/openai native vision is phase 2 — those presets fall back to the
85
+ // MCP tool pattern.
86
+
87
+ const DEFAULT_VISION_MODEL = 'opencode-go/mimo-v2.5'
88
+ const ZEN_BASE_URL = 'https://opencode.ai/zen/v1'
89
+ const ZEN_GO_BASE_URL = 'https://opencode.ai/zen/go/v1'
90
+ const NATIVE_PROMPT =
91
+ 'Descreva fielmente esta imagem em detalhes (texto visível, cores, layout, objetos).'
92
+ const NATIVE_TIMEOUT_MS = 20_000
93
+ const NATIVE_MAX_TOKENS = 500
94
+ const MAX_NATIVE_IMAGE_BYTES = 25 * 1024 * 1024
95
+
96
+ /**
97
+ * Descriptions produced by the native path are cached per session, keyed by
98
+ * the user prompt + the content hashes of the attached images (see
99
+ * `buildDescriptionCacheKey`). TTL bounds staleness: an identical re-ask
100
+ * within the window reuses the stored text without another gateway call;
101
+ * after expiry the multimodal model is called again.
102
+ */
103
+ export const DESCRIPTION_CACHE_TTL_MS = 30 * 60 * 1000
104
+
105
+ // ─── Types ─────────────────────────────────────────────────────────────────
106
+
107
+ type VisionConfig = {
108
+ models?: string[]
109
+ imageAnalysisTool?: string
110
+ promptTemplate?: string
111
+ mode?: VisionMode
112
+ visionModel?: string
113
+ }
114
+
115
+ export type ModelInfo = { providerID: string; modelID: string }
116
+
117
+ export type NativeVisionTarget = { modelID: string; baseURL: string; apiKey: string }
118
+
119
+ type NativeVisionEndpoint = { modelID: string; baseURL: string }
120
+
121
+ export type VisionMode = 'native' | 'tool' | 'auto'
122
+
123
+ /** Return whether OpenCode's current model is known to accept image input. */
124
+ export function modelAcceptsImages(model: ModelInfo | undefined): boolean {
125
+ if (!model) return true
126
+ try {
127
+ return hasVision(`${model.providerID}/${model.modelID}`)
128
+ } catch {
129
+ // Unknown providers/models should retain OpenCode's native behavior.
130
+ return true
131
+ }
132
+ }
133
+
134
+ type SavedImage = {
135
+ path: string
136
+ partId: string
137
+ temporary: boolean
138
+ }
139
+
140
+ type NativeImagePayload = { url: string }
141
+
142
+ // ─── Config loading (project > user) ──────────────────────────────────────
143
+
144
+ // ─── OpenCode auth store (fallback credential source) ────────────────────
145
+ // `opencode auth login` stores provider tokens in <data dir>/auth.json.
146
+ // The vision plugin reads that store so connected users don't need to export
147
+ // any env var. Mirrors the usage-bar pattern (src/plugins/tui) but stays
148
+ // local to this module — no TUI imports.
149
+
150
+ function defaultAuthStorePath(): string {
151
+ return join(homedir(), '.local', 'share', 'opencode', 'auth.json')
152
+ }
153
+
154
+ /**
155
+ * Read a provider token from opencode's auth store (`opencode auth login`).
156
+ * `statePath` is the opencode data dir (default ~/.local/share/opencode); the
157
+ * auth.json inside it maps provider IDs to `{ access, expires, ... }`.
158
+ * Returns the first non-expired `access` token across `providerIDs`, or null
159
+ * when the file is missing/corrupted or no entry has a usable token — callers
160
+ * fall back to the MCP tool pattern. The token is never logged.
161
+ */
162
+ export async function readOpencodeAuthToken(
163
+ providerIDs: readonly string[],
164
+ statePath?: string,
165
+ ): Promise<string | null> {
166
+ const authPath = statePath ? join(statePath, 'auth.json') : defaultAuthStorePath()
167
+ let raw: string
168
+ try {
169
+ raw = await readFile(authPath, 'utf8')
170
+ } catch {
171
+ return null // missing/unreadable → no crash
172
+ }
173
+ let auth: Record<string, { access?: unknown; expires?: unknown } | undefined>
174
+ try {
175
+ auth = JSON.parse(raw) as typeof auth
176
+ } catch {
177
+ return null // corrupted → no crash
178
+ }
179
+ for (const id of providerIDs) {
180
+ const entry = auth[id]
181
+ const access = typeof entry?.access === 'string' ? entry.access.trim() : ''
182
+ if (access === '') continue
183
+ const expires = entry?.expires
184
+ if (typeof expires === 'number' && expires > 0 && Date.now() >= expires) continue
185
+ return access
186
+ }
187
+ return null
188
+ }
189
+
190
+ function userConfigPath(): string {
191
+ const xdg = process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config')
192
+ return join(xdg, 'opencode', CONFIG_FILENAME)
193
+ }
194
+
195
+ async function readConfigFile(configPath: string): Promise<VisionConfig | null> {
196
+ if (!existsSync(configPath)) return null
197
+ try {
198
+ const parsed = JSON.parse(await readFile(configPath, 'utf8')) as Record<string, unknown>
199
+ const config: VisionConfig = {}
200
+ if (Array.isArray(parsed.models)) {
201
+ const models = parsed.models.filter((m): m is string => typeof m === 'string')
202
+ if (models.length > 0) config.models = models
203
+ }
204
+ if (typeof parsed.imageAnalysisTool === 'string' && parsed.imageAnalysisTool.trim() !== '') {
205
+ config.imageAnalysisTool = parsed.imageAnalysisTool
206
+ }
207
+ const template = parsed.promptTemplate
208
+ if (
209
+ typeof template === 'string' &&
210
+ PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))
211
+ ) {
212
+ config.promptTemplate = template
213
+ }
214
+ if (parsed.mode === 'native' || parsed.mode === 'tool' || parsed.mode === 'auto') {
215
+ config.mode = parsed.mode
216
+ }
217
+ if (typeof parsed.visionModel === 'string' && parsed.visionModel.trim() !== '') {
218
+ config.visionModel = parsed.visionModel.trim()
219
+ }
220
+ return Object.keys(config).length > 0 ? config : null
221
+ } catch {
222
+ return null
223
+ }
224
+ }
225
+
226
+ export async function loadVisionConfig(directory: string): Promise<VisionConfig> {
227
+ const project = await readConfigFile(join(directory, '.opencode', CONFIG_FILENAME))
228
+ const user = await readConfigFile(userConfigPath())
229
+ const merged: VisionConfig = {}
230
+ if (project?.models) merged.models = project.models
231
+ else if (user?.models) merged.models = user.models
232
+ if (project?.imageAnalysisTool) merged.imageAnalysisTool = project.imageAnalysisTool
233
+ else if (user?.imageAnalysisTool) merged.imageAnalysisTool = user.imageAnalysisTool
234
+ if (project?.promptTemplate) merged.promptTemplate = project.promptTemplate
235
+ else if (user?.promptTemplate) merged.promptTemplate = user.promptTemplate
236
+ if (project?.mode) merged.mode = project.mode
237
+ else if (user?.mode) merged.mode = user.mode
238
+ if (project?.visionModel) merged.visionModel = project.visionModel
239
+ else if (user?.visionModel) merged.visionModel = user.visionModel
240
+ return merged
241
+ }
242
+
243
+ // ─── Active-preset resolution (native vision model) ────────────────────────
244
+ // Same candidate order as the plugin config hook, so the native vision model
245
+ // matches the active preset (resolved.vision?.model) exactly.
246
+
247
+ export function activePresetCandidates(): string[] {
248
+ const home = process.env.HOME ?? ''
249
+ const xdg = process.env.XDG_CONFIG_HOME ?? `${home}/.config`
250
+ return [
251
+ `${process.cwd()}/.pantheon/active-preset.json`,
252
+ `${xdg}/opencode/.pantheon/active-preset.json`,
253
+ `${home}/.opencode/.pantheon/active-preset.json`,
254
+ ]
255
+ }
256
+
257
+ /**
258
+ * Resolve the vision mode: env PANTHEON_VISION_MODE, then the config file's
259
+ * `mode` field, then default `auto`. `auto` = try native vision first (when a
260
+ * key exists), fall back to the MCP tool pattern. `tool` forces the legacy
261
+ * tool pattern. `native` tries native first and still falls back on failure
262
+ * (never break a user turn).
263
+ */
264
+ export function getVisionMode(
265
+ env: Record<string, string | undefined> = process.env,
266
+ configMode?: string,
267
+ ): VisionMode {
268
+ const envMode = env.PANTHEON_VISION_MODE?.trim().toLowerCase()
269
+ if (envMode === 'native' || envMode === 'tool') return envMode
270
+ const cfgMode = configMode?.trim().toLowerCase()
271
+ if (cfgMode === 'native' || cfgMode === 'tool') return cfgMode
272
+ return 'auto'
273
+ }
274
+
275
+ // ─── Wildcard model matching ───────────────────────────────────────────────
276
+ // Patterns: `*`, `prefix*`, `*suffix`, `*contains*`, `provider/*`, `*/model`.
277
+
278
+ export function matchesWildcardPattern(pattern: string, value: string): boolean {
279
+ const p = pattern.toLowerCase()
280
+ const v = value.toLowerCase()
281
+ if (p === '*') return true
282
+ if (p.startsWith('*') && p.endsWith('*') && p.length > 2) return v.includes(p.slice(1, -1))
283
+ if (p.endsWith('*')) return v.startsWith(p.slice(0, -1))
284
+ if (p.startsWith('*')) return v.endsWith(p.slice(1))
285
+ return v === p
286
+ }
287
+
288
+ export function matchesModelPattern(pattern: string, model: ModelInfo): boolean {
289
+ if (pattern === '*') return true
290
+ const slash = pattern.indexOf('/')
291
+ if (slash === -1) {
292
+ return (
293
+ matchesWildcardPattern(pattern, model.providerID) ||
294
+ matchesWildcardPattern(pattern, model.modelID)
295
+ )
296
+ }
297
+ const provider = pattern.slice(0, slash)
298
+ const modelPattern = pattern.slice(slash + 1)
299
+ return (
300
+ matchesWildcardPattern(provider, model.providerID) &&
301
+ matchesWildcardPattern(modelPattern, model.modelID)
302
+ )
303
+ }
304
+
305
+ export function modelMatchesAnyPattern(
306
+ model: ModelInfo | undefined,
307
+ patterns: readonly string[],
308
+ ): boolean {
309
+ if (!model) return false
310
+ return patterns.some((pattern) => matchesModelPattern(pattern, model))
311
+ }
312
+
313
+ // ─── Type guards ───────────────────────────────────────────────────────────
314
+
315
+ export function isImageFilePart(part: unknown): part is FilePart {
316
+ if (!part || typeof part !== 'object') return false
317
+ const value = part as { type?: unknown; mime?: unknown }
318
+ return (
319
+ value.type === 'file' &&
320
+ typeof value.mime === 'string' &&
321
+ value.mime.toLowerCase().startsWith('image/')
322
+ )
323
+ }
324
+
325
+ function isSupportedImageFilePart(part: unknown): part is FilePart {
326
+ if (!isImageFilePart(part)) return false
327
+ return SUPPORTED_MIME_TYPES.has(part.mime.toLowerCase())
328
+ }
329
+
330
+ function isTextPart(part: Part): part is TextPart {
331
+ return part.type === 'text'
332
+ }
333
+
334
+ function syntheticPartId(): string {
335
+ return `prt${randomUUID()}`
336
+ }
337
+
338
+ // ─── Image target resolution ───────────────────────────────────────────────
339
+ // file:// → local path used directly; data: → decoded + saved to temp;
340
+ // http(s):// → URL passed through unchanged.
341
+
342
+ function getExtensionForMime(mime: string): string {
343
+ return MIME_TO_EXTENSION[mime.toLowerCase()] ?? 'png'
344
+ }
345
+
346
+ // Same set as the paste-and-ask interception and the Pantheon Vision MCP.
347
+ const EXTENSION_TO_MIME: Record<string, string> = {
348
+ png: 'image/png',
349
+ jpg: 'image/jpeg',
350
+ jpeg: 'image/jpeg',
351
+ webp: 'image/webp',
352
+ gif: 'image/gif',
353
+ }
354
+
355
+ /**
356
+ * Guess a MIME type from a file extension (.png/.jpg/.jpeg/.webp/.gif).
357
+ * Returns null for unknown extensions — callers surface a friendly error.
358
+ */
359
+ export function getMimeForPath(path: string): string | null {
360
+ const match = /\.([a-z0-9]+)$/i.exec(path)
361
+ if (!match) return null
362
+ const extension = (match[1] ?? '').toLowerCase()
363
+ return EXTENSION_TO_MIME[extension] ?? null
364
+ }
365
+
366
+ // ─── Temp-image LRU (global disk cap) ─────────────────────────────────────
367
+ // Content-hash filenames mean identical re-pasted images share one file; the
368
+ // LRU registry keeps the total on disk bounded. Insertion order = recency:
369
+ // touching a path deletes + re-sets it so the oldest entries sit at the front.
370
+
371
+ /** LRU registry: temp image path → last touch time. Exposed for tests. */
372
+ export const tempFileLRU = new Map<string, number>()
373
+
374
+ /** Mark a temp file as recently used (recency = map insertion order). */
375
+ export function touchTempFile(path: string, now: number = Date.now()): void {
376
+ tempFileLRU.delete(path)
377
+ tempFileLRU.set(path, now)
378
+ }
379
+
380
+ /** Drop a temp file from the registry (after it has been unlinked). */
381
+ export function forgetTempFile(path: string): void {
382
+ tempFileLRU.delete(path)
383
+ }
384
+
385
+ /**
386
+ * Enforce the global temp-file cap: when the registry exceeds `maxFiles`,
387
+ * unlink the least-recently-used files — but NEVER a file younger than
388
+ * `graceMs`, because an active auto-continue turn may still reference it.
389
+ * The temp directory itself is never removed (other sessions/processes may
390
+ * share it). Returns how many files were evicted.
391
+ */
392
+ export async function enforceTempFileCap(
393
+ maxFiles: number = TEMP_MAX_FILES,
394
+ graceMs: number = TEMP_FILE_GRACE_MS,
395
+ now: number = Date.now(),
396
+ ): Promise<number> {
397
+ let evicted = 0
398
+ for (const [path, touchedAt] of tempFileLRU) {
399
+ if (tempFileLRU.size <= maxFiles) break
400
+ if (now - touchedAt <= graceMs) continue
401
+ tempFileLRU.delete(path)
402
+ try {
403
+ await unlink(path)
404
+ } catch {
405
+ // Best effort.
406
+ }
407
+ evicted += 1
408
+ }
409
+ return evicted
410
+ }
411
+
412
+ export async function saveDataUrlImage(dataUrl: string, mime: string): Promise<string | null> {
413
+ const match = /^data:([^;,]+);base64,(.+)$/s.exec(dataUrl)
414
+ if (!match) return null
415
+ const data = Buffer.from(match[2] ?? '', 'base64')
416
+ if (data.length === 0) return null
417
+ const dir = join(tmpdir(), TEMP_DIR_NAME)
418
+ await mkdir(dir, { recursive: true })
419
+ // Content-hash filename: identical re-pasted images reuse the same file
420
+ // (dedup — less disk, and it stabilizes the per-image description cache).
421
+ const filePath = join(dir, `${hashContent(data)}.${getExtensionForMime(mime)}`)
422
+ if (!existsSync(filePath)) await writeFile(filePath, data)
423
+ touchTempFile(filePath)
424
+ await enforceTempFileCap()
425
+ return filePath
426
+ }
427
+
428
+ async function resolveImageTarget(filePart: FilePart): Promise<SavedImage | null> {
429
+ const url = filePart.url
430
+ if (!url) return null
431
+ if (url.startsWith('file://')) {
432
+ return { path: url.slice('file://'.length), partId: filePart.id, temporary: false }
433
+ }
434
+ if (url.startsWith('data:')) {
435
+ const path = await saveDataUrlImage(url, filePart.mime)
436
+ if (!path) return null
437
+ return { path, partId: filePart.id, temporary: true }
438
+ }
439
+ if (url.startsWith('http://') || url.startsWith('https://')) {
440
+ return { path: url, partId: filePart.id, temporary: false }
441
+ }
442
+ return null
443
+ }
444
+
445
+ // ─── Native vision (provider-direct) ───────────────────────────────────────
446
+
447
+ /** First non-empty (trimmed) value across the priority list, else `fallback`. */
448
+ function firstNonEmpty(fallback: string, ...values: Array<string | undefined>): string {
449
+ for (const value of values) {
450
+ const trimmed = value?.trim()
451
+ if (trimmed) return trimmed
452
+ }
453
+ return fallback
454
+ }
455
+
456
+ /**
457
+ * Resolve the native vision endpoint: model ID + Zen base URL (no key I/O).
458
+ *
459
+ * Model precedence: env PANTHEON_VISION_MODEL > config file `visionModel` >
460
+ * active preset `vision.model` > DEFAULT_VISION_MODEL. Returns null when the
461
+ * model's provider is not opencode-go/opencode (anthropic/openai are phase 2).
462
+ */
463
+ function resolveNativeVisionEndpoint(
464
+ preset: Pick<ResolvedPreset, 'vision'> | null | undefined,
465
+ env: Record<string, string | undefined>,
466
+ configVisionModel?: string,
467
+ ): NativeVisionEndpoint | null {
468
+ const resolved = preset ?? resolveActivePreset({ env, candidates: activePresetCandidates() })
469
+ const modelID = firstNonEmpty(
470
+ DEFAULT_VISION_MODEL,
471
+ env.PANTHEON_VISION_MODEL,
472
+ configVisionModel,
473
+ resolved?.vision?.model,
474
+ )
475
+ const providerID = modelID.slice(0, modelID.indexOf('/'))
476
+ let baseURL: string | null = null
477
+ if (providerID.startsWith('opencode-go')) baseURL = ZEN_GO_BASE_URL
478
+ else if (providerID === 'opencode') baseURL = ZEN_BASE_URL
479
+ if (!baseURL) return null
480
+ return { modelID, baseURL }
481
+ }
482
+
483
+ /**
484
+ * Resolve the native vision target: model + Zen endpoint + API key.
485
+ *
486
+ * Key comes from env only (PANTHEON_OPENCODE_API_KEY then OPENCODE_API_KEY).
487
+ * Returns null when the model's provider is not opencode-go/opencode or when
488
+ * no env key is set — callers fall back to the MCP tool. For the auth-store
489
+ * fallback (`opencode auth login`) use `resolveNativeVisionTarget`.
490
+ */
491
+ export function resolveNativeVisionConfig(
492
+ preset: Pick<ResolvedPreset, 'vision'> | null | undefined = undefined,
493
+ env: Record<string, string | undefined> = process.env,
494
+ configVisionModel?: string,
495
+ ): NativeVisionTarget | null {
496
+ const endpoint = resolveNativeVisionEndpoint(preset, env, configVisionModel)
497
+ if (!endpoint) return null
498
+ const apiKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
499
+ if (!apiKey || apiKey.trim() === '') return null
500
+ return { ...endpoint, apiKey }
501
+ }
502
+
503
+ /**
504
+ * Full native-vision target resolution for the chat hook (async): model +
505
+ * endpoint via `resolveNativeVisionConfig`, then the API key from env
506
+ * (PANTHEON_OPENCODE_API_KEY ?? OPENCODE_API_KEY) falling back to the opencode
507
+ * auth store (`opencode auth login`) — 'opencode-go' entry first, then
508
+ * 'opencode' — so connected users don't need to export anything. Returns null
509
+ * when the provider isn't opencode-go/opencode or no usable key exists —
510
+ * callers fall back to the MCP tool.
511
+ */
512
+ export async function resolveNativeVisionTarget(
513
+ input: {
514
+ preset?: Pick<ResolvedPreset, 'vision'> | null
515
+ env?: Record<string, string | undefined>
516
+ config?: VisionConfig | null
517
+ authStatePath?: string
518
+ } = {},
519
+ ): Promise<NativeVisionTarget | null> {
520
+ const env = input.env ?? process.env
521
+ const endpoint = resolveNativeVisionEndpoint(input.preset, env, input.config?.visionModel)
522
+ if (!endpoint) return null
523
+ const envKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
524
+ if (envKey && envKey.trim() !== '') return { ...endpoint, apiKey: envKey }
525
+ const authToken = await readOpencodeAuthToken(['opencode-go', 'opencode'], input.authStatePath)
526
+ if (!authToken) return null
527
+ return { ...endpoint, apiKey: authToken }
528
+ }
529
+
530
+ /**
531
+ * Resolve a single image source into an OpenAI-compatible image_url payload:
532
+ * - http(s) URL → passed through unchanged
533
+ * - data: URI → validated + size-capped, used as-is
534
+ * - local path → bytes read, converted to a data URI with the given MIME
535
+ * Returns null when the source is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
536
+ * callers fall back to the MCP tool pattern (or surface a friendly error).
537
+ */
538
+ export async function resolveImageSourcePayload(
539
+ source: string,
540
+ mime: string,
541
+ ): Promise<NativeImagePayload | null> {
542
+ if (source.startsWith('http://') || source.startsWith('https://')) {
543
+ return { url: source }
544
+ }
545
+ if (source.startsWith('data:')) {
546
+ const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/.exec(source)
547
+ if (!match) return null
548
+ const approxBytes = Math.ceil(((match[2]?.length ?? 0) * 3) / 4)
549
+ if (approxBytes > MAX_NATIVE_IMAGE_BYTES) return null
550
+ return { url: source }
551
+ }
552
+ // Local path (from a file:// FilePart or a standalone tool path).
553
+ try {
554
+ const data = await readFile(source)
555
+ if (data.length === 0 || data.length > MAX_NATIVE_IMAGE_BYTES) return null
556
+ return { url: `data:${mime.toLowerCase()};base64,${data.toString('base64')}` }
557
+ } catch {
558
+ return null
559
+ }
560
+ }
561
+
562
+ /**
563
+ * Build OpenAI-compatible image_url payloads for the native vision call.
564
+ * - data: URI → used as-is (validated + size-capped).
565
+ * - file:// path → read bytes, converted to a data URI.
566
+ * - http(s) URL → passed through unchanged.
567
+ * Returns null when any image is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
568
+ * the caller falls back to the MCP tool pattern.
569
+ */
570
+ async function buildNativeImagePayloads(
571
+ resolved: ReadonlyArray<{ filePart: FilePart; target: SavedImage }>,
572
+ ): Promise<NativeImagePayload[] | null> {
573
+ const payloads: NativeImagePayload[] = []
574
+ for (const { filePart, target } of resolved) {
575
+ const payload = await resolveImageSourcePayload(target.path, filePart.mime)
576
+ if (!payload) return null
577
+ payloads.push(payload)
578
+ }
579
+ return payloads
580
+ }
581
+
582
+ /**
583
+ * Strip the provider prefix from a model ID before sending it to the Zen
584
+ * gateway. The endpoint URL already encodes the provider (see
585
+ * `resolveNativeVisionEndpoint`), and a qualified model ID such as
586
+ * `opencode-go/mimo-v2.5` is rejected by the gateway with a 401
587
+ * ("Model ... is not supported"). Models without a `/` are returned unchanged.
588
+ */
589
+ export function stripProviderPrefix(modelID: string): string {
590
+ const slash = modelID.lastIndexOf('/')
591
+ return slash === -1 ? modelID : modelID.slice(slash + 1)
592
+ }
593
+
594
+ /**
595
+ * Call the multimodal model directly (OpenAI-compatible /chat/completions on
596
+ * the opencode Zen endpoint). Returns the raw description text, or null on any
597
+ * failure (HTTP error, network error, timeout, empty content) — the caller
598
+ * falls back to the MCP tool pattern. The image never reaches the main
599
+ * provider: it is sent only here and replaced by the returned text.
600
+ *
601
+ * `prompt` overrides the default describe prompt (e.g. OCR extraction);
602
+ * `maxTokens` caps the completion (OCR needs more headroom); `signal` links
603
+ * an external abort (e.g. the plugin-tool ToolContext.abort) to the request.
604
+ */
605
+ export async function describeImagesNative(
606
+ payloads: readonly NativeImagePayload[],
607
+ target: NativeVisionTarget,
608
+ prompt: string = NATIVE_PROMPT,
609
+ maxTokens: number = NATIVE_MAX_TOKENS,
610
+ signal?: AbortSignal,
611
+ ): Promise<string | null> {
612
+ const controller = new AbortController()
613
+ const timer = setTimeout(() => controller.abort(), NATIVE_TIMEOUT_MS)
614
+ const onExternalAbort = () => controller.abort()
615
+ if (signal) {
616
+ if (signal.aborted) controller.abort()
617
+ else signal.addEventListener('abort', onExternalAbort, { once: true })
618
+ }
619
+ try {
620
+ const response = await fetch(`${target.baseURL}/chat/completions`, {
621
+ method: 'POST',
622
+ headers: {
623
+ 'Content-Type': 'application/json',
624
+ Authorization: `Bearer ${target.apiKey}`,
625
+ },
626
+ body: JSON.stringify({
627
+ model: stripProviderPrefix(target.modelID),
628
+ messages: [
629
+ {
630
+ role: 'user',
631
+ content: [
632
+ { type: 'text', text: prompt },
633
+ ...payloads.map((payload) => ({
634
+ type: 'image_url',
635
+ image_url: { url: payload.url },
636
+ })),
637
+ ],
638
+ },
639
+ ],
640
+ max_tokens: maxTokens,
641
+ }),
642
+ signal: controller.signal,
643
+ })
644
+ if (!response.ok) return null
645
+ const data = (await response.json()) as {
646
+ choices?: Array<{ message?: { content?: string } }>
647
+ }
648
+ const content = data.choices?.[0]?.message?.content
649
+ if (typeof content !== 'string' || content.trim() === '') return null
650
+ return content.trim()
651
+ } catch {
652
+ return null
653
+ } finally {
654
+ clearTimeout(timer)
655
+ if (signal) signal.removeEventListener('abort', onExternalAbort)
656
+ }
657
+ }
658
+
659
+ // ─── Session description cache ─────────────────────────────────────────────
660
+ // The native path stores the gateway's text answer per session, keyed by the
661
+ // user prompt + the sorted content hashes of every image payload. An identical
662
+ // re-ask within the TTL reuses the stored text and never contacts the gateway.
663
+ // Only textual descriptions are cached — never image bytes — and a cached
664
+ // description is injected as text (it is never re-sent as an image payload).
665
+
666
+ export type DescriptionCacheEntry = { text: string; storedAt: number }
667
+
668
+ /** sha256 hex of arbitrary content (prompt, data-URI, mime+bytes). */
669
+ export function hashContent(content: string | Buffer): string {
670
+ return createHash('sha256').update(content).digest('hex')
671
+ }
672
+
673
+ /**
674
+ * Composite cache key: hash of the user prompt + the sorted content hashes of
675
+ * every image payload. The prompt is part of the key so a NEW question about
676
+ * the same image never reuses a stale description; sorting makes the key
677
+ * order-insensitive (the same image set in any order maps to one key). For
678
+ * data-URI/local payloads the URL encodes both mime and bytes, so hashing it
679
+ * covers "mime+bytes"; remote URLs hash the URL itself.
680
+ */
681
+ export function buildDescriptionCacheKey(
682
+ prompt: string,
683
+ payloads: readonly { url: string }[],
684
+ ): string {
685
+ const promptHash = hashContent(prompt)
686
+ const imageHashes = payloads.map((payload) => hashContent(payload.url)).sort()
687
+ return [promptHash, ...imageHashes].join(':')
688
+ }
689
+
690
+ /**
691
+ * Look up a cached description. Returns null on miss or when the entry is
692
+ * older than `ttlMs` — callers then hit the gateway and store the result.
693
+ */
694
+ export function getCachedDescription(
695
+ cache: ReadonlyMap<string, DescriptionCacheEntry>,
696
+ key: string,
697
+ now: number = Date.now(),
698
+ ttlMs: number = DESCRIPTION_CACHE_TTL_MS,
699
+ ): string | null {
700
+ const entry = cache.get(key)
701
+ if (!entry) return null
702
+ if (now - entry.storedAt >= ttlMs) return null
703
+ return entry.text
704
+ }
705
+
706
+ /** Store a description with the current timestamp (overwrites on re-ask). */
707
+ export function setCachedDescription(
708
+ cache: Map<string, DescriptionCacheEntry>,
709
+ key: string,
710
+ text: string,
711
+ now: number = Date.now(),
712
+ ): void {
713
+ cache.set(key, { text, storedAt: now })
714
+ }
715
+
716
+ // ─── Prompt generation ─────────────────────────────────────────────────────
717
+
718
+ export function generateInjectionPrompt(
719
+ images: Array<{ path: string }>,
720
+ userText: string,
721
+ toolName: string,
722
+ template?: string,
723
+ ): string {
724
+ if (!Array.isArray(images)) {
725
+ throw new TypeError('Vision image resolution must return an array')
726
+ }
727
+ const imageList = images.map((image, index) => `- Image ${index + 1}: ${image.path}`).join('\n')
728
+ if (template && PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))) {
729
+ return template
730
+ .replaceAll('{imageList}', imageList)
731
+ .replaceAll('{imageCount}', String(images.length))
732
+ .replaceAll('{toolName}', toolName)
733
+ .replaceAll('{userText}', userText)
734
+ }
735
+ const plural = images.length > 1
736
+ const heading = plural
737
+ ? `The user has shared ${images.length} images. The images are available at:`
738
+ : 'The user has shared an image. The image is available at:'
739
+ const analyze = plural ? 'each image' : 'this image'
740
+ return [
741
+ heading,
742
+ imageList,
743
+ '',
744
+ `Use the \`${toolName}\` tool to analyze ${analyze}.`,
745
+ '',
746
+ `User's request: ${userText || '(analyze the image)'}`,
747
+ ].join('\n')
748
+ }
749
+
750
+ /**
751
+ * Injection text for the native-vision path: the description returned by the
752
+ * multimodal model, without any tool instruction — the main model already has
753
+ * the visual content in text form.
754
+ */
755
+ export function generateNativeInjection(
756
+ description: string,
757
+ userText: string,
758
+ modelID: string,
759
+ ): string {
760
+ return [
761
+ `The user shared an image. Vision description (native, ${modelID}):`,
762
+ description,
763
+ '',
764
+ `User's request: ${userText || '(analyze the image)'}`,
765
+ ].join('\n')
766
+ }
767
+
768
+ // ─── Intent-calibrated native prompts + structured responses ───────────────
769
+ // The native vision path calibrates its prompt to the user's actual request
770
+ // (compare / ocr / reconstruct / bugs / describe) and asks the multimodal
771
+ // model for a structured, parseable answer: one `<item id="N"><description>…`
772
+ // block per image plus an optional `<context>` block for cross-image analysis.
773
+ // The parser maps each item back to its image in the text injected into the
774
+ // main model; any parse failure falls back to treating the whole gateway
775
+ // answer as a single description — the turn never breaks.
776
+
777
+ export type VisionIntent = 'compare' | 'ocr' | 'reconstruct' | 'bugs' | 'describe'
778
+
779
+ const INTENT_PATTERNS: Record<Exclude<VisionIntent, 'describe'>, RegExp> = {
780
+ compare:
781
+ /\b(compare|comparar|comparação|diferença|diferenças|difference|differences|qual é melhor|qual e melhor|which is better|versus|vs)\b/i,
782
+ ocr: /\b(texto|ler|read|ocr|extrair|extraia|extract|transcrever|transcreva|transcribe)\b/i,
783
+ reconstruct:
784
+ /\b(código|codigo|code|implementar|implemente|reconstruir|reconstrua|html|css|component|ui|tela|screen)\b/i,
785
+ bugs: /\b(bug|erro|problema|issue|debug|não funciona|nao funciona|not working)\b/i,
786
+ }
787
+
788
+ /**
789
+ * Classify the user's request into a vision intent, in priority order:
790
+ * `compare` (requires 2+ images) → `ocr` → `reconstruct` → `bugs` → `describe`.
791
+ * Term matching is case-insensitive, word-boundary based, PT + EN.
792
+ */
793
+ export function detectVisionIntent(userText: string, imageCount: number): VisionIntent {
794
+ if (imageCount >= 2 && INTENT_PATTERNS.compare.test(userText)) return 'compare'
795
+ if (INTENT_PATTERNS.ocr.test(userText)) return 'ocr'
796
+ if (INTENT_PATTERNS.reconstruct.test(userText)) return 'reconstruct'
797
+ if (INTENT_PATTERNS.bugs.test(userText)) return 'bugs'
798
+ return 'describe'
799
+ }
800
+
801
+ /** Escape `&`, `<`, `>` as XML entities (& first) — for template interpolation. */
802
+ export function escapeXml(text: string): string {
803
+ return text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')
804
+ }
805
+
806
+ /** Reverse of `escapeXml` — applied to parsed `<description>`/`<context>` bodies. */
807
+ function unescapeXml(text: string): string {
808
+ return text.replaceAll('&lt;', '<').replaceAll('&gt;', '>').replaceAll('&amp;', '&')
809
+ }
810
+
811
+ const INTENT_TASK_INSTRUCTIONS: Record<VisionIntent, string> = {
812
+ describe:
813
+ 'Descreva cada imagem fielmente e em detalhes: texto visível verbatim, cores, layout, objetos.',
814
+ compare:
815
+ 'Descreva cada imagem no seu próprio bloco <item>; depois coloque uma comparação direta e exaustiva no <context> (diferenças por região/elemento), terminando com um veredito claro.',
816
+ ocr: 'Transcreva literalmente todo o texto visível em cada imagem, preservando ordem e estrutura. A transcrição vai dentro do <description> da imagem correspondente.',
817
+ reconstruct:
818
+ 'Descreva cada imagem com detalhe de reconstrução para reprodução em código: layout completo (regiões, alinhamento, espaçamento), todo texto visível verbatim, paleta de cores exata em hex, fontes/tamanhos/pesos por elemento, estrutura de componentes e estados.',
819
+ bugs: 'Descreva o sintoma visual observado em cada imagem: o que está errado, onde, esperado vs real, com referências de região precisas e contexto suficiente para localizar e reproduzir.',
820
+ }
821
+
822
+ const INTENT_CONTEXT_INSTRUCTIONS: Record<VisionIntent, string> = {
823
+ compare:
824
+ 'Depois dos itens, adicione UM bloco <context> com a análise cruzada das imagens (comparação, veredito).',
825
+ describe: '',
826
+ ocr: '',
827
+ reconstruct: '',
828
+ bugs: '',
829
+ }
830
+
831
+ /**
832
+ * Build the calibrated prompt for the native vision gateway call. The model is
833
+ * instructed to reply with N `<item id="N"><description>…</description></item>`
834
+ * blocks (N = imageCount) and, for `compare`, one extra `<context>` block.
835
+ * The user's text is XML-escaped when interpolated so it cannot break the
836
+ * format instructions. `intent` is auto-detected from `userText` +
837
+ * `imageCount` when omitted.
838
+ */
839
+ export function buildNativeVisionPrompt(
840
+ userText: string,
841
+ imageCount: number,
842
+ intent: VisionIntent = detectVisionIntent(userText, imageCount),
843
+ ): string {
844
+ const count = Math.max(1, imageCount)
845
+ const request =
846
+ escapeXml(userText.trim()) || '(sem pedido explícito — descreva as imagens com precisão)'
847
+ const contextInstruction = INTENT_CONTEXT_INSTRUCTIONS[intent]
848
+ const contextLine = contextInstruction ? ` ${contextInstruction}` : ''
849
+ return [
850
+ 'You are a multimodal analysis specialist embedded inside a coding assistant. The coding',
851
+ 'assistant CANNOT perceive the images in this turn — your output is its ONLY view of them.',
852
+ 'It will act on exactly what you write.',
853
+ '',
854
+ "READ THE USER REQUEST FIRST: infer the user's goal and calibrate the depth of each",
855
+ 'description to it.',
856
+ '',
857
+ INTENT_TASK_INSTRUCTIONS[intent],
858
+ '',
859
+ 'OUTPUT FORMAT — binding, no exceptions:',
860
+ `Responda com UM bloco <item id="N"> para cada uma das ${count} imagens anexadas`,
861
+ `(N = 1..${count}), em ordem. Cada bloco contém um <description>.${contextLine}`,
862
+ '',
863
+ '<item id="1">',
864
+ '<description>...apenas fatos sobre a imagem 1...</description>',
865
+ '</item>',
866
+ '<item id="2">',
867
+ '<description>...apenas fatos sobre a imagem 2...</description>',
868
+ '</item>',
869
+ '',
870
+ 'RULES',
871
+ '- O primeiro caractere da sua resposta deve ser "<". Sem preâmbulo, sem "Aqui está",',
872
+ ' sem observações finais, sem wrap em blocos de código.',
873
+ '- ESCAPING — obrigatório: dentro dos corpos de <description> e <context>, escape TODO',
874
+ ' "&", "<" e ">" literais como "&amp;", "&lt;", "&gt;". Isso vale para qualquer texto',
875
+ ' tag-like reproduzido verbatim (HTML, XML, erros com colchetes angulares). NÃO escape',
876
+ ' as tags estruturais <item>, <description>, <context>.',
877
+ '- Nunca invente detalhes que não consegue perceber. Se um texto estiver ilegível,',
878
+ ' escreva "ilegível". Coloque análises cruzadas (comparação, veredito) no <context>,',
879
+ ' nunca dentro de um <description>.',
880
+ '',
881
+ `User's request: ${request}`,
882
+ ].join('\n')
883
+ }
884
+
885
+ export type StructuredVisionItem = { id: number; description: string }
886
+ export type StructuredVisionResponse = { items: StructuredVisionItem[]; context: string | null }
887
+
888
+ const ITEM_BLOCK_RE = /<item\b[^>]*>([\s\S]*?)<\/item>/gi
889
+ const OPEN_TAG_RE = /<item\b[^>]*>/i
890
+ const DESCRIPTION_BODY_RE = /<description>([\s\S]*?)<\/description>/i
891
+ const CONTEXT_BODY_RE = /<context>([\s\S]*?)<\/context>/i
892
+ const ITEM_ID_ATTR_RE = /id\s*=\s*["']?(\d+)["']?/i
893
+ const BARE_DESCRIPTION_RE = /<description>([\s\S]*?)<\/description>/gi
894
+
895
+ /**
896
+ * Parse the gateway's structured answer into per-image `<item>` descriptions
897
+ * plus an optional `<context>` block. Resistant to formatting variations
898
+ * (whitespace, quotes, tag case, missing id → sequential ids, markdown fences
899
+ * around the XML). XML entities in the bodies are unescaped for the main
900
+ * model. Returns null when no `<item>`/`<description>` block is found —
901
+ * callers then treat the whole answer as a single description (the
902
+ * pre-structured behavior, no extra gateway call).
903
+ */
904
+ export function parseStructuredVisionResponse(text: string): StructuredVisionResponse | null {
905
+ const items: StructuredVisionItem[] = []
906
+ let match = ITEM_BLOCK_RE.exec(text)
907
+ while (match !== null) {
908
+ const desc = DESCRIPTION_BODY_RE.exec(match[1] ?? '')
909
+ if (desc) {
910
+ const openTag = OPEN_TAG_RE.exec(match[0] ?? '')?.[0] ?? ''
911
+ const rawId = ITEM_ID_ATTR_RE.exec(openTag)?.[1]
912
+ const description = unescapeXml((desc[1] ?? '').trim())
913
+ items.push({
914
+ id: rawId !== undefined ? Number(rawId) : items.length + 1,
915
+ description,
916
+ })
917
+ }
918
+ match = ITEM_BLOCK_RE.exec(text)
919
+ }
920
+ if (items.length === 0) {
921
+ let bareMatch = BARE_DESCRIPTION_RE.exec(text)
922
+ while (bareMatch !== null) {
923
+ items.push({
924
+ id: items.length + 1,
925
+ description: unescapeXml((bareMatch[1] ?? '').trim()),
926
+ })
927
+ bareMatch = BARE_DESCRIPTION_RE.exec(text)
928
+ }
929
+ }
930
+ const contextMatch = CONTEXT_BODY_RE.exec(text)
931
+ const context = contextMatch ? unescapeXml((contextMatch[1] ?? '').trim()) : null
932
+ return items.length > 0 ? { items, context } : null
933
+ }
934
+
935
+ /**
936
+ * Build the text injected into the main model from a parsed structured
937
+ * response: each `<item id>` is mapped to its image (by id, falling back to
938
+ * position when an id is missing), followed by the optional cross-image
939
+ * `<context>` block. The fallback single-description path stays in
940
+ * `generateNativeInjection`.
941
+ */
942
+ export function buildStructuredInjection(
943
+ items: readonly StructuredVisionItem[],
944
+ context: string | null,
945
+ userText: string,
946
+ modelID: string,
947
+ imageCount: number,
948
+ ): string {
949
+ const count = Math.max(1, imageCount)
950
+ const byId = new Map(items.map((item) => [item.id, item.description]))
951
+ const header =
952
+ count > 1
953
+ ? `The user shared ${count} images. Vision descriptions (native, ${modelID}):`
954
+ : `The user shared an image. Vision description (native, ${modelID}):`
955
+ const lines = [header]
956
+ for (let i = 1; i <= count; i += 1) {
957
+ const description = byId.get(i) ?? items[i - 1]?.description
958
+ lines.push(`- Image ${i}: ${description ?? '(no description provided)'}`)
959
+ }
960
+ if (context) {
961
+ lines.push('', 'Cross-image context:', context)
962
+ }
963
+ lines.push('', `User's request: ${userText || '(analyze the image)'}`)
964
+ return lines.join('\n')
965
+ }
966
+
967
+ // ─── Tool name resolution ──────────────────────────────────────────────────
968
+ // env PANTHEON_VISION_TOOL > config imageAnalysisTool > dynamic detection
969
+ // (prefer the canonical Pantheon MCP) > canonical Pantheon MCP default.
970
+
971
+ async function resolveImageAnalysisTool(
972
+ client: PluginInput['client'],
973
+ directory: string,
974
+ configPromise: Promise<VisionConfig>,
975
+ ): Promise<string | null> {
976
+ const envTool = process.env.PANTHEON_VISION_TOOL?.trim()
977
+ if (envTool) return envTool
978
+ const config = await configPromise
979
+ if (config.imageAnalysisTool) return config.imageAnalysisTool
980
+ try {
981
+ const response = await client.tool?.ids?.({ query: { directory } })
982
+ const ids = response?.data
983
+ if (Array.isArray(ids)) {
984
+ // Exact canonical ID first (fast path), then the underscore form and the
985
+ // legacy mcp__ hyphen form (both match the pantheon[-_]vision + action
986
+ // regexes), then the loose describe fallback for older ID shapes.
987
+ const match =
988
+ ids.find((id) => id === DEFAULT_IMAGE_ANALYSIS_TOOL) ??
989
+ ids.find((id) => isPantheonVisionServerTool(id) && isPantheonVisionActionTool(id)) ??
990
+ ids.find((id) => isPantheonVisionServerTool(id) && /describe/i.test(id))
991
+ // OpenCode 1.18.11 keeps MCP tools in the MCP service, NOT in
992
+ // client.tool.ids() — an empty/absent list is the NORMAL runtime state,
993
+ // so it must resolve to the canonical default tool instead of the
994
+ // 'Vision fallback unavailable' message. Only a thrown exception (the
995
+ // detection genuinely failed) returns null below.
996
+ return match ?? DEFAULT_IMAGE_ANALYSIS_TOOL
997
+ }
998
+ } catch {
999
+ // Detection failed (network/MCP error). Returning null keeps the explicit
1000
+ // 'Vision fallback unavailable' safe-failure message in the turn instead
1001
+ // of guessing at a tool we could not verify — the turn never breaks
1002
+ // (interception is best-effort). Returning DEFAULT here too would hide
1003
+ // real detection failures behind a tool name the model might not have.
1004
+ return null
1005
+ }
1006
+ return DEFAULT_IMAGE_ANALYSIS_TOOL
1007
+ }
1008
+
1009
+ // ─── Temp image lifecycle ──────────────────────────────────────────────────
1010
+
1011
+ /**
1012
+ * Age-guarded cleanup of a session's temp images.
1013
+ *
1014
+ * A `session.idle` can fire between auto-continue turns — not just at real
1015
+ * session end — so a temp file referenced by the next turn (seconds old) must
1016
+ * survive. Only paths older than `graceMs` are unlinked; younger paths stay
1017
+ * registered and are swept by the next idle. `server.instance.disposed` and
1018
+ * `session.deleted` pass `graceMs = 0` to unlink everything while the session
1019
+ * is gone / process is dying. The temp directory itself is never removed: it
1020
+ * may hold files from other sessions or processes. The session's description
1021
+ * cache is always dropped (cache is per-session state).
1022
+ */
1023
+ export async function cleanupSessionTempImages(
1024
+ sessionTempFiles: Map<string, Map<string, number>>,
1025
+ sessionVisionText: Map<string, Map<string, string>>,
1026
+ nativeVisionSessions: Set<string>,
1027
+ sessionID: string,
1028
+ graceMs: number = TEMP_FILE_GRACE_MS,
1029
+ sessionDescriptionCache?: Map<string, Map<string, DescriptionCacheEntry>>,
1030
+ ): Promise<void> {
1031
+ const files = sessionTempFiles.get(sessionID)
1032
+ if (files) {
1033
+ const now = Date.now()
1034
+ for (const [path, createdAt] of files) {
1035
+ // graceMs === 0 means "unlink everything now" (session gone / process
1036
+ // dying); a positive grace skips files younger than the threshold so the
1037
+ // next auto-continue turn can still read them.
1038
+ if (graceMs > 0 && now - createdAt <= graceMs) continue
1039
+ try {
1040
+ await unlink(path)
1041
+ } catch {
1042
+ // Best effort.
1043
+ }
1044
+ forgetTempFile(path)
1045
+ files.delete(path)
1046
+ }
1047
+ if (files.size === 0) sessionTempFiles.delete(sessionID)
1048
+ }
1049
+ sessionVisionText.delete(sessionID)
1050
+ nativeVisionSessions.delete(sessionID)
1051
+ sessionDescriptionCache?.delete(sessionID)
1052
+ }
1053
+
1054
+ type VisionHistoryMessage = {
1055
+ info?: { sessionID?: string; id?: string }
1056
+ sessionID?: string
1057
+ id?: string
1058
+ parts?: unknown
1059
+ content?: unknown
1060
+ [key: string]: unknown
1061
+ }
1062
+
1063
+ function syntheticHistoryPart(message: VisionHistoryMessage, text: string): TextPart {
1064
+ return {
1065
+ id: syntheticPartId(),
1066
+ sessionID: message.info?.sessionID ?? '',
1067
+ messageID: message.info?.id ?? '',
1068
+ type: 'text',
1069
+ text,
1070
+ synthetic: true,
1071
+ }
1072
+ }
1073
+
1074
+ /**
1075
+ * Remove image-bearing parts before a provider serializer can turn them into
1076
+ * image_url content. This hook is deliberately synchronous in spirit: it only
1077
+ * reuses text cached by chat.message and never calls a vision tool/model.
1078
+ */
1079
+ export function sanitizeVisionHistory(
1080
+ output: { messages?: unknown },
1081
+ sessionVisionText: ReadonlyMap<string, ReadonlyMap<string, string>>,
1082
+ nativeVisionSessions: ReadonlySet<string> = new Set(),
1083
+ ): void {
1084
+ if (!Array.isArray(output.messages)) return
1085
+
1086
+ for (const message of output.messages) {
1087
+ if (!message || typeof message !== 'object') continue
1088
+ const historyMessage = message as VisionHistoryMessage
1089
+ const sessionID = historyMessage.info?.sessionID ?? historyMessage.sessionID ?? ''
1090
+ if (nativeVisionSessions.has(sessionID)) continue
1091
+ const descriptions = sessionVisionText.get(sessionID)
1092
+ const sanitize = (value: unknown, inParts: boolean): unknown => {
1093
+ if (Array.isArray(value)) return value.map((item) => sanitize(item, inParts))
1094
+ if (!value || typeof value !== 'object') return value
1095
+
1096
+ const raw = value as { type?: unknown; id?: unknown; mime?: unknown }
1097
+ const isImageUrl = raw.type === 'image_url'
1098
+ const isImageFile =
1099
+ raw.type === 'file' &&
1100
+ typeof raw.mime === 'string' &&
1101
+ raw.mime.toLowerCase().startsWith('image/')
1102
+ if (isImageUrl || isImageFile) {
1103
+ const partID = typeof raw.id === 'string' ? raw.id : ''
1104
+ const text =
1105
+ descriptions?.get(partID) ??
1106
+ 'Imagem removida do histórico antes do envio ao provedor textual.'
1107
+ return inParts ? syntheticHistoryPart(historyMessage, text) : { type: 'text', text }
1108
+ }
1109
+
1110
+ const copy: Record<string, unknown> = {}
1111
+ for (const [key, child] of Object.entries(value)) {
1112
+ copy[key] = sanitize(child, key === 'parts')
1113
+ }
1114
+ return copy
1115
+ }
1116
+
1117
+ const sanitized = sanitize(historyMessage, false) as VisionHistoryMessage
1118
+ for (const [key, value] of Object.entries(sanitized)) {
1119
+ historyMessage[key] = value
1120
+ }
1121
+ }
1122
+ }
1123
+
1124
+ // ─── Handler factory ───────────────────────────────────────────────────────
1125
+
1126
+ export function createVisionHandler(input: PluginInput) {
1127
+ const { client, directory } = input
1128
+ const sessionTempFiles = new Map<string, Map<string, number>>()
1129
+ const sessionVisionText = new Map<string, Map<string, string>>()
1130
+ const nativeVisionSessions = new Set<string>()
1131
+ const sessionDescriptionCache = new Map<string, Map<string, DescriptionCacheEntry>>()
1132
+ let configPromise: Promise<VisionConfig> | null = null
1133
+ let toolNamePromise: Promise<string | null> | null = null
1134
+
1135
+ const getConfig = (): Promise<VisionConfig> => {
1136
+ configPromise ??= loadVisionConfig(directory)
1137
+ return configPromise
1138
+ }
1139
+
1140
+ const getToolName = (): Promise<string | null> => {
1141
+ toolNamePromise ??= resolveImageAnalysisTool(client, directory, getConfig())
1142
+ return toolNamePromise
1143
+ }
1144
+
1145
+ const chatMessage: NonNullable<Hooks['chat.message']> = async (hookInput, output) => {
1146
+ try {
1147
+ const message: UserMessage | undefined = output?.message
1148
+ if (!message || message.role !== 'user') return
1149
+ const model = hookInput?.model ?? message.model
1150
+ if (modelAcceptsImages(model)) {
1151
+ nativeVisionSessions.add(hookInput.sessionID)
1152
+ return
1153
+ }
1154
+ nativeVisionSessions.delete(hookInput.sessionID)
1155
+ const config = await getConfig()
1156
+ const patterns =
1157
+ config.models && config.models.length > 0 ? config.models : DEFAULT_MODEL_PATTERNS
1158
+ if (!modelMatchesAnyPattern(model, patterns)) {
1159
+ // Pattern configuration may disable enrichment, but it must never
1160
+ // disable the text-only provider safety boundary.
1161
+ const currentTurn = { messages: [{ ...message, parts: output.parts }] }
1162
+ sanitizeVisionHistory(currentTurn, sessionVisionText)
1163
+ const sanitized = currentTurn.messages as VisionHistoryMessage[]
1164
+ output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
1165
+ return
1166
+ }
1167
+
1168
+ const parts: Part[] = Array.isArray(output.parts) ? output.parts : []
1169
+ const images = parts.filter(isImageFilePart)
1170
+ if (images.length === 0) {
1171
+ // Some runtimes hand the current turn to plugins already serialized as
1172
+ // provider content. Do not let that shape bypass the final sanitizer.
1173
+ const currentTurn = { messages: [{ ...message, parts: output.parts }] }
1174
+ sanitizeVisionHistory(currentTurn, sessionVisionText)
1175
+ const sanitized = currentTurn.messages as VisionHistoryMessage[]
1176
+ output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
1177
+ return
1178
+ }
1179
+
1180
+ const userText = parts
1181
+ .filter(isTextPart)
1182
+ .map((part) => part.text)
1183
+ .join('\n')
1184
+ .trim()
1185
+
1186
+ const resolved: Array<{ filePart: FilePart; target: SavedImage }> = []
1187
+ for (const image of images.filter(isSupportedImageFilePart)) {
1188
+ const target = await resolveImageTarget(image)
1189
+ if (target) resolved.push({ filePart: image, target })
1190
+ }
1191
+ const saved = resolved.map((item) => item.target)
1192
+
1193
+ const tempPaths = saved.filter((item) => item.temporary).map((item) => item.path)
1194
+ if (tempPaths.length > 0) {
1195
+ const createdAt = Date.now()
1196
+ const existing = sessionTempFiles.get(hookInput.sessionID) ?? new Map<string, number>()
1197
+ for (const path of tempPaths) existing.set(path, createdAt)
1198
+ sessionTempFiles.set(hookInput.sessionID, existing)
1199
+ }
1200
+
1201
+ // Native-first: call the multimodal model directly when a key exists
1202
+ // (mode auto/native). Any failure — no key, unsupported provider,
1203
+ // oversized/invalid image, HTTP/network error — falls back to the tool.
1204
+ // The prompt is calibrated to the user's intent (compare/ocr/
1205
+ // reconstruct/bugs/describe) and asks for a structured answer
1206
+ // (<item>/<context>), which is parsed and mapped back to each image.
1207
+ // Descriptions are cached per session: the key is the calibrated prompt
1208
+ // + the content hashes of the images, so an identical re-ask within the
1209
+ // TTL reuses the stored (possibly structured) text without another
1210
+ // gateway call, while a new question about the same image misses and is
1211
+ // answered freshly.
1212
+ let injection: string | null = null
1213
+ if (getVisionMode(process.env, config.mode) !== 'tool') {
1214
+ try {
1215
+ const target = await resolveNativeVisionTarget({ config })
1216
+ if (target) {
1217
+ const payloads = await buildNativeImagePayloads(resolved)
1218
+ if (payloads) {
1219
+ let sessionCache = sessionDescriptionCache.get(hookInput.sessionID)
1220
+ if (!sessionCache) {
1221
+ sessionCache = new Map()
1222
+ sessionDescriptionCache.set(hookInput.sessionID, sessionCache)
1223
+ }
1224
+ const intent = detectVisionIntent(userText, payloads.length)
1225
+ const prompt = buildNativeVisionPrompt(userText, payloads.length, intent)
1226
+ const cacheKey = buildDescriptionCacheKey(prompt, payloads)
1227
+ let rawText = getCachedDescription(sessionCache, cacheKey)
1228
+ if (rawText === null) {
1229
+ rawText = await describeImagesNative(payloads, target, prompt)
1230
+ if (rawText !== null) setCachedDescription(sessionCache, cacheKey, rawText)
1231
+ }
1232
+ if (rawText !== null) {
1233
+ const structured = parseStructuredVisionResponse(rawText)
1234
+ injection = structured
1235
+ ? buildStructuredInjection(
1236
+ structured.items,
1237
+ structured.context,
1238
+ userText,
1239
+ target.modelID,
1240
+ payloads.length,
1241
+ )
1242
+ : generateNativeInjection(rawText, userText, target.modelID)
1243
+ }
1244
+ }
1245
+ }
1246
+ } catch {
1247
+ // Native vision is best-effort; fall back to the tool pattern.
1248
+ }
1249
+ }
1250
+
1251
+ if (injection === null && saved.length > 0) {
1252
+ const toolName = await getToolName()
1253
+ injection = toolName
1254
+ ? generateInjectionPrompt(saved, userText, toolName, config.promptTemplate)
1255
+ : 'Vision fallback unavailable: pantheon-vision MCP is unavailable. The image was removed safely; please configure the MCP server and retry.'
1256
+ }
1257
+ if (injection === null) {
1258
+ injection = 'A imagem compartilhada foi removida antes do envio ao provedor textual.'
1259
+ }
1260
+
1261
+ const descriptions = sessionVisionText.get(hookInput.sessionID) ?? new Map<string, string>()
1262
+ for (const image of images) descriptions.set(image.id, injection)
1263
+ sessionVisionText.set(hookInput.sessionID, descriptions)
1264
+
1265
+ const injectionPart: TextPart = {
1266
+ id: syntheticPartId(),
1267
+ sessionID: message.sessionID,
1268
+ messageID: message.id,
1269
+ type: 'text',
1270
+ text: injection,
1271
+ synthetic: true,
1272
+ }
1273
+ const kept = parts.filter((part) => !isImageFilePart(part))
1274
+ // Mutate in place: drop image parts, lead with the vision text
1275
+ // (native description or tool instruction).
1276
+ output.parts.splice(0, output.parts.length, injectionPart, ...kept)
1277
+ sanitizeVisionHistory({ messages: [{ ...message, parts: output.parts }] }, sessionVisionText)
1278
+ } catch {
1279
+ // Vision interception is best-effort; never fail a user turn.
1280
+ }
1281
+ }
1282
+
1283
+ const event: NonNullable<Hooks['event']> = async ({ event: ev }) => {
1284
+ if (ev.type === 'session.idle') {
1285
+ // An idle can fire between auto-continue turns: temp files younger than
1286
+ // the grace period survive, but the per-session description cache is
1287
+ // dropped (it is pure optimization, safe to reset on every idle).
1288
+ const sessionID = ev.properties?.sessionID
1289
+ if (sessionID)
1290
+ await cleanupSessionTempImages(
1291
+ sessionTempFiles,
1292
+ sessionVisionText,
1293
+ nativeVisionSessions,
1294
+ sessionID,
1295
+ TEMP_FILE_GRACE_MS,
1296
+ sessionDescriptionCache,
1297
+ )
1298
+ } else if (ev.type === 'session.deleted') {
1299
+ // The session is gone for good: no active turn can reference its temp
1300
+ // images, so unlink them all immediately and drop its cached state.
1301
+ const sessionID = ev.properties.info.id
1302
+ await cleanupSessionTempImages(
1303
+ sessionTempFiles,
1304
+ sessionVisionText,
1305
+ nativeVisionSessions,
1306
+ sessionID,
1307
+ 0,
1308
+ sessionDescriptionCache,
1309
+ )
1310
+ } else if (ev.type === 'server.instance.disposed') {
1311
+ const sessions = [...sessionTempFiles.keys()]
1312
+ for (const sessionID of sessions) {
1313
+ await cleanupSessionTempImages(
1314
+ sessionTempFiles,
1315
+ sessionVisionText,
1316
+ nativeVisionSessions,
1317
+ sessionID,
1318
+ 0, // The process is dying: unlink everything immediately.
1319
+ sessionDescriptionCache,
1320
+ )
1321
+ }
1322
+ sessionVisionText.clear()
1323
+ sessionDescriptionCache.clear()
1324
+ }
1325
+ }
1326
+
1327
+ const messagesTransform: NonNullable<Hooks['experimental.chat.messages.transform']> = async (
1328
+ _hookInput,
1329
+ output,
1330
+ ) => {
1331
+ try {
1332
+ sanitizeVisionHistory(output, sessionVisionText, nativeVisionSessions)
1333
+ } catch {
1334
+ // History sanitization is best effort; never break runtimes that expose
1335
+ // the experimental hook with a partial output shape.
1336
+ }
1337
+ }
1338
+
1339
+ return { chatMessage, messagesTransform, event }
1340
+ }