pantheon-opencode 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +101 -3
  2. package/bin/pantheon-init.mjs +79 -0
  3. package/commands/pantheon-vision.md +67 -0
  4. package/opencode.json +13 -9
  5. package/package.json +12 -5
  6. package/scripts/_pantheon_paths.py +0 -2
  7. package/scripts/doctor.mjs +4 -2
  8. package/scripts/install/health-check.mjs +101 -1
  9. package/scripts/install/model-picker.mjs +118 -0
  10. package/scripts/install/opencode.mjs +102 -19
  11. package/scripts/install/venv.mjs +25 -2
  12. package/scripts/install-mcp.mjs +98 -13
  13. package/scripts/mcp_persistence_server.py +1 -1
  14. package/scripts/secret-scan.mjs +88 -0
  15. package/scripts/uninstall.mjs +3 -3
  16. package/scripts/validate-routing.mjs +15 -0
  17. package/src/agents/hermes.md +2 -2
  18. package/src/agents/mnemosyne.md +1 -1
  19. package/src/agents/zeus.md +1 -1
  20. package/src/mcp/_pantheon_paths.py +0 -2
  21. package/src/mcp/install-mcp.mjs +6 -913
  22. package/src/mcp/mcp_persistence_server.py +2 -1
  23. package/src/mcp/pantheon_vision_server.py +495 -0
  24. package/src/mcp/requirements-vision.txt +3 -0
  25. package/src/pantheon/auto-wake.ts +91 -0
  26. package/src/pantheon/background-job-board.ts +568 -0
  27. package/src/pantheon/file-persistence.ts +95 -0
  28. package/src/pantheon/presets.d.mts +108 -0
  29. package/src/pantheon/presets.mjs +554 -0
  30. package/src/pantheon/vision.ts +1437 -0
  31. package/src/plugin.ts +69 -25
  32. package/src/plugins/tui/package.json +3 -3
  33. package/src/plugins/tui/src/index.tsx +866 -121
  34. package/src/routing.yml +148 -0
  35. package/src/skills/context-compression/SKILL.md +1 -1
  36. package/src/skills/loop-engineering/SKILL.md +1 -1
  37. package/src/skills/orchestration-workflow/SKILL.md +11 -11
  38. package/scripts/scrub_secrets.py +0 -281
  39. package/src/plugins/tui/dist/tui.d.ts +0 -8
  40. package/src/plugins/tui/dist/tui.d.ts.map +0 -1
  41. package/src/plugins/tui/dist/tui.js +0 -609
  42. package/src/plugins/tui/dist/tui.js.map +0 -1
  43. package/src/plugins/tui/dist/tui.tsx +0 -444
@@ -0,0 +1,1437 @@
1
+ import { createHash, randomUUID } from 'node:crypto'
2
+ import { existsSync } from 'node:fs'
3
+ import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'
4
+ import { homedir, tmpdir } from 'node:os'
5
+ import { join } from 'node:path'
6
+ import { fileURLToPath } from 'node:url'
7
+ import type { Hooks, PluginInput } from '@opencode-ai/plugin'
8
+ import type { FilePart, Part, TextPart, UserMessage } from '@opencode-ai/sdk'
9
+ import type { ResolvedPreset } from './presets.mjs'
10
+ import { hasVision, resolveActivePreset, visionBrokenOnGateway } from './presets.mjs'
11
+
12
+ // ─── Constants ────────────────────────────────────────────────────────────
13
+
14
+ const CONFIG_FILENAME = 'opencode-vision.json'
15
+ const TEMP_DIR_NAME = 'pantheon-vision'
16
+
17
+ /**
18
+ * Temp images younger than this survive a `session.idle` fired between
19
+ * auto-continue turns; abandoned files are swept by the next idle.
20
+ */
21
+ export const TEMP_FILE_GRACE_MS = 120_000
22
+
23
+ /**
24
+ * Global cap on temp images in `/tmp/pantheon-vision`. When the LRU registry
25
+ * (see `enforceTempFileCap`) exceeds this, the least-recently-used files older
26
+ * than TEMP_FILE_GRACE_MS are unlinked — never a young file that an active
27
+ * turn may still reference.
28
+ */
29
+ export const TEMP_MAX_FILES = 200
30
+
31
+ // Tool the model is instructed to call. Configurable via:
32
+ // 1. env PANTHEON_VISION_TOOL
33
+ // 2. config file imageAnalysisTool (project > user)
34
+ // 3. dynamic detection of an available MCP vision tool
35
+ // 4. the canonical Pantheon Vision MCP tool
36
+ //
37
+ // OpenCode 1.18.11 generates MCP tool IDs as sanitize(clientName) + '_' +
38
+ // sanitize(name): the `pantheon-vision` MCP (FastMCP name) yields
39
+ // `pantheon_vision_vision_describe` (underscores) — there is no `mcp__` prefix
40
+ // in that version. The legacy `mcp__pantheon-vision__vision_describe` form is
41
+ // still accepted when explicitly configured or detected.
42
+ const DEFAULT_IMAGE_ANALYSIS_TOOL = 'pantheon_vision_vision_describe'
43
+
44
+ /** True for IDs referencing the Pantheon Vision MCP — hyphen OR underscore server form. */
45
+ const isPantheonVisionServerTool = (id: string): boolean => /[pP]antheon[-_]vision/.test(id)
46
+
47
+ /** True for the describe/ocr/analyze tool suffix (sanitized MCP tool name). */
48
+ const isPantheonVisionActionTool = (id: string): boolean =>
49
+ /vision_(describe|ocr|analyze)/i.test(id)
50
+
51
+ // Enabled for ALL models by default ("paste and ask" universal). The config
52
+ // file can restrict this with wildcard patterns.
53
+ const DEFAULT_MODEL_PATTERNS: readonly string[] = ['*']
54
+
55
+ // Only PNG/JPEG/WebP are intercepted (data: URLs need a known extension).
56
+ const SUPPORTED_MIME_TYPES = new Set([
57
+ 'image/png',
58
+ 'image/jpeg',
59
+ 'image/jpg',
60
+ 'image/webp',
61
+ 'image/gif',
62
+ ])
63
+ const MIME_TO_EXTENSION: Record<string, string> = {
64
+ 'image/png': 'png',
65
+ 'image/jpeg': 'jpg',
66
+ 'image/jpg': 'jpg',
67
+ 'image/webp': 'webp',
68
+ 'image/gif': 'gif',
69
+ }
70
+
71
+ const PROMPT_TEMPLATE_VARIABLES = [
72
+ '{imageList}',
73
+ '{imageCount}',
74
+ '{toolName}',
75
+ '{userText}',
76
+ ] as const
77
+
78
+ // ─── Native vision (provider-direct, OpenAI-compatible) ─────────────────────
79
+ // Preferred path: when an API key is available the plugin calls the multimodal
80
+ // model DIRECTLY via the opencode Zen OpenAI-compatible endpoint, so any user
81
+ // with a provider key gets vision WITHOUT an MCP tool. The MCP tool pattern
82
+ // (imageAnalysisTool) remains the fallback for missing keys / provider errors.
83
+ //
84
+ // MVP scope: only opencode-go / opencode providers (the user's stack).
85
+ // anthropic/openai native vision is phase 2 — those presets fall back to the
86
+ // MCP tool pattern.
87
+
88
+ const DEFAULT_VISION_MODEL = 'opencode-go/mimo-v2.5'
89
+ const ZEN_BASE_URL = 'https://opencode.ai/zen/v1'
90
+ const ZEN_GO_BASE_URL = 'https://opencode.ai/zen/go/v1'
91
+ const NATIVE_PROMPT =
92
+ 'Descreva fielmente esta imagem em detalhes (texto visível, cores, layout, objetos).'
93
+ const NATIVE_TIMEOUT_MS = 20_000
94
+ const NATIVE_MAX_TOKENS = 500
95
+ const MAX_NATIVE_IMAGE_BYTES = 25 * 1024 * 1024
96
+
97
+ /**
98
+ * Descriptions produced by the native path are cached per session, keyed by
99
+ * the user prompt + the content hashes of the attached images (see
100
+ * `buildDescriptionCacheKey`). TTL bounds staleness: an identical re-ask
101
+ * within the window reuses the stored text without another gateway call;
102
+ * after expiry the multimodal model is called again.
103
+ */
104
+ export const DESCRIPTION_CACHE_TTL_MS = 30 * 60 * 1000
105
+
106
+ // ─── Types ─────────────────────────────────────────────────────────────────
107
+
108
+ type VisionConfig = {
109
+ models?: string[]
110
+ imageAnalysisTool?: string
111
+ promptTemplate?: string
112
+ mode?: VisionMode
113
+ visionModel?: string
114
+ }
115
+
116
+ export type ModelInfo = { providerID: string; modelID: string }
117
+
118
+ export type NativeVisionTarget = { modelID: string; baseURL: string; apiKey: string }
119
+
120
+ type NativeVisionEndpoint = { modelID: string; baseURL: string }
121
+
122
+ export type VisionMode = 'native' | 'tool' | 'auto'
123
+
124
+ /** Return whether OpenCode's current model is known to accept image input. */
125
+ export function modelAcceptsImages(model: ModelInfo | undefined): boolean {
126
+ if (!model) return true
127
+ try {
128
+ const id = `${model.providerID}/${model.modelID}`
129
+ // Provider/gateway-aware: a model can be vision-capable per models.dev
130
+ // yet BROKEN on its runtime gateway (qwen3.7-plus on opencode-go returns
131
+ // HTTP 500 on image turns). Such models are treated as text-only so the
132
+ // image is intercepted and routed to the multimodal fallback instead of
133
+ // being sent directly to the gateway.
134
+ return hasVision(id) && !visionBrokenOnGateway(id, model.providerID)
135
+ } catch {
136
+ // Unknown providers/models should retain OpenCode's native behavior.
137
+ return true
138
+ }
139
+ }
140
+
141
+ type SavedImage = {
142
+ path: string
143
+ partId: string
144
+ temporary: boolean
145
+ }
146
+
147
+ type NativeImagePayload = { url: string }
148
+
149
+ // ─── Config loading (project > user) ──────────────────────────────────────
150
+
151
+ // ─── OpenCode auth store (fallback credential source) ────────────────────
152
+ // `opencode auth login` stores provider tokens in <data dir>/auth.json.
153
+ // The vision plugin reads that store so connected users don't need to export
154
+ // any env var. Mirrors the usage-bar pattern (src/plugins/tui) but stays
155
+ // local to this module — no TUI imports.
156
+
157
+ function defaultAuthStorePath(): string {
158
+ return join(homedir(), '.local', 'share', 'opencode', 'auth.json')
159
+ }
160
+
161
+ /**
162
+ * Read a provider token from opencode's auth store (`opencode auth login`).
163
+ * `statePath` is the opencode data dir (default ~/.local/share/opencode); the
164
+ * auth.json inside it maps provider IDs to `{ access, expires, ... }`.
165
+ * Returns the first non-expired `access` token across `providerIDs`, or null
166
+ * when the file is missing/corrupted or no entry has a usable token — callers
167
+ * fall back to the MCP tool pattern. The token is never logged.
168
+ */
169
+ export async function readOpencodeAuthToken(
170
+ providerIDs: readonly string[],
171
+ statePath?: string,
172
+ ): Promise<string | null> {
173
+ const authPath = statePath ? join(statePath, 'auth.json') : defaultAuthStorePath()
174
+ let raw: string
175
+ try {
176
+ raw = await readFile(authPath, 'utf8')
177
+ } catch {
178
+ return null // missing/unreadable → no crash
179
+ }
180
+ let auth: Record<string, { access?: unknown; expires?: unknown } | undefined>
181
+ try {
182
+ auth = JSON.parse(raw) as typeof auth
183
+ } catch {
184
+ return null // corrupted → no crash
185
+ }
186
+ for (const id of providerIDs) {
187
+ const entry = auth[id]
188
+ const access = typeof entry?.access === 'string' ? entry.access.trim() : ''
189
+ if (access === '') continue
190
+ const expires = entry?.expires
191
+ if (typeof expires === 'number' && expires > 0 && Date.now() >= expires) continue
192
+ return access
193
+ }
194
+ return null
195
+ }
196
+
197
+ function userConfigPath(): string {
198
+ const xdg = process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config')
199
+ return join(xdg, 'opencode', CONFIG_FILENAME)
200
+ }
201
+
202
+ async function readConfigFile(configPath: string): Promise<VisionConfig | null> {
203
+ if (!existsSync(configPath)) return null
204
+ try {
205
+ const parsed = JSON.parse(await readFile(configPath, 'utf8')) as Record<string, unknown>
206
+ const config: VisionConfig = {}
207
+ if (Array.isArray(parsed.models)) {
208
+ const models = parsed.models.filter((m): m is string => typeof m === 'string')
209
+ if (models.length > 0) config.models = models
210
+ }
211
+ if (typeof parsed.imageAnalysisTool === 'string' && parsed.imageAnalysisTool.trim() !== '') {
212
+ config.imageAnalysisTool = parsed.imageAnalysisTool
213
+ }
214
+ const template = parsed.promptTemplate
215
+ if (
216
+ typeof template === 'string' &&
217
+ PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))
218
+ ) {
219
+ config.promptTemplate = template
220
+ }
221
+ if (parsed.mode === 'native' || parsed.mode === 'tool' || parsed.mode === 'auto') {
222
+ config.mode = parsed.mode
223
+ }
224
+ if (typeof parsed.visionModel === 'string' && parsed.visionModel.trim() !== '') {
225
+ config.visionModel = parsed.visionModel.trim()
226
+ }
227
+ return Object.keys(config).length > 0 ? config : null
228
+ } catch {
229
+ return null
230
+ }
231
+ }
232
+
233
+ export async function loadVisionConfig(directory: string): Promise<VisionConfig> {
234
+ const project = await readConfigFile(join(directory, '.opencode', CONFIG_FILENAME))
235
+ const user = await readConfigFile(userConfigPath())
236
+ const merged: VisionConfig = {}
237
+ if (project?.models) merged.models = project.models
238
+ else if (user?.models) merged.models = user.models
239
+ if (project?.imageAnalysisTool) merged.imageAnalysisTool = project.imageAnalysisTool
240
+ else if (user?.imageAnalysisTool) merged.imageAnalysisTool = user.imageAnalysisTool
241
+ if (project?.promptTemplate) merged.promptTemplate = project.promptTemplate
242
+ else if (user?.promptTemplate) merged.promptTemplate = user.promptTemplate
243
+ if (project?.mode) merged.mode = project.mode
244
+ else if (user?.mode) merged.mode = user.mode
245
+ if (project?.visionModel) merged.visionModel = project.visionModel
246
+ else if (user?.visionModel) merged.visionModel = user.visionModel
247
+ return merged
248
+ }
249
+
250
+ // ─── Active-preset resolution (native vision model) ────────────────────────
251
+ // Same candidate order as the plugin config hook, so the native vision model
252
+ // matches the active preset (resolved.vision?.model) exactly.
253
+
254
+ export function activePresetCandidates(): string[] {
255
+ const home = process.env.HOME ?? ''
256
+ const xdg = process.env.XDG_CONFIG_HOME ?? `${home}/.config`
257
+ return [
258
+ `${process.cwd()}/.pantheon/active-preset.json`,
259
+ `${xdg}/opencode/.pantheon/active-preset.json`,
260
+ `${home}/.opencode/.pantheon/active-preset.json`,
261
+ ]
262
+ }
263
+
264
+ /**
265
+ * Resolve the vision mode: env PANTHEON_VISION_MODE, then the config file's
266
+ * `mode` field, then default `auto`. `auto` = try native vision first (when a
267
+ * key exists), fall back to the MCP tool pattern. `tool` forces the legacy
268
+ * tool pattern. `native` tries native first and still falls back on failure
269
+ * (never break a user turn).
270
+ */
271
+ export function getVisionMode(
272
+ env: Record<string, string | undefined> = process.env,
273
+ configMode?: string,
274
+ ): VisionMode {
275
+ const envMode = env.PANTHEON_VISION_MODE?.trim().toLowerCase()
276
+ if (envMode === 'native' || envMode === 'tool') return envMode
277
+ const cfgMode = configMode?.trim().toLowerCase()
278
+ if (cfgMode === 'native' || cfgMode === 'tool') return cfgMode
279
+ return 'auto'
280
+ }
281
+
282
+ // ─── Wildcard model matching ───────────────────────────────────────────────
283
+ // Patterns: `*`, `prefix*`, `*suffix`, `*contains*`, `provider/*`, `*/model`.
284
+
285
+ export function matchesWildcardPattern(pattern: string, value: string): boolean {
286
+ const p = pattern.toLowerCase()
287
+ const v = value.toLowerCase()
288
+ if (p === '*') return true
289
+ if (p.startsWith('*') && p.endsWith('*') && p.length > 2) return v.includes(p.slice(1, -1))
290
+ if (p.endsWith('*')) return v.startsWith(p.slice(0, -1))
291
+ if (p.startsWith('*')) return v.endsWith(p.slice(1))
292
+ return v === p
293
+ }
294
+
295
+ export function matchesModelPattern(pattern: string, model: ModelInfo): boolean {
296
+ if (pattern === '*') return true
297
+ const slash = pattern.indexOf('/')
298
+ if (slash === -1) {
299
+ return (
300
+ matchesWildcardPattern(pattern, model.providerID) ||
301
+ matchesWildcardPattern(pattern, model.modelID)
302
+ )
303
+ }
304
+ const provider = pattern.slice(0, slash)
305
+ const modelPattern = pattern.slice(slash + 1)
306
+ return (
307
+ matchesWildcardPattern(provider, model.providerID) &&
308
+ matchesWildcardPattern(modelPattern, model.modelID)
309
+ )
310
+ }
311
+
312
+ export function modelMatchesAnyPattern(
313
+ model: ModelInfo | undefined,
314
+ patterns: readonly string[],
315
+ ): boolean {
316
+ if (!model) return false
317
+ return patterns.some((pattern) => matchesModelPattern(pattern, model))
318
+ }
319
+
320
+ // ─── Type guards ───────────────────────────────────────────────────────────
321
+
322
+ export function isImageFilePart(part: unknown): part is FilePart {
323
+ if (!part || typeof part !== 'object') return false
324
+ const value = part as { type?: unknown; mime?: unknown }
325
+ return (
326
+ value.type === 'file' &&
327
+ typeof value.mime === 'string' &&
328
+ value.mime.toLowerCase().startsWith('image/')
329
+ )
330
+ }
331
+
332
+ function isSupportedImageFilePart(part: unknown): part is FilePart {
333
+ if (!isImageFilePart(part)) return false
334
+ return SUPPORTED_MIME_TYPES.has(part.mime.toLowerCase())
335
+ }
336
+
337
+ function isTextPart(part: Part): part is TextPart {
338
+ return part.type === 'text'
339
+ }
340
+
341
+ function syntheticPartId(): string {
342
+ return `prt${randomUUID()}`
343
+ }
344
+
345
+ // ─── Image target resolution ───────────────────────────────────────────────
346
+ // file:// → local path used directly; data: → decoded + saved to temp;
347
+ // http(s):// → URL passed through unchanged.
348
+
349
+ function getExtensionForMime(mime: string): string {
350
+ return MIME_TO_EXTENSION[mime.toLowerCase()] ?? 'png'
351
+ }
352
+
353
+ // Same set as the paste-and-ask interception and the Pantheon Vision MCP.
354
+ const EXTENSION_TO_MIME: Record<string, string> = {
355
+ png: 'image/png',
356
+ jpg: 'image/jpeg',
357
+ jpeg: 'image/jpeg',
358
+ webp: 'image/webp',
359
+ gif: 'image/gif',
360
+ }
361
+
362
+ /**
363
+ * Guess a MIME type from a file extension (.png/.jpg/.jpeg/.webp/.gif).
364
+ * Returns null for unknown extensions — callers surface a friendly error.
365
+ */
366
+ export function getMimeForPath(path: string): string | null {
367
+ const match = /\.([a-z0-9]+)$/i.exec(path)
368
+ if (!match) return null
369
+ const extension = (match[1] ?? '').toLowerCase()
370
+ return EXTENSION_TO_MIME[extension] ?? null
371
+ }
372
+
373
+ // ─── Temp-image LRU (global disk cap) ─────────────────────────────────────
374
+ // Content-hash filenames mean identical re-pasted images share one file; the
375
+ // LRU registry keeps the total on disk bounded. Insertion order = recency:
376
+ // touching a path deletes + re-sets it so the oldest entries sit at the front.
377
+
378
+ /** LRU registry: temp image path → last touch time. Exposed for tests. */
379
+ export const tempFileLRU = new Map<string, number>()
380
+
381
+ /** Mark a temp file as recently used (recency = map insertion order). */
382
+ export function touchTempFile(path: string, now: number = Date.now()): void {
383
+ tempFileLRU.delete(path)
384
+ tempFileLRU.set(path, now)
385
+ }
386
+
387
+ /** Drop a temp file from the registry (after it has been unlinked). */
388
+ export function forgetTempFile(path: string): void {
389
+ tempFileLRU.delete(path)
390
+ }
391
+
392
+ /**
393
+ * Enforce the global temp-file cap: when the registry exceeds `maxFiles`,
394
+ * unlink the least-recently-used files — but NEVER a file younger than
395
+ * `graceMs`, because an active auto-continue turn may still reference it.
396
+ * The temp directory itself is never removed (other sessions/processes may
397
+ * share it). Returns how many files were evicted.
398
+ */
399
+ export async function enforceTempFileCap(
400
+ maxFiles: number = TEMP_MAX_FILES,
401
+ graceMs: number = TEMP_FILE_GRACE_MS,
402
+ now: number = Date.now(),
403
+ ): Promise<number> {
404
+ let evicted = 0
405
+ for (const [path, touchedAt] of tempFileLRU) {
406
+ if (tempFileLRU.size <= maxFiles) break
407
+ if (now - touchedAt <= graceMs) continue
408
+ tempFileLRU.delete(path)
409
+ try {
410
+ await unlink(path)
411
+ } catch {
412
+ // Best effort.
413
+ }
414
+ evicted += 1
415
+ }
416
+ return evicted
417
+ }
418
+
419
+ export async function saveDataUrlImage(dataUrl: string, mime: string): Promise<string | null> {
420
+ const match = /^data:([^;,]+);base64,(.+)$/s.exec(dataUrl)
421
+ if (!match) return null
422
+ const data = Buffer.from(match[2] ?? '', 'base64')
423
+ if (data.length === 0) return null
424
+ const dir = join(tmpdir(), TEMP_DIR_NAME)
425
+ // Screenshots may hold sensitive content: force private modes even under a
426
+ // permissive umask. mkdir/writeFile honor the umask, so chmod is applied
427
+ // AFTER creation to guarantee 0700/0600 regardless of the environment.
428
+ await mkdir(dir, { recursive: true, mode: 0o700 })
429
+ await chmod(dir, 0o700)
430
+ // Content-hash filename: identical re-pasted images reuse the same file
431
+ // (dedup — less disk, and it stabilizes the per-image description cache).
432
+ const filePath = join(dir, `${hashContent(data)}.${getExtensionForMime(mime)}`)
433
+ if (!existsSync(filePath)) await writeFile(filePath, data, { mode: 0o600 })
434
+ await chmod(filePath, 0o600)
435
+ touchTempFile(filePath)
436
+ await enforceTempFileCap()
437
+ return filePath
438
+ }
439
+
440
+ async function resolveImageTarget(filePart: FilePart): Promise<SavedImage | null> {
441
+ const url = filePart.url
442
+ if (!url) return null
443
+ if (url.startsWith('file://')) {
444
+ // fileURLToPath decodes percent-encoding (file:///tmp/my%20screen.png →
445
+ // /tmp/my screen.png) and normalizes Windows drive-letter URLs
446
+ // (file:///C:/... → C:\... on Windows). A raw slice('file://') would
447
+ // keep the literal %20 in the path, so reading the file would fail.
448
+ try {
449
+ return { path: fileURLToPath(url), partId: filePart.id, temporary: false }
450
+ } catch {
451
+ return null // malformed file URL → unresolvable
452
+ }
453
+ }
454
+ if (url.startsWith('data:')) {
455
+ const path = await saveDataUrlImage(url, filePart.mime)
456
+ if (!path) return null
457
+ return { path, partId: filePart.id, temporary: true }
458
+ }
459
+ if (url.startsWith('http://') || url.startsWith('https://')) {
460
+ return { path: url, partId: filePart.id, temporary: false }
461
+ }
462
+ return null
463
+ }
464
+
465
+ // ─── Native vision (provider-direct) ───────────────────────────────────────
466
+
467
+ /** First non-empty (trimmed) value across the priority list, else `fallback`. */
468
+ function firstNonEmpty(fallback: string, ...values: Array<string | undefined>): string {
469
+ for (const value of values) {
470
+ const trimmed = value?.trim()
471
+ if (trimmed) return trimmed
472
+ }
473
+ return fallback
474
+ }
475
+
476
+ /**
477
+ * Resolve the native vision endpoint: model ID + Zen base URL (no key I/O).
478
+ *
479
+ * Model precedence: env PANTHEON_VISION_MODEL > config file `visionModel` >
480
+ * active preset `vision.model` > DEFAULT_VISION_MODEL. Returns null when the
481
+ * model's provider is not opencode-go/opencode (anthropic/openai are phase 2).
482
+ */
483
+ function resolveNativeVisionEndpoint(
484
+ preset: Pick<ResolvedPreset, 'vision'> | null | undefined,
485
+ env: Record<string, string | undefined>,
486
+ configVisionModel?: string,
487
+ ): NativeVisionEndpoint | null {
488
+ const resolved = preset ?? resolveActivePreset({ env, candidates: activePresetCandidates() })
489
+ const modelID = firstNonEmpty(
490
+ DEFAULT_VISION_MODEL,
491
+ env.PANTHEON_VISION_MODEL,
492
+ configVisionModel,
493
+ resolved?.vision?.model,
494
+ )
495
+ const providerID = modelID.slice(0, modelID.indexOf('/'))
496
+ let baseURL: string | null = null
497
+ if (providerID.startsWith('opencode-go')) baseURL = ZEN_GO_BASE_URL
498
+ else if (providerID === 'opencode') baseURL = ZEN_BASE_URL
499
+ if (!baseURL) return null
500
+ return { modelID, baseURL }
501
+ }
502
+
503
+ /**
504
+ * Resolve the native vision target: model + Zen endpoint + API key.
505
+ *
506
+ * Key comes from env only (PANTHEON_OPENCODE_API_KEY then OPENCODE_API_KEY).
507
+ * Returns null when the model's provider is not opencode-go/opencode or when
508
+ * no env key is set — callers fall back to the MCP tool. For the auth-store
509
+ * fallback (`opencode auth login`) use `resolveNativeVisionTarget`.
510
+ */
511
+ export function resolveNativeVisionConfig(
512
+ preset: Pick<ResolvedPreset, 'vision'> | null | undefined = undefined,
513
+ env: Record<string, string | undefined> = process.env,
514
+ configVisionModel?: string,
515
+ ): NativeVisionTarget | null {
516
+ const endpoint = resolveNativeVisionEndpoint(preset, env, configVisionModel)
517
+ if (!endpoint) return null
518
+ const apiKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
519
+ if (!apiKey || apiKey.trim() === '') return null
520
+ return { ...endpoint, apiKey }
521
+ }
522
+
523
+ /**
524
+ * Full native-vision target resolution for the chat hook (async): model +
525
+ * endpoint via `resolveNativeVisionConfig`, then the API key from env
526
+ * (PANTHEON_OPENCODE_API_KEY ?? OPENCODE_API_KEY) falling back to the opencode
527
+ * auth store (`opencode auth login`) — 'opencode-go' entry first, then
528
+ * 'opencode' — so connected users don't need to export anything. Returns null
529
+ * when the provider isn't opencode-go/opencode or no usable key exists —
530
+ * callers fall back to the MCP tool.
531
+ */
532
+ export async function resolveNativeVisionTarget(
533
+ input: {
534
+ preset?: Pick<ResolvedPreset, 'vision'> | null
535
+ env?: Record<string, string | undefined>
536
+ config?: VisionConfig | null
537
+ authStatePath?: string
538
+ } = {},
539
+ ): Promise<NativeVisionTarget | null> {
540
+ const env = input.env ?? process.env
541
+ const endpoint = resolveNativeVisionEndpoint(input.preset, env, input.config?.visionModel)
542
+ if (!endpoint) return null
543
+ const envKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
544
+ if (envKey && envKey.trim() !== '') return { ...endpoint, apiKey: envKey }
545
+ const authToken = await readOpencodeAuthToken(['opencode-go', 'opencode'], input.authStatePath)
546
+ if (!authToken) return null
547
+ return { ...endpoint, apiKey: authToken }
548
+ }
549
+
550
+ /**
551
+ * Resolve a single image source into an OpenAI-compatible image_url payload:
552
+ * - http(s) URL → passed through unchanged
553
+ * - data: URI → validated + size-capped, used as-is
554
+ * - local path → bytes read, converted to a data URI with the given MIME
555
+ * Returns null when the source is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
556
+ * callers fall back to the MCP tool pattern (or surface a friendly error).
557
+ */
558
+ export async function resolveImageSourcePayload(
559
+ source: string,
560
+ mime: string,
561
+ ): Promise<NativeImagePayload | null> {
562
+ if (source.startsWith('http://') || source.startsWith('https://')) {
563
+ return { url: source }
564
+ }
565
+ if (source.startsWith('data:')) {
566
+ const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/.exec(source)
567
+ if (!match) return null
568
+ const approxBytes = Math.ceil(((match[2]?.length ?? 0) * 3) / 4)
569
+ if (approxBytes > MAX_NATIVE_IMAGE_BYTES) return null
570
+ return { url: source }
571
+ }
572
+ // Local path (from a file:// FilePart or a standalone tool path).
573
+ try {
574
+ const data = await readFile(source)
575
+ if (data.length === 0 || data.length > MAX_NATIVE_IMAGE_BYTES) return null
576
+ return { url: `data:${mime.toLowerCase()};base64,${data.toString('base64')}` }
577
+ } catch {
578
+ return null
579
+ }
580
+ }
581
+
582
+ /**
583
+ * Build OpenAI-compatible image_url payloads for the native vision call.
584
+ * - data: URI → used as-is (validated + size-capped).
585
+ * - file:// path → read bytes, converted to a data URI.
586
+ * - http(s) URL → passed through unchanged.
587
+ * Returns null when any image is invalid or exceeds MAX_NATIVE_IMAGE_BYTES —
588
+ * the caller falls back to the MCP tool pattern.
589
+ */
590
+ async function buildNativeImagePayloads(
591
+ resolved: ReadonlyArray<{ filePart: FilePart; target: SavedImage }>,
592
+ ): Promise<NativeImagePayload[] | null> {
593
+ const payloads: NativeImagePayload[] = []
594
+ for (const { filePart, target } of resolved) {
595
+ const payload = await resolveImageSourcePayload(target.path, filePart.mime)
596
+ if (!payload) return null
597
+ payloads.push(payload)
598
+ }
599
+ return payloads
600
+ }
601
+
602
+ /**
603
+ * Strip the provider prefix from a model ID before sending it to the Zen
604
+ * gateway. The endpoint URL already encodes the provider (see
605
+ * `resolveNativeVisionEndpoint`), and a qualified model ID such as
606
+ * `opencode-go/mimo-v2.5` is rejected by the gateway with a 401
607
+ * ("Model ... is not supported"). Models without a `/` are returned unchanged.
608
+ */
609
+ export function stripProviderPrefix(modelID: string): string {
610
+ const slash = modelID.lastIndexOf('/')
611
+ return slash === -1 ? modelID : modelID.slice(slash + 1)
612
+ }
613
+
614
+ /**
615
+ * Call the multimodal model directly (OpenAI-compatible /chat/completions on
616
+ * the opencode Zen endpoint). Returns the raw description text, or null on any
617
+ * failure (HTTP error, network error, timeout, empty content) — the caller
618
+ * falls back to the MCP tool pattern. The image never reaches the main
619
+ * provider: it is sent only here and replaced by the returned text.
620
+ *
621
+ * `prompt` overrides the default describe prompt (e.g. OCR extraction);
622
+ * `maxTokens` caps the completion (OCR needs more headroom); `signal` links
623
+ * an external abort (e.g. the plugin-tool ToolContext.abort) to the request.
624
+ */
625
+ export async function describeImagesNative(
626
+ payloads: readonly NativeImagePayload[],
627
+ target: NativeVisionTarget,
628
+ prompt: string = NATIVE_PROMPT,
629
+ maxTokens: number = NATIVE_MAX_TOKENS,
630
+ signal?: AbortSignal,
631
+ ): Promise<string | null> {
632
+ const controller = new AbortController()
633
+ const timer = setTimeout(() => controller.abort(), NATIVE_TIMEOUT_MS)
634
+ const onExternalAbort = () => controller.abort()
635
+ if (signal) {
636
+ if (signal.aborted) controller.abort()
637
+ else signal.addEventListener('abort', onExternalAbort, { once: true })
638
+ }
639
+ try {
640
+ const response = await fetch(`${target.baseURL}/chat/completions`, {
641
+ method: 'POST',
642
+ headers: {
643
+ 'Content-Type': 'application/json',
644
+ Authorization: `Bearer ${target.apiKey}`,
645
+ },
646
+ body: JSON.stringify({
647
+ model: stripProviderPrefix(target.modelID),
648
+ messages: [
649
+ {
650
+ role: 'user',
651
+ content: [
652
+ { type: 'text', text: prompt },
653
+ ...payloads.map((payload) => ({
654
+ type: 'image_url',
655
+ image_url: { url: payload.url },
656
+ })),
657
+ ],
658
+ },
659
+ ],
660
+ max_tokens: maxTokens,
661
+ }),
662
+ signal: controller.signal,
663
+ })
664
+ if (!response.ok) return null
665
+ const data = (await response.json()) as {
666
+ choices?: Array<{ message?: { content?: string } }>
667
+ }
668
+ const content = data.choices?.[0]?.message?.content
669
+ if (typeof content !== 'string' || content.trim() === '') return null
670
+ return content.trim()
671
+ } catch {
672
+ return null
673
+ } finally {
674
+ clearTimeout(timer)
675
+ if (signal) signal.removeEventListener('abort', onExternalAbort)
676
+ }
677
+ }
678
+
679
+ // ─── Session description cache ─────────────────────────────────────────────
680
+ // The native path stores the gateway's text answer per session, keyed by the
681
+ // user prompt + the sorted content hashes of every image payload. An identical
682
+ // re-ask within the TTL reuses the stored text and never contacts the gateway.
683
+ // Only textual descriptions are cached — never image bytes — and a cached
684
+ // description is injected as text (it is never re-sent as an image payload).
685
+
686
+ export type DescriptionCacheEntry = { text: string; storedAt: number }
687
+
688
+ /** sha256 hex of arbitrary content (prompt, data-URI, mime+bytes). */
689
+ export function hashContent(content: string | Buffer): string {
690
+ return createHash('sha256').update(content).digest('hex')
691
+ }
692
+
693
+ /**
694
+ * Composite cache key: hash of the user prompt + the sorted content hashes of
695
+ * every image payload. The prompt is part of the key so a NEW question about
696
+ * the same image never reuses a stale description; sorting makes the key
697
+ * order-insensitive (the same image set in any order maps to one key). For
698
+ * data-URI/local payloads the URL encodes both mime and bytes, so hashing it
699
+ * covers "mime+bytes"; remote URLs hash the URL itself.
700
+ */
701
+ export function buildDescriptionCacheKey(
702
+ prompt: string,
703
+ payloads: readonly { url: string }[],
704
+ ): string {
705
+ const promptHash = hashContent(prompt)
706
+ const imageHashes = payloads.map((payload) => hashContent(payload.url)).sort()
707
+ return [promptHash, ...imageHashes].join(':')
708
+ }
709
+
710
+ /**
711
+ * Look up a cached description. Returns null on miss or when the entry is
712
+ * older than `ttlMs` — callers then hit the gateway and store the result.
713
+ */
714
+ export function getCachedDescription(
715
+ cache: ReadonlyMap<string, DescriptionCacheEntry>,
716
+ key: string,
717
+ now: number = Date.now(),
718
+ ttlMs: number = DESCRIPTION_CACHE_TTL_MS,
719
+ ): string | null {
720
+ const entry = cache.get(key)
721
+ if (!entry) return null
722
+ if (now - entry.storedAt >= ttlMs) return null
723
+ return entry.text
724
+ }
725
+
726
+ /** Store a description with the current timestamp (overwrites on re-ask). */
727
+ export function setCachedDescription(
728
+ cache: Map<string, DescriptionCacheEntry>,
729
+ key: string,
730
+ text: string,
731
+ now: number = Date.now(),
732
+ ): void {
733
+ cache.set(key, { text, storedAt: now })
734
+ }
735
+
736
+ // ─── Prompt generation ─────────────────────────────────────────────────────
737
+
738
+ export function generateInjectionPrompt(
739
+ images: Array<{ path: string }>,
740
+ userText: string,
741
+ toolName: string,
742
+ template?: string,
743
+ ): string {
744
+ if (!Array.isArray(images)) {
745
+ throw new TypeError('Vision image resolution must return an array')
746
+ }
747
+ const imageList = images.map((image, index) => `- Image ${index + 1}: ${image.path}`).join('\n')
748
+ if (template && PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))) {
749
+ return template
750
+ .replaceAll('{imageList}', imageList)
751
+ .replaceAll('{imageCount}', String(images.length))
752
+ .replaceAll('{toolName}', toolName)
753
+ .replaceAll('{userText}', userText)
754
+ }
755
+ const plural = images.length > 1
756
+ const heading = plural
757
+ ? `The user has shared ${images.length} images. The images are available at:`
758
+ : 'The user has shared an image. The image is available at:'
759
+ const analyze = plural ? 'each image' : 'this image'
760
+ return [
761
+ heading,
762
+ imageList,
763
+ '',
764
+ `Use the \`${toolName}\` tool to analyze ${analyze}.`,
765
+ '',
766
+ `User's request: ${userText || '(analyze the image)'}`,
767
+ ].join('\n')
768
+ }
769
+
770
+ /**
771
+ * Injection text for the native-vision path: the description returned by the
772
+ * multimodal model, without any tool instruction — the main model already has
773
+ * the visual content in text form.
774
+ */
775
+ export function generateNativeInjection(
776
+ description: string,
777
+ userText: string,
778
+ modelID: string,
779
+ ): string {
780
+ return [
781
+ `The user shared an image. Vision description (native, ${modelID}):`,
782
+ description,
783
+ '',
784
+ `User's request: ${userText || '(analyze the image)'}`,
785
+ ].join('\n')
786
+ }
787
+
788
+ // ─── Intent-calibrated native prompts + structured responses ───────────────
789
+ // The native vision path calibrates its prompt to the user's actual request
790
+ // (compare / ocr / reconstruct / bugs / describe) and asks the multimodal
791
+ // model for a structured, parseable answer: one `<item id="N"><description>…`
792
+ // block per image plus an optional `<context>` block for cross-image analysis.
793
+ // The parser maps each item back to its image in the text injected into the
794
+ // main model; any parse failure falls back to treating the whole gateway
795
+ // answer as a single description — the turn never breaks.
796
+
797
+ export type VisionIntent = 'compare' | 'ocr' | 'reconstruct' | 'bugs' | 'describe'
798
+
799
+ const INTENT_PATTERNS: Record<Exclude<VisionIntent, 'describe'>, RegExp> = {
800
+ compare:
801
+ /\b(compare|comparar|comparação|diferença|diferenças|difference|differences|qual é melhor|qual e melhor|which is better|versus|vs)\b/i,
802
+ ocr: /\b(texto|ler|read|ocr|extrair|extraia|extract|transcrever|transcreva|transcribe)\b/i,
803
+ reconstruct:
804
+ /\b(código|codigo|code|implementar|implemente|reconstruir|reconstrua|html|css|component|ui|tela|screen)\b/i,
805
+ bugs: /\b(bug|erro|problema|issue|debug|não funciona|nao funciona|not working)\b/i,
806
+ }
807
+
808
+ /**
809
+ * Classify the user's request into a vision intent, in priority order:
810
+ * `compare` (requires 2+ images) → `ocr` → `reconstruct` → `bugs` → `describe`.
811
+ * Term matching is case-insensitive, word-boundary based, PT + EN.
812
+ */
813
+ export function detectVisionIntent(userText: string, imageCount: number): VisionIntent {
814
+ if (imageCount >= 2 && INTENT_PATTERNS.compare.test(userText)) return 'compare'
815
+ if (INTENT_PATTERNS.ocr.test(userText)) return 'ocr'
816
+ if (INTENT_PATTERNS.reconstruct.test(userText)) return 'reconstruct'
817
+ if (INTENT_PATTERNS.bugs.test(userText)) return 'bugs'
818
+ return 'describe'
819
+ }
820
+
821
+ /** Escape `&`, `<`, `>` as XML entities (& first) — for template interpolation. */
822
+ export function escapeXml(text: string): string {
823
+ return text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;')
824
+ }
825
+
826
+ /** Reverse of `escapeXml` — applied to parsed `<description>`/`<context>` bodies. */
827
+ function unescapeXml(text: string): string {
828
+ return text.replaceAll('&lt;', '<').replaceAll('&gt;', '>').replaceAll('&amp;', '&')
829
+ }
830
+
831
+ const INTENT_TASK_INSTRUCTIONS: Record<VisionIntent, string> = {
832
+ describe:
833
+ 'Descreva cada imagem fielmente e em detalhes: texto visível verbatim, cores, layout, objetos.',
834
+ compare:
835
+ 'Descreva cada imagem no seu próprio bloco <item>; depois coloque uma comparação direta e exaustiva no <context> (diferenças por região/elemento), terminando com um veredito claro.',
836
+ ocr: 'Transcreva literalmente todo o texto visível em cada imagem, preservando ordem e estrutura. A transcrição vai dentro do <description> da imagem correspondente.',
837
+ reconstruct:
838
+ 'Descreva cada imagem com detalhe de reconstrução para reprodução em código: layout completo (regiões, alinhamento, espaçamento), todo texto visível verbatim, paleta de cores exata em hex, fontes/tamanhos/pesos por elemento, estrutura de componentes e estados.',
839
+ bugs: 'Descreva o sintoma visual observado em cada imagem: o que está errado, onde, esperado vs real, com referências de região precisas e contexto suficiente para localizar e reproduzir.',
840
+ }
841
+
842
+ const INTENT_CONTEXT_INSTRUCTIONS: Record<VisionIntent, string> = {
843
+ compare:
844
+ 'Depois dos itens, adicione UM bloco <context> com a análise cruzada das imagens (comparação, veredito).',
845
+ describe: '',
846
+ ocr: '',
847
+ reconstruct: '',
848
+ bugs: '',
849
+ }
850
+
851
+ /**
852
+ * Build the calibrated prompt for the native vision gateway call. The model is
853
+ * instructed to reply with N `<item id="N"><description>…</description></item>`
854
+ * blocks (N = imageCount) and, for `compare`, one extra `<context>` block.
855
+ * The user's text is XML-escaped when interpolated so it cannot break the
856
+ * format instructions. `intent` is auto-detected from `userText` +
857
+ * `imageCount` when omitted.
858
+ */
859
+ export function buildNativeVisionPrompt(
860
+ userText: string,
861
+ imageCount: number,
862
+ intent: VisionIntent = detectVisionIntent(userText, imageCount),
863
+ ): string {
864
+ const count = Math.max(1, imageCount)
865
+ const request =
866
+ escapeXml(userText.trim()) || '(sem pedido explícito — descreva as imagens com precisão)'
867
+ const contextInstruction = INTENT_CONTEXT_INSTRUCTIONS[intent]
868
+ const contextLine = contextInstruction ? ` ${contextInstruction}` : ''
869
+ return [
870
+ 'You are a multimodal analysis specialist embedded inside a coding assistant. The coding',
871
+ 'assistant CANNOT perceive the images in this turn — your output is its ONLY view of them.',
872
+ 'It will act on exactly what you write.',
873
+ '',
874
+ "READ THE USER REQUEST FIRST: infer the user's goal and calibrate the depth of each",
875
+ 'description to it.',
876
+ '',
877
+ INTENT_TASK_INSTRUCTIONS[intent],
878
+ '',
879
+ 'OUTPUT FORMAT — binding, no exceptions:',
880
+ `Responda com UM bloco <item id="N"> para cada uma das ${count} imagens anexadas`,
881
+ `(N = 1..${count}), em ordem. Cada bloco contém um <description>.${contextLine}`,
882
+ '',
883
+ '<item id="1">',
884
+ '<description>...apenas fatos sobre a imagem 1...</description>',
885
+ '</item>',
886
+ '<item id="2">',
887
+ '<description>...apenas fatos sobre a imagem 2...</description>',
888
+ '</item>',
889
+ '',
890
+ 'RULES',
891
+ '- O primeiro caractere da sua resposta deve ser "<". Sem preâmbulo, sem "Aqui está",',
892
+ ' sem observações finais, sem wrap em blocos de código.',
893
+ '- ESCAPING — obrigatório: dentro dos corpos de <description> e <context>, escape TODO',
894
+ ' "&", "<" e ">" literais como "&amp;", "&lt;", "&gt;". Isso vale para qualquer texto',
895
+ ' tag-like reproduzido verbatim (HTML, XML, erros com colchetes angulares). NÃO escape',
896
+ ' as tags estruturais <item>, <description>, <context>.',
897
+ '- Nunca invente detalhes que não consegue perceber. Se um texto estiver ilegível,',
898
+ ' escreva "ilegível". Coloque análises cruzadas (comparação, veredito) no <context>,',
899
+ ' nunca dentro de um <description>.',
900
+ '',
901
+ `User's request: ${request}`,
902
+ ].join('\n')
903
+ }
904
+
905
+ export type StructuredVisionItem = { id: number; description: string }
906
+ export type StructuredVisionResponse = { items: StructuredVisionItem[]; context: string | null }
907
+
908
+ const ITEM_BLOCK_RE = /<item\b[^>]*>([\s\S]*?)<\/item>/gi
909
+ const OPEN_TAG_RE = /<item\b[^>]*>/i
910
+ const DESCRIPTION_BODY_RE = /<description>([\s\S]*?)<\/description>/i
911
+ const CONTEXT_BODY_RE = /<context>([\s\S]*?)<\/context>/i
912
+ const ITEM_ID_ATTR_RE = /id\s*=\s*["']?(\d+)["']?/i
913
+ const BARE_DESCRIPTION_RE = /<description>([\s\S]*?)<\/description>/gi
914
+
915
+ /**
916
+ * Parse the gateway's structured answer into per-image `<item>` descriptions
917
+ * plus an optional `<context>` block. Resistant to formatting variations
918
+ * (whitespace, quotes, tag case, missing id → sequential ids, markdown fences
919
+ * around the XML). XML entities in the bodies are unescaped for the main
920
+ * model. Returns null when no `<item>`/`<description>` block is found —
921
+ * callers then treat the whole answer as a single description (the
922
+ * pre-structured behavior, no extra gateway call).
923
+ */
924
+ export function parseStructuredVisionResponse(text: string): StructuredVisionResponse | null {
925
+ const items: StructuredVisionItem[] = []
926
+ let match = ITEM_BLOCK_RE.exec(text)
927
+ while (match !== null) {
928
+ const desc = DESCRIPTION_BODY_RE.exec(match[1] ?? '')
929
+ if (desc) {
930
+ const openTag = OPEN_TAG_RE.exec(match[0] ?? '')?.[0] ?? ''
931
+ const rawId = ITEM_ID_ATTR_RE.exec(openTag)?.[1]
932
+ const description = unescapeXml((desc[1] ?? '').trim())
933
+ items.push({
934
+ id: rawId !== undefined ? Number(rawId) : items.length + 1,
935
+ description,
936
+ })
937
+ }
938
+ match = ITEM_BLOCK_RE.exec(text)
939
+ }
940
+ if (items.length === 0) {
941
+ let bareMatch = BARE_DESCRIPTION_RE.exec(text)
942
+ while (bareMatch !== null) {
943
+ items.push({
944
+ id: items.length + 1,
945
+ description: unescapeXml((bareMatch[1] ?? '').trim()),
946
+ })
947
+ bareMatch = BARE_DESCRIPTION_RE.exec(text)
948
+ }
949
+ }
950
+ const contextMatch = CONTEXT_BODY_RE.exec(text)
951
+ const context = contextMatch ? unescapeXml((contextMatch[1] ?? '').trim()) : null
952
+ return items.length > 0 ? { items, context } : null
953
+ }
954
+
955
+ /**
956
+ * Build the text injected into the main model from a parsed structured
957
+ * response: each `<item id>` is mapped to its image (by id, falling back to
958
+ * position when an id is missing), followed by the optional cross-image
959
+ * `<context>` block. The fallback single-description path stays in
960
+ * `generateNativeInjection`.
961
+ */
962
+ export function buildStructuredInjection(
963
+ items: readonly StructuredVisionItem[],
964
+ context: string | null,
965
+ userText: string,
966
+ modelID: string,
967
+ imageCount: number,
968
+ ): string {
969
+ const count = Math.max(1, imageCount)
970
+ const byId = new Map(items.map((item) => [item.id, item.description]))
971
+ const header =
972
+ count > 1
973
+ ? `The user shared ${count} images. Vision descriptions (native, ${modelID}):`
974
+ : `The user shared an image. Vision description (native, ${modelID}):`
975
+ const lines = [header]
976
+ for (let i = 1; i <= count; i += 1) {
977
+ const description = byId.get(i) ?? items[i - 1]?.description
978
+ lines.push(`- Image ${i}: ${description ?? '(no description provided)'}`)
979
+ }
980
+ if (context) {
981
+ lines.push('', 'Cross-image context:', context)
982
+ }
983
+ lines.push('', `User's request: ${userText || '(analyze the image)'}`)
984
+ return lines.join('\n')
985
+ }
986
+
987
+ // ─── Tool name resolution ──────────────────────────────────────────────────
988
+ // env PANTHEON_VISION_TOOL > config imageAnalysisTool > dynamic detection
989
+ // (prefer the canonical Pantheon MCP) > canonical Pantheon MCP default —
990
+ // but ONLY when real evidence says the MCP is available (P2-3).
991
+
992
+ /**
993
+ * Real evidence that the pantheon-vision MCP is available at runtime:
994
+ * - its tools appear in client.tool.ids() (direct), OR
995
+ * - client.mcp.status() lists it with status 'connected' (registration).
996
+ * With --no-mcp / an uninstalled MCP there is NO such evidence, and
997
+ * defaulting to the canonical tool would make the model call a tool that
998
+ * does not exist — callers must return null instead ('Vision fallback
999
+ * unavailable').
1000
+ */
1001
+ async function hasPantheonVisionMcp(
1002
+ client: PluginInput['client'],
1003
+ directory: string,
1004
+ ): Promise<boolean> {
1005
+ try {
1006
+ const idsResponse = await client.tool?.ids?.({ query: { directory } })
1007
+ const ids = idsResponse?.data
1008
+ if (Array.isArray(ids) && ids.some((id) => isPantheonVisionServerTool(id))) return true
1009
+ } catch {
1010
+ // Fall through to the MCP status map.
1011
+ }
1012
+ try {
1013
+ const mcpResponse = await client.mcp?.status?.({ query: { directory } })
1014
+ const statuses = mcpResponse?.data
1015
+ if (statuses && typeof statuses === 'object') {
1016
+ for (const [name, status] of Object.entries(statuses)) {
1017
+ if (!/[pP]antheon[-_]vision/.test(name)) continue
1018
+ if ((status as { status?: string } | undefined)?.status === 'connected') return true
1019
+ }
1020
+ }
1021
+ } catch {
1022
+ // Fall through — no evidence.
1023
+ }
1024
+ return false
1025
+ }
1026
+
1027
+ async function resolveImageAnalysisTool(
1028
+ client: PluginInput['client'],
1029
+ directory: string,
1030
+ configPromise: Promise<VisionConfig>,
1031
+ ): Promise<string | null> {
1032
+ const envTool = process.env.PANTHEON_VISION_TOOL?.trim()
1033
+ if (envTool) return envTool
1034
+ const config = await configPromise
1035
+ if (config.imageAnalysisTool) return config.imageAnalysisTool
1036
+ try {
1037
+ const response = await client.tool?.ids?.({ query: { directory } })
1038
+ const ids = response?.data
1039
+ if (Array.isArray(ids)) {
1040
+ // Exact canonical ID first (fast path), then the underscore form and the
1041
+ // legacy mcp__ hyphen form (both match the pantheon[-_]vision + action
1042
+ // regexes), then the loose describe fallback for older ID shapes.
1043
+ const match =
1044
+ ids.find((id) => id === DEFAULT_IMAGE_ANALYSIS_TOOL) ??
1045
+ ids.find((id) => isPantheonVisionServerTool(id) && isPantheonVisionActionTool(id)) ??
1046
+ ids.find((id) => isPantheonVisionServerTool(id) && /describe/i.test(id))
1047
+ if (match) return match
1048
+ }
1049
+ } catch {
1050
+ // Detection failed (network/MCP error). Returning null keeps the explicit
1051
+ // 'Vision fallback unavailable' safe-failure message in the turn instead
1052
+ // of guessing at a tool we could not verify — the turn never breaks
1053
+ // (interception is best-effort). Returning DEFAULT here too would hide
1054
+ // real detection failures behind a tool name the model might not have.
1055
+ return null
1056
+ }
1057
+ // OpenCode 1.18.11 keeps MCP tools in the MCP service, NOT in
1058
+ // client.tool.ids() — an empty/absent list is the NORMAL runtime state, so
1059
+ // fall back to the canonical default ONLY when there is independent
1060
+ // evidence the pantheon-vision MCP is installed and connected. Without it
1061
+ // (--no-mcp, uninstalled, disabled) instructing the canonical tool would
1062
+ // fail the turn with a phantom tool — return null instead.
1063
+ const available = await hasPantheonVisionMcp(client, directory)
1064
+ return available ? DEFAULT_IMAGE_ANALYSIS_TOOL : null
1065
+ }
1066
+
1067
+ // ─── Temp image lifecycle ──────────────────────────────────────────────────
1068
+
1069
+ /**
1070
+ * Age-guarded, REFCOUNTED cleanup of a session's temp images.
1071
+ *
1072
+ * A `session.idle` can fire between auto-continue turns — not just at real
1073
+ * session end — so a temp file referenced by the next turn (seconds old) must
1074
+ * survive. Only paths older than `graceMs` are unlinked; younger paths stay
1075
+ * registered and are swept by the next idle. `server.instance.disposed` and
1076
+ * `session.deleted` pass `graceMs = 0` to unlink everything while the session
1077
+ * is gone / process is dying.
1078
+ *
1079
+ * Content-hash filenames dedup identical images ACROSS sessions: two sessions
1080
+ * can reference the SAME path. `tempFileRefs` (path → set of sessions) makes
1081
+ * the unlink decision reference-aware — the file is only unlinked when the
1082
+ * LAST referencing session releases it; otherwise the other session's MCP
1083
+ * tool call would hit file-not-found (P2-4). When `tempFileRefs` is omitted
1084
+ * (legacy callers) the unconditional behavior is kept.
1085
+ *
1086
+ * The temp directory itself is never removed: it may hold files from other
1087
+ * sessions or processes. The session's description cache is always dropped
1088
+ * (cache is per-session state).
1089
+ */
1090
+ export async function cleanupSessionTempImages(
1091
+ sessionTempFiles: Map<string, Map<string, number>>,
1092
+ sessionVisionText: Map<string, Map<string, string>>,
1093
+ nativeVisionSessions: Set<string>,
1094
+ sessionID: string,
1095
+ graceMs: number = TEMP_FILE_GRACE_MS,
1096
+ sessionDescriptionCache?: Map<string, Map<string, DescriptionCacheEntry>>,
1097
+ tempFileRefs?: Map<string, Set<string>>,
1098
+ ): Promise<void> {
1099
+ const files = sessionTempFiles.get(sessionID)
1100
+ if (files) {
1101
+ const now = Date.now()
1102
+ for (const [path, createdAt] of files) {
1103
+ // graceMs === 0 means "unlink everything now" (session gone / process
1104
+ // dying); a positive grace skips files younger than the threshold so the
1105
+ // next auto-continue turn can still read them.
1106
+ if (graceMs > 0 && now - createdAt <= graceMs) continue
1107
+ const refs = tempFileRefs?.get(path)
1108
+ if (refs && tempFileRefs) {
1109
+ refs.delete(sessionID)
1110
+ // Another session still references this content-hash path: keep the
1111
+ // file on disk and drop this session's tracking of it.
1112
+ if (refs.size > 0) {
1113
+ files.delete(path)
1114
+ continue
1115
+ }
1116
+ tempFileRefs.delete(path)
1117
+ }
1118
+ try {
1119
+ await unlink(path)
1120
+ } catch {
1121
+ // Best effort.
1122
+ }
1123
+ forgetTempFile(path)
1124
+ files.delete(path)
1125
+ }
1126
+ if (files.size === 0) sessionTempFiles.delete(sessionID)
1127
+ }
1128
+ sessionVisionText.delete(sessionID)
1129
+ nativeVisionSessions.delete(sessionID)
1130
+ sessionDescriptionCache?.delete(sessionID)
1131
+ }
1132
+
1133
+ type VisionHistoryMessage = {
1134
+ info?: { sessionID?: string; id?: string }
1135
+ sessionID?: string
1136
+ id?: string
1137
+ parts?: unknown
1138
+ content?: unknown
1139
+ [key: string]: unknown
1140
+ }
1141
+
1142
+ function syntheticHistoryPart(message: VisionHistoryMessage, text: string): TextPart {
1143
+ return {
1144
+ id: syntheticPartId(),
1145
+ sessionID: message.info?.sessionID ?? '',
1146
+ messageID: message.info?.id ?? '',
1147
+ type: 'text',
1148
+ text,
1149
+ synthetic: true,
1150
+ }
1151
+ }
1152
+
1153
+ /**
1154
+ * Remove image-bearing parts before a provider serializer can turn them into
1155
+ * image_url content. This hook is deliberately synchronous in spirit: it only
1156
+ * reuses text cached by chat.message and never calls a vision tool/model.
1157
+ */
1158
+ export function sanitizeVisionHistory(
1159
+ output: { messages?: unknown },
1160
+ sessionVisionText: ReadonlyMap<string, ReadonlyMap<string, string>>,
1161
+ nativeVisionSessions: ReadonlySet<string> = new Set(),
1162
+ ): void {
1163
+ if (!Array.isArray(output.messages)) return
1164
+
1165
+ for (const message of output.messages) {
1166
+ if (!message || typeof message !== 'object') continue
1167
+ const historyMessage = message as VisionHistoryMessage
1168
+ const sessionID = historyMessage.info?.sessionID ?? historyMessage.sessionID ?? ''
1169
+ if (nativeVisionSessions.has(sessionID)) continue
1170
+ const descriptions = sessionVisionText.get(sessionID)
1171
+ const sanitize = (value: unknown, inParts: boolean): unknown => {
1172
+ if (Array.isArray(value)) return value.map((item) => sanitize(item, inParts))
1173
+ if (!value || typeof value !== 'object') return value
1174
+
1175
+ const raw = value as { type?: unknown; id?: unknown; mime?: unknown }
1176
+ const isImageUrl = raw.type === 'image_url'
1177
+ const isImageFile =
1178
+ raw.type === 'file' &&
1179
+ typeof raw.mime === 'string' &&
1180
+ raw.mime.toLowerCase().startsWith('image/')
1181
+ if (isImageUrl || isImageFile) {
1182
+ const partID = typeof raw.id === 'string' ? raw.id : ''
1183
+ const text =
1184
+ descriptions?.get(partID) ??
1185
+ 'Imagem removida do histórico antes do envio ao provedor textual.'
1186
+ return inParts ? syntheticHistoryPart(historyMessage, text) : { type: 'text', text }
1187
+ }
1188
+
1189
+ const copy: Record<string, unknown> = {}
1190
+ for (const [key, child] of Object.entries(value)) {
1191
+ copy[key] = sanitize(child, key === 'parts')
1192
+ }
1193
+ return copy
1194
+ }
1195
+
1196
+ const sanitized = sanitize(historyMessage, false) as VisionHistoryMessage
1197
+ for (const [key, value] of Object.entries(sanitized)) {
1198
+ historyMessage[key] = value
1199
+ }
1200
+ }
1201
+ }
1202
+
1203
+ // ─── Handler factory ───────────────────────────────────────────────────────
1204
+
1205
+ export function createVisionHandler(input: PluginInput) {
1206
+ const { client, directory } = input
1207
+ const sessionTempFiles = new Map<string, Map<string, number>>()
1208
+ // Content-hash dedup makes identical images share one path ACROSS sessions;
1209
+ // this map (path → sessions holding it) makes cleanup reference-aware so a
1210
+ // file is only unlinked when the LAST session releases it (P2-4).
1211
+ const tempFileRefs = new Map<string, Set<string>>()
1212
+ const sessionVisionText = new Map<string, Map<string, string>>()
1213
+ const nativeVisionSessions = new Set<string>()
1214
+ const sessionDescriptionCache = new Map<string, Map<string, DescriptionCacheEntry>>()
1215
+ let configPromise: Promise<VisionConfig> | null = null
1216
+ let toolNamePromise: Promise<string | null> | null = null
1217
+
1218
+ const getConfig = (): Promise<VisionConfig> => {
1219
+ configPromise ??= loadVisionConfig(directory)
1220
+ return configPromise
1221
+ }
1222
+
1223
+ const getToolName = (): Promise<string | null> => {
1224
+ toolNamePromise ??= resolveImageAnalysisTool(client, directory, getConfig())
1225
+ return toolNamePromise
1226
+ }
1227
+
1228
+ const chatMessage: NonNullable<Hooks['chat.message']> = async (hookInput, output) => {
1229
+ try {
1230
+ const message: UserMessage | undefined = output?.message
1231
+ if (!message || message.role !== 'user') return
1232
+ const model = hookInput?.model ?? message.model
1233
+ if (modelAcceptsImages(model)) {
1234
+ nativeVisionSessions.add(hookInput.sessionID)
1235
+ return
1236
+ }
1237
+ nativeVisionSessions.delete(hookInput.sessionID)
1238
+ const config = await getConfig()
1239
+ const patterns =
1240
+ config.models && config.models.length > 0 ? config.models : DEFAULT_MODEL_PATTERNS
1241
+ if (!modelMatchesAnyPattern(model, patterns)) {
1242
+ // Pattern configuration may disable enrichment, but it must never
1243
+ // disable the text-only provider safety boundary.
1244
+ const currentTurn = { messages: [{ ...message, parts: output.parts }] }
1245
+ sanitizeVisionHistory(currentTurn, sessionVisionText)
1246
+ const sanitized = currentTurn.messages as VisionHistoryMessage[]
1247
+ output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
1248
+ return
1249
+ }
1250
+
1251
+ const parts: Part[] = Array.isArray(output.parts) ? output.parts : []
1252
+ const images = parts.filter(isImageFilePart)
1253
+ if (images.length === 0) {
1254
+ // Some runtimes hand the current turn to plugins already serialized as
1255
+ // provider content. Do not let that shape bypass the final sanitizer.
1256
+ const currentTurn = { messages: [{ ...message, parts: output.parts }] }
1257
+ sanitizeVisionHistory(currentTurn, sessionVisionText)
1258
+ const sanitized = currentTurn.messages as VisionHistoryMessage[]
1259
+ output.parts.splice(0, output.parts.length, ...((sanitized[0]?.parts as Part[]) ?? []))
1260
+ return
1261
+ }
1262
+
1263
+ const userText = parts
1264
+ .filter(isTextPart)
1265
+ .map((part) => part.text)
1266
+ .join('\n')
1267
+ .trim()
1268
+
1269
+ const resolved: Array<{ filePart: FilePart; target: SavedImage }> = []
1270
+ for (const image of images.filter(isSupportedImageFilePart)) {
1271
+ const target = await resolveImageTarget(image)
1272
+ if (target) resolved.push({ filePart: image, target })
1273
+ }
1274
+ const saved = resolved.map((item) => item.target)
1275
+
1276
+ const tempPaths = saved.filter((item) => item.temporary).map((item) => item.path)
1277
+ if (tempPaths.length > 0) {
1278
+ const createdAt = Date.now()
1279
+ const existing = sessionTempFiles.get(hookInput.sessionID) ?? new Map<string, number>()
1280
+ for (const path of tempPaths) {
1281
+ existing.set(path, createdAt)
1282
+ // Track the session as a holder of this (content-hash) path so
1283
+ // cleanup only unlinks when the LAST holder releases it.
1284
+ let refs = tempFileRefs.get(path)
1285
+ if (!refs) {
1286
+ refs = new Set()
1287
+ tempFileRefs.set(path, refs)
1288
+ }
1289
+ refs.add(hookInput.sessionID)
1290
+ }
1291
+ sessionTempFiles.set(hookInput.sessionID, existing)
1292
+ }
1293
+
1294
+ // Native-first: call the multimodal model directly when a key exists
1295
+ // (mode auto/native). Any failure — no key, unsupported provider,
1296
+ // oversized/invalid image, HTTP/network error — falls back to the tool.
1297
+ // The prompt is calibrated to the user's intent (compare/ocr/
1298
+ // reconstruct/bugs/describe) and asks for a structured answer
1299
+ // (<item>/<context>), which is parsed and mapped back to each image.
1300
+ // Descriptions are cached per session: the key is the calibrated prompt
1301
+ // + the content hashes of the images, so an identical re-ask within the
1302
+ // TTL reuses the stored (possibly structured) text without another
1303
+ // gateway call, while a new question about the same image misses and is
1304
+ // answered freshly.
1305
+ let injection: string | null = null
1306
+ if (getVisionMode(process.env, config.mode) !== 'tool') {
1307
+ try {
1308
+ const target = await resolveNativeVisionTarget({ config })
1309
+ if (target) {
1310
+ const payloads = await buildNativeImagePayloads(resolved)
1311
+ if (payloads) {
1312
+ let sessionCache = sessionDescriptionCache.get(hookInput.sessionID)
1313
+ if (!sessionCache) {
1314
+ sessionCache = new Map()
1315
+ sessionDescriptionCache.set(hookInput.sessionID, sessionCache)
1316
+ }
1317
+ const intent = detectVisionIntent(userText, payloads.length)
1318
+ const prompt = buildNativeVisionPrompt(userText, payloads.length, intent)
1319
+ const cacheKey = buildDescriptionCacheKey(prompt, payloads)
1320
+ let rawText = getCachedDescription(sessionCache, cacheKey)
1321
+ if (rawText === null) {
1322
+ rawText = await describeImagesNative(payloads, target, prompt)
1323
+ if (rawText !== null) setCachedDescription(sessionCache, cacheKey, rawText)
1324
+ }
1325
+ if (rawText !== null) {
1326
+ const structured = parseStructuredVisionResponse(rawText)
1327
+ injection = structured
1328
+ ? buildStructuredInjection(
1329
+ structured.items,
1330
+ structured.context,
1331
+ userText,
1332
+ target.modelID,
1333
+ payloads.length,
1334
+ )
1335
+ : generateNativeInjection(rawText, userText, target.modelID)
1336
+ }
1337
+ }
1338
+ }
1339
+ } catch {
1340
+ // Native vision is best-effort; fall back to the tool pattern.
1341
+ }
1342
+ }
1343
+
1344
+ if (injection === null && saved.length > 0) {
1345
+ const toolName = await getToolName()
1346
+ injection = toolName
1347
+ ? generateInjectionPrompt(saved, userText, toolName, config.promptTemplate)
1348
+ : 'Vision fallback unavailable: pantheon-vision MCP is unavailable. The image was removed safely; please configure the MCP server and retry.'
1349
+ }
1350
+ if (injection === null) {
1351
+ injection = 'A imagem compartilhada foi removida antes do envio ao provedor textual.'
1352
+ }
1353
+
1354
+ const descriptions = sessionVisionText.get(hookInput.sessionID) ?? new Map<string, string>()
1355
+ for (const image of images) descriptions.set(image.id, injection)
1356
+ sessionVisionText.set(hookInput.sessionID, descriptions)
1357
+
1358
+ const injectionPart: TextPart = {
1359
+ id: syntheticPartId(),
1360
+ sessionID: message.sessionID,
1361
+ messageID: message.id,
1362
+ type: 'text',
1363
+ text: injection,
1364
+ synthetic: true,
1365
+ }
1366
+ const kept = parts.filter((part) => !isImageFilePart(part))
1367
+ // Mutate in place: drop image parts, lead with the vision text
1368
+ // (native description or tool instruction).
1369
+ output.parts.splice(0, output.parts.length, injectionPart, ...kept)
1370
+ sanitizeVisionHistory({ messages: [{ ...message, parts: output.parts }] }, sessionVisionText)
1371
+ } catch {
1372
+ // Vision interception is best-effort; never fail a user turn.
1373
+ }
1374
+ }
1375
+
1376
+ const event: NonNullable<Hooks['event']> = async ({ event: ev }) => {
1377
+ if (ev.type === 'session.idle') {
1378
+ // An idle can fire between auto-continue turns: temp files younger than
1379
+ // the grace period survive, but the per-session description cache is
1380
+ // dropped (it is pure optimization, safe to reset on every idle).
1381
+ const sessionID = ev.properties?.sessionID
1382
+ if (sessionID)
1383
+ await cleanupSessionTempImages(
1384
+ sessionTempFiles,
1385
+ sessionVisionText,
1386
+ nativeVisionSessions,
1387
+ sessionID,
1388
+ TEMP_FILE_GRACE_MS,
1389
+ sessionDescriptionCache,
1390
+ tempFileRefs,
1391
+ )
1392
+ } else if (ev.type === 'session.deleted') {
1393
+ // The session is gone for good: no active turn can reference its temp
1394
+ // images, so unlink them all immediately and drop its cached state.
1395
+ // Shared paths survive while another session still references them.
1396
+ const sessionID = ev.properties.info.id
1397
+ await cleanupSessionTempImages(
1398
+ sessionTempFiles,
1399
+ sessionVisionText,
1400
+ nativeVisionSessions,
1401
+ sessionID,
1402
+ 0,
1403
+ sessionDescriptionCache,
1404
+ tempFileRefs,
1405
+ )
1406
+ } else if (ev.type === 'server.instance.disposed') {
1407
+ const sessions = [...sessionTempFiles.keys()]
1408
+ for (const sessionID of sessions) {
1409
+ await cleanupSessionTempImages(
1410
+ sessionTempFiles,
1411
+ sessionVisionText,
1412
+ nativeVisionSessions,
1413
+ sessionID,
1414
+ 0, // The process is dying: unlink everything immediately.
1415
+ sessionDescriptionCache,
1416
+ tempFileRefs,
1417
+ )
1418
+ }
1419
+ sessionVisionText.clear()
1420
+ sessionDescriptionCache.clear()
1421
+ }
1422
+ }
1423
+
1424
+ const messagesTransform: NonNullable<Hooks['experimental.chat.messages.transform']> = async (
1425
+ _hookInput,
1426
+ output,
1427
+ ) => {
1428
+ try {
1429
+ sanitizeVisionHistory(output, sessionVisionText, nativeVisionSessions)
1430
+ } catch {
1431
+ // History sanitization is best effort; never break runtimes that expose
1432
+ // the experimental hook with a partial output shape.
1433
+ }
1434
+ }
1435
+
1436
+ return { chatMessage, messagesTransform, event }
1437
+ }