pantheon-opencode 1.6.0-beta.1 → 1.6.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.global.md +182 -0
- package/AGENTS.md +10 -891
- package/CHANGELOG.md +326 -0
- package/README.md +496 -32
- package/README.pt-BR.md +285 -20
- package/bin/pantheon-init.mjs +26 -3
- package/commands/pantheon-audit.md +7 -1
- package/commands/pantheon-consolidate.md +1 -1
- package/commands/pantheon-deepwork.md +14 -3
- package/commands/pantheon-focus.md +17 -0
- package/commands/pantheon-vision.md +61 -12
- package/package.json +21 -14
- package/plugin.json +2 -2
- package/pyproject.toml +8 -6
- package/scripts/build-agents-md.mjs +374 -56
- package/scripts/changelog-merge.mjs +161 -0
- package/scripts/{code_mode_server.py → code_mode.py} +3 -3
- package/scripts/cost.mjs +244 -41
- package/scripts/doctor.mjs +151 -1
- package/scripts/install/agents-md.mjs +4 -4
- package/scripts/install/config-migration.mjs +317 -4
- package/scripts/install/health-check.mjs +21 -19
- package/scripts/install/interactive.mjs +1 -1
- package/scripts/install/opencode-version.mjs +246 -4
- package/scripts/install/opencode.mjs +627 -179
- package/scripts/install/plugin.mjs +43 -9
- package/scripts/install/shared.mjs +120 -12
- package/scripts/install/vision-prompt.mjs +161 -0
- package/scripts/install-mcp.mjs +28 -9
- package/scripts/{mcp_persistence_server.py → mcp_persistence.py} +2 -2
- package/{src/mcp/mcp_resources_server.py → scripts/mcp_resources.py} +2 -2
- package/scripts/{memory_mcp_server.py → memory_mcp.py} +2 -2
- package/scripts/probe-context-rehydrate.mjs +8 -2
- package/scripts/probe-pantheon-cost.mjs +71 -15
- package/scripts/run-node-tests.mjs +189 -0
- package/scripts/sync-artifacts.mjs +26 -13
- package/scripts/sync-tui.mjs +24 -3
- package/scripts/test-opencode-v2-sandbox.sh +180 -26
- package/scripts/validate-routing.mjs +3 -3
- package/scripts/validate-sandbox-install.mjs +138 -0
- package/scripts/versioning.mjs +13 -1
- package/scripts/vision-status.mjs +682 -0
- package/skills-lock.json +8 -8
- package/src/agents/aphrodite.md +20 -37
- package/src/agents/apollo.md +7 -3
- package/src/agents/athena.md +63 -35
- package/src/agents/demeter.md +8 -28
- package/src/agents/gaia.md +1 -1
- package/src/agents/hephaestus.md +10 -21
- package/src/agents/hermes.md +15 -38
- package/src/agents/iris.md +1 -1
- package/src/agents/mnemosyne.md +18 -54
- package/src/agents/nyx.md +1 -1
- package/src/agents/prometheus.md +16 -20
- package/src/agents/talos.md +10 -2
- package/src/agents/themis.md +5 -5
- package/src/agents/zeus.md +99 -129
- package/src/instructions/agent-return-format.instructions.md +18 -3
- package/src/instructions/backend-standards.instructions.md +2 -0
- package/src/instructions/frontend-standards.instructions.md +3 -1
- package/src/instructions/memory-protocol.instructions.md +11 -110
- package/src/instructions/zeus-anti-stall.instructions.md +14 -61
- package/src/instructions/zeus-communication-rules.instructions.md +0 -1
- package/src/instructions/zeus-council-synthesis.instructions.md +32 -260
- package/src/instructions/zeus-memory-operations.instructions.md +26 -0
- package/src/instructions/zeus-timeout-retry.instructions.md +40 -87
- package/src/mcp/{code_mode_server.py → code_mode.py} +3 -3
- package/src/mcp/eval_store.py +2 -2
- package/src/mcp/{mcp_persistence_server.py → mcp_persistence.py} +2 -2
- package/{scripts/mcp_resources_server.py → src/mcp/mcp_resources.py} +2 -2
- package/src/mcp/{memory_mcp_server.py → memory_mcp.py} +2 -2
- package/src/mcp/{pantheon_vision_server.py → pantheon_vision.py} +286 -42
- package/src/mcp/requirements-mcp.txt +6 -4
- package/src/mcp/requirements-vision.txt +2 -2
- package/src/pantheon/cost-command.ts +399 -101
- package/src/pantheon/delegation-enforce.ts +10 -1
- package/src/pantheon/presets.d.mts +0 -8
- package/src/pantheon/presets.mjs +5 -32
- package/src/pantheon/v2-bounded-read.ts +640 -0
- package/src/pantheon/v2-delegation-enforce.ts +214 -0
- package/src/pantheon/v2-execute-after.ts +186 -0
- package/src/pantheon/v2-tools.ts +429 -0
- package/src/pantheon/v2-unsupported.d.mts +11 -0
- package/src/pantheon/v2-unsupported.mjs +90 -0
- package/src/pantheon/v2-vision.ts +622 -0
- package/src/pantheon/vision-local.ts +531 -0
- package/src/pantheon/vision.ts +1160 -115
- package/src/plugin-v2.ts +1145 -219
- package/src/plugin.ts +12 -9
- package/src/plugins/tui/dist/server.d.ts +1 -2
- package/src/plugins/tui/dist/server.d.ts.map +1 -1
- package/src/plugins/tui/dist/tui-v2.d.ts +154 -0
- package/src/plugins/tui/dist/tui-v2.d.ts.map +1 -0
- package/src/plugins/tui/dist/tui-v2.js +659 -0
- package/src/plugins/tui/dist/tui-v2.js.map +1 -0
- package/src/plugins/tui/dist/tui.d.ts +85 -82
- package/src/plugins/tui/dist/tui.d.ts.map +1 -1
- package/src/plugins/tui/dist/tui.js +72 -59
- package/src/plugins/tui/index-v2.ts +6 -0
- package/src/plugins/tui/package-lock.json +5516 -1353
- package/src/plugins/tui/package.json +22 -12
- package/src/plugins/tui/src/index-v2.tsx +734 -0
- package/src/plugins/tui/src/index.tsx +106 -85
- package/src/plugins/tui/src/opencode-v2.d.ts +13 -0
- package/src/plugins/tui/tsconfig.build.json +5 -0
- package/src/plugins/tui/tsconfig.typecheck.json +5 -0
- package/src/plugins/tui/tsdown.config.ts +22 -2
- package/src/plugins/tui/tui-v2.ts +2 -0
- package/src/plugins/tui/tui.ts +3 -0
- package/src/routing.yml +11 -21
- package/src/skills/README.md +3 -3
- package/src/skills/agent-coordination/SKILL.md +7 -2
- package/src/skills/artifact-management/SKILL.md +12 -5
- package/src/skills/auto-continue/SKILL.md +39 -30
- package/src/skills/code-review-checklist/SKILL.md +5 -5
- package/src/skills/context-compression/SKILL.md +26 -879
- package/src/skills/session-goal/SKILL.md +14 -5
- package/src/skills/spec-driven-development/SKILL.md +3 -1
- package/src/skills/tdd-with-agents/SKILL.md +18 -10
- package/opencode.json +0 -58
package/src/pantheon/vision.ts
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
chmodSync,
|
|
4
|
+
existsSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
readFileSync,
|
|
7
|
+
renameSync,
|
|
8
|
+
unlinkSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs'
|
|
11
|
+
import { readFile, unlink } from 'node:fs/promises'
|
|
4
12
|
import { homedir, tmpdir } from 'node:os'
|
|
5
13
|
import { join } from 'node:path'
|
|
6
14
|
import { fileURLToPath } from 'node:url'
|
|
@@ -13,6 +21,7 @@ import {
|
|
|
13
21
|
resolveActivePreset,
|
|
14
22
|
visionBrokenOnGateway,
|
|
15
23
|
} from './presets.mjs'
|
|
24
|
+
import { buildLocalVisionBlock } from './vision-local.ts'
|
|
16
25
|
|
|
17
26
|
// ─── Constants ────────────────────────────────────────────────────────────
|
|
18
27
|
|
|
@@ -82,23 +91,77 @@ const PROMPT_TEMPLATE_VARIABLES = [
|
|
|
82
91
|
|
|
83
92
|
// ─── Native vision (provider-direct, OpenAI-compatible) ─────────────────────
|
|
84
93
|
// Preferred path: when an API key is available the plugin calls the multimodal
|
|
85
|
-
// model DIRECTLY via
|
|
86
|
-
//
|
|
94
|
+
// model DIRECTLY via an OpenAI-compatible endpoint and the image is replaced
|
|
95
|
+
// by the returned text description (no MCP tool). The MCP tool pattern
|
|
87
96
|
// (imageAnalysisTool) remains the fallback for missing keys / provider errors.
|
|
88
97
|
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
98
|
+
// Model + endpoint come from the host catalog (GET /config/providers) when no
|
|
99
|
+
// explicit model is configured (env > config > preset > discovery > DEFAULT);
|
|
100
|
+
// the Zen endpoints stay as the no-catalog fallback. anthropic/openai native
|
|
101
|
+
// vision works only when their catalog entries declare an OpenAI-compatible
|
|
102
|
+
// wire format (see NATIVE_VISION_NPM_ALLOWLIST) — otherwise they fall back to
|
|
103
|
+
// the MCP tool pattern.
|
|
92
104
|
|
|
93
105
|
const DEFAULT_VISION_MODEL = 'opencode-go/mimo-v2.5'
|
|
94
106
|
const ZEN_BASE_URL = 'https://opencode.ai/zen/v1'
|
|
95
107
|
const ZEN_GO_BASE_URL = 'https://opencode.ai/zen/go/v1'
|
|
96
108
|
const NATIVE_PROMPT =
|
|
97
109
|
'Descreva fielmente esta imagem em detalhes (texto visível, cores, layout, objetos).'
|
|
98
|
-
|
|
99
|
-
|
|
110
|
+
/**
|
|
111
|
+
* Base timeout for the native vision gateway call (small/unknown payloads).
|
|
112
|
+
* 60s, not 20s: the vision model is a REASONING model (mimo-v2.6-flash) and a
|
|
113
|
+
* measured round-trip for a ~380 KB image spends 1.0k–1.2k tokens on reasoning
|
|
114
|
+
* alone, completing in 20–30s. A 20s base aborted legitimate completed
|
|
115
|
+
* descriptions as `network`/timeout (observed at exactly 20000ms). The per-MB
|
|
116
|
+
* term below still scales larger payloads; THIS base carries the reasoning
|
|
117
|
+
* overhead that no payload term captures.
|
|
118
|
+
*/
|
|
119
|
+
const NATIVE_TIMEOUT_MS = 60_000
|
|
120
|
+
/** Extra timeout per whole MB of image payload (linear scaling from the base). */
|
|
121
|
+
const NATIVE_TIMEOUT_PER_MB_MS = 10_000
|
|
122
|
+
/** Hard ceiling for the native vision gateway call (showlotus reference: 300s). */
|
|
123
|
+
const NATIVE_TIMEOUT_MAX_MS = 300_000
|
|
124
|
+
const BYTES_PER_MB = 1024 * 1024
|
|
125
|
+
/**
|
|
126
|
+
* Completion-token ceiling for the native vision call. The model spends this
|
|
127
|
+
* budget on REASONING tokens FIRST: measured 1023 reasoning tokens against a
|
|
128
|
+
* 1024 cap (content came back null with finish_reason "length"), while 2048
|
|
129
|
+
* finished normally (`stop`) with a complete description. The former 500
|
|
130
|
+
* truncated EVERY reasoning call. Exported so the V2 tool surface shares one
|
|
131
|
+
* value instead of hard-coding its own copy.
|
|
132
|
+
*/
|
|
133
|
+
export const NATIVE_MAX_TOKENS = 2048
|
|
100
134
|
const MAX_NATIVE_IMAGE_BYTES = 25 * 1024 * 1024
|
|
101
135
|
|
|
136
|
+
/**
|
|
137
|
+
* Dynamic timeout for a native vision call, scaled by the payload size
|
|
138
|
+
* (adapted from showlotus/opencode-image-vision's 120s→300s scheme to this
|
|
139
|
+
* codebase's 60s base): base for small/unknown payloads, +10s per whole MB,
|
|
140
|
+
* hard-capped at 300s. Large base64 uploads take measurably longer to
|
|
141
|
+
* reach/read on the gateway side, and the previous fixed 20s budget dropped
|
|
142
|
+
* legitimate large-image descriptions as timeouts.
|
|
143
|
+
*/
|
|
144
|
+
export function computeNativeTimeoutMs(totalBytes: number): number {
|
|
145
|
+
const bytes = Number.isFinite(totalBytes) && totalBytes > 0 ? totalBytes : 0
|
|
146
|
+
const mb = Math.floor(bytes / BYTES_PER_MB)
|
|
147
|
+
return Math.min(NATIVE_TIMEOUT_MAX_MS, NATIVE_TIMEOUT_MS + mb * NATIVE_TIMEOUT_PER_MB_MS)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Approximate decoded byte size of every payload in the call, summed.
|
|
152
|
+
* Data URIs are measured by their base64 length (×3/4); remote http(s) URLs
|
|
153
|
+
* contribute 0 — their size is unknown without a network round-trip, so they
|
|
154
|
+
* get the base budget (fail-safe, not optimistic).
|
|
155
|
+
*/
|
|
156
|
+
function estimateNativePayloadsBytes(payloads: readonly NativeImagePayload[]): number {
|
|
157
|
+
let total = 0
|
|
158
|
+
for (const payload of payloads) {
|
|
159
|
+
const match = /^data:[^;,]+;base64,([A-Za-z0-9+/=]+)$/.exec(payload.url)
|
|
160
|
+
total += match ? Math.ceil(((match[1]?.length ?? 0) * 3) / 4) : 0
|
|
161
|
+
}
|
|
162
|
+
return total
|
|
163
|
+
}
|
|
164
|
+
|
|
102
165
|
/**
|
|
103
166
|
* Descriptions produced by the native path are cached per session, keyed by
|
|
104
167
|
* the user prompt + the content hashes of the attached images (see
|
|
@@ -120,10 +183,187 @@ type VisionConfig = {
|
|
|
120
183
|
|
|
121
184
|
export type ModelInfo = { providerID: string; modelID: string }
|
|
122
185
|
|
|
123
|
-
export type NativeVisionTarget = {
|
|
186
|
+
export type NativeVisionTarget = {
|
|
187
|
+
modelID: string
|
|
188
|
+
baseURL: string
|
|
189
|
+
apiKey: string
|
|
190
|
+
/**
|
|
191
|
+
* OpenCode session id, sent as the `x-opencode-session` header — and ONLY on
|
|
192
|
+
* Zen endpoints (`https://opencode.ai/zen/...`). The Zen gateway rejects a
|
|
193
|
+
* request without it (HTTP 400 `MissingSessionID`). Optional: callers attach
|
|
194
|
+
* it at the request site (`{ ...target, sessionID }`); a target without it
|
|
195
|
+
* never carries the header. Non-Zen providers must never receive it.
|
|
196
|
+
*/
|
|
197
|
+
sessionID?: string
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Non-sensitive reason a native describe produced no text. Carries ONLY the
|
|
202
|
+
* HTTP status and the model's `finish_reason` (never the key, payload, or
|
|
203
|
+
* gateway body) so a caller can build an actionable error without leaking
|
|
204
|
+
* anything.
|
|
205
|
+
*/
|
|
206
|
+
export type NativeVisionFailure = {
|
|
207
|
+
reason: 'http' | 'network' | 'timeout' | 'empty' | 'cancelled'
|
|
208
|
+
status?: number
|
|
209
|
+
/**
|
|
210
|
+
* OpenAI-compatible `finish_reason` (e.g. `length` when a reasoning model was
|
|
211
|
+
* cut off by `max_tokens`) — surfaced so a "no text" failure becomes
|
|
212
|
+
* actionable instead of opaque.
|
|
213
|
+
*/
|
|
214
|
+
finishReason?: string
|
|
215
|
+
/**
|
|
216
|
+
* The budget that elapsed when OUR dynamic timer aborted the request. Set
|
|
217
|
+
* only for `reason: 'timeout'`; a genuine transport drop stays `network`, so
|
|
218
|
+
* a timed-out reasoning call no longer reads as a vague "network error".
|
|
219
|
+
*/
|
|
220
|
+
timeoutMs?: number
|
|
221
|
+
}
|
|
124
222
|
|
|
125
223
|
type NativeVisionEndpoint = { modelID: string; baseURL: string }
|
|
126
224
|
|
|
225
|
+
// ─── Model catalog (host discovery, GET /config/providers) ──────────────────
|
|
226
|
+
// Defensive subset of the SDK `Model`/`Provider` shapes (see
|
|
227
|
+
// node_modules/@opencode-ai/sdk/dist/gen/types.gen.d.ts). Parsed with optional
|
|
228
|
+
// chaining — a malformed catalog entry is skipped, never a crash.
|
|
229
|
+
|
|
230
|
+
export type VisionCatalogModel = {
|
|
231
|
+
id?: unknown
|
|
232
|
+
providerID?: unknown
|
|
233
|
+
status?: unknown
|
|
234
|
+
capabilities?: { input?: { image?: unknown } } | undefined
|
|
235
|
+
api?: { url?: unknown; npm?: unknown } | undefined
|
|
236
|
+
cost?: { input?: unknown } | undefined
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export type VisionCatalogProvider = {
|
|
240
|
+
id?: unknown
|
|
241
|
+
models?: unknown
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Body of GET /config/providers (defensive subset), or any garbage. */
|
|
245
|
+
export type VisionCatalog =
|
|
246
|
+
| {
|
|
247
|
+
providers?: unknown
|
|
248
|
+
models?: unknown
|
|
249
|
+
}
|
|
250
|
+
| null
|
|
251
|
+
| undefined
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* npm provider packages whose wire format is OpenAI-compatible
|
|
255
|
+
* (`POST {base}/chat/completions` with `image_url` content parts) — the only
|
|
256
|
+
* shape `describeImagesNative` speaks. `@ai-sdk/openai-compatible` is what
|
|
257
|
+
* opencode's Zen providers use; `@ai-sdk/openai` is the same wire format.
|
|
258
|
+
* Packages like `@ai-sdk/anthropic` are NOT compatible and must never be
|
|
259
|
+
* picked by discovery (Q2 conservative allowlist).
|
|
260
|
+
*/
|
|
261
|
+
const NATIVE_VISION_NPM_ALLOWLIST: ReadonlySet<string> = new Set([
|
|
262
|
+
'@ai-sdk/openai-compatible',
|
|
263
|
+
'@ai-sdk/openai',
|
|
264
|
+
])
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Legacy Zen auth-store provider IDs. Zen endpoints ONLY — a non-Zen provider
|
|
268
|
+
* resolved through the npm allowlist must use its own auth-store entry, never
|
|
269
|
+
* these. The historical Zen order is kept as a fallback for Zen models.
|
|
270
|
+
*/
|
|
271
|
+
const LEGACY_VISION_AUTH_PROVIDER_IDS: readonly string[] = ['opencode-go', 'opencode']
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Whether `providerID` is an OpenCode-hosted (Zen) provider. Only Zen providers
|
|
275
|
+
* may use the shared credential sources — the `PANTHEON_OPENCODE_API_KEY` /
|
|
276
|
+
* `OPENCODE_API_KEY` env keys and the legacy `opencode-go` / `opencode` auth
|
|
277
|
+
* entries. A non-Zen provider (discovered or explicit) must resolve its OWN
|
|
278
|
+
* auth-store entry, else the native path returns null and the image falls
|
|
279
|
+
* through the tool→local chain.
|
|
280
|
+
*/
|
|
281
|
+
function isZenProvider(providerID: string): boolean {
|
|
282
|
+
return providerID === 'opencode' || providerID.startsWith('opencode-go')
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Provider prefix of a qualified `provider/model` id (the id itself if bare). */
|
|
286
|
+
function providerOf(modelID: string): string {
|
|
287
|
+
const slash = modelID.indexOf('/')
|
|
288
|
+
return slash === -1 ? modelID : modelID.slice(0, slash)
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Whether a resolved native target points at an OpenCode Zen endpoint. The
|
|
293
|
+
* gateway host is the security boundary for the `x-opencode-session` header
|
|
294
|
+
* (the header travels to `baseURL`, not to the model id): the session identity
|
|
295
|
+
* must reach `https://opencode.ai/zen/...` and NOTHING else. A target qualifies
|
|
296
|
+
* only when BOTH the model's provider is Zen AND the base URL is a Zen
|
|
297
|
+
* endpoint — a mismatch degrades to "no header" (the gateway then answers with
|
|
298
|
+
* its actionable HTTP 400, never a cross-provider leak).
|
|
299
|
+
*/
|
|
300
|
+
export function isZenNativeTarget(
|
|
301
|
+
target: Pick<NativeVisionTarget, 'modelID' | 'baseURL'>,
|
|
302
|
+
): boolean {
|
|
303
|
+
if (!isZenProvider(providerOf(target.modelID))) return false
|
|
304
|
+
return /^https:\/\/opencode\.ai\/zen\//i.test(target.baseURL)
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Build an actionable, non-sensitive message for a failed native describe.
|
|
309
|
+
* Maps the gateway status/`finish_reason` to the known Zen failures —
|
|
310
|
+
* 400 `MissingSessionID` (session header absent), 403 (free-tier model usable
|
|
311
|
+
* only from inside OpenCode, or the account has no paid access), 401
|
|
312
|
+
* (credential/model) — separates OUR budget expiring (`timeout`) from a genuine
|
|
313
|
+
* transport drop (`network`), and surfaces `finish_reason` when the model
|
|
314
|
+
* returned no text (e.g. `length` = truncated by `max_tokens`). An EXTERNAL
|
|
315
|
+
* cancellation (`cancelled`, e.g. a `ToolContext.signal` abort) gets a neutral
|
|
316
|
+
* message with no "retry" guidance — the abort was deliberate, not transient.
|
|
317
|
+
* Never includes the credential, image payload, or session id.
|
|
318
|
+
*/
|
|
319
|
+
export function nativeVisionFailureMessage(
|
|
320
|
+
failure: NativeVisionFailure | undefined,
|
|
321
|
+
modelID: string,
|
|
322
|
+
): string {
|
|
323
|
+
if (failure?.reason === 'timeout') {
|
|
324
|
+
const budget = failure.timeoutMs ? ` after ${Math.round(failure.timeoutMs / 1000)}s` : ''
|
|
325
|
+
return `the multimodal model request timed out${budget}: the image-description budget elapsed before a response — retry, or raise the timeout budget`
|
|
326
|
+
}
|
|
327
|
+
if (failure?.reason === 'cancelled') {
|
|
328
|
+
return 'the multimodal model request was cancelled before it completed'
|
|
329
|
+
}
|
|
330
|
+
if (failure?.reason === 'network') {
|
|
331
|
+
return 'the multimodal model request failed (network error): the connection was dropped before the description completed — retry the describe'
|
|
332
|
+
}
|
|
333
|
+
const status = failure?.reason === 'http' ? failure.status : undefined
|
|
334
|
+
if (status === 400) {
|
|
335
|
+
return 'the Zen gateway rejected the request (HTTP 400 MissingSessionID): the x-opencode-session header is missing — the request must carry a real OpenCode session id'
|
|
336
|
+
}
|
|
337
|
+
if (status === 401) {
|
|
338
|
+
return `the OpenCode Zen gateway rejected the credential (HTTP 401); verify the credential and that ${modelID} supports image input on its endpoint`
|
|
339
|
+
}
|
|
340
|
+
if (status === 403) {
|
|
341
|
+
return `the OpenCode Zen gateway denied access (HTTP 403) to ${modelID}: the model may be free-tier (usable only from inside OpenCode) or this account has no paid access to it`
|
|
342
|
+
}
|
|
343
|
+
if (status !== undefined) {
|
|
344
|
+
return `the Zen gateway rejected the request (HTTP ${status}) for ${modelID}`
|
|
345
|
+
}
|
|
346
|
+
if (failure?.reason === 'empty') {
|
|
347
|
+
const finish = failure.finishReason
|
|
348
|
+
if (finish === 'length') {
|
|
349
|
+
return 'the model returned no text (finish_reason: length) — the response was truncated by max_tokens; raise max_tokens or choose a non-reasoning image model'
|
|
350
|
+
}
|
|
351
|
+
if (finish) {
|
|
352
|
+
return `the model returned no text (finish_reason: ${finish})`
|
|
353
|
+
}
|
|
354
|
+
return 'the multimodal model returned no text'
|
|
355
|
+
}
|
|
356
|
+
return 'the multimodal model returned no text or the request failed'
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* How long a fetched providers catalog stays cached, per client (process).
|
|
361
|
+
* Short TTL: the catalog is one HTTP round-trip to the local opencode server
|
|
362
|
+
* and must NOT be refetched on every user turn, yet must pick up model
|
|
363
|
+
* changes within a session without a restart.
|
|
364
|
+
*/
|
|
365
|
+
export const PROVIDERS_CACHE_TTL_MS = 60_000
|
|
366
|
+
|
|
127
367
|
export type VisionMode = 'native' | 'tool' | 'auto'
|
|
128
368
|
|
|
129
369
|
/**
|
|
@@ -169,52 +409,362 @@ type SavedImage = {
|
|
|
169
409
|
|
|
170
410
|
type NativeImagePayload = { url: string }
|
|
171
411
|
|
|
412
|
+
// ─── Catalog parsing & discovery ───────────────────────────────────────────
|
|
413
|
+
|
|
414
|
+
function asRecord(value: unknown): Record<string, unknown> | null {
|
|
415
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
416
|
+
? (value as Record<string, unknown>)
|
|
417
|
+
: null
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
type CatalogEntry = {
|
|
421
|
+
/** Qualified id: `providerID/modelID`. */
|
|
422
|
+
modelID: string
|
|
423
|
+
providerID: string
|
|
424
|
+
image: boolean
|
|
425
|
+
url: string | null
|
|
426
|
+
npm: string | null
|
|
427
|
+
costInput: number | null
|
|
428
|
+
deprecated: boolean
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Parse one catalog model defensively; null when the entry has no usable id. */
|
|
432
|
+
function parseCatalogEntry(raw: Record<string, unknown>, providerID: string): CatalogEntry | null {
|
|
433
|
+
const id = typeof raw.id === 'string' && raw.id.trim() !== '' ? raw.id.trim() : null
|
|
434
|
+
if (!id) return null
|
|
435
|
+
const entryProvider =
|
|
436
|
+
typeof raw.providerID === 'string' && raw.providerID.trim() !== ''
|
|
437
|
+
? raw.providerID.trim()
|
|
438
|
+
: providerID
|
|
439
|
+
if (!entryProvider) return null
|
|
440
|
+
const api = asRecord(raw.api)
|
|
441
|
+
const url = typeof api?.url === 'string' && api.url.trim() !== '' ? api.url.trim() : null
|
|
442
|
+
const npm = typeof api?.npm === 'string' && api.npm.trim() !== '' ? api.npm.trim() : null
|
|
443
|
+
const input = asRecord(asRecord(raw.capabilities)?.input)
|
|
444
|
+
const cost = asRecord(raw.cost)
|
|
445
|
+
return {
|
|
446
|
+
modelID: `${entryProvider}/${id}`,
|
|
447
|
+
providerID: entryProvider,
|
|
448
|
+
image: input?.image === true,
|
|
449
|
+
url,
|
|
450
|
+
npm,
|
|
451
|
+
costInput: typeof cost?.input === 'number' && Number.isFinite(cost.input) ? cost.input : null,
|
|
452
|
+
deprecated: raw.status === 'deprecated',
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Flatten a catalog payload into entries. Accepts the canonical
|
|
458
|
+
* `{ providers: [{ id, models: { [id]: Model } }] }` shape and, defensively,
|
|
459
|
+
* a top-level `{ models: Model[] }` array.
|
|
460
|
+
*/
|
|
461
|
+
function catalogEntries(catalog: unknown): CatalogEntry[] {
|
|
462
|
+
const root = asRecord(catalog)
|
|
463
|
+
if (!root) return []
|
|
464
|
+
const entries: CatalogEntry[] = []
|
|
465
|
+
if (Array.isArray(root.providers)) {
|
|
466
|
+
for (const provider of root.providers) {
|
|
467
|
+
const p = asRecord(provider)
|
|
468
|
+
if (!p) continue
|
|
469
|
+
const pid = typeof p.id === 'string' ? p.id : ''
|
|
470
|
+
const models = asRecord(p.models)
|
|
471
|
+
if (!models) continue
|
|
472
|
+
for (const model of Object.values(models)) {
|
|
473
|
+
const m = asRecord(model)
|
|
474
|
+
if (!m) continue
|
|
475
|
+
const entry = parseCatalogEntry(m, pid)
|
|
476
|
+
if (entry) entries.push(entry)
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
if (Array.isArray(root.models)) {
|
|
481
|
+
for (const model of root.models) {
|
|
482
|
+
const m = asRecord(model)
|
|
483
|
+
if (!m) continue
|
|
484
|
+
const entry = parseCatalogEntry(m, '')
|
|
485
|
+
if (entry) entries.push(entry)
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
return entries
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* Whether the provider speaks an OpenAI-compatible wire format the native
|
|
493
|
+
* vision call can use: the known Zen providers, or an npm package on the
|
|
494
|
+
* conservative allowlist. Unknown/incompatible → false → callers fall back
|
|
495
|
+
* to the MCP tool pattern (Q2).
|
|
496
|
+
*/
|
|
497
|
+
function providerSupportsNativeVision(providerID: string, npm: string | null): boolean {
|
|
498
|
+
if (isZenProvider(providerID)) return true
|
|
499
|
+
return npm !== null && NATIVE_VISION_NPM_ALLOWLIST.has(npm)
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Whether a catalog entry is FREE-TIER. Free-tier Zen models are rejected by
|
|
504
|
+
* the direct gateway (HTTP 403 "free tier can only be used from within
|
|
505
|
+
* OpenCode"), so they must never be picked on the native describe path. Two
|
|
506
|
+
* conservative signals: an explicit `cost.input <= 0`, OR a `-free` id suffix.
|
|
507
|
+
* `<= 0` (not `=== 0`) matches V2's `isFreeVisionModel` so a malformed negative
|
|
508
|
+
* cost can never be treated as paid on one path and free on the other.
|
|
509
|
+
*/
|
|
510
|
+
function catalogEntryIsFree(entry: CatalogEntry): boolean {
|
|
511
|
+
return (entry.costInput !== null && entry.costInput <= 0) || /-free$/i.test(entry.modelID)
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Pick the vision model from a host catalog: image-capable, wire-compatible,
|
|
516
|
+
* not deprecated, and NOT free-tier (cheapest first, ties broken
|
|
517
|
+
* lexicographically for determinism). Free-tier models are excluded because the
|
|
518
|
+
* direct gateway cannot use them. Returns the qualified model ID and its
|
|
519
|
+
* catalog base URL, or null when nothing qualifies (callers then use
|
|
520
|
+
* DEFAULT_VISION_MODEL with the legacy Zen endpoint).
|
|
521
|
+
*/
|
|
522
|
+
export function discoverVisionModel(catalog: unknown): { modelID: string; baseURL: string } | null {
|
|
523
|
+
const candidates = catalogEntries(catalog).filter(
|
|
524
|
+
(entry) =>
|
|
525
|
+
entry.image &&
|
|
526
|
+
!entry.deprecated &&
|
|
527
|
+
!catalogEntryIsFree(entry) &&
|
|
528
|
+
providerSupportsNativeVision(entry.providerID, entry.npm) &&
|
|
529
|
+
entry.url !== null &&
|
|
530
|
+
/^https?:\/\//i.test(entry.url),
|
|
531
|
+
)
|
|
532
|
+
if (candidates.length === 0) return null
|
|
533
|
+
candidates.sort((left, right) => {
|
|
534
|
+
const price = (entry: CatalogEntry): number =>
|
|
535
|
+
entry.costInput === null ? Number.POSITIVE_INFINITY : entry.costInput
|
|
536
|
+
return price(left) - price(right) || left.modelID.localeCompare(right.modelID)
|
|
537
|
+
})
|
|
538
|
+
const pick = candidates[0]
|
|
539
|
+
return pick?.url ? { modelID: pick.modelID, baseURL: pick.url } : null
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Catalog base URL for an EXPLICITLY configured model, honoring the same
|
|
544
|
+
* compatibility gate. Returns null when the model is absent from the catalog
|
|
545
|
+
* or its entry is unusable — the caller then applies the legacy Zen mapping
|
|
546
|
+
* (the catalog never gates a Zen model that the old code already supported).
|
|
547
|
+
*/
|
|
548
|
+
function catalogEndpointFor(modelID: string, catalog: unknown): string | null {
|
|
549
|
+
const slash = modelID.indexOf('/')
|
|
550
|
+
if (slash < 1) return null
|
|
551
|
+
for (const entry of catalogEntries(catalog)) {
|
|
552
|
+
if (entry.modelID !== modelID) continue
|
|
553
|
+
if (!providerSupportsNativeVision(entry.providerID, entry.npm)) return null
|
|
554
|
+
return entry.url !== null && /^https?:\/\//i.test(entry.url) ? entry.url : null
|
|
555
|
+
}
|
|
556
|
+
return null
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Historical hardcoded Zen endpoint mapping (opencode-go* → zen/go/v1,
|
|
561
|
+
* opencode → zen/v1, else null). Fallback only — used when there is no
|
|
562
|
+
* catalog (or the model is not in it) so existing setups keep working.
|
|
563
|
+
*/
|
|
564
|
+
function legacyZenBaseURL(modelID: string): string | null {
|
|
565
|
+
const providerID = modelID.slice(0, modelID.indexOf('/'))
|
|
566
|
+
if (providerID.startsWith('opencode-go')) return ZEN_GO_BASE_URL
|
|
567
|
+
if (providerID === 'opencode') return ZEN_BASE_URL
|
|
568
|
+
return null
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const providersCatalogCache = new WeakMap<object, { payload: unknown; at: number }>()
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Fetch (and TTL-cache per client) the host provider catalog via
|
|
575
|
+
* `client.config.providers()` (GET /config/providers). Never throws: any
|
|
576
|
+
* failure — client absent, HTTP error, malformed payload — yields null, and
|
|
577
|
+
* the null itself is cached so a broken server is not hammered per request.
|
|
578
|
+
* Discovery talks ONLY to the local opencode server — never via bifrost.
|
|
579
|
+
*/
|
|
580
|
+
export async function fetchProvidersCatalog(
|
|
581
|
+
client: PluginInput['client'] | undefined | null,
|
|
582
|
+
): Promise<unknown> {
|
|
583
|
+
if (!client || typeof client !== 'object') return null
|
|
584
|
+
const cached = providersCatalogCache.get(client)
|
|
585
|
+
const now = Date.now()
|
|
586
|
+
if (cached && now - cached.at < PROVIDERS_CACHE_TTL_MS) return cached.payload
|
|
587
|
+
let payload: unknown = null
|
|
588
|
+
try {
|
|
589
|
+
const providers = asRecord(client)?.config as
|
|
590
|
+
| { providers?: (options?: unknown) => Promise<unknown> }
|
|
591
|
+
| undefined
|
|
592
|
+
if (typeof providers?.providers === 'function') {
|
|
593
|
+
const response = await providers.providers()
|
|
594
|
+
payload = asRecord(response)?.data ?? null
|
|
595
|
+
}
|
|
596
|
+
} catch {
|
|
597
|
+
payload = null
|
|
598
|
+
}
|
|
599
|
+
providersCatalogCache.set(client, { payload, at: now })
|
|
600
|
+
return payload
|
|
601
|
+
}
|
|
602
|
+
|
|
172
603
|
// ─── Config loading (project > user) ──────────────────────────────────────
|
|
173
604
|
|
|
174
605
|
// ─── OpenCode auth store (fallback credential source) ────────────────────
|
|
175
|
-
// `opencode auth login` stores provider tokens in <data dir>/auth.json
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
//
|
|
606
|
+
// `opencode auth login` stores provider tokens in <data dir>/auth.json on V1
|
|
607
|
+
// and in a `credential` table inside <data dir>/opencode.db on V2. The vision
|
|
608
|
+
// plugin reads BOTH stores so a connected user needs no exported env var, and
|
|
609
|
+
// so every vision path (plugin native, MCP, picker) resolves the same
|
|
610
|
+
// credential. Mirrors the usage-bar pattern (src/plugins/tui) but stays local
|
|
611
|
+
// to this module — no TUI imports.
|
|
612
|
+
|
|
613
|
+
function defaultAuthStateDir(): string {
|
|
614
|
+
// Honor XDG_DATA_HOME so a relocated data root resolves the same auth store
|
|
615
|
+
// (auth.json) and V2 credential db (opencode.db) as the plugin and the MCP.
|
|
616
|
+
// Mirrors v2-vision.ts; OPENCODE_DB still names the exact db file.
|
|
617
|
+
const dataHome = process.env.XDG_DATA_HOME || join(homedir(), '.local', 'share')
|
|
618
|
+
return join(dataHome, 'opencode')
|
|
619
|
+
}
|
|
179
620
|
|
|
180
|
-
|
|
181
|
-
|
|
621
|
+
/**
|
|
622
|
+
* Path to the OpenCode V2 credential database. `OPENCODE_DB` names the exact
|
|
623
|
+
* file when the host relocated it (e.g. a sandbox with its own db); otherwise
|
|
624
|
+
* the default `<data dir>/opencode.db` sibling of `auth.json` is used.
|
|
625
|
+
*/
|
|
626
|
+
function v2CredentialDbPath(stateDir: string, env: Record<string, string | undefined>): string {
|
|
627
|
+
const override = env.OPENCODE_DB?.trim()
|
|
628
|
+
return override ? override : join(stateDir, 'opencode.db')
|
|
182
629
|
}
|
|
183
630
|
|
|
184
631
|
/**
|
|
185
|
-
*
|
|
186
|
-
* `
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
* when the file is missing/corrupted or no entry has a usable token — callers
|
|
190
|
-
* fall back to the MCP tool pattern. The token is never logged.
|
|
632
|
+
* Credential field names accepted in an OpenCode auth entry. V1 OAuth entries
|
|
633
|
+
* carry `access`; V1/V2 API-key entries carry `key`/`apiKey`. Mirrors the
|
|
634
|
+
* Pantheon Vision MCP's `_extract_key` so the native path sees the same
|
|
635
|
+
* credential shapes as the tool path.
|
|
191
636
|
*/
|
|
192
|
-
|
|
637
|
+
const AUTH_TOKEN_FIELDS: readonly string[] = [
|
|
638
|
+
'access',
|
|
639
|
+
'key',
|
|
640
|
+
'apiKey',
|
|
641
|
+
'api_key',
|
|
642
|
+
'token',
|
|
643
|
+
'accessToken',
|
|
644
|
+
'access_token',
|
|
645
|
+
]
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Extract a usable token from an OpenCode credential value (a raw string or a
|
|
649
|
+
* JSON object), skipping an expired OAuth entry. Returns null when no field
|
|
650
|
+
* carries a non-empty token. The token is never logged.
|
|
651
|
+
*/
|
|
652
|
+
function extractAuthToken(value: unknown): string | null {
|
|
653
|
+
if (typeof value === 'string') return value.trim() === '' ? null : value
|
|
654
|
+
if (!value || typeof value !== 'object') return null
|
|
655
|
+
const record = value as Record<string, unknown>
|
|
656
|
+
const expires = record.expires
|
|
657
|
+
if (typeof expires === 'number' && expires > 0 && Date.now() >= expires) return null
|
|
658
|
+
for (const field of AUTH_TOKEN_FIELDS) {
|
|
659
|
+
const candidate = record[field]
|
|
660
|
+
if (typeof candidate === 'string' && candidate.trim() !== '') return candidate
|
|
661
|
+
}
|
|
662
|
+
return null
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* Extract a token from a V2 `credential.value` cell: the cell is JSON text
|
|
667
|
+
* (e.g. `{"type":"key","key":"..."}` or an OAuth blob), so it is parsed first.
|
|
668
|
+
* A non-JSON or unusable value yields null. The token is never logged.
|
|
669
|
+
*/
|
|
670
|
+
function extractCredentialCellToken(value: unknown): string | null {
|
|
671
|
+
if (typeof value !== 'string') return null
|
|
672
|
+
try {
|
|
673
|
+
return extractAuthToken(JSON.parse(value))
|
|
674
|
+
} catch {
|
|
675
|
+
return null // not JSON → not a usable credential cell
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Read a provider token from the OpenCode V2 SQLite credential store
|
|
681
|
+
* (`<data dir>/opencode.db`, table `credential`). OpenCode V2 moved credentials
|
|
682
|
+
* out of `auth.json`; a V2 host has no JSON file, so an auth.json-only read left
|
|
683
|
+
* the native path unresolvable even though the MCP (which reads this table) and
|
|
684
|
+
* the picker both worked.
|
|
685
|
+
*
|
|
686
|
+
* Best-effort and READ-ONLY (`mode=ro`): a missing `node:sqlite` builtin, a
|
|
687
|
+
* missing/locked database, or a missing row yields null and the caller falls
|
|
688
|
+
* back to the tool→local chain. Rows are tried in `providerIDs` order; the
|
|
689
|
+
* stored value is never logged.
|
|
690
|
+
*/
|
|
691
|
+
async function readV2CredentialToken(
|
|
193
692
|
providerIDs: readonly string[],
|
|
194
|
-
|
|
693
|
+
stateDir: string,
|
|
694
|
+
env: Record<string, string | undefined>,
|
|
195
695
|
): Promise<string | null> {
|
|
196
|
-
const
|
|
197
|
-
|
|
696
|
+
const dbPath = v2CredentialDbPath(stateDir, env)
|
|
697
|
+
if (!existsSync(dbPath)) return null
|
|
698
|
+
let sqlite: typeof import('node:sqlite') | undefined
|
|
198
699
|
try {
|
|
199
|
-
|
|
700
|
+
sqlite = await import('node:sqlite')
|
|
200
701
|
} catch {
|
|
201
|
-
return null //
|
|
702
|
+
return null // runtime without node:sqlite → no V2 store to read
|
|
202
703
|
}
|
|
203
|
-
|
|
704
|
+
const DatabaseSync = sqlite.DatabaseSync
|
|
705
|
+
if (typeof DatabaseSync !== 'function') return null
|
|
706
|
+
let db: import('node:sqlite').DatabaseSync | undefined
|
|
204
707
|
try {
|
|
205
|
-
|
|
708
|
+
db = new DatabaseSync(`file:${dbPath}?mode=ro`, { readOnly: true })
|
|
709
|
+
const statement = db.prepare(
|
|
710
|
+
'SELECT value FROM credential WHERE integration_id = ? AND active = 1 LIMIT 1',
|
|
711
|
+
)
|
|
712
|
+
for (const id of providerIDs) {
|
|
713
|
+
const row = statement.get(id) as { value?: unknown } | undefined
|
|
714
|
+
const token = extractCredentialCellToken(row?.value)
|
|
715
|
+
if (token) return token
|
|
716
|
+
}
|
|
717
|
+
return null
|
|
206
718
|
} catch {
|
|
207
|
-
return null //
|
|
719
|
+
return null // missing table / locked / corrupt → no crash
|
|
720
|
+
} finally {
|
|
721
|
+
try {
|
|
722
|
+
db?.close()
|
|
723
|
+
} catch {
|
|
724
|
+
// best-effort close
|
|
725
|
+
}
|
|
208
726
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/**
|
|
730
|
+
* Read a provider token from opencode's credential stores.
|
|
731
|
+
*
|
|
732
|
+
* `statePath` is the opencode data dir (default `$XDG_DATA_HOME/opencode`, else
|
|
733
|
+
* `~/.local/share/opencode`). The
|
|
734
|
+
* V1 `auth.json` inside it maps provider IDs to `{ access | key, expires }`;
|
|
735
|
+
* the V2 `opencode.db` (or the file named by `OPENCODE_DB`) holds the same
|
|
736
|
+
* secrets in a `credential` table. The V1 file is read first (it is absent on a
|
|
737
|
+
* V2 host), then the V2 store.
|
|
738
|
+
*
|
|
739
|
+
* Returns the first non-expired token across `providerIDs`, or null when both
|
|
740
|
+
* stores are missing/corrupted or no entry has a usable token — callers fall
|
|
741
|
+
* back to the MCP tool pattern. The token is never logged.
|
|
742
|
+
*/
|
|
743
|
+
export async function readOpencodeAuthToken(
|
|
744
|
+
providerIDs: readonly string[],
|
|
745
|
+
statePath?: string,
|
|
746
|
+
env: Record<string, string | undefined> = process.env,
|
|
747
|
+
): Promise<string | null> {
|
|
748
|
+
const stateDir = statePath ?? defaultAuthStateDir()
|
|
749
|
+
const raw = await readFile(join(stateDir, 'auth.json'), 'utf8').catch(() => null)
|
|
750
|
+
if (raw !== null) {
|
|
751
|
+
let auth: Record<string, unknown> | null = null
|
|
752
|
+
try {
|
|
753
|
+
const parsed = JSON.parse(raw) as unknown
|
|
754
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
755
|
+
auth = parsed as Record<string, unknown>
|
|
756
|
+
}
|
|
757
|
+
} catch {
|
|
758
|
+
auth = null // corrupted → treat as absent, then try the V2 store
|
|
759
|
+
}
|
|
760
|
+
if (auth) {
|
|
761
|
+
for (const id of providerIDs) {
|
|
762
|
+
const token = extractAuthToken(auth[id])
|
|
763
|
+
if (token) return token
|
|
764
|
+
}
|
|
765
|
+
}
|
|
216
766
|
}
|
|
217
|
-
return
|
|
767
|
+
return readV2CredentialToken(providerIDs, stateDir, env)
|
|
218
768
|
}
|
|
219
769
|
|
|
220
770
|
function userConfigPath(): string {
|
|
@@ -439,27 +989,106 @@ export async function enforceTempFileCap(
|
|
|
439
989
|
return evicted
|
|
440
990
|
}
|
|
441
991
|
|
|
442
|
-
|
|
443
|
-
|
|
992
|
+
/**
|
|
993
|
+
* Decode a base64 image data URI to its bytes. The regex accepts OPTIONAL
|
|
994
|
+
* parameters between the mime type and the `;base64` marker (e.g.
|
|
995
|
+
* `data:image/png;charset=utf-8;base64,…`), so a URI a browser or clipboard
|
|
996
|
+
* decorates with parameters still decodes. The declared mime and its parameters
|
|
997
|
+
* are ignored: callers pass the authoritative mime separately (an explicit
|
|
998
|
+
* filename/mime wins over the data URI's own declaration). Never throws;
|
|
999
|
+
* returns null for malformed input or an empty payload.
|
|
1000
|
+
*/
|
|
1001
|
+
function decodeDataUrlImage(dataUrl: string): Buffer | null {
|
|
1002
|
+
const match = /^data:([^;,]+)(?:;[^;,]*)*;base64,(.+)$/s.exec(dataUrl)
|
|
444
1003
|
if (!match) return null
|
|
445
1004
|
const data = Buffer.from(match[2] ?? '', 'base64')
|
|
446
1005
|
if (data.length === 0) return null
|
|
1006
|
+
return data
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
/**
|
|
1010
|
+
* Atomically write `data` to `filePath` via a sibling temp file + rename.
|
|
1011
|
+
*
|
|
1012
|
+
* The temp file lives in the SAME directory, so `rename` is an atomic
|
|
1013
|
+
* same-filesystem replace that overwrites any existing destination. A crash
|
|
1014
|
+
* mid-write can therefore leave at most a stray `<name>.tmp-<pid>` file —
|
|
1015
|
+
* never a truncated destination that a later content-hash dedup would trust.
|
|
1016
|
+
* Private mode 0600 is forced AFTER creation to beat a permissive umask.
|
|
1017
|
+
*/
|
|
1018
|
+
function writeFileAtomicSync(filePath: string, data: Buffer): void {
|
|
1019
|
+
const tempPath = `${filePath}.tmp-${process.pid}`
|
|
1020
|
+
try {
|
|
1021
|
+
writeFileSync(tempPath, data, { mode: 0o600 })
|
|
1022
|
+
chmodSync(tempPath, 0o600)
|
|
1023
|
+
renameSync(tempPath, filePath)
|
|
1024
|
+
} catch (err) {
|
|
1025
|
+
try {
|
|
1026
|
+
unlinkSync(tempPath)
|
|
1027
|
+
} catch {
|
|
1028
|
+
// The temp file may not exist yet (the write failed first) — best effort.
|
|
1029
|
+
}
|
|
1030
|
+
throw err
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/** Whether `filePath` already holds exactly `data` (dedup fast path). */
|
|
1035
|
+
function fileHasContent(filePath: string, data: Buffer): boolean {
|
|
1036
|
+
try {
|
|
1037
|
+
return readFileSync(filePath).equals(data)
|
|
1038
|
+
} catch {
|
|
1039
|
+
return false
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/**
|
|
1044
|
+
* Write decoded image bytes to the shared temp dir under a content-hash
|
|
1045
|
+
* filename (dedup: identical re-pasted images reuse one file), with private
|
|
1046
|
+
* modes forced AFTER creation. The write is ATOMIC (sibling temp + rename) so
|
|
1047
|
+
* an interrupted process can never leave a truncated file that a later dedup
|
|
1048
|
+
* would reuse as valid. Registers the path as most-recently-used in the LRU
|
|
1049
|
+
* registry. Screenshots may hold sensitive content, so modes are 0700/0600
|
|
1050
|
+
* regardless of a permissive umask; the bytes are never logged.
|
|
1051
|
+
*/
|
|
1052
|
+
export function writeTempImageSync(data: Buffer, mime: string): string {
|
|
447
1053
|
const dir = join(tmpdir(), TEMP_DIR_NAME)
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
// AFTER creation to guarantee 0700/0600 regardless of the environment.
|
|
451
|
-
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
452
|
-
await chmod(dir, 0o700)
|
|
453
|
-
// Content-hash filename: identical re-pasted images reuse the same file
|
|
454
|
-
// (dedup — less disk, and it stabilizes the per-image description cache).
|
|
1054
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
1055
|
+
chmodSync(dir, 0o700)
|
|
455
1056
|
const filePath = join(dir, `${hashContent(data)}.${getExtensionForMime(mime)}`)
|
|
456
|
-
|
|
457
|
-
|
|
1057
|
+
// Dedup: a content-hash name whose bytes already match is the object every
|
|
1058
|
+
// other session expects — skip the rewrite. Different bytes under the same
|
|
1059
|
+
// name (or an unreadable/truncated leftover) are healed by the atomic write.
|
|
1060
|
+
if (!fileHasContent(filePath, data)) writeFileAtomicSync(filePath, data)
|
|
1061
|
+
chmodSync(filePath, 0o600)
|
|
458
1062
|
touchTempFile(filePath)
|
|
1063
|
+
return filePath
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
/** Async temp materialization used by the V1 chat hook (awaits the LRU cap). */
|
|
1067
|
+
export async function saveDataUrlImage(dataUrl: string, mime: string): Promise<string | null> {
|
|
1068
|
+
const data = decodeDataUrlImage(dataUrl)
|
|
1069
|
+
if (!data) return null
|
|
1070
|
+
const filePath = writeTempImageSync(data, mime)
|
|
459
1071
|
await enforceTempFileCap()
|
|
460
1072
|
return filePath
|
|
461
1073
|
}
|
|
462
1074
|
|
|
1075
|
+
/**
|
|
1076
|
+
* Synchronous sibling of `saveDataUrlImage` for callers that run inside a
|
|
1077
|
+
* synchronous host hook — namely the V2 `prompt` admission, which must hand a
|
|
1078
|
+
* path-based vision tool/MCP a real reference for an inline (clipboard)
|
|
1079
|
+
* attachment whose bytes exist only in memory as a data URI. Bytes go to the
|
|
1080
|
+
* SAME temp dir + LRU registry, so V1 and V2 attachment cleanup share one
|
|
1081
|
+
* mechanism. The global cap is enforced best-effort (a sync caller cannot await
|
|
1082
|
+
* it): the eviction runs off the returned tick and never rejects.
|
|
1083
|
+
*/
|
|
1084
|
+
export function saveDataUrlImageSync(dataUrl: string, mime: string): string | null {
|
|
1085
|
+
const data = decodeDataUrlImage(dataUrl)
|
|
1086
|
+
if (!data) return null
|
|
1087
|
+
const filePath = writeTempImageSync(data, mime)
|
|
1088
|
+
void enforceTempFileCap().catch(() => undefined)
|
|
1089
|
+
return filePath
|
|
1090
|
+
}
|
|
1091
|
+
|
|
463
1092
|
async function resolveImageTarget(filePart: FilePart): Promise<SavedImage | null> {
|
|
464
1093
|
const url = filePart.url
|
|
465
1094
|
if (!url) return null
|
|
@@ -497,60 +1126,93 @@ function firstNonEmpty(fallback: string, ...values: Array<string | undefined>):
|
|
|
497
1126
|
}
|
|
498
1127
|
|
|
499
1128
|
/**
|
|
500
|
-
* Resolve the native vision endpoint: model ID +
|
|
1129
|
+
* Resolve the native vision endpoint: model ID + base URL (no key I/O).
|
|
501
1130
|
*
|
|
502
1131
|
* Model precedence: env PANTHEON_VISION_MODEL > config file `visionModel` >
|
|
503
|
-
* active preset `vision.model` >
|
|
504
|
-
* model
|
|
1132
|
+
* active preset `vision.model` > catalog discovery > DEFAULT_VISION_MODEL.
|
|
1133
|
+
* When an explicit model is set, its base URL comes from the catalog entry
|
|
1134
|
+
* (`api.url`) when present, else the legacy Zen mapping. Discovery (no
|
|
1135
|
+
* explicit model) only picks wire-compatible image-capable models and uses
|
|
1136
|
+
* their `api.url`; with no qualifying entry it falls back to
|
|
1137
|
+
* DEFAULT_VISION_MODEL + Zen. Returns null when the model's provider has no
|
|
1138
|
+
* usable endpoint (unknown providers without a catalog — anthropic/openai
|
|
1139
|
+
* stay phase 2).
|
|
505
1140
|
*/
|
|
506
1141
|
function resolveNativeVisionEndpoint(
|
|
507
1142
|
preset: Pick<ResolvedPreset, 'vision'> | null | undefined,
|
|
508
1143
|
env: Record<string, string | undefined>,
|
|
509
1144
|
configVisionModel?: string,
|
|
1145
|
+
catalog?: unknown,
|
|
510
1146
|
): NativeVisionEndpoint | null {
|
|
511
1147
|
const resolved = preset ?? resolveActivePreset({ env, candidates: activePresetCandidates() })
|
|
512
|
-
|
|
513
|
-
|
|
1148
|
+
// Explicit model wins over discovery; empty string = nothing configured.
|
|
1149
|
+
const explicitModel = firstNonEmpty(
|
|
1150
|
+
'',
|
|
514
1151
|
env.PANTHEON_VISION_MODEL,
|
|
515
1152
|
configVisionModel,
|
|
516
1153
|
resolved?.vision?.model,
|
|
517
1154
|
)
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
1155
|
+
if (explicitModel) {
|
|
1156
|
+
const baseURL = catalogEndpointFor(explicitModel, catalog) ?? legacyZenBaseURL(explicitModel)
|
|
1157
|
+
if (!baseURL) return null
|
|
1158
|
+
return { modelID: explicitModel, baseURL }
|
|
1159
|
+
}
|
|
1160
|
+
const discovered = discoverVisionModel(catalog)
|
|
1161
|
+
if (discovered) return discovered
|
|
1162
|
+
const modelID = DEFAULT_VISION_MODEL
|
|
1163
|
+
const baseURL = legacyZenBaseURL(modelID)
|
|
522
1164
|
if (!baseURL) return null
|
|
523
1165
|
return { modelID, baseURL }
|
|
524
1166
|
}
|
|
525
1167
|
|
|
526
1168
|
/**
|
|
527
|
-
* Resolve the native vision target: model +
|
|
1169
|
+
* Resolve the native vision target: model + endpoint + API key.
|
|
528
1170
|
*
|
|
529
|
-
*
|
|
530
|
-
*
|
|
531
|
-
*
|
|
532
|
-
*
|
|
1171
|
+
* The key comes from the shared Zen env keys only (`PANTHEON_OPENCODE_API_KEY`
|
|
1172
|
+
* then `OPENCODE_API_KEY`), and only when the resolved endpoint is Zen. `catalog`
|
|
1173
|
+
* (optional, shape of GET /config/providers) enables discovery when no explicit
|
|
1174
|
+
* model is configured (env > config > preset > discovery > DEFAULT). Returns
|
|
1175
|
+
* null when the model has no usable endpoint, is non-Zen, or no env key is set —
|
|
1176
|
+
* callers fall back to the MCP tool. For the auth-store fallback (`opencode auth
|
|
1177
|
+
* login`) use `resolveNativeVisionTarget`.
|
|
533
1178
|
*/
|
|
534
1179
|
export function resolveNativeVisionConfig(
|
|
535
1180
|
preset: Pick<ResolvedPreset, 'vision'> | null | undefined = undefined,
|
|
536
1181
|
env: Record<string, string | undefined> = process.env,
|
|
537
1182
|
configVisionModel?: string,
|
|
1183
|
+
catalog?: unknown,
|
|
538
1184
|
): NativeVisionTarget | null {
|
|
539
|
-
const endpoint = resolveNativeVisionEndpoint(preset, env, configVisionModel)
|
|
1185
|
+
const endpoint = resolveNativeVisionEndpoint(preset, env, configVisionModel, catalog)
|
|
540
1186
|
if (!endpoint) return null
|
|
1187
|
+
// Non-Zen endpoints never receive the shared Zen env keys (cross-provider
|
|
1188
|
+
// credential leak); they must resolve their own auth store via the async path.
|
|
1189
|
+
const providerID = endpoint.modelID.slice(0, endpoint.modelID.indexOf('/'))
|
|
1190
|
+
if (!isZenProvider(providerID)) return null
|
|
541
1191
|
const apiKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
|
|
542
1192
|
if (!apiKey || apiKey.trim() === '') return null
|
|
543
1193
|
return { ...endpoint, apiKey }
|
|
544
1194
|
}
|
|
545
1195
|
|
|
546
1196
|
/**
|
|
547
|
-
* Full native-vision
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
*
|
|
553
|
-
*
|
|
1197
|
+
* Full native-vision resolution for the chat hook (async): model + endpoint
|
|
1198
|
+
* via `resolveNativeVisionConfig` (env > config > preset > catalog discovery
|
|
1199
|
+
* > DEFAULT), then the API key.
|
|
1200
|
+
*
|
|
1201
|
+
* Credential precedence:
|
|
1202
|
+
* - Zen endpoints (`opencode` / `opencode-go*`): the env keys
|
|
1203
|
+
* (`PANTHEON_OPENCODE_API_KEY` ?? `OPENCODE_API_KEY`) first, then the auth
|
|
1204
|
+
* store (`auth.json` on V1, the V2 `opencode.db` credential table) — the
|
|
1205
|
+
* resolved provider entry first, then the legacy Zen entries (`opencode-go`,
|
|
1206
|
+
* then `opencode`) — so connected users don't need to export anything.
|
|
1207
|
+
* - Non-Zen endpoints (discovered or explicit via the npm allowlist): ONLY the
|
|
1208
|
+
* resolved provider's own auth-store entry. The shared env keys and the legacy
|
|
1209
|
+
* Zen entries are never attached to a non-Zen base URL; absent an own
|
|
1210
|
+
* credential this returns null and the caller falls back to the tool→local
|
|
1211
|
+
* chain (no cross-provider credential leak).
|
|
1212
|
+
*
|
|
1213
|
+
* `catalog` enables host-catalog discovery; without it the behavior is exactly
|
|
1214
|
+
* the pre-discovery one. Returns null when the model has no usable endpoint or
|
|
1215
|
+
* no usable key exists — callers fall back to the MCP tool.
|
|
554
1216
|
*/
|
|
555
1217
|
export async function resolveNativeVisionTarget(
|
|
556
1218
|
input: {
|
|
@@ -558,14 +1220,28 @@ export async function resolveNativeVisionTarget(
|
|
|
558
1220
|
env?: Record<string, string | undefined>
|
|
559
1221
|
config?: VisionConfig | null
|
|
560
1222
|
authStatePath?: string
|
|
1223
|
+
catalog?: unknown
|
|
561
1224
|
} = {},
|
|
562
1225
|
): Promise<NativeVisionTarget | null> {
|
|
563
1226
|
const env = input.env ?? process.env
|
|
564
|
-
const endpoint = resolveNativeVisionEndpoint(
|
|
1227
|
+
const endpoint = resolveNativeVisionEndpoint(
|
|
1228
|
+
input.preset,
|
|
1229
|
+
env,
|
|
1230
|
+
input.config?.visionModel,
|
|
1231
|
+
input.catalog,
|
|
1232
|
+
)
|
|
565
1233
|
if (!endpoint) return null
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
|
|
1234
|
+
const resolvedProvider = endpoint.modelID.slice(0, endpoint.modelID.indexOf('/'))
|
|
1235
|
+
const zen = isZenProvider(resolvedProvider)
|
|
1236
|
+
if (zen) {
|
|
1237
|
+
const envKey = env.PANTHEON_OPENCODE_API_KEY ?? env.OPENCODE_API_KEY
|
|
1238
|
+
if (envKey && envKey.trim() !== '') return { ...endpoint, apiKey: envKey }
|
|
1239
|
+
}
|
|
1240
|
+
// Own provider entry always; the legacy Zen entries ONLY for a Zen endpoint.
|
|
1241
|
+
const authProviderIDs = [
|
|
1242
|
+
...new Set(zen ? [resolvedProvider, ...LEGACY_VISION_AUTH_PROVIDER_IDS] : [resolvedProvider]),
|
|
1243
|
+
].filter((id) => Boolean(id))
|
|
1244
|
+
const authToken = await readOpencodeAuthToken(authProviderIDs, input.authStatePath, env)
|
|
569
1245
|
if (!authToken) return null
|
|
570
1246
|
return { ...endpoint, apiKey: authToken }
|
|
571
1247
|
}
|
|
@@ -644,6 +1320,15 @@ export function stripProviderPrefix(modelID: string): string {
|
|
|
644
1320
|
* `prompt` overrides the default describe prompt (e.g. OCR extraction);
|
|
645
1321
|
* `maxTokens` caps the completion (OCR needs more headroom); `signal` links
|
|
646
1322
|
* an external abort (e.g. the plugin-tool ToolContext.abort) to the request.
|
|
1323
|
+
*
|
|
1324
|
+
* When `target.sessionID` is set AND the target is Zen (`isZenNativeTarget`),
|
|
1325
|
+
* the request carries the `x-opencode-session` header — the Zen gateway rejects
|
|
1326
|
+
* a sessionless request with HTTP 400 `MissingSessionID`. The header is NEVER
|
|
1327
|
+
* sent to a non-Zen provider (no cross-provider session leak).
|
|
1328
|
+
*
|
|
1329
|
+
* The abort budget is DYNAMIC (see `computeNativeTimeoutMs`): it scales with
|
|
1330
|
+
* the total decoded size of `payloads` so large images are not dropped as
|
|
1331
|
+
* timeouts by the fixed base budget.
|
|
647
1332
|
*/
|
|
648
1333
|
export async function describeImagesNative(
|
|
649
1334
|
payloads: readonly NativeImagePayload[],
|
|
@@ -651,21 +1336,54 @@ export async function describeImagesNative(
|
|
|
651
1336
|
prompt: string = NATIVE_PROMPT,
|
|
652
1337
|
maxTokens: number = NATIVE_MAX_TOKENS,
|
|
653
1338
|
signal?: AbortSignal,
|
|
1339
|
+
onFailure?: (failure: NativeVisionFailure) => void,
|
|
654
1340
|
): Promise<string | null> {
|
|
1341
|
+
const report = (failure: NativeVisionFailure): void => {
|
|
1342
|
+
try {
|
|
1343
|
+
onFailure?.(failure)
|
|
1344
|
+
} catch {
|
|
1345
|
+
// A diagnostic sink must never break the describe result.
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const headers: Record<string, string> = {
|
|
1349
|
+
'Content-Type': 'application/json',
|
|
1350
|
+
Authorization: `Bearer ${target.apiKey}`,
|
|
1351
|
+
}
|
|
1352
|
+
// Zen-only session header: the gateway requires it for `opencode-go*` and
|
|
1353
|
+
// `opencode` models, and a non-Zen host must never see the session identity.
|
|
1354
|
+
if (target.sessionID && isZenNativeTarget(target)) {
|
|
1355
|
+
headers['x-opencode-session'] = target.sessionID
|
|
1356
|
+
}
|
|
655
1357
|
const controller = new AbortController()
|
|
656
|
-
const
|
|
657
|
-
|
|
1358
|
+
const budgetMs = computeNativeTimeoutMs(estimateNativePayloadsBytes(payloads))
|
|
1359
|
+
// Track WHICH abort fired: our budget expiring (a timeout the caller can act
|
|
1360
|
+
// on), an EXTERNAL cancellation (a deliberate `ToolContext.signal` abort), or
|
|
1361
|
+
// a genuine transport drop. Collapsing all three into `network` is what made
|
|
1362
|
+
// a timed-out reasoning call read as the vague "network error" the agent then
|
|
1363
|
+
// paraphrased — and is what made an external cancellation advertise a bogus
|
|
1364
|
+
// "retry". External cancellation wins: it is deliberate, never transient.
|
|
1365
|
+
let timedOut = false
|
|
1366
|
+
let cancelled = false
|
|
1367
|
+
const timer = setTimeout(() => {
|
|
1368
|
+
timedOut = true
|
|
1369
|
+
controller.abort()
|
|
1370
|
+
}, budgetMs)
|
|
1371
|
+
const onExternalAbort = () => {
|
|
1372
|
+
cancelled = true
|
|
1373
|
+
controller.abort()
|
|
1374
|
+
}
|
|
658
1375
|
if (signal) {
|
|
659
|
-
if (signal.aborted)
|
|
660
|
-
|
|
1376
|
+
if (signal.aborted) {
|
|
1377
|
+
cancelled = true
|
|
1378
|
+
controller.abort()
|
|
1379
|
+
} else {
|
|
1380
|
+
signal.addEventListener('abort', onExternalAbort, { once: true })
|
|
1381
|
+
}
|
|
661
1382
|
}
|
|
662
1383
|
try {
|
|
663
1384
|
const response = await fetch(`${target.baseURL}/chat/completions`, {
|
|
664
1385
|
method: 'POST',
|
|
665
|
-
headers
|
|
666
|
-
'Content-Type': 'application/json',
|
|
667
|
-
Authorization: `Bearer ${target.apiKey}`,
|
|
668
|
-
},
|
|
1386
|
+
headers,
|
|
669
1387
|
body: JSON.stringify({
|
|
670
1388
|
model: stripProviderPrefix(target.modelID),
|
|
671
1389
|
messages: [
|
|
@@ -684,14 +1402,38 @@ export async function describeImagesNative(
|
|
|
684
1402
|
}),
|
|
685
1403
|
signal: controller.signal,
|
|
686
1404
|
})
|
|
687
|
-
if (!response.ok)
|
|
1405
|
+
if (!response.ok) {
|
|
1406
|
+
report({ reason: 'http', status: response.status })
|
|
1407
|
+
return null
|
|
1408
|
+
}
|
|
688
1409
|
const data = (await response.json()) as {
|
|
689
|
-
choices?: Array<{ message?: { content?: string } }>
|
|
1410
|
+
choices?: Array<{ message?: { content?: string }; finish_reason?: string }>
|
|
1411
|
+
}
|
|
1412
|
+
const choice = data.choices?.[0]
|
|
1413
|
+
const content = choice?.message?.content
|
|
1414
|
+
if (typeof content !== 'string' || content.trim() === '') {
|
|
1415
|
+
// Surface finish_reason (e.g. `length`) so a reasoning model cut off by
|
|
1416
|
+
// max_tokens yields an actionable message instead of a bare "no text".
|
|
1417
|
+
report({
|
|
1418
|
+
reason: 'empty',
|
|
1419
|
+
...(typeof choice?.finish_reason === 'string'
|
|
1420
|
+
? { finishReason: choice.finish_reason }
|
|
1421
|
+
: {}),
|
|
1422
|
+
})
|
|
1423
|
+
return null
|
|
690
1424
|
}
|
|
691
|
-
const content = data.choices?.[0]?.message?.content
|
|
692
|
-
if (typeof content !== 'string' || content.trim() === '') return null
|
|
693
1425
|
return content.trim()
|
|
694
1426
|
} catch {
|
|
1427
|
+
// Our dynamic budget elapsed → `timeout` (with the budget for the message);
|
|
1428
|
+
// an external abort → `cancelled` (deliberate, no "retry" guidance);
|
|
1429
|
+
// anything else (connection reset, TLS, JSON parse) → `network`.
|
|
1430
|
+
report(
|
|
1431
|
+
cancelled
|
|
1432
|
+
? { reason: 'cancelled' }
|
|
1433
|
+
: timedOut
|
|
1434
|
+
? { reason: 'timeout', timeoutMs: budgetMs }
|
|
1435
|
+
: { reason: 'network' },
|
|
1436
|
+
)
|
|
695
1437
|
return null
|
|
696
1438
|
} finally {
|
|
697
1439
|
clearTimeout(timer)
|
|
@@ -716,10 +1458,13 @@ export function hashContent(content: string | Buffer): string {
|
|
|
716
1458
|
/**
|
|
717
1459
|
* Composite cache key: hash of the user prompt + the sorted content hashes of
|
|
718
1460
|
* every image payload. The prompt is part of the key so a NEW question about
|
|
719
|
-
* the same image never reuses a stale description
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
*
|
|
1461
|
+
* the same image never reuses a stale description — same image + new question
|
|
1462
|
+
* means a NEW gateway describe call (by design, aligned with the council's
|
|
1463
|
+
* describe-once-per-turn flow); only an identical re-ask within the TTL hits
|
|
1464
|
+
* the cache. Sorting makes the key order-insensitive (the same image set in
|
|
1465
|
+
* any order maps to one key). For data-URI/local payloads the URL encodes both
|
|
1466
|
+
* mime and bytes, so hashing it covers "mime+bytes"; remote URLs hash the URL
|
|
1467
|
+
* itself.
|
|
723
1468
|
*/
|
|
724
1469
|
export function buildDescriptionCacheKey(
|
|
725
1470
|
prompt: string,
|
|
@@ -763,31 +1508,37 @@ export function generateInjectionPrompt(
|
|
|
763
1508
|
userText: string,
|
|
764
1509
|
toolName: string,
|
|
765
1510
|
template?: string,
|
|
1511
|
+
sessionID?: string,
|
|
766
1512
|
): string {
|
|
767
1513
|
if (!Array.isArray(images)) {
|
|
768
1514
|
throw new TypeError('Vision image resolution must return an array')
|
|
769
1515
|
}
|
|
770
1516
|
const imageList = images.map((image, index) => `- Image ${index + 1}: ${image.path}`).join('\n')
|
|
1517
|
+
// The MCP runs in a SEPARATE process the host does not hand a session id to,
|
|
1518
|
+
// and the Zen gateway (zen/go, zen/v1) rejects a sessionless request with
|
|
1519
|
+
// HTTP 400 `MissingSessionID`. The plugin relays the live session id through
|
|
1520
|
+
// the tool call so the MCP can send the `x-opencode-session` header. The id
|
|
1521
|
+
// is JSON-quoted so it cannot spoof the surrounding instruction layout.
|
|
1522
|
+
const sessionLine = sessionID
|
|
1523
|
+
? `When you call the tool, pass session_id=${JSON.stringify(sessionID)} so the Zen gateway accepts the request.`
|
|
1524
|
+
: undefined
|
|
771
1525
|
if (template && PROMPT_TEMPLATE_VARIABLES.some((variable) => template.includes(variable))) {
|
|
772
|
-
|
|
1526
|
+
const rendered = template
|
|
773
1527
|
.replaceAll('{imageList}', imageList)
|
|
774
1528
|
.replaceAll('{imageCount}', String(images.length))
|
|
775
1529
|
.replaceAll('{toolName}', toolName)
|
|
776
1530
|
.replaceAll('{userText}', userText)
|
|
1531
|
+
return sessionLine ? `${rendered}\n${sessionLine}` : rendered
|
|
777
1532
|
}
|
|
778
1533
|
const plural = images.length > 1
|
|
779
1534
|
const heading = plural
|
|
780
1535
|
? `The user has shared ${images.length} images. The images are available at:`
|
|
781
1536
|
: 'The user has shared an image. The image is available at:'
|
|
782
1537
|
const analyze = plural ? 'each image' : 'this image'
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
`Use the \`${toolName}\` tool to analyze ${analyze}.`,
|
|
788
|
-
'',
|
|
789
|
-
`User's request: ${userText || '(analyze the image)'}`,
|
|
790
|
-
].join('\n')
|
|
1538
|
+
const lines = [heading, imageList, '', `Use the \`${toolName}\` tool to analyze ${analyze}.`, '']
|
|
1539
|
+
if (sessionLine) lines.push(sessionLine, '')
|
|
1540
|
+
lines.push(`User's request: ${userText || '(analyze the image)'}`)
|
|
1541
|
+
return lines.join('\n')
|
|
791
1542
|
}
|
|
792
1543
|
|
|
793
1544
|
/**
|
|
@@ -851,6 +1602,32 @@ function unescapeXml(text: string): string {
|
|
|
851
1602
|
return text.replaceAll('<', '<').replaceAll('>', '>').replaceAll('&', '&')
|
|
852
1603
|
}
|
|
853
1604
|
|
|
1605
|
+
/**
|
|
1606
|
+
* Payload bound for the user's question interpolated into vision prompts:
|
|
1607
|
+
* prompts longer than this are clipped (with an ellipsis marker) so a long
|
|
1608
|
+
* turn never bloats the gateway request. Exported because the V2 path
|
|
1609
|
+
* (v2-vision.ts) applies the same cap to its own describe prompt.
|
|
1610
|
+
*/
|
|
1611
|
+
export const MAX_PROMPT_USER_TEXT_CHARS = 2000
|
|
1612
|
+
|
|
1613
|
+
/**
|
|
1614
|
+
* Trim and cap the user's question at `maxChars` characters TOTAL, appending a
|
|
1615
|
+
* single "…" when clipped (the marker counts against the cap, so the result is
|
|
1616
|
+
* never longer than `maxChars`). The cap applies to the raw text BEFORE XML
|
|
1617
|
+
* escaping (escaping happens at the interpolation site). Empty/whitespace
|
|
1618
|
+
* input yields '' so callers keep their generic-prompt placeholder.
|
|
1619
|
+
* Never logged — the question is prompt content, not telemetry.
|
|
1620
|
+
*/
|
|
1621
|
+
export function truncateUserText(
|
|
1622
|
+
text: string,
|
|
1623
|
+
maxChars: number = MAX_PROMPT_USER_TEXT_CHARS,
|
|
1624
|
+
): string {
|
|
1625
|
+
const trimmed = text.trim()
|
|
1626
|
+
if (trimmed.length <= maxChars) return trimmed
|
|
1627
|
+
const marker = '…'
|
|
1628
|
+
return `${trimmed.slice(0, Math.max(0, maxChars - marker.length))}${marker}`
|
|
1629
|
+
}
|
|
1630
|
+
|
|
854
1631
|
const INTENT_TASK_INSTRUCTIONS: Record<VisionIntent, string> = {
|
|
855
1632
|
describe:
|
|
856
1633
|
'Descreva cada imagem fielmente e em detalhes: texto visível verbatim, cores, layout, objetos.',
|
|
@@ -871,22 +1648,189 @@ const INTENT_CONTEXT_INSTRUCTIONS: Record<VisionIntent, string> = {
|
|
|
871
1648
|
bugs: '',
|
|
872
1649
|
}
|
|
873
1650
|
|
|
1651
|
+
// ─── EXIF orientation detection (zero-dep, JPEG only) ──────────────────────
|
|
1652
|
+
// DETECTION, not correction: no pixel data is ever rewritten (no sharp/jimp —
|
|
1653
|
+
// YAGNI). When a payload's EXIF orientation tag ≠ 1, the native prompt carries
|
|
1654
|
+
// a textual hint instructing the vision model to describe the image in its
|
|
1655
|
+
// correct viewing orientation. The parser reads only the first kilobytes of
|
|
1656
|
+
// the JPEG (APP1 EXIF lives at the very start) and is fail-safe: ANY malformed
|
|
1657
|
+
// input yields null, never a throw.
|
|
1658
|
+
|
|
1659
|
+
/** EXIF orientation tag id (IFD0). */
|
|
1660
|
+
const EXIF_ORIENTATION_TAG = 0x0112
|
|
1661
|
+
/**
|
|
1662
|
+
* Base64 chars decoded for the EXIF scan (≈75 KB of binary — far more than
|
|
1663
|
+
* any APP1 segment needs; keeps `payloadOrientation` O(1) even for 25 MB URIs).
|
|
1664
|
+
*/
|
|
1665
|
+
const EXIF_SCAN_BASE64_CHARS = 100_000
|
|
1666
|
+
|
|
1667
|
+
/** How to mentally correct each non-1 EXIF orientation to view the image upright. */
|
|
1668
|
+
const ORIENTATION_TRANSFORM_HINTS: Record<number, string> = {
|
|
1669
|
+
2: 'stored mirrored horizontally — apply a horizontal mirror to view it upright',
|
|
1670
|
+
3: 'stored upside down — apply a 180-degree rotation to view it upright',
|
|
1671
|
+
4: 'stored mirrored vertically — apply a vertical mirror to view it upright',
|
|
1672
|
+
5: 'stored mirrored and rotated — apply a horizontal mirror then a 90-degree clockwise rotation to view it upright',
|
|
1673
|
+
6: 'stored rotated — apply a 90-degree clockwise rotation to view it upright',
|
|
1674
|
+
7: 'stored mirrored and rotated — apply a horizontal mirror then a 90-degree counter-clockwise rotation to view it upright',
|
|
1675
|
+
8: 'stored rotated — apply a 90-degree counter-clockwise rotation to view it upright',
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
/**
|
|
1679
|
+
* Read the EXIF orientation (tag 0x0112) from raw JPEG bytes. Walks the
|
|
1680
|
+
* segment chain (SOI → APP0/APP1/…) for the first APP1 segment starting with
|
|
1681
|
+
* "Exif\0\0", then parses the TIFF byte-order header, the IFD0 entry count and
|
|
1682
|
+
* the 12-byte entries until the orientation tag is found. Supports both TIFF
|
|
1683
|
+
* byte orders ("II" little-endian and "MM" big-endian). Returns 1–8, or null
|
|
1684
|
+
* for anything else — non-JPEG, no EXIF, no orientation tag, or ANY malformed
|
|
1685
|
+
* input. Never throws (fail-safe by construction: every read is bounds-checked).
|
|
1686
|
+
*/
|
|
1687
|
+
export function readExifOrientation(bytes: Uint8Array): number | null {
|
|
1688
|
+
try {
|
|
1689
|
+
// JPEG must start with the SOI marker; anything else is not ours to parse.
|
|
1690
|
+
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null
|
|
1691
|
+
let offset = 2
|
|
1692
|
+
while (offset + 4 <= bytes.length) {
|
|
1693
|
+
if (bytes[offset] !== 0xff) return null // marker sync lost → malformed
|
|
1694
|
+
const marker = bytes[offset + 1] ?? 0
|
|
1695
|
+
// Standalone markers carry no length field (SOI repeat, RSTn, TEM).
|
|
1696
|
+
if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
|
|
1697
|
+
offset += 2
|
|
1698
|
+
continue
|
|
1699
|
+
}
|
|
1700
|
+
if (marker === 0xda) return null // start of scan — no EXIF ahead
|
|
1701
|
+
const length = ((bytes[offset + 2] ?? 0) << 8) | (bytes[offset + 3] ?? 0)
|
|
1702
|
+
if (length < 2) return null // segment length must cover its own 2 bytes
|
|
1703
|
+
const segmentStart = offset + 4
|
|
1704
|
+
if (marker === 0xe1 && segmentStart + 6 <= bytes.length) {
|
|
1705
|
+
const isExifHeader =
|
|
1706
|
+
bytes[segmentStart] === 0x45 && // "E"
|
|
1707
|
+
bytes[segmentStart + 1] === 0x78 && // "x"
|
|
1708
|
+
bytes[segmentStart + 2] === 0x69 && // "i"
|
|
1709
|
+
bytes[segmentStart + 3] === 0x66 && // "f"
|
|
1710
|
+
bytes[segmentStart + 4] === 0x00 &&
|
|
1711
|
+
bytes[segmentStart + 5] === 0x00
|
|
1712
|
+
if (isExifHeader) {
|
|
1713
|
+
return readTiffOrientation(bytes, segmentStart + 6)
|
|
1714
|
+
}
|
|
1715
|
+
// Non-EXIF APP1 (e.g. XMP) → skip and keep walking.
|
|
1716
|
+
}
|
|
1717
|
+
offset += 2 + length
|
|
1718
|
+
}
|
|
1719
|
+
return null
|
|
1720
|
+
} catch {
|
|
1721
|
+
return null // belt-and-suspenders: never let detection fail a turn
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
/** Parse the TIFF block (byte-order header + IFD0) for the orientation tag. */
|
|
1726
|
+
function readTiffOrientation(bytes: Uint8Array, tiffStart: number): number | null {
|
|
1727
|
+
if (tiffStart + 8 > bytes.length) return null
|
|
1728
|
+
const bom = bytes[tiffStart]
|
|
1729
|
+
const little = bom === 0x49 // "I" → little-endian; "M" (0x4d) → big-endian
|
|
1730
|
+
if (!little && bom !== 0x4d) return null
|
|
1731
|
+
const u16 = (pos: number): number =>
|
|
1732
|
+
little
|
|
1733
|
+
? (bytes[pos] ?? 0) | ((bytes[pos + 1] ?? 0) << 8)
|
|
1734
|
+
: ((bytes[pos] ?? 0) << 8) | (bytes[pos + 1] ?? 0)
|
|
1735
|
+
const u32 = (pos: number): number =>
|
|
1736
|
+
little
|
|
1737
|
+
? ((bytes[pos] ?? 0) |
|
|
1738
|
+
((bytes[pos + 1] ?? 0) << 8) |
|
|
1739
|
+
((bytes[pos + 2] ?? 0) << 16) |
|
|
1740
|
+
((bytes[pos + 3] ?? 0) << 24)) >>>
|
|
1741
|
+
0
|
|
1742
|
+
: (((bytes[pos] ?? 0) << 24) |
|
|
1743
|
+
((bytes[pos + 1] ?? 0) << 16) |
|
|
1744
|
+
((bytes[pos + 2] ?? 0) << 8) |
|
|
1745
|
+
(bytes[pos + 3] ?? 0)) >>>
|
|
1746
|
+
0
|
|
1747
|
+
if (u16(tiffStart + 2) !== 42) return null // TIFF magic
|
|
1748
|
+
const ifdStart = tiffStart + u32(tiffStart + 4)
|
|
1749
|
+
if (ifdStart + 2 > bytes.length) return null
|
|
1750
|
+
const entryCount = u16(ifdStart)
|
|
1751
|
+
for (let i = 0; i < entryCount; i++) {
|
|
1752
|
+
const entry = ifdStart + 2 + i * 12
|
|
1753
|
+
if (entry + 12 > bytes.length) return null
|
|
1754
|
+
if (u16(entry) === EXIF_ORIENTATION_TAG) {
|
|
1755
|
+
// Type SHORT (3), count 1 → the value sits inline in the first 2 bytes
|
|
1756
|
+
// of the entry's value/offset field, encoded in the TIFF byte order.
|
|
1757
|
+
const value = u16(entry + 8)
|
|
1758
|
+
return value >= 1 && value <= 8 ? value : null
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
return null
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
/**
|
|
1765
|
+
* Best-effort EXIF orientation for one native payload. Only local JPEG data
|
|
1766
|
+
* URIs are inspected (remote http(s) URLs would need a network round-trip —
|
|
1767
|
+
* out of scope for zero-dep detection; non-JPEG payloads short-circuit before
|
|
1768
|
+
* any parsing). Returns 1–8 or null; never throws.
|
|
1769
|
+
*/
|
|
1770
|
+
export function payloadOrientation(payload: { url: string }): number | null {
|
|
1771
|
+
if (payload.url.startsWith('http://') || payload.url.startsWith('https://')) return null
|
|
1772
|
+
const match = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/.exec(payload.url)
|
|
1773
|
+
if (!match || (match[1]?.toLowerCase() ?? '') !== 'image/jpeg') return null
|
|
1774
|
+
try {
|
|
1775
|
+
const prefix = Buffer.from((match[2] ?? '').slice(0, EXIF_SCAN_BASE64_CHARS), 'base64')
|
|
1776
|
+
return readExifOrientation(prefix)
|
|
1777
|
+
} catch {
|
|
1778
|
+
return null
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
/**
|
|
1783
|
+
* Build the textual orientation hint appended to the native prompt. One line
|
|
1784
|
+
* per image whose detected orientation ≠ 1 (the transform the vision model
|
|
1785
|
+
* must apply mentally to see the image upright); empty string when every
|
|
1786
|
+
* image is upright or unknown — the prompt then stays byte-identical to the
|
|
1787
|
+
* no-EXIF behavior. The hint is pure static text (no user content), so it can
|
|
1788
|
+
* never re-expand template variables nor break the prompt's XML structure.
|
|
1789
|
+
*/
|
|
1790
|
+
export function buildOrientationHint(orientations: ReadonlyArray<number | null>): string {
|
|
1791
|
+
const notes: string[] = []
|
|
1792
|
+
orientations.forEach((orientation, index) => {
|
|
1793
|
+
const transform = orientation === null ? undefined : ORIENTATION_TRANSFORM_HINTS[orientation]
|
|
1794
|
+
if (transform)
|
|
1795
|
+
notes.push(`- image ${index + 1}: EXIF orientation ${orientation} — ${transform}.`)
|
|
1796
|
+
})
|
|
1797
|
+
return notes.join('\n')
|
|
1798
|
+
}
|
|
1799
|
+
|
|
874
1800
|
/**
|
|
875
1801
|
* Build the calibrated prompt for the native vision gateway call. The model is
|
|
876
1802
|
* instructed to reply with N `<item id="N"><description>…</description></item>`
|
|
877
1803
|
* blocks (N = imageCount) and, for `compare`, one extra `<context>` block.
|
|
878
|
-
* The user's text is
|
|
879
|
-
*
|
|
880
|
-
* `
|
|
1804
|
+
* The user's text is trimmed, capped at `MAX_PROMPT_USER_TEXT_CHARS`
|
|
1805
|
+
* (see `truncateUserText`) and XML-escaped when interpolated so it can neither
|
|
1806
|
+
* break the format instructions nor bloat the payload. `intent` is
|
|
1807
|
+
* auto-detected from `userText` + `imageCount` when omitted.
|
|
1808
|
+
*
|
|
1809
|
+
* `orientationHint` (see `buildOrientationHint`) appends an EXIF orientation
|
|
1810
|
+
* block when any image is stored rotated/mirrored — DETECTION, not correction:
|
|
1811
|
+
* the vision model is instructed to describe each image in its corrected
|
|
1812
|
+
* viewing orientation; no pixels are rewritten. An empty hint leaves the
|
|
1813
|
+
* prompt byte-identical to the previous behavior.
|
|
1814
|
+
*
|
|
1815
|
+
* Cache interplay: the returned prompt IS the description-cache key's prompt
|
|
1816
|
+
* component (`buildDescriptionCacheKey`), so the SAME image asked a NEW
|
|
1817
|
+
* question produces a different key and a fresh gateway describe — by design,
|
|
1818
|
+
* aligned with the council's describe-once-per-turn flow; only an identical
|
|
1819
|
+
* re-ask (same question, same images) within the TTL reuses the stored text.
|
|
1820
|
+
* The orientation hint travels INSIDE the returned prompt, so two calls for
|
|
1821
|
+
* the same image whose detected orientations differ necessarily get different
|
|
1822
|
+
* cache keys — a stale hint can never be served (and vice versa).
|
|
881
1823
|
*/
|
|
882
1824
|
export function buildNativeVisionPrompt(
|
|
883
1825
|
userText: string,
|
|
884
1826
|
imageCount: number,
|
|
885
1827
|
intent: VisionIntent = detectVisionIntent(userText, imageCount),
|
|
1828
|
+
orientationHint: string = '',
|
|
886
1829
|
): string {
|
|
887
1830
|
const count = Math.max(1, imageCount)
|
|
888
1831
|
const request =
|
|
889
|
-
escapeXml(userText
|
|
1832
|
+
escapeXml(truncateUserText(userText)) ||
|
|
1833
|
+
'(sem pedido explícito — descreva as imagens com precisão)'
|
|
890
1834
|
const contextInstruction = INTENT_CONTEXT_INSTRUCTIONS[intent]
|
|
891
1835
|
const contextLine = contextInstruction ? ` ${contextInstruction}` : ''
|
|
892
1836
|
return [
|
|
@@ -920,6 +1864,15 @@ export function buildNativeVisionPrompt(
|
|
|
920
1864
|
'- Nunca invente detalhes que não consegue perceber. Se um texto estiver ilegível,',
|
|
921
1865
|
' escreva "ilegível". Coloque análises cruzadas (comparação, veredito) no <context>,',
|
|
922
1866
|
' nunca dentro de um <description>.',
|
|
1867
|
+
...(orientationHint.trim()
|
|
1868
|
+
? [
|
|
1869
|
+
'',
|
|
1870
|
+
'IMAGE ORIENTATION (EXIF) — storage metadata, not content:',
|
|
1871
|
+
...orientationHint.trim().split('\n'),
|
|
1872
|
+
'Describe each image as it appears AFTER applying the correction above;',
|
|
1873
|
+
'the rotation/mirroring is not a defect and must not be reported as one.',
|
|
1874
|
+
]
|
|
1875
|
+
: []),
|
|
923
1876
|
'',
|
|
924
1877
|
`User's request: ${request}`,
|
|
925
1878
|
].join('\n')
|
|
@@ -1248,6 +2201,65 @@ export function createVisionHandler(input: PluginInput) {
|
|
|
1248
2201
|
return toolNamePromise
|
|
1249
2202
|
}
|
|
1250
2203
|
|
|
2204
|
+
/**
|
|
2205
|
+
* Host provider catalog for native-vision model discovery (TTL-cached per
|
|
2206
|
+
* client inside `fetchProvidersCatalog`): one round-trip per TTL window,
|
|
2207
|
+
* never per request. Null on any failure → legacy behavior.
|
|
2208
|
+
*/
|
|
2209
|
+
const getProvidersCatalog = (): Promise<unknown> => fetchProvidersCatalog(client)
|
|
2210
|
+
|
|
2211
|
+
/** Lazily create (and return) the per-session description cache. */
|
|
2212
|
+
const ensureSessionCache = (sessionID: string): Map<string, DescriptionCacheEntry> => {
|
|
2213
|
+
let cache = sessionDescriptionCache.get(sessionID)
|
|
2214
|
+
if (!cache) {
|
|
2215
|
+
cache = new Map()
|
|
2216
|
+
sessionDescriptionCache.set(sessionID, cache)
|
|
2217
|
+
}
|
|
2218
|
+
return cache
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
/**
|
|
2222
|
+
* Last rung of the fallback chain: deterministic local extraction (metadata
|
|
2223
|
+
* + best-effort colors/OCR) for when BOTH the native model and the MCP tool
|
|
2224
|
+
* are unavailable. Reuses `resolveImageSourcePayload` so data URIs and local
|
|
2225
|
+
* paths share the native pipeline's byte/size handling. The block is cached
|
|
2226
|
+
* per session (keyed by content hashes) because it is deterministic for the
|
|
2227
|
+
* same file. Returns null when no image could be read — callers then emit an
|
|
2228
|
+
* explicit unavailable message. Never logs payload content. The user's
|
|
2229
|
+
* question is trimmed, capped at `MAX_PROMPT_USER_TEXT_CHARS` and XML-escaped
|
|
2230
|
+
* (same `truncateUserText` + `escapeXml` as the native prompt) so it can
|
|
2231
|
+
* neither bloat the block nor be interpreted as markup.
|
|
2232
|
+
*/
|
|
2233
|
+
const buildLocalInjection = async (
|
|
2234
|
+
sessionID: string,
|
|
2235
|
+
images: ReadonlyArray<{ filePart: FilePart; target: SavedImage }>,
|
|
2236
|
+
userText: string,
|
|
2237
|
+
): Promise<string | null> => {
|
|
2238
|
+
const urls: string[] = []
|
|
2239
|
+
for (const { filePart, target } of images) {
|
|
2240
|
+
const payload = await resolveImageSourcePayload(target.path, filePart.mime)
|
|
2241
|
+
if (payload) urls.push(payload.url)
|
|
2242
|
+
}
|
|
2243
|
+
if (urls.length === 0) return null
|
|
2244
|
+
const cache = ensureSessionCache(sessionID)
|
|
2245
|
+
const cacheKey = `local:${urls
|
|
2246
|
+
.map((url) => hashContent(url))
|
|
2247
|
+
.sort()
|
|
2248
|
+
.join(':')}`
|
|
2249
|
+
let block = getCachedDescription(cache, cacheKey)
|
|
2250
|
+
if (block === null) {
|
|
2251
|
+
block = await buildLocalVisionBlock(urls)
|
|
2252
|
+
if (block === null) return null
|
|
2253
|
+
setCachedDescription(cache, cacheKey, block)
|
|
2254
|
+
}
|
|
2255
|
+
return [
|
|
2256
|
+
'Visão nativa e tool MCP de visão indisponíveis; foi usada extração local determinística (metadados + best-effort), sem conteúdo inventado:',
|
|
2257
|
+
block,
|
|
2258
|
+
'',
|
|
2259
|
+
`User's request: ${escapeXml(truncateUserText(userText)) || '(analyze the image)'}`,
|
|
2260
|
+
].join('\n')
|
|
2261
|
+
}
|
|
2262
|
+
|
|
1251
2263
|
const chatMessage: NonNullable<Hooks['chat.message']> = async (hookInput, output) => {
|
|
1252
2264
|
try {
|
|
1253
2265
|
const message: UserMessage | undefined = output?.message
|
|
@@ -1328,21 +2340,38 @@ export function createVisionHandler(input: PluginInput) {
|
|
|
1328
2340
|
let injection: string | null = null
|
|
1329
2341
|
if (getVisionMode(process.env, config.mode) !== 'tool') {
|
|
1330
2342
|
try {
|
|
1331
|
-
const target = await resolveNativeVisionTarget({
|
|
2343
|
+
const target = await resolveNativeVisionTarget({
|
|
2344
|
+
config,
|
|
2345
|
+
catalog: await getProvidersCatalog(),
|
|
2346
|
+
})
|
|
1332
2347
|
if (target) {
|
|
1333
2348
|
const payloads = await buildNativeImagePayloads(resolved)
|
|
1334
2349
|
if (payloads) {
|
|
1335
|
-
|
|
1336
|
-
if (!sessionCache) {
|
|
1337
|
-
sessionCache = new Map()
|
|
1338
|
-
sessionDescriptionCache.set(hookInput.sessionID, sessionCache)
|
|
1339
|
-
}
|
|
2350
|
+
const sessionCache = ensureSessionCache(hookInput.sessionID)
|
|
1340
2351
|
const intent = detectVisionIntent(userText, payloads.length)
|
|
1341
|
-
|
|
2352
|
+
// EXIF orientation detection (zero-dep, JPEG only): a non-1 tag
|
|
2353
|
+
// adds a textual hint to the prompt so the vision model describes
|
|
2354
|
+
// the corrected view. The hint lives INSIDE the prompt, so the
|
|
2355
|
+
// cache key below varies with it — no stale-orientation hits.
|
|
2356
|
+
const orientationHint = buildOrientationHint(
|
|
2357
|
+
payloads.map((payload) => payloadOrientation(payload)),
|
|
2358
|
+
)
|
|
2359
|
+
const prompt = buildNativeVisionPrompt(
|
|
2360
|
+
userText,
|
|
2361
|
+
payloads.length,
|
|
2362
|
+
intent,
|
|
2363
|
+
orientationHint,
|
|
2364
|
+
)
|
|
1342
2365
|
const cacheKey = buildDescriptionCacheKey(prompt, payloads)
|
|
1343
2366
|
let rawText = getCachedDescription(sessionCache, cacheKey)
|
|
1344
2367
|
if (rawText === null) {
|
|
1345
|
-
|
|
2368
|
+
// Attach the live session id so the Zen gateway accepts the
|
|
2369
|
+
// request (Zen-only; see describeImagesNative).
|
|
2370
|
+
const requestTarget: NativeVisionTarget = {
|
|
2371
|
+
...target,
|
|
2372
|
+
...(hookInput.sessionID ? { sessionID: hookInput.sessionID } : {}),
|
|
2373
|
+
}
|
|
2374
|
+
rawText = await describeImagesNative(payloads, requestTarget, prompt)
|
|
1346
2375
|
if (rawText !== null) setCachedDescription(sessionCache, cacheKey, rawText)
|
|
1347
2376
|
}
|
|
1348
2377
|
if (rawText !== null) {
|
|
@@ -1366,9 +2395,25 @@ export function createVisionHandler(input: PluginInput) {
|
|
|
1366
2395
|
|
|
1367
2396
|
if (injection === null && saved.length > 0) {
|
|
1368
2397
|
const toolName = await getToolName()
|
|
2398
|
+
// Chain: native → tool → local. The MCP tool instruction is used when
|
|
2399
|
+
// a tool is configured/detected; otherwise fall to deterministic local
|
|
2400
|
+
// extraction (metadata + best-effort colors/OCR) instead of a bare
|
|
2401
|
+
// unavailable message. A null tool name (env/config absent AND no MCP
|
|
2402
|
+
// evidence) is the only path that reaches local extraction, so the
|
|
2403
|
+
// env > config > preset > discovery > DEFAULT precedence is untouched.
|
|
1369
2404
|
injection = toolName
|
|
1370
|
-
? generateInjectionPrompt(
|
|
1371
|
-
|
|
2405
|
+
? generateInjectionPrompt(
|
|
2406
|
+
saved,
|
|
2407
|
+
userText,
|
|
2408
|
+
toolName,
|
|
2409
|
+
config.promptTemplate,
|
|
2410
|
+
hookInput.sessionID,
|
|
2411
|
+
)
|
|
2412
|
+
: await buildLocalInjection(hookInput.sessionID, resolved, userText)
|
|
2413
|
+
if (injection === null) {
|
|
2414
|
+
injection =
|
|
2415
|
+
'Vision fallback unavailable: pantheon-vision MCP is unavailable and local image extraction also failed. The image was removed safely; please configure the MCP server and retry.'
|
|
2416
|
+
}
|
|
1372
2417
|
}
|
|
1373
2418
|
if (injection === null) {
|
|
1374
2419
|
injection = 'A imagem compartilhada foi removida antes do envio ao provedor textual.'
|