@bitkyc08/opencodex 2.30.0-preview.20260821 → 2.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/gui/dist/assets/index-DkcRs1fL.js +102 -0
- package/gui/dist/index.html +1 -1
- package/package.json +1 -1
- package/src/adapters/cursor/cursor-errors.ts +65 -6
- package/src/adapters/cursor/discovery.ts +7 -2
- package/src/adapters/cursor/effort-map.ts +6 -0
- package/src/adapters/cursor/h2-pool.ts +123 -0
- package/src/adapters/cursor/live-models.ts +21 -26
- package/src/adapters/cursor/live-transport.ts +213 -3
- package/src/adapters/cursor/native-exec-common.ts +17 -0
- package/src/adapters/cursor/native-exec.ts +9 -4
- package/src/adapters/cursor/protobuf-events.ts +5 -1
- package/src/adapters/cursor/protobuf-request.ts +11 -4
- package/src/adapters/cursor/tool-definitions.ts +20 -0
- package/src/adapters/cursor/transport.ts +10 -0
- package/src/adapters/cursor.ts +23 -5
- package/src/adapters/google.ts +16 -3
- package/src/adapters/openai-responses.ts +66 -20
- package/src/adapters/xai-web-search.ts +185 -0
- package/src/cli/agent.ts +2 -1
- package/src/cli/dispatch.ts +2 -2
- package/src/cli/doctor.ts +89 -0
- package/src/cli/help.ts +2 -0
- package/src/cli/registry.ts +7 -2
- package/src/codex/auth-context.ts +41 -2
- package/src/codex/catalog/effort.ts +1 -1
- package/src/codex/catalog/parsing.ts +2 -0
- package/src/codex/catalog/provider-fetch.ts +20 -5
- package/src/codex/coordinator-doctor.ts +332 -0
- package/src/codex/inject-coordination.ts +39 -6
- package/src/codex/transition-state.ts +12 -12
- package/src/generated/compatibility-version.json +74 -50
- package/src/lib/errors.ts +8 -2
- package/src/oauth/cursor.ts +21 -0
- package/src/providers/cursor-pool.ts +72 -0
- package/src/providers/derive.ts +3 -0
- package/src/providers/fastwire.ts +12 -1
- package/src/providers/openai-sidecar.ts +1 -0
- package/src/providers/registry.ts +25 -0
- package/src/providers/service-tier.ts +22 -7
- package/src/responses/custom-tool-compat.ts +24 -8
- package/src/responses/namespace-tool-compat.ts +2 -3
- package/src/router.ts +3 -0
- package/src/server/chat-completions.ts +4 -0
- package/src/server/chat-native.ts +20 -0
- package/src/server/management/agent-settings-routes.ts +16 -5
- package/src/server/management/config-routes.ts +25 -5
- package/src/server/management/vision-sidecar-options.ts +54 -19
- package/src/server/responses/compact.ts +1 -2
- package/src/server/responses/core.ts +54 -13
- package/src/service.ts +122 -14
- package/src/types/config.ts +9 -3
- package/src/types/provider.ts +6 -0
- package/src/usage/cost.ts +52 -38
- package/src/usage/expected-prices.ts +79 -9
- package/src/vision/backends.ts +97 -0
- package/src/vision/eligibility.ts +43 -22
- package/src/vision/index.ts +73 -5
- package/src/vision/routed-describe.ts +175 -0
- package/gui/dist/assets/index-eBA05kYB.js +0 -102
|
@@ -181,6 +181,30 @@ export const EXPECTED_PRICE_OVERLAYS: readonly ExpectedPriceOverlay[] = [
|
|
|
181
181
|
{ provider: "cursor", modelId: "auto", cost4: { input: 1.25, output: 6, cacheRead: 0.25, cacheWrite: 1.25 }, source: "https://docs.cursor.com/account/pricing + https://cursor.com/blog/aug-2025-pricing", verifiedAt: "2026-07-20", status: "verified" },
|
|
182
182
|
];
|
|
183
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Exact official corrections for stale nonzero catalog rows. These are intentionally separate
|
|
186
|
+
* from fallback overlays: they win over the bundled row only for the declared provider/model and
|
|
187
|
+
* therefore cannot reprice routed resellers that reuse the same model slug.
|
|
188
|
+
*/
|
|
189
|
+
export const VERIFIED_PRICE_OVERRIDES: readonly ExpectedPriceOverlay[] = [
|
|
190
|
+
{
|
|
191
|
+
provider: "xai",
|
|
192
|
+
modelId: "grok-4.6",
|
|
193
|
+
cost4: { input: 2, output: 6, cacheRead: 0.5, cacheWrite: 0 },
|
|
194
|
+
source: "https://docs.x.ai/developers/pricing",
|
|
195
|
+
verifiedAt: "2026-08-18",
|
|
196
|
+
status: "verified",
|
|
197
|
+
},
|
|
198
|
+
];
|
|
199
|
+
|
|
200
|
+
export function findVerifiedPriceOverride(
|
|
201
|
+
provider: string,
|
|
202
|
+
modelId: string,
|
|
203
|
+
overrides: readonly ExpectedPriceOverlay[] = VERIFIED_PRICE_OVERRIDES,
|
|
204
|
+
): ExpectedPriceOverlay | undefined {
|
|
205
|
+
return overrides.find(row => row.provider === provider && row.modelId === modelId);
|
|
206
|
+
}
|
|
207
|
+
|
|
184
208
|
/**
|
|
185
209
|
* Exact-key overlay lookup. Returns verified first, then verified-derived.
|
|
186
210
|
* NEVER returns "unverified" rows — fail-closed is enforced in code, not just docs.
|
|
@@ -196,12 +220,7 @@ export function findExpectedPriceOverlay(
|
|
|
196
220
|
?? exact.find(row => row.status === "verified-derived");
|
|
197
221
|
}
|
|
198
222
|
|
|
199
|
-
/**
|
|
200
|
-
* OpenAI Fast mode (`service_tier=priority`) price multipliers by model slug.
|
|
201
|
-
* Source: https://openai.com/api-fast-mode/ (2026-07-31).
|
|
202
|
-
* Fast pricing applies uniformly to all token types (input, output, cache).
|
|
203
|
-
* Models not listed here fall back to 1× (no multiplier).
|
|
204
|
-
*/
|
|
223
|
+
/** OpenAI Fast price multipliers retained as a compatibility export. */
|
|
205
224
|
export const PRIORITY_MULTIPLIERS: Readonly<Record<string, number>> = {
|
|
206
225
|
"gpt-5.6-sol": 2,
|
|
207
226
|
// Post-price-cut Fast tables (https://openai.com/api-fast-mode/, 2026-08-05):
|
|
@@ -219,6 +238,52 @@ export function resolvePriorityMultiplier(modelId: string): number {
|
|
|
219
238
|
return PRIORITY_MULTIPLIERS[modelId] ?? 1;
|
|
220
239
|
}
|
|
221
240
|
|
|
241
|
+
export interface PriorityPricingRule {
|
|
242
|
+
provider: string;
|
|
243
|
+
modelId: string;
|
|
244
|
+
multiplier: number;
|
|
245
|
+
/** Apply the premium only after the upstream response confirms this tier. */
|
|
246
|
+
requiresResponseConfirmation?: true;
|
|
247
|
+
source: string;
|
|
248
|
+
verifiedAt: string;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const OPENAI_FAST_PRICING = "https://openai.com/api-fast-mode/";
|
|
252
|
+
const XAI_PRIORITY_PRICING = "https://docs.x.ai/developers/advanced-api-usage/priority-processing";
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Exact provider/model priority premiums. Routed resellers never inherit a vendor rule merely
|
|
256
|
+
* because they reuse its model slug. Multipliers apply uniformly after cache discounts.
|
|
257
|
+
*/
|
|
258
|
+
export const PRIORITY_PRICING_RULES: readonly PriorityPricingRule[] = [
|
|
259
|
+
...["openai", "openai-apikey"].flatMap(provider =>
|
|
260
|
+
Object.entries(PRIORITY_MULTIPLIERS).map(([modelId, multiplier]): PriorityPricingRule => ({
|
|
261
|
+
provider,
|
|
262
|
+
modelId,
|
|
263
|
+
multiplier,
|
|
264
|
+
source: OPENAI_FAST_PRICING,
|
|
265
|
+
verifiedAt: "2026-08-05",
|
|
266
|
+
})),
|
|
267
|
+
),
|
|
268
|
+
...["grok-4.5", "grok-4.6"].map((modelId): PriorityPricingRule => ({
|
|
269
|
+
provider: "xai",
|
|
270
|
+
modelId,
|
|
271
|
+
multiplier: 2,
|
|
272
|
+
requiresResponseConfirmation: true,
|
|
273
|
+
source: XAI_PRIORITY_PRICING,
|
|
274
|
+
verifiedAt: "2026-08-18",
|
|
275
|
+
})),
|
|
276
|
+
];
|
|
277
|
+
|
|
278
|
+
/** Exact provider/model priority-pricing lookup. */
|
|
279
|
+
export function findPriorityPricingRule(
|
|
280
|
+
provider: string,
|
|
281
|
+
modelId: string,
|
|
282
|
+
rules: readonly PriorityPricingRule[] = PRIORITY_PRICING_RULES,
|
|
283
|
+
): PriorityPricingRule | undefined {
|
|
284
|
+
return rules.find(rule => rule.provider === provider && rule.modelId === modelId);
|
|
285
|
+
}
|
|
286
|
+
|
|
222
287
|
/**
|
|
223
288
|
* Long-context pricing tiers (#908). Several vendors reprice the ENTIRE request
|
|
224
289
|
* once the prompt crosses a published input-token threshold, so a flat Cost4
|
|
@@ -244,6 +309,8 @@ export interface ContextTier {
|
|
|
244
309
|
inclusive: boolean;
|
|
245
310
|
/** Per-field factor from the short rate to the published long rate. */
|
|
246
311
|
multiplier: Cost4;
|
|
312
|
+
/** Published relationship between confirmed priority and long-context bands. */
|
|
313
|
+
confirmedPriorityRelation?: "exclusive" | "lower-bound";
|
|
247
314
|
source: string;
|
|
248
315
|
verifiedAt: string;
|
|
249
316
|
}
|
|
@@ -277,6 +344,7 @@ export const CONTEXT_TIERS: readonly ContextTier[] = [
|
|
|
277
344
|
thresholdInputTokens: 272_000,
|
|
278
345
|
inclusive: false,
|
|
279
346
|
multiplier: OPENAI_LONG_CONTEXT,
|
|
347
|
+
confirmedPriorityRelation: "exclusive",
|
|
280
348
|
source: OPENAI_PRICING_DOC,
|
|
281
349
|
verifiedAt: "2026-08-03",
|
|
282
350
|
})),
|
|
@@ -287,19 +355,21 @@ export const CONTEXT_TIERS: readonly ContextTier[] = [
|
|
|
287
355
|
thresholdInputTokens: 200_000,
|
|
288
356
|
inclusive: true,
|
|
289
357
|
multiplier: UNIFORM_DOUBLE,
|
|
358
|
+
confirmedPriorityRelation: "lower-bound",
|
|
290
359
|
source: "https://docs.x.ai/developers/pricing",
|
|
291
360
|
verifiedAt: "2026-08-03",
|
|
292
361
|
},
|
|
293
362
|
{
|
|
294
|
-
//
|
|
295
|
-
//
|
|
363
|
+
// xAI publishes the whole-request >=200k band for grok-4.6. Its combination with
|
|
364
|
+
// Priority Processing is not published, so confirmed priority uses this row as a lower bound.
|
|
296
365
|
provider: "xai",
|
|
297
366
|
modelId: "grok-4.6",
|
|
298
367
|
thresholdInputTokens: 200_000,
|
|
299
368
|
inclusive: true,
|
|
300
369
|
multiplier: UNIFORM_DOUBLE,
|
|
370
|
+
confirmedPriorityRelation: "lower-bound",
|
|
301
371
|
source: "https://docs.x.ai/developers/pricing",
|
|
302
|
-
verifiedAt: "2026-08-
|
|
372
|
+
verifiedAt: "2026-08-18",
|
|
303
373
|
},
|
|
304
374
|
{
|
|
305
375
|
// daybreak-blue-latest aliases gpt-5.6-sol, which publishes the full long-context row
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which backends may DESCRIBE images for the vision sidecar, and which
|
|
3
|
+
* candidate rows each can describe through (#2188 vision rules; roadmap 170
|
|
4
|
+
* REVISED: the "routed" backend).
|
|
5
|
+
*
|
|
6
|
+
* A SIBLING of WEB_SEARCH_BACKENDS, not a shared table: vision has no
|
|
7
|
+
* per-model probe (rule 2 is "− provably text-only", enforced by
|
|
8
|
+
* modelAcceptsImageInput, not here), carries per-side baseline models, and
|
|
9
|
+
* excludes non-LLM backends like exa.
|
|
10
|
+
*
|
|
11
|
+
* Three backends, not one per provider: "openai" and "anthropic" carry auth
|
|
12
|
+
* semantics loopback routing cannot replicate (forwarded ChatGPT headers,
|
|
13
|
+
* OAuth beta fences) and their defaults must not drift. Every OTHER
|
|
14
|
+
* picker-visible provider row reaches the describer through "routed" — a
|
|
15
|
+
* loopback self-fetch of the proxy's own /v1/chat/completions, where the
|
|
16
|
+
* router and adapters already speak each provider's wire. That is what makes
|
|
17
|
+
* this table closed under provider growth: a new provider needs no new
|
|
18
|
+
* describe executor.
|
|
19
|
+
*/
|
|
20
|
+
import type { OcxConfig } from "../types";
|
|
21
|
+
import type { SidecarAuthState } from "../sidecar/auth";
|
|
22
|
+
import { listOpenAiForwardSidecarCandidates } from "../providers/openai-sidecar";
|
|
23
|
+
import type { VisionCandidateModel, VisionSidecarBackend } from "./eligibility";
|
|
24
|
+
|
|
25
|
+
export interface VisionBackendDescriptor {
|
|
26
|
+
backend: VisionSidecarBackend;
|
|
27
|
+
/** Liveness signal for this backend. */
|
|
28
|
+
isActive(auth: SidecarAuthState, config: OcxConfig): boolean;
|
|
29
|
+
/** Which candidate rows this backend's describe executor can actually run. */
|
|
30
|
+
candidateMatch(candidate: VisionCandidateModel, auth: SidecarAuthState): boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Default entry for this side: cheap, image-capable, present in every
|
|
33
|
+
* deployment. Only the two universal sides carry one — "routed" has no
|
|
34
|
+
* universal model to name.
|
|
35
|
+
*/
|
|
36
|
+
baseline?: string;
|
|
37
|
+
/** Stable option ordering (baselines first within a side). */
|
|
38
|
+
rank: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export const VISION_BACKENDS: readonly VisionBackendDescriptor[] = [
|
|
42
|
+
{
|
|
43
|
+
backend: "openai",
|
|
44
|
+
// The OpenAI describer needs a CANONICAL ChatGPT forward provider, not
|
|
45
|
+
// merely a provider keyed "openai" — same predicate the runtime sidecar
|
|
46
|
+
// resolver uses. Deliberately NOT auth.isCodexAuth: tightening to a live
|
|
47
|
+
// credential here would change which options a fresh install sees, and
|
|
48
|
+
// the options list is a suggestion surface, not the write gate.
|
|
49
|
+
isActive: (_auth, config) => listOpenAiForwardSidecarCandidates(config).length > 0,
|
|
50
|
+
candidateMatch: candidate => candidate.native === true || candidate.provider === "openai",
|
|
51
|
+
baseline: "gpt-5.6-luna",
|
|
52
|
+
rank: 0,
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
backend: "anthropic",
|
|
56
|
+
isActive: auth => auth.isAnthropicAuth,
|
|
57
|
+
// The runtime dispatches through exactly ONE Anthropic provider — the
|
|
58
|
+
// resolved OAuth row. Same-adapter keyed rows are unreachable (see
|
|
59
|
+
// visionBackendForCandidate's original stance).
|
|
60
|
+
candidateMatch: (candidate, auth) => candidate.provider === auth.anthropicProviderName,
|
|
61
|
+
baseline: "claude-haiku-4-5",
|
|
62
|
+
rank: 1,
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
backend: "routed",
|
|
66
|
+
// Always offered: options only materialize when a matching picker row
|
|
67
|
+
// exists, and the row's own provider config is the liveness signal — the
|
|
68
|
+
// loopback request fails closed through ordinary routing errors.
|
|
69
|
+
isActive: () => true,
|
|
70
|
+
// Any row the other two executors do NOT own. Auth-slot rows are
|
|
71
|
+
// entitlements of the openai/anthropic sides and never route here.
|
|
72
|
+
candidateMatch: (candidate, auth) =>
|
|
73
|
+
candidate.native !== true
|
|
74
|
+
&& candidate.provider !== "openai"
|
|
75
|
+
&& candidate.provider !== auth.anthropicProviderName,
|
|
76
|
+
rank: 2,
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
export function visionBackendDescriptor(backend: VisionSidecarBackend): VisionBackendDescriptor {
|
|
81
|
+
const descriptor = VISION_BACKENDS.find(entry => entry.backend === backend);
|
|
82
|
+
if (!descriptor) throw new Error(`unknown vision backend "${backend}"`);
|
|
83
|
+
return descriptor;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The active backend set for option generation. Falls back to the two
|
|
88
|
+
* UNIVERSAL sides when neither is active (fresh install: picker stays
|
|
89
|
+
* populated, permissive-unknown rule); "routed" is active by construction.
|
|
90
|
+
*/
|
|
91
|
+
export function activeVisionBackends(auth: SidecarAuthState, config: OcxConfig): VisionSidecarBackend[] {
|
|
92
|
+
const active = VISION_BACKENDS.filter(entry => entry.isActive(auth, config)).map(entry => entry.backend);
|
|
93
|
+
return active.includes("openai") || active.includes("anthropic")
|
|
94
|
+
? active
|
|
95
|
+
: ["openai", "anthropic", ...active.filter(backend => backend === "routed")];
|
|
96
|
+
}
|
|
97
|
+
|
|
@@ -26,15 +26,27 @@ import { nativeInputModalities } from "../codex/catalog/metadata";
|
|
|
26
26
|
import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models";
|
|
27
27
|
import { enrichProviderFromRegistry } from "../providers/derive";
|
|
28
28
|
|
|
29
|
-
/**
|
|
30
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The wire protocols `planVisionSidecar` can dispatch to (#2188 roadmap 170
|
|
31
|
+
* REVISED). "routed" describes through the proxy's OWN router via loopback —
|
|
32
|
+
* one executor for every non-forward, non-OAuth-Anthropic provider row.
|
|
33
|
+
*/
|
|
34
|
+
export type VisionSidecarBackend = "openai" | "anthropic" | "routed";
|
|
35
|
+
|
|
36
|
+
/** The two sides every deployment has; also the empty-auth fallback set. */
|
|
37
|
+
export type UniversalVisionBackend = "openai" | "anthropic";
|
|
31
38
|
|
|
32
39
|
/**
|
|
33
40
|
* Default entry per backend: cheap, image-capable, and present in every deployment. Offered
|
|
34
41
|
* whenever its side is enabled, and withheld only when that provider explicitly lists it as a
|
|
35
42
|
* model the sidecar describes FOR — never merely because a metadata table stayed silent.
|
|
43
|
+
*
|
|
44
|
+
* Keyed by the UNIVERSAL subset on purpose (roadmap 170, audit blocker A):
|
|
45
|
+
* xai/gemini are auth-gated sides whose catalogs are present whenever the side
|
|
46
|
+
* is, so they carry no baseline, and a narrow-key total record documents that
|
|
47
|
+
* without sprinkling non-null assertions at the consumers.
|
|
36
48
|
*/
|
|
37
|
-
export const BASELINE_VISION_MODELS: Record<
|
|
49
|
+
export const BASELINE_VISION_MODELS: Record<UniversalVisionBackend, string> = {
|
|
38
50
|
openai: "gpt-5.6-luna",
|
|
39
51
|
anthropic: "claude-haiku-4-5",
|
|
40
52
|
};
|
|
@@ -146,7 +158,7 @@ function isVisionEligibleModelWithCache(
|
|
|
146
158
|
return modelAcceptsImageInputWithCache(config, candidate, cache) !== false;
|
|
147
159
|
}
|
|
148
160
|
|
|
149
|
-
/** Which executor can describe through this row
|
|
161
|
+
/** Which executor can describe through this row. */
|
|
150
162
|
export function visionBackendForCandidate(
|
|
151
163
|
config: Pick<OcxConfig, "providers">,
|
|
152
164
|
candidate: VisionCandidateModel,
|
|
@@ -158,17 +170,17 @@ export function visionBackendForCandidate(
|
|
|
158
170
|
// Messages wire is not enough: a key-auth row of the same adapter is unreachable, and an
|
|
159
171
|
// option that cannot be dispatched is worse than a missing one, because selecting it fails
|
|
160
172
|
// at describe time rather than at pick time.
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
//
|
|
165
|
-
//
|
|
166
|
-
|
|
167
|
-
return
|
|
173
|
+
if (anthropicProviderName !== undefined && candidate.provider === anthropicProviderName) {
|
|
174
|
+
return "anthropic";
|
|
175
|
+
}
|
|
176
|
+
// EVERY other provider row describes through the proxy's own router
|
|
177
|
+
// (roadmap 170 revised): the loopback executor covers all provider wires,
|
|
178
|
+
// so no row is left without an executor.
|
|
179
|
+
return "routed";
|
|
168
180
|
}
|
|
169
181
|
|
|
170
182
|
function baselineCandidate(
|
|
171
|
-
backend:
|
|
183
|
+
backend: UniversalVisionBackend,
|
|
172
184
|
anthropicProviderName: string | undefined,
|
|
173
185
|
): VisionCandidateModel {
|
|
174
186
|
return {
|
|
@@ -181,12 +193,13 @@ function baselineCandidate(
|
|
|
181
193
|
}
|
|
182
194
|
|
|
183
195
|
/**
|
|
184
|
-
* The picker's option list: every eligible row reachable by
|
|
185
|
-
*
|
|
186
|
-
* excluded, de-duplicated and stably ordered (
|
|
187
|
-
* within a side). Anthropic rows must belong to the OAuth
|
|
188
|
-
* actually execute them, so `anthropicProviderName` is
|
|
189
|
-
* catalog rows eligible at all
|
|
196
|
+
* The picker's option list: every eligible row reachable by an enabled
|
|
197
|
+
* executor, plus each enabled universal side's baseline unless that baseline
|
|
198
|
+
* is explicitly excluded, de-duplicated and stably ordered (side rank order,
|
|
199
|
+
* baselines first within a side). Anthropic rows must belong to the OAuth
|
|
200
|
+
* provider that would actually execute them, so `anthropicProviderName` is
|
|
201
|
+
* what makes that side's catalog rows eligible at all; xai/gemini rows map by
|
|
202
|
+
* provider identity and appear only when the caller enabled those backends.
|
|
190
203
|
*
|
|
191
204
|
* This is the SUGGESTION list (narrow): it emits only rows an executor can reach
|
|
192
205
|
* and some source has heard of. It is deliberately NOT the same set as the write
|
|
@@ -208,7 +221,7 @@ export function visionEligibleModelOptions(
|
|
|
208
221
|
const byValue = new Map<string, VisionModelOption>();
|
|
209
222
|
const enrichedProviders: EnrichedProviderCache = new Map();
|
|
210
223
|
|
|
211
|
-
for (const backend of
|
|
224
|
+
for (const backend of Object.keys(BASELINE_VISION_MODELS) as UniversalVisionBackend[]) {
|
|
212
225
|
if (!enabled.has(backend)) continue;
|
|
213
226
|
const candidate = baselineCandidate(backend, anthropicProviderName);
|
|
214
227
|
if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
|
|
@@ -219,11 +232,19 @@ export function visionEligibleModelOptions(
|
|
|
219
232
|
const backend = visionBackendForCandidate(config, candidate, anthropicProviderName);
|
|
220
233
|
if (!backend || !enabled.has(backend)) continue;
|
|
221
234
|
if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
|
|
222
|
-
|
|
223
|
-
|
|
235
|
+
// Routed rows carry NAMESPACED values ("provider/model") so the loopback
|
|
236
|
+
// dispatch is unambiguous under routeModel; the legacy sides keep bare ids
|
|
237
|
+
// (GUI current-value compatibility, and the forward/OAuth executors POST
|
|
238
|
+
// the string verbatim). De-dup stays keyed by the emitted value.
|
|
239
|
+
const value = backend === "routed" ? `${candidate.provider}/${candidate.id}` : candidate.id;
|
|
240
|
+
if (byValue.has(value)) continue;
|
|
241
|
+
byValue.set(value, { value, label: value, backend });
|
|
224
242
|
}
|
|
225
243
|
|
|
244
|
+
// Two slots per side (baseline first), ranked openai < anthropic < routed
|
|
245
|
+
// so widening the union appends rather than interleaves (roadmap 170).
|
|
246
|
+
const sideRank: Record<VisionSidecarBackend, number> = { openai: 0, anthropic: 2, routed: 4 };
|
|
226
247
|
const order = (option: VisionModelOption) =>
|
|
227
|
-
|
|
248
|
+
sideRank[option.backend] + (option.baseline ? 0 : 1);
|
|
228
249
|
return [...byValue.values()].sort((a, b) => order(a) - order(b) || a.value.localeCompare(b.value));
|
|
229
250
|
}
|
package/src/vision/index.ts
CHANGED
|
@@ -5,6 +5,8 @@ import { modelRecordValue } from "../reasoning-effort";
|
|
|
5
5
|
import type { VisionReasoningEffort } from "../reasoning-effort";
|
|
6
6
|
import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe";
|
|
7
7
|
import { describeImageAnthropic } from "./anthropic-describe";
|
|
8
|
+
import { describeImageRouted } from "./routed-describe";
|
|
9
|
+
import { modelAcceptsImageInput } from "./eligibility";
|
|
8
10
|
import { normalizeVisionReasoningForModel } from "./reasoning";
|
|
9
11
|
import type { CodexAuthContext } from "../codex/auth-context";
|
|
10
12
|
import { resolveSidecarAuth } from "../sidecar/auth";
|
|
@@ -226,16 +228,23 @@ export function findAnthropicVisionProvider(config: OcxConfig): AnthropicVisionP
|
|
|
226
228
|
}
|
|
227
229
|
|
|
228
230
|
export function resolveVisionBackend(
|
|
229
|
-
explicit: "openai" | "anthropic" | undefined,
|
|
231
|
+
explicit: "openai" | "anthropic" | "routed" | undefined,
|
|
230
232
|
anthropicSidecar: AnthropicVisionProvider | undefined,
|
|
231
233
|
): "openai" | "anthropic" {
|
|
232
234
|
if (explicit === "openai" || explicit === "anthropic") return explicit;
|
|
235
|
+
// "routed" collapses to the legacy default order until its describe executor
|
|
236
|
+
// lands (roadmap 170 → 180 revised): a persisted routed backend without a
|
|
237
|
+
// dispatchable arm degrades exactly like unset rather than crashing. wp3
|
|
238
|
+
// replaces this collapse with the real routed arm in planVisionSidecar.
|
|
233
239
|
return anthropicSidecar ? "anthropic" : "openai";
|
|
234
240
|
}
|
|
235
241
|
|
|
236
242
|
/** Native model used by the OpenAI vision helper, including its bounded default. */
|
|
237
243
|
export function resolveOpenAiVisionModel(config: Pick<OcxConfig, "visionSidecar">): string {
|
|
238
|
-
|
|
244
|
+
const configured = config.visionSidecar?.model;
|
|
245
|
+
// Namespaced routed ids never reach the forward executor (see
|
|
246
|
+
// resolveEffectiveVisionModel).
|
|
247
|
+
return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL;
|
|
239
248
|
}
|
|
240
249
|
|
|
241
250
|
/** Effective describer model for the backend `planVisionSidecar` selected. */
|
|
@@ -243,9 +252,15 @@ export function resolveEffectiveVisionModel(
|
|
|
243
252
|
config: Pick<OcxConfig, "visionSidecar">,
|
|
244
253
|
backend: "openai" | "anthropic",
|
|
245
254
|
): string {
|
|
255
|
+
const configured = config.visionSidecar?.model;
|
|
256
|
+
// A namespaced "provider/model" id belongs to the routed backend only; the
|
|
257
|
+
// forward/OAuth executors POST the model string verbatim, so it falls back
|
|
258
|
+
// to the side's default here (PUT coherence rejects new writes of this
|
|
259
|
+
// shape, but a legacy or hand-edited config must not break the executor).
|
|
260
|
+
const usable = configured && !configured.includes("/") ? configured : undefined;
|
|
246
261
|
return backend === "anthropic"
|
|
247
|
-
?
|
|
248
|
-
:
|
|
262
|
+
? usable || DEFAULT_ANTHROPIC_VISION_MODEL
|
|
263
|
+
: usable || DEFAULT_VISION_MODEL;
|
|
249
264
|
}
|
|
250
265
|
|
|
251
266
|
/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
|
|
@@ -271,9 +286,13 @@ export function shouldResolveOpenAiVisionSidecar(
|
|
|
271
286
|
}
|
|
272
287
|
|
|
273
288
|
export interface VisionPlan {
|
|
274
|
-
backend: "openai" | "anthropic";
|
|
289
|
+
backend: "openai" | "anthropic" | "routed";
|
|
275
290
|
forwardSidecar?: ResolvedOpenAiForwardSidecar;
|
|
276
291
|
anthropicSidecar?: AnthropicVisionProvider;
|
|
292
|
+
/** Namespaced "provider/model" describer for the routed backend (roadmap 180). */
|
|
293
|
+
routedModel?: string;
|
|
294
|
+
/** Loopback dispatch inputs for the routed backend. */
|
|
295
|
+
routedConfig?: Pick<OcxConfig, "port" | "apiKeys">;
|
|
277
296
|
settings: VisionSettings;
|
|
278
297
|
maxDescriptionsPerTurn: number;
|
|
279
298
|
}
|
|
@@ -295,8 +314,45 @@ export function planVisionSidecar(
|
|
|
295
314
|
if (!messagesHaveImage(parsed)) return undefined;
|
|
296
315
|
const cfg = config.visionSidecar ?? {};
|
|
297
316
|
if (cfg.enabled === false) return undefined;
|
|
317
|
+
|
|
318
|
+
// Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit
|
|
319
|
+
// model only — never inferred from credential availability. Plan-time
|
|
320
|
+
// fence: the target must not be provably blind, and must not itself be a
|
|
321
|
+
// model this planner would re-enter for (belt; the terminal marker on the
|
|
322
|
+
// loopback request is the braces).
|
|
323
|
+
if (cfg.backend === "routed") {
|
|
324
|
+
const routedModel = cfg.model;
|
|
325
|
+
const sep = routedModel ? routedModel.indexOf("/") : -1;
|
|
326
|
+
if (routedModel && sep > 0) {
|
|
327
|
+
const targetProvider = routedModel.slice(0, sep);
|
|
328
|
+
const targetId = routedModel.slice(sep + 1);
|
|
329
|
+
const targetProviderConfig = config.providers?.[targetProvider];
|
|
330
|
+
const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false
|
|
331
|
+
&& !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId));
|
|
332
|
+
if (targetVisible) {
|
|
333
|
+
return {
|
|
334
|
+
backend: "routed",
|
|
335
|
+
routedModel,
|
|
336
|
+
routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) },
|
|
337
|
+
settings: {
|
|
338
|
+
model: routedModel,
|
|
339
|
+
reasoning: DEFAULT_REASONING,
|
|
340
|
+
timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
|
|
341
|
+
},
|
|
342
|
+
maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn),
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
// Misconfigured routed backend (bare id, unknown provider, or provably
|
|
347
|
+
// blind target): fall through to the legacy default order below rather
|
|
348
|
+
// than dispatching a describe that cannot work.
|
|
349
|
+
}
|
|
350
|
+
|
|
298
351
|
const anthropicSidecar = findAnthropicVisionProvider(config);
|
|
299
352
|
const backend = resolveVisionBackend(cfg.backend, anthropicSidecar);
|
|
353
|
+
// A namespaced routed model must never reach the forward/OAuth executors
|
|
354
|
+
// (they POST the string verbatim); the effective-model resolver falls back
|
|
355
|
+
// to each side's default in that case.
|
|
300
356
|
const model = resolveEffectiveVisionModel(config, backend);
|
|
301
357
|
const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn);
|
|
302
358
|
|
|
@@ -447,6 +503,18 @@ async function executeDescription(
|
|
|
447
503
|
abortSignal?: AbortSignal,
|
|
448
504
|
recordSidecarOutcome?: SidecarOutcomeRecorder,
|
|
449
505
|
): Promise<DescribeOutcome> {
|
|
506
|
+
if (plan.backend === "routed") {
|
|
507
|
+
if (!plan.routedModel || !plan.routedConfig) return { text: "", error: "routed vision sidecar is unavailable" };
|
|
508
|
+
return describeImageRouted(
|
|
509
|
+
job.imageUrl,
|
|
510
|
+
job.detail,
|
|
511
|
+
job.contextText,
|
|
512
|
+
plan.routedModel,
|
|
513
|
+
plan.routedConfig,
|
|
514
|
+
plan.settings,
|
|
515
|
+
abortSignal,
|
|
516
|
+
);
|
|
517
|
+
}
|
|
450
518
|
if (plan.backend === "anthropic") {
|
|
451
519
|
const sidecar = plan.anthropicSidecar;
|
|
452
520
|
if (!sidecar) return { text: "", error: "anthropic vision sidecar is unavailable" };
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe ONE image via a ROUTED model through the proxy's own
|
|
3
|
+
* /v1/chat/completions on loopback (#2188 roadmap 180 revised).
|
|
4
|
+
*
|
|
5
|
+
* One executor for every provider the router can reach: the chat inbound
|
|
6
|
+
* translates image_url parts and each adapter compiles its own wire
|
|
7
|
+
* (Anthropic blocks, Antigravity inlineData, xai Responses input_image, plain
|
|
8
|
+
* openai-chat), so provider coverage is the router's job, not this file's.
|
|
9
|
+
*
|
|
10
|
+
* Recursion fence: the request carries `x-opencodex-vision-describe: 1`.
|
|
11
|
+
* The Chat surface detects the raw header before its bridge rebuilds headers
|
|
12
|
+
* and carries it into handleResponses as `visionDescribeTerminal`; a marked
|
|
13
|
+
* request STRIPS images instead of planning another describe (depth cap 1,
|
|
14
|
+
* holds under predicate drift and combo re-resolution — audit rounds 2-4).
|
|
15
|
+
*
|
|
16
|
+
* Admission ladder (audit round 3): configuredApiAuthToken() (env token) ||
|
|
17
|
+
* service token file || first config.apiKeys entry, sent as
|
|
18
|
+
* `x-opencodex-api-key` — never Authorization (gateway-cache.ts rule: an
|
|
19
|
+
* admission secret in a forwardable header is a forwarding hazard). Loopback
|
|
20
|
+
* binds require no token at all (resolveApiAuth admits loopback).
|
|
21
|
+
*
|
|
22
|
+
* Known limitation (recorded in roadmap 170): a bindHost where 127.0.0.1
|
|
23
|
+
* does not answer cannot reach its own loopback — same latent limitation
|
|
24
|
+
* gateway-cache has.
|
|
25
|
+
*/
|
|
26
|
+
import type { OcxConfig } from "../types";
|
|
27
|
+
import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
|
|
28
|
+
import { redactSecretString } from "../lib/redact";
|
|
29
|
+
import { sidecarEnter } from "../lib/sidecar-tracker";
|
|
30
|
+
import { configuredApiAuthToken, configuredPort } from "../server/auth-cors";
|
|
31
|
+
import { loadServiceTokenFromFile } from "../lib/service-secrets";
|
|
32
|
+
import type { DescribeOutcome, VisionSettings } from "./describe";
|
|
33
|
+
|
|
34
|
+
export const VISION_DESCRIBE_TERMINAL_HEADER = "x-opencodex-vision-describe";
|
|
35
|
+
|
|
36
|
+
const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]);
|
|
37
|
+
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
38
|
+
/** Bound the loopback JSON response; descriptions are clamped to ~2k chars by the caller anyway. */
|
|
39
|
+
const MAX_ROUTED_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
40
|
+
|
|
41
|
+
const DESCRIBE_INSTRUCTION =
|
|
42
|
+
"You are a vision describer for a text-only model that cannot see the image. Describe the image "
|
|
43
|
+
+ "thoroughly and factually so that model can fully reason about it: transcribe any visible text "
|
|
44
|
+
+ "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on "
|
|
45
|
+
+ "what's relevant to the user's request. Output only the description.";
|
|
46
|
+
|
|
47
|
+
function validateImageUrl(url: string): string | null {
|
|
48
|
+
if (url.startsWith("data:")) {
|
|
49
|
+
const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url);
|
|
50
|
+
if (!match) return "malformed data URL";
|
|
51
|
+
const mime = match[1].toLowerCase();
|
|
52
|
+
if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`;
|
|
53
|
+
if (match[2]) {
|
|
54
|
+
const bytes = Math.floor((match[3].length * 3) / 4);
|
|
55
|
+
if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`;
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
if (url.startsWith("https://")) return null;
|
|
60
|
+
return "unsupported image URL scheme (expected data: or https:)";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** The admission ladder: env token, service token file, first configured API key. */
|
|
64
|
+
export function routedDescribeAdmissionToken(config: Pick<OcxConfig, "apiKeys">): string | undefined {
|
|
65
|
+
const envToken = configuredApiAuthToken();
|
|
66
|
+
if (envToken) return envToken;
|
|
67
|
+
const fileToken = loadServiceTokenFromFile(process.env);
|
|
68
|
+
if (fileToken) return fileToken;
|
|
69
|
+
const first = config.apiKeys?.[0]?.key?.trim();
|
|
70
|
+
return first || undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Base URL seam for tests; production always self-fetches loopback. */
|
|
74
|
+
export function routedDescribeBaseUrl(config: Pick<OcxConfig, "port">): string {
|
|
75
|
+
// config.port can be 0 (ephemeral bind, tests) or stale after a live port
|
|
76
|
+
// override; the server records its ACTUAL bound port via setCorsOrigin at
|
|
77
|
+
// startup, so prefer that when config carries no positive port.
|
|
78
|
+
const port = config.port && config.port > 0 ? String(config.port) : configuredPort();
|
|
79
|
+
return `http://127.0.0.1:${port}`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function describeImageRouted(
|
|
83
|
+
imageUrl: string,
|
|
84
|
+
_detail: string | undefined,
|
|
85
|
+
contextText: string,
|
|
86
|
+
routedModel: string,
|
|
87
|
+
config: Pick<OcxConfig, "port" | "apiKeys">,
|
|
88
|
+
settings: VisionSettings,
|
|
89
|
+
abortSignal?: AbortSignal,
|
|
90
|
+
baseUrlOverride?: string,
|
|
91
|
+
): Promise<DescribeOutcome> {
|
|
92
|
+
const invalid = validateImageUrl(imageUrl);
|
|
93
|
+
if (invalid) return { text: "", error: invalid };
|
|
94
|
+
|
|
95
|
+
const headers: Record<string, string> = {
|
|
96
|
+
"Content-Type": "application/json",
|
|
97
|
+
[VISION_DESCRIBE_TERMINAL_HEADER]: "1",
|
|
98
|
+
};
|
|
99
|
+
const admission = routedDescribeAdmissionToken(config);
|
|
100
|
+
if (admission) headers["x-opencodex-api-key"] = admission;
|
|
101
|
+
|
|
102
|
+
const requestBody = {
|
|
103
|
+
model: routedModel,
|
|
104
|
+
stream: false,
|
|
105
|
+
messages: [
|
|
106
|
+
{ role: "system", content: DESCRIBE_INSTRUCTION },
|
|
107
|
+
{
|
|
108
|
+
role: "user",
|
|
109
|
+
content: [
|
|
110
|
+
...(contextText ? [{ type: "text", text: `User's request context: ${contextText}` }] : []),
|
|
111
|
+
{ type: "image_url", image_url: { url: imageUrl } },
|
|
112
|
+
],
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
|
|
118
|
+
const sidecarExit = sidecarEnter("vision");
|
|
119
|
+
const t0 = Date.now();
|
|
120
|
+
try {
|
|
121
|
+
const res = await fetch(`${baseUrlOverride ?? routedDescribeBaseUrl(config)}/v1/chat/completions`, {
|
|
122
|
+
method: "POST",
|
|
123
|
+
headers,
|
|
124
|
+
body: JSON.stringify(requestBody),
|
|
125
|
+
signal: linkedSignal.signal,
|
|
126
|
+
redirect: "manual",
|
|
127
|
+
});
|
|
128
|
+
const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
|
|
129
|
+
try {
|
|
130
|
+
const raw = await res.text();
|
|
131
|
+
if (raw.length > MAX_ROUTED_RESPONSE_BYTES) {
|
|
132
|
+
return { text: "", error: "routed describe response exceeded byte bound" };
|
|
133
|
+
}
|
|
134
|
+
if (!res.ok) {
|
|
135
|
+
return { text: "", error: `routed describe HTTP ${res.status}: ${redactSecretString(raw.slice(0, 200))}` };
|
|
136
|
+
}
|
|
137
|
+
let payload: unknown;
|
|
138
|
+
try { payload = JSON.parse(raw); } catch {
|
|
139
|
+
return { text: "", error: "routed describe returned non-JSON" };
|
|
140
|
+
}
|
|
141
|
+
const content = extractChatContent(payload);
|
|
142
|
+
if (!content) return { text: "", error: "routed describe returned no text" };
|
|
143
|
+
return { text: content };
|
|
144
|
+
} finally {
|
|
145
|
+
detachBodyGuard();
|
|
146
|
+
}
|
|
147
|
+
} catch (e) {
|
|
148
|
+
const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
|
|
149
|
+
console.warn(`[vision] routed describe ${kind} (${Date.now() - t0}ms)`);
|
|
150
|
+
return { text: "", error: redactSecretString(e instanceof Error ? e.message : String(e)) };
|
|
151
|
+
} finally {
|
|
152
|
+
sidecarExit();
|
|
153
|
+
linkedSignal.cleanup();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function extractChatContent(payload: unknown): string | undefined {
|
|
158
|
+
if (!payload || typeof payload !== "object") return undefined;
|
|
159
|
+
const choices = (payload as { choices?: unknown }).choices;
|
|
160
|
+
if (!Array.isArray(choices) || choices.length === 0) return undefined;
|
|
161
|
+
const message = (choices[0] as { message?: unknown })?.message;
|
|
162
|
+
if (!message || typeof message !== "object") return undefined;
|
|
163
|
+
const content = (message as { content?: unknown }).content;
|
|
164
|
+
if (typeof content === "string" && content.trim().length > 0) return content;
|
|
165
|
+
// Some adapters emit content parts; join text parts.
|
|
166
|
+
if (Array.isArray(content)) {
|
|
167
|
+
const joined = content
|
|
168
|
+
.map(part => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
|
|
169
|
+
? (part as { text: string }).text
|
|
170
|
+
: ""))
|
|
171
|
+
.join("");
|
|
172
|
+
if (joined.trim().length > 0) return joined;
|
|
173
|
+
}
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|