llm-cli-gateway 2.16.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/least-cost-routing/SKILL.md +123 -0
- package/CHANGELOG.md +28 -0
- package/dist/acp/client.js +5 -0
- package/dist/acp/flight-redaction.d.ts +2 -0
- package/dist/acp/flight-redaction.js +2 -0
- package/dist/acp/runtime.d.ts +1 -0
- package/dist/acp/runtime.js +20 -0
- package/dist/acp/types.d.ts +52 -0
- package/dist/acp/types.js +8 -0
- package/dist/api-provider.d.ts +1 -0
- package/dist/api-provider.js +1 -0
- package/dist/async-job-manager.d.ts +16 -0
- package/dist/async-job-manager.js +70 -22
- package/dist/claude-mcp-config.js +1 -1
- package/dist/compressor/transforms/ansi.js +12 -2
- package/dist/config.d.ts +31 -0
- package/dist/config.js +142 -7
- package/dist/db.js +4 -4
- package/dist/doctor.d.ts +34 -1
- package/dist/doctor.js +85 -3
- package/dist/executor.d.ts +5 -0
- package/dist/executor.js +11 -1
- package/dist/flight-recorder.d.ts +10 -0
- package/dist/flight-recorder.js +68 -2
- package/dist/http-transport.js +19 -21
- package/dist/index.d.ts +41 -1
- package/dist/index.js +658 -30
- package/dist/job-store.d.ts +10 -2
- package/dist/job-store.js +154 -43
- package/dist/lcr-priors.d.ts +60 -0
- package/dist/lcr-priors.js +190 -0
- package/dist/lcr-router-env.d.ts +20 -0
- package/dist/lcr-router-env.js +133 -0
- package/dist/lcr-telemetry.d.ts +2 -0
- package/dist/lcr-telemetry.js +17 -0
- package/dist/least-cost-router.d.ts +86 -0
- package/dist/least-cost-router.js +296 -0
- package/dist/least-cost-types.d.ts +34 -0
- package/dist/least-cost-types.js +1 -0
- package/dist/migrate-sessions.js +1 -1
- package/dist/migrate.js +1 -1
- package/dist/model-registry.js +1 -1
- package/dist/postgres-job-store-worker.js +56 -13
- package/dist/pricing.d.ts +17 -0
- package/dist/pricing.js +167 -0
- package/dist/provider-definitions.d.ts +4 -0
- package/dist/provider-definitions.js +28 -0
- package/dist/request-helpers.d.ts +2 -2
- package/dist/resources.d.ts +37 -2
- package/dist/resources.js +96 -1
- package/dist/retry.d.ts +1 -0
- package/dist/retry.js +1 -1
- package/dist/token-estimator.d.ts +6 -0
- package/dist/token-estimator.js +59 -0
- package/dist/upstream-contracts.js +1 -1
- package/dist/validation-receipt.js +1 -1
- package/dist/validation-tools.d.ts +6 -0
- package/dist/validation-tools.js +174 -54
- package/npm-shrinkwrap.json +2 -2
- package/package.json +9 -4
- package/setup/status.schema.json +68 -0
package/dist/pricing.js
CHANGED
|
@@ -116,3 +116,170 @@ export function estimateCacheSavingsUsd(cli, model, cacheReadTokens) {
|
|
|
116
116
|
const savedPerToken = (p.inputUsd * (1 - p.cacheReadMultiplier)) / 1_000_000;
|
|
117
117
|
return cacheReadTokens * savedPerToken;
|
|
118
118
|
}
|
|
119
|
+
export function modelIdToFamily(model) {
|
|
120
|
+
const lower = model.toLowerCase();
|
|
121
|
+
if (lower.includes("sonnet"))
|
|
122
|
+
return "claude-sonnet";
|
|
123
|
+
if (lower.includes("opus"))
|
|
124
|
+
return "claude-opus";
|
|
125
|
+
if (lower.includes("haiku"))
|
|
126
|
+
return "claude-haiku";
|
|
127
|
+
if (lower.includes("gpt-5") || lower.includes("o3"))
|
|
128
|
+
return "openai-gpt5";
|
|
129
|
+
const geminiSpecialty = lower.includes("lite") ||
|
|
130
|
+
lower.includes("image") ||
|
|
131
|
+
lower.includes("audio") ||
|
|
132
|
+
lower.includes("tts");
|
|
133
|
+
if (!geminiSpecialty) {
|
|
134
|
+
if (lower.includes("2.5-flash"))
|
|
135
|
+
return "gemini-2.5-flash";
|
|
136
|
+
if (lower.includes("2.5-pro"))
|
|
137
|
+
return "gemini-2.5-pro";
|
|
138
|
+
if (lower.includes("gemini-3") && lower.includes("pro"))
|
|
139
|
+
return "gemini-3-pro";
|
|
140
|
+
}
|
|
141
|
+
if (lower.includes("grok-build") || lower.includes("grok-code"))
|
|
142
|
+
return "grok-build";
|
|
143
|
+
if (lower.includes("grok-4") || lower.includes("grok-3") || lower.includes("grok-latest")) {
|
|
144
|
+
return "grok-4";
|
|
145
|
+
}
|
|
146
|
+
if (lower.includes("devstral-small"))
|
|
147
|
+
return "mistral-devstral";
|
|
148
|
+
if (lower.includes("mistral-medium-3.5"))
|
|
149
|
+
return "mistral-medium";
|
|
150
|
+
return "unknown";
|
|
151
|
+
}
|
|
152
|
+
const FAMILY_PRICING = new Map([
|
|
153
|
+
["claude-sonnet", ANTHROPIC_SONNET],
|
|
154
|
+
["claude-opus", ANTHROPIC_OPUS],
|
|
155
|
+
["claude-haiku", ANTHROPIC_HAIKU],
|
|
156
|
+
["openai-gpt5", OPENAI_GPT5],
|
|
157
|
+
["gemini-2.5-pro", GEMINI_25_PRO],
|
|
158
|
+
["gemini-2.5-flash", GEMINI_FLASH],
|
|
159
|
+
["gemini-3-pro", GEMINI_3_PRO],
|
|
160
|
+
["grok-4", GROK_4],
|
|
161
|
+
["grok-build", GROK_BUILD],
|
|
162
|
+
["mistral-medium", MISTRAL_MEDIUM],
|
|
163
|
+
["mistral-devstral", MISTRAL_DEVSTRAL],
|
|
164
|
+
]);
|
|
165
|
+
export const API_CATALOG_AS_OF = "2026-07-11";
|
|
166
|
+
export const API_CATALOG = new Map([
|
|
167
|
+
[
|
|
168
|
+
"openai/gpt-5.5",
|
|
169
|
+
{
|
|
170
|
+
inputUsdPerMTok: 1.25,
|
|
171
|
+
outputUsdPerMTok: 10,
|
|
172
|
+
cacheReadMultiplier: 0.5,
|
|
173
|
+
family: "openai-gpt5",
|
|
174
|
+
},
|
|
175
|
+
],
|
|
176
|
+
[
|
|
177
|
+
"anthropic/claude-sonnet-4.5",
|
|
178
|
+
{
|
|
179
|
+
inputUsdPerMTok: 3,
|
|
180
|
+
outputUsdPerMTok: 15,
|
|
181
|
+
cacheReadMultiplier: 0.1,
|
|
182
|
+
accountingMode: "disjoint",
|
|
183
|
+
family: "claude-sonnet",
|
|
184
|
+
},
|
|
185
|
+
],
|
|
186
|
+
[
|
|
187
|
+
"google/gemini-2.5-flash",
|
|
188
|
+
{
|
|
189
|
+
inputUsdPerMTok: 0.3,
|
|
190
|
+
outputUsdPerMTok: 2.5,
|
|
191
|
+
cacheReadMultiplier: 0.1,
|
|
192
|
+
family: "gemini-2.5-flash",
|
|
193
|
+
},
|
|
194
|
+
],
|
|
195
|
+
[
|
|
196
|
+
"grok-build-0.1",
|
|
197
|
+
{ inputUsdPerMTok: 1, outputUsdPerMTok: 2, cacheReadMultiplier: 0.2, family: "grok-build" },
|
|
198
|
+
],
|
|
199
|
+
[
|
|
200
|
+
"grok-4",
|
|
201
|
+
{ inputUsdPerMTok: 1.25, outputUsdPerMTok: 2.5, cacheReadMultiplier: 0.16, family: "grok-4" },
|
|
202
|
+
],
|
|
203
|
+
]);
|
|
204
|
+
function tableModelCost(family, rate) {
|
|
205
|
+
const accountingMode = family.startsWith("claude-") ? "disjoint" : "inclusive";
|
|
206
|
+
return {
|
|
207
|
+
inputUsdPerMTok: rate.inputUsd,
|
|
208
|
+
outputUsdPerMTok: rate.outputUsd,
|
|
209
|
+
cacheReadMultiplier: rate.cacheReadMultiplier,
|
|
210
|
+
cacheWriteUsdPerMTok: rate.inputUsd,
|
|
211
|
+
accountingMode,
|
|
212
|
+
family,
|
|
213
|
+
source: "table",
|
|
214
|
+
asOf: PRICING_AS_OF,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
function catalogModelCost(model, entry) {
|
|
218
|
+
const family = entry.family ?? modelIdToFamily(model);
|
|
219
|
+
return {
|
|
220
|
+
inputUsdPerMTok: entry.inputUsdPerMTok,
|
|
221
|
+
outputUsdPerMTok: entry.outputUsdPerMTok,
|
|
222
|
+
cacheReadMultiplier: entry.cacheReadMultiplier,
|
|
223
|
+
cacheWriteUsdPerMTok: entry.inputUsdPerMTok,
|
|
224
|
+
accountingMode: entry.accountingMode ?? "inclusive",
|
|
225
|
+
family: family === "unknown" ? model.toLowerCase() : family,
|
|
226
|
+
source: "api-catalog",
|
|
227
|
+
asOf: API_CATALOG_AS_OF,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const UNKNOWN_MODEL_COST = {
|
|
231
|
+
inputUsdPerMTok: 0,
|
|
232
|
+
outputUsdPerMTok: 0,
|
|
233
|
+
cacheReadMultiplier: 0,
|
|
234
|
+
cacheWriteUsdPerMTok: 0,
|
|
235
|
+
accountingMode: "inclusive",
|
|
236
|
+
family: "unknown",
|
|
237
|
+
source: "unknown",
|
|
238
|
+
asOf: PRICING_AS_OF,
|
|
239
|
+
};
|
|
240
|
+
export function getModelCost(provider, model, opts) {
|
|
241
|
+
const catalog = opts?.catalog ?? API_CATALOG;
|
|
242
|
+
const preferCatalog = opts?.preferCatalog ?? true;
|
|
243
|
+
const catalogEntry = catalog.get(model.toLowerCase());
|
|
244
|
+
const family = modelIdToFamily(model);
|
|
245
|
+
const familyRate = FAMILY_PRICING.get(family);
|
|
246
|
+
if (catalogEntry !== undefined && (preferCatalog || familyRate === undefined)) {
|
|
247
|
+
return catalogModelCost(model, catalogEntry);
|
|
248
|
+
}
|
|
249
|
+
if (familyRate !== undefined) {
|
|
250
|
+
return tableModelCost(family, familyRate);
|
|
251
|
+
}
|
|
252
|
+
return { ...UNKNOWN_MODEL_COST };
|
|
253
|
+
}
|
|
254
|
+
function composeTotalUsd(inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens, modelCost, inputIsWholePrompt) {
|
|
255
|
+
const M = 1_000_000;
|
|
256
|
+
const { inputUsdPerMTok, outputUsdPerMTok, cacheWriteUsdPerMTok, cacheReadMultiplier } = modelCost;
|
|
257
|
+
const outputCost = ((outputTokens + reasoningTokens) * outputUsdPerMTok) / M;
|
|
258
|
+
const cacheWriteCost = (cacheWriteTokens * cacheWriteUsdPerMTok) / M;
|
|
259
|
+
if (modelCost.accountingMode === "disjoint") {
|
|
260
|
+
const freshTokens = inputIsWholePrompt
|
|
261
|
+
? inputTokens - cacheReadTokens - cacheWriteTokens
|
|
262
|
+
: inputTokens;
|
|
263
|
+
const freshCost = (freshTokens * inputUsdPerMTok) / M;
|
|
264
|
+
const cacheReadCost = (cacheReadTokens * inputUsdPerMTok * cacheReadMultiplier) / M;
|
|
265
|
+
return freshCost + cacheWriteCost + cacheReadCost + outputCost;
|
|
266
|
+
}
|
|
267
|
+
const baseCost = (inputTokens * inputUsdPerMTok) / M;
|
|
268
|
+
const cacheReadDiscount = (cacheReadTokens * inputUsdPerMTok * (1 - cacheReadMultiplier)) / M;
|
|
269
|
+
return baseCost + cacheWriteCost + outputCost - cacheReadDiscount;
|
|
270
|
+
}
|
|
271
|
+
export function composeCost(counts, estimate, modelCost) {
|
|
272
|
+
if (counts?.reportedCostUsd != null) {
|
|
273
|
+
return {
|
|
274
|
+
costUsd: counts.reportedCostUsd,
|
|
275
|
+
cost_basis: "provider-reported",
|
|
276
|
+
confidence: "high",
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
if (counts != null && modelCost.source !== "unknown") {
|
|
280
|
+
const costUsd = composeTotalUsd(counts.inputTokens, counts.outputTokens, counts.cacheReadTokens ?? 0, counts.cacheCreationTokens ?? 0, counts.reasoningTokens ?? 0, modelCost, false);
|
|
281
|
+
return { costUsd, cost_basis: "derived-from-tokens", confidence: "high" };
|
|
282
|
+
}
|
|
283
|
+
const costUsd = composeTotalUsd(estimate.estInputTokens, estimate.estOutputTokens, estimate.estCacheReadTokens ?? 0, estimate.estCacheWriteTokens ?? 0, 0, modelCost, true);
|
|
284
|
+
return { costUsd, cost_basis: "pre-flight-estimate", confidence: "low" };
|
|
285
|
+
}
|
|
@@ -97,6 +97,10 @@ export interface ProviderRequestSurface {
|
|
|
97
97
|
readonly acpCapable: boolean;
|
|
98
98
|
readonly syncToolName: string;
|
|
99
99
|
readonly asyncToolName: string;
|
|
100
|
+
readonly acceptsImages: boolean;
|
|
101
|
+
readonly acceptsAttachments: boolean;
|
|
102
|
+
readonly toolCalling: boolean;
|
|
103
|
+
readonly jsonSchema: boolean;
|
|
100
104
|
}
|
|
101
105
|
export interface ProviderDefinition {
|
|
102
106
|
readonly id: ProviderId;
|
|
@@ -28,6 +28,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
28
28
|
acpCapable: false,
|
|
29
29
|
syncToolName: "claude_request",
|
|
30
30
|
asyncToolName: "claude_request_async",
|
|
31
|
+
acceptsImages: false,
|
|
32
|
+
acceptsAttachments: false,
|
|
33
|
+
toolCalling: true,
|
|
34
|
+
jsonSchema: true,
|
|
31
35
|
},
|
|
32
36
|
docs: { primary: ["https://code.claude.com/docs/en/cli-reference"] },
|
|
33
37
|
discovery: {
|
|
@@ -127,6 +131,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
127
131
|
acpCapable: false,
|
|
128
132
|
syncToolName: "codex_request",
|
|
129
133
|
asyncToolName: "codex_request_async",
|
|
134
|
+
acceptsImages: true,
|
|
135
|
+
acceptsAttachments: true,
|
|
136
|
+
toolCalling: true,
|
|
137
|
+
jsonSchema: true,
|
|
130
138
|
},
|
|
131
139
|
docs: {
|
|
132
140
|
primary: [
|
|
@@ -264,6 +272,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
264
272
|
acpCapable: false,
|
|
265
273
|
syncToolName: "gemini_request",
|
|
266
274
|
asyncToolName: "gemini_request_async",
|
|
275
|
+
acceptsImages: false,
|
|
276
|
+
acceptsAttachments: false,
|
|
277
|
+
toolCalling: true,
|
|
278
|
+
jsonSchema: false,
|
|
267
279
|
},
|
|
268
280
|
docs: {
|
|
269
281
|
primary: [
|
|
@@ -363,6 +375,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
363
375
|
acpCapable: true,
|
|
364
376
|
syncToolName: "grok_request",
|
|
365
377
|
asyncToolName: "grok_request_async",
|
|
378
|
+
acceptsImages: false,
|
|
379
|
+
acceptsAttachments: false,
|
|
380
|
+
toolCalling: true,
|
|
381
|
+
jsonSchema: true,
|
|
366
382
|
},
|
|
367
383
|
docs: {
|
|
368
384
|
primary: [
|
|
@@ -470,6 +486,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
470
486
|
acpCapable: true,
|
|
471
487
|
syncToolName: "mistral_request",
|
|
472
488
|
asyncToolName: "mistral_request_async",
|
|
489
|
+
acceptsImages: false,
|
|
490
|
+
acceptsAttachments: false,
|
|
491
|
+
toolCalling: true,
|
|
492
|
+
jsonSchema: false,
|
|
473
493
|
},
|
|
474
494
|
docs: { primary: ["https://github.com/mistralai/mistral-vibe"] },
|
|
475
495
|
discovery: {
|
|
@@ -586,6 +606,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
586
606
|
acpCapable: true,
|
|
587
607
|
syncToolName: "devin_request",
|
|
588
608
|
asyncToolName: "devin_request_async",
|
|
609
|
+
acceptsImages: false,
|
|
610
|
+
acceptsAttachments: false,
|
|
611
|
+
toolCalling: true,
|
|
612
|
+
jsonSchema: false,
|
|
589
613
|
},
|
|
590
614
|
docs: { primary: ["https://docs.devin.ai/cli/reference/commands"] },
|
|
591
615
|
discovery: {
|
|
@@ -697,6 +721,10 @@ const PROVIDER_DEFINITIONS = {
|
|
|
697
721
|
acpCapable: true,
|
|
698
722
|
syncToolName: "cursor_request",
|
|
699
723
|
asyncToolName: "cursor_request_async",
|
|
724
|
+
acceptsImages: false,
|
|
725
|
+
acceptsAttachments: false,
|
|
726
|
+
toolCalling: true,
|
|
727
|
+
jsonSchema: false,
|
|
700
728
|
},
|
|
701
729
|
docs: { primary: ["https://docs.cursor.com/en/cli/overview"] },
|
|
702
730
|
discovery: {
|
|
@@ -229,8 +229,8 @@ export declare const CODEX_HIGH_IMPACT_PARAMS_SCHEMA: z.ZodObject<{
|
|
|
229
229
|
disable: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
230
230
|
}, "strip", z.ZodTypeAny, {
|
|
231
231
|
search?: boolean | undefined;
|
|
232
|
-
outputSchema?: string | Record<string, unknown> | undefined;
|
|
233
232
|
profile?: string | undefined;
|
|
233
|
+
outputSchema?: string | Record<string, unknown> | undefined;
|
|
234
234
|
configOverrides?: Record<string, string> | undefined;
|
|
235
235
|
ephemeral?: boolean | undefined;
|
|
236
236
|
images?: string[] | undefined;
|
|
@@ -240,8 +240,8 @@ export declare const CODEX_HIGH_IMPACT_PARAMS_SCHEMA: z.ZodObject<{
|
|
|
240
240
|
disable?: string[] | undefined;
|
|
241
241
|
}, {
|
|
242
242
|
search?: boolean | undefined;
|
|
243
|
-
outputSchema?: string | Record<string, unknown> | undefined;
|
|
244
243
|
profile?: string | undefined;
|
|
244
|
+
outputSchema?: string | Record<string, unknown> | undefined;
|
|
245
245
|
configOverrides?: Record<string, string> | undefined;
|
|
246
246
|
ephemeral?: boolean | undefined;
|
|
247
247
|
images?: string[] | undefined;
|
package/dist/resources.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { type CliType } from "./session-manager.js";
|
|
|
3
3
|
import { PerformanceMetrics } from "./metrics.js";
|
|
4
4
|
import { FlightRecorderQuery } from "./flight-recorder.js";
|
|
5
5
|
import { type GlobalCacheStats, type PrefixCacheStats, type SessionCacheStats } from "./cache-stats.js";
|
|
6
|
-
import { type CacheAwarenessConfig, type ProvidersConfig } from "./config.js";
|
|
6
|
+
import { type CacheAwarenessConfig, type LeastCostConfig, type ProvidersConfig } from "./config.js";
|
|
7
7
|
import type { AcpConfig } from "./config.js";
|
|
8
8
|
import { type ResolvedProviderCapability } from "./provider-capability-resolver.js";
|
|
9
9
|
export interface ResourceDefinition {
|
|
@@ -31,7 +31,9 @@ export declare class ResourceProvider {
|
|
|
31
31
|
private providers;
|
|
32
32
|
private capabilityPeek;
|
|
33
33
|
private acpConfig;
|
|
34
|
-
|
|
34
|
+
private leastCost;
|
|
35
|
+
constructor(sessionManager: ISessionManager, performanceMetrics: PerformanceMetrics, flightRecorder?: FlightRecorderQuery, cacheAwareness?: CacheAwarenessConfig | null, providers?: ProvidersConfig | null, capabilityPeek?: (id: CliType) => ResolvedProviderCapability | null, acpConfig?: AcpConfig | null, leastCost?: LeastCostConfig | undefined);
|
|
36
|
+
private routingResourcesEnabled;
|
|
35
37
|
getFlightRecorderQuery(): FlightRecorderQuery;
|
|
36
38
|
private apiRuntimes;
|
|
37
39
|
private continuityTrackedApiRuntimes;
|
|
@@ -42,7 +44,40 @@ export declare class ResourceProvider {
|
|
|
42
44
|
readCacheStateSession(sessionId: string): SessionCacheStats;
|
|
43
45
|
readCacheStateForPrefix(stablePrefixHash: string): PrefixCacheStats;
|
|
44
46
|
listResources(): ResourceDefinition[];
|
|
47
|
+
private readRoutingDecisions;
|
|
48
|
+
private readRoutingPriors;
|
|
45
49
|
private ownedSessions;
|
|
46
50
|
private ownedActiveId;
|
|
47
51
|
readResource(uri: string): Promise<ResourceContents | null>;
|
|
48
52
|
}
|
|
53
|
+
export interface RoutingDecision {
|
|
54
|
+
provider: string;
|
|
55
|
+
model: string;
|
|
56
|
+
tier: string;
|
|
57
|
+
estCostUsd: number | null;
|
|
58
|
+
costBasis: string | null;
|
|
59
|
+
confidence: string | null;
|
|
60
|
+
reason: string | null;
|
|
61
|
+
considered: number | null;
|
|
62
|
+
reroutes: number | null;
|
|
63
|
+
at: string;
|
|
64
|
+
}
|
|
65
|
+
export interface RoutingPriorsPayload {
|
|
66
|
+
priorsScope: string;
|
|
67
|
+
priceAsOf: {
|
|
68
|
+
table: string;
|
|
69
|
+
apiCatalog: string;
|
|
70
|
+
};
|
|
71
|
+
outputPriors: Array<{
|
|
72
|
+
candidate: string;
|
|
73
|
+
median: number;
|
|
74
|
+
p90: number;
|
|
75
|
+
samples: number;
|
|
76
|
+
}>;
|
|
77
|
+
calibration: Array<{
|
|
78
|
+
bucket: string;
|
|
79
|
+
k: number;
|
|
80
|
+
samples: number;
|
|
81
|
+
confidence: string;
|
|
82
|
+
}>;
|
|
83
|
+
}
|
package/dist/resources.js
CHANGED
|
@@ -4,6 +4,9 @@ import { getRequestContext, principalCanAccess, resolveOwnerPrincipal } from "./
|
|
|
4
4
|
import { getAvailableCliInfo } from "./model-registry.js";
|
|
5
5
|
import { computeGlobalCacheStats, computePrefixCacheStats, computeSessionCacheStats, computeTtlRemaining, } from "./cache-stats.js";
|
|
6
6
|
import { enabledApiProviders, } from "./config.js";
|
|
7
|
+
import { computeLcrPriorsFromDb } from "./lcr-priors.js";
|
|
8
|
+
import { telemetryTierFor } from "./lcr-telemetry.js";
|
|
9
|
+
import { PRICING_AS_OF, API_CATALOG_AS_OF } from "./pricing.js";
|
|
7
10
|
import { apiContinuityForKind } from "./api-provider.js";
|
|
8
11
|
import { apiProviderCatalogEntry } from "./api-request.js";
|
|
9
12
|
import { buildProviderSubcommandsCompactCatalog, getCliSubcommandContract, serializeCliSubcommandContract, } from "./upstream-contracts.js";
|
|
@@ -20,7 +23,8 @@ export class ResourceProvider {
|
|
|
20
23
|
providers;
|
|
21
24
|
capabilityPeek;
|
|
22
25
|
acpConfig;
|
|
23
|
-
|
|
26
|
+
leastCost;
|
|
27
|
+
constructor(sessionManager, performanceMetrics, flightRecorder = { queryRequests: () => [] }, cacheAwareness = null, providers = null, capabilityPeek = peekProviderCapabilitySet, acpConfig = null, leastCost = undefined) {
|
|
24
28
|
this.sessionManager = sessionManager;
|
|
25
29
|
this.performanceMetrics = performanceMetrics;
|
|
26
30
|
this.flightRecorder = flightRecorder;
|
|
@@ -28,6 +32,10 @@ export class ResourceProvider {
|
|
|
28
32
|
this.providers = providers;
|
|
29
33
|
this.capabilityPeek = capabilityPeek;
|
|
30
34
|
this.acpConfig = acpConfig;
|
|
35
|
+
this.leastCost = leastCost;
|
|
36
|
+
}
|
|
37
|
+
routingResourcesEnabled() {
|
|
38
|
+
return this.leastCost?.enabled === true;
|
|
31
39
|
}
|
|
32
40
|
getFlightRecorderQuery() {
|
|
33
41
|
return this.flightRecorder;
|
|
@@ -172,8 +180,79 @@ export class ResourceProvider {
|
|
|
172
180
|
priority: 0.7,
|
|
173
181
|
},
|
|
174
182
|
})),
|
|
183
|
+
...(this.routingResourcesEnabled()
|
|
184
|
+
? [
|
|
185
|
+
{
|
|
186
|
+
uri: "routing://decisions",
|
|
187
|
+
name: "Routing Decisions",
|
|
188
|
+
title: "Least-Cost Routing Decisions",
|
|
189
|
+
description: "Recent redacted least-cost routing decisions (provider/model/tier/est cost/confidence)",
|
|
190
|
+
mimeType: "application/json",
|
|
191
|
+
annotations: {
|
|
192
|
+
audience: ["user", "assistant"],
|
|
193
|
+
priority: 0.7,
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
uri: "routing://priors",
|
|
198
|
+
name: "Routing Priors",
|
|
199
|
+
title: "Least-Cost Routing Priors",
|
|
200
|
+
description: "Learned output-token priors + input-token calibration (k, samples, quality) + price asOf",
|
|
201
|
+
mimeType: "application/json",
|
|
202
|
+
annotations: {
|
|
203
|
+
audience: ["user", "assistant"],
|
|
204
|
+
priority: 0.7,
|
|
205
|
+
},
|
|
206
|
+
},
|
|
207
|
+
]
|
|
208
|
+
: []),
|
|
175
209
|
];
|
|
176
210
|
}
|
|
211
|
+
readRoutingDecisions() {
|
|
212
|
+
const rows = this.flightRecorder.queryRequests(`SELECT r.cli, r.model, r.datetime_utc, r.cost_basis,
|
|
213
|
+
m.route_est_cost_usd, m.route_est_confidence, m.route_reason,
|
|
214
|
+
m.route_considered, m.route_reroutes
|
|
215
|
+
FROM requests r
|
|
216
|
+
LEFT JOIN gateway_metadata m ON m.request_id = r.id
|
|
217
|
+
WHERE m.routed = 1
|
|
218
|
+
ORDER BY r.datetime_utc DESC
|
|
219
|
+
LIMIT 50`);
|
|
220
|
+
return rows.map(row => ({
|
|
221
|
+
provider: row.cli,
|
|
222
|
+
model: row.model,
|
|
223
|
+
tier: telemetryTierFor(row.cli),
|
|
224
|
+
estCostUsd: row.route_est_cost_usd ?? null,
|
|
225
|
+
costBasis: row.cost_basis ?? null,
|
|
226
|
+
confidence: row.route_est_confidence ?? null,
|
|
227
|
+
reason: row.route_reason ?? null,
|
|
228
|
+
considered: row.route_considered ?? null,
|
|
229
|
+
reroutes: row.route_reroutes ?? null,
|
|
230
|
+
at: row.datetime_utc,
|
|
231
|
+
}));
|
|
232
|
+
}
|
|
233
|
+
readRoutingPriors() {
|
|
234
|
+
const scope = this.leastCost?.priorsScope ?? "off";
|
|
235
|
+
const priors = computeLcrPriorsFromDb(this.flightRecorder, {
|
|
236
|
+
priorsScope: scope,
|
|
237
|
+
ownerPrincipal: resolveOwnerPrincipal(getRequestContext()),
|
|
238
|
+
});
|
|
239
|
+
return {
|
|
240
|
+
priorsScope: scope,
|
|
241
|
+
priceAsOf: { table: PRICING_AS_OF, apiCatalog: API_CATALOG_AS_OF },
|
|
242
|
+
outputPriors: Array.from(priors.outputPriors, ([key, prior]) => ({
|
|
243
|
+
candidate: key,
|
|
244
|
+
median: prior.median,
|
|
245
|
+
p90: prior.p90,
|
|
246
|
+
samples: prior.samples,
|
|
247
|
+
})),
|
|
248
|
+
calibration: Array.from(priors.calibration, ([key, bucket]) => ({
|
|
249
|
+
bucket: key,
|
|
250
|
+
k: bucket.k,
|
|
251
|
+
samples: bucket.samples,
|
|
252
|
+
confidence: bucket.confidence,
|
|
253
|
+
})),
|
|
254
|
+
};
|
|
255
|
+
}
|
|
177
256
|
ownedSessions(sessions) {
|
|
178
257
|
const caller = resolveOwnerPrincipal(getRequestContext());
|
|
179
258
|
const owned = sessions.filter(s => principalCanAccess(s.ownerPrincipal, caller));
|
|
@@ -256,6 +335,22 @@ export class ResourceProvider {
|
|
|
256
335
|
};
|
|
257
336
|
}
|
|
258
337
|
}
|
|
338
|
+
if (this.routingResourcesEnabled()) {
|
|
339
|
+
if (uri === "routing://decisions") {
|
|
340
|
+
return {
|
|
341
|
+
uri,
|
|
342
|
+
mimeType: "application/json",
|
|
343
|
+
text: JSON.stringify({ decisions: this.readRoutingDecisions() }, null, 2),
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
if (uri === "routing://priors") {
|
|
347
|
+
return {
|
|
348
|
+
uri,
|
|
349
|
+
mimeType: "application/json",
|
|
350
|
+
text: JSON.stringify(this.readRoutingPriors(), null, 2),
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
}
|
|
259
354
|
if (uri === "metrics://performance") {
|
|
260
355
|
return {
|
|
261
356
|
uri,
|
package/dist/retry.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ export interface RetryOptions {
|
|
|
19
19
|
isTransient: (error: any) => boolean;
|
|
20
20
|
onRetry: (error: any, attempt: number, delay: number) => void;
|
|
21
21
|
}
|
|
22
|
+
export declare const isDefaultTransient: (error: any) => boolean;
|
|
22
23
|
export declare function createCircuitBreaker(options?: {
|
|
23
24
|
resetTimeout?: number;
|
|
24
25
|
failureThreshold?: number;
|
package/dist/retry.js
CHANGED
|
@@ -4,7 +4,7 @@ export var CircuitBreakerState;
|
|
|
4
4
|
CircuitBreakerState["OPEN"] = "OPEN";
|
|
5
5
|
CircuitBreakerState["HALF_OPEN"] = "HALF_OPEN";
|
|
6
6
|
})(CircuitBreakerState || (CircuitBreakerState = {}));
|
|
7
|
-
const isDefaultTransient = (error) => {
|
|
7
|
+
export const isDefaultTransient = (error) => {
|
|
8
8
|
if (!error) {
|
|
9
9
|
return false;
|
|
10
10
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
const CJK_RE = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef\uac00-\ud7af]/g;
|
|
2
|
+
const CODE_SYMBOL_RE = /[{}[\]()<>;=+\-*/\\|&%$#@`~]/g;
|
|
3
|
+
const CODE_KEYWORD_RE = /\b(function|const|let|var|return|import|export|class|def|public|private|static|void|for|while|switch|case)\b/;
|
|
4
|
+
const DIVISOR_BY_TYPE = new Map([
|
|
5
|
+
["prose", 4],
|
|
6
|
+
["code", 3],
|
|
7
|
+
["cjk", 1.5],
|
|
8
|
+
]);
|
|
9
|
+
const FAMILY_MULTIPLIERS = new Map([
|
|
10
|
+
["openai", 1.0],
|
|
11
|
+
["o200k", 1.0],
|
|
12
|
+
["cl100k", 1.02],
|
|
13
|
+
["claude", 1.08],
|
|
14
|
+
["gemini", 0.98],
|
|
15
|
+
["sentencepiece", 0.98],
|
|
16
|
+
["grok", 1.0],
|
|
17
|
+
["mistral", 1.05],
|
|
18
|
+
]);
|
|
19
|
+
export function classifyContent(text) {
|
|
20
|
+
if (!text)
|
|
21
|
+
return "prose";
|
|
22
|
+
const nonSpace = text.replace(/\s+/g, "").length;
|
|
23
|
+
if (nonSpace === 0)
|
|
24
|
+
return "prose";
|
|
25
|
+
const cjkMatches = text.match(CJK_RE);
|
|
26
|
+
const cjkCount = cjkMatches ? cjkMatches.length : 0;
|
|
27
|
+
if (cjkCount / nonSpace >= 0.2)
|
|
28
|
+
return "cjk";
|
|
29
|
+
const trimmed = text.trim();
|
|
30
|
+
const looksJson = /^[[{]/.test(trimmed) && /[}\]]/.test(trimmed) && /[:,]/.test(trimmed);
|
|
31
|
+
const symbolMatches = trimmed.match(CODE_SYMBOL_RE);
|
|
32
|
+
const symbolCount = symbolMatches ? symbolMatches.length : 0;
|
|
33
|
+
const symbolRatio = symbolCount / nonSpace;
|
|
34
|
+
const hasKeyword = CODE_KEYWORD_RE.test(trimmed);
|
|
35
|
+
if (looksJson || symbolRatio >= 0.08 || (hasKeyword && symbolRatio >= 0.03)) {
|
|
36
|
+
return "code";
|
|
37
|
+
}
|
|
38
|
+
return "prose";
|
|
39
|
+
}
|
|
40
|
+
function familyMultiplier(family) {
|
|
41
|
+
if (!family)
|
|
42
|
+
return 1;
|
|
43
|
+
const f = family.toLowerCase();
|
|
44
|
+
for (const [key, multiplier] of FAMILY_MULTIPLIERS) {
|
|
45
|
+
if (f === key || f.includes(key))
|
|
46
|
+
return multiplier;
|
|
47
|
+
}
|
|
48
|
+
return 1;
|
|
49
|
+
}
|
|
50
|
+
export function estimateInputTokens(text, opts) {
|
|
51
|
+
if (!text)
|
|
52
|
+
return 0;
|
|
53
|
+
const type = classifyContent(text);
|
|
54
|
+
const divisor = DIVISOR_BY_TYPE.get(type) ?? 4;
|
|
55
|
+
const base = text.length / divisor;
|
|
56
|
+
const familyMult = familyMultiplier(opts?.family);
|
|
57
|
+
const k = opts?.calibrationK ?? 1;
|
|
58
|
+
return Math.ceil(base * familyMult * k);
|
|
59
|
+
}
|
|
@@ -1691,7 +1691,7 @@ export const UPSTREAM_CLI_CONTRACTS = {
|
|
|
1691
1691
|
env: {
|
|
1692
1692
|
VIBE_ACTIVE_MODEL: {
|
|
1693
1693
|
arity: "one",
|
|
1694
|
-
pattern: /^[^\s\
|
|
1694
|
+
pattern: /^[^\s\p{Cc}]+$/u,
|
|
1695
1695
|
description: "Active model selector; Vibe uses env instead of a --model flag",
|
|
1696
1696
|
},
|
|
1697
1697
|
},
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { z } from "zod/v3";
|
|
2
2
|
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
3
|
import type { AsyncJobManager } from "./async-job-manager.js";
|
|
4
|
+
import { PerformanceMetrics } from "./metrics.js";
|
|
5
|
+
import { type LeastCostConfig } from "./config.js";
|
|
6
|
+
import type { FlightRecorderQuery } from "./flight-recorder.js";
|
|
4
7
|
import { type ValidationOrchestratorDeps } from "./validation-orchestrator.js";
|
|
5
8
|
export interface ValidationToolDeps extends ValidationOrchestratorDeps {
|
|
6
9
|
asyncJobManager: AsyncJobManager;
|
|
10
|
+
leastCost?: LeastCostConfig;
|
|
11
|
+
performanceMetrics?: PerformanceMetrics;
|
|
12
|
+
flightRecorder?: FlightRecorderQuery;
|
|
7
13
|
}
|
|
8
14
|
export declare function buildValidationSchemas(deps: ValidationToolDeps): {
|
|
9
15
|
providerSchema: z.ZodEnum<[string, ...string[]]>;
|