auto-model-router 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +24 -0
- package/.github/workflows/publish.yml +40 -0
- package/.omp-plugin/marketplace.json +30 -0
- package/LICENSE +21 -0
- package/README.md +639 -0
- package/bun.lock +32 -0
- package/docs/claude-anthropic-wire.md +116 -0
- package/omp-extension/configure-logic.ts +128 -0
- package/omp-extension/embed-logic.ts +141 -0
- package/omp-extension/router-configure.ts +111 -0
- package/omp-extension/router-embed.ts +118 -0
- package/omp-extension/router-toast.ts +130 -0
- package/omp-extension/toast-logic.ts +136 -0
- package/package.json +56 -0
- package/src/catalog/openrouter-catalog.ts +428 -0
- package/src/catalog/types.ts +104 -0
- package/src/cli/args.ts +105 -0
- package/src/cli/config-cmd.ts +362 -0
- package/src/cli/config-wizard.ts +636 -0
- package/src/cli/explain.ts +167 -0
- package/src/cli/models.ts +240 -0
- package/src/cli/stats.ts +69 -0
- package/src/config/defaults.ts +136 -0
- package/src/config/load.ts +143 -0
- package/src/config/omp-credentials.ts +124 -0
- package/src/config/schema.ts +161 -0
- package/src/config/types.ts +244 -0
- package/src/cost/blended.ts +80 -0
- package/src/cost/forecast.ts +129 -0
- package/src/cost/ledger.ts +291 -0
- package/src/cost/types.ts +148 -0
- package/src/index.ts +93 -0
- package/src/router/cache-control.ts +66 -0
- package/src/router/candidates.ts +246 -0
- package/src/router/classify.ts +329 -0
- package/src/router/escalate.ts +264 -0
- package/src/router/features.ts +225 -0
- package/src/router/index.ts +99 -0
- package/src/router/select.ts +365 -0
- package/src/router/state.ts +118 -0
- package/src/router/tier-plan.ts +151 -0
- package/src/router/types.ts +222 -0
- package/src/server/http.ts +343 -0
- package/src/server/turn.ts +393 -0
- package/src/tokens/estimate.ts +74 -0
- package/src/upstream/openrouter.ts +221 -0
- package/src/upstream/sse-parse.ts +208 -0
- package/src/upstream/types.ts +75 -0
- package/src/util/hash.ts +0 -0
- package/src/util/log.ts +53 -0
- package/src/util/sqlite.ts +140 -0
- package/src/util/sse.ts +23 -0
- package/src/wire/openai/errors.ts +48 -0
- package/src/wire/openai/models.ts +37 -0
- package/src/wire/openai/request.ts +279 -0
- package/src/wire/openai/sink.ts +213 -0
- package/src/wire/types.ts +156 -0
- package/test/catalog.test.ts +319 -0
- package/test/classify.test.ts +269 -0
- package/test/config-wizard.test.ts +482 -0
- package/test/config.test.ts +121 -0
- package/test/configure-logic.test.ts +151 -0
- package/test/cost.test.ts +137 -0
- package/test/embed-logic.test.ts +107 -0
- package/test/escalate.test.ts +223 -0
- package/test/failover.test.ts +494 -0
- package/test/features.test.ts +228 -0
- package/test/fixtures/openrouter-models.json +15340 -0
- package/test/models-yml.test.ts +186 -0
- package/test/omp-credentials.test.ts +185 -0
- package/test/select.test.ts +538 -0
- package/test/sse-parse.test.ts +142 -0
- package/test/tier-plan.test.ts +302 -0
- package/test/toast-logic.test.ts +160 -0
- package/test/tokens.test.ts +160 -0
- package/test/trust-attribution.test.ts +175 -0
- package/test/turn.test.ts +498 -0
- package/test/wire-request.test.ts +297 -0
- package/test/wire-sink.test.ts +179 -0
- package/tools/install.ts +140 -0
- package/tools/mock-openrouter.ts +269 -0
- package/tools/smoke.ts +326 -0
- package/tsconfig.json +23 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feature extraction: the cheap signals that drive classification.
|
|
3
|
+
*
|
|
4
|
+
* Everything here is a pure function of the normalized request. No tokenizer,
|
|
5
|
+
* no I/O, no model call — this runs on every agent turn, so the work is a few
|
|
6
|
+
* linear scans over the message list and at most one pass over the newest
|
|
7
|
+
* user-authored text.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { NormMessage, NormRequest } from "../wire/types.ts";
|
|
11
|
+
import type { Features } from "./types.ts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Complexity signals. Deliberately small: each hit pushes the turn toward a
|
|
15
|
+
* more expensive tier, so precision beats recall. Word-boundary matches,
|
|
16
|
+
* evaluated only against the newest user-authored content — never history.
|
|
17
|
+
*/
|
|
18
|
+
const COMPLEXITY_KEYWORDS: ReadonlyArray<readonly [string, RegExp]> = [
|
|
19
|
+
["architecture", /\barchitecture\b/i],
|
|
20
|
+
["refactor", /\brefactor\w*/i],
|
|
21
|
+
["debug", /\bdebug\w*/i],
|
|
22
|
+
// "race" (condition); inflected forms are too rare in agent traffic to matter.
|
|
23
|
+
["race", /\brace\b/i],
|
|
24
|
+
["deadlock", /\bdeadlock\w*/i],
|
|
25
|
+
// Causal "why" questions demand reasoning, not retrieval.
|
|
26
|
+
["why", /\bwhy\b/i],
|
|
27
|
+
["root cause", /\broot cause\b/i],
|
|
28
|
+
["design", /\bdesign\w*/i],
|
|
29
|
+
["optimize", /\boptimi[sz]\w*/i],
|
|
30
|
+
["security", /\bsecurity\b/i],
|
|
31
|
+
["migrate", /\bmigrat\w*/i],
|
|
32
|
+
["concurrency", /\bconcurren\w*/i],
|
|
33
|
+
["invariant", /\binvariant\w*/i],
|
|
34
|
+
["proof", /\bproof\w*/i],
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
/** Triviality signals: mechanical edits a small model cannot fumble. */
|
|
38
|
+
const TRIVIALITY_KEYWORDS: ReadonlyArray<readonly [string, RegExp]> = [
|
|
39
|
+
["rename", /\brename\w*/i],
|
|
40
|
+
["typo", /\btypos?\b/i],
|
|
41
|
+
["format", /\bformat\w*/i],
|
|
42
|
+
// Dependency/version bumps.
|
|
43
|
+
["bump", /\bbump\w*/i],
|
|
44
|
+
["comment", /\bcomments?\b/i],
|
|
45
|
+
["changelog", /\bchangelogs?\b/i],
|
|
46
|
+
["add a test", /\badd (?:a|an|some) tests?\b/i],
|
|
47
|
+
["lint", /\blint(?:ing|ed)?\b/i],
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Error markers scanned in tail tool results. Kept short and literal on
|
|
52
|
+
* purpose: a false positive escalates a turn that was actually fine, which
|
|
53
|
+
* costs real money; a false negative merely routes like a clean continuation.
|
|
54
|
+
*/
|
|
55
|
+
const TOOL_FAILURE_MARKERS: ReadonlyArray<RegExp> = [
|
|
56
|
+
// "error:"/"Error:" at a line start — the near-universal tool error prefix.
|
|
57
|
+
/(?:^|\n)\s*(?:error|Error):/,
|
|
58
|
+
// Python crash dump.
|
|
59
|
+
/Traceback \(most recent call last\)/,
|
|
60
|
+
// Shell: missing binary.
|
|
61
|
+
/command not found/,
|
|
62
|
+
// Non-zero process exit. "exit code 0" is success and never matches.
|
|
63
|
+
/exit(?:ed with)? code [1-9]\d*/i,
|
|
64
|
+
// GNU make: "make: *** [target] Error 2". "Error" sits mid-line so the
|
|
65
|
+
// prefix rule above misses it, and the "*** " literal is distinctive
|
|
66
|
+
// enough that successful builds never produce it.
|
|
67
|
+
/(?:^|\n)[^\n]*\*\*\* \[[^\]]*\] Error \d+/,
|
|
68
|
+
];
|
|
69
|
+
|
|
70
|
+
// Unified-diff headers. Plain "--- "/"+++ " are excluded: markdown rules and
|
|
71
|
+
// lists would false-positive. `diff --git`, hunks, and a/ b/ paths are real diffs.
|
|
72
|
+
const DIFF_RE = /(?:^|\n)(?:diff --git |@@ -\d|\+{3} b\/|-{3} a\/)/;
|
|
73
|
+
|
|
74
|
+
// A "terse instruction" is one short sentence; longer text carries real requirements.
|
|
75
|
+
const TERSE_MAX_BYTES = 128;
|
|
76
|
+
|
|
77
|
+
/** Index after the last message of the trailing run matching `pred`. */
|
|
78
|
+
function trailingRunStart(messages: NormMessage[], pred: (m: NormMessage) => boolean): number {
|
|
79
|
+
let i = messages.length - 1;
|
|
80
|
+
while (i >= 0) {
|
|
81
|
+
const m = messages[i];
|
|
82
|
+
if (m === undefined || !pred(m)) break;
|
|
83
|
+
i--;
|
|
84
|
+
}
|
|
85
|
+
return i + 1;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function extractFeatures(req: NormRequest, promptTokens: number): Features {
|
|
89
|
+
const messages = req.messages;
|
|
90
|
+
const tail = messages[messages.length - 1];
|
|
91
|
+
|
|
92
|
+
// The volatile tail: either the newest user-authored content (a trailing
|
|
93
|
+
// run of user messages — what the human just supplied) or, when the tail is
|
|
94
|
+
// tool output, a mechanical agent-loop continuation with no new user content.
|
|
95
|
+
const isToolResultContinuation = tail?.role === "tool";
|
|
96
|
+
let newContentBytes = 0;
|
|
97
|
+
let newestUserText = "";
|
|
98
|
+
if (isToolResultContinuation) {
|
|
99
|
+
const start = trailingRunStart(messages, (m) => m.role === "tool");
|
|
100
|
+
for (let i = start; i < messages.length; i++) newContentBytes += messages[i]?.textBytes ?? 0;
|
|
101
|
+
} else if (tail?.role === "user") {
|
|
102
|
+
const start = trailingRunStart(messages, (m) => m.role === "user");
|
|
103
|
+
const parts: string[] = [];
|
|
104
|
+
for (let i = start; i < messages.length; i++) {
|
|
105
|
+
const m = messages[i];
|
|
106
|
+
if (m === undefined) continue;
|
|
107
|
+
newContentBytes += m.textBytes;
|
|
108
|
+
parts.push(m.text);
|
|
109
|
+
}
|
|
110
|
+
newestUserText = parts.join("\n");
|
|
111
|
+
}
|
|
112
|
+
// Proportional share of the caller's prompt estimate, so this inherits
|
|
113
|
+
// whatever tokenizer calibration the estimate already applied.
|
|
114
|
+
const newContentTokens =
|
|
115
|
+
req.promptBytes > 0 && newContentBytes > 0
|
|
116
|
+
? Math.max(1, Math.round(promptTokens * (newContentBytes / req.promptBytes)))
|
|
117
|
+
: 0;
|
|
118
|
+
|
|
119
|
+
// Depth of the current agent loop: trailing tool results plus the assistant
|
|
120
|
+
// tool-call turns interleaved with them.
|
|
121
|
+
let toolLoopDepth = 0;
|
|
122
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
123
|
+
const m = messages[i];
|
|
124
|
+
if (m === undefined) break;
|
|
125
|
+
if (m.role === "tool" || (m.role === "assistant" && m.toolCalls.length > 0)) toolLoopDepth++;
|
|
126
|
+
else break;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let turnDepth = 0;
|
|
130
|
+
let toolSchemaBytes = 0;
|
|
131
|
+
const toolNames = new Set<string>();
|
|
132
|
+
for (const t of req.tools) toolSchemaBytes += t.schemaBytes;
|
|
133
|
+
for (const m of messages) {
|
|
134
|
+
if (m.role === "user" || m.role === "assistant") turnDepth++;
|
|
135
|
+
if (m.role === "assistant") for (const tc of m.toolCalls) toolNames.add(tc.name);
|
|
136
|
+
if (m.toolName !== undefined) toolNames.add(m.toolName);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// The last two assistant tool calls, in conversation order.
|
|
140
|
+
let lastName: string | null = null;
|
|
141
|
+
let lastArgs = "";
|
|
142
|
+
let prevName: string | null = null;
|
|
143
|
+
let prevArgs = "";
|
|
144
|
+
scanCalls: for (let i = messages.length - 1; i >= 0; i--) {
|
|
145
|
+
const m = messages[i];
|
|
146
|
+
if (m === undefined || m.role !== "assistant") continue;
|
|
147
|
+
for (let j = m.toolCalls.length - 1; j >= 0; j--) {
|
|
148
|
+
const tc = m.toolCalls[j];
|
|
149
|
+
if (tc === undefined) continue;
|
|
150
|
+
if (lastName === null) {
|
|
151
|
+
lastName = tc.name;
|
|
152
|
+
lastArgs = tc.argsJson;
|
|
153
|
+
} else {
|
|
154
|
+
prevName = tc.name;
|
|
155
|
+
prevArgs = tc.argsJson;
|
|
156
|
+
break scanCalls;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const repeatedToolCall = lastName !== null && lastName === prevName && lastArgs === prevArgs;
|
|
161
|
+
|
|
162
|
+
let lastToolFailed = false;
|
|
163
|
+
if (isToolResultContinuation) {
|
|
164
|
+
scanResults: for (let i = messages.length - 1; i >= 0; i--) {
|
|
165
|
+
const m = messages[i];
|
|
166
|
+
if (m === undefined || m.role !== "tool") break;
|
|
167
|
+
for (const re of TOOL_FAILURE_MARKERS) {
|
|
168
|
+
if (re.test(m.text)) {
|
|
169
|
+
lastToolFailed = true;
|
|
170
|
+
break scanResults;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Fenced code blocks: odd segments of a ``` split. Byte count includes the
|
|
177
|
+
// language tag line — close enough for a signal, and allocation-free per block.
|
|
178
|
+
let codeBlocks = 0;
|
|
179
|
+
let codeBytes = 0;
|
|
180
|
+
if (newestUserText.includes("```")) {
|
|
181
|
+
const parts = newestUserText.split("```");
|
|
182
|
+
codeBlocks = (parts.length - 1) >> 1;
|
|
183
|
+
for (let i = 1; i + 1 < parts.length; i += 2) codeBytes += Buffer.byteLength(parts[i] ?? "");
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let questionCount = 0;
|
|
187
|
+
for (let i = 0; i < newestUserText.length; i++) {
|
|
188
|
+
if (newestUserText.charCodeAt(i) === 63) questionCount++;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const complexityKeywords: string[] = [];
|
|
192
|
+
for (const [id, re] of COMPLEXITY_KEYWORDS) if (re.test(newestUserText)) complexityKeywords.push(id);
|
|
193
|
+
const trivialityKeywords: string[] = [];
|
|
194
|
+
for (const [id, re] of TRIVIALITY_KEYWORDS) if (re.test(newestUserText)) trivialityKeywords.push(id);
|
|
195
|
+
|
|
196
|
+
const trimmed = newestUserText.trim();
|
|
197
|
+
const terminators = trimmed.match(/[.!?]+(?:\s|$)/g);
|
|
198
|
+
const isTerseInstruction =
|
|
199
|
+
trimmed.length > 0 &&
|
|
200
|
+
Buffer.byteLength(trimmed) <= TERSE_MAX_BYTES &&
|
|
201
|
+
codeBlocks === 0 &&
|
|
202
|
+
(terminators === null ? 0 : terminators.length) <= 1;
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
promptTokens,
|
|
206
|
+
newContentTokens,
|
|
207
|
+
turnDepth,
|
|
208
|
+
toolCount: req.tools.length,
|
|
209
|
+
toolSchemaBytes,
|
|
210
|
+
isToolResultContinuation,
|
|
211
|
+
toolLoopDepth,
|
|
212
|
+
distinctToolsUsed: toolNames.size,
|
|
213
|
+
lastToolFailed,
|
|
214
|
+
repeatedToolCall,
|
|
215
|
+
hasImages: req.hasImages,
|
|
216
|
+
codeBlocks,
|
|
217
|
+
codeBytes,
|
|
218
|
+
looksLikeDiff: DIFF_RE.test(newestUserText),
|
|
219
|
+
complexityKeywords,
|
|
220
|
+
trivialityKeywords,
|
|
221
|
+
requestedReasoning: req.reasoning,
|
|
222
|
+
questionCount,
|
|
223
|
+
isTerseInstruction,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Router assembly: estimate -> features -> classify -> select.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately READ-ONLY with respect to conversation state. `server/turn.ts`
|
|
5
|
+
* owns every write, because only it knows the committed outcome (which model
|
|
6
|
+
* actually served, what it cost, whether the cache went warm). Routing that
|
|
7
|
+
* also persisted would race that save and lose the sticky window.
|
|
8
|
+
*
|
|
9
|
+
* The read-only property is what makes `auto-model-router explain` safe: it routes a
|
|
10
|
+
* real request without perturbing the conversation it belongs to.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { CatalogSource } from "../catalog/types.ts";
|
|
14
|
+
import type { ProfileConfig, RouterConfig } from "../config/types.ts";
|
|
15
|
+
import type { Ledger } from "../cost/types.ts";
|
|
16
|
+
import { estimatePromptTokens } from "../tokens/estimate.ts";
|
|
17
|
+
import type { UpstreamClient } from "../upstream/types.ts";
|
|
18
|
+
import type { NormRequest } from "../wire/types.ts";
|
|
19
|
+
import { classify, classifyTask } from "./classify.ts";
|
|
20
|
+
import { extractFeatures } from "./features.ts";
|
|
21
|
+
import { select } from "./select.ts";
|
|
22
|
+
import { TIER_ORDER, type Classification, type ConversationStore, type Decision, type Router, type Tier } from "./types.ts";
|
|
23
|
+
|
|
24
|
+
export interface RouterDeps {
|
|
25
|
+
config: RouterConfig;
|
|
26
|
+
catalog: CatalogSource;
|
|
27
|
+
ledger: Ledger;
|
|
28
|
+
conversations: ConversationStore;
|
|
29
|
+
upstream: UpstreamClient;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Tokenizer used for prompt estimation before a model is chosen.
|
|
34
|
+
*
|
|
35
|
+
* Chicken-and-egg: the estimate feeds candidate filtering, so it cannot depend
|
|
36
|
+
* on the winner. The conversation's previous model is the best available proxy,
|
|
37
|
+
* and family ratios differ by only a few percent anyway.
|
|
38
|
+
*/
|
|
39
|
+
const NEUTRAL_TOKENIZER = "gpt";
|
|
40
|
+
|
|
41
|
+
function resolveProfile(cfg: RouterConfig, requestedModel: string): ProfileConfig {
|
|
42
|
+
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
43
|
+
if (exact !== undefined) return exact;
|
|
44
|
+
const fallback = cfg.profiles[0];
|
|
45
|
+
if (fallback === undefined) throw new Error("no router profiles configured");
|
|
46
|
+
return fallback;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function createRouter(deps: RouterDeps): Router {
|
|
50
|
+
const { config, catalog, ledger, conversations, upstream } = deps;
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
async route(
|
|
54
|
+
req: NormRequest,
|
|
55
|
+
opts: { attempt: number; escalateFrom?: Tier; excludeSlugs?: readonly string[] },
|
|
56
|
+
): Promise<Decision> {
|
|
57
|
+
const state = conversations.get(req.conversationKey) ?? conversations.load(req.conversationKey);
|
|
58
|
+
const snapshot = await catalog.get();
|
|
59
|
+
|
|
60
|
+
const priorTokenizer =
|
|
61
|
+
state.currentSlug === null ? undefined : catalog.find(state.currentSlug)?.tokenizer;
|
|
62
|
+
const promptTokens = estimatePromptTokens(req, priorTokenizer ?? NEUTRAL_TOKENIZER, ledger);
|
|
63
|
+
const features = extractFeatures(req, promptTokens);
|
|
64
|
+
|
|
65
|
+
let classification: Classification;
|
|
66
|
+
if (opts.escalateFrom !== undefined) {
|
|
67
|
+
// An escalation is not a re-judgement: the probe already proved the
|
|
68
|
+
// cheaper tier failed, so force strictly upward rather than letting
|
|
69
|
+
// the classifier re-derive the same losing answer.
|
|
70
|
+
const nextIdx = Math.min(TIER_ORDER.indexOf(opts.escalateFrom) + 1, TIER_ORDER.length - 1);
|
|
71
|
+
const forced = TIER_ORDER[nextIdx];
|
|
72
|
+
if (forced === undefined) throw new Error(`unresolvable escalation tier from ${opts.escalateFrom}`);
|
|
73
|
+
classification = {
|
|
74
|
+
tier: forced,
|
|
75
|
+
task: classifyTask(features),
|
|
76
|
+
confidence: 1,
|
|
77
|
+
source: "escalation",
|
|
78
|
+
score: 1,
|
|
79
|
+
reasons: [`escalated from ${opts.escalateFrom} after attempt ${opts.attempt - 1} was rejected`],
|
|
80
|
+
};
|
|
81
|
+
} else {
|
|
82
|
+
classification = await classify(req, features, config, { upstream, ledger, catalog });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return select({
|
|
86
|
+
req,
|
|
87
|
+
features,
|
|
88
|
+
classification,
|
|
89
|
+
profile: resolveProfile(config, req.requestedModel),
|
|
90
|
+
state,
|
|
91
|
+
snapshot,
|
|
92
|
+
ledger,
|
|
93
|
+
cfg: config,
|
|
94
|
+
nowMs: Date.now(),
|
|
95
|
+
...(opts.excludeSlugs === undefined ? {} : { excludeSlugs: opts.excludeSlugs }),
|
|
96
|
+
});
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Selection: turn a classification into a concrete model decision.
|
|
3
|
+
*
|
|
4
|
+
* This is the economic core of the router: profile clamping, hysteresis,
|
|
5
|
+
* candidate widening, the cache-aware stay/switch arithmetic, the budget
|
|
6
|
+
* guard, fallbacks, cache breakpoints, probe planning, and capability clamps.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { CatalogSnapshot } from "../catalog/types.ts";
|
|
10
|
+
import type { ProfileConfig, RouterConfig } from "../config/types.ts";
|
|
11
|
+
import { priceAt } from "../cost/forecast.ts";
|
|
12
|
+
import type { Ledger } from "../cost/types.ts";
|
|
13
|
+
import type { NormRequest, ReasoningLevel } from "../wire/types.ts";
|
|
14
|
+
import { planCacheBreakpoints } from "./cache-control.ts";
|
|
15
|
+
import { buildCandidates } from "./candidates.ts";
|
|
16
|
+
import {
|
|
17
|
+
TIER_ORDER,
|
|
18
|
+
type Candidate,
|
|
19
|
+
type Classification,
|
|
20
|
+
type ConversationState,
|
|
21
|
+
type Decision,
|
|
22
|
+
type Features,
|
|
23
|
+
type ProbePlan,
|
|
24
|
+
type Rejection,
|
|
25
|
+
type Tier,
|
|
26
|
+
} from "./types.ts";
|
|
27
|
+
|
|
28
|
+
export interface SelectArgs {
|
|
29
|
+
req: NormRequest;
|
|
30
|
+
features: Features;
|
|
31
|
+
classification: Classification;
|
|
32
|
+
profile: ProfileConfig;
|
|
33
|
+
state: ConversationState;
|
|
34
|
+
snapshot: CatalogSnapshot;
|
|
35
|
+
ledger: Ledger | null;
|
|
36
|
+
cfg: RouterConfig;
|
|
37
|
+
nowMs: number;
|
|
38
|
+
/**
|
|
39
|
+
* Slugs that already failed on this turn. Passed straight to
|
|
40
|
+
* `buildCandidates` so a failover retry lands on a different model.
|
|
41
|
+
*/
|
|
42
|
+
excludeSlugs?: readonly string[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Thrown when the budget guard rejects a turn. `status`/`code` mirror the
|
|
47
|
+
* WireError shape so the server can render it as a 402-class response.
|
|
48
|
+
*/
|
|
49
|
+
export class BudgetExceededError extends Error {
|
|
50
|
+
readonly status = 402;
|
|
51
|
+
readonly code = "budget_exceeded";
|
|
52
|
+
constructor(message: string) {
|
|
53
|
+
super(message);
|
|
54
|
+
this.name = "BudgetExceededError";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Mid-range completion assumption for forecasts. Long generations amortize
|
|
59
|
+
// into prompt-dominated cost anyway; precision here does not move rankings.
|
|
60
|
+
const EXPECTED_COMPLETION_TOKENS = 1024;
|
|
61
|
+
const DAY_MS = 86_400_000;
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Authors known to accept replayed assistant reasoning over chat completions:
|
|
65
|
+
* Anthropic requires thinking-block replay for tool-use continuity, and Google
|
|
66
|
+
* requires thought signatures. Everything else gets reasoning stripped — a
|
|
67
|
+
* rejected replay costs a whole turn.
|
|
68
|
+
*/
|
|
69
|
+
const REASONING_REPLAY_AUTHORS: Record<string, true> = {
|
|
70
|
+
anthropic: true,
|
|
71
|
+
google: true,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
function tierIdx(t: Tier): number {
|
|
75
|
+
return TIER_ORDER.indexOf(t);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function tierAt(i: number): Tier | null {
|
|
79
|
+
const t = TIER_ORDER[i];
|
|
80
|
+
return t === undefined ? null : t;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Tier search order: the tier itself, then one up, one down, two up, ... within [minTier, maxTier]. */
|
|
84
|
+
function wideningOrder(tier: Tier, minTier: Tier, maxTier: Tier): Tier[] {
|
|
85
|
+
const lo = tierIdx(minTier);
|
|
86
|
+
const hi = tierIdx(maxTier);
|
|
87
|
+
const c = tierIdx(tier);
|
|
88
|
+
const out: Tier[] = [];
|
|
89
|
+
for (let d = 0; d <= Math.max(hi - c, c - lo); d++) {
|
|
90
|
+
const up = tierAt(c + d);
|
|
91
|
+
const down = tierAt(c - d);
|
|
92
|
+
if (d === 0) {
|
|
93
|
+
if (up !== null) out.push(up);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (up !== null && c + d <= hi) out.push(up);
|
|
97
|
+
if (down !== null && c - d >= lo) out.push(down);
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function select(args: SelectArgs): Decision {
|
|
103
|
+
const { req, features, classification, profile, state, snapshot, ledger, cfg, nowMs } = args;
|
|
104
|
+
const reasons: string[] = [];
|
|
105
|
+
const minI = tierIdx(profile.minTier);
|
|
106
|
+
const maxI = tierIdx(profile.maxTier);
|
|
107
|
+
const clampTier = (t: Tier): Tier => tierAt(Math.min(Math.max(tierIdx(t), minI), maxI)) ?? t;
|
|
108
|
+
|
|
109
|
+
// 1. Clamp the classified tier to the requesting profile's envelope.
|
|
110
|
+
let effective = clampTier(classification.tier);
|
|
111
|
+
if (effective !== classification.tier) {
|
|
112
|
+
reasons.push(`classified ${classification.tier}, clamped to profile ${profile.id} [${profile.minTier}..${profile.maxTier}] → ${effective}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// 2. Hysteresis: while the sticky window is open, never route below the
|
|
116
|
+
// held tier — per-turn flapping would repeatedly cold-start prompt caches.
|
|
117
|
+
let cls = classification;
|
|
118
|
+
if (state.stickyUntilTurn > state.turn && state.currentTier !== null && tierIdx(state.currentTier) >= tierIdx(effective)) {
|
|
119
|
+
const held = clampTier(state.currentTier);
|
|
120
|
+
if (held !== effective) {
|
|
121
|
+
reasons.push(`hysteresis: holding ${held} until turn ${state.stickyUntilTurn} (classified ${effective})`);
|
|
122
|
+
cls = {
|
|
123
|
+
...classification,
|
|
124
|
+
tier: held,
|
|
125
|
+
source: "sticky",
|
|
126
|
+
reasons: [`hysteresis hold ${held} until turn ${state.stickyUntilTurn}`, ...classification.reasons],
|
|
127
|
+
};
|
|
128
|
+
effective = held;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// Never downgrade more than maxDowngradePerTurn tiers in one turn, so
|
|
132
|
+
// quality never falls off a cliff on a single odd classification.
|
|
133
|
+
if (state.currentTier !== null) {
|
|
134
|
+
const floor = Math.max(tierIdx(state.currentTier) - cfg.hysteresis.maxDowngradePerTurn, minI);
|
|
135
|
+
if (tierIdx(effective) < floor) {
|
|
136
|
+
const clamped = tierAt(floor);
|
|
137
|
+
if (clamped !== null) {
|
|
138
|
+
reasons.push(`downgrade limited to ${cfg.hysteresis.maxDowngradePerTurn} tier(s)/turn: ${effective} → ${clamped}`);
|
|
139
|
+
effective = clamped;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// 3. Candidates for the effective tier; widen one tier upward, then
|
|
145
|
+
// downward, and only fail when the whole profile envelope is exhausted.
|
|
146
|
+
// The task type selects the quality axis and capability filters; the tier
|
|
147
|
+
// still bounds cost (task selects, tier budgets).
|
|
148
|
+
const warmSlug =
|
|
149
|
+
state.cacheWarmSlug !== null && nowMs - state.cacheWarmAtMs <= cfg.hysteresis.cacheWarmTtlMs
|
|
150
|
+
? state.cacheWarmSlug
|
|
151
|
+
: null;
|
|
152
|
+
const build = (t: Tier, relaxLevel = 0): { candidates: Candidate[]; rejected: Rejection[] } =>
|
|
153
|
+
buildCandidates({
|
|
154
|
+
req,
|
|
155
|
+
features,
|
|
156
|
+
tier: t,
|
|
157
|
+
task: classification.task,
|
|
158
|
+
snapshot,
|
|
159
|
+
ledger,
|
|
160
|
+
cfg,
|
|
161
|
+
expectedCompletionTokens: EXPECTED_COMPLETION_TOKENS,
|
|
162
|
+
warmSlug,
|
|
163
|
+
relaxLevel,
|
|
164
|
+
...(args.excludeSlugs === undefined ? {} : { excludeSlugs: args.excludeSlugs }),
|
|
165
|
+
});
|
|
166
|
+
let chosenTier = effective;
|
|
167
|
+
let built: { candidates: Candidate[]; rejected: Rejection[] } | null = null;
|
|
168
|
+
// Relax level the tier rescue used (0 = no rescue). The budget downgrade
|
|
169
|
+
// search must rebuild at the same level, or it re-applies the strict config
|
|
170
|
+
// that excluded every model and throws instead of downgrading.
|
|
171
|
+
let rescuedRelax = 0;
|
|
172
|
+
for (const t of wideningOrder(effective, profile.minTier, profile.maxTier)) {
|
|
173
|
+
const b = build(t);
|
|
174
|
+
if (b.candidates.length > 0) {
|
|
175
|
+
built = b;
|
|
176
|
+
chosenTier = t;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
reasons.push(`no candidates in ${t} (${b.rejected.length} rejected)`);
|
|
180
|
+
built ??= b;
|
|
181
|
+
}
|
|
182
|
+
// Tier rescue: the configured envelopes (price ceilings, quality floors,
|
|
183
|
+
// trust bar) were tuned against the full catalog, and a guardrail can shrink
|
|
184
|
+
// availability so no configured tier admits anything. Rather than 500, relax
|
|
185
|
+
// the tier's economic constraints — in order, price → quality → trust — until
|
|
186
|
+
// some AVAILABLE model qualifies. Hard capability filters (tools/images/
|
|
187
|
+
// context) and the key-scoped allowlist are never lifted.
|
|
188
|
+
if (built === null || built.candidates.length === 0) {
|
|
189
|
+
const envelope = wideningOrder(effective, profile.minTier, profile.maxTier);
|
|
190
|
+
let rescued = false;
|
|
191
|
+
for (let relax = 1; relax <= 3 && !rescued; relax++) {
|
|
192
|
+
for (const t of envelope) {
|
|
193
|
+
const b = build(t, relax);
|
|
194
|
+
if (b.candidates.length > 0) {
|
|
195
|
+
built = b;
|
|
196
|
+
chosenTier = t;
|
|
197
|
+
rescued = true;
|
|
198
|
+
rescuedRelax = relax;
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (rescued) {
|
|
204
|
+
const first = built!.candidates[0];
|
|
205
|
+
const label =
|
|
206
|
+
rescuedRelax === 1
|
|
207
|
+
? "price ceilings"
|
|
208
|
+
: rescuedRelax === 2
|
|
209
|
+
? "price ceilings + quality floors"
|
|
210
|
+
: "price ceilings + quality floors + trust bar";
|
|
211
|
+
reasons.push(
|
|
212
|
+
`tier rescue: strict config excluded all available models; relaxed ${label} to pick ${first!.model.slug} (${chosenTier})`,
|
|
213
|
+
);
|
|
214
|
+
} else if (built === null || built.candidates.length === 0) {
|
|
215
|
+
throw new Error(`no viable model: catalog exhausted across profile ${profile.id}`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (chosenTier !== effective) reasons.push(`widened ${effective} → ${chosenTier}`);
|
|
219
|
+
// After the rescue block, `built` is guaranteed non-null: the branch either
|
|
220
|
+
// rescued a non-empty candidate set, or threw the `catalog exhausted` error.
|
|
221
|
+
const resolved = built as { candidates: Candidate[]; rejected: Rejection[] };
|
|
222
|
+
let candidates = resolved.candidates;
|
|
223
|
+
const first = candidates[0];
|
|
224
|
+
if (first === undefined) throw new Error(`no viable model: catalog exhausted across profile ${profile.id}`);
|
|
225
|
+
let chosen = first;
|
|
226
|
+
|
|
227
|
+
// 4. Cache-aware switch decision. Staying prices the previous turn's prompt
|
|
228
|
+
// at the warm model's cache-read rate; switching prices the full current
|
|
229
|
+
// prompt at the new model's cold rate plus its cache-write premium (we
|
|
230
|
+
// assume the whole prompt is written). Switch only when the saving
|
|
231
|
+
// clears switchMargin.
|
|
232
|
+
let sticky = false;
|
|
233
|
+
if (warmSlug !== null && chosen.model.slug !== warmSlug) {
|
|
234
|
+
const warm = candidates.find((c) => c.model.slug === warmSlug);
|
|
235
|
+
if (warm !== undefined) {
|
|
236
|
+
const warmPrice = priceAt(warm.model, Math.max(1, state.lastPromptTokens));
|
|
237
|
+
const newPrice = priceAt(chosen.model, Math.max(1, features.promptTokens));
|
|
238
|
+
const stayCost = state.lastPromptTokens * (warmPrice.cacheRead ?? warmPrice.prompt);
|
|
239
|
+
const switchCost = features.promptTokens * (newPrice.prompt + (newPrice.cacheWrite ?? 0));
|
|
240
|
+
if (stayCost > switchCost * cfg.hysteresis.switchMargin) {
|
|
241
|
+
reasons.push(
|
|
242
|
+
`cache: switch ${warmSlug} → ${chosen.model.slug} (stay $${stayCost.toFixed(4)} > switch $${switchCost.toFixed(4)} × ${cfg.hysteresis.switchMargin})`,
|
|
243
|
+
);
|
|
244
|
+
} else {
|
|
245
|
+
chosen = warm;
|
|
246
|
+
sticky = true;
|
|
247
|
+
reasons.push(
|
|
248
|
+
`cache: keeping warm ${warmSlug} (stay $${stayCost.toFixed(4)} ≤ switch $${switchCost.toFixed(4)} × ${cfg.hysteresis.switchMargin})`,
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// 5. Budget guard, against the COLD forecast: a budget must survive a cache miss.
|
|
255
|
+
const budget = {
|
|
256
|
+
perTurnUsd: profile.budget?.perTurnUsd ?? cfg.budget.perTurnUsd,
|
|
257
|
+
perConversationUsd: profile.budget?.perConversationUsd ?? cfg.budget.perConversationUsd,
|
|
258
|
+
perDayUsd: profile.budget?.perDayUsd ?? cfg.budget.perDayUsd,
|
|
259
|
+
onExceeded: profile.budget?.onExceeded ?? cfg.budget.onExceeded,
|
|
260
|
+
};
|
|
261
|
+
const breach = (c: Candidate): string | null => {
|
|
262
|
+
if (budget.perTurnUsd !== undefined && c.forecast.coldUsd > budget.perTurnUsd) {
|
|
263
|
+
return `cold forecast $${c.forecast.coldUsd.toFixed(4)} > per-turn budget $${budget.perTurnUsd}`;
|
|
264
|
+
}
|
|
265
|
+
if (budget.perConversationUsd !== undefined && state.spentUsd + c.forecast.coldUsd > budget.perConversationUsd) {
|
|
266
|
+
return `conversation spend $${state.spentUsd.toFixed(4)} + cold forecast > per-conversation budget $${budget.perConversationUsd}`;
|
|
267
|
+
}
|
|
268
|
+
if (budget.perDayUsd !== undefined) {
|
|
269
|
+
// Scope the rolling 24h ceiling to the requesting harness when it
|
|
270
|
+
// identifies itself, so multiple harnesses sharing one router each get
|
|
271
|
+
// their own daily budget instead of one exhausting it for the others.
|
|
272
|
+
const daySpend = ledger?.spendSince(nowMs - DAY_MS, req.harnessId) ?? 0;
|
|
273
|
+
if (daySpend + c.forecast.coldUsd > budget.perDayUsd) {
|
|
274
|
+
return `24h spend $${daySpend.toFixed(4)} + cold forecast > per-day budget $${budget.perDayUsd}`;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return null;
|
|
278
|
+
};
|
|
279
|
+
let budgetDowngraded = false;
|
|
280
|
+
const why = breach(chosen);
|
|
281
|
+
if (why !== null) {
|
|
282
|
+
if (budget.onExceeded === "reject") throw new BudgetExceededError(why);
|
|
283
|
+
// Downgrade: the cheapest candidate in the cheapest tier that fits.
|
|
284
|
+
let rescue: { tier: Tier; candidate: Candidate; candidates: Candidate[] } | null = null;
|
|
285
|
+
for (const t of wideningOrder(profile.minTier, profile.minTier, profile.maxTier)) {
|
|
286
|
+
const b = build(t, rescuedRelax);
|
|
287
|
+
let cheapest: Candidate | null = null;
|
|
288
|
+
for (const c of b.candidates) {
|
|
289
|
+
if (cheapest === null || c.forecast.coldUsd < cheapest.forecast.coldUsd) cheapest = c;
|
|
290
|
+
}
|
|
291
|
+
if (cheapest !== null && breach(cheapest) === null) {
|
|
292
|
+
rescue = { tier: t, candidate: cheapest, candidates: b.candidates };
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (rescue === null) throw new BudgetExceededError(`${why}; no cheaper candidate fits the budget`);
|
|
297
|
+
reasons.push(`budget: ${why}; downgraded ${chosenTier} → ${rescue.tier} (${rescue.candidate.model.slug})`);
|
|
298
|
+
chosen = rescue.candidate;
|
|
299
|
+
chosenTier = rescue.tier;
|
|
300
|
+
candidates = rescue.candidates;
|
|
301
|
+
budgetDowngraded = true;
|
|
302
|
+
sticky = false;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// 6. Same-tier fallbacks for OpenRouter's transient-error `models[]` cascade.
|
|
306
|
+
const fallbacks: string[] = [];
|
|
307
|
+
for (const c of candidates) {
|
|
308
|
+
if (c.model.slug === chosen.model.slug) continue;
|
|
309
|
+
fallbacks.push(c.model.slug);
|
|
310
|
+
if (fallbacks.length >= 2) break;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// 7. Cache breakpoints.
|
|
314
|
+
const cacheBreakpointMessageIndices = planCacheBreakpoints(req, chosen.model, cfg);
|
|
315
|
+
|
|
316
|
+
// 8. Guarded probe: only tiers configured for probing, and only when a
|
|
317
|
+
// strictly higher tier exists inside the profile envelope to escalate into.
|
|
318
|
+
const nextTier = tierAt(tierIdx(chosenTier) + 1);
|
|
319
|
+
const escalateTo = nextTier !== null && tierIdx(nextTier) <= maxI ? nextTier : null;
|
|
320
|
+
const probeEnabled = cfg.escalation.enabled && escalateTo !== null && cfg.escalation.probeTiers.includes(chosenTier);
|
|
321
|
+
const probe: ProbePlan = {
|
|
322
|
+
enabled: probeEnabled,
|
|
323
|
+
maxTokens: cfg.escalation.probeTokens,
|
|
324
|
+
maxHoldMs: cfg.escalation.maxHoldMs,
|
|
325
|
+
escalateTo: probeEnabled ? escalateTo : null,
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
// 9. Clamp reasoning and output budget to what the target supports.
|
|
329
|
+
let reasoning: ReasoningLevel | undefined = req.reasoning;
|
|
330
|
+
if (reasoning !== undefined && !chosen.model.supportsReasoning) {
|
|
331
|
+
if (reasoning !== "off") reasons.push(`dropped reasoning=${reasoning}: ${chosen.model.slug} does not support it`);
|
|
332
|
+
reasoning = undefined;
|
|
333
|
+
}
|
|
334
|
+
if (chosen.model.reasoningMandatory && (reasoning === undefined || reasoning === "off")) {
|
|
335
|
+
reasoning = "minimal";
|
|
336
|
+
reasons.push(`reasoning forced to minimal: ${chosen.model.slug} has mandatory reasoning`);
|
|
337
|
+
}
|
|
338
|
+
let maxTokens: number | undefined = req.maxTokens;
|
|
339
|
+
const ceiling = chosen.model.maxCompletionTokens;
|
|
340
|
+
if (ceiling !== undefined) {
|
|
341
|
+
// The ceiling is a hard limit anyway; passing it explicitly also caps runaway completions.
|
|
342
|
+
maxTokens = maxTokens === undefined ? ceiling : Math.min(maxTokens, ceiling);
|
|
343
|
+
}
|
|
344
|
+
const stripAssistantReasoning = !(chosen.model.supportsReasoning && REASONING_REPLAY_AUTHORS[chosen.model.author] === true);
|
|
345
|
+
|
|
346
|
+
return {
|
|
347
|
+
slug: chosen.model.slug,
|
|
348
|
+
fallbacks,
|
|
349
|
+
tier: chosenTier,
|
|
350
|
+
classification: cls,
|
|
351
|
+
features,
|
|
352
|
+
forecast: chosen.forecast,
|
|
353
|
+
sessionId: state.sessionId,
|
|
354
|
+
sticky,
|
|
355
|
+
cacheBreakpointMessageIndices,
|
|
356
|
+
reasoning,
|
|
357
|
+
maxTokens,
|
|
358
|
+
stripAssistantReasoning,
|
|
359
|
+
probe,
|
|
360
|
+
considered: candidates,
|
|
361
|
+
rejected: resolved.rejected,
|
|
362
|
+
reasons,
|
|
363
|
+
budgetDowngraded,
|
|
364
|
+
};
|
|
365
|
+
}
|