@alexeiled/claude-router 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +90 -0
- package/README.md +97 -75
- package/docs/architecture.md +435 -0
- package/docs/configuration.md +151 -153
- package/docs/evaluation.md +83 -0
- package/docs/tier-share.svg +73 -0
- package/docs/user-guide.md +177 -92
- package/lib/config.mjs +105 -14
- package/lib/cost.mjs +53 -8
- package/lib/facts.mjs +37 -6
- package/lib/gateway.mjs +33 -30
- package/lib/jev.mjs +7 -4
- package/lib/policy.mjs +16 -12
- package/lib/rewrite.mjs +93 -3
- package/lib/router.mjs +92 -29
- package/lib/runtime.mjs +72 -7
- package/lib/status.mjs +45 -3
- package/lib/store.mjs +2 -1
- package/package.json +3 -2
- package/scripts/ensure-gateway.mjs +34 -8
- package/scripts/gateway.mjs +12 -2
- package/scripts/status.mjs +16 -3
- package/scripts/statusline.mjs +6 -2
- package/docs/design.md +0 -254
package/lib/facts.mjs
CHANGED
|
@@ -5,11 +5,20 @@ import { createHash } from 'node:crypto';
|
|
|
5
5
|
const EDIT_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit', 'Bash']);
|
|
6
6
|
const FAILURE_WINDOW = 40;
|
|
7
7
|
const REMINDER = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
8
|
+
const CLIP_MARKER = ' […] ';
|
|
9
|
+
// A typed /router:<tier> reaches the gateway as a command block at the start of a text block. Anchored, so a pasted
|
|
10
|
+
// file or quoted log that mentions the marker cannot pin a tier.
|
|
11
|
+
const PIN = /^<command-message>router:([a-z]+)<\/command-message>/;
|
|
8
12
|
|
|
9
13
|
export function factsFromRequest(body, memory, { recentTurns, maxTextChars }) {
|
|
10
14
|
const messages = Array.isArray(body.messages) ? body.messages : [];
|
|
11
|
-
|
|
15
|
+
// Claude Code puts hook output and tool additions in `system` messages after the user message, and keeps them in
|
|
16
|
+
// the history: the turn is the last user or assistant message.
|
|
17
|
+
const lastIndex = messages.findLastIndex((m) => m?.role !== 'system');
|
|
18
|
+
const last = messages[lastIndex];
|
|
12
19
|
const lastBlocks = blocks(last?.content);
|
|
20
|
+
// The current user message goes to Jev as the prompt; `turns` holds only the dialogue before it.
|
|
21
|
+
const current = last?.role === 'user' ? lastIndex : -1;
|
|
13
22
|
const turns = [];
|
|
14
23
|
const errors = [];
|
|
15
24
|
const edits = [];
|
|
@@ -22,16 +31,21 @@ export function factsFromRequest(body, memory, { recentTurns, maxTextChars }) {
|
|
|
22
31
|
for (const block of content) if (block.type === 'tool_use' && EDIT_TOOLS.has(block.name)) edits.push(index);
|
|
23
32
|
}
|
|
24
33
|
const text = textOf(content);
|
|
25
|
-
if (text && (message.role === 'user' || message.role === 'assistant'))
|
|
26
|
-
turns.push({ role: message.role, text: text
|
|
34
|
+
if (index !== current && text && (message.role === 'user' || message.role === 'assistant'))
|
|
35
|
+
turns.push({ role: message.role, text: clip(text, maxTextChars) });
|
|
27
36
|
});
|
|
28
37
|
return {
|
|
29
38
|
turns: turns.slice(-recentTurns),
|
|
30
|
-
prompt:
|
|
39
|
+
prompt: current === -1 ? '' : clip(textOf(lastBlocks), maxTextChars),
|
|
31
40
|
continuation: last?.role === 'user' && lastBlocks.some((b) => b.type === 'tool_result'),
|
|
32
|
-
// The
|
|
33
|
-
|
|
41
|
+
// The tier named by a /router:<tier> in the current message; the router checks that it is a tier.
|
|
42
|
+
pin: current === -1 ? null : pinOf(lastBlocks),
|
|
43
|
+
// The same position and the same last user message: Claude Code resent the request (429, 529, a dropped stream, or
|
|
44
|
+
// a 400 it answers by dropping a feature from the system message after it).
|
|
45
|
+
turnKey: last?.role === 'user' ? `${lastIndex + 1}:${hash(JSON.stringify(last.content))}` : null,
|
|
34
46
|
failure: repeatedFailure(errors, edits, messages.length - 1),
|
|
47
|
+
// The effort Claude Code sent: a route without its own effort keeps it, and it is part of the cache key.
|
|
48
|
+
effort: body.output_config?.effort ?? null,
|
|
35
49
|
lastRoute: memory.lastRoute,
|
|
36
50
|
lastRequest: memory.lastRequest,
|
|
37
51
|
models: memory.models,
|
|
@@ -51,6 +65,15 @@ function repeatedFailure(errors, edits, lastIndex) {
|
|
|
51
65
|
return null;
|
|
52
66
|
}
|
|
53
67
|
|
|
68
|
+
// At most `max` characters: the head and the tail of a long text, where a request and its question usually are.
|
|
69
|
+
function clip(text, max) {
|
|
70
|
+
if (text.length <= max) return text;
|
|
71
|
+
if (max <= CLIP_MARKER.length) return text.slice(0, max);
|
|
72
|
+
const room = max - CLIP_MARKER.length;
|
|
73
|
+
const head = Math.ceil(room / 2);
|
|
74
|
+
return `${text.slice(0, head)}${CLIP_MARKER}${text.slice(text.length - (room - head))}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
54
77
|
function signatureOf(block) {
|
|
55
78
|
return textOf(blocks(block.content)).toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ').slice(0, 120);
|
|
56
79
|
}
|
|
@@ -71,3 +94,11 @@ function textOf(content) {
|
|
|
71
94
|
.join('\n')
|
|
72
95
|
.trim();
|
|
73
96
|
}
|
|
97
|
+
|
|
98
|
+
function pinOf(content) {
|
|
99
|
+
for (const block of content) {
|
|
100
|
+
const match = block.type === 'text' && typeof block.text === 'string' ? PIN.exec(block.text) : null;
|
|
101
|
+
if (match) return match[1];
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
package/lib/gateway.mjs
CHANGED
|
@@ -5,23 +5,20 @@ import { createServer, request as httpRequest } from 'node:http';
|
|
|
5
5
|
import { request as httpsRequest } from 'node:https';
|
|
6
6
|
import { pipeline } from 'node:stream';
|
|
7
7
|
import { IdleTracker } from './idle.mjs';
|
|
8
|
+
import { adaptBetas } from './rewrite.mjs';
|
|
9
|
+
import { isSideRequest } from './router.mjs';
|
|
8
10
|
import { UsageReader } from './sse.mjs';
|
|
9
11
|
import { ROUTER_DISPLAY_NAME, STATUS_PATH, statusSnapshot } from './status.mjs';
|
|
10
12
|
|
|
11
13
|
const HOP_BY_HOP = new Set(['host', 'connection', 'content-length', 'accept-encoding', 'transfer-encoding']);
|
|
12
14
|
const LOOPBACK = new Set(['127.0.0.1', 'localhost', '[::1]']);
|
|
13
15
|
const MESSAGES_PATH = '/v1/messages';
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const model = Object.values(config.models).find((m) => m.id === modelId);
|
|
21
|
-
const beta = headers['anthropic-beta'];
|
|
22
|
-
if (!model || model.contextWindow >= LONG_CONTEXT_WINDOW || typeof beta !== 'string') return;
|
|
23
|
-
const kept = beta.split(',').filter((flag) => !LONG_CONTEXT_BETA.test(flag.trim()));
|
|
24
|
-
if (kept.length) headers['anthropic-beta'] = kept.join(',');
|
|
16
|
+
|
|
17
|
+
// The betas the routed model accepts: rewrite.mjs owns what each model takes.
|
|
18
|
+
function withSupportedBetas(headers, modelId, config) {
|
|
19
|
+
if (typeof headers['anthropic-beta'] !== 'string') return;
|
|
20
|
+
const kept = adaptBetas(headers['anthropic-beta'], modelId, config);
|
|
21
|
+
if (kept) headers['anthropic-beta'] = kept;
|
|
25
22
|
else delete headers['anthropic-beta'];
|
|
26
23
|
}
|
|
27
24
|
|
|
@@ -58,7 +55,12 @@ export function createGateway({
|
|
|
58
55
|
const parsed = parseJson(body);
|
|
59
56
|
const session = sessionKey(req, parsed);
|
|
60
57
|
const turn = isMessagesCall(req);
|
|
61
|
-
|
|
58
|
+
// Only the conversation itself waits for tool results; a side request on the same session says nothing about it.
|
|
59
|
+
// Only the hint header decides here: by body shape a turn with thinking off looks like a side request, and
|
|
60
|
+
// keeping a wait too long costs less than exiting under a permission prompt.
|
|
61
|
+
const { requestClass } = routingHints(req);
|
|
62
|
+
const side = Boolean(requestClass) && isSideRequest(parsed ?? {}, requestClass);
|
|
63
|
+
if (turn && !side) activity.turnResumed(session);
|
|
62
64
|
let routed = null;
|
|
63
65
|
if (parsed && router.isRouted(parsed)) {
|
|
64
66
|
if (turn) {
|
|
@@ -66,7 +68,7 @@ export function createGateway({
|
|
|
66
68
|
routed = await router.route(parsed, { sessionId: session, ...routingHints(req) });
|
|
67
69
|
} catch (error) {
|
|
68
70
|
onError(error);
|
|
69
|
-
routed = router.fallback(parsed); // never forward the alias upstream
|
|
71
|
+
routed = router.fallback(parsed, session, routingHints(req)); // never forward the alias upstream
|
|
70
72
|
}
|
|
71
73
|
} else routed = router.resolveModel(parsed, session);
|
|
72
74
|
body = Buffer.from(JSON.stringify(routed.body));
|
|
@@ -76,7 +78,7 @@ export function createGateway({
|
|
|
76
78
|
for (const [name, value] of Object.entries(req.headers)) if (!HOP_BY_HOP.has(name)) headers[name] = value;
|
|
77
79
|
headers.host = target.host;
|
|
78
80
|
headers['content-length'] = String(body.length);
|
|
79
|
-
if (routed)
|
|
81
|
+
if (routed) withSupportedBetas(headers, routed.body.model, router.config);
|
|
80
82
|
up = send(
|
|
81
83
|
{ host: target.hostname, port: target.port || undefined, path: req.url, method: req.method, headers },
|
|
82
84
|
(upRes) => relay(upRes, res, { routed, session, turn, clientLeft: () => closedEarly }),
|
|
@@ -108,18 +110,26 @@ export function createGateway({
|
|
|
108
110
|
if (upstreamBroke) onError(new Error(`upstream stream: ${error.message}`));
|
|
109
111
|
return;
|
|
110
112
|
}
|
|
111
|
-
if (!reader
|
|
113
|
+
if (!reader) return;
|
|
114
|
+
if (upRes.statusCode >= 300) {
|
|
115
|
+
if (routed) tryRecord(() => router.recordFailure(session, routed, upRes.statusCode));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
112
118
|
const usage = reader.end();
|
|
113
119
|
if (reader.stopReason === 'tool_use') activity.turnPaused(session);
|
|
114
120
|
if (!routed || routed.auxiliary) return;
|
|
115
|
-
|
|
116
|
-
router.recordResponse(session, routed.tier, usage);
|
|
117
|
-
} catch (recordError) {
|
|
118
|
-
onError(recordError);
|
|
119
|
-
}
|
|
121
|
+
tryRecord(() => router.recordResponse(session, routed.tier, usage, routed.effort));
|
|
120
122
|
});
|
|
121
123
|
}
|
|
122
124
|
|
|
125
|
+
function tryRecord(record) {
|
|
126
|
+
try {
|
|
127
|
+
record();
|
|
128
|
+
} catch (error) {
|
|
129
|
+
onError(error);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
123
133
|
function fail(res, error) {
|
|
124
134
|
onError(error);
|
|
125
135
|
if (!res.headersSent) res.writeHead(502);
|
|
@@ -154,17 +164,10 @@ function routingHints(req) {
|
|
|
154
164
|
};
|
|
155
165
|
}
|
|
156
166
|
|
|
157
|
-
// A web page can reach a loopback port too: by DNS rebinding (a foreign Host) or by a
|
|
158
|
-
// Origin
|
|
167
|
+
// A web page can reach a loopback port too: by DNS rebinding (a foreign Host) or by a request from a page, which
|
|
168
|
+
// always carries an Origin, even one served from another loopback port. Claude Code sends a loopback Host and no Origin.
|
|
159
169
|
function isLocalClient(req) {
|
|
160
|
-
|
|
161
|
-
const origin = req.headers.origin;
|
|
162
|
-
if (origin === undefined) return true;
|
|
163
|
-
try {
|
|
164
|
-
return LOOPBACK.has(new URL(origin).hostname);
|
|
165
|
-
} catch {
|
|
166
|
-
return false;
|
|
167
|
-
}
|
|
170
|
+
return LOOPBACK.has(hostOf(req.headers.host)) && req.headers.origin === undefined;
|
|
168
171
|
}
|
|
169
172
|
|
|
170
173
|
function hostOf(host = '') {
|
package/lib/jev.mjs
CHANGED
|
@@ -75,18 +75,21 @@ export async function askJev({ fetchFn, config, apiKey, prompt, turns, now = Dat
|
|
|
75
75
|
}
|
|
76
76
|
if (response.ok) return parseAnswers(await response.json());
|
|
77
77
|
if (!TRANSIENT.has(response.status) || attempt === 1) throw new Error(`jev http ${response.status}`);
|
|
78
|
-
const wait = retryAfterMs(response.headers) ?? RETRY_DELAY_MS;
|
|
78
|
+
const wait = retryAfterMs(response.headers, now()) ?? RETRY_DELAY_MS;
|
|
79
79
|
if (wait >= deadline - now()) throw new Error(`jev http ${response.status}, retry-after beyond the budget`);
|
|
80
80
|
await sleep(wait);
|
|
81
81
|
}
|
|
82
82
|
throw new Error('jev unreachable');
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
// Delay-seconds or an HTTP-date (RFC 9110). Seconds first: Date.parse also reads a bare number, as a year.
|
|
86
|
+
function retryAfterMs(headers, now) {
|
|
86
87
|
const value = headers?.get?.('retry-after');
|
|
87
|
-
if (value === null || value === undefined || value === '') return null;
|
|
88
|
+
if (value === null || value === undefined || value.trim() === '') return null;
|
|
88
89
|
const seconds = Number(value);
|
|
89
|
-
|
|
90
|
+
if (Number.isFinite(seconds)) return seconds >= 0 ? seconds * 1000 : null;
|
|
91
|
+
const at = Date.parse(value);
|
|
92
|
+
return Number.isFinite(at) ? Math.max(0, at - now) : null;
|
|
90
93
|
}
|
|
91
94
|
|
|
92
95
|
export function parseAnswers(json) {
|
package/lib/policy.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// Switching policy v0 (docs/
|
|
1
|
+
// Switching policy v0 (docs/architecture.md): stickiness, escalation floor, cost-gated votes.
|
|
2
2
|
import { rank, TIERS } from './config.mjs';
|
|
3
|
-
import { coldWriteUsd, isWarm, nextContextTokens, switchingTaxUsd } from './cost.mjs';
|
|
3
|
+
import { cacheState, coldWriteUsd, isWarm, nextContextTokens, routeCacheKey, switchingTaxUsd } from './cost.mjs';
|
|
4
4
|
|
|
5
5
|
export function initialState() {
|
|
6
6
|
return { turn: 0, votes: [], holdUntilTurn: 0, escalatedSignature: null };
|
|
@@ -34,15 +34,13 @@ export function decide({ config, facts, advice, state, baseline, now }) {
|
|
|
34
34
|
if (rank(gated.fallback) <= rank(incumbent)) return stay('cash-gate', gated.estimate);
|
|
35
35
|
return result(gated.fallback, 'cash-gate', next, gated.estimate);
|
|
36
36
|
}
|
|
37
|
-
const tax = Math.max(
|
|
38
|
-
0,
|
|
39
|
-
switchingTaxUsd(config, config.routes[choice].model, config.routes[incumbent].model, facts, now),
|
|
40
|
-
);
|
|
37
|
+
const tax = Math.max(0, switchingTaxUsd(config, choice, incumbent, facts, now));
|
|
41
38
|
const threshold =
|
|
42
39
|
config.policy.upgradeBase + config.policy.upgradeSlope * (tax / (tax + config.policy.upgradePivotUsd));
|
|
43
40
|
const jump = rank(choice) - rank(incumbent) >= 2 && upgrade >= config.policy.jumpConfidence;
|
|
44
41
|
const streak = trailing(next.votes, (v) => rank(v.tier) > rank(incumbent));
|
|
45
|
-
const
|
|
42
|
+
const cache = { candidate: cacheOf(config, choice, facts, now), incumbent: cacheOf(config, incumbent, facts, now) };
|
|
43
|
+
const estimate = { taxUsd: tax, threshold, upgradeMass: upgrade, streak, cache };
|
|
46
44
|
if (jump || (streak >= config.policy.upgradeVotes && upgrade >= threshold))
|
|
47
45
|
return result(choice, jump ? 'jump' : 'upgrade', next, estimate);
|
|
48
46
|
return stay('upgrade-pending', estimate);
|
|
@@ -73,16 +71,22 @@ export function fitTier(config, tier, tokens) {
|
|
|
73
71
|
);
|
|
74
72
|
}
|
|
75
73
|
|
|
76
|
-
//
|
|
74
|
+
// Cold-write guard: an automatic route to a credits-billed model whose cache is not warm must not start with a cache
|
|
75
|
+
// write above `cashCapUsd`. It bounds that one estimated write, not the spend of the turn: a warm cache passes, and
|
|
76
|
+
// output is not counted.
|
|
77
77
|
function cashGate(config, tier, facts, now) {
|
|
78
78
|
const alias = config.routes[tier].model;
|
|
79
|
-
|
|
80
|
-
if (
|
|
81
|
-
if (isWarm(facts.models[model.id], now, config.cache)) return { blocked: false };
|
|
79
|
+
if (config.models[alias].billing !== 'credits') return { blocked: false };
|
|
80
|
+
if (isWarm(facts.models[routeCacheKey(config, tier, facts.effort)], now, config.cache)) return { blocked: false };
|
|
82
81
|
const cold = coldWriteUsd(config, alias, nextContextTokens(facts), facts);
|
|
83
82
|
if (cold <= config.policy.cashCapUsd) return { blocked: false };
|
|
84
83
|
const fallback = [...TIERS].reverse().find((t) => config.models[config.routes[t].model].billing === 'plan');
|
|
85
|
-
|
|
84
|
+
const estimate = { coldUsd: cold, cap: config.policy.cashCapUsd, cache: cacheOf(config, tier, facts, now) };
|
|
85
|
+
return { blocked: true, fallback, estimate };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function cacheOf(config, tier, facts, now) {
|
|
89
|
+
return cacheState(facts.models[routeCacheKey(config, tier, facts.effort)], now, config.cache);
|
|
86
90
|
}
|
|
87
91
|
|
|
88
92
|
function trailing(votes, predicate) {
|
package/lib/rewrite.mjs
CHANGED
|
@@ -1,12 +1,26 @@
|
|
|
1
|
-
// Turn a request for the router alias into a request for a concrete model.
|
|
2
|
-
// `output_config.effort`, `thinking` and thinking edits in `context_management` change
|
|
3
|
-
//
|
|
1
|
+
// Turn a request for the router alias into a request for a concrete model. `model`, `max_tokens`,
|
|
2
|
+
// `output_config.effort`, `thinking` and thinking edits in `context_management` change. For a model without a
|
|
3
|
+
// request feature that Claude Code uses (config FEATURES), the messages and the beta header lose that feature,
|
|
4
|
+
// as Claude Code itself does when the model answers 400; system prompt and tools are never touched.
|
|
4
5
|
import { EFFORTS } from './config.mjs';
|
|
5
6
|
|
|
7
|
+
const TOOL_CHANGE_BLOCKS = new Set(['tool_addition', 'tool_removal']);
|
|
8
|
+
const LONG_CONTEXT_BETA = /^context-1m-/;
|
|
9
|
+
const LONG_CONTEXT_WINDOW = 1_000_000;
|
|
10
|
+
// Each feature is a beta; a model without the feature answers the beta with 400.
|
|
11
|
+
const FEATURE_BETAS = {
|
|
12
|
+
'mid-conversation-system': /^mid-conversation-system-/,
|
|
13
|
+
'per-turn-control': /^per-turn-control-/,
|
|
14
|
+
'mid-conversation-tool-changes': /^mid-conversation-tool-changes-/,
|
|
15
|
+
};
|
|
16
|
+
|
|
6
17
|
export function rewriteRequest(body, tier, config) {
|
|
7
18
|
const route = config.routes[tier];
|
|
8
19
|
const model = config.models[route.model];
|
|
9
20
|
const out = { ...body, model: model.id };
|
|
21
|
+
// A route's own effort also goes into per-turn control, which overrides the request's effort on the model.
|
|
22
|
+
const routeEffort = route.effort ? clampEffort(route.effort, model.efforts) : null;
|
|
23
|
+
if (Array.isArray(body.messages)) out.messages = adaptMessages(body.messages, model.features ?? [], routeEffort);
|
|
10
24
|
if (model.maxOutput && out.max_tokens > model.maxOutput) out.max_tokens = model.maxOutput;
|
|
11
25
|
if (model.efforts.length === 0) return withoutThinking(out, body.output_config);
|
|
12
26
|
const effort = clampEffort(route.effort ?? body.output_config?.effort, model.efforts);
|
|
@@ -14,6 +28,82 @@ export function rewriteRequest(body, tier, config) {
|
|
|
14
28
|
return out;
|
|
15
29
|
}
|
|
16
30
|
|
|
31
|
+
// The `anthropic-beta` header for the routed model, or null when nothing is left. `jev-router[1m]` makes Claude Code
|
|
32
|
+
// send the 1M context beta on every request; a model with a smaller window answers it with 400.
|
|
33
|
+
export function adaptBetas(beta, modelId, config) {
|
|
34
|
+
const model = Object.values(config.models).find((m) => m.id === modelId);
|
|
35
|
+
if (!model) return beta;
|
|
36
|
+
const features = model.features ?? [];
|
|
37
|
+
const dropped = Object.entries(FEATURE_BETAS)
|
|
38
|
+
.filter(([feature]) => !features.includes(feature))
|
|
39
|
+
.map(([, pattern]) => pattern);
|
|
40
|
+
if (model.contextWindow < LONG_CONTEXT_WINDOW) dropped.push(LONG_CONTEXT_BETA);
|
|
41
|
+
const kept = beta.split(',').filter((flag) => !dropped.some((pattern) => pattern.test(flag.trim())));
|
|
42
|
+
return kept.length ? kept.join(',') : null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Pure and deterministic: the same history adapts the same way on every turn, so the cached prefix holds.
|
|
46
|
+
function adaptMessages(messages, features, routeEffort) {
|
|
47
|
+
let out = messages;
|
|
48
|
+
if (!features.includes('per-turn-control')) out = out.map(withoutTurnControl).filter(Boolean);
|
|
49
|
+
else if (routeEffort && out.some((m) => turnEffortOf(m) && turnEffortOf(m) !== routeEffort))
|
|
50
|
+
out = out.map((m) => (turnEffortOf(m) ? { ...m, output_config: { ...m.output_config, effort: routeEffort } } : m));
|
|
51
|
+
if (!features.includes('mid-conversation-tool-changes')) out = out.map(withoutToolChanges).filter(Boolean);
|
|
52
|
+
if (!features.includes('mid-conversation-system')) out = foldSystemMessages(out);
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Per-turn control is an `output_config` (the turn's effort) on a message, usually the system message after the prompt.
|
|
57
|
+
// A message that carried only that is dropped: the API rejects empty content.
|
|
58
|
+
function withoutTurnControl(message) {
|
|
59
|
+
if (!message || !Object.hasOwn(message, 'output_config')) return message;
|
|
60
|
+
const { output_config: _turn, ...rest } = message;
|
|
61
|
+
return Array.isArray(rest.content) && rest.content.length === 0 ? null : rest;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function turnEffortOf(message) {
|
|
65
|
+
return message?.output_config?.effort ?? null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Drops tool_addition and tool_removal blocks. A dropped block's cache breakpoint moves to the block before it, or the
|
|
69
|
+
// turn loses its last breakpoint; a message left empty is dropped.
|
|
70
|
+
function withoutToolChanges(message) {
|
|
71
|
+
if (!Array.isArray(message.content) || !message.content.some((b) => TOOL_CHANGE_BLOCKS.has(b?.type))) return message;
|
|
72
|
+
const content = [];
|
|
73
|
+
for (const block of message.content) {
|
|
74
|
+
if (!TOOL_CHANGE_BLOCKS.has(block?.type)) content.push(block);
|
|
75
|
+
else if (block.cache_control && content.length && !content.at(-1).cache_control)
|
|
76
|
+
content[content.length - 1] = { ...content.at(-1), cache_control: block.cache_control };
|
|
77
|
+
}
|
|
78
|
+
return content.length ? { ...message, content } : null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// A `system` message becomes text in the user message next to it, wrapped as a system reminder: the form Claude Code
|
|
82
|
+
// uses for hook output when the model takes no system messages.
|
|
83
|
+
function foldSystemMessages(messages) {
|
|
84
|
+
const out = [];
|
|
85
|
+
let lastFolded = false;
|
|
86
|
+
for (const message of messages) {
|
|
87
|
+
const folded = message.role === 'system';
|
|
88
|
+
const next = folded ? { role: 'user', content: blocks(message.content).map(asReminder) } : message;
|
|
89
|
+
const prev = out.at(-1);
|
|
90
|
+
if (prev?.role === 'user' && next.role === 'user' && (folded || lastFolded))
|
|
91
|
+
out[out.length - 1] = { ...prev, content: [...blocks(prev.content), ...next.content] };
|
|
92
|
+
else out.push(next);
|
|
93
|
+
lastFolded = folded;
|
|
94
|
+
}
|
|
95
|
+
return out;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function blocks(content) {
|
|
99
|
+
if (typeof content === 'string') return [{ type: 'text', text: content }];
|
|
100
|
+
return Array.isArray(content) ? content : [];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function asReminder(block) {
|
|
104
|
+
return block?.type === 'text' ? { ...block, text: `<system-reminder>\n${block.text}\n</system-reminder>` } : block;
|
|
105
|
+
}
|
|
106
|
+
|
|
17
107
|
// Highest supported level at or below the wanted one, as Claude Code does for its own models.
|
|
18
108
|
export function clampEffort(wanted, supported) {
|
|
19
109
|
if (!wanted || supported.length === 0) return null;
|
package/lib/router.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Per-request orchestration: facts -> advice -> policy -> rewritten body. Knows nothing about HTTP.
|
|
2
|
-
import { LEGACY_ALIAS } from './config.mjs';
|
|
3
|
-
import { nextContextTokens } from './cost.mjs';
|
|
2
|
+
import { LEGACY_ALIAS, TIERS } from './config.mjs';
|
|
3
|
+
import { cacheKey, nextContextTokens, shadowEconomics } from './cost.mjs';
|
|
4
4
|
import { factsFromRequest } from './facts.mjs';
|
|
5
5
|
import { askJev } from './jev.mjs';
|
|
6
6
|
import { decide, fitTier, initialState } from './policy.mjs';
|
|
@@ -20,14 +20,27 @@ export function emptyMemory() {
|
|
|
20
20
|
return {
|
|
21
21
|
lastRoute: null,
|
|
22
22
|
lastReason: null,
|
|
23
|
+
lastEstimate: null,
|
|
23
24
|
lastEffort: null,
|
|
24
25
|
lastRequest: null,
|
|
25
26
|
lastTurnKey: null,
|
|
27
|
+
lastMessageCount: null,
|
|
26
28
|
models: {},
|
|
27
29
|
state: null,
|
|
30
|
+
// Set by a pinned turn: the route to decide the next prompt from.
|
|
31
|
+
pin: null,
|
|
28
32
|
};
|
|
29
33
|
}
|
|
30
34
|
|
|
35
|
+
// The history the gateway saw is gone: a compaction or a rewind. The cached prefixes are unknown, and the votes and
|
|
36
|
+
// the escalation hold were about turns that are no longer in it. The route stays until the next decision.
|
|
37
|
+
// `lastRequest` stays: until the next response it is the only size there is, and as an upper bound it errs safe.
|
|
38
|
+
// A rewind can keep most of a long context, and a size of zero would let context-fit send it to a small window.
|
|
39
|
+
function restartHistory(memory) {
|
|
40
|
+
memory.models = {};
|
|
41
|
+
if (memory.state) memory.state = { ...memory.state, votes: [], holdUntilTurn: 0, escalatedSignature: null };
|
|
42
|
+
}
|
|
43
|
+
|
|
31
44
|
export class Router {
|
|
32
45
|
constructor({ config, fetchFn, dataDir, now = Date.now, onError = () => {} }) {
|
|
33
46
|
this.config = config;
|
|
@@ -49,28 +62,49 @@ export class Router {
|
|
|
49
62
|
// The hints come from Claude Code's gateway headers (CLAUDE_CODE_GATEWAY_HINT_HEADERS=1); each may be missing.
|
|
50
63
|
async route(body, { sessionId, requestClass = null, agentType = null, contextCompacted = null }) {
|
|
51
64
|
const memory = this.memory(sessionId);
|
|
52
|
-
|
|
53
|
-
|
|
65
|
+
const auxiliary = isSideRequest(body, requestClass);
|
|
66
|
+
// Messages only grow within one history. Fewer than the last main request: Claude Code compacted the conversation
|
|
67
|
+
// or the user rewound it. Context editing clears tool results but keeps the messages, so it is not a break.
|
|
68
|
+
const messageCount = Array.isArray(body.messages) ? body.messages.length : 0;
|
|
69
|
+
const rewound = !auxiliary && memory.lastMessageCount !== null && messageCount < memory.lastMessageCount;
|
|
70
|
+
if (contextCompacted || rewound) {
|
|
71
|
+
restartHistory(memory);
|
|
72
|
+
this.log({ session: sessionId, historyBreak: contextCompacted ? 'compaction' : 'shorter-history' });
|
|
73
|
+
}
|
|
54
74
|
const facts = factsFromRequest(body, memory, this.config.context);
|
|
55
|
-
const auxiliary = requestClass ? SIDE_REQUEST_CLASSES.has(requestClass) : isAuxiliaryShape(body);
|
|
56
75
|
let decision;
|
|
57
76
|
if (auxiliary) decision = { tier: this.config.gateway.auxiliaryTier, reason: 'auxiliary', state: memory.state };
|
|
58
77
|
else if (facts.continuation && memory.lastRoute)
|
|
59
78
|
decision = { tier: memory.lastRoute, reason: 'tool-continuation', state: memory.state };
|
|
60
79
|
else if (facts.turnKey && facts.turnKey === memory.lastTurnKey && memory.lastRoute)
|
|
61
80
|
decision = { tier: memory.lastRoute, reason: 'retry', state: memory.state };
|
|
62
|
-
else
|
|
81
|
+
else {
|
|
82
|
+
// A pin serves its turn and the turn's tool calls; the next prompt is decided from the route before the pin.
|
|
83
|
+
if (memory.pin) {
|
|
84
|
+
memory.lastRoute = memory.pin.returnTo;
|
|
85
|
+
facts.lastRoute = memory.pin.returnTo;
|
|
86
|
+
memory.pin = null;
|
|
87
|
+
}
|
|
88
|
+
decision = await this.decideTurn(facts, memory);
|
|
89
|
+
}
|
|
90
|
+
const pinnedTurn = decision.reason === 'pinned';
|
|
63
91
|
// Main turns and compaction carry the main conversation; other side requests have their own, unknown size.
|
|
64
92
|
if (!auxiliary || requestClass === 'compaction') {
|
|
65
93
|
const tier = fitTier(this.config, decision.tier, nextContextTokens(facts));
|
|
66
94
|
if (tier !== decision.tier) decision = { ...decision, tier, reason: 'context-fit' };
|
|
67
95
|
}
|
|
68
96
|
const rewritten = rewriteRequest(body, decision.tier, this.config);
|
|
97
|
+
const effort = rewritten.output_config?.effort ?? null;
|
|
69
98
|
if (!auxiliary) {
|
|
99
|
+
if (pinnedTurn) memory.pin = { returnTo: memory.lastRoute };
|
|
70
100
|
memory.lastRoute = decision.tier;
|
|
71
|
-
memory.lastEffort =
|
|
101
|
+
memory.lastEffort = effort;
|
|
72
102
|
memory.lastTurnKey = facts.turnKey;
|
|
73
|
-
|
|
103
|
+
memory.lastMessageCount = messageCount;
|
|
104
|
+
if (!REUSED_ROUTE.has(decision.reason)) {
|
|
105
|
+
memory.lastReason = decision.reason;
|
|
106
|
+
memory.lastEstimate = decision.estimate ?? null;
|
|
107
|
+
}
|
|
74
108
|
memory.state = decision.state;
|
|
75
109
|
this.persist(sessionId, memory);
|
|
76
110
|
}
|
|
@@ -79,8 +113,11 @@ export class Router {
|
|
|
79
113
|
requestClass,
|
|
80
114
|
agentType,
|
|
81
115
|
tier: decision.tier,
|
|
116
|
+
model: rewritten.model,
|
|
117
|
+
effort,
|
|
82
118
|
reason: decision.reason,
|
|
83
119
|
estimate: decision.estimate ?? null,
|
|
120
|
+
shadow: decision.shadow ?? null,
|
|
84
121
|
advice: decision.advice ?? null,
|
|
85
122
|
adviceError: decision.adviceError ?? null,
|
|
86
123
|
contextTokens: memory.lastRequest?.tokens ?? 0,
|
|
@@ -88,15 +125,28 @@ export class Router {
|
|
|
88
125
|
return {
|
|
89
126
|
body: rewritten,
|
|
90
127
|
tier: decision.tier,
|
|
128
|
+
effort,
|
|
91
129
|
reason: decision.reason,
|
|
92
130
|
auxiliary,
|
|
93
131
|
};
|
|
94
132
|
}
|
|
95
133
|
|
|
96
|
-
// Used when routing itself failed: the alias must never reach Anthropic.
|
|
97
|
-
|
|
134
|
+
// Used when routing itself failed: the alias must never reach Anthropic. The log line has the reason only: an
|
|
135
|
+
// error message can quote the request.
|
|
136
|
+
fallback(body, sessionId, { requestClass = null } = {}) {
|
|
98
137
|
const tier = this.config.gateway.baselineTier;
|
|
99
|
-
|
|
138
|
+
const rewritten = rewriteRequest(body, tier, this.config);
|
|
139
|
+
const effort = rewritten.output_config?.effort ?? null;
|
|
140
|
+
this.log({ session: sessionId, requestClass, tier, model: rewritten.model, effort, reason: 'error' });
|
|
141
|
+
return { body: rewritten, tier, effort, reason: 'error', auxiliary: true };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// A routed turn that Anthropic answered with an error status: no usage to record, but the turn is visible.
|
|
145
|
+
recordFailure(sessionId, routed, status) {
|
|
146
|
+
this.log({
|
|
147
|
+
session: sessionId,
|
|
148
|
+
failed: { status, tier: routed.tier, model: routed.body.model, effort: routed.effort ?? null },
|
|
149
|
+
});
|
|
100
150
|
}
|
|
101
151
|
|
|
102
152
|
// Side endpoints (count_tokens) name the alias too. They get the session's current model: no Jev call, no vote.
|
|
@@ -109,6 +159,9 @@ export class Router {
|
|
|
109
159
|
const state = memory.state ?? initialState();
|
|
110
160
|
const { forcedTier, apiKey, gateway } = this.config;
|
|
111
161
|
if (forcedTier) return { tier: forcedTier, reason: 'forced', state: { ...state, turn: state.turn + 1 } };
|
|
162
|
+
// A typed /router:<tier> pins this turn. Claude Code no longer switches the model from the skill's frontmatter.
|
|
163
|
+
if (TIERS.includes(facts.pin))
|
|
164
|
+
return { tier: facts.pin, reason: 'pinned', state: { ...state, turn: state.turn + 1 } };
|
|
112
165
|
let advice = null;
|
|
113
166
|
let adviceError = null;
|
|
114
167
|
if (apiKey && facts.prompt) {
|
|
@@ -130,15 +183,15 @@ export class Router {
|
|
|
130
183
|
}
|
|
131
184
|
}
|
|
132
185
|
}
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
return { ...decision, advice, adviceError };
|
|
186
|
+
const now = this.now();
|
|
187
|
+
const decision = decide({ config: this.config, facts, advice, state, baseline: gateway.baselineTier, now });
|
|
188
|
+
// Shadow only: what following Jev's choice instead of staying would cost at list prices. The policy ignores it.
|
|
189
|
+
const incumbent = TIERS.includes(facts.lastRoute) ? facts.lastRoute : gateway.baselineTier;
|
|
190
|
+
const shadow =
|
|
191
|
+
TIERS.includes(advice?.choice) && advice.choice !== incumbent
|
|
192
|
+
? shadowEconomics(this.config, advice.choice, incumbent, facts, now)
|
|
193
|
+
: null;
|
|
194
|
+
return { ...decision, advice, adviceError, shadow };
|
|
142
195
|
}
|
|
143
196
|
|
|
144
197
|
jevSucceeded() {
|
|
@@ -160,15 +213,19 @@ export class Router {
|
|
|
160
213
|
);
|
|
161
214
|
}
|
|
162
215
|
|
|
163
|
-
// Called with the usage the gateway read from a forwarded main-conversation response
|
|
164
|
-
|
|
216
|
+
// Called with the usage the gateway read from a forwarded main-conversation response, and the effort the gateway
|
|
217
|
+
// sent with that request: the cache is keyed by both.
|
|
218
|
+
recordResponse(sessionId, tier, usage, effort = null) {
|
|
165
219
|
if (!usage) return;
|
|
166
220
|
const memory = this.memory(sessionId);
|
|
167
221
|
const modelId = usage.model ?? this.config.models[this.config.routes[tier].model].id;
|
|
168
222
|
const at = this.now();
|
|
169
|
-
// A context that shrank
|
|
170
|
-
// the
|
|
171
|
-
if (memory.lastRequest && usage.tokens < memory.lastRequest.tokens * COMPACTION_SHRINK)
|
|
223
|
+
// A context that shrank by more than a fifth: a compaction, or context editing that cleared old tool results.
|
|
224
|
+
// Either way the cached prefixes no longer match. The votes stay: a shorter history in messages resets them.
|
|
225
|
+
if (memory.lastRequest && usage.tokens < memory.lastRequest.tokens * COMPACTION_SHRINK) {
|
|
226
|
+
memory.models = {};
|
|
227
|
+
this.log({ session: sessionId, cacheReset: 'context-shrink' });
|
|
228
|
+
}
|
|
172
229
|
memory.lastRequest = {
|
|
173
230
|
model: modelId,
|
|
174
231
|
tokens: usage.tokens,
|
|
@@ -177,9 +234,13 @@ export class Router {
|
|
|
177
234
|
ttl: usage.ttl,
|
|
178
235
|
at,
|
|
179
236
|
};
|
|
180
|
-
memory.models[modelId] = {
|
|
237
|
+
memory.models[cacheKey(modelId, effort)] = {
|
|
238
|
+
lastAt: at,
|
|
239
|
+
prefixTokens: usage.tokens + usage.outputTokens,
|
|
240
|
+
ttl: usage.ttl,
|
|
241
|
+
};
|
|
181
242
|
this.persist(sessionId, memory);
|
|
182
|
-
this.log({ session: sessionId, observed: { ...usage, model: modelId, tier } });
|
|
243
|
+
this.log({ session: sessionId, observed: { ...usage, model: modelId, tier, effort } });
|
|
183
244
|
}
|
|
184
245
|
|
|
185
246
|
memory(sessionId) {
|
|
@@ -206,7 +267,9 @@ export class Router {
|
|
|
206
267
|
}
|
|
207
268
|
}
|
|
208
269
|
|
|
209
|
-
//
|
|
210
|
-
|
|
270
|
+
// A request outside the conversation (title, classifier, compaction), by the request-class hint header. Without the
|
|
271
|
+
// header: side requests turn thinking off or ask for a schema.
|
|
272
|
+
export function isSideRequest(body, requestClass) {
|
|
273
|
+
if (requestClass) return SIDE_REQUEST_CLASSES.has(requestClass);
|
|
211
274
|
return body.thinking?.type === 'disabled' || Boolean(body.output_config?.format);
|
|
212
275
|
}
|