@alexeiled/claude-router 0.2.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +4 -4
- package/README.md +41 -15
- package/docs/configuration.md +47 -18
- package/docs/design.md +42 -2
- package/docs/user-guide.md +26 -12
- package/hooks/hooks.json +12 -0
- package/lib/config.mjs +10 -5
- package/lib/facts.mjs +7 -0
- package/lib/gateway.mjs +143 -58
- package/lib/idle.mjs +41 -0
- package/lib/jev.mjs +29 -8
- package/lib/policy.mjs +17 -0
- package/lib/router.mjs +82 -18
- package/lib/sse.mjs +7 -2
- package/lib/status.mjs +27 -7
- package/lib/store.mjs +40 -1
- package/package.json +2 -2
- package/scripts/ensure-gateway.mjs +78 -16
- package/scripts/gateway.mjs +64 -5
- package/skills/setup/SKILL.md +19 -14
- package/skills/status/SKILL.md +2 -1
package/lib/gateway.mjs
CHANGED
|
@@ -1,69 +1,119 @@
|
|
|
1
1
|
// Local Anthropic Messages gateway: byte-for-byte passthrough, except a routed request gets its
|
|
2
2
|
// model, effort and thinking rewritten. Never re-serializes responses; reads usage on a tee.
|
|
3
|
+
// Upstream errors (429, 529, 5xx) pass through unchanged: Claude Code owns retries and backoff.
|
|
3
4
|
import { createServer, request as httpRequest } from 'node:http';
|
|
4
5
|
import { request as httpsRequest } from 'node:https';
|
|
6
|
+
import { pipeline } from 'node:stream';
|
|
7
|
+
import { IdleTracker } from './idle.mjs';
|
|
5
8
|
import { UsageReader } from './sse.mjs';
|
|
6
9
|
import { ROUTER_DISPLAY_NAME, STATUS_PATH, statusSnapshot } from './status.mjs';
|
|
7
10
|
|
|
8
11
|
const HOP_BY_HOP = new Set(['host', 'connection', 'content-length', 'accept-encoding', 'transfer-encoding']);
|
|
12
|
+
const LOOPBACK = new Set(['127.0.0.1', 'localhost', '[::1]']);
|
|
13
|
+
const MESSAGES_PATH = '/v1/messages';
|
|
9
14
|
|
|
10
|
-
export function createGateway({
|
|
15
|
+
export function createGateway({
|
|
16
|
+
router,
|
|
17
|
+
upstream = 'https://api.anthropic.com',
|
|
18
|
+
onError = () => {},
|
|
19
|
+
activity = new IdleTracker(),
|
|
20
|
+
}) {
|
|
11
21
|
const target = new URL(upstream);
|
|
12
22
|
const send = target.protocol === 'https:' ? httpsRequest : httpRequest;
|
|
13
23
|
|
|
14
24
|
return createServer((req, res) => {
|
|
25
|
+
activity.requestStarted();
|
|
26
|
+
res.once('close', () => activity.requestEnded());
|
|
27
|
+
if (!isLocalClient(req))
|
|
28
|
+
return reply(res, 403, errorBody('permission_error', 'router: the gateway serves local clients only'));
|
|
15
29
|
if (req.method === 'GET' && req.url.startsWith('/v1/models')) return discovery(router, res);
|
|
16
30
|
if (req.method === 'GET' && req.url.startsWith(STATUS_PATH)) return status(router, req, res);
|
|
31
|
+
let up = null;
|
|
32
|
+
let closedEarly = false;
|
|
33
|
+
// The client left before its response was complete (Esc in Claude Code, a client timeout). Before the upstream
|
|
34
|
+
// answers, abort the upstream request here; after, pipeline() below does it. Either way the model stops generating.
|
|
35
|
+
res.on('close', () => {
|
|
36
|
+
if (res.writableFinished) return;
|
|
37
|
+
closedEarly = true;
|
|
38
|
+
if (!res.headersSent) up?.destroy();
|
|
39
|
+
});
|
|
17
40
|
const chunks = [];
|
|
18
41
|
req.on('data', (c) => chunks.push(c));
|
|
19
42
|
req.on('end', async () => {
|
|
20
|
-
let body = Buffer.concat(chunks);
|
|
21
|
-
let parsed = null;
|
|
22
43
|
try {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
44
|
+
let body = Buffer.concat(chunks);
|
|
45
|
+
const parsed = parseJson(body);
|
|
46
|
+
const session = sessionKey(req, parsed);
|
|
47
|
+
const turn = isMessagesCall(req);
|
|
48
|
+
if (turn) activity.turnResumed(session);
|
|
49
|
+
let routed = null;
|
|
50
|
+
if (parsed && router.isRouted(parsed)) {
|
|
51
|
+
if (turn) {
|
|
52
|
+
try {
|
|
53
|
+
routed = await router.route(parsed, {
|
|
54
|
+
sessionId: session,
|
|
55
|
+
requestClass: req.headers['x-claude-code-request-class'] ?? null,
|
|
56
|
+
});
|
|
57
|
+
} catch (error) {
|
|
58
|
+
onError(error);
|
|
59
|
+
routed = router.fallback(parsed); // never forward the alias upstream
|
|
60
|
+
}
|
|
61
|
+
} else routed = router.resolveModel(parsed, session);
|
|
62
|
+
body = Buffer.from(JSON.stringify(routed.body));
|
|
38
63
|
}
|
|
39
|
-
|
|
64
|
+
if (closedEarly) return; // the client left while the route was decided
|
|
65
|
+
const headers = {};
|
|
66
|
+
for (const [name, value] of Object.entries(req.headers)) if (!HOP_BY_HOP.has(name)) headers[name] = value;
|
|
67
|
+
headers.host = target.host;
|
|
68
|
+
headers['content-length'] = String(body.length);
|
|
69
|
+
up = send(
|
|
70
|
+
{ host: target.hostname, port: target.port || undefined, path: req.url, method: req.method, headers },
|
|
71
|
+
(upRes) => relay(upRes, res, { routed, session, turn, clientLeft: () => closedEarly }),
|
|
72
|
+
);
|
|
73
|
+
up.on('error', (error) => {
|
|
74
|
+
if (closedEarly) return; // aborted above because the client left
|
|
75
|
+
fail(res, new Error(`upstream: ${error.message}`));
|
|
76
|
+
});
|
|
77
|
+
up.end(body);
|
|
78
|
+
} catch (error) {
|
|
79
|
+
fail(res, error);
|
|
40
80
|
}
|
|
41
|
-
const headers = {};
|
|
42
|
-
for (const [name, value] of Object.entries(req.headers)) if (!HOP_BY_HOP.has(name)) headers[name] = value;
|
|
43
|
-
headers.host = target.host;
|
|
44
|
-
headers['content-length'] = String(body.length);
|
|
45
|
-
const up = send(
|
|
46
|
-
{ host: target.hostname, port: target.port || undefined, path: req.url, method: req.method, headers },
|
|
47
|
-
(upRes) => {
|
|
48
|
-
const reader = routed && !routed.auxiliary ? new UsageReader() : null;
|
|
49
|
-
res.writeHead(upRes.statusCode, upRes.headers);
|
|
50
|
-
upRes.on('data', (c) => {
|
|
51
|
-
if (reader) reader.feed(c.toString('utf8'));
|
|
52
|
-
});
|
|
53
|
-
upRes.on('end', () => {
|
|
54
|
-
if (reader && upRes.statusCode < 300) router.recordResponse(session, routed.tier, reader.end());
|
|
55
|
-
});
|
|
56
|
-
upRes.pipe(res);
|
|
57
|
-
},
|
|
58
|
-
);
|
|
59
|
-
up.on('error', (error) => {
|
|
60
|
-
onError(error);
|
|
61
|
-
if (!res.headersSent) res.writeHead(502);
|
|
62
|
-
res.end();
|
|
63
|
-
});
|
|
64
|
-
up.end(body);
|
|
65
81
|
});
|
|
66
82
|
});
|
|
83
|
+
|
|
84
|
+
function relay(upRes, res, { routed, session, turn, clientLeft }) {
|
|
85
|
+
const reader = turn ? new UsageReader() : null;
|
|
86
|
+
// pipeline() reports ECONNRESET for both kinds of failure; only the side that closed first says whose it was.
|
|
87
|
+
let upstreamBroke = false;
|
|
88
|
+
upRes.once('close', () => {
|
|
89
|
+
upstreamBroke = !upRes.complete && !clientLeft();
|
|
90
|
+
});
|
|
91
|
+
res.writeHead(upRes.statusCode, upRes.headers);
|
|
92
|
+
if (reader) upRes.on('data', (c) => reader.feed(c.toString('utf8')));
|
|
93
|
+
// pipeline() tears down both sides on failure. An upstream reset reaches the client as a reset it retries at once,
|
|
94
|
+
// where pipe() left it waiting for bytes that never came; a client that leaves stops the upstream response.
|
|
95
|
+
pipeline(upRes, res, (error) => {
|
|
96
|
+
if (error) {
|
|
97
|
+
if (upstreamBroke) onError(new Error(`upstream stream: ${error.message}`));
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
if (!reader || upRes.statusCode >= 300) return;
|
|
101
|
+
const usage = reader.end();
|
|
102
|
+
if (reader.stopReason === 'tool_use') activity.turnPaused(session);
|
|
103
|
+
if (!routed || routed.auxiliary) return;
|
|
104
|
+
try {
|
|
105
|
+
router.recordResponse(session, routed.tier, usage);
|
|
106
|
+
} catch (recordError) {
|
|
107
|
+
onError(recordError);
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function fail(res, error) {
|
|
113
|
+
onError(error);
|
|
114
|
+
if (!res.headersSent) res.writeHead(502);
|
|
115
|
+
res.end();
|
|
116
|
+
}
|
|
67
117
|
}
|
|
68
118
|
|
|
69
119
|
// Claude Code sends its session id as a header; the metadata field is the fallback.
|
|
@@ -77,29 +127,64 @@ export function sessionKey(req, body) {
|
|
|
77
127
|
}
|
|
78
128
|
}
|
|
79
129
|
|
|
130
|
+
// A web page can reach a loopback port too: by DNS rebinding (a foreign Host) or by a cross-site request (a foreign
|
|
131
|
+
// Origin). Claude Code sends a loopback Host and no Origin.
|
|
132
|
+
function isLocalClient(req) {
|
|
133
|
+
if (!LOOPBACK.has(hostOf(req.headers.host))) return false;
|
|
134
|
+
const origin = req.headers.origin;
|
|
135
|
+
if (origin === undefined) return true;
|
|
136
|
+
try {
|
|
137
|
+
return LOOPBACK.has(new URL(origin).hostname);
|
|
138
|
+
} catch {
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function hostOf(host = '') {
|
|
144
|
+
return host.replace(/:\d+$/, '').toLowerCase();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Only a Messages call is a turn. Side endpoints such as count_tokens carry the alias too, but no decision.
|
|
148
|
+
function isMessagesCall(req) {
|
|
149
|
+
return req.method === 'POST' && req.url.split('?')[0] === MESSAGES_PATH;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function parseJson(buffer) {
|
|
153
|
+
if (!buffer.length) return null;
|
|
154
|
+
try {
|
|
155
|
+
return JSON.parse(buffer.toString('utf8'));
|
|
156
|
+
} catch {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
80
161
|
function discovery(router, res) {
|
|
81
162
|
const { alias } = router.config.gateway;
|
|
82
|
-
res
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
data: [
|
|
86
|
-
{
|
|
87
|
-
id: alias,
|
|
88
|
-
display_name: ROUTER_DISPLAY_NAME,
|
|
89
|
-
description: routesDescription(router.config),
|
|
90
|
-
},
|
|
91
|
-
],
|
|
92
|
-
}),
|
|
93
|
-
);
|
|
163
|
+
reply(res, 200, {
|
|
164
|
+
data: [{ id: alias, display_name: ROUTER_DISPLAY_NAME, description: routesDescription(router.config) }],
|
|
165
|
+
});
|
|
94
166
|
}
|
|
95
167
|
|
|
96
168
|
function status(router, req, res) {
|
|
97
169
|
const session = new URL(req.url, 'http://localhost').searchParams.get('session');
|
|
98
|
-
|
|
99
|
-
res
|
|
170
|
+
const pausedUntil = router.jevPausedUntil > router.now() ? new Date(router.jevPausedUntil).toISOString() : null;
|
|
171
|
+
reply(res, 200, {
|
|
172
|
+
...statusSnapshot(router.config, session ? router.memory(session) : null),
|
|
173
|
+
pid: process.pid,
|
|
174
|
+
jevPausedUntil: pausedUntil,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function reply(res, statusCode, payload) {
|
|
179
|
+
res.writeHead(statusCode, { 'content-type': 'application/json' });
|
|
180
|
+
res.end(JSON.stringify(payload));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function errorBody(type, message) {
|
|
184
|
+
return { type: 'error', error: { type, message } };
|
|
100
185
|
}
|
|
101
186
|
|
|
102
|
-
// "Picks claude-opus-5-5 / claude-sonnet-
|
|
187
|
+
// "Picks claude-opus-5-5 / claude-sonnet-5 / claude-haiku-4-5 and the effort for each turn"
|
|
103
188
|
export function routesDescription(config) {
|
|
104
189
|
const ids = [...new Set(Object.values(config.routes).map((r) => config.models[r.model].id))];
|
|
105
190
|
return `Picks ${ids.join(' / ')} and the effort for each turn`;
|
package/lib/idle.mjs
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// When the gateway may exit on its own. Claude Code holds no connection open between requests, so an idle session
|
|
2
|
+
// and a closed one look the same here: the signal is time without requests. Two cases must still keep the gateway:
|
|
3
|
+
// a request in flight, and a turn that waits for a tool result (a permission prompt or a question nobody answered
|
|
4
|
+
// yet), whose next request comes without a prompt and so without the hook that would start a gateway again.
|
|
5
|
+
// A session killed mid-turn stops counting after TURN_WAIT_MS.
|
|
6
|
+
export const TURN_WAIT_MS = 24 * 3_600_000;
|
|
7
|
+
|
|
8
|
+
export class IdleTracker {
|
|
9
|
+
constructor({ now = Date.now } = {}) {
|
|
10
|
+
this.now = now;
|
|
11
|
+
this.inFlight = 0;
|
|
12
|
+
this.lastActive = now();
|
|
13
|
+
this.waiting = new Map(); // session -> when its last response asked for a tool
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
requestStarted() {
|
|
17
|
+
this.inFlight += 1;
|
|
18
|
+
this.lastActive = this.now();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
requestEnded() {
|
|
22
|
+
this.inFlight -= 1;
|
|
23
|
+
this.lastActive = this.now();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// A Messages request for the session: whatever it waited for has arrived.
|
|
27
|
+
turnResumed(session) {
|
|
28
|
+
this.waiting.delete(session);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
turnPaused(session) {
|
|
32
|
+
this.waiting.set(session, this.now());
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
idle(idleMs) {
|
|
36
|
+
const now = this.now();
|
|
37
|
+
if (this.inFlight > 0 || now - this.lastActive < idleMs) return false;
|
|
38
|
+
for (const [session, at] of this.waiting) if (now - at >= TURN_WAIT_MS) this.waiting.delete(session);
|
|
39
|
+
return this.waiting.size === 0;
|
|
40
|
+
}
|
|
41
|
+
}
|
package/lib/jev.mjs
CHANGED
|
@@ -34,6 +34,8 @@ const ROUTE_INSTRUCTIONS = {
|
|
|
34
34
|
const CONTINUATION_INSTRUCTIONS =
|
|
35
35
|
'Is `currentRequest.text` a continuation of the task in `recentDialogue` (for example "continue", "yes", "now fix the tests"), rather than a new task?';
|
|
36
36
|
const TRANSIENT = new Set([408, 429, 500, 502, 503, 504]);
|
|
37
|
+
const RETRY_DELAY_MS = 100;
|
|
38
|
+
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
39
|
|
|
38
40
|
export function buildRequest(config, prompt, turns) {
|
|
39
41
|
const criteria = {};
|
|
@@ -49,25 +51,44 @@ export function buildRequest(config, prompt, turns) {
|
|
|
49
51
|
};
|
|
50
52
|
}
|
|
51
53
|
|
|
52
|
-
// Returns { choice, confidence, probabilities, continuation } or throws. One retry on a
|
|
53
|
-
|
|
54
|
+
// Returns { choice, confidence, probabilities, continuation } or throws. One retry on a network error or a transient
|
|
55
|
+
// status, after Retry-After when the server sends one; a wait that would not fit the budget is not taken.
|
|
56
|
+
export async function askJev({ fetchFn, config, apiKey, prompt, turns, now = Date.now, sleep = delay }) {
|
|
54
57
|
const deadline = now() + config.jev.timeoutMs;
|
|
55
58
|
const body = JSON.stringify(buildRequest(config, prompt, turns));
|
|
56
59
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
57
60
|
const remaining = deadline - now();
|
|
58
61
|
if (remaining <= 0) throw new Error('jev timeout');
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
let response;
|
|
63
|
+
try {
|
|
64
|
+
response = await fetchFn(config.jev.endpoint, {
|
|
65
|
+
method: 'POST',
|
|
66
|
+
headers: { Authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
|
|
67
|
+
body,
|
|
68
|
+
signal: AbortSignal.timeout(remaining),
|
|
69
|
+
});
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error.name === 'TimeoutError') throw new Error('jev timeout');
|
|
72
|
+
if (attempt === 1) throw new Error(`jev unreachable: ${error.message}`);
|
|
73
|
+
await sleep(RETRY_DELAY_MS);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
65
76
|
if (response.ok) return parseAnswers(await response.json());
|
|
66
77
|
if (!TRANSIENT.has(response.status) || attempt === 1) throw new Error(`jev http ${response.status}`);
|
|
78
|
+
const wait = retryAfterMs(response.headers) ?? RETRY_DELAY_MS;
|
|
79
|
+
if (wait >= deadline - now()) throw new Error(`jev http ${response.status}, retry-after beyond the budget`);
|
|
80
|
+
await sleep(wait);
|
|
67
81
|
}
|
|
68
82
|
throw new Error('jev unreachable');
|
|
69
83
|
}
|
|
70
84
|
|
|
85
|
+
function retryAfterMs(headers) {
|
|
86
|
+
const value = headers?.get?.('retry-after');
|
|
87
|
+
if (value === null || value === undefined || value === '') return null;
|
|
88
|
+
const seconds = Number(value);
|
|
89
|
+
return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : null;
|
|
90
|
+
}
|
|
91
|
+
|
|
71
92
|
export function parseAnswers(json) {
|
|
72
93
|
const route = json?.answers?.route;
|
|
73
94
|
if (route?.type !== 'choice' || typeof route.probabilities !== 'object') throw new Error('jev malformed answer');
|
package/lib/policy.mjs
CHANGED
|
@@ -56,6 +56,23 @@ export function decide({ config, facts, advice, state, baseline, now }) {
|
|
|
56
56
|
return stay('downgrade-pending', estimate);
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
// Share of a model's window the next request may fill. The estimate leaves out the new prompt and tool
|
|
60
|
+
// results, so a model drops out before the conversation reaches its limit.
|
|
61
|
+
const CONTEXT_FILL = 0.8;
|
|
62
|
+
|
|
63
|
+
// The lowest tier at or above `tier` whose model fits `tokens` of context. Routes need not grow in window
|
|
64
|
+
// with rank, so the search falls back to any tier that fits, then to the largest window, keeping `tier` on a tie.
|
|
65
|
+
export function fitTier(config, tier, tokens) {
|
|
66
|
+
const windowOf = (t) => config.models[config.routes[t].model].contextWindow;
|
|
67
|
+
const fits = (t) => tokens <= windowOf(t) * CONTEXT_FILL;
|
|
68
|
+
if (fits(tier)) return tier;
|
|
69
|
+
return (
|
|
70
|
+
TIERS.slice(rank(tier)).find(fits) ??
|
|
71
|
+
TIERS.find(fits) ??
|
|
72
|
+
TIERS.reduce((best, t) => (windowOf(t) > windowOf(best) ? t : best), tier)
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
59
76
|
// Automatic routing to a credits-billed model must fit the cash cap unless its cache is warm.
|
|
60
77
|
function cashGate(config, tier, facts, now) {
|
|
61
78
|
const alias = config.routes[tier].model;
|
package/lib/router.mjs
CHANGED
|
@@ -1,23 +1,40 @@
|
|
|
1
1
|
// Per-request orchestration: facts -> advice -> policy -> rewritten body. Knows nothing about HTTP.
|
|
2
|
+
import { nextContextTokens } from './cost.mjs';
|
|
2
3
|
import { factsFromRequest } from './facts.mjs';
|
|
3
4
|
import { askJev } from './jev.mjs';
|
|
4
|
-
import { decide, initialState } from './policy.mjs';
|
|
5
|
+
import { decide, fitTier, initialState } from './policy.mjs';
|
|
5
6
|
import { rewriteRequest } from './rewrite.mjs';
|
|
6
7
|
import { appendLog, loadMemory, saveMemory } from './store.mjs';
|
|
7
8
|
|
|
8
9
|
const COMPACTION_SHRINK = 0.8;
|
|
10
|
+
// Jev down: after this many failures in a row, new turns skip it for the cooldown instead of waiting out its timeout.
|
|
11
|
+
export const JEV_FAILURES_TO_PAUSE = 3;
|
|
12
|
+
export const JEV_PAUSE_MS = 60_000;
|
|
13
|
+
// Reasons that reuse the route of the turn and must not replace the reason that chose it.
|
|
14
|
+
const REUSED_ROUTE = new Set(['tool-continuation', 'retry']);
|
|
9
15
|
|
|
10
16
|
export function emptyMemory() {
|
|
11
|
-
return {
|
|
17
|
+
return {
|
|
18
|
+
lastRoute: null,
|
|
19
|
+
lastReason: null,
|
|
20
|
+
lastEffort: null,
|
|
21
|
+
lastRequest: null,
|
|
22
|
+
lastTurnKey: null,
|
|
23
|
+
models: {},
|
|
24
|
+
state: null,
|
|
25
|
+
};
|
|
12
26
|
}
|
|
13
27
|
|
|
14
28
|
export class Router {
|
|
15
|
-
constructor({ config, fetchFn, dataDir, now = Date.now }) {
|
|
29
|
+
constructor({ config, fetchFn, dataDir, now = Date.now, onError = () => {} }) {
|
|
16
30
|
this.config = config;
|
|
17
31
|
this.fetchFn = fetchFn;
|
|
18
32
|
this.dataDir = dataDir;
|
|
19
33
|
this.now = now;
|
|
34
|
+
this.onError = onError;
|
|
20
35
|
this.memories = new Map();
|
|
36
|
+
this.jevFailures = 0;
|
|
37
|
+
this.jevPausedUntil = 0;
|
|
21
38
|
}
|
|
22
39
|
|
|
23
40
|
isRouted(body) {
|
|
@@ -34,13 +51,20 @@ export class Router {
|
|
|
34
51
|
if (auxiliary) decision = { tier: this.config.gateway.auxiliaryTier, reason: 'auxiliary', state: memory.state };
|
|
35
52
|
else if (facts.continuation && memory.lastRoute)
|
|
36
53
|
decision = { tier: memory.lastRoute, reason: 'tool-continuation', state: memory.state };
|
|
54
|
+
else if (facts.turnKey && facts.turnKey === memory.lastTurnKey && memory.lastRoute)
|
|
55
|
+
decision = { tier: memory.lastRoute, reason: 'retry', state: memory.state };
|
|
37
56
|
else decision = await this.decideTurn(facts, memory);
|
|
57
|
+
// Main turns and compaction carry the main conversation; other side requests have their own, unknown size.
|
|
58
|
+
if (!auxiliary || requestClass === 'compaction') {
|
|
59
|
+
const tier = fitTier(this.config, decision.tier, nextContextTokens(facts));
|
|
60
|
+
if (tier !== decision.tier) decision = { ...decision, tier, reason: 'context-fit' };
|
|
61
|
+
}
|
|
38
62
|
const rewritten = rewriteRequest(body, decision.tier, this.config);
|
|
39
63
|
if (!auxiliary) {
|
|
40
64
|
memory.lastRoute = decision.tier;
|
|
41
65
|
memory.lastEffort = rewritten.output_config?.effort ?? null;
|
|
42
|
-
|
|
43
|
-
if (decision.reason
|
|
66
|
+
memory.lastTurnKey = facts.turnKey;
|
|
67
|
+
if (!REUSED_ROUTE.has(decision.reason)) memory.lastReason = decision.reason;
|
|
44
68
|
memory.state = decision.state;
|
|
45
69
|
this.persist(sessionId, memory);
|
|
46
70
|
}
|
|
@@ -68,6 +92,12 @@ export class Router {
|
|
|
68
92
|
return { body: rewriteRequest(body, tier, this.config), tier, reason: 'error', auxiliary: true };
|
|
69
93
|
}
|
|
70
94
|
|
|
95
|
+
// Side endpoints (count_tokens) name the alias too. They get the session's current model: no Jev call, no vote.
|
|
96
|
+
resolveModel(body, sessionId) {
|
|
97
|
+
const tier = this.memory(sessionId).lastRoute ?? this.config.gateway.baselineTier;
|
|
98
|
+
return { body: rewriteRequest(body, tier, this.config), tier, reason: 'side-endpoint', auxiliary: true };
|
|
99
|
+
}
|
|
100
|
+
|
|
71
101
|
async decideTurn(facts, memory) {
|
|
72
102
|
const state = memory.state ?? initialState();
|
|
73
103
|
const { forcedTier, apiKey, gateway } = this.config;
|
|
@@ -75,17 +105,22 @@ export class Router {
|
|
|
75
105
|
let advice = null;
|
|
76
106
|
let adviceError = null;
|
|
77
107
|
if (apiKey && facts.prompt) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
108
|
+
if (this.now() < this.jevPausedUntil) adviceError = 'jev paused after repeated failures';
|
|
109
|
+
else {
|
|
110
|
+
try {
|
|
111
|
+
advice = await askJev({
|
|
112
|
+
fetchFn: this.fetchFn,
|
|
113
|
+
config: this.config,
|
|
114
|
+
apiKey,
|
|
115
|
+
prompt: facts.prompt,
|
|
116
|
+
turns: facts.turns,
|
|
117
|
+
now: this.now,
|
|
118
|
+
});
|
|
119
|
+
this.jevSucceeded();
|
|
120
|
+
} catch (error) {
|
|
121
|
+
adviceError = error.message;
|
|
122
|
+
this.jevFailed(error);
|
|
123
|
+
}
|
|
89
124
|
}
|
|
90
125
|
}
|
|
91
126
|
const decision = decide({
|
|
@@ -99,6 +134,25 @@ export class Router {
|
|
|
99
134
|
return { ...decision, advice, adviceError };
|
|
100
135
|
}
|
|
101
136
|
|
|
137
|
+
jevSucceeded() {
|
|
138
|
+
if (this.jevFailures >= JEV_FAILURES_TO_PAUSE) this.onError(new Error('jev answers again, routing resumed'));
|
|
139
|
+
this.jevFailures = 0;
|
|
140
|
+
this.jevPausedUntil = 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// The count stays at or above the threshold while Jev keeps failing: after a pause, one failed try pauses again.
|
|
144
|
+
jevFailed(error) {
|
|
145
|
+
this.jevFailures += 1;
|
|
146
|
+
if (this.jevFailures < JEV_FAILURES_TO_PAUSE) return;
|
|
147
|
+
this.jevPausedUntil = this.now() + JEV_PAUSE_MS;
|
|
148
|
+
if (this.jevFailures === JEV_FAILURES_TO_PAUSE)
|
|
149
|
+
this.onError(
|
|
150
|
+
new Error(
|
|
151
|
+
`jev failed ${JEV_FAILURES_TO_PAUSE} times in a row (${error.message}): new turns use the baseline, one try every ${JEV_PAUSE_MS / 1000}s`,
|
|
152
|
+
),
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
|
|
102
156
|
// Called with the usage the gateway read from a forwarded main-conversation response.
|
|
103
157
|
recordResponse(sessionId, tier, usage) {
|
|
104
158
|
if (!usage) return;
|
|
@@ -125,12 +179,22 @@ export class Router {
|
|
|
125
179
|
return this.memories.get(sessionId);
|
|
126
180
|
}
|
|
127
181
|
|
|
182
|
+
// Disk trouble (full, read-only) must not cost the routing decision: the in-memory state stays authoritative.
|
|
128
183
|
persist(sessionId, memory) {
|
|
129
|
-
|
|
184
|
+
try {
|
|
185
|
+
saveMemory(this.dataDir, sessionId, memory);
|
|
186
|
+
} catch (error) {
|
|
187
|
+
this.onError(error);
|
|
188
|
+
}
|
|
130
189
|
}
|
|
131
190
|
|
|
132
191
|
log(entry) {
|
|
133
|
-
if (this.config.log)
|
|
192
|
+
if (!this.config.log) return;
|
|
193
|
+
try {
|
|
194
|
+
appendLog(this.dataDir, { at: new Date(this.now()).toISOString(), ...entry });
|
|
195
|
+
} catch (error) {
|
|
196
|
+
this.onError(error);
|
|
197
|
+
}
|
|
134
198
|
}
|
|
135
199
|
}
|
|
136
200
|
|
package/lib/sse.mjs
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
// Incremental usage reader for a Messages response: streaming SSE or a plain JSON body.
|
|
1
|
+
// Incremental usage and stop-reason reader for a Messages response: streaming SSE or a plain JSON body.
|
|
2
2
|
export class UsageReader {
|
|
3
3
|
constructor() {
|
|
4
4
|
this.buffer = '';
|
|
5
5
|
this.usage = null;
|
|
6
6
|
this.model = null;
|
|
7
|
+
this.stopReason = null;
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
feed(chunk) {
|
|
@@ -34,7 +35,11 @@ export class UsageReader {
|
|
|
34
35
|
this.model = message.model ?? this.model;
|
|
35
36
|
this.usage = { ...(this.usage ?? {}), ...message.usage };
|
|
36
37
|
}
|
|
37
|
-
if (
|
|
38
|
+
if (message?.stop_reason) this.stopReason = message.stop_reason;
|
|
39
|
+
if (event.type === 'message_delta') {
|
|
40
|
+
if (event.usage) this.usage = { ...(this.usage ?? {}), ...event.usage };
|
|
41
|
+
if (event.delta?.stop_reason) this.stopReason = event.delta.stop_reason;
|
|
42
|
+
}
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
result() {
|
package/lib/status.mjs
CHANGED
|
@@ -1,8 +1,20 @@
|
|
|
1
1
|
// Router status: the snapshot the gateway serves at GET /router/status, and its text forms for the
|
|
2
2
|
// status line and /router:status. The snapshot never carries the API key, only whether one is set.
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
3
4
|
import { TIERS } from './config.mjs';
|
|
4
5
|
import { clampEffort } from './rewrite.mjs';
|
|
5
6
|
|
|
7
|
+
const _require = createRequire(import.meta.url);
|
|
8
|
+
export const ROUTER_VERSION = _require('../package.json').version;
|
|
9
|
+
|
|
10
|
+
// True when `running` is older than `current`. Gateways before 0.3.0 report no version: they count as older.
|
|
11
|
+
export function isOlderVersion(running, current) {
|
|
12
|
+
const parse = (v) => (/^\d+\.\d+\.\d+/.test(v ?? '') ? v.split('.').map((n) => Number.parseInt(n, 10)) : [0, 0, 0]);
|
|
13
|
+
const [a, b] = [parse(running), parse(current)];
|
|
14
|
+
for (let i = 0; i < 3; i += 1) if (a[i] !== b[i]) return a[i] < b[i];
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
|
|
6
18
|
export const STATUS_PATH = '/router/status';
|
|
7
19
|
// The name of the alias in the /model picker and in discovery.
|
|
8
20
|
export const ROUTER_DISPLAY_NAME = 'Router (auto)';
|
|
@@ -11,6 +23,7 @@ const FETCH_TIMEOUT_MS = 300;
|
|
|
11
23
|
export function statusSnapshot(config, memory) {
|
|
12
24
|
const { alias, port, baselineTier } = config.gateway;
|
|
13
25
|
return {
|
|
26
|
+
version: ROUTER_VERSION,
|
|
14
27
|
alias,
|
|
15
28
|
port,
|
|
16
29
|
keySet: Boolean(config.apiKey),
|
|
@@ -38,23 +51,24 @@ function routeRow(config, tier) {
|
|
|
38
51
|
return { tier, model: model.id, effort };
|
|
39
52
|
}
|
|
40
53
|
|
|
41
|
-
// One status-line segment, e.g. "router ▸ opus-5-5 · xhigh
|
|
54
|
+
// One status-line segment, e.g. "router ▸ opus-5-5 · xhigh".
|
|
42
55
|
export function statusSegment(status) {
|
|
43
|
-
if (!status) return 'router: gateway
|
|
56
|
+
if (!status) return 'router: gateway off, the next prompt starts it';
|
|
44
57
|
const last = status.session;
|
|
45
58
|
if (!last) return `${status.alias}: no turn yet`;
|
|
46
59
|
const model = shortModel(last.model ?? status.routes.find((r) => r.tier === last.tier)?.model);
|
|
47
60
|
const effort = last.effort ? ` · ${last.effort}` : '';
|
|
48
|
-
return `${status.alias} ▸ ${model}${effort}
|
|
61
|
+
return `${status.alias} ▸ ${model}${effort}`;
|
|
49
62
|
}
|
|
50
63
|
|
|
51
64
|
// Markdown for /router:status.
|
|
52
65
|
export function statusReport(status) {
|
|
53
|
-
if (!status)
|
|
66
|
+
if (!status)
|
|
67
|
+
return 'The router gateway is not running. The next prompt starts it. If it does not start, read `gateway.log` in the plugin data directory.';
|
|
54
68
|
const lines = [
|
|
55
|
-
`
|
|
56
|
-
`Jev
|
|
57
|
-
`
|
|
69
|
+
`Router v${status.version}, gateway: http://127.0.0.1:${status.port}, alias \`${status.alias}\`.`,
|
|
70
|
+
`Jev routing: ${jevState(status)}.`,
|
|
71
|
+
`Default tier: ${status.baselineTier}.${status.forcedTier ? ` Forced tier: ${status.forcedTier}.` : ''}`,
|
|
58
72
|
'',
|
|
59
73
|
'| Tier | Model | Effort |',
|
|
60
74
|
'| ---- | ----- | ------ |',
|
|
@@ -85,6 +99,12 @@ export async function fetchStatus(port, sessionId) {
|
|
|
85
99
|
}
|
|
86
100
|
}
|
|
87
101
|
|
|
102
|
+
function jevState(status) {
|
|
103
|
+
if (!status.keySet) return 'inactive, no key: every turn runs on the default tier';
|
|
104
|
+
if (status.jevPausedUntil) return `paused after repeated failures, next try at ${status.jevPausedUntil}`;
|
|
105
|
+
return 'active';
|
|
106
|
+
}
|
|
107
|
+
|
|
88
108
|
function shortModel(id) {
|
|
89
109
|
return id ? id.replace(/^claude-/, '') : 'unknown';
|
|
90
110
|
}
|