@drsamonline/router 0.0.0-stage → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +26 -4
- package/src/breaker.js +113 -0
- package/src/classify.js +88 -0
- package/src/explain.js +74 -0
- package/src/index.js +5 -0
- package/src/router.js +963 -0
- package/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drsamonline/router",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.12.2",
|
|
4
|
+
"description": "Routing decisions, strategies, circuit breaker.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "./src/index.js",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": "./src/index.js",
|
|
10
|
+
"./src/*": "./src/*"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"src",
|
|
14
|
+
"web",
|
|
15
|
+
"bin",
|
|
16
|
+
"LICENSE"
|
|
17
|
+
],
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git://github.com/drsamonline/sui.git",
|
|
21
|
+
"directory": "packages/router"
|
|
22
|
+
},
|
|
23
|
+
"homepage": "https://github.com/drsamonline/sui#readme",
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public",
|
|
26
|
+
"registry": "https://registry.npmjs.org/"
|
|
27
|
+
}
|
|
28
|
+
}
|
package/src/breaker.js
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-target circuit breaker.
|
|
3
|
+
*
|
|
4
|
+
* closed -> (failures >= threshold) -> open
|
|
5
|
+
* open -> (cooldown elapsed) -> half-open (one trial request allowed)
|
|
6
|
+
* half-open -> success -> closed
|
|
7
|
+
* half-open -> failure -> open, cooldown restarts
|
|
8
|
+
*
|
|
9
|
+
* The breaker is the reason a rate-limited provider costs one failed request
|
|
10
|
+
* instead of every request until its quota resets.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const CLOSED = 'closed';
|
|
14
|
+
const OPEN = 'open';
|
|
15
|
+
const HALF_OPEN = 'half_open';
|
|
16
|
+
|
|
17
|
+
export class CircuitBreaker {
|
|
18
|
+
/**
|
|
19
|
+
* @param {{failures?: number, cooldownMs?: number, halfOpenSuccesses?: number, now?: () => number}} [opts]
|
|
20
|
+
*/
|
|
21
|
+
constructor(opts = {}) {
|
|
22
|
+
this.threshold = opts.failures ?? 3;
|
|
23
|
+
this.cooldownMs = opts.cooldownMs ?? 30000;
|
|
24
|
+
this.halfOpenSuccesses = opts.halfOpenSuccesses ?? 1;
|
|
25
|
+
this.now = opts.now ?? Date.now;
|
|
26
|
+
/** @type {Map<string, {state: string, failures: number, openedAt: number, successes: number, lastError: string|null}>} */
|
|
27
|
+
this.targets = new Map();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** @param {string} key */
|
|
31
|
+
#entry(key) {
|
|
32
|
+
let e = this.targets.get(key);
|
|
33
|
+
if (!e) {
|
|
34
|
+
e = { state: CLOSED, failures: 0, openedAt: 0, successes: 0, lastError: null };
|
|
35
|
+
this.targets.set(key, e);
|
|
36
|
+
}
|
|
37
|
+
return e;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** @param {string} key */
|
|
41
|
+
canAttempt(key) {
|
|
42
|
+
const e = this.#entry(key);
|
|
43
|
+
if (e.state === CLOSED) return true;
|
|
44
|
+
if (e.state === OPEN) {
|
|
45
|
+
if (this.now() - e.openedAt >= this.cooldownMs) {
|
|
46
|
+
e.state = HALF_OPEN;
|
|
47
|
+
e.successes = 0;
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
// half-open: allow the trial through
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** @param {string} key */
|
|
57
|
+
recordSuccess(key) {
|
|
58
|
+
const e = this.#entry(key);
|
|
59
|
+
if (e.state === HALF_OPEN) {
|
|
60
|
+
e.successes += 1;
|
|
61
|
+
if (e.successes >= this.halfOpenSuccesses) {
|
|
62
|
+
e.state = CLOSED;
|
|
63
|
+
e.failures = 0;
|
|
64
|
+
e.lastError = null;
|
|
65
|
+
}
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
e.state = CLOSED;
|
|
69
|
+
e.failures = 0;
|
|
70
|
+
e.lastError = null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* @param {string} key
|
|
75
|
+
* @param {string} [reason]
|
|
76
|
+
*/
|
|
77
|
+
recordFailure(key, reason) {
|
|
78
|
+
const e = this.#entry(key);
|
|
79
|
+
e.failures += 1;
|
|
80
|
+
e.lastError = reason ?? e.lastError;
|
|
81
|
+
if (e.state === HALF_OPEN || e.failures >= this.threshold) {
|
|
82
|
+
e.state = OPEN;
|
|
83
|
+
e.openedAt = this.now();
|
|
84
|
+
e.successes = 0;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** @param {string} key */
|
|
89
|
+
reset(key) {
|
|
90
|
+
this.targets.delete(key);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
resetAll() {
|
|
94
|
+
this.targets.clear();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** @param {string} key */
|
|
98
|
+
stateOf(key) {
|
|
99
|
+
const e = this.#entry(key);
|
|
100
|
+
const cooling = e.state === OPEN ? Math.max(0, this.cooldownMs - (this.now() - e.openedAt)) : 0;
|
|
101
|
+
return {
|
|
102
|
+
key,
|
|
103
|
+
state: e.state,
|
|
104
|
+
failures: e.failures,
|
|
105
|
+
lastError: e.lastError,
|
|
106
|
+
retryInMs: cooling,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
snapshot() {
|
|
111
|
+
return [...this.targets.keys()].map((k) => this.stateOf(k));
|
|
112
|
+
}
|
|
113
|
+
}
|
package/src/classify.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @drsamonline/router/classify — the 50 ms classifier (B3.5).
|
|
3
|
+
*
|
|
4
|
+
* Before a request goes anywhere, decide whether it needs the expensive model.
|
|
5
|
+
* The classifier is deterministic and has no model of its own: it reads the
|
|
6
|
+
* prompt's shape, and it must answer in well under 50 ms, because a classifier
|
|
7
|
+
* that costs a request has defeated the point of saving one.
|
|
8
|
+
*
|
|
9
|
+
* Signals, each of which can be pointed at in a test:
|
|
10
|
+
*
|
|
11
|
+
* length a long prompt is usually a real task, not a lookup
|
|
12
|
+
* code fenced code, or a request to write/debug code
|
|
13
|
+
* multi-step "first … then", numbered steps, "after that"
|
|
14
|
+
* reasoning arithmetic, comparison, "why", "prove", "explain the trade-off"
|
|
15
|
+
* schema JSON/schema/tool output wanted
|
|
16
|
+
* terse a short factual question with no verbs that ask for work
|
|
17
|
+
*
|
|
18
|
+
* The output is a decision, not a score: `simple` or `complex`, plus the signals
|
|
19
|
+
* that produced it so `sui explain` can say why.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const COMPLEX_MARKERS = [
|
|
23
|
+
'first', 'then', 'after that', 'step by step', 'steps', 'walk me through',
|
|
24
|
+
'prove', 'derive', 'trade-off', 'tradeoff', 'compare', 'why does', 'why is',
|
|
25
|
+
'architect', 'design', 'migrate', 'refactor', 'optimi', 'debug', 'root cause',
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
const CODE_MARKERS = [
|
|
29
|
+
'```', 'function', 'class ', 'import ', 'select ', 'regex', 'sql', 'typescript',
|
|
30
|
+
'javascript', 'python', 'rust', 'golang', 'compile', 'stack trace', 'unit test',
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
const SCHEMA_MARKERS = ['json', 'schema', 'tool_call', 'function call', 'yaml', 'csv', 'table'];
|
|
34
|
+
|
|
35
|
+
const TRIVIAL_MARKERS = ['what is', 'who is', 'when did', 'how many', 'capital of', 'define'];
|
|
36
|
+
|
|
37
|
+
/** Cheap token estimate: the classifier must not parse, just count. */
|
|
38
|
+
function words(text) {
|
|
39
|
+
let count = 0;
|
|
40
|
+
for (let i = 0; i < text.length; i++) if (text[i] === ' ' || text[i] === '\n') count += 1;
|
|
41
|
+
return count + 1;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @param {string} text
|
|
46
|
+
* @param {{maxMs?: number, longWords?: number}} [opts]
|
|
47
|
+
* @returns {{complexity: 'simple'|'complex', signals: string[], ms: number, confidence: number}}
|
|
48
|
+
*/
|
|
49
|
+
export function classifyPrompt(text, opts = {}) {
|
|
50
|
+
const started = process.hrtime.bigint();
|
|
51
|
+
const prompt = String(text ?? '');
|
|
52
|
+
const lower = prompt.toLowerCase();
|
|
53
|
+
const signals = [];
|
|
54
|
+
|
|
55
|
+
const wordCount = words(prompt);
|
|
56
|
+
const longWords = opts.longWords ?? 120;
|
|
57
|
+
|
|
58
|
+
if (wordCount > longWords) signals.push(`long prompt (${wordCount} words)`);
|
|
59
|
+
if (CODE_MARKERS.some((m) => lower.includes(m))) signals.push('code');
|
|
60
|
+
if (SCHEMA_MARKERS.some((m) => lower.includes(m))) signals.push('schema');
|
|
61
|
+
const multiStep = COMPLEX_MARKERS.filter((m) => lower.includes(m));
|
|
62
|
+
if (multiStep.length) signals.push(`multi-step (${multiStep.slice(0, 3).join(', ')})`);
|
|
63
|
+
if (/\d+\s*[-+*/x×]\s*\d+/.test(prompt)) signals.push('arithmetic');
|
|
64
|
+
if (/\?[^?]*\?/.test(prompt)) signals.push('multiple questions');
|
|
65
|
+
|
|
66
|
+
// A short, single-question lookup with none of the work markers is simple.
|
|
67
|
+
const looksTrivial =
|
|
68
|
+
wordCount < 40 &&
|
|
69
|
+
!signals.includes('code') &&
|
|
70
|
+
!signals.includes('schema') &&
|
|
71
|
+
multiStep.length === 0 &&
|
|
72
|
+
TRIVIAL_MARKERS.some((m) => lower.startsWith(m) || lower.includes(m));
|
|
73
|
+
|
|
74
|
+
const complexity = signals.length === 0 || (looksTrivial && signals.length <= 1) ? 'simple' : 'complex';
|
|
75
|
+
|
|
76
|
+
// Confidence is about the distance from the decision, not about being right.
|
|
77
|
+
const confidence =
|
|
78
|
+
complexity === 'complex'
|
|
79
|
+
? Math.min(0.95, 0.5 + 0.15 * signals.length)
|
|
80
|
+
: Math.max(0.55, 0.9 - 0.1 * signals.length);
|
|
81
|
+
|
|
82
|
+
const ms = Number(process.hrtime.bigint() - started) / 1e6;
|
|
83
|
+
if (opts.maxMs && ms > opts.maxMs) {
|
|
84
|
+
signals.push(`classifier took ${ms.toFixed(1)}ms (budget ${opts.maxMs}ms)`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return { complexity, signals, ms: Math.round(ms * 1000) / 1000, confidence: Math.round(confidence * 100) / 100 };
|
|
88
|
+
}
|
package/src/explain.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @drsamonline/router/explain — B7.4, explain-my-route in plain language.
|
|
3
|
+
*
|
|
4
|
+
* A route decision is only trustworthy if it can be narrated. These functions
|
|
5
|
+
* turn the plan (a pure data structure) and a turn's meta into sentences a person
|
|
6
|
+
* can read, without a second query and without storing anything extra.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {{combo: any, ordered: any[], resting: any[], skipped: any[]}} plan
|
|
11
|
+
* @returns {string[]}
|
|
12
|
+
*/
|
|
13
|
+
export function explainPlan(plan) {
|
|
14
|
+
const lines = [];
|
|
15
|
+
const combo = plan.combo;
|
|
16
|
+
lines.push(`"${combo.name}" is a ${combo.strategy} route with ${combo.targets.length} target${combo.targets.length === 1 ? '' : 's'}.`);
|
|
17
|
+
|
|
18
|
+
if (plan.ordered?.length) {
|
|
19
|
+
const names = plan.ordered.map((t) => t.modelId);
|
|
20
|
+
lines.push(`It would try ${names.join(' → ')}.`);
|
|
21
|
+
if (combo.strategy === 'cheapest') lines.push('Targets are ordered by blended cost, after the quality floor.');
|
|
22
|
+
if (combo.strategy === 'weighted') lines.push('The first target is picked by weight; the rest are fallbacks in weight order.');
|
|
23
|
+
if (combo.strategy === 'latency') lines.push('Targets are ordered by measured p50 from the ledger.');
|
|
24
|
+
if (plan.learned) {
|
|
25
|
+
lines.push('Fallback learning is on for this combo: targets with enough recent samples are ordered by success rate, then latency, and the declared order breaks ties.');
|
|
26
|
+
}
|
|
27
|
+
} else {
|
|
28
|
+
lines.push('No target is available right now.');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
for (const skip of plan.skipped ?? []) {
|
|
32
|
+
lines.push(`Skipped ${skip.modelId}: ${skip.reason}.`);
|
|
33
|
+
}
|
|
34
|
+
for (const resting of plan.resting ?? []) {
|
|
35
|
+
lines.push(`Held back ${resting.modelId}: its breaker is open, so it is only tried if everything else fails.`);
|
|
36
|
+
}
|
|
37
|
+
return lines;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {any} meta a completed turn's meta
|
|
42
|
+
* @returns {string[]}
|
|
43
|
+
*/
|
|
44
|
+
export function explainTurn(meta) {
|
|
45
|
+
if (!meta) return ['No turn to explain.'];
|
|
46
|
+
const lines = [];
|
|
47
|
+
lines.push(`Answered by ${meta.modelId} (${meta.providerName ?? meta.provider}) in ${meta.latencyMs} ms${meta.cached ? ', from cache' : ''}.`);
|
|
48
|
+
|
|
49
|
+
for (const attempt of meta.attempts ?? []) {
|
|
50
|
+
lines.push(`${attempt.modelId} did not answer: ${attempt.reason}.`);
|
|
51
|
+
}
|
|
52
|
+
if (meta.escalation) lines.push(meta.escalation);
|
|
53
|
+
|
|
54
|
+
if (meta.compression?.preset || meta.compression?.style) {
|
|
55
|
+
const bits = [];
|
|
56
|
+
if (meta.compression.preset) bits.push(`compression ${meta.compression.preset} (${Math.round((meta.compression.ratio ?? 0) * 100)}% smaller)`);
|
|
57
|
+
if (meta.compression.style) bits.push(`output style ${meta.compression.style}`);
|
|
58
|
+
lines.push(`Shaped the request: ${bits.join(', ')}.`);
|
|
59
|
+
}
|
|
60
|
+
if (meta.verification) {
|
|
61
|
+
lines.push(
|
|
62
|
+
meta.verification.verified
|
|
63
|
+
? `Confidence ${meta.verification.confidence} clears the ${meta.verification.threshold} threshold.`
|
|
64
|
+
: `Confidence ${meta.verification.confidence} is below ${meta.verification.threshold}: ${(meta.verification.reasons ?? []).map((r) => r.detail).join('; ') || 'no detail'}. Treated as unverified.`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
if (meta.promptCache) {
|
|
68
|
+
lines.push(meta.promptCache.hint ? `Prompt cache: ${meta.promptCache.hint}.` : `Prompt cache unavailable: ${meta.promptCache.unsupported}.`);
|
|
69
|
+
if (meta.promptCache.volatile?.length) {
|
|
70
|
+
lines.push(`The prefix carries ${meta.promptCache.volatile.map((v) => v.name).join(', ')}, which breaks provider caching.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return lines;
|
|
74
|
+
}
|
package/src/index.js
ADDED
package/src/router.js
ADDED
|
@@ -0,0 +1,963 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The router.
|
|
3
|
+
*
|
|
4
|
+
* A request names a *combo* — a virtual model — and the router decides which
|
|
5
|
+
* real provider answers it, in what order, and what to do when one of them is
|
|
6
|
+
* resting, out of credit or over budget.
|
|
7
|
+
*
|
|
8
|
+
* Strategies:
|
|
9
|
+
* fallback declared order, first healthy target wins
|
|
10
|
+
* weighted one target picked by weight (A/B test), the rest as fallback
|
|
11
|
+
* cheapest lowest blended cost among targets that clear the quality floor
|
|
12
|
+
* latency fastest measured p50 among healthy targets
|
|
13
|
+
*
|
|
14
|
+
* Every attempt is written to the ledger, including failures, so "why did this
|
|
15
|
+
* go to Gemini?" is answerable afterwards.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { ConfigError, ExhaustedError, ProviderError, SuiError } from '@drsamonline/core';
|
|
19
|
+
import { estimatedCost, estimateTokens, isPaidProvider, privacyRefusal, providerHasCredentials, providerMeetsPrivacyTier, providerMeetsRegion, regionRefusal, resolveCombo, tierRank } from '@drsamonline/core';
|
|
20
|
+
import { ResponseCache } from '@drsamonline/telemetry';
|
|
21
|
+
import { RecallStore } from '@drsamonline/compression';
|
|
22
|
+
import { PrefixMemory, preparePromptCache } from '@drsamonline/promptcache';
|
|
23
|
+
import { classifyPrompt } from './classify.js';
|
|
24
|
+
import { checkGrounding, scoreAnswer, shouldEscalate, verdict as verifyVerdict } from '@drsamonline/verify';
|
|
25
|
+
import { createAdapter, displayName } from '@drsamonline/providers';
|
|
26
|
+
import { checkPrompt, clampMaxTokens, inspectToolChannel } from '@drsamonline/guardrails';
|
|
27
|
+
import { applyStyle, choosePreset, chooseStyle, compress as compressText } from '@drsamonline/compression';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @typedef {{
|
|
31
|
+
* provider: string, providerName: string, model: string, modelId: string, combo: string,
|
|
32
|
+
* arm: string|null, cached: boolean, verified: boolean, latencyMs: number, ttftMs: number|null,
|
|
33
|
+
* costUsd: number, usage: {prompt_tokens: number, completion_tokens: number},
|
|
34
|
+
* strategy: string, attempts: Array<{modelId: string, provider: string, reason: string}>
|
|
35
|
+
* }} TurnMeta
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export class Router {
|
|
39
|
+
/**
|
|
40
|
+
* @param {{
|
|
41
|
+
* config: any, ledger: any, breaker: any, cache: any,
|
|
42
|
+
* logger?: {debug: Function, info: Function, warn: Function},
|
|
43
|
+
* now?: () => number, random?: () => number
|
|
44
|
+
* }} deps
|
|
45
|
+
*/
|
|
46
|
+
constructor(deps) {
|
|
47
|
+
this.config = deps.config;
|
|
48
|
+
this.ledger = deps.ledger;
|
|
49
|
+
this.breaker = deps.breaker;
|
|
50
|
+
this.cache = deps.cache;
|
|
51
|
+
this.now = deps.now ?? Date.now;
|
|
52
|
+
this.random = deps.random ?? Math.random;
|
|
53
|
+
this.logger = deps.logger ?? { debug() {}, info() {}, warn() {} };
|
|
54
|
+
this.canary = deps.canary ?? null;
|
|
55
|
+
/** @type {Map<string, {until: number, note: string}>} B9.4: provider quota windows */
|
|
56
|
+
this.quotaResets = new Map();
|
|
57
|
+
/** @type {Map<string, any>} */
|
|
58
|
+
this.adapters = new Map();
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** @param {string} providerId */
|
|
62
|
+
adapterFor(providerId) {
|
|
63
|
+
let a = this.adapters.get(providerId);
|
|
64
|
+
if (!a) {
|
|
65
|
+
const provider = this.config.providers[providerId];
|
|
66
|
+
if (!provider) {
|
|
67
|
+
throw new ConfigError(`Model points at provider "${providerId}", which is not declared.`, {
|
|
68
|
+
path: 'providers',
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
a = createAdapter(providerId, provider, this.config.env);
|
|
72
|
+
this.adapters.set(providerId, a);
|
|
73
|
+
}
|
|
74
|
+
return a;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Rebuild adapters after a config change. */
|
|
78
|
+
refresh() {
|
|
79
|
+
this.adapters.clear();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Work out the attempt order for a combo, and why anything was left out.
|
|
84
|
+
* Pure decision-making: no network, no side effects.
|
|
85
|
+
*
|
|
86
|
+
* Two invariants live here:
|
|
87
|
+
* I12 a provider marked tos:avoid is never auto-routed
|
|
88
|
+
* I14 if every target is ejected by a breaker, the request is still
|
|
89
|
+
* attempted against the unfiltered enabled list rather than refused
|
|
90
|
+
*
|
|
91
|
+
* @param {string} comboName
|
|
92
|
+
* @param {{promptTokens?: number, keyMeta?: any, exclude?: string[], allowAvoid?: boolean}} [opts]
|
|
93
|
+
*/
|
|
94
|
+
plan(comboName, opts = {}) {
|
|
95
|
+
const privacyTier = opts.privacyTier ?? this.config.privacy?.defaultTier ?? 'any';
|
|
96
|
+
const region = opts.region ?? this.config.privacy?.region ?? 'any';
|
|
97
|
+
let combo = resolveCombo(this.config, comboName);
|
|
98
|
+
|
|
99
|
+
// B3.5: a cascade combo chooses its own targets from the prompt's shape.
|
|
100
|
+
let cascade = null;
|
|
101
|
+
if (combo.cascade?.enabled) {
|
|
102
|
+
const decision = classifyPrompt(opts.promptText ?? '', { maxMs: 50, longWords: combo.cascade.longWords });
|
|
103
|
+
const chosen = decision.complexity === 'complex' ? combo.cascade.complex : combo.cascade.simple;
|
|
104
|
+
if (Array.isArray(chosen) && chosen.length) {
|
|
105
|
+
const mapped = chosen.map((id) => combo.targets.find((t) => t.modelId === id)).filter(Boolean);
|
|
106
|
+
// A cascade list that names models the combo does not have must not empty
|
|
107
|
+
// the route: keep the declared order and say so in the signals.
|
|
108
|
+
if (mapped.length) combo = { ...combo, targets: mapped };
|
|
109
|
+
else cascade = { ...decision, signals: [...decision.signals, `cascade list "${chosen.join(', ')}" matched no target — using the declared order`] };
|
|
110
|
+
}
|
|
111
|
+
cascade = cascade ?? decision;
|
|
112
|
+
}
|
|
113
|
+
const promptTokens = opts.promptTokens ?? 0;
|
|
114
|
+
const exclude = new Set(opts.exclude ?? []);
|
|
115
|
+
/** @type {Array<{modelId: string, provider: string, reason: string}>} */
|
|
116
|
+
const skipped = [];
|
|
117
|
+
const allowedModels = opts.keyMeta?.allowedModels ? new Set(opts.keyMeta.allowedModels) : null;
|
|
118
|
+
|
|
119
|
+
const dailyBudget = this.config.budgets.dailyUsd;
|
|
120
|
+
const spentToday = this.ledger.todaySpendUsd();
|
|
121
|
+
const budgetGone = dailyBudget != null && spentToday >= Number(dailyBudget);
|
|
122
|
+
|
|
123
|
+
/** @type {Array<any>} */
|
|
124
|
+
const usable = [];
|
|
125
|
+
/** @type {Array<any>} */
|
|
126
|
+
const resting = [];
|
|
127
|
+
|
|
128
|
+
for (const target of combo.targets) {
|
|
129
|
+
const { modelId, model } = target;
|
|
130
|
+
const providerId = model?.provider;
|
|
131
|
+
const provider = this.config.providers[providerId];
|
|
132
|
+
|
|
133
|
+
if (!model) {
|
|
134
|
+
skipped.push({ modelId, provider: String(providerId ?? '?'), reason: 'not declared in models' });
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (exclude.has(modelId)) {
|
|
138
|
+
skipped.push({ modelId, provider: providerId, reason: 'excluded by policy' });
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (allowedModels && !allowedModels.has(modelId)) {
|
|
142
|
+
skipped.push({ modelId, provider: providerId, reason: 'not allowed for this API key' });
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (provider?.tos === 'avoid' && !opts.allowAvoid) {
|
|
146
|
+
skipped.push({ modelId, provider: providerId, reason: 'terms of service marked avoid (never auto-routed)' });
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
// B9.4: a provider that told us when to come back is not tried before then.
|
|
150
|
+
const reset = this.quotaResets.get(modelId);
|
|
151
|
+
if (reset) {
|
|
152
|
+
if (reset.until > this.now()) {
|
|
153
|
+
const waitS = Math.max(1, Math.ceil((reset.until - this.now()) / 1000));
|
|
154
|
+
skipped.push({ modelId, provider: providerId, reason: `quota window closed, reopens in ${waitS}s` });
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
this.quotaResets.delete(modelId);
|
|
158
|
+
this.logger.info?.(`${displayName(providerId)} (${modelId}) quota window reopened`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// B6.5: checked before credentials, because sending a prompt to a provider
|
|
162
|
+
// that trains on it is not recoverable.
|
|
163
|
+
if (!providerMeetsPrivacyTier(provider, privacyTier)) {
|
|
164
|
+
skipped.push({ modelId, provider: providerId, reason: privacyRefusal(provider, privacyTier) });
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
// B6.6: residency, checked for the same reason
|
|
168
|
+
if (!providerMeetsRegion(provider, region)) {
|
|
169
|
+
skipped.push({ modelId, provider: providerId, reason: regionRefusal(provider, region) });
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
if (!providerHasCredentials(provider, this.config.env, this.config)) {
|
|
173
|
+
skipped.push({
|
|
174
|
+
modelId,
|
|
175
|
+
provider: providerId,
|
|
176
|
+
reason: isPaidProvider(provider) && this.config.paidTiers?.enabled !== true
|
|
177
|
+
? 'paid tier is closed (paid_tiers.enabled is false)'
|
|
178
|
+
: provider?.apiKeyEnv
|
|
179
|
+
? `no API key set (${provider.apiKeyEnv})`
|
|
180
|
+
: 'provider not configured',
|
|
181
|
+
});
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
// Money is checked before health: a resting provider that is also over
|
|
185
|
+
// budget must not be woken up by the I14 escape hatch.
|
|
186
|
+
if (budgetGone) {
|
|
187
|
+
skipped.push({ modelId, provider: providerId, reason: 'daily budget reached' });
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
const providerCap = this.config.budgets.perProvider?.[providerId];
|
|
191
|
+
if (providerCap != null && this.ledger.providerSpendToday(providerId) >= Number(providerCap)) {
|
|
192
|
+
skipped.push({ modelId, provider: providerId, reason: `provider budget reached ($${providerCap}/day)` });
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (combo.strategy === 'cheapest' && Number(model.quality ?? 5) < Number(combo.minQuality ?? 1)) {
|
|
196
|
+
skipped.push({
|
|
197
|
+
modelId,
|
|
198
|
+
provider: providerId,
|
|
199
|
+
reason: `quality ${model.quality ?? '?'} is below the floor of ${combo.minQuality}`,
|
|
200
|
+
});
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (combo.maxCostPer1k != null) {
|
|
204
|
+
const blended = Number(model.costPer1kIn ?? 0) + Number(model.costPer1kOut ?? 0);
|
|
205
|
+
if (blended > Number(combo.maxCostPer1k)) {
|
|
206
|
+
skipped.push({ modelId, provider: providerId, reason: `costs more than the $${combo.maxCostPer1k}/1k cap` });
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const entry = { ...target, providerId, provider, estimate: this.#estimateCost(model, promptTokens) };
|
|
212
|
+
|
|
213
|
+
if (!this.breaker.canAttempt(modelId)) {
|
|
214
|
+
// I14: not a skip. Held back as a last resort so a health blip cannot
|
|
215
|
+
// become a total outage.
|
|
216
|
+
resting.push(entry);
|
|
217
|
+
const st = this.breaker.stateOf(modelId);
|
|
218
|
+
skipped.push({
|
|
219
|
+
modelId,
|
|
220
|
+
provider: providerId,
|
|
221
|
+
reason: `resting after a failure (retrying in ${Math.ceil(st.retryInMs / 1000)}s) — held back, not skipped`,
|
|
222
|
+
});
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
usable.push(entry);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// B3.1: free targets before paid ones, whatever order they were declared in.
|
|
230
|
+
// A combo can opt out; the gateway default is on, because a paid key that
|
|
231
|
+
// happens to be first in a list should not be the thing that spends money.
|
|
232
|
+
const freeFirst = combo.freeFirst ?? this.config.routing?.freeFirst !== false;
|
|
233
|
+
let ordered = usable;
|
|
234
|
+
if (freeFirst && usable.length > 1) {
|
|
235
|
+
ordered = usable
|
|
236
|
+
.map((target, index) => ({ target, index, rank: tierRank(target.model, this.config) }))
|
|
237
|
+
.sort((a, b) => (a.rank !== b.rank ? a.rank - b.rank : a.index - b.index))
|
|
238
|
+
.map((x) => x.target);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// B9.3: fallback learning. Opt-in per combo, because reordering changes what
|
|
242
|
+
// answers your traffic and that should be a decision, not a surprise.
|
|
243
|
+
const learning = combo.learning ?? null;
|
|
244
|
+
if (learning?.enabled && ordered.length > 1) {
|
|
245
|
+
const minSamples = Number(learning.minSamples ?? 10);
|
|
246
|
+
const windowMs = Number(learning.windowMs ?? 24 * 3600_000);
|
|
247
|
+
const since = this.now() - windowMs;
|
|
248
|
+
const rows = this.ledger.entries({ since });
|
|
249
|
+
const stats = new Map();
|
|
250
|
+
for (const row of rows) {
|
|
251
|
+
if (!row.modelId) continue;
|
|
252
|
+
const entry = stats.get(row.modelId) ?? { ok: 0, total: 0, latencies: [] };
|
|
253
|
+
entry.total += 1;
|
|
254
|
+
if (row.ok) {
|
|
255
|
+
entry.ok += 1;
|
|
256
|
+
if (Number.isFinite(row.latencyMs)) entry.latencies.push(row.latencyMs);
|
|
257
|
+
}
|
|
258
|
+
stats.set(row.modelId, entry);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const score = (modelId) => {
|
|
262
|
+
const entry = stats.get(modelId);
|
|
263
|
+
if (!entry || entry.total < minSamples) return null; // not enough evidence: keep the declared position
|
|
264
|
+
const successRate = entry.ok / entry.total;
|
|
265
|
+
const sorted = [...entry.latencies].sort((a, b) => a - b);
|
|
266
|
+
const p50 = sorted.length ? sorted[Math.floor(sorted.length / 2)] : Number.MAX_SAFE_INTEGER;
|
|
267
|
+
return { successRate, p50, total: entry.total };
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
const withScore = ordered.map((target, index) => ({ target, index, score: score(target.modelId) }));
|
|
271
|
+
withScore.sort((a, b) => {
|
|
272
|
+
if (a.score && !b.score) return -1; // measured beats unmeasured
|
|
273
|
+
if (!a.score && b.score) return 1;
|
|
274
|
+
if (a.score && b.score) {
|
|
275
|
+
if (Math.abs(b.score.successRate - a.score.successRate) > 0.05) return b.score.successRate - a.score.successRate;
|
|
276
|
+
if (a.score.p50 !== b.score.p50) return a.score.p50 - b.score.p50;
|
|
277
|
+
}
|
|
278
|
+
return a.index - b.index; // stable: declared order breaks every tie
|
|
279
|
+
});
|
|
280
|
+
ordered = withScore.map((x) => x.target);
|
|
281
|
+
}
|
|
282
|
+
if (combo.strategy === 'cheapest') {
|
|
283
|
+
ordered = [...usable].sort((a, b) => a.estimate - b.estimate || Number(b.model.quality ?? 0) - Number(a.model.quality ?? 0));
|
|
284
|
+
} else if (combo.strategy === 'latency') {
|
|
285
|
+
ordered = [...usable].sort((a, b) => this.#p50(a.modelId) - this.#p50(b.modelId));
|
|
286
|
+
} else if (combo.strategy === 'weighted' && usable.length) {
|
|
287
|
+
const primary = pickWeighted(usable, this.random);
|
|
288
|
+
ordered = [primary, ...usable.filter((t) => t.modelId !== primary.modelId).sort((a, b) => b.weight - a.weight)];
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return {
|
|
292
|
+
combo,
|
|
293
|
+
ordered,
|
|
294
|
+
resting,
|
|
295
|
+
skipped,
|
|
296
|
+
spentToday,
|
|
297
|
+
dailyBudget,
|
|
298
|
+
privacyTier,
|
|
299
|
+
region,
|
|
300
|
+
learned: Boolean(combo.learning?.enabled),
|
|
301
|
+
cascade,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Compress everything except the last user message.
|
|
307
|
+
*
|
|
308
|
+
* The preset is chosen from the provider budget as a quota proxy until real
|
|
309
|
+
* quota telemetry lands (B2.3 follow-on): a provider at 90% of its daily cap
|
|
310
|
+
* is a provider whose remaining tokens are worth more.
|
|
311
|
+
*
|
|
312
|
+
* @param {Array<{role: string, content: string}>} messages
|
|
313
|
+
* @param {string} comboName
|
|
314
|
+
*/
|
|
315
|
+
#shapeContext(messages, comboName) {
|
|
316
|
+
const settings = this.config.compression ?? {};
|
|
317
|
+
const style = settings.outputStyle ?? 'off';
|
|
318
|
+
const wantsStyle = style !== 'off';
|
|
319
|
+
|
|
320
|
+
if (!settings.enabled && !wantsStyle) return { messages, meta: null };
|
|
321
|
+
|
|
322
|
+
const lastUserIndex = settings.keepLastUserMessage
|
|
323
|
+
? messages.map((m) => m.role).lastIndexOf('user')
|
|
324
|
+
: -1;
|
|
325
|
+
|
|
326
|
+
let preset = settings.preset;
|
|
327
|
+
let presetReason = null;
|
|
328
|
+
if (preset === 'auto') {
|
|
329
|
+
const quota = this.#quotaFraction(comboName);
|
|
330
|
+
const chosen = choosePreset({ fraction: quota });
|
|
331
|
+
preset = chosen.preset;
|
|
332
|
+
presetReason = chosen.reason;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const store = this.recallStore ?? (this.recallStore = new RecallStore({ secret: this.config.recallSecret }));
|
|
336
|
+
|
|
337
|
+
// B4.5 cache-aware compression: a provider caches a prefix, and compressing
|
|
338
|
+
// that prefix changes it, which invalidates the cache. So when prompt caching
|
|
339
|
+
// is on, the system message — the most stable part of any prompt — is left
|
|
340
|
+
// exactly as written, and compression works on the turns after it.
|
|
341
|
+
const protectPrefix = settings.preserveCachedPrefix !== false && this.config.promptCache?.enabled !== false;
|
|
342
|
+
const prefixEnd = protectPrefix ? firstNonSystemIndex(messages) : 0;
|
|
343
|
+
|
|
344
|
+
let originalChars = 0;
|
|
345
|
+
let compressedChars = 0;
|
|
346
|
+
const steps = [];
|
|
347
|
+
const next = messages.map((message, index) => {
|
|
348
|
+
if (index === lastUserIndex) return message;
|
|
349
|
+
if (index < prefixEnd) return message; // the cacheable prefix stays untouched
|
|
350
|
+
if (typeof message.content !== 'string' || message.content.length < settings.minChars) return message;
|
|
351
|
+
const result = compressText(message.content, { preset, store });
|
|
352
|
+
originalChars += result.originalChars;
|
|
353
|
+
compressedChars += result.compressedChars;
|
|
354
|
+
for (const step of result.steps) if (step.droppedChars) steps.push({ ...step, message: index });
|
|
355
|
+
return { ...message, content: result.text };
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
// B4.4: the output axis. 'auto' follows the same quota reading as the preset.
|
|
359
|
+
let resolvedStyle = style;
|
|
360
|
+
let styleReason = null;
|
|
361
|
+
if (style === 'auto') {
|
|
362
|
+
const chosen = chooseStyle({ fraction: this.#quotaFraction(comboName), taskHint: messages[messages.length - 1]?.content });
|
|
363
|
+
resolvedStyle = chosen.style;
|
|
364
|
+
styleReason = chosen.reason;
|
|
365
|
+
}
|
|
366
|
+
const styled = resolvedStyle !== 'off' ? applyStyle(next, resolvedStyle) : next;
|
|
367
|
+
|
|
368
|
+
const meta = {
|
|
369
|
+
preset: settings.enabled ? preset : null,
|
|
370
|
+
prefixProtected: protectPrefix && prefixEnd > 0,
|
|
371
|
+
reason: presetReason,
|
|
372
|
+
style: resolvedStyle === 'off' ? null : resolvedStyle,
|
|
373
|
+
styleReason,
|
|
374
|
+
ratio: originalChars ? Math.round(((originalChars - compressedChars) / originalChars) * 1000) / 1000 : 0,
|
|
375
|
+
savedChars: Math.max(0, originalChars - compressedChars),
|
|
376
|
+
steps,
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
// Say what happened even when nothing was compressed: "the prefix was
|
|
380
|
+
// protected" is a different fact from "there was nothing to do".
|
|
381
|
+
const skipped = steps.length
|
|
382
|
+
? null
|
|
383
|
+
: protectPrefix && prefixEnd > 0
|
|
384
|
+
? 'the cacheable prefix was left untouched so the provider cache still hits'
|
|
385
|
+
: 'nothing in this prompt was worth compressing';
|
|
386
|
+
|
|
387
|
+
if (!settings.enabled && resolvedStyle === 'off') return { messages: next, meta: null };
|
|
388
|
+
meta.skipped = skipped;
|
|
389
|
+
return { messages: styled, meta };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** B9.4: the quota windows currently being honoured. */
|
|
393
|
+
quotaSnapshot() {
|
|
394
|
+
const now = this.now();
|
|
395
|
+
return [...this.quotaResets.entries()].map(([modelId, reset]) => ({
|
|
396
|
+
modelId,
|
|
397
|
+
reopensInMs: Math.max(0, reset.until - now),
|
|
398
|
+
reopensAt: reset.until,
|
|
399
|
+
}));
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** How many targets are currently resting — the signal that a failure is an outage. */
|
|
403
|
+
#breakersOpen() {
|
|
404
|
+
return this.breaker.snapshot().filter((b) => b.state === 'open').length;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** 1 = plenty of budget left, 0 = none. Used to pick a compression preset. */
|
|
408
|
+
#quotaFraction(comboName) {
|
|
409
|
+
const combo = this.config.combos?.[comboName];
|
|
410
|
+
const providerId = combo ? this.config.models?.[combo.targets?.[0]]?.provider : null;
|
|
411
|
+
const cap = providerId ? this.config.budgets?.perProvider?.[providerId] : null;
|
|
412
|
+
if (cap == null) {
|
|
413
|
+
const daily = this.config.budgets?.dailyUsd;
|
|
414
|
+
if (daily == null) return 1;
|
|
415
|
+
return Math.max(0, 1 - this.ledger.todaySpendUsd() / Number(daily));
|
|
416
|
+
}
|
|
417
|
+
return Math.max(0, 1 - this.ledger.providerSpendToday(providerId) / Number(cap));
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
#estimateCost(model, promptTokens) {
|
|
421
|
+
const blended = (Number(model.costPer1kIn ?? 0) + Number(model.costPer1kOut ?? 0)) / 2;
|
|
422
|
+
return (blended * Math.max(promptTokens, 1)) / 1000;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
#p50(modelId) {
|
|
426
|
+
const rows = this.ledger.entries().filter((r) => r.modelId === modelId && r.ok);
|
|
427
|
+
if (!rows.length) return Number.POSITIVE_INFINITY;
|
|
428
|
+
const sorted = rows.map((r) => r.latencyMs).sort((a, b) => a - b);
|
|
429
|
+
return sorted[Math.floor(sorted.length / 2)];
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Run a chat turn.
|
|
434
|
+
*
|
|
435
|
+
* @param {{
|
|
436
|
+
* combo: string, body: any, keyMeta?: any,
|
|
437
|
+
* onToken?: (delta: string) => void, now?: () => number
|
|
438
|
+
* }} input
|
|
439
|
+
* @returns {Promise<{text: string, usage: {prompt_tokens: number, completion_tokens: number}, finishReason: string|null, meta: TurnMeta}>}
|
|
440
|
+
*/
|
|
441
|
+
async chat(input) {
|
|
442
|
+
const started = this.now();
|
|
443
|
+
const body = input.body ?? {};
|
|
444
|
+
|
|
445
|
+
// B9.2: the canary decides which combo answers, before anything else runs.
|
|
446
|
+
// Named `canaryPick`, not `arm`: `arm` is the A/B arm of a weighted target
|
|
447
|
+
// inside the loop, and shadowing it here silently broke attribution.
|
|
448
|
+
const canaryPick = this.canary ? this.canary.pick() : null;
|
|
449
|
+
const comboName = canaryPick?.arm === 'candidate' ? canaryPick.combo : input.combo;
|
|
450
|
+
const messages = normaliseMessages(body.messages);
|
|
451
|
+
const promptText = messages.map((m) => m.content).join('\n');
|
|
452
|
+
const streamed = Boolean(body.stream);
|
|
453
|
+
|
|
454
|
+
checkPrompt({ messages, guardrails: this.config.guardrails });
|
|
455
|
+
|
|
456
|
+
// B6.3: the prompt is not the only channel. Tool definitions are text the
|
|
457
|
+
// model reads, and tool results are text an attacker may control entirely.
|
|
458
|
+
const toolInspection = inspectToolChannel({
|
|
459
|
+
messages,
|
|
460
|
+
tools: body.tools,
|
|
461
|
+
guardrails: this.config.guardrails,
|
|
462
|
+
});
|
|
463
|
+
if (toolInspection.blocked) {
|
|
464
|
+
const first = toolInspection.findings.find((f) => f.severity === 'high');
|
|
465
|
+
throw new SuiError(`A tool message was refused: ${first.detail}.`, {
|
|
466
|
+
code: 'tool_channel_blocked',
|
|
467
|
+
status: 400,
|
|
468
|
+
detail: toolInspection.findings.map((f) => `${f.channel}/${f.where}: ${f.signal}`).join('; '),
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// B4.1/B4.4: compress the older context, never the question being asked,
|
|
473
|
+
// then shape the output axis if it is on.
|
|
474
|
+
const compressed = this.#shapeContext(messages, comboName);
|
|
475
|
+
const effectiveMessages = compressed.messages;
|
|
476
|
+
const promptTokens = estimateTokens(effectiveMessages.map((m) => m.content).join('\n'));
|
|
477
|
+
|
|
478
|
+
const { combo, ordered, resting, skipped, privacyTier, region, learned, cascade } = this.plan(comboName, {
|
|
479
|
+
region: input.region,
|
|
480
|
+
promptText,
|
|
481
|
+
privacyTier: input.privacyTier,
|
|
482
|
+
promptTokens,
|
|
483
|
+
keyMeta: input.keyMeta,
|
|
484
|
+
allowAvoid: Boolean(input.allowAvoid),
|
|
485
|
+
// plan() has always supported an exclusion list; nothing could reach it
|
|
486
|
+
exclude: input.exclude ?? [],
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
const cacheKey = this.cache?.enabled && !streamed ? ResponseCache.key({ ...body, combo: comboName }) : null;
|
|
490
|
+
if (cacheKey) {
|
|
491
|
+
const hit = this.cache.get(cacheKey, promptText);
|
|
492
|
+
if (hit) {
|
|
493
|
+
const meta = {
|
|
494
|
+
...hit.value.meta,
|
|
495
|
+
cached: true,
|
|
496
|
+
cacheKind: hit.kind,
|
|
497
|
+
latencyMs: this.now() - started,
|
|
498
|
+
ttftMs: 0,
|
|
499
|
+
attempts: [],
|
|
500
|
+
};
|
|
501
|
+
this.ledger.record({
|
|
502
|
+
provider: meta.provider,
|
|
503
|
+
model: meta.model,
|
|
504
|
+
modelId: meta.modelId,
|
|
505
|
+
combo: comboName,
|
|
506
|
+
arm: meta.arm,
|
|
507
|
+
ok: true,
|
|
508
|
+
promptTokens: hit.value.usage.prompt_tokens,
|
|
509
|
+
completionTokens: hit.value.usage.completion_tokens,
|
|
510
|
+
costUsd: hit.value.meta.costUsd,
|
|
511
|
+
latencyMs: meta.latencyMs,
|
|
512
|
+
ttftMs: 0,
|
|
513
|
+
cached: true,
|
|
514
|
+
verified: meta.verified,
|
|
515
|
+
promptText,
|
|
516
|
+
streamed: false,
|
|
517
|
+
});
|
|
518
|
+
return { text: hit.value.text, usage: hit.value.usage, finishReason: hit.value.finishReason, meta };
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
if (!ordered.length && !resting.length) {
|
|
523
|
+
throw exhausted(comboName, skipped, this.config);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** @type {Array<{modelId: string, provider: string, reason: string}>} */
|
|
527
|
+
const attempts = [];
|
|
528
|
+
let lastError = null;
|
|
529
|
+
let usedResting = false;
|
|
530
|
+
const verification = this.config.verification ?? {};
|
|
531
|
+
let escalationsLeft = Number(verification.maxEscalations ?? 1);
|
|
532
|
+
/** @type {{result: any, meta: any}|null} */
|
|
533
|
+
let bestLow = null;
|
|
534
|
+
|
|
535
|
+
// Healthy targets first; the held-back ones only if every healthy target
|
|
536
|
+
// failed. This is the I14 escape hatch, and it is recorded when used.
|
|
537
|
+
const queue = [...ordered, ...resting.map((t) => ({ ...t, restingFallback: true }))];
|
|
538
|
+
|
|
539
|
+
for (const target of queue) {
|
|
540
|
+
const { modelId, model, providerId, arm } = target;
|
|
541
|
+
const adapter = this.adapterFor(providerId);
|
|
542
|
+
const emptyRetryAllowed = this.config.streaming?.emptyTurnRetry !== false;
|
|
543
|
+
let retriedEmpty = false;
|
|
544
|
+
|
|
545
|
+
// B3.3: an empty turn is a failed turn. Some free tiers answer a 200 with
|
|
546
|
+
// nothing in it when they are throttling — retry the same target once,
|
|
547
|
+
// then move on, rather than handing the user a blank reply.
|
|
548
|
+
for (;;) {
|
|
549
|
+
const attemptStart = this.now();
|
|
550
|
+
let ttft = null;
|
|
551
|
+
let committed = false;
|
|
552
|
+
|
|
553
|
+
/** @type {(delta: string) => void} */
|
|
554
|
+
const emit = (delta) => {
|
|
555
|
+
if (!committed) {
|
|
556
|
+
committed = true;
|
|
557
|
+
ttft = this.now() - attemptStart;
|
|
558
|
+
}
|
|
559
|
+
input.onToken?.(delta);
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
try {
|
|
563
|
+
input.onAttempt?.({ modelId, providerId, providerName: adapter.name, model: model.model, arm: arm ?? null });
|
|
564
|
+
|
|
565
|
+
// B4.6: what the prefix cache can do with this provider.
|
|
566
|
+
const cache = preparePromptCache({
|
|
567
|
+
messages: effectiveMessages,
|
|
568
|
+
providerType: adapter.type,
|
|
569
|
+
memory: this.prefixMemory ?? (this.prefixMemory = new PrefixMemory({ maxEntries: this.config.server?.cache?.maxEntries ?? 500 })),
|
|
570
|
+
enabled: this.config.promptCache?.enabled !== false,
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
const result = await adapter.chat(
|
|
574
|
+
{
|
|
575
|
+
model: model.model,
|
|
576
|
+
messages: effectiveMessages,
|
|
577
|
+
stream: streamed,
|
|
578
|
+
temperature: body.temperature,
|
|
579
|
+
// B6.6: a key may carry tighter guardrails than the gateway.
|
|
580
|
+
max_tokens: clampMaxTokens(body.max_tokens, input.keyMeta?.guardrails ?? this.config.guardrails),
|
|
581
|
+
top_p: body.top_p,
|
|
582
|
+
stop: body.stop,
|
|
583
|
+
cacheHints: cache.hints,
|
|
584
|
+
},
|
|
585
|
+
{ onToken: streamed ? emit : undefined },
|
|
586
|
+
);
|
|
587
|
+
|
|
588
|
+
const latencyMs = this.now() - attemptStart;
|
|
589
|
+
const isEmpty = !committed && String(result.text ?? '').trim().length === 0;
|
|
590
|
+
|
|
591
|
+
if (isEmpty && emptyRetryAllowed && !retriedEmpty) {
|
|
592
|
+
retriedEmpty = true;
|
|
593
|
+
attempts.push({ modelId, provider: providerId, reason: 'empty turn — retried once on the same target' });
|
|
594
|
+
this.ledger.record({
|
|
595
|
+
provider: providerId,
|
|
596
|
+
model: model.model,
|
|
597
|
+
modelId,
|
|
598
|
+
combo: comboName,
|
|
599
|
+
arm: arm ?? null,
|
|
600
|
+
ok: false,
|
|
601
|
+
error: 'empty_turn',
|
|
602
|
+
promptTokens: result.usage?.prompt_tokens ?? promptTokens,
|
|
603
|
+
completionTokens: 0,
|
|
604
|
+
costUsd: 0,
|
|
605
|
+
latencyMs,
|
|
606
|
+
ttftMs: ttft,
|
|
607
|
+
cached: false,
|
|
608
|
+
verified: false,
|
|
609
|
+
promptText,
|
|
610
|
+
streamed,
|
|
611
|
+
});
|
|
612
|
+
this.logger.warn(`${displayName(providerId)} (${modelId}) returned an empty turn — retrying once`);
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
if (isEmpty) {
|
|
617
|
+
// Still empty after the retry: record it and try the next target.
|
|
618
|
+
attempts.push({ modelId, provider: providerId, reason: 'empty turn (no retry left)' });
|
|
619
|
+
this.ledger.record({
|
|
620
|
+
provider: providerId,
|
|
621
|
+
model: model.model,
|
|
622
|
+
modelId,
|
|
623
|
+
combo: comboName,
|
|
624
|
+
arm: arm ?? null,
|
|
625
|
+
ok: false,
|
|
626
|
+
error: 'empty_turn',
|
|
627
|
+
promptTokens: result.usage?.prompt_tokens ?? promptTokens,
|
|
628
|
+
completionTokens: 0,
|
|
629
|
+
costUsd: 0,
|
|
630
|
+
latencyMs,
|
|
631
|
+
ttftMs: ttft,
|
|
632
|
+
cached: false,
|
|
633
|
+
verified: false,
|
|
634
|
+
promptText,
|
|
635
|
+
streamed,
|
|
636
|
+
});
|
|
637
|
+
break;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
const costUsd = estimatedCost(model, result.usage.prompt_tokens, result.usage.completion_tokens);
|
|
641
|
+
const verified = adapter.type !== 'mock';
|
|
642
|
+
|
|
643
|
+
// B7.2: what the compression saved, valued at this model's input price.
|
|
644
|
+
const savedTokens = Math.ceil((compressed.meta?.savedChars ?? 0) / 4);
|
|
645
|
+
const savedUsd = savedTokens ? Math.round((Number(model.costPer1kIn ?? 0) * savedTokens) / 1000 * 1e6) / 1e6 : 0;
|
|
646
|
+
|
|
647
|
+
this.breaker.recordSuccess(modelId);
|
|
648
|
+
if (target.restingFallback) usedResting = true;
|
|
649
|
+
|
|
650
|
+
/** @type {TurnMeta} */
|
|
651
|
+
const meta = {
|
|
652
|
+
provider: providerId,
|
|
653
|
+
providerName: adapter.name,
|
|
654
|
+
model: model.model,
|
|
655
|
+
modelId,
|
|
656
|
+
combo: comboName,
|
|
657
|
+
arm: arm ?? null,
|
|
658
|
+
cached: false,
|
|
659
|
+
cacheKind: null,
|
|
660
|
+
verified,
|
|
661
|
+
latencyMs,
|
|
662
|
+
ttftMs: streamed ? ttft : null,
|
|
663
|
+
costUsd,
|
|
664
|
+
usage: result.usage,
|
|
665
|
+
strategy: combo.strategy,
|
|
666
|
+
attempts,
|
|
667
|
+
escalation: usedResting ? 'I14 — every healthy target failed, a resting one was tried' : null,
|
|
668
|
+
compression: compressed.meta,
|
|
669
|
+
verification: null,
|
|
670
|
+
toolInspection: toolInspection.findings.length ? { mode: toolInspection.mode, findings: toolInspection.findings } : null,
|
|
671
|
+
canary: canaryPick ? { id: this.canary.id, arm: canaryPick.arm, reason: canaryPick.reason } : null,
|
|
672
|
+
savedUsd,
|
|
673
|
+
route: {
|
|
674
|
+
strategy: combo.strategy,
|
|
675
|
+
privacyTier,
|
|
676
|
+
region,
|
|
677
|
+
learned,
|
|
678
|
+
cascade,
|
|
679
|
+
considered: [
|
|
680
|
+
...ordered.map((t) => ({ modelId: t.modelId, provider: t.providerId, outcome: t.modelId === modelId ? 'answered' : 'available' })),
|
|
681
|
+
...resting.map((t) => ({ modelId: t.modelId, provider: t.providerId, outcome: 'resting' })),
|
|
682
|
+
...skipped.map((s) => ({ modelId: s.modelId, provider: s.provider, outcome: 'skipped', reason: s.reason })),
|
|
683
|
+
],
|
|
684
|
+
},
|
|
685
|
+
promptCache: cache.detected
|
|
686
|
+
? {
|
|
687
|
+
warm: cache.warm,
|
|
688
|
+
sends: cache.sends,
|
|
689
|
+
prefixChars: cache.detected.prefixChars,
|
|
690
|
+
fingerprint: cache.detected.fingerprint,
|
|
691
|
+
volatile: cache.detected.volatile,
|
|
692
|
+
hint: cache.hints.supported ? cache.hints.reason : null,
|
|
693
|
+
unsupported: cache.hints.supported ? null : cache.hints.reason,
|
|
694
|
+
}
|
|
695
|
+
: null,
|
|
696
|
+
};
|
|
697
|
+
|
|
698
|
+
this.ledger.record({
|
|
699
|
+
provider: providerId,
|
|
700
|
+
model: model.model,
|
|
701
|
+
modelId,
|
|
702
|
+
combo: comboName,
|
|
703
|
+
arm: arm ?? null,
|
|
704
|
+
ok: true,
|
|
705
|
+
promptTokens: result.usage.prompt_tokens,
|
|
706
|
+
completionTokens: result.usage.completion_tokens,
|
|
707
|
+
costUsd,
|
|
708
|
+
latencyMs,
|
|
709
|
+
ttftMs: streamed ? ttft : null,
|
|
710
|
+
cached: false,
|
|
711
|
+
verified,
|
|
712
|
+
promptText,
|
|
713
|
+
streamed,
|
|
714
|
+
compressionSavedTokens: savedTokens,
|
|
715
|
+
compressionSavedUsd: savedUsd,
|
|
716
|
+
compressionPreset: compressed.meta?.preset ?? null,
|
|
717
|
+
promptCacheWarm: cache.warm,
|
|
718
|
+
apiKey: input.keyMeta?.name ?? null,
|
|
719
|
+
user: input.attribution?.user ?? null,
|
|
720
|
+
tool: input.attribution?.tool ?? null,
|
|
721
|
+
});
|
|
722
|
+
|
|
723
|
+
// B3.4: score the answer, and escalate if it is not convincing.
|
|
724
|
+
if (verification.enabled) {
|
|
725
|
+
const score = scoreAnswer({
|
|
726
|
+
text: result.text,
|
|
727
|
+
prompt: promptText,
|
|
728
|
+
finishReason: result.finishReason,
|
|
729
|
+
wantsJson: body.response_format?.type === 'json_object',
|
|
730
|
+
});
|
|
731
|
+
// B7.6: grounding runs before the verdict, because an ungrounded
|
|
732
|
+
// answer is not a confident one however fluent it reads.
|
|
733
|
+
const groundingMode = verification.grounding ?? 'off';
|
|
734
|
+
let grounding = null;
|
|
735
|
+
if (groundingMode !== 'off') {
|
|
736
|
+
const contextText = messages
|
|
737
|
+
.filter((m) => m.role === 'tool' || m.role === 'function' || m.role === 'system')
|
|
738
|
+
.map((m) => (typeof m.content === 'string' ? m.content : ''))
|
|
739
|
+
.join('\n');
|
|
740
|
+
const check = checkGrounding({ answer: result.text, context: contextText });
|
|
741
|
+
grounding = { mode: groundingMode, ...check };
|
|
742
|
+
if (check.checked && !check.supported) {
|
|
743
|
+
// a fabricated URL or number is a hard signal, not a hint
|
|
744
|
+
score.reasons.push({
|
|
745
|
+
signal: 'ungrounded',
|
|
746
|
+
penalty: 0.45,
|
|
747
|
+
detail: `${check.findings.length} claim(s) not found in the context: ${check.findings.slice(0, 3).map((f) => f.claim).join(', ')}`,
|
|
748
|
+
});
|
|
749
|
+
score.confidence = Math.max(0, Math.round((score.confidence - 0.45) * 100) / 100);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
const v = verifyVerdict({ ...score, threshold: Number(verification.threshold ?? 0.7) });
|
|
754
|
+
meta.verification = {
|
|
755
|
+
confidence: v.confidence,
|
|
756
|
+
threshold: v.threshold,
|
|
757
|
+
verified: v.verified,
|
|
758
|
+
reasons: score.reasons,
|
|
759
|
+
escalated: false,
|
|
760
|
+
grounding,
|
|
761
|
+
};
|
|
762
|
+
|
|
763
|
+
if (!v.ok && shouldEscalate({ confidence: v.confidence, threshold: v.threshold, escalationsLeft })) {
|
|
764
|
+
escalationsLeft -= 1;
|
|
765
|
+
meta.verification.escalated = true;
|
|
766
|
+
attempts.push({
|
|
767
|
+
modelId,
|
|
768
|
+
provider: providerId,
|
|
769
|
+
reason: `answered, but confidence ${v.confidence} is below ${v.threshold} — escalating`,
|
|
770
|
+
});
|
|
771
|
+
// Keep the best low-confidence answer in case nothing better comes.
|
|
772
|
+
if (!bestLow || v.confidence > bestLow.meta.verification.confidence) bestLow = { result, meta };
|
|
773
|
+
this.ledger.record({
|
|
774
|
+
provider: providerId,
|
|
775
|
+
model: model.model,
|
|
776
|
+
modelId,
|
|
777
|
+
combo: comboName,
|
|
778
|
+
arm: arm ?? null,
|
|
779
|
+
ok: true,
|
|
780
|
+
error: null,
|
|
781
|
+
promptTokens: result.usage.prompt_tokens,
|
|
782
|
+
completionTokens: result.usage.completion_tokens,
|
|
783
|
+
costUsd,
|
|
784
|
+
latencyMs,
|
|
785
|
+
ttftMs: streamed ? ttft : null,
|
|
786
|
+
cached: false,
|
|
787
|
+
verified,
|
|
788
|
+
promptText,
|
|
789
|
+
streamed,
|
|
790
|
+
apiKey: input.keyMeta?.name ?? null,
|
|
791
|
+
user: input.attribution?.user ?? null,
|
|
792
|
+
tool: input.attribution?.tool ?? null,
|
|
793
|
+
});
|
|
794
|
+
break; // leave this target; the outer loop moves to the next one
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
if (!v.ok) {
|
|
798
|
+
// Fail-closed: the answer is returned, but never presented as verified.
|
|
799
|
+
meta.verified = false;
|
|
800
|
+
meta.escalation = `confidence ${v.confidence} is below ${v.threshold}; returned unverified (fail-closed)`;
|
|
801
|
+
this.logger.warn(`${modelId} answered with confidence ${v.confidence} (threshold ${v.threshold}); returning it marked unverified`);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
if (cacheKey) {
|
|
806
|
+
this.cache.set(
|
|
807
|
+
cacheKey,
|
|
808
|
+
{ text: result.text, usage: result.usage, finishReason: result.finishReason, meta },
|
|
809
|
+
{ promptText },
|
|
810
|
+
);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
if (canaryPick?.arm === 'candidate') {
|
|
814
|
+
const verdict = this.canary.observe({
|
|
815
|
+
ok: true,
|
|
816
|
+
latencyMs,
|
|
817
|
+
// an outage is the provider's problem, not the candidate's
|
|
818
|
+
outage: this.#breakersOpen() > 0,
|
|
819
|
+
});
|
|
820
|
+
if (verdict.event) {
|
|
821
|
+
this.logger.warn(`canary ${this.canary.id}: ${verdict.event} — ${verdict.detail}`);
|
|
822
|
+
meta.canary.outcome = verdict.event;
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
return { text: result.text, usage: result.usage, finishReason: result.finishReason, meta };
|
|
827
|
+
} catch (err) {
|
|
828
|
+
const e = /** @type {any} */ (err);
|
|
829
|
+
const latencyMs = this.now() - attemptStart;
|
|
830
|
+
|
|
831
|
+
// I13: only 5xx and transport failures trip the breaker. A 401 or a 404
|
|
832
|
+
// is a fact about the key or the model name, not a health signal, and
|
|
833
|
+
// tripping on it would hide a config bug behind a cooldown.
|
|
834
|
+
if (tripsBreaker(e)) this.breaker.recordFailure(modelId, e.message);
|
|
835
|
+
|
|
836
|
+
attempts.push({ modelId, provider: providerId, reason: e.message ?? String(err) });
|
|
837
|
+
lastError = e;
|
|
838
|
+
|
|
839
|
+
// B9.2: a failure is an observation too — but only if it is about this
|
|
840
|
+
// candidate. If other targets are already resting, this is an outage
|
|
841
|
+
// and rolling the candidate back would blame it for someone else's
|
|
842
|
+
// downtime.
|
|
843
|
+
// B9.4: honour a stated quota window instead of guessing from failures.
|
|
844
|
+
if (e?.retryAfterMs > 0) {
|
|
845
|
+
const until = this.now() + e.retryAfterMs;
|
|
846
|
+
this.quotaResets.set(modelId, { until, note: `${Math.round(e.retryAfterMs / 1000)}s` });
|
|
847
|
+
this.logger.warn(`${displayName(providerId)} (${modelId}) closed its quota window for ${Math.round(e.retryAfterMs / 1000)}s`);
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
if (canaryPick?.arm === 'candidate') {
|
|
851
|
+
const othersResting = this.breaker.snapshot().filter((b) => b.state === 'open' && b.key !== modelId).length;
|
|
852
|
+
const verdict = this.canary.observe({ ok: false, latencyMs, outage: othersResting > 0 });
|
|
853
|
+
if (verdict.event) this.logger.warn(`canary ${this.canary.id}: ${verdict.event} — ${verdict.detail}`);
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
this.ledger.record({
|
|
857
|
+
provider: providerId,
|
|
858
|
+
model: model.model,
|
|
859
|
+
modelId,
|
|
860
|
+
combo: comboName,
|
|
861
|
+
arm: arm ?? null,
|
|
862
|
+
ok: false,
|
|
863
|
+
error: e.message ?? String(err),
|
|
864
|
+
promptTokens,
|
|
865
|
+
completionTokens: 0,
|
|
866
|
+
costUsd: 0,
|
|
867
|
+
latencyMs,
|
|
868
|
+
ttftMs: ttft,
|
|
869
|
+
cached: false,
|
|
870
|
+
verified: false,
|
|
871
|
+
promptText,
|
|
872
|
+
streamed,
|
|
873
|
+
});
|
|
874
|
+
|
|
875
|
+
this.logger.warn(`${displayName(providerId)} (${modelId}) failed: ${e.message}`);
|
|
876
|
+
|
|
877
|
+
if (committed) {
|
|
878
|
+
// Tokens already went to the client. Switching provider now would
|
|
879
|
+
// splice two different answers together, so stop and say so.
|
|
880
|
+
throw new SuiError(
|
|
881
|
+
`${displayName(providerId)} dropped the connection mid-answer. Nothing was retried because part of the reply had already been sent.`,
|
|
882
|
+
{ code: 'stream_interrupted', status: 502, retryable: false, detail: e.message },
|
|
883
|
+
);
|
|
884
|
+
}
|
|
885
|
+
// A non-retryable provider error (401, 404, a malformed request) is a
|
|
886
|
+
// fact about that provider, not a reason to abandon the whole combo:
|
|
887
|
+
// the next target is a different provider with a different key.
|
|
888
|
+
if (!(e instanceof ProviderError)) throw e;
|
|
889
|
+
break; // this target is done; the outer loop moves to the next one
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
// Nothing cleared the bar, but something answered: fail-closed means mark it
|
|
895
|
+
// unverified, not that the user gets nothing.
|
|
896
|
+
if (bestLow) {
|
|
897
|
+
bestLow.meta.verified = false; // fail-closed, on the way out too
|
|
898
|
+
bestLow.meta.escalation = `confidence never reached ${verification.threshold}; returning the best answer, marked unverified`;
|
|
899
|
+
return { text: bestLow.result.text, usage: bestLow.result.usage, finishReason: bestLow.result.finishReason, meta: bestLow.meta };
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
const err = exhausted(comboName, skipped, this.config, attempts);
|
|
903
|
+
if (lastError && !(lastError instanceof ProviderError)) throw lastError;
|
|
904
|
+
throw err;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/** Index of the first message that is not part of the stable system prefix. */
|
|
909
|
+
function firstNonSystemIndex(messages) {
|
|
910
|
+
let i = 0;
|
|
911
|
+
while (i < messages.length && messages[i].role === 'system') i += 1;
|
|
912
|
+
return i;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* Build the plain-language "nothing left" error. This is the message a user
|
|
917
|
+
* actually reads, so it says what is resting, why, and what to do next.
|
|
918
|
+
*/
|
|
919
|
+
function exhausted(comboName, skipped, config, attempts = []) {
|
|
920
|
+
const parts = [];
|
|
921
|
+
for (const s of skipped) parts.push(`${s.modelId} — ${s.reason}`);
|
|
922
|
+
for (const a of attempts) parts.push(`${a.modelId} — tried and failed: ${a.reason}`);
|
|
923
|
+
const tried = attempts.length ? `Tried ${attempts.length}, all failed. ` : '';
|
|
924
|
+
return new ExhaustedError(
|
|
925
|
+
`No model could answer "${comboName}" right now. ${tried}${parts.length ? `Details: ${parts.join('; ')}.` : ''} Add a provider key, wait for the cooldown, or raise the daily budget.`,
|
|
926
|
+
{ attempts: [...attempts, ...skipped.map((s) => ({ model: s.modelId, reason: s.reason }))] },
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/** @param {any[]} messages */
|
|
931
|
+
function normaliseMessages(messages) {
|
|
932
|
+
if (!Array.isArray(messages) || !messages.length) {
|
|
933
|
+
throw new SuiError('messages must be a non-empty array.', { code: 'bad_request', status: 400 });
|
|
934
|
+
}
|
|
935
|
+
return messages.map((m) => ({
|
|
936
|
+
role: m.role ?? 'user',
|
|
937
|
+
content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content ?? ''),
|
|
938
|
+
}));
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/** @param {any[]} items @param {() => number} random */
|
|
942
|
+
function pickWeighted(items, random) {
|
|
943
|
+
const total = items.reduce((a, i) => a + (Number(i.weight) || 1), 0);
|
|
944
|
+
let r = random() * total;
|
|
945
|
+
for (const item of items) {
|
|
946
|
+
r -= Number(item.weight) || 1;
|
|
947
|
+
if (r <= 0) return item;
|
|
948
|
+
}
|
|
949
|
+
return items[items.length - 1];
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* I13 — does this failure say something about the provider's health?
|
|
954
|
+
* Yes for 5xx, timeouts and transport errors. No for 400/401/403/404/422/429.
|
|
955
|
+
* @param {any} err
|
|
956
|
+
*/
|
|
957
|
+
export function tripsBreaker(err) {
|
|
958
|
+
if (!err) return false;
|
|
959
|
+
const code = String(err.code ?? '');
|
|
960
|
+
if (code === 'provider_timeout' || code === 'provider_unreachable') return true;
|
|
961
|
+
const status = Number(err.status ?? 0);
|
|
962
|
+
return status >= 500;
|
|
963
|
+
}
|
package/README.md
DELETED