@drsamonline/router 0.0.0-stage → 0.12.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,37 @@
1
1
  {
2
2
  "name": "@drsamonline/router",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.12.3",
4
+ "description": "Routing decisions, strategies, circuit breaker.",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "main": "./src/index.js",
8
+ "exports": {
9
+ ".": "./src/index.js",
10
+ "./src/*": "./src/*"
11
+ },
12
+ "files": [
13
+ "src",
14
+ "web",
15
+ "bin",
16
+ "LICENSE"
17
+ ],
18
+ "repository": {
19
+ "type": "git",
20
+ "url": "git://github.com/drsamonline/sui.git",
21
+ "directory": "packages/router"
22
+ },
23
+ "homepage": "https://github.com/drsamonline/sui#readme",
24
+ "publishConfig": {
25
+ "access": "public",
26
+ "registry": "https://registry.npmjs.org/"
27
+ },
28
+ "dependencies": {
29
+ "@drsamonline/compression": "^0.12.3",
30
+ "@drsamonline/core": "^0.12.3",
31
+ "@drsamonline/guardrails": "^0.12.3",
32
+ "@drsamonline/promptcache": "^0.12.3",
33
+ "@drsamonline/providers": "^0.12.3",
34
+ "@drsamonline/telemetry": "^0.12.3",
35
+ "@drsamonline/verify": "^0.12.3"
36
+ }
37
+ }
package/src/breaker.js ADDED
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Per-target circuit breaker.
3
+ *
4
+ * closed -> (failures >= threshold) -> open
5
+ * open -> (cooldown elapsed) -> half-open (one trial request allowed)
6
+ * half-open -> success -> closed
7
+ * half-open -> failure -> open, cooldown restarts
8
+ *
9
+ * The breaker is the reason a rate-limited provider costs one failed request
10
+ * instead of every request until its quota resets.
11
+ */
12
+
13
+ const CLOSED = 'closed';
14
+ const OPEN = 'open';
15
+ const HALF_OPEN = 'half_open';
16
+
17
+ export class CircuitBreaker {
18
+ /**
19
+ * @param {{failures?: number, cooldownMs?: number, halfOpenSuccesses?: number, now?: () => number}} [opts]
20
+ */
21
+ constructor(opts = {}) {
22
+ this.threshold = opts.failures ?? 3;
23
+ this.cooldownMs = opts.cooldownMs ?? 30000;
24
+ this.halfOpenSuccesses = opts.halfOpenSuccesses ?? 1;
25
+ this.now = opts.now ?? Date.now;
26
+ /** @type {Map<string, {state: string, failures: number, openedAt: number, successes: number, lastError: string|null}>} */
27
+ this.targets = new Map();
28
+ }
29
+
30
+ /** @param {string} key */
31
+ #entry(key) {
32
+ let e = this.targets.get(key);
33
+ if (!e) {
34
+ e = { state: CLOSED, failures: 0, openedAt: 0, successes: 0, lastError: null };
35
+ this.targets.set(key, e);
36
+ }
37
+ return e;
38
+ }
39
+
40
+ /** @param {string} key */
41
+ canAttempt(key) {
42
+ const e = this.#entry(key);
43
+ if (e.state === CLOSED) return true;
44
+ if (e.state === OPEN) {
45
+ if (this.now() - e.openedAt >= this.cooldownMs) {
46
+ e.state = HALF_OPEN;
47
+ e.successes = 0;
48
+ return true;
49
+ }
50
+ return false;
51
+ }
52
+ // half-open: allow the trial through
53
+ return true;
54
+ }
55
+
56
+ /** @param {string} key */
57
+ recordSuccess(key) {
58
+ const e = this.#entry(key);
59
+ if (e.state === HALF_OPEN) {
60
+ e.successes += 1;
61
+ if (e.successes >= this.halfOpenSuccesses) {
62
+ e.state = CLOSED;
63
+ e.failures = 0;
64
+ e.lastError = null;
65
+ }
66
+ return;
67
+ }
68
+ e.state = CLOSED;
69
+ e.failures = 0;
70
+ e.lastError = null;
71
+ }
72
+
73
+ /**
74
+ * @param {string} key
75
+ * @param {string} [reason]
76
+ */
77
+ recordFailure(key, reason) {
78
+ const e = this.#entry(key);
79
+ e.failures += 1;
80
+ e.lastError = reason ?? e.lastError;
81
+ if (e.state === HALF_OPEN || e.failures >= this.threshold) {
82
+ e.state = OPEN;
83
+ e.openedAt = this.now();
84
+ e.successes = 0;
85
+ }
86
+ }
87
+
88
+ /** @param {string} key */
89
+ reset(key) {
90
+ this.targets.delete(key);
91
+ }
92
+
93
+ resetAll() {
94
+ this.targets.clear();
95
+ }
96
+
97
+ /** @param {string} key */
98
+ stateOf(key) {
99
+ const e = this.#entry(key);
100
+ const cooling = e.state === OPEN ? Math.max(0, this.cooldownMs - (this.now() - e.openedAt)) : 0;
101
+ return {
102
+ key,
103
+ state: e.state,
104
+ failures: e.failures,
105
+ lastError: e.lastError,
106
+ retryInMs: cooling,
107
+ };
108
+ }
109
+
110
+ snapshot() {
111
+ return [...this.targets.keys()].map((k) => this.stateOf(k));
112
+ }
113
+ }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * @drsamonline/router/classify — the 50 ms classifier (B3.5).
3
+ *
4
+ * Before a request goes anywhere, decide whether it needs the expensive model.
5
+ * The classifier is deterministic and has no model of its own: it reads the
6
+ * prompt's shape, and it must answer in well under 50 ms, because a classifier
7
+ * that costs a request has defeated the point of saving one.
8
+ *
9
+ * Signals, each of which can be pointed at in a test:
10
+ *
11
+ * length a long prompt is usually a real task, not a lookup
12
+ * code fenced code, or a request to write/debug code
13
+ * multi-step "first … then", numbered steps, "after that"
14
+ * reasoning arithmetic, comparison, "why", "prove", "explain the trade-off"
15
+ * schema JSON/schema/tool output wanted
16
+ * terse a short factual question with no verbs that ask for work
17
+ *
18
+ * The output is a decision, not a score: `simple` or `complex`, plus the signals
19
+ * that produced it so `sui explain` can say why.
20
+ */
21
+
22
+ const COMPLEX_MARKERS = [
23
+ 'first', 'then', 'after that', 'step by step', 'steps', 'walk me through',
24
+ 'prove', 'derive', 'trade-off', 'tradeoff', 'compare', 'why does', 'why is',
25
+ 'architect', 'design', 'migrate', 'refactor', 'optimi', 'debug', 'root cause',
26
+ ];
27
+
28
+ const CODE_MARKERS = [
29
+ '```', 'function', 'class ', 'import ', 'select ', 'regex', 'sql', 'typescript',
30
+ 'javascript', 'python', 'rust', 'golang', 'compile', 'stack trace', 'unit test',
31
+ ];
32
+
33
+ const SCHEMA_MARKERS = ['json', 'schema', 'tool_call', 'function call', 'yaml', 'csv', 'table'];
34
+
35
+ const TRIVIAL_MARKERS = ['what is', 'who is', 'when did', 'how many', 'capital of', 'define'];
36
+
37
+ /** Cheap token estimate: the classifier must not parse, just count. */
38
+ function words(text) {
39
+ let count = 0;
40
+ for (let i = 0; i < text.length; i++) if (text[i] === ' ' || text[i] === '\n') count += 1;
41
+ return count + 1;
42
+ }
43
+
44
+ /**
45
+ * @param {string} text
46
+ * @param {{maxMs?: number, longWords?: number}} [opts]
47
+ * @returns {{complexity: 'simple'|'complex', signals: string[], ms: number, confidence: number}}
48
+ */
49
+ export function classifyPrompt(text, opts = {}) {
50
+ const started = process.hrtime.bigint();
51
+ const prompt = String(text ?? '');
52
+ const lower = prompt.toLowerCase();
53
+ const signals = [];
54
+
55
+ const wordCount = words(prompt);
56
+ const longWords = opts.longWords ?? 120;
57
+
58
+ if (wordCount > longWords) signals.push(`long prompt (${wordCount} words)`);
59
+ if (CODE_MARKERS.some((m) => lower.includes(m))) signals.push('code');
60
+ if (SCHEMA_MARKERS.some((m) => lower.includes(m))) signals.push('schema');
61
+ const multiStep = COMPLEX_MARKERS.filter((m) => lower.includes(m));
62
+ if (multiStep.length) signals.push(`multi-step (${multiStep.slice(0, 3).join(', ')})`);
63
+ if (/\d+\s*[-+*/x×]\s*\d+/.test(prompt)) signals.push('arithmetic');
64
+ if (/\?[^?]*\?/.test(prompt)) signals.push('multiple questions');
65
+
66
+ // A short, single-question lookup with none of the work markers is simple.
67
+ const looksTrivial =
68
+ wordCount < 40 &&
69
+ !signals.includes('code') &&
70
+ !signals.includes('schema') &&
71
+ multiStep.length === 0 &&
72
+ TRIVIAL_MARKERS.some((m) => lower.startsWith(m) || lower.includes(m));
73
+
74
+ const complexity = signals.length === 0 || (looksTrivial && signals.length <= 1) ? 'simple' : 'complex';
75
+
76
+ // Confidence is about the distance from the decision, not about being right.
77
+ const confidence =
78
+ complexity === 'complex'
79
+ ? Math.min(0.95, 0.5 + 0.15 * signals.length)
80
+ : Math.max(0.55, 0.9 - 0.1 * signals.length);
81
+
82
+ const ms = Number(process.hrtime.bigint() - started) / 1e6;
83
+ if (opts.maxMs && ms > opts.maxMs) {
84
+ signals.push(`classifier took ${ms.toFixed(1)}ms (budget ${opts.maxMs}ms)`);
85
+ }
86
+
87
+ return { complexity, signals, ms: Math.round(ms * 1000) / 1000, confidence: Math.round(confidence * 100) / 100 };
88
+ }
package/src/explain.js ADDED
@@ -0,0 +1,74 @@
1
+ /**
2
+ * @drsamonline/router/explain — B7.4, explain-my-route in plain language.
3
+ *
4
+ * A route decision is only trustworthy if it can be narrated. These functions
5
+ * turn the plan (a pure data structure) and a turn's meta into sentences a person
6
+ * can read, without a second query and without storing anything extra.
7
+ */
8
+
9
+ /**
10
+ * @param {{combo: any, ordered: any[], resting: any[], skipped: any[]}} plan
11
+ * @returns {string[]}
12
+ */
13
+ export function explainPlan(plan) {
14
+ const lines = [];
15
+ const combo = plan.combo;
16
+ lines.push(`"${combo.name}" is a ${combo.strategy} route with ${combo.targets.length} target${combo.targets.length === 1 ? '' : 's'}.`);
17
+
18
+ if (plan.ordered?.length) {
19
+ const names = plan.ordered.map((t) => t.modelId);
20
+ lines.push(`It would try ${names.join(' → ')}.`);
21
+ if (combo.strategy === 'cheapest') lines.push('Targets are ordered by blended cost, after the quality floor.');
22
+ if (combo.strategy === 'weighted') lines.push('The first target is picked by weight; the rest are fallbacks in weight order.');
23
+ if (combo.strategy === 'latency') lines.push('Targets are ordered by measured p50 from the ledger.');
24
+ if (plan.learned) {
25
+ lines.push('Fallback learning is on for this combo: targets with enough recent samples are ordered by success rate, then latency, and the declared order breaks ties.');
26
+ }
27
+ } else {
28
+ lines.push('No target is available right now.');
29
+ }
30
+
31
+ for (const skip of plan.skipped ?? []) {
32
+ lines.push(`Skipped ${skip.modelId}: ${skip.reason}.`);
33
+ }
34
+ for (const resting of plan.resting ?? []) {
35
+ lines.push(`Held back ${resting.modelId}: its breaker is open, so it is only tried if everything else fails.`);
36
+ }
37
+ return lines;
38
+ }
39
+
40
+ /**
41
+ * @param {any} meta a completed turn's meta
42
+ * @returns {string[]}
43
+ */
44
+ export function explainTurn(meta) {
45
+ if (!meta) return ['No turn to explain.'];
46
+ const lines = [];
47
+ lines.push(`Answered by ${meta.modelId} (${meta.providerName ?? meta.provider}) in ${meta.latencyMs} ms${meta.cached ? ', from cache' : ''}.`);
48
+
49
+ for (const attempt of meta.attempts ?? []) {
50
+ lines.push(`${attempt.modelId} did not answer: ${attempt.reason}.`);
51
+ }
52
+ if (meta.escalation) lines.push(meta.escalation);
53
+
54
+ if (meta.compression?.preset || meta.compression?.style) {
55
+ const bits = [];
56
+ if (meta.compression.preset) bits.push(`compression ${meta.compression.preset} (${Math.round((meta.compression.ratio ?? 0) * 100)}% smaller)`);
57
+ if (meta.compression.style) bits.push(`output style ${meta.compression.style}`);
58
+ lines.push(`Shaped the request: ${bits.join(', ')}.`);
59
+ }
60
+ if (meta.verification) {
61
+ lines.push(
62
+ meta.verification.verified
63
+ ? `Confidence ${meta.verification.confidence} clears the ${meta.verification.threshold} threshold.`
64
+ : `Confidence ${meta.verification.confidence} is below ${meta.verification.threshold}: ${(meta.verification.reasons ?? []).map((r) => r.detail).join('; ') || 'no detail'}. Treated as unverified.`,
65
+ );
66
+ }
67
+ if (meta.promptCache) {
68
+ lines.push(meta.promptCache.hint ? `Prompt cache: ${meta.promptCache.hint}.` : `Prompt cache unavailable: ${meta.promptCache.unsupported}.`);
69
+ if (meta.promptCache.volatile?.length) {
70
+ lines.push(`The prefix carries ${meta.promptCache.volatile.map((v) => v.name).join(', ')}, which breaks provider caching.`);
71
+ }
72
+ }
73
+ return lines;
74
+ }
package/src/index.js ADDED
@@ -0,0 +1,5 @@
1
+ /** @drsamonline/router — routing decisions, strategies, and the circuit breaker. */
2
+ export * from './router.js';
3
+ export * from './breaker.js';
4
+ export * from './explain.js';
5
+ export * from './classify.js';
package/src/router.js ADDED
@@ -0,0 +1,963 @@
1
+ /**
2
+ * The router.
3
+ *
4
+ * A request names a *combo* — a virtual model — and the router decides which
5
+ * real provider answers it, in what order, and what to do when one of them is
6
+ * resting, out of credit or over budget.
7
+ *
8
+ * Strategies:
9
+ * fallback declared order, first healthy target wins
10
+ * weighted one target picked by weight (A/B test), the rest as fallback
11
+ * cheapest lowest blended cost among targets that clear the quality floor
12
+ * latency fastest measured p50 among healthy targets
13
+ *
14
+ * Every attempt is written to the ledger, including failures, so "why did this
15
+ * go to Gemini?" is answerable afterwards.
16
+ */
17
+
18
+ import { ConfigError, ExhaustedError, ProviderError, SuiError } from '@drsamonline/core';
19
+ import { estimatedCost, estimateTokens, isPaidProvider, privacyRefusal, providerHasCredentials, providerMeetsPrivacyTier, providerMeetsRegion, regionRefusal, resolveCombo, tierRank } from '@drsamonline/core';
20
+ import { ResponseCache } from '@drsamonline/telemetry';
21
+ import { RecallStore } from '@drsamonline/compression';
22
+ import { PrefixMemory, preparePromptCache } from '@drsamonline/promptcache';
23
+ import { classifyPrompt } from './classify.js';
24
+ import { checkGrounding, scoreAnswer, shouldEscalate, verdict as verifyVerdict } from '@drsamonline/verify';
25
+ import { createAdapter, displayName } from '@drsamonline/providers';
26
+ import { checkPrompt, clampMaxTokens, inspectToolChannel } from '@drsamonline/guardrails';
27
+ import { applyStyle, choosePreset, chooseStyle, compress as compressText } from '@drsamonline/compression';
28
+
29
+ /**
30
+ * @typedef {{
31
+ * provider: string, providerName: string, model: string, modelId: string, combo: string,
32
+ * arm: string|null, cached: boolean, verified: boolean, latencyMs: number, ttftMs: number|null,
33
+ * costUsd: number, usage: {prompt_tokens: number, completion_tokens: number},
34
+ * strategy: string, attempts: Array<{modelId: string, provider: string, reason: string}>
35
+ * }} TurnMeta
36
+ */
37
+
38
+ export class Router {
39
+ /**
40
+ * @param {{
41
+ * config: any, ledger: any, breaker: any, cache: any,
42
+ * logger?: {debug: Function, info: Function, warn: Function},
43
+ * now?: () => number, random?: () => number
44
+ * }} deps
45
+ */
46
+ constructor(deps) {
47
+ this.config = deps.config;
48
+ this.ledger = deps.ledger;
49
+ this.breaker = deps.breaker;
50
+ this.cache = deps.cache;
51
+ this.now = deps.now ?? Date.now;
52
+ this.random = deps.random ?? Math.random;
53
+ this.logger = deps.logger ?? { debug() {}, info() {}, warn() {} };
54
+ this.canary = deps.canary ?? null;
55
+ /** @type {Map<string, {until: number, note: string}>} B9.4: provider quota windows */
56
+ this.quotaResets = new Map();
57
+ /** @type {Map<string, any>} */
58
+ this.adapters = new Map();
59
+ }
60
+
61
+ /** @param {string} providerId */
62
+ adapterFor(providerId) {
63
+ let a = this.adapters.get(providerId);
64
+ if (!a) {
65
+ const provider = this.config.providers[providerId];
66
+ if (!provider) {
67
+ throw new ConfigError(`Model points at provider "${providerId}", which is not declared.`, {
68
+ path: 'providers',
69
+ });
70
+ }
71
+ a = createAdapter(providerId, provider, this.config.env);
72
+ this.adapters.set(providerId, a);
73
+ }
74
+ return a;
75
+ }
76
+
77
+ /** Rebuild adapters after a config change. */
78
+ refresh() {
79
+ this.adapters.clear();
80
+ }
81
+
82
+ /**
83
+ * Work out the attempt order for a combo, and why anything was left out.
84
+ * Pure decision-making: no network, no side effects.
85
+ *
86
+ * Two invariants live here:
87
+ * I12 a provider marked tos:avoid is never auto-routed
88
+ * I14 if every target is ejected by a breaker, the request is still
89
+ * attempted against the unfiltered enabled list rather than refused
90
+ *
91
+ * @param {string} comboName
92
+ * @param {{promptTokens?: number, keyMeta?: any, exclude?: string[], allowAvoid?: boolean}} [opts]
93
+ */
94
+ plan(comboName, opts = {}) {
95
+ const privacyTier = opts.privacyTier ?? this.config.privacy?.defaultTier ?? 'any';
96
+ const region = opts.region ?? this.config.privacy?.region ?? 'any';
97
+ let combo = resolveCombo(this.config, comboName);
98
+
99
+ // B3.5: a cascade combo chooses its own targets from the prompt's shape.
100
+ let cascade = null;
101
+ if (combo.cascade?.enabled) {
102
+ const decision = classifyPrompt(opts.promptText ?? '', { maxMs: 50, longWords: combo.cascade.longWords });
103
+ const chosen = decision.complexity === 'complex' ? combo.cascade.complex : combo.cascade.simple;
104
+ if (Array.isArray(chosen) && chosen.length) {
105
+ const mapped = chosen.map((id) => combo.targets.find((t) => t.modelId === id)).filter(Boolean);
106
+ // A cascade list that names models the combo does not have must not empty
107
+ // the route: keep the declared order and say so in the signals.
108
+ if (mapped.length) combo = { ...combo, targets: mapped };
109
+ else cascade = { ...decision, signals: [...decision.signals, `cascade list "${chosen.join(', ')}" matched no target — using the declared order`] };
110
+ }
111
+ cascade = cascade ?? decision;
112
+ }
113
+ const promptTokens = opts.promptTokens ?? 0;
114
+ const exclude = new Set(opts.exclude ?? []);
115
+ /** @type {Array<{modelId: string, provider: string, reason: string}>} */
116
+ const skipped = [];
117
+ const allowedModels = opts.keyMeta?.allowedModels ? new Set(opts.keyMeta.allowedModels) : null;
118
+
119
+ const dailyBudget = this.config.budgets.dailyUsd;
120
+ const spentToday = this.ledger.todaySpendUsd();
121
+ const budgetGone = dailyBudget != null && spentToday >= Number(dailyBudget);
122
+
123
+ /** @type {Array<any>} */
124
+ const usable = [];
125
+ /** @type {Array<any>} */
126
+ const resting = [];
127
+
128
+ for (const target of combo.targets) {
129
+ const { modelId, model } = target;
130
+ const providerId = model?.provider;
131
+ const provider = this.config.providers[providerId];
132
+
133
+ if (!model) {
134
+ skipped.push({ modelId, provider: String(providerId ?? '?'), reason: 'not declared in models' });
135
+ continue;
136
+ }
137
+ if (exclude.has(modelId)) {
138
+ skipped.push({ modelId, provider: providerId, reason: 'excluded by policy' });
139
+ continue;
140
+ }
141
+ if (allowedModels && !allowedModels.has(modelId)) {
142
+ skipped.push({ modelId, provider: providerId, reason: 'not allowed for this API key' });
143
+ continue;
144
+ }
145
+ if (provider?.tos === 'avoid' && !opts.allowAvoid) {
146
+ skipped.push({ modelId, provider: providerId, reason: 'terms of service marked avoid (never auto-routed)' });
147
+ continue;
148
+ }
149
+ // B9.4: a provider that told us when to come back is not tried before then.
150
+ const reset = this.quotaResets.get(modelId);
151
+ if (reset) {
152
+ if (reset.until > this.now()) {
153
+ const waitS = Math.max(1, Math.ceil((reset.until - this.now()) / 1000));
154
+ skipped.push({ modelId, provider: providerId, reason: `quota window closed, reopens in ${waitS}s` });
155
+ continue;
156
+ }
157
+ this.quotaResets.delete(modelId);
158
+ this.logger.info?.(`${displayName(providerId)} (${modelId}) quota window reopened`);
159
+ }
160
+
161
+ // B6.5: checked before credentials, because sending a prompt to a provider
162
+ // that trains on it is not recoverable.
163
+ if (!providerMeetsPrivacyTier(provider, privacyTier)) {
164
+ skipped.push({ modelId, provider: providerId, reason: privacyRefusal(provider, privacyTier) });
165
+ continue;
166
+ }
167
+ // B6.6: residency, checked for the same reason
168
+ if (!providerMeetsRegion(provider, region)) {
169
+ skipped.push({ modelId, provider: providerId, reason: regionRefusal(provider, region) });
170
+ continue;
171
+ }
172
+ if (!providerHasCredentials(provider, this.config.env, this.config)) {
173
+ skipped.push({
174
+ modelId,
175
+ provider: providerId,
176
+ reason: isPaidProvider(provider) && this.config.paidTiers?.enabled !== true
177
+ ? 'paid tier is closed (paid_tiers.enabled is false)'
178
+ : provider?.apiKeyEnv
179
+ ? `no API key set (${provider.apiKeyEnv})`
180
+ : 'provider not configured',
181
+ });
182
+ continue;
183
+ }
184
+ // Money is checked before health: a resting provider that is also over
185
+ // budget must not be woken up by the I14 escape hatch.
186
+ if (budgetGone) {
187
+ skipped.push({ modelId, provider: providerId, reason: 'daily budget reached' });
188
+ continue;
189
+ }
190
+ const providerCap = this.config.budgets.perProvider?.[providerId];
191
+ if (providerCap != null && this.ledger.providerSpendToday(providerId) >= Number(providerCap)) {
192
+ skipped.push({ modelId, provider: providerId, reason: `provider budget reached ($${providerCap}/day)` });
193
+ continue;
194
+ }
195
+ if (combo.strategy === 'cheapest' && Number(model.quality ?? 5) < Number(combo.minQuality ?? 1)) {
196
+ skipped.push({
197
+ modelId,
198
+ provider: providerId,
199
+ reason: `quality ${model.quality ?? '?'} is below the floor of ${combo.minQuality}`,
200
+ });
201
+ continue;
202
+ }
203
+ if (combo.maxCostPer1k != null) {
204
+ const blended = Number(model.costPer1kIn ?? 0) + Number(model.costPer1kOut ?? 0);
205
+ if (blended > Number(combo.maxCostPer1k)) {
206
+ skipped.push({ modelId, provider: providerId, reason: `costs more than the $${combo.maxCostPer1k}/1k cap` });
207
+ continue;
208
+ }
209
+ }
210
+
211
+ const entry = { ...target, providerId, provider, estimate: this.#estimateCost(model, promptTokens) };
212
+
213
+ if (!this.breaker.canAttempt(modelId)) {
214
+ // I14: not a skip. Held back as a last resort so a health blip cannot
215
+ // become a total outage.
216
+ resting.push(entry);
217
+ const st = this.breaker.stateOf(modelId);
218
+ skipped.push({
219
+ modelId,
220
+ provider: providerId,
221
+ reason: `resting after a failure (retrying in ${Math.ceil(st.retryInMs / 1000)}s) — held back, not skipped`,
222
+ });
223
+ continue;
224
+ }
225
+
226
+ usable.push(entry);
227
+ }
228
+
229
+ // B3.1: free targets before paid ones, whatever order they were declared in.
230
+ // A combo can opt out; the gateway default is on, because a paid key that
231
+ // happens to be first in a list should not be the thing that spends money.
232
+ const freeFirst = combo.freeFirst ?? this.config.routing?.freeFirst !== false;
233
+ let ordered = usable;
234
+ if (freeFirst && usable.length > 1) {
235
+ ordered = usable
236
+ .map((target, index) => ({ target, index, rank: tierRank(target.model, this.config) }))
237
+ .sort((a, b) => (a.rank !== b.rank ? a.rank - b.rank : a.index - b.index))
238
+ .map((x) => x.target);
239
+ }
240
+
241
+ // B9.3: fallback learning. Opt-in per combo, because reordering changes what
242
+ // answers your traffic and that should be a decision, not a surprise.
243
+ const learning = combo.learning ?? null;
244
+ if (learning?.enabled && ordered.length > 1) {
245
+ const minSamples = Number(learning.minSamples ?? 10);
246
+ const windowMs = Number(learning.windowMs ?? 24 * 3600_000);
247
+ const since = this.now() - windowMs;
248
+ const rows = this.ledger.entries({ since });
249
+ const stats = new Map();
250
+ for (const row of rows) {
251
+ if (!row.modelId) continue;
252
+ const entry = stats.get(row.modelId) ?? { ok: 0, total: 0, latencies: [] };
253
+ entry.total += 1;
254
+ if (row.ok) {
255
+ entry.ok += 1;
256
+ if (Number.isFinite(row.latencyMs)) entry.latencies.push(row.latencyMs);
257
+ }
258
+ stats.set(row.modelId, entry);
259
+ }
260
+
261
+ const score = (modelId) => {
262
+ const entry = stats.get(modelId);
263
+ if (!entry || entry.total < minSamples) return null; // not enough evidence: keep the declared position
264
+ const successRate = entry.ok / entry.total;
265
+ const sorted = [...entry.latencies].sort((a, b) => a - b);
266
+ const p50 = sorted.length ? sorted[Math.floor(sorted.length / 2)] : Number.MAX_SAFE_INTEGER;
267
+ return { successRate, p50, total: entry.total };
268
+ };
269
+
270
+ const withScore = ordered.map((target, index) => ({ target, index, score: score(target.modelId) }));
271
+ withScore.sort((a, b) => {
272
+ if (a.score && !b.score) return -1; // measured beats unmeasured
273
+ if (!a.score && b.score) return 1;
274
+ if (a.score && b.score) {
275
+ if (Math.abs(b.score.successRate - a.score.successRate) > 0.05) return b.score.successRate - a.score.successRate;
276
+ if (a.score.p50 !== b.score.p50) return a.score.p50 - b.score.p50;
277
+ }
278
+ return a.index - b.index; // stable: declared order breaks every tie
279
+ });
280
+ ordered = withScore.map((x) => x.target);
281
+ }
282
+ if (combo.strategy === 'cheapest') {
283
+ ordered = [...usable].sort((a, b) => a.estimate - b.estimate || Number(b.model.quality ?? 0) - Number(a.model.quality ?? 0));
284
+ } else if (combo.strategy === 'latency') {
285
+ ordered = [...usable].sort((a, b) => this.#p50(a.modelId) - this.#p50(b.modelId));
286
+ } else if (combo.strategy === 'weighted' && usable.length) {
287
+ const primary = pickWeighted(usable, this.random);
288
+ ordered = [primary, ...usable.filter((t) => t.modelId !== primary.modelId).sort((a, b) => b.weight - a.weight)];
289
+ }
290
+
291
+ return {
292
+ combo,
293
+ ordered,
294
+ resting,
295
+ skipped,
296
+ spentToday,
297
+ dailyBudget,
298
+ privacyTier,
299
+ region,
300
+ learned: Boolean(combo.learning?.enabled),
301
+ cascade,
302
+ };
303
+ }
304
+
305
+ /**
306
+ * Compress everything except the last user message.
307
+ *
308
+ * The preset is chosen from the provider budget as a quota proxy until real
309
+ * quota telemetry lands (B2.3 follow-on): a provider at 90% of its daily cap
310
+ * is a provider whose remaining tokens are worth more.
311
+ *
312
+ * @param {Array<{role: string, content: string}>} messages
313
+ * @param {string} comboName
314
+ */
315
+ #shapeContext(messages, comboName) {
316
+ const settings = this.config.compression ?? {};
317
+ const style = settings.outputStyle ?? 'off';
318
+ const wantsStyle = style !== 'off';
319
+
320
+ if (!settings.enabled && !wantsStyle) return { messages, meta: null };
321
+
322
+ const lastUserIndex = settings.keepLastUserMessage
323
+ ? messages.map((m) => m.role).lastIndexOf('user')
324
+ : -1;
325
+
326
+ let preset = settings.preset;
327
+ let presetReason = null;
328
+ if (preset === 'auto') {
329
+ const quota = this.#quotaFraction(comboName);
330
+ const chosen = choosePreset({ fraction: quota });
331
+ preset = chosen.preset;
332
+ presetReason = chosen.reason;
333
+ }
334
+
335
+ const store = this.recallStore ?? (this.recallStore = new RecallStore({ secret: this.config.recallSecret }));
336
+
337
+ // B4.5 cache-aware compression: a provider caches a prefix, and compressing
338
+ // that prefix changes it, which invalidates the cache. So when prompt caching
339
+ // is on, the system message — the most stable part of any prompt — is left
340
+ // exactly as written, and compression works on the turns after it.
341
+ const protectPrefix = settings.preserveCachedPrefix !== false && this.config.promptCache?.enabled !== false;
342
+ const prefixEnd = protectPrefix ? firstNonSystemIndex(messages) : 0;
343
+
344
+ let originalChars = 0;
345
+ let compressedChars = 0;
346
+ const steps = [];
347
+ const next = messages.map((message, index) => {
348
+ if (index === lastUserIndex) return message;
349
+ if (index < prefixEnd) return message; // the cacheable prefix stays untouched
350
+ if (typeof message.content !== 'string' || message.content.length < settings.minChars) return message;
351
+ const result = compressText(message.content, { preset, store });
352
+ originalChars += result.originalChars;
353
+ compressedChars += result.compressedChars;
354
+ for (const step of result.steps) if (step.droppedChars) steps.push({ ...step, message: index });
355
+ return { ...message, content: result.text };
356
+ });
357
+
358
+ // B4.4: the output axis. 'auto' follows the same quota reading as the preset.
359
+ let resolvedStyle = style;
360
+ let styleReason = null;
361
+ if (style === 'auto') {
362
+ const chosen = chooseStyle({ fraction: this.#quotaFraction(comboName), taskHint: messages[messages.length - 1]?.content });
363
+ resolvedStyle = chosen.style;
364
+ styleReason = chosen.reason;
365
+ }
366
+ const styled = resolvedStyle !== 'off' ? applyStyle(next, resolvedStyle) : next;
367
+
368
+ const meta = {
369
+ preset: settings.enabled ? preset : null,
370
+ prefixProtected: protectPrefix && prefixEnd > 0,
371
+ reason: presetReason,
372
+ style: resolvedStyle === 'off' ? null : resolvedStyle,
373
+ styleReason,
374
+ ratio: originalChars ? Math.round(((originalChars - compressedChars) / originalChars) * 1000) / 1000 : 0,
375
+ savedChars: Math.max(0, originalChars - compressedChars),
376
+ steps,
377
+ };
378
+
379
+ // Say what happened even when nothing was compressed: "the prefix was
380
+ // protected" is a different fact from "there was nothing to do".
381
+ const skipped = steps.length
382
+ ? null
383
+ : protectPrefix && prefixEnd > 0
384
+ ? 'the cacheable prefix was left untouched so the provider cache still hits'
385
+ : 'nothing in this prompt was worth compressing';
386
+
387
+ if (!settings.enabled && resolvedStyle === 'off') return { messages: next, meta: null };
388
+ meta.skipped = skipped;
389
+ return { messages: styled, meta };
390
+ }
391
+
392
+ /** B9.4: the quota windows currently being honoured. */
393
+ quotaSnapshot() {
394
+ const now = this.now();
395
+ return [...this.quotaResets.entries()].map(([modelId, reset]) => ({
396
+ modelId,
397
+ reopensInMs: Math.max(0, reset.until - now),
398
+ reopensAt: reset.until,
399
+ }));
400
+ }
401
+
402
+ /** How many targets are currently resting — the signal that a failure is an outage. */
403
+ #breakersOpen() {
404
+ return this.breaker.snapshot().filter((b) => b.state === 'open').length;
405
+ }
406
+
407
+ /** 1 = plenty of budget left, 0 = none. Used to pick a compression preset. */
408
+ #quotaFraction(comboName) {
409
+ const combo = this.config.combos?.[comboName];
410
+ const providerId = combo ? this.config.models?.[combo.targets?.[0]]?.provider : null;
411
+ const cap = providerId ? this.config.budgets?.perProvider?.[providerId] : null;
412
+ if (cap == null) {
413
+ const daily = this.config.budgets?.dailyUsd;
414
+ if (daily == null) return 1;
415
+ return Math.max(0, 1 - this.ledger.todaySpendUsd() / Number(daily));
416
+ }
417
+ return Math.max(0, 1 - this.ledger.providerSpendToday(providerId) / Number(cap));
418
+ }
419
+
420
+ #estimateCost(model, promptTokens) {
421
+ const blended = (Number(model.costPer1kIn ?? 0) + Number(model.costPer1kOut ?? 0)) / 2;
422
+ return (blended * Math.max(promptTokens, 1)) / 1000;
423
+ }
424
+
425
+ #p50(modelId) {
426
+ const rows = this.ledger.entries().filter((r) => r.modelId === modelId && r.ok);
427
+ if (!rows.length) return Number.POSITIVE_INFINITY;
428
+ const sorted = rows.map((r) => r.latencyMs).sort((a, b) => a - b);
429
+ return sorted[Math.floor(sorted.length / 2)];
430
+ }
431
+
432
+ /**
433
+ * Run a chat turn.
434
+ *
435
+ * @param {{
436
+ * combo: string, body: any, keyMeta?: any,
437
+ * onToken?: (delta: string) => void, now?: () => number
438
+ * }} input
439
+ * @returns {Promise<{text: string, usage: {prompt_tokens: number, completion_tokens: number}, finishReason: string|null, meta: TurnMeta}>}
440
+ */
441
+ async chat(input) {
442
+ const started = this.now();
443
+ const body = input.body ?? {};
444
+
445
+ // B9.2: the canary decides which combo answers, before anything else runs.
446
+ // Named `canaryPick`, not `arm`: `arm` is the A/B arm of a weighted target
447
+ // inside the loop, and shadowing it here silently broke attribution.
448
+ const canaryPick = this.canary ? this.canary.pick() : null;
449
+ const comboName = canaryPick?.arm === 'candidate' ? canaryPick.combo : input.combo;
450
+ const messages = normaliseMessages(body.messages);
451
+ const promptText = messages.map((m) => m.content).join('\n');
452
+ const streamed = Boolean(body.stream);
453
+
454
+ checkPrompt({ messages, guardrails: this.config.guardrails });
455
+
456
+ // B6.3: the prompt is not the only channel. Tool definitions are text the
457
+ // model reads, and tool results are text an attacker may control entirely.
458
+ const toolInspection = inspectToolChannel({
459
+ messages,
460
+ tools: body.tools,
461
+ guardrails: this.config.guardrails,
462
+ });
463
+ if (toolInspection.blocked) {
464
+ const first = toolInspection.findings.find((f) => f.severity === 'high');
465
+ throw new SuiError(`A tool message was refused: ${first.detail}.`, {
466
+ code: 'tool_channel_blocked',
467
+ status: 400,
468
+ detail: toolInspection.findings.map((f) => `${f.channel}/${f.where}: ${f.signal}`).join('; '),
469
+ });
470
+ }
471
+
472
+ // B4.1/B4.4: compress the older context, never the question being asked,
473
+ // then shape the output axis if it is on.
474
+ const compressed = this.#shapeContext(messages, comboName);
475
+ const effectiveMessages = compressed.messages;
476
+ const promptTokens = estimateTokens(effectiveMessages.map((m) => m.content).join('\n'));
477
+
478
+ const { combo, ordered, resting, skipped, privacyTier, region, learned, cascade } = this.plan(comboName, {
479
+ region: input.region,
480
+ promptText,
481
+ privacyTier: input.privacyTier,
482
+ promptTokens,
483
+ keyMeta: input.keyMeta,
484
+ allowAvoid: Boolean(input.allowAvoid),
485
+ // plan() has always supported an exclusion list; nothing could reach it
486
+ exclude: input.exclude ?? [],
487
+ });
488
+
489
+ const cacheKey = this.cache?.enabled && !streamed ? ResponseCache.key({ ...body, combo: comboName }) : null;
490
+ if (cacheKey) {
491
+ const hit = this.cache.get(cacheKey, promptText);
492
+ if (hit) {
493
+ const meta = {
494
+ ...hit.value.meta,
495
+ cached: true,
496
+ cacheKind: hit.kind,
497
+ latencyMs: this.now() - started,
498
+ ttftMs: 0,
499
+ attempts: [],
500
+ };
501
+ this.ledger.record({
502
+ provider: meta.provider,
503
+ model: meta.model,
504
+ modelId: meta.modelId,
505
+ combo: comboName,
506
+ arm: meta.arm,
507
+ ok: true,
508
+ promptTokens: hit.value.usage.prompt_tokens,
509
+ completionTokens: hit.value.usage.completion_tokens,
510
+ costUsd: hit.value.meta.costUsd,
511
+ latencyMs: meta.latencyMs,
512
+ ttftMs: 0,
513
+ cached: true,
514
+ verified: meta.verified,
515
+ promptText,
516
+ streamed: false,
517
+ });
518
+ return { text: hit.value.text, usage: hit.value.usage, finishReason: hit.value.finishReason, meta };
519
+ }
520
+ }
521
+
522
+ if (!ordered.length && !resting.length) {
523
+ throw exhausted(comboName, skipped, this.config);
524
+ }
525
+
526
+ /** @type {Array<{modelId: string, provider: string, reason: string}>} */
527
+ const attempts = [];
528
+ let lastError = null;
529
+ let usedResting = false;
530
+ const verification = this.config.verification ?? {};
531
+ let escalationsLeft = Number(verification.maxEscalations ?? 1);
532
+ /** @type {{result: any, meta: any}|null} */
533
+ let bestLow = null;
534
+
535
+ // Healthy targets first; the held-back ones only if every healthy target
536
+ // failed. This is the I14 escape hatch, and it is recorded when used.
537
+ const queue = [...ordered, ...resting.map((t) => ({ ...t, restingFallback: true }))];
538
+
539
+ for (const target of queue) {
540
+ const { modelId, model, providerId, arm } = target;
541
+ const adapter = this.adapterFor(providerId);
542
+ const emptyRetryAllowed = this.config.streaming?.emptyTurnRetry !== false;
543
+ let retriedEmpty = false;
544
+
545
+ // B3.3: an empty turn is a failed turn. Some free tiers answer a 200 with
546
+ // nothing in it when they are throttling — retry the same target once,
547
+ // then move on, rather than handing the user a blank reply.
548
+ for (;;) {
549
+ const attemptStart = this.now();
550
+ let ttft = null;
551
+ let committed = false;
552
+
553
+ /** @type {(delta: string) => void} */
554
+ const emit = (delta) => {
555
+ if (!committed) {
556
+ committed = true;
557
+ ttft = this.now() - attemptStart;
558
+ }
559
+ input.onToken?.(delta);
560
+ };
561
+
562
+ try {
563
+ input.onAttempt?.({ modelId, providerId, providerName: adapter.name, model: model.model, arm: arm ?? null });
564
+
565
+ // B4.6: what the prefix cache can do with this provider.
566
+ const cache = preparePromptCache({
567
+ messages: effectiveMessages,
568
+ providerType: adapter.type,
569
+ memory: this.prefixMemory ?? (this.prefixMemory = new PrefixMemory({ maxEntries: this.config.server?.cache?.maxEntries ?? 500 })),
570
+ enabled: this.config.promptCache?.enabled !== false,
571
+ });
572
+
573
+ const result = await adapter.chat(
574
+ {
575
+ model: model.model,
576
+ messages: effectiveMessages,
577
+ stream: streamed,
578
+ temperature: body.temperature,
579
+ // B6.6: a key may carry tighter guardrails than the gateway.
580
+ max_tokens: clampMaxTokens(body.max_tokens, input.keyMeta?.guardrails ?? this.config.guardrails),
581
+ top_p: body.top_p,
582
+ stop: body.stop,
583
+ cacheHints: cache.hints,
584
+ },
585
+ { onToken: streamed ? emit : undefined },
586
+ );
587
+
588
+ const latencyMs = this.now() - attemptStart;
589
+ const isEmpty = !committed && String(result.text ?? '').trim().length === 0;
590
+
591
+ if (isEmpty && emptyRetryAllowed && !retriedEmpty) {
592
+ retriedEmpty = true;
593
+ attempts.push({ modelId, provider: providerId, reason: 'empty turn — retried once on the same target' });
594
+ this.ledger.record({
595
+ provider: providerId,
596
+ model: model.model,
597
+ modelId,
598
+ combo: comboName,
599
+ arm: arm ?? null,
600
+ ok: false,
601
+ error: 'empty_turn',
602
+ promptTokens: result.usage?.prompt_tokens ?? promptTokens,
603
+ completionTokens: 0,
604
+ costUsd: 0,
605
+ latencyMs,
606
+ ttftMs: ttft,
607
+ cached: false,
608
+ verified: false,
609
+ promptText,
610
+ streamed,
611
+ });
612
+ this.logger.warn(`${displayName(providerId)} (${modelId}) returned an empty turn — retrying once`);
613
+ continue;
614
+ }
615
+
616
+ if (isEmpty) {
617
+ // Still empty after the retry: record it and try the next target.
618
+ attempts.push({ modelId, provider: providerId, reason: 'empty turn (no retry left)' });
619
+ this.ledger.record({
620
+ provider: providerId,
621
+ model: model.model,
622
+ modelId,
623
+ combo: comboName,
624
+ arm: arm ?? null,
625
+ ok: false,
626
+ error: 'empty_turn',
627
+ promptTokens: result.usage?.prompt_tokens ?? promptTokens,
628
+ completionTokens: 0,
629
+ costUsd: 0,
630
+ latencyMs,
631
+ ttftMs: ttft,
632
+ cached: false,
633
+ verified: false,
634
+ promptText,
635
+ streamed,
636
+ });
637
+ break;
638
+ }
639
+
640
+ const costUsd = estimatedCost(model, result.usage.prompt_tokens, result.usage.completion_tokens);
641
+ const verified = adapter.type !== 'mock';
642
+
643
+ // B7.2: what the compression saved, valued at this model's input price.
644
+ const savedTokens = Math.ceil((compressed.meta?.savedChars ?? 0) / 4);
645
+ const savedUsd = savedTokens ? Math.round((Number(model.costPer1kIn ?? 0) * savedTokens) / 1000 * 1e6) / 1e6 : 0;
646
+
647
+ this.breaker.recordSuccess(modelId);
648
+ if (target.restingFallback) usedResting = true;
649
+
650
+ /** @type {TurnMeta} */
651
+ const meta = {
652
+ provider: providerId,
653
+ providerName: adapter.name,
654
+ model: model.model,
655
+ modelId,
656
+ combo: comboName,
657
+ arm: arm ?? null,
658
+ cached: false,
659
+ cacheKind: null,
660
+ verified,
661
+ latencyMs,
662
+ ttftMs: streamed ? ttft : null,
663
+ costUsd,
664
+ usage: result.usage,
665
+ strategy: combo.strategy,
666
+ attempts,
667
+ escalation: usedResting ? 'I14 — every healthy target failed, a resting one was tried' : null,
668
+ compression: compressed.meta,
669
+ verification: null,
670
+ toolInspection: toolInspection.findings.length ? { mode: toolInspection.mode, findings: toolInspection.findings } : null,
671
+ canary: canaryPick ? { id: this.canary.id, arm: canaryPick.arm, reason: canaryPick.reason } : null,
672
+ savedUsd,
673
+ route: {
674
+ strategy: combo.strategy,
675
+ privacyTier,
676
+ region,
677
+ learned,
678
+ cascade,
679
+ considered: [
680
+ ...ordered.map((t) => ({ modelId: t.modelId, provider: t.providerId, outcome: t.modelId === modelId ? 'answered' : 'available' })),
681
+ ...resting.map((t) => ({ modelId: t.modelId, provider: t.providerId, outcome: 'resting' })),
682
+ ...skipped.map((s) => ({ modelId: s.modelId, provider: s.provider, outcome: 'skipped', reason: s.reason })),
683
+ ],
684
+ },
685
+ promptCache: cache.detected
686
+ ? {
687
+ warm: cache.warm,
688
+ sends: cache.sends,
689
+ prefixChars: cache.detected.prefixChars,
690
+ fingerprint: cache.detected.fingerprint,
691
+ volatile: cache.detected.volatile,
692
+ hint: cache.hints.supported ? cache.hints.reason : null,
693
+ unsupported: cache.hints.supported ? null : cache.hints.reason,
694
+ }
695
+ : null,
696
+ };
697
+
698
+ this.ledger.record({
699
+ provider: providerId,
700
+ model: model.model,
701
+ modelId,
702
+ combo: comboName,
703
+ arm: arm ?? null,
704
+ ok: true,
705
+ promptTokens: result.usage.prompt_tokens,
706
+ completionTokens: result.usage.completion_tokens,
707
+ costUsd,
708
+ latencyMs,
709
+ ttftMs: streamed ? ttft : null,
710
+ cached: false,
711
+ verified,
712
+ promptText,
713
+ streamed,
714
+ compressionSavedTokens: savedTokens,
715
+ compressionSavedUsd: savedUsd,
716
+ compressionPreset: compressed.meta?.preset ?? null,
717
+ promptCacheWarm: cache.warm,
718
+ apiKey: input.keyMeta?.name ?? null,
719
+ user: input.attribution?.user ?? null,
720
+ tool: input.attribution?.tool ?? null,
721
+ });
722
+
723
+ // B3.4: score the answer, and escalate if it is not convincing.
724
+ if (verification.enabled) {
725
+ const score = scoreAnswer({
726
+ text: result.text,
727
+ prompt: promptText,
728
+ finishReason: result.finishReason,
729
+ wantsJson: body.response_format?.type === 'json_object',
730
+ });
731
+ // B7.6: grounding runs before the verdict, because an ungrounded
732
+ // answer is not a confident one however fluent it reads.
733
+ const groundingMode = verification.grounding ?? 'off';
734
+ let grounding = null;
735
+ if (groundingMode !== 'off') {
736
+ const contextText = messages
737
+ .filter((m) => m.role === 'tool' || m.role === 'function' || m.role === 'system')
738
+ .map((m) => (typeof m.content === 'string' ? m.content : ''))
739
+ .join('\n');
740
+ const check = checkGrounding({ answer: result.text, context: contextText });
741
+ grounding = { mode: groundingMode, ...check };
742
+ if (check.checked && !check.supported) {
743
+ // a fabricated URL or number is a hard signal, not a hint
744
+ score.reasons.push({
745
+ signal: 'ungrounded',
746
+ penalty: 0.45,
747
+ detail: `${check.findings.length} claim(s) not found in the context: ${check.findings.slice(0, 3).map((f) => f.claim).join(', ')}`,
748
+ });
749
+ score.confidence = Math.max(0, Math.round((score.confidence - 0.45) * 100) / 100);
750
+ }
751
+ }
752
+
753
+ const v = verifyVerdict({ ...score, threshold: Number(verification.threshold ?? 0.7) });
754
+ meta.verification = {
755
+ confidence: v.confidence,
756
+ threshold: v.threshold,
757
+ verified: v.verified,
758
+ reasons: score.reasons,
759
+ escalated: false,
760
+ grounding,
761
+ };
762
+
763
+ if (!v.ok && shouldEscalate({ confidence: v.confidence, threshold: v.threshold, escalationsLeft })) {
764
+ escalationsLeft -= 1;
765
+ meta.verification.escalated = true;
766
+ attempts.push({
767
+ modelId,
768
+ provider: providerId,
769
+ reason: `answered, but confidence ${v.confidence} is below ${v.threshold} — escalating`,
770
+ });
771
+ // Keep the best low-confidence answer in case nothing better comes.
772
+ if (!bestLow || v.confidence > bestLow.meta.verification.confidence) bestLow = { result, meta };
773
+ this.ledger.record({
774
+ provider: providerId,
775
+ model: model.model,
776
+ modelId,
777
+ combo: comboName,
778
+ arm: arm ?? null,
779
+ ok: true,
780
+ error: null,
781
+ promptTokens: result.usage.prompt_tokens,
782
+ completionTokens: result.usage.completion_tokens,
783
+ costUsd,
784
+ latencyMs,
785
+ ttftMs: streamed ? ttft : null,
786
+ cached: false,
787
+ verified,
788
+ promptText,
789
+ streamed,
790
+ apiKey: input.keyMeta?.name ?? null,
791
+ user: input.attribution?.user ?? null,
792
+ tool: input.attribution?.tool ?? null,
793
+ });
794
+ break; // leave this target; the outer loop moves to the next one
795
+ }
796
+
797
+ if (!v.ok) {
798
+ // Fail-closed: the answer is returned, but never presented as verified.
799
+ meta.verified = false;
800
+ meta.escalation = `confidence ${v.confidence} is below ${v.threshold}; returned unverified (fail-closed)`;
801
+ this.logger.warn(`${modelId} answered with confidence ${v.confidence} (threshold ${v.threshold}); returning it marked unverified`);
802
+ }
803
+ }
804
+
805
+ if (cacheKey) {
806
+ this.cache.set(
807
+ cacheKey,
808
+ { text: result.text, usage: result.usage, finishReason: result.finishReason, meta },
809
+ { promptText },
810
+ );
811
+ }
812
+
813
+ if (canaryPick?.arm === 'candidate') {
814
+ const verdict = this.canary.observe({
815
+ ok: true,
816
+ latencyMs,
817
+ // an outage is the provider's problem, not the candidate's
818
+ outage: this.#breakersOpen() > 0,
819
+ });
820
+ if (verdict.event) {
821
+ this.logger.warn(`canary ${this.canary.id}: ${verdict.event} — ${verdict.detail}`);
822
+ meta.canary.outcome = verdict.event;
823
+ }
824
+ }
825
+
826
+ return { text: result.text, usage: result.usage, finishReason: result.finishReason, meta };
827
+ } catch (err) {
828
+ const e = /** @type {any} */ (err);
829
+ const latencyMs = this.now() - attemptStart;
830
+
831
+ // I13: only 5xx and transport failures trip the breaker. A 401 or a 404
832
+ // is a fact about the key or the model name, not a health signal, and
833
+ // tripping on it would hide a config bug behind a cooldown.
834
+ if (tripsBreaker(e)) this.breaker.recordFailure(modelId, e.message);
835
+
836
+ attempts.push({ modelId, provider: providerId, reason: e.message ?? String(err) });
837
+ lastError = e;
838
+
839
+ // B9.2: a failure is an observation too — but only if it is about this
840
+ // candidate. If other targets are already resting, this is an outage
841
+ // and rolling the candidate back would blame it for someone else's
842
+ // downtime.
843
+ // B9.4: honour a stated quota window instead of guessing from failures.
844
+ if (e?.retryAfterMs > 0) {
845
+ const until = this.now() + e.retryAfterMs;
846
+ this.quotaResets.set(modelId, { until, note: `${Math.round(e.retryAfterMs / 1000)}s` });
847
+ this.logger.warn(`${displayName(providerId)} (${modelId}) closed its quota window for ${Math.round(e.retryAfterMs / 1000)}s`);
848
+ }
849
+
850
+ if (canaryPick?.arm === 'candidate') {
851
+ const othersResting = this.breaker.snapshot().filter((b) => b.state === 'open' && b.key !== modelId).length;
852
+ const verdict = this.canary.observe({ ok: false, latencyMs, outage: othersResting > 0 });
853
+ if (verdict.event) this.logger.warn(`canary ${this.canary.id}: ${verdict.event} — ${verdict.detail}`);
854
+ }
855
+
856
+ this.ledger.record({
857
+ provider: providerId,
858
+ model: model.model,
859
+ modelId,
860
+ combo: comboName,
861
+ arm: arm ?? null,
862
+ ok: false,
863
+ error: e.message ?? String(err),
864
+ promptTokens,
865
+ completionTokens: 0,
866
+ costUsd: 0,
867
+ latencyMs,
868
+ ttftMs: ttft,
869
+ cached: false,
870
+ verified: false,
871
+ promptText,
872
+ streamed,
873
+ });
874
+
875
+ this.logger.warn(`${displayName(providerId)} (${modelId}) failed: ${e.message}`);
876
+
877
+ if (committed) {
878
+ // Tokens already went to the client. Switching provider now would
879
+ // splice two different answers together, so stop and say so.
880
+ throw new SuiError(
881
+ `${displayName(providerId)} dropped the connection mid-answer. Nothing was retried because part of the reply had already been sent.`,
882
+ { code: 'stream_interrupted', status: 502, retryable: false, detail: e.message },
883
+ );
884
+ }
885
+ // A non-retryable provider error (401, 404, a malformed request) is a
886
+ // fact about that provider, not a reason to abandon the whole combo:
887
+ // the next target is a different provider with a different key.
888
+ if (!(e instanceof ProviderError)) throw e;
889
+ break; // this target is done; the outer loop moves to the next one
890
+ }
891
+ }
892
+ }
893
+
894
+ // Nothing cleared the bar, but something answered: fail-closed means mark it
895
+ // unverified, not that the user gets nothing.
896
+ if (bestLow) {
897
+ bestLow.meta.verified = false; // fail-closed, on the way out too
898
+ bestLow.meta.escalation = `confidence never reached ${verification.threshold}; returning the best answer, marked unverified`;
899
+ return { text: bestLow.result.text, usage: bestLow.result.usage, finishReason: bestLow.result.finishReason, meta: bestLow.meta };
900
+ }
901
+
902
+ const err = exhausted(comboName, skipped, this.config, attempts);
903
+ if (lastError && !(lastError instanceof ProviderError)) throw lastError;
904
+ throw err;
905
+ }
906
+ }
907
+
908
+ /** Index of the first message that is not part of the stable system prefix. */
909
+ function firstNonSystemIndex(messages) {
910
+ let i = 0;
911
+ while (i < messages.length && messages[i].role === 'system') i += 1;
912
+ return i;
913
+ }
914
+
915
+ /**
916
+ * Build the plain-language "nothing left" error. This is the message a user
917
+ * actually reads, so it says what is resting, why, and what to do next.
918
+ */
919
+ function exhausted(comboName, skipped, config, attempts = []) {
920
+ const parts = [];
921
+ for (const s of skipped) parts.push(`${s.modelId} — ${s.reason}`);
922
+ for (const a of attempts) parts.push(`${a.modelId} — tried and failed: ${a.reason}`);
923
+ const tried = attempts.length ? `Tried ${attempts.length}, all failed. ` : '';
924
+ return new ExhaustedError(
925
+ `No model could answer "${comboName}" right now. ${tried}${parts.length ? `Details: ${parts.join('; ')}.` : ''} Add a provider key, wait for the cooldown, or raise the daily budget.`,
926
+ { attempts: [...attempts, ...skipped.map((s) => ({ model: s.modelId, reason: s.reason }))] },
927
+ );
928
+ }
929
+
930
+ /** @param {any[]} messages */
931
+ function normaliseMessages(messages) {
932
+ if (!Array.isArray(messages) || !messages.length) {
933
+ throw new SuiError('messages must be a non-empty array.', { code: 'bad_request', status: 400 });
934
+ }
935
+ return messages.map((m) => ({
936
+ role: m.role ?? 'user',
937
+ content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content ?? ''),
938
+ }));
939
+ }
940
+
941
+ /** @param {any[]} items @param {() => number} random */
942
+ function pickWeighted(items, random) {
943
+ const total = items.reduce((a, i) => a + (Number(i.weight) || 1), 0);
944
+ let r = random() * total;
945
+ for (const item of items) {
946
+ r -= Number(item.weight) || 1;
947
+ if (r <= 0) return item;
948
+ }
949
+ return items[items.length - 1];
950
+ }
951
+
952
+ /**
953
+ * I13 — does this failure say something about the provider's health?
954
+ * Yes for 5xx, timeouts and transport errors. No for 400/401/403/404/422/429.
955
+ * @param {any} err
956
+ */
957
+ export function tripsBreaker(err) {
958
+ if (!err) return false;
959
+ const code = String(err.code ?? '');
960
+ if (code === 'provider_timeout' || code === 'provider_unreachable') return true;
961
+ const status = Number(err.status ?? 0);
962
+ return status >= 500;
963
+ }
package/README.md DELETED
@@ -1,3 +0,0 @@
1
- # Temporary Holding Version
2
-
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.