@goodandready/dsh-key-rotation 0.8.75 → 0.8.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/lib/client.js +13 -1
- package/lib/error-taxonomy.js +59 -10
- package/lib/errors.js +302 -0
- package/lib/index.js +3 -0
- package/lib/ops-config-io.js +32 -5
- package/lib/pool-selection.js +92 -17
- package/lib/pool.js +39 -5
- package/lib/rotate.js +61 -11
- package/lib/safe-fetch.js +161 -28
- package/lib/session-router.js +265 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.77 (2026-10-30)
|
|
4
|
+
|
|
5
|
+
### Session Routing, Typed Error Taxonomy & Pre-First-Token Failover (Block 5 Release)
|
|
6
|
+
- **Pre-First-Token Failover & Request Attempt Loop (Issue #527)**: Introduced transaction-safe pre-first-token retry loop in `lib/rotate.js` with canonical `isContentChunk` detector (distinguishes model-visible tokens from stream envelopes and metadata). Stream metadata (such as `block-start`, `response-metadata`) emitted prior to the first content chunk is securely buffered; upon upstream failures (HTTP 429, 502, 503, connection drops) before any content token yields, the buffered metadata is cleanly discarded and the request seamlessly transitions to the next available healthy key without chat UI disruption or duplicate blocks.
|
|
7
|
+
- **Typed Error Taxonomy & Canonical classifyResponseError (Issue #547)**: Implemented `lib/errors.js` with structured envelope parsing for OpenAI, Anthropic, Google Gemini, Groq, and Ollama. Added robust RFC 1123 HTTP-Date and numeric seconds parser for `Retry-After` headers converting to exact millisecond cooldowns; mapped provider failures into typed `@deepseek-ai/dsh-llm` `LlmError` instances (`QUOTA`, `ACCOUNT_QUOTA`, `RATE_LIMIT`, `CONTEXT_WINDOW_EXCEEDED`, `INVALID_CREDENTIAL`, `SERVER`, `TIMEOUT`); updated `isSwitchableError` and `classifyFailure` to replace brittle text regex matching with structured JSON payload classification.
|
|
8
|
+
- **Bounded SessionRouter with LRU & Proactive Unsticking (Issue #548)**: Created standalone domain service `SessionRouter` (`lib/session-router.js`) with capacity limit (`MAX_SESSIONS = 1000`) and automatic 24-hour TTL expiration (`SESSION_TTL_MS`). Integrated Least-Used Initial Assignment to distribute new incoming sessions across least-loaded keys; implemented `unstickKey(failedKey)` in `lib/pool-selection.js` and key penalization to proactively unbind all sessions pinned to failed or cooling keys, guaranteeing seamless transparent failover on consecutive prompts.
|
|
9
|
+
|
|
10
|
+
## 0.8.76 (2026-10-30)
|
|
11
|
+
|
|
12
|
+
### Security Hardening, DNS Pinning & Mutation Integrity (Block 4 Release)
|
|
13
|
+
- **SSRF DNS Rebinding Prevention (Issue #557)**: Replaced single-shot DNS precheck with connect-time socket IP validation (`createSafeLookup`) via native HTTPS request handling (`fetchNativeJson`). Enforces loopback and private IP rejection at the exact moment the socket establishes connection across all attempts and redirects, completely neutralizing TOCTOU DNS rebinding with zero runtime dependencies.
|
|
14
|
+
- **Write-Origin Guard & Desktop Spoofing Elimination (Issue #516)**: Added `hasNonLoopbackForwarding` to detect requests proxied through LAN reverse proxies (`x-forwarded-for`, `x-real-ip`). Refused mutating requests (POST/PUT/DELETE/PATCH) lacking Origin or attempting to spoof Desktop headers (`@deepseek-ai/dsh-desktop/` User-Agent, `dsh-auth-` cookies, or `dsh-app://app` Origin) over LAN; ensured `/auto-import` strictly validates bodies and fails closed.
|
|
15
|
+
- **Config Import Optimistic Concurrency & Revision Integrity (Issue #553)**: End-to-end optimistic revision tracking in config import; dry-run preview returns settings revision, WebUI client (`50-card-main.js`) stores preview revision and transmits `expectedRevision` upon apply; backend validates revision match and converts `SETTINGS_CONFLICT` during `settings.replace` into HTTP 409 Conflict with clear preview invalidation guidance instead of generic 500 errors.
|
|
16
|
+
|
|
3
17
|
## 0.8.75 (2026-10-30)
|
|
4
18
|
|
|
5
19
|
### Dependencies, Cleanup & Audit Finalization (Block 3 Release)
|
package/lib/client.js
CHANGED
|
@@ -1316,6 +1316,7 @@ window.__ModuleLoader__.load({
|
|
|
1316
1316
|
const [importPassphrase, setImportPassphrase] = React.useState('');
|
|
1317
1317
|
const [importMode, setImportMode] = React.useState('merge');
|
|
1318
1318
|
const [importPreview, setImportPreview] = React.useState(null);
|
|
1319
|
+
const [importRevision, setImportRevision] = React.useState(null);
|
|
1319
1320
|
const [importError, setImportError] = React.useState('');
|
|
1320
1321
|
const [importRunning, setImportRunning] = React.useState(false);
|
|
1321
1322
|
|
|
@@ -1358,6 +1359,7 @@ window.__ModuleLoader__.load({
|
|
|
1358
1359
|
const runImportPreview = async () => {
|
|
1359
1360
|
setImportError('');
|
|
1360
1361
|
setImportPreview(null);
|
|
1362
|
+
setImportRevision(null);
|
|
1361
1363
|
setImportRunning(true);
|
|
1362
1364
|
try {
|
|
1363
1365
|
let parsed;
|
|
@@ -1381,6 +1383,7 @@ window.__ModuleLoader__.load({
|
|
|
1381
1383
|
const data = await res.json();
|
|
1382
1384
|
if (!res.ok || !data.ok) throw new Error(data?.error?.message || 'Import preview failed');
|
|
1383
1385
|
setImportPreview(data.preview);
|
|
1386
|
+
setImportRevision(data.revision ?? null);
|
|
1384
1387
|
} catch (err) {
|
|
1385
1388
|
setImportError(String(err?.message ?? err));
|
|
1386
1389
|
} finally {
|
|
@@ -1406,14 +1409,23 @@ window.__ModuleLoader__.load({
|
|
|
1406
1409
|
encrypted: parsed.encrypted,
|
|
1407
1410
|
passphrase: importPassphrase.trim() || undefined,
|
|
1408
1411
|
mode: importMode,
|
|
1412
|
+
expectedRevision: importRevision !== null ? importRevision : undefined,
|
|
1409
1413
|
dryRun: false,
|
|
1410
1414
|
}),
|
|
1411
1415
|
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(10000) : undefined,
|
|
1412
1416
|
});
|
|
1413
1417
|
const data = await res.json();
|
|
1414
|
-
if (!res.ok || !data.ok)
|
|
1418
|
+
if (!res.ok || !data.ok) {
|
|
1419
|
+
if (res.status === 409 || data?.error?.code === 'conflict') {
|
|
1420
|
+
setImportPreview(null);
|
|
1421
|
+
setImportRevision(null);
|
|
1422
|
+
throw new Error(data?.error?.message || 'Configuration was modified concurrently. Please preview again before applying.');
|
|
1423
|
+
}
|
|
1424
|
+
throw new Error(data?.error?.message || 'Import apply failed');
|
|
1425
|
+
}
|
|
1415
1426
|
setImportModal(false);
|
|
1416
1427
|
setImportPreview(null);
|
|
1428
|
+
setImportRevision(null);
|
|
1417
1429
|
setImportText('');
|
|
1418
1430
|
void load();
|
|
1419
1431
|
} catch (err) {
|
package/lib/error-taxonomy.js
CHANGED
|
@@ -1,13 +1,35 @@
|
|
|
1
|
-
|
|
2
|
-
// lib/error-taxonomy.js — complete failure classification (#267).
|
|
1
|
+
// lib/error-taxonomy.js — complete failure classification (#267, #547).
|
|
3
2
|
// action: 'switch' | 'surface' | 'cooldown'
|
|
4
3
|
|
|
4
|
+
import { SOFT_FAILURE_CODES } from './pool.js';
|
|
5
|
+
import {
|
|
6
|
+
classifyResponseError,
|
|
7
|
+
parseErrorBody,
|
|
8
|
+
retryAfterMs,
|
|
9
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
10
|
+
QUOTA_EXCEEDED_CODE,
|
|
11
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
12
|
+
EMPTY_RESPONSE_CODE,
|
|
13
|
+
INVALID_CREDENTIAL_CODE,
|
|
14
|
+
} from './errors.js';
|
|
15
|
+
|
|
16
|
+
export {
|
|
17
|
+
classifyResponseError,
|
|
18
|
+
parseErrorBody,
|
|
19
|
+
retryAfterMs,
|
|
20
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
21
|
+
QUOTA_EXCEEDED_CODE,
|
|
22
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
23
|
+
EMPTY_RESPONSE_CODE,
|
|
24
|
+
INVALID_CREDENTIAL_CODE,
|
|
25
|
+
};
|
|
26
|
+
|
|
5
27
|
const SWITCH_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
|
|
6
28
|
const SURFACE_STATUS = new Set([400, 404, 422]);
|
|
7
29
|
const AUTH_STATUS = new Set([401, 403]);
|
|
8
30
|
|
|
9
31
|
const SWITCH_CODES = new Set([
|
|
10
|
-
'QUOTA', 'RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT', 'EMPTY_RESPONSE',
|
|
32
|
+
'QUOTA', 'ACCOUNT_QUOTA', 'RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT', 'EMPTY_RESPONSE',
|
|
11
33
|
'UNKNOWN_MODEL', 'AUTH', 'RESOURCE_EXHAUSTED', 'UNAVAILABLE', 'INTERNAL',
|
|
12
34
|
'DEADLINE_EXCEEDED', 'UNAUTHENTICATED', 'PERMISSION_DENIED', 'ABORTED',
|
|
13
35
|
]);
|
|
@@ -19,7 +41,7 @@ const SOCKET_SWITCH = new Set([
|
|
|
19
41
|
/**
|
|
20
42
|
* Classify a failure for rotation.
|
|
21
43
|
* @param {any} failureOrPayload
|
|
22
|
-
* @returns {{ action: 'switch'|'surface'|'cooldown', code: string, soft: boolean, reason: string }}
|
|
44
|
+
* @returns {{ action: 'switch'|'surface'|'cooldown', code: string, soft: boolean, reason: string, retryAfterMs?: number }}
|
|
23
45
|
*/
|
|
24
46
|
export function classifyFailure(failureOrPayload) {
|
|
25
47
|
if (!failureOrPayload) {
|
|
@@ -29,18 +51,46 @@ export function classifyFailure(failureOrPayload) {
|
|
|
29
51
|
const status = Number(failure.status ?? failure.statusCode ?? failure.httpStatus ?? 0);
|
|
30
52
|
const code = String(failure.code ?? failure.reason ?? failure.name ?? '').toUpperCase();
|
|
31
53
|
const message = String(failure.message ?? failureOrPayload.message ?? '');
|
|
54
|
+
const headers = failure.headers ?? failureOrPayload.headers ?? failure.response?.headers;
|
|
55
|
+
const retryMs = retryAfterMs(headers);
|
|
56
|
+
|
|
57
|
+
// Check structured body/error through classifyResponseError if available
|
|
58
|
+
if (status || failure.body || (code && code !== 'ERROR')) {
|
|
59
|
+
const typed = classifyResponseError(status, failure.body ?? message, headers);
|
|
60
|
+
if (typed.code === CONTEXT_WINDOW_EXCEEDED_CODE) {
|
|
61
|
+
return { action: 'surface', code: CONTEXT_WINDOW_EXCEEDED_CODE, soft: false, reason: 'context_window' };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
32
64
|
|
|
33
|
-
if (AUTH_STATUS.has(status) || code === 'AUTH' || code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED') {
|
|
34
|
-
return {
|
|
65
|
+
if (AUTH_STATUS.has(status) || code === 'AUTH' || code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED' || code === INVALID_CREDENTIAL_CODE) {
|
|
66
|
+
return {
|
|
67
|
+
action: 'switch',
|
|
68
|
+
code: status ? String(status) : (code || 'AUTH'),
|
|
69
|
+
soft: false,
|
|
70
|
+
reason: 'auth',
|
|
71
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
72
|
+
};
|
|
35
73
|
}
|
|
36
|
-
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA') {
|
|
37
|
-
return {
|
|
74
|
+
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA' || code === 'ACCOUNT_QUOTA' || code === QUOTA_EXCEEDED_CODE) {
|
|
75
|
+
return {
|
|
76
|
+
action: 'switch',
|
|
77
|
+
code: status ? '429' : (code || 'RATE_LIMIT'),
|
|
78
|
+
soft: false,
|
|
79
|
+
reason: 'quota',
|
|
80
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
81
|
+
};
|
|
38
82
|
}
|
|
39
83
|
const numCode = Number(code);
|
|
40
84
|
if (SWITCH_STATUS.has(status) || (numCode > 0 && SWITCH_STATUS.has(numCode)) || SOFT_FAILURE_CODES.has(code)) {
|
|
41
85
|
const effCode = status ? String(status) : (numCode > 0 ? String(numCode) : code);
|
|
42
86
|
const soft = SOFT_FAILURE_CODES.has(effCode);
|
|
43
|
-
return {
|
|
87
|
+
return {
|
|
88
|
+
action: 'switch',
|
|
89
|
+
code: effCode,
|
|
90
|
+
soft,
|
|
91
|
+
reason: 'http',
|
|
92
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
93
|
+
};
|
|
44
94
|
}
|
|
45
95
|
if (SURFACE_STATUS.has(status)) {
|
|
46
96
|
return { action: 'surface', code: String(status), soft: false, reason: 'client' };
|
|
@@ -63,4 +113,3 @@ export function classifyFailure(failureOrPayload) {
|
|
|
63
113
|
return { action: 'surface', code: code || 'UNKNOWN', soft: false, reason: 'unclassified' };
|
|
64
114
|
}
|
|
65
115
|
|
|
66
|
-
/** True when classification says switch to next key. */
|
package/lib/errors.js
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
// lib/errors.js — Canonical error classification and taxonomy for DeepSeek Harness (#547).
|
|
2
|
+
// Unifies error shapes across OpenAI, Anthropic, Gemini, Ollama, Groq and OpenCode into standard LlmError.
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
HarnessError as DshHarnessError,
|
|
6
|
+
LlmError as DshLlmError,
|
|
7
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
8
|
+
QUOTA_EXCEEDED_CODE,
|
|
9
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
10
|
+
EMPTY_RESPONSE_CODE,
|
|
11
|
+
INVALID_CREDENTIAL_CODE,
|
|
12
|
+
isContextWindowExceededError,
|
|
13
|
+
isQuotaExceededError,
|
|
14
|
+
} from '@deepseek-ai/dsh-llm';
|
|
15
|
+
|
|
16
|
+
export {
|
|
17
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
18
|
+
QUOTA_EXCEEDED_CODE,
|
|
19
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
20
|
+
EMPTY_RESPONSE_CODE,
|
|
21
|
+
INVALID_CREDENTIAL_CODE,
|
|
22
|
+
isContextWindowExceededError,
|
|
23
|
+
isQuotaExceededError,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
// Fallback HarnessError and LlmError if ever decoupled from runtime
|
|
27
|
+
export class HarnessError extends (DshHarnessError || Error) {
|
|
28
|
+
constructor(message, code, options) {
|
|
29
|
+
super(message, code, options);
|
|
30
|
+
this.code = code;
|
|
31
|
+
this.name = this.constructor.name;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class LlmError extends (DshLlmError || HarnessError) {
|
|
36
|
+
constructor(message, code, options) {
|
|
37
|
+
super(message, code, options);
|
|
38
|
+
this.status = options?.status;
|
|
39
|
+
this.providerRetryAfterMs = options?.providerRetryAfterMs;
|
|
40
|
+
this.code = code;
|
|
41
|
+
this.name = this.constructor.name;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Robust parser for structured error bodies (JSON, object, text).
|
|
47
|
+
* Handles OpenAI, Anthropic, Gemini, Groq, Ollama envelopes.
|
|
48
|
+
*
|
|
49
|
+
* @param {unknown} body
|
|
50
|
+
* @returns {{ message?: string, type?: string, code?: string, status?: number, raw?: unknown }}
|
|
51
|
+
*/
|
|
52
|
+
export function parseErrorBody(body) {
|
|
53
|
+
if (!body) return {};
|
|
54
|
+
if (typeof body === 'object') {
|
|
55
|
+
return extractErrorFields(body);
|
|
56
|
+
}
|
|
57
|
+
if (typeof body === 'string') {
|
|
58
|
+
const trimmed = body.trim();
|
|
59
|
+
if (!trimmed) return {};
|
|
60
|
+
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
|
61
|
+
try {
|
|
62
|
+
const parsed = JSON.parse(trimmed);
|
|
63
|
+
return extractErrorFields(parsed);
|
|
64
|
+
} catch {
|
|
65
|
+
// Not valid JSON, return as plain message
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { message: trimmed.slice(0, 1000) };
|
|
69
|
+
}
|
|
70
|
+
return { message: String(body).slice(0, 1000) };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function extractErrorFields(obj) {
|
|
74
|
+
if (!obj || typeof obj !== 'object') return {};
|
|
75
|
+
const outer = obj;
|
|
76
|
+
const nested = outer.error;
|
|
77
|
+
|
|
78
|
+
let message = outer.message;
|
|
79
|
+
let type = outer.type;
|
|
80
|
+
let code = outer.code;
|
|
81
|
+
let status = outer.status;
|
|
82
|
+
|
|
83
|
+
if (typeof nested === 'string') {
|
|
84
|
+
message = nested;
|
|
85
|
+
} else if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
|
|
86
|
+
if (typeof nested.message === 'string') message = nested.message;
|
|
87
|
+
if (typeof nested.type === 'string') type = nested.type;
|
|
88
|
+
if (nested.code !== undefined && nested.code !== null) code = String(nested.code);
|
|
89
|
+
if (typeof nested.status === 'number' || typeof nested.status === 'string') {
|
|
90
|
+
const s = Number(nested.status);
|
|
91
|
+
if (Number.isFinite(s)) status = s;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Handle gRPC style status object: { code: 8, message: "Resource exhausted", details: [...] }
|
|
96
|
+
if (outer.details && Array.isArray(outer.details) && outer.details.length > 0) {
|
|
97
|
+
const firstDetail = outer.details[0];
|
|
98
|
+
if (firstDetail && typeof firstDetail === 'object') {
|
|
99
|
+
if (!message && typeof firstDetail.message === 'string') message = firstDetail.message;
|
|
100
|
+
if (!type && typeof firstDetail.reason === 'string') type = firstDetail.reason;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
...(message ? { message: String(message) } : {}),
|
|
106
|
+
...(type ? { type: String(type) } : {}),
|
|
107
|
+
...(code !== undefined && code !== null ? { code: String(code) } : {}),
|
|
108
|
+
...(typeof status === 'number' ? { status } : {}),
|
|
109
|
+
raw: obj,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Parse Retry-After value from Headers, headers object, or string into milliseconds.
|
|
115
|
+
*
|
|
116
|
+
* @param {Headers | Record<string, string | string[]> | string | null | undefined} headers
|
|
117
|
+
* @param {number} [now=Date.now()]
|
|
118
|
+
* @returns {number | undefined}
|
|
119
|
+
*/
|
|
120
|
+
export function retryAfterMs(headers, now = Date.now()) {
|
|
121
|
+
if (!headers) return undefined;
|
|
122
|
+
let raw = '';
|
|
123
|
+
if (typeof headers === 'string') {
|
|
124
|
+
const m = headers.match(/retry-after\s*[:=]\s*(.+)/i);
|
|
125
|
+
raw = m ? m[1].trim().split(/[\n\r;]/)[0].trim() : headers.trim();
|
|
126
|
+
} else if (typeof headers.get === 'function') {
|
|
127
|
+
raw = headers.get('retry-after') || headers.get('x-ratelimit-reset-after') || '';
|
|
128
|
+
} else if (typeof headers === 'object') {
|
|
129
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
130
|
+
if (k.toLowerCase() === 'retry-after' || k.toLowerCase() === 'x-ratelimit-reset-after') {
|
|
131
|
+
raw = Array.isArray(v) ? v[0] : String(v || '');
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
raw = String(raw).trim();
|
|
138
|
+
if (!raw) return undefined;
|
|
139
|
+
|
|
140
|
+
// 1. Seconds format: integer or float
|
|
141
|
+
const seconds = Number(raw);
|
|
142
|
+
if (Number.isFinite(seconds) && seconds >= 0 && seconds <= 86400 * 7) {
|
|
143
|
+
return Math.trunc(seconds * 1000);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// 2. HTTP-Date format: RFC 1123 / RFC 850
|
|
147
|
+
const parsedTs = Date.parse(raw);
|
|
148
|
+
if (Number.isFinite(parsedTs)) {
|
|
149
|
+
const diff = parsedTs - now;
|
|
150
|
+
return diff > 0 && diff < 86400 * 7 * 1000 ? Math.trunc(diff) : 0;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Determine if text/detail indicates quota or balance exhaustion.
|
|
158
|
+
*
|
|
159
|
+
* @param {string} detail
|
|
160
|
+
* @returns {boolean}
|
|
161
|
+
*/
|
|
162
|
+
export function isQuotaDetail(detail) {
|
|
163
|
+
if (!detail) return false;
|
|
164
|
+
if (isQuotaExceededError(detail)) return true;
|
|
165
|
+
const d = detail.toLowerCase();
|
|
166
|
+
return (
|
|
167
|
+
d.includes('insufficient_quota') ||
|
|
168
|
+
d.includes('insufficient balance') ||
|
|
169
|
+
d.includes('insufficient credits') ||
|
|
170
|
+
d.includes('out of credits') ||
|
|
171
|
+
d.includes('out of budget') ||
|
|
172
|
+
d.includes('quota exceeded') ||
|
|
173
|
+
d.includes('usage limit exceeded') ||
|
|
174
|
+
d.includes('resource_exhausted') ||
|
|
175
|
+
d.includes('credit balance') ||
|
|
176
|
+
d.includes('monthly limit') ||
|
|
177
|
+
d.includes('weekly limit') ||
|
|
178
|
+
d.includes('payment required')
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Canonical classifier translating status, body and headers into typed LlmError (#547).
|
|
184
|
+
*
|
|
185
|
+
* @param {number} status HTTP status code
|
|
186
|
+
* @param {unknown} body Response body (string, object, buffer)
|
|
187
|
+
* @param {Headers | Record<string, unknown> | null | undefined} headers Response headers
|
|
188
|
+
* @param {{ now?: number }} [options]
|
|
189
|
+
* @returns {LlmError}
|
|
190
|
+
*/
|
|
191
|
+
export function classifyResponseError(status, body, headers, options = {}) {
|
|
192
|
+
const now = options.now ?? Date.now();
|
|
193
|
+
const parsed = parseErrorBody(body);
|
|
194
|
+
const type = parsed.type ?? '';
|
|
195
|
+
const message = parsed.message ?? (status ? `HTTP ${status}` : 'Unknown upstream error');
|
|
196
|
+
const code = (parsed.code ?? '').toUpperCase();
|
|
197
|
+
const detail = `${type} ${code} ${message}`.trim();
|
|
198
|
+
const retryMs = retryAfterMs(headers, now);
|
|
199
|
+
|
|
200
|
+
const validStatus = Number.isInteger(status) && status >= 100 && status <= 599 ? status : undefined;
|
|
201
|
+
const errorOptions = {
|
|
202
|
+
...(validStatus !== undefined ? { status: validStatus } : {}),
|
|
203
|
+
...(retryMs !== undefined ? { providerRetryAfterMs: retryMs } : {}),
|
|
204
|
+
...(parsed.raw ? { cause: parsed.raw } : {}),
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// 1. Context window exceeded
|
|
208
|
+
if (code === CONTEXT_WINDOW_EXCEEDED_CODE || isContextWindowExceededError(detail)) {
|
|
209
|
+
return new LlmError(message, CONTEXT_WINDOW_EXCEEDED_CODE, errorOptions);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 2. Authentication & Credentials (401, 403, invalid_api_key)
|
|
213
|
+
if (
|
|
214
|
+
status === 401 ||
|
|
215
|
+
status === 403 ||
|
|
216
|
+
code === 'INVALID_API_KEY' ||
|
|
217
|
+
code === 'UNAUTHENTICATED' ||
|
|
218
|
+
code === 'PERMISSION_DENIED' ||
|
|
219
|
+
type === 'authentication_error' ||
|
|
220
|
+
type === 'permission_denied'
|
|
221
|
+
) {
|
|
222
|
+
const isMalformed = /malformed|invalid|bad|incorrect/i.test(detail);
|
|
223
|
+
return new LlmError(
|
|
224
|
+
message,
|
|
225
|
+
isMalformed ? INVALID_CREDENTIAL_CODE : 'AUTH',
|
|
226
|
+
errorOptions
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// 3. Quota & Balance exhaustion (429 or 402 with quota phrasing)
|
|
231
|
+
if (
|
|
232
|
+
status === 402 ||
|
|
233
|
+
code === 'QUOTA' ||
|
|
234
|
+
code === 'RESOURCE_EXHAUSTED' ||
|
|
235
|
+
code === 'INSUFFICIENT_QUOTA' ||
|
|
236
|
+
isQuotaDetail(detail)
|
|
237
|
+
) {
|
|
238
|
+
const isAccount = /account|billing|subscription|replenish/i.test(detail);
|
|
239
|
+
return new LlmError(
|
|
240
|
+
message,
|
|
241
|
+
isAccount ? ACCOUNT_QUOTA_EXCEEDED_CODE : QUOTA_EXCEEDED_CODE,
|
|
242
|
+
errorOptions
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// 4. Rate Limiting (429 without quota exhaustion, concurrency limits)
|
|
247
|
+
if (
|
|
248
|
+
status === 429 ||
|
|
249
|
+
code === 'RATE_LIMIT' ||
|
|
250
|
+
code === 'RATE_LIMIT_EXCEEDED' ||
|
|
251
|
+
type === 'rate_limit_error' ||
|
|
252
|
+
/rate[\s_-]?limit|too many requests|concurrency[\s_-]+limit|rpm|tpm/i.test(detail)
|
|
253
|
+
) {
|
|
254
|
+
return new LlmError(message, 'RATE_LIMIT', errorOptions);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// 5. Unknown Model (explicit model_not_found or invalid_model)
|
|
258
|
+
if (
|
|
259
|
+
code === 'MODEL_NOT_FOUND' ||
|
|
260
|
+
code === 'UNKNOWN_MODEL' ||
|
|
261
|
+
type === 'invalid_model' ||
|
|
262
|
+
/model[_\s]+not[_\s]+found|unknown[_\s]+model/i.test(detail)
|
|
263
|
+
) {
|
|
264
|
+
return new LlmError(message, 'UNKNOWN_MODEL', errorOptions);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// 6. Timeouts (408, 504, DEADLINE_EXCEEDED)
|
|
268
|
+
if (
|
|
269
|
+
status === 408 ||
|
|
270
|
+
status === 504 ||
|
|
271
|
+
code === 'TIMEOUT' ||
|
|
272
|
+
code === 'DEADLINE_EXCEEDED' ||
|
|
273
|
+
/timeout|timed[\s_-]?out/i.test(detail)
|
|
274
|
+
) {
|
|
275
|
+
return new LlmError(message, 'TIMEOUT', errorOptions);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// 7. Server Errors (500, 502, 503)
|
|
279
|
+
if (
|
|
280
|
+
(status >= 500 && status <= 503) ||
|
|
281
|
+
code === 'SERVER' ||
|
|
282
|
+
code === 'INTERNAL' ||
|
|
283
|
+
code === 'UNAVAILABLE' ||
|
|
284
|
+
type === 'server_error' ||
|
|
285
|
+
type === 'api_error'
|
|
286
|
+
) {
|
|
287
|
+
return new LlmError(message, 'SERVER', errorOptions);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// 8. Empty response
|
|
291
|
+
if (code === EMPTY_RESPONSE_CODE || (!message && status === 200)) {
|
|
292
|
+
return new LlmError(message || 'Upstream returned empty completion', EMPTY_RESPONSE_CODE, errorOptions);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// 9. Default client error (400, 422) or generic provider error
|
|
296
|
+
if (status >= 400 && status < 500) {
|
|
297
|
+
return new LlmError(message, code || String(status), errorOptions);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return new LlmError(message, code || 'PROVIDER_ERROR', errorOptions);
|
|
301
|
+
}
|
|
302
|
+
|
package/lib/index.js
CHANGED
|
@@ -57,6 +57,9 @@ export const name = '@goodandready/dsh-key-rotation';
|
|
|
57
57
|
export const inject = ['llm', 'webServer', 'settings', 'credentials'];
|
|
58
58
|
export { keyTail, KEY_TAIL_CHARS, KEY_TAIL_PLACEHOLDER, isLoopbackAddress, isTrustedBridgeRequest, isDesktopShellRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage, getRuntime };
|
|
59
59
|
export { notifySwitch, notifyExhaustion, notifyEarlyWarning };
|
|
60
|
+
export { classifyResponseError } from './errors.js';
|
|
61
|
+
export { SessionRouter, defaultSessionRouter, unstickFailedKey } from './session-router.js';
|
|
62
|
+
export { isContentChunk } from './rotate.js';
|
|
60
63
|
|
|
61
64
|
const NS = 'dsh-key-rotation';
|
|
62
65
|
|
package/lib/ops-config-io.js
CHANGED
|
@@ -349,15 +349,24 @@ export function registerConfigIoRoutes(ctx, deps) {
|
|
|
349
349
|
}
|
|
350
350
|
const rawCurrent = plainConfig(currentConfig);
|
|
351
351
|
|
|
352
|
-
// Concurrency control:
|
|
353
|
-
if (
|
|
354
|
-
if (body.expectedRevision
|
|
352
|
+
// Concurrency control: validate expectedRevision format and match (#553)
|
|
353
|
+
if (body.expectedRevision !== undefined && body.expectedRevision !== null) {
|
|
354
|
+
if (!Number.isSafeInteger(body.expectedRevision) || body.expectedRevision < 0) {
|
|
355
|
+
json(res, 400, {
|
|
356
|
+
error: {
|
|
357
|
+
code: 'bad-request',
|
|
358
|
+
message: 'expectedRevision must be a non-negative integer',
|
|
359
|
+
},
|
|
360
|
+
});
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
if (currentRevision !== null && body.expectedRevision !== currentRevision) {
|
|
355
364
|
json(res, 409, {
|
|
356
365
|
error: {
|
|
357
366
|
code: 'conflict',
|
|
358
367
|
message: `Configuration has been modified concurrently (expected revision ${body.expectedRevision}, current is ${currentRevision})`,
|
|
359
368
|
currentRevision,
|
|
360
|
-
}
|
|
369
|
+
},
|
|
361
370
|
});
|
|
362
371
|
return;
|
|
363
372
|
}
|
|
@@ -406,8 +415,26 @@ export function registerConfigIoRoutes(ctx, deps) {
|
|
|
406
415
|
}
|
|
407
416
|
|
|
408
417
|
try {
|
|
409
|
-
await settings.replace(NS, mergedConfig, currentRevision ?? undefined);
|
|
418
|
+
await settings.replace(NS, mergedConfig, body.expectedRevision ?? currentRevision ?? undefined);
|
|
410
419
|
} catch (err) {
|
|
420
|
+
const isConflict = err?.code === 'SETTINGS_CONFLICT' ||
|
|
421
|
+
err?.status === 409 ||
|
|
422
|
+
err?.code === 'CONFLICT' ||
|
|
423
|
+
/conflict|revision|stale/i.test(err?.message || '');
|
|
424
|
+
if (isConflict) {
|
|
425
|
+
let latestRevision = null;
|
|
426
|
+
try {
|
|
427
|
+
latestRevision = settings.describe({ redactSecrets: false })?.find((c) => c.ns === NS)?.revision ?? null;
|
|
428
|
+
} catch (_) {}
|
|
429
|
+
json(res, 409, {
|
|
430
|
+
error: {
|
|
431
|
+
code: 'conflict',
|
|
432
|
+
message: `Configuration has been modified concurrently: ${err.message}`,
|
|
433
|
+
currentRevision: latestRevision ?? currentRevision,
|
|
434
|
+
},
|
|
435
|
+
});
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
411
438
|
json(res, 500, { error: { code: 'settings-write-failed', message: err.message } });
|
|
412
439
|
return;
|
|
413
440
|
}
|
package/lib/pool-selection.js
CHANGED
|
@@ -1,30 +1,91 @@
|
|
|
1
|
-
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512).
|
|
1
|
+
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548).
|
|
2
2
|
// Extracted from lib/pool.js to enforce < 600 line modularity standard.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
import {
|
|
5
|
+
SessionRouter,
|
|
6
|
+
defaultSessionRouter,
|
|
7
|
+
MAX_SESSIONS,
|
|
8
|
+
SESSION_TTL_MS,
|
|
9
|
+
} from './session-router.js';
|
|
5
10
|
|
|
6
|
-
export
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
export {
|
|
12
|
+
SessionRouter,
|
|
13
|
+
defaultSessionRouter,
|
|
14
|
+
MAX_SESSIONS,
|
|
15
|
+
SESSION_TTL_MS,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export const MAX_SESSION_STICKY_ENTRIES = MAX_SESSIONS;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Get or initialize the SessionRouter on pool.state or fallback to default.
|
|
22
|
+
* @param {object} [pool]
|
|
23
|
+
* @returns {SessionRouter}
|
|
24
|
+
*/
|
|
25
|
+
export function getOrCreateSessionRouter(pool) {
|
|
26
|
+
if (!pool?.state) return defaultSessionRouter;
|
|
27
|
+
if (!pool.state.sessionRouter) {
|
|
28
|
+
pool.state.sessionRouter = new SessionRouter({
|
|
29
|
+
maxSessions: MAX_SESSIONS,
|
|
30
|
+
sessionTtlMs: SESSION_TTL_MS,
|
|
31
|
+
});
|
|
10
32
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
33
|
+
return pool.state.sessionRouter;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function setSessionSticky(pool, sessionId, candidate) {
|
|
37
|
+
if (!sessionId || !candidate) return;
|
|
38
|
+
const router = getOrCreateSessionRouter(pool);
|
|
39
|
+
router.setSession(sessionId, candidate);
|
|
40
|
+
if (pool?.state) {
|
|
41
|
+
if (!(pool.state.sessionSticky instanceof Map)) {
|
|
42
|
+
pool.state.sessionSticky = new Map();
|
|
43
|
+
}
|
|
44
|
+
if (pool.state.sessionSticky.has(sessionId)) {
|
|
45
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
46
|
+
} else if (pool.state.sessionSticky.size >= MAX_SESSION_STICKY_ENTRIES) {
|
|
47
|
+
const oldest = pool.state.sessionSticky.keys().next().value;
|
|
48
|
+
if (oldest !== undefined) pool.state.sessionSticky.delete(oldest);
|
|
49
|
+
}
|
|
50
|
+
pool.state.sessionSticky.set(sessionId, candidate);
|
|
16
51
|
}
|
|
17
|
-
pool.state.sessionSticky.set(sessionId, candidate);
|
|
18
52
|
}
|
|
19
53
|
|
|
20
54
|
export function getSessionSticky(pool, sessionId) {
|
|
21
|
-
if (!sessionId
|
|
22
|
-
|
|
55
|
+
if (!sessionId) return null;
|
|
56
|
+
const router = getOrCreateSessionRouter(pool);
|
|
57
|
+
const fromRouter = router.getSession(sessionId);
|
|
58
|
+
if (fromRouter) return fromRouter;
|
|
59
|
+
return pool?.state?.sessionSticky?.get?.(sessionId) ?? null;
|
|
23
60
|
}
|
|
24
61
|
|
|
25
62
|
export function clearSessionSticky(pool, sessionId) {
|
|
26
|
-
if (!sessionId
|
|
27
|
-
pool
|
|
63
|
+
if (!sessionId) return;
|
|
64
|
+
const router = getOrCreateSessionRouter(pool);
|
|
65
|
+
router.unstick(sessionId);
|
|
66
|
+
if (pool?.state?.sessionSticky) {
|
|
67
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Proactively unstick all sessions pinned to a cooling or failed key (#548).
|
|
73
|
+
* @param {object} pool
|
|
74
|
+
* @param {string} keyRef
|
|
75
|
+
* @returns {number}
|
|
76
|
+
*/
|
|
77
|
+
export function unstickFailedKey(pool, keyRef) {
|
|
78
|
+
if (!keyRef) return 0;
|
|
79
|
+
const router = getOrCreateSessionRouter(pool);
|
|
80
|
+
const count = router.unstickKey(keyRef);
|
|
81
|
+
if (pool?.state?.sessionSticky) {
|
|
82
|
+
for (const [sId, ref] of pool.state.sessionSticky.entries()) {
|
|
83
|
+
if (ref === keyRef) {
|
|
84
|
+
pool.state.sessionSticky.delete(sId);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return count;
|
|
28
89
|
}
|
|
29
90
|
|
|
30
91
|
/** Pick a key pool for a (provider, model) pair. */
|
|
@@ -48,10 +109,23 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
|
48
109
|
if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
|
|
49
110
|
const list = refs.slice();
|
|
50
111
|
if (strategy === 'sticky') {
|
|
51
|
-
const
|
|
112
|
+
const router = getOrCreateSessionRouter(deps.pool);
|
|
113
|
+
let chosenKey = null;
|
|
114
|
+
|
|
115
|
+
if (deps.sessionId) {
|
|
116
|
+
const resolved = router.resolveSessionKey(deps.sessionId, list);
|
|
117
|
+
if (resolved && resolved.keyRef) {
|
|
118
|
+
chosenKey = resolved.keyRef;
|
|
119
|
+
if (deps.pool) setSessionSticky(deps.pool, deps.sessionId, chosenKey);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const last = chosenKey ||
|
|
124
|
+
deps.sessionSticky ||
|
|
52
125
|
(deps.sessionId && (deps.pool?.state?.sessionSticky?.get?.(deps.sessionId) || deps.sessionStickyMap?.get?.(deps.sessionId))) ||
|
|
53
126
|
deps.lastUsed ||
|
|
54
127
|
(deps.pool && deps.pool.state && deps.pool.state.lastUsed);
|
|
128
|
+
|
|
55
129
|
if (last && list.includes(last)) {
|
|
56
130
|
const idx = list.indexOf(last);
|
|
57
131
|
if (idx > 0) {
|
|
@@ -147,3 +221,4 @@ export function budgetVerdict(spend, budget) {
|
|
|
147
221
|
const ratio = spend / budget;
|
|
148
222
|
return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
|
|
149
223
|
}
|
|
224
|
+
|