@goodandready/dsh-key-rotation 0.8.76 → 0.8.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/lib/error-taxonomy.js +59 -10
- package/lib/errors.js +302 -0
- package/lib/index.js +3 -0
- package/lib/pool-selection.js +92 -17
- package/lib/pool.js +19 -0
- package/lib/rotate.js +61 -11
- package/lib/session-router.js +265 -0
- package/package.json +10 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.77 (2026-10-30)
|
|
4
|
+
|
|
5
|
+
### Session Routing, Typed Error Taxonomy & Pre-First-Token Failover (Block 5 Release)
|
|
6
|
+
- **Pre-First-Token Failover & Request Attempt Loop (Issue #527)**: Introduced transaction-safe pre-first-token retry loop in `lib/rotate.js` with canonical `isContentChunk` detector (distinguishes model-visible tokens from stream envelopes and metadata). Stream metadata (such as `block-start`, `response-metadata`) emitted prior to the first content chunk is securely buffered; upon upstream failures (HTTP 429, 502, 503, connection drops) before any content token yields, the buffered metadata is cleanly discarded and the request seamlessly transitions to the next available healthy key without chat UI disruption or duplicate blocks.
|
|
7
|
+
- **Typed Error Taxonomy & Canonical classifyResponseError (Issue #547)**: Implemented `lib/errors.js` with structured envelope parsing for OpenAI, Anthropic, Google Gemini, Groq, and Ollama. Added robust RFC 1123 HTTP-Date and numeric seconds parser for `Retry-After` headers converting to exact millisecond cooldowns; mapped provider failures into typed `@deepseek-ai/dsh-llm` `LlmError` instances (`QUOTA`, `ACCOUNT_QUOTA`, `RATE_LIMIT`, `CONTEXT_WINDOW_EXCEEDED`, `INVALID_CREDENTIAL`, `SERVER`, `TIMEOUT`); updated `isSwitchableError` and `classifyFailure` to replace brittle text regex matching with structured JSON payload classification.
|
|
8
|
+
- **Bounded SessionRouter with LRU & Proactive Unsticking (Issue #548)**: Created standalone domain service `SessionRouter` (`lib/session-router.js`) with capacity limit (`MAX_SESSIONS = 1000`) and automatic 24-hour TTL expiration (`SESSION_TTL_MS`). Integrated Least-Used Initial Assignment to distribute new incoming sessions across least-loaded keys; implemented `unstickKey(failedKey)` in `lib/pool-selection.js` and key penalization to proactively unbind all sessions pinned to failed or cooling keys, guaranteeing seamless transparent failover on consecutive prompts.
|
|
9
|
+
|
|
3
10
|
## 0.8.76 (2026-10-30)
|
|
4
11
|
|
|
5
12
|
### Security Hardening, DNS Pinning & Mutation Integrity (Block 4 Release)
|
package/lib/error-taxonomy.js
CHANGED
|
@@ -1,13 +1,35 @@
|
|
|
1
|
-
|
|
2
|
-
// lib/error-taxonomy.js — complete failure classification (#267).
|
|
1
|
+
// lib/error-taxonomy.js — complete failure classification (#267, #547).
|
|
3
2
|
// action: 'switch' | 'surface' | 'cooldown'
|
|
4
3
|
|
|
4
|
+
import { SOFT_FAILURE_CODES } from './pool.js';
|
|
5
|
+
import {
|
|
6
|
+
classifyResponseError,
|
|
7
|
+
parseErrorBody,
|
|
8
|
+
retryAfterMs,
|
|
9
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
10
|
+
QUOTA_EXCEEDED_CODE,
|
|
11
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
12
|
+
EMPTY_RESPONSE_CODE,
|
|
13
|
+
INVALID_CREDENTIAL_CODE,
|
|
14
|
+
} from './errors.js';
|
|
15
|
+
|
|
16
|
+
export {
|
|
17
|
+
classifyResponseError,
|
|
18
|
+
parseErrorBody,
|
|
19
|
+
retryAfterMs,
|
|
20
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
21
|
+
QUOTA_EXCEEDED_CODE,
|
|
22
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
23
|
+
EMPTY_RESPONSE_CODE,
|
|
24
|
+
INVALID_CREDENTIAL_CODE,
|
|
25
|
+
};
|
|
26
|
+
|
|
5
27
|
const SWITCH_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
|
|
6
28
|
const SURFACE_STATUS = new Set([400, 404, 422]);
|
|
7
29
|
const AUTH_STATUS = new Set([401, 403]);
|
|
8
30
|
|
|
9
31
|
const SWITCH_CODES = new Set([
|
|
10
|
-
'QUOTA', 'RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT', 'EMPTY_RESPONSE',
|
|
32
|
+
'QUOTA', 'ACCOUNT_QUOTA', 'RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT', 'EMPTY_RESPONSE',
|
|
11
33
|
'UNKNOWN_MODEL', 'AUTH', 'RESOURCE_EXHAUSTED', 'UNAVAILABLE', 'INTERNAL',
|
|
12
34
|
'DEADLINE_EXCEEDED', 'UNAUTHENTICATED', 'PERMISSION_DENIED', 'ABORTED',
|
|
13
35
|
]);
|
|
@@ -19,7 +41,7 @@ const SOCKET_SWITCH = new Set([
|
|
|
19
41
|
/**
|
|
20
42
|
* Classify a failure for rotation.
|
|
21
43
|
* @param {any} failureOrPayload
|
|
22
|
-
* @returns {{ action: 'switch'|'surface'|'cooldown', code: string, soft: boolean, reason: string }}
|
|
44
|
+
* @returns {{ action: 'switch'|'surface'|'cooldown', code: string, soft: boolean, reason: string, retryAfterMs?: number }}
|
|
23
45
|
*/
|
|
24
46
|
export function classifyFailure(failureOrPayload) {
|
|
25
47
|
if (!failureOrPayload) {
|
|
@@ -29,18 +51,46 @@ export function classifyFailure(failureOrPayload) {
|
|
|
29
51
|
const status = Number(failure.status ?? failure.statusCode ?? failure.httpStatus ?? 0);
|
|
30
52
|
const code = String(failure.code ?? failure.reason ?? failure.name ?? '').toUpperCase();
|
|
31
53
|
const message = String(failure.message ?? failureOrPayload.message ?? '');
|
|
54
|
+
const headers = failure.headers ?? failureOrPayload.headers ?? failure.response?.headers;
|
|
55
|
+
const retryMs = retryAfterMs(headers);
|
|
56
|
+
|
|
57
|
+
// Check structured body/error through classifyResponseError if available
|
|
58
|
+
if (status || failure.body || (code && code !== 'ERROR')) {
|
|
59
|
+
const typed = classifyResponseError(status, failure.body ?? message, headers);
|
|
60
|
+
if (typed.code === CONTEXT_WINDOW_EXCEEDED_CODE) {
|
|
61
|
+
return { action: 'surface', code: CONTEXT_WINDOW_EXCEEDED_CODE, soft: false, reason: 'context_window' };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
32
64
|
|
|
33
|
-
if (AUTH_STATUS.has(status) || code === 'AUTH' || code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED') {
|
|
34
|
-
return {
|
|
65
|
+
if (AUTH_STATUS.has(status) || code === 'AUTH' || code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED' || code === INVALID_CREDENTIAL_CODE) {
|
|
66
|
+
return {
|
|
67
|
+
action: 'switch',
|
|
68
|
+
code: status ? String(status) : (code || 'AUTH'),
|
|
69
|
+
soft: false,
|
|
70
|
+
reason: 'auth',
|
|
71
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
72
|
+
};
|
|
35
73
|
}
|
|
36
|
-
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA') {
|
|
37
|
-
return {
|
|
74
|
+
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA' || code === 'ACCOUNT_QUOTA' || code === QUOTA_EXCEEDED_CODE) {
|
|
75
|
+
return {
|
|
76
|
+
action: 'switch',
|
|
77
|
+
code: status ? '429' : (code || 'RATE_LIMIT'),
|
|
78
|
+
soft: false,
|
|
79
|
+
reason: 'quota',
|
|
80
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
81
|
+
};
|
|
38
82
|
}
|
|
39
83
|
const numCode = Number(code);
|
|
40
84
|
if (SWITCH_STATUS.has(status) || (numCode > 0 && SWITCH_STATUS.has(numCode)) || SOFT_FAILURE_CODES.has(code)) {
|
|
41
85
|
const effCode = status ? String(status) : (numCode > 0 ? String(numCode) : code);
|
|
42
86
|
const soft = SOFT_FAILURE_CODES.has(effCode);
|
|
43
|
-
return {
|
|
87
|
+
return {
|
|
88
|
+
action: 'switch',
|
|
89
|
+
code: effCode,
|
|
90
|
+
soft,
|
|
91
|
+
reason: 'http',
|
|
92
|
+
...(retryMs !== undefined ? { retryAfterMs: retryMs } : {}),
|
|
93
|
+
};
|
|
44
94
|
}
|
|
45
95
|
if (SURFACE_STATUS.has(status)) {
|
|
46
96
|
return { action: 'surface', code: String(status), soft: false, reason: 'client' };
|
|
@@ -63,4 +113,3 @@ export function classifyFailure(failureOrPayload) {
|
|
|
63
113
|
return { action: 'surface', code: code || 'UNKNOWN', soft: false, reason: 'unclassified' };
|
|
64
114
|
}
|
|
65
115
|
|
|
66
|
-
/** True when classification says switch to next key. */
|
package/lib/errors.js
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
// lib/errors.js — Canonical error classification and taxonomy for DeepSeek Harness (#547).
|
|
2
|
+
// Unifies error shapes across OpenAI, Anthropic, Gemini, Ollama, Groq and OpenCode into standard LlmError.
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
HarnessError as DshHarnessError,
|
|
6
|
+
LlmError as DshLlmError,
|
|
7
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
8
|
+
QUOTA_EXCEEDED_CODE,
|
|
9
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
10
|
+
EMPTY_RESPONSE_CODE,
|
|
11
|
+
INVALID_CREDENTIAL_CODE,
|
|
12
|
+
isContextWindowExceededError,
|
|
13
|
+
isQuotaExceededError,
|
|
14
|
+
} from '@deepseek-ai/dsh-llm';
|
|
15
|
+
|
|
16
|
+
export {
|
|
17
|
+
CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
18
|
+
QUOTA_EXCEEDED_CODE,
|
|
19
|
+
ACCOUNT_QUOTA_EXCEEDED_CODE,
|
|
20
|
+
EMPTY_RESPONSE_CODE,
|
|
21
|
+
INVALID_CREDENTIAL_CODE,
|
|
22
|
+
isContextWindowExceededError,
|
|
23
|
+
isQuotaExceededError,
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
// Fallback HarnessError and LlmError if ever decoupled from runtime
|
|
27
|
+
export class HarnessError extends (DshHarnessError || Error) {
|
|
28
|
+
constructor(message, code, options) {
|
|
29
|
+
super(message, code, options);
|
|
30
|
+
this.code = code;
|
|
31
|
+
this.name = this.constructor.name;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class LlmError extends (DshLlmError || HarnessError) {
|
|
36
|
+
constructor(message, code, options) {
|
|
37
|
+
super(message, code, options);
|
|
38
|
+
this.status = options?.status;
|
|
39
|
+
this.providerRetryAfterMs = options?.providerRetryAfterMs;
|
|
40
|
+
this.code = code;
|
|
41
|
+
this.name = this.constructor.name;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Robust parser for structured error bodies (JSON, object, text).
|
|
47
|
+
* Handles OpenAI, Anthropic, Gemini, Groq, Ollama envelopes.
|
|
48
|
+
*
|
|
49
|
+
* @param {unknown} body
|
|
50
|
+
* @returns {{ message?: string, type?: string, code?: string, status?: number, raw?: unknown }}
|
|
51
|
+
*/
|
|
52
|
+
export function parseErrorBody(body) {
|
|
53
|
+
if (!body) return {};
|
|
54
|
+
if (typeof body === 'object') {
|
|
55
|
+
return extractErrorFields(body);
|
|
56
|
+
}
|
|
57
|
+
if (typeof body === 'string') {
|
|
58
|
+
const trimmed = body.trim();
|
|
59
|
+
if (!trimmed) return {};
|
|
60
|
+
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
|
61
|
+
try {
|
|
62
|
+
const parsed = JSON.parse(trimmed);
|
|
63
|
+
return extractErrorFields(parsed);
|
|
64
|
+
} catch {
|
|
65
|
+
// Not valid JSON, return as plain message
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { message: trimmed.slice(0, 1000) };
|
|
69
|
+
}
|
|
70
|
+
return { message: String(body).slice(0, 1000) };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function extractErrorFields(obj) {
|
|
74
|
+
if (!obj || typeof obj !== 'object') return {};
|
|
75
|
+
const outer = obj;
|
|
76
|
+
const nested = outer.error;
|
|
77
|
+
|
|
78
|
+
let message = outer.message;
|
|
79
|
+
let type = outer.type;
|
|
80
|
+
let code = outer.code;
|
|
81
|
+
let status = outer.status;
|
|
82
|
+
|
|
83
|
+
if (typeof nested === 'string') {
|
|
84
|
+
message = nested;
|
|
85
|
+
} else if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
|
|
86
|
+
if (typeof nested.message === 'string') message = nested.message;
|
|
87
|
+
if (typeof nested.type === 'string') type = nested.type;
|
|
88
|
+
if (nested.code !== undefined && nested.code !== null) code = String(nested.code);
|
|
89
|
+
if (typeof nested.status === 'number' || typeof nested.status === 'string') {
|
|
90
|
+
const s = Number(nested.status);
|
|
91
|
+
if (Number.isFinite(s)) status = s;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Handle gRPC style status object: { code: 8, message: "Resource exhausted", details: [...] }
|
|
96
|
+
if (outer.details && Array.isArray(outer.details) && outer.details.length > 0) {
|
|
97
|
+
const firstDetail = outer.details[0];
|
|
98
|
+
if (firstDetail && typeof firstDetail === 'object') {
|
|
99
|
+
if (!message && typeof firstDetail.message === 'string') message = firstDetail.message;
|
|
100
|
+
if (!type && typeof firstDetail.reason === 'string') type = firstDetail.reason;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
...(message ? { message: String(message) } : {}),
|
|
106
|
+
...(type ? { type: String(type) } : {}),
|
|
107
|
+
...(code !== undefined && code !== null ? { code: String(code) } : {}),
|
|
108
|
+
...(typeof status === 'number' ? { status } : {}),
|
|
109
|
+
raw: obj,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Parse Retry-After value from Headers, headers object, or string into milliseconds.
|
|
115
|
+
*
|
|
116
|
+
* @param {Headers | Record<string, string | string[]> | string | null | undefined} headers
|
|
117
|
+
* @param {number} [now=Date.now()]
|
|
118
|
+
* @returns {number | undefined}
|
|
119
|
+
*/
|
|
120
|
+
export function retryAfterMs(headers, now = Date.now()) {
|
|
121
|
+
if (!headers) return undefined;
|
|
122
|
+
let raw = '';
|
|
123
|
+
if (typeof headers === 'string') {
|
|
124
|
+
const m = headers.match(/retry-after\s*[:=]\s*(.+)/i);
|
|
125
|
+
raw = m ? m[1].trim().split(/[\n\r;]/)[0].trim() : headers.trim();
|
|
126
|
+
} else if (typeof headers.get === 'function') {
|
|
127
|
+
raw = headers.get('retry-after') || headers.get('x-ratelimit-reset-after') || '';
|
|
128
|
+
} else if (typeof headers === 'object') {
|
|
129
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
130
|
+
if (k.toLowerCase() === 'retry-after' || k.toLowerCase() === 'x-ratelimit-reset-after') {
|
|
131
|
+
raw = Array.isArray(v) ? v[0] : String(v || '');
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
raw = String(raw).trim();
|
|
138
|
+
if (!raw) return undefined;
|
|
139
|
+
|
|
140
|
+
// 1. Seconds format: integer or float
|
|
141
|
+
const seconds = Number(raw);
|
|
142
|
+
if (Number.isFinite(seconds) && seconds >= 0 && seconds <= 86400 * 7) {
|
|
143
|
+
return Math.trunc(seconds * 1000);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// 2. HTTP-Date format: RFC 1123 / RFC 850
|
|
147
|
+
const parsedTs = Date.parse(raw);
|
|
148
|
+
if (Number.isFinite(parsedTs)) {
|
|
149
|
+
const diff = parsedTs - now;
|
|
150
|
+
return diff > 0 && diff < 86400 * 7 * 1000 ? Math.trunc(diff) : 0;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Determine if text/detail indicates quota or balance exhaustion.
|
|
158
|
+
*
|
|
159
|
+
* @param {string} detail
|
|
160
|
+
* @returns {boolean}
|
|
161
|
+
*/
|
|
162
|
+
export function isQuotaDetail(detail) {
|
|
163
|
+
if (!detail) return false;
|
|
164
|
+
if (isQuotaExceededError(detail)) return true;
|
|
165
|
+
const d = detail.toLowerCase();
|
|
166
|
+
return (
|
|
167
|
+
d.includes('insufficient_quota') ||
|
|
168
|
+
d.includes('insufficient balance') ||
|
|
169
|
+
d.includes('insufficient credits') ||
|
|
170
|
+
d.includes('out of credits') ||
|
|
171
|
+
d.includes('out of budget') ||
|
|
172
|
+
d.includes('quota exceeded') ||
|
|
173
|
+
d.includes('usage limit exceeded') ||
|
|
174
|
+
d.includes('resource_exhausted') ||
|
|
175
|
+
d.includes('credit balance') ||
|
|
176
|
+
d.includes('monthly limit') ||
|
|
177
|
+
d.includes('weekly limit') ||
|
|
178
|
+
d.includes('payment required')
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Canonical classifier translating status, body and headers into typed LlmError (#547).
|
|
184
|
+
*
|
|
185
|
+
* @param {number} status HTTP status code
|
|
186
|
+
* @param {unknown} body Response body (string, object, buffer)
|
|
187
|
+
* @param {Headers | Record<string, unknown> | null | undefined} headers Response headers
|
|
188
|
+
* @param {{ now?: number }} [options]
|
|
189
|
+
* @returns {LlmError}
|
|
190
|
+
*/
|
|
191
|
+
export function classifyResponseError(status, body, headers, options = {}) {
|
|
192
|
+
const now = options.now ?? Date.now();
|
|
193
|
+
const parsed = parseErrorBody(body);
|
|
194
|
+
const type = parsed.type ?? '';
|
|
195
|
+
const message = parsed.message ?? (status ? `HTTP ${status}` : 'Unknown upstream error');
|
|
196
|
+
const code = (parsed.code ?? '').toUpperCase();
|
|
197
|
+
const detail = `${type} ${code} ${message}`.trim();
|
|
198
|
+
const retryMs = retryAfterMs(headers, now);
|
|
199
|
+
|
|
200
|
+
const validStatus = Number.isInteger(status) && status >= 100 && status <= 599 ? status : undefined;
|
|
201
|
+
const errorOptions = {
|
|
202
|
+
...(validStatus !== undefined ? { status: validStatus } : {}),
|
|
203
|
+
...(retryMs !== undefined ? { providerRetryAfterMs: retryMs } : {}),
|
|
204
|
+
...(parsed.raw ? { cause: parsed.raw } : {}),
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// 1. Context window exceeded
|
|
208
|
+
if (code === CONTEXT_WINDOW_EXCEEDED_CODE || isContextWindowExceededError(detail)) {
|
|
209
|
+
return new LlmError(message, CONTEXT_WINDOW_EXCEEDED_CODE, errorOptions);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 2. Authentication & Credentials (401, 403, invalid_api_key)
|
|
213
|
+
if (
|
|
214
|
+
status === 401 ||
|
|
215
|
+
status === 403 ||
|
|
216
|
+
code === 'INVALID_API_KEY' ||
|
|
217
|
+
code === 'UNAUTHENTICATED' ||
|
|
218
|
+
code === 'PERMISSION_DENIED' ||
|
|
219
|
+
type === 'authentication_error' ||
|
|
220
|
+
type === 'permission_denied'
|
|
221
|
+
) {
|
|
222
|
+
const isMalformed = /malformed|invalid|bad|incorrect/i.test(detail);
|
|
223
|
+
return new LlmError(
|
|
224
|
+
message,
|
|
225
|
+
isMalformed ? INVALID_CREDENTIAL_CODE : 'AUTH',
|
|
226
|
+
errorOptions
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// 3. Quota & Balance exhaustion (429 or 402 with quota phrasing)
|
|
231
|
+
if (
|
|
232
|
+
status === 402 ||
|
|
233
|
+
code === 'QUOTA' ||
|
|
234
|
+
code === 'RESOURCE_EXHAUSTED' ||
|
|
235
|
+
code === 'INSUFFICIENT_QUOTA' ||
|
|
236
|
+
isQuotaDetail(detail)
|
|
237
|
+
) {
|
|
238
|
+
const isAccount = /account|billing|subscription|replenish/i.test(detail);
|
|
239
|
+
return new LlmError(
|
|
240
|
+
message,
|
|
241
|
+
isAccount ? ACCOUNT_QUOTA_EXCEEDED_CODE : QUOTA_EXCEEDED_CODE,
|
|
242
|
+
errorOptions
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// 4. Rate Limiting (429 without quota exhaustion, concurrency limits)
|
|
247
|
+
if (
|
|
248
|
+
status === 429 ||
|
|
249
|
+
code === 'RATE_LIMIT' ||
|
|
250
|
+
code === 'RATE_LIMIT_EXCEEDED' ||
|
|
251
|
+
type === 'rate_limit_error' ||
|
|
252
|
+
/rate[\s_-]?limit|too many requests|concurrency[\s_-]+limit|rpm|tpm/i.test(detail)
|
|
253
|
+
) {
|
|
254
|
+
return new LlmError(message, 'RATE_LIMIT', errorOptions);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// 5. Unknown Model (explicit model_not_found or invalid_model)
|
|
258
|
+
if (
|
|
259
|
+
code === 'MODEL_NOT_FOUND' ||
|
|
260
|
+
code === 'UNKNOWN_MODEL' ||
|
|
261
|
+
type === 'invalid_model' ||
|
|
262
|
+
/model[_\s]+not[_\s]+found|unknown[_\s]+model/i.test(detail)
|
|
263
|
+
) {
|
|
264
|
+
return new LlmError(message, 'UNKNOWN_MODEL', errorOptions);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// 6. Timeouts (408, 504, DEADLINE_EXCEEDED)
|
|
268
|
+
if (
|
|
269
|
+
status === 408 ||
|
|
270
|
+
status === 504 ||
|
|
271
|
+
code === 'TIMEOUT' ||
|
|
272
|
+
code === 'DEADLINE_EXCEEDED' ||
|
|
273
|
+
/timeout|timed[\s_-]?out/i.test(detail)
|
|
274
|
+
) {
|
|
275
|
+
return new LlmError(message, 'TIMEOUT', errorOptions);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// 7. Server Errors (500, 502, 503)
|
|
279
|
+
if (
|
|
280
|
+
(status >= 500 && status <= 503) ||
|
|
281
|
+
code === 'SERVER' ||
|
|
282
|
+
code === 'INTERNAL' ||
|
|
283
|
+
code === 'UNAVAILABLE' ||
|
|
284
|
+
type === 'server_error' ||
|
|
285
|
+
type === 'api_error'
|
|
286
|
+
) {
|
|
287
|
+
return new LlmError(message, 'SERVER', errorOptions);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// 8. Empty response
|
|
291
|
+
if (code === EMPTY_RESPONSE_CODE || (!message && status === 200)) {
|
|
292
|
+
return new LlmError(message || 'Upstream returned empty completion', EMPTY_RESPONSE_CODE, errorOptions);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// 9. Default client error (400, 422) or generic provider error
|
|
296
|
+
if (status >= 400 && status < 500) {
|
|
297
|
+
return new LlmError(message, code || String(status), errorOptions);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return new LlmError(message, code || 'PROVIDER_ERROR', errorOptions);
|
|
301
|
+
}
|
|
302
|
+
|
package/lib/index.js
CHANGED
|
@@ -57,6 +57,9 @@ export const name = '@goodandready/dsh-key-rotation';
|
|
|
57
57
|
export const inject = ['llm', 'webServer', 'settings', 'credentials'];
|
|
58
58
|
export { keyTail, KEY_TAIL_CHARS, KEY_TAIL_PLACEHOLDER, isLoopbackAddress, isTrustedBridgeRequest, isDesktopShellRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage, getRuntime };
|
|
59
59
|
export { notifySwitch, notifyExhaustion, notifyEarlyWarning };
|
|
60
|
+
export { classifyResponseError } from './errors.js';
|
|
61
|
+
export { SessionRouter, defaultSessionRouter, unstickFailedKey } from './session-router.js';
|
|
62
|
+
export { isContentChunk } from './rotate.js';
|
|
60
63
|
|
|
61
64
|
const NS = 'dsh-key-rotation';
|
|
62
65
|
|
package/lib/pool-selection.js
CHANGED
|
@@ -1,30 +1,91 @@
|
|
|
1
|
-
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512).
|
|
1
|
+
// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548).
|
|
2
2
|
// Extracted from lib/pool.js to enforce < 600 line modularity standard.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
import {
|
|
5
|
+
SessionRouter,
|
|
6
|
+
defaultSessionRouter,
|
|
7
|
+
MAX_SESSIONS,
|
|
8
|
+
SESSION_TTL_MS,
|
|
9
|
+
} from './session-router.js';
|
|
5
10
|
|
|
6
|
-
export
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
export {
|
|
12
|
+
SessionRouter,
|
|
13
|
+
defaultSessionRouter,
|
|
14
|
+
MAX_SESSIONS,
|
|
15
|
+
SESSION_TTL_MS,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export const MAX_SESSION_STICKY_ENTRIES = MAX_SESSIONS;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Get or initialize the SessionRouter on pool.state or fallback to default.
|
|
22
|
+
* @param {object} [pool]
|
|
23
|
+
* @returns {SessionRouter}
|
|
24
|
+
*/
|
|
25
|
+
export function getOrCreateSessionRouter(pool) {
|
|
26
|
+
if (!pool?.state) return defaultSessionRouter;
|
|
27
|
+
if (!pool.state.sessionRouter) {
|
|
28
|
+
pool.state.sessionRouter = new SessionRouter({
|
|
29
|
+
maxSessions: MAX_SESSIONS,
|
|
30
|
+
sessionTtlMs: SESSION_TTL_MS,
|
|
31
|
+
});
|
|
10
32
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
33
|
+
return pool.state.sessionRouter;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function setSessionSticky(pool, sessionId, candidate) {
|
|
37
|
+
if (!sessionId || !candidate) return;
|
|
38
|
+
const router = getOrCreateSessionRouter(pool);
|
|
39
|
+
router.setSession(sessionId, candidate);
|
|
40
|
+
if (pool?.state) {
|
|
41
|
+
if (!(pool.state.sessionSticky instanceof Map)) {
|
|
42
|
+
pool.state.sessionSticky = new Map();
|
|
43
|
+
}
|
|
44
|
+
if (pool.state.sessionSticky.has(sessionId)) {
|
|
45
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
46
|
+
} else if (pool.state.sessionSticky.size >= MAX_SESSION_STICKY_ENTRIES) {
|
|
47
|
+
const oldest = pool.state.sessionSticky.keys().next().value;
|
|
48
|
+
if (oldest !== undefined) pool.state.sessionSticky.delete(oldest);
|
|
49
|
+
}
|
|
50
|
+
pool.state.sessionSticky.set(sessionId, candidate);
|
|
16
51
|
}
|
|
17
|
-
pool.state.sessionSticky.set(sessionId, candidate);
|
|
18
52
|
}
|
|
19
53
|
|
|
20
54
|
export function getSessionSticky(pool, sessionId) {
|
|
21
|
-
if (!sessionId
|
|
22
|
-
|
|
55
|
+
if (!sessionId) return null;
|
|
56
|
+
const router = getOrCreateSessionRouter(pool);
|
|
57
|
+
const fromRouter = router.getSession(sessionId);
|
|
58
|
+
if (fromRouter) return fromRouter;
|
|
59
|
+
return pool?.state?.sessionSticky?.get?.(sessionId) ?? null;
|
|
23
60
|
}
|
|
24
61
|
|
|
25
62
|
export function clearSessionSticky(pool, sessionId) {
|
|
26
|
-
if (!sessionId
|
|
27
|
-
pool
|
|
63
|
+
if (!sessionId) return;
|
|
64
|
+
const router = getOrCreateSessionRouter(pool);
|
|
65
|
+
router.unstick(sessionId);
|
|
66
|
+
if (pool?.state?.sessionSticky) {
|
|
67
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Proactively unstick all sessions pinned to a cooling or failed key (#548).
|
|
73
|
+
* @param {object} pool
|
|
74
|
+
* @param {string} keyRef
|
|
75
|
+
* @returns {number}
|
|
76
|
+
*/
|
|
77
|
+
export function unstickFailedKey(pool, keyRef) {
|
|
78
|
+
if (!keyRef) return 0;
|
|
79
|
+
const router = getOrCreateSessionRouter(pool);
|
|
80
|
+
const count = router.unstickKey(keyRef);
|
|
81
|
+
if (pool?.state?.sessionSticky) {
|
|
82
|
+
for (const [sId, ref] of pool.state.sessionSticky.entries()) {
|
|
83
|
+
if (ref === keyRef) {
|
|
84
|
+
pool.state.sessionSticky.delete(sId);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return count;
|
|
28
89
|
}
|
|
29
90
|
|
|
30
91
|
/** Pick a key pool for a (provider, model) pair. */
|
|
@@ -48,10 +109,23 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
|
48
109
|
if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
|
|
49
110
|
const list = refs.slice();
|
|
50
111
|
if (strategy === 'sticky') {
|
|
51
|
-
const
|
|
112
|
+
const router = getOrCreateSessionRouter(deps.pool);
|
|
113
|
+
let chosenKey = null;
|
|
114
|
+
|
|
115
|
+
if (deps.sessionId) {
|
|
116
|
+
const resolved = router.resolveSessionKey(deps.sessionId, list);
|
|
117
|
+
if (resolved && resolved.keyRef) {
|
|
118
|
+
chosenKey = resolved.keyRef;
|
|
119
|
+
if (deps.pool) setSessionSticky(deps.pool, deps.sessionId, chosenKey);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const last = chosenKey ||
|
|
124
|
+
deps.sessionSticky ||
|
|
52
125
|
(deps.sessionId && (deps.pool?.state?.sessionSticky?.get?.(deps.sessionId) || deps.sessionStickyMap?.get?.(deps.sessionId))) ||
|
|
53
126
|
deps.lastUsed ||
|
|
54
127
|
(deps.pool && deps.pool.state && deps.pool.state.lastUsed);
|
|
128
|
+
|
|
55
129
|
if (last && list.includes(last)) {
|
|
56
130
|
const idx = list.indexOf(last);
|
|
57
131
|
if (idx > 0) {
|
|
@@ -147,3 +221,4 @@ export function budgetVerdict(spend, budget) {
|
|
|
147
221
|
const ratio = spend / budget;
|
|
148
222
|
return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
|
|
149
223
|
}
|
|
224
|
+
|
package/lib/pool.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
// lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
|
|
2
2
|
import {
|
|
3
3
|
MAX_SESSION_STICKY_ENTRIES, setSessionSticky, getSessionSticky, clearSessionSticky,
|
|
4
|
+
unstickFailedKey, SessionRouter, defaultSessionRouter, MAX_SESSIONS, SESSION_TTL_MS,
|
|
4
5
|
selectPool, sortAttemptList, expiringSoon, shouldNotifyDaily,
|
|
5
6
|
costForDay, costForWeek, budgetVerdict,
|
|
6
7
|
} from './pool-selection.js';
|
|
7
8
|
|
|
8
9
|
export {
|
|
9
10
|
MAX_SESSION_STICKY_ENTRIES, setSessionSticky, getSessionSticky, clearSessionSticky,
|
|
11
|
+
unstickFailedKey, SessionRouter, defaultSessionRouter, MAX_SESSIONS, SESSION_TTL_MS,
|
|
10
12
|
selectPool, sortAttemptList, expiringSoon, shouldNotifyDaily,
|
|
11
13
|
costForDay, costForWeek, budgetVerdict,
|
|
12
14
|
};
|
|
13
15
|
|
|
16
|
+
import { classifyResponseError, CONTEXT_WINDOW_EXCEEDED_CODE } from './errors.js';
|
|
17
|
+
export { classifyResponseError };
|
|
14
18
|
import { resetModelQuotaIfNeeded } from './model-quota.js';
|
|
15
19
|
import { decryptSecret } from './crypto-storage.js';
|
|
16
20
|
import { bestEffort } from './best-effort.js';
|
|
@@ -190,6 +194,7 @@ export function pickNext(pool, now, refsCount = pool.refs.length) {
|
|
|
190
194
|
|
|
191
195
|
/** Apply a failed-key cooldown to pool state. Pure: returns next state shape. */
|
|
192
196
|
export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
|
|
197
|
+
unstickFailedKey(pool, ref);
|
|
193
198
|
return {
|
|
194
199
|
...pool,
|
|
195
200
|
state: {
|
|
@@ -218,6 +223,7 @@ export function computeBackoff(baseMs, failCount, maxMs, isSoft = false) {
|
|
|
218
223
|
|
|
219
224
|
/** Record a failure for `ref` in `pool.state`. */
|
|
220
225
|
export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jitter = false) {
|
|
226
|
+
unstickFailedKey(pool, ref);
|
|
221
227
|
if (!pool.state.failCounts) pool.state.failCounts = new Map();
|
|
222
228
|
let next = pool.state.failCounts.get(ref) ?? 0;
|
|
223
229
|
if (!isSoft) {
|
|
@@ -494,6 +500,18 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
|
|
|
494
500
|
status = Number(rawCode);
|
|
495
501
|
}
|
|
496
502
|
|
|
503
|
+
// 0. Structured response error classification (#547)
|
|
504
|
+
if (status || failure.body || message.startsWith('{') || (code && code !== 'ERROR')) {
|
|
505
|
+
const typed = classifyResponseError(status, failure.body ?? message, failure.headers ?? failure.response?.headers);
|
|
506
|
+
if (typed.code === CONTEXT_WINDOW_EXCEEDED_CODE) return false;
|
|
507
|
+
if ((typed.code === 'QUOTA' || typed.code === 'ACCOUNT_QUOTA') && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT') || switchCodes.has('429'))) return true;
|
|
508
|
+
if (typed.code === 'RATE_LIMIT' && (switchCodes.has('RATE_LIMIT') || switchCodes.has('429'))) return true;
|
|
509
|
+
if ((typed.code === 'AUTH' || typed.code === 'INVALID_CREDENTIAL') && (switchCodes.has('AUTH') || switchCodes.has('401') || switchCodes.has('403'))) return true;
|
|
510
|
+
if (typed.code === 'SERVER' && (switchCodes.has('SERVER') || (status >= 500 && status <= 504 && switchCodes.has(String(status))))) return true;
|
|
511
|
+
if (typed.code === 'TIMEOUT' && (switchCodes.has('TIMEOUT') || switchCodes.has('408') || switchCodes.has('504'))) return true;
|
|
512
|
+
if (switchCodes.has(typed.code)) return true;
|
|
513
|
+
}
|
|
514
|
+
|
|
497
515
|
// 1. Direct HTTP status codes (#267: also 408/425 transient, #545: numeric status/code mapped to semantic switch codes)
|
|
498
516
|
if ((status === 429 || status === 408 || status === 425) && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429') || switchCodes.has('TIMEOUT') || switchCodes.has(String(status)))) return true;
|
|
499
517
|
if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401') || switchCodes.has('403') || switchCodes.has(String(status)))) return true;
|
|
@@ -576,3 +594,4 @@ export function isKeyBroken(pool, ref, now = Date.now()) {
|
|
|
576
594
|
if (pool.basePool) return isKeyBroken(pool.basePool, ref, now);
|
|
577
595
|
return false;
|
|
578
596
|
}
|
|
597
|
+
|
package/lib/rotate.js
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
getSessionSticky,
|
|
16
16
|
setSessionSticky,
|
|
17
17
|
clearSessionSticky,
|
|
18
|
+
unstickFailedKey,
|
|
18
19
|
} from './pool.js';
|
|
19
20
|
import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
|
|
20
21
|
import { createRequestBudget, BUDGET_EXHAUSTED_TIMEOUT, BUDGET_EXHAUSTED_ATTEMPTS } from './request-budget.js';
|
|
@@ -210,6 +211,7 @@ export function createRotate(deps) {
|
|
|
210
211
|
const penalizeRef = (targetRef, errCode, errMsg, rawHeaders) => {
|
|
211
212
|
if (!targetRef) return;
|
|
212
213
|
if (sessionId) clearSessionSticky(pool, sessionId);
|
|
214
|
+
unstickFailedKey(pool, targetRef);
|
|
213
215
|
if (errCode === LOCAL_QUOTA_CODE) return;
|
|
214
216
|
let headerResetMs = null;
|
|
215
217
|
if (rawHeaders) {
|
|
@@ -411,6 +413,7 @@ export function createRotate(deps) {
|
|
|
411
413
|
let yielded = false;
|
|
412
414
|
let switching = false;
|
|
413
415
|
let inner;
|
|
416
|
+
const preTokenBuffer = [];
|
|
414
417
|
try {
|
|
415
418
|
// #464 & #461: Record attempt & trace dispatch
|
|
416
419
|
budget.recordAttempt();
|
|
@@ -450,10 +453,13 @@ export function createRotate(deps) {
|
|
|
450
453
|
if (ok) reqStore.concurrencyRef = fallbackRef;
|
|
451
454
|
}
|
|
452
455
|
}
|
|
453
|
-
//
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
456
|
+
// Pre-First-Token Failover (#527): Lock the stream only upon first content delta.
|
|
457
|
+
if (chunk && isContentChunk(chunk)) {
|
|
458
|
+
if (!yielded) {
|
|
459
|
+
yielded = true;
|
|
460
|
+
for (const prev of preTokenBuffer) yield prev;
|
|
461
|
+
preTokenBuffer.length = 0;
|
|
462
|
+
}
|
|
457
463
|
yield chunk;
|
|
458
464
|
continue;
|
|
459
465
|
}
|
|
@@ -465,15 +471,16 @@ export function createRotate(deps) {
|
|
|
465
471
|
const message = failure?.message ?? '';
|
|
466
472
|
const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
|
|
467
473
|
const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
|
|
468
|
-
const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
|
|
474
|
+
const activeRef = reqStore.pickedRef ?? pool.state.lastUsed ?? attemptList[attempt];
|
|
469
475
|
if (switchable) {
|
|
476
|
+
preTokenBuffer.length = 0; // Discard metadata from aborted attempt (#527)
|
|
470
477
|
penalizeRef(activeRef, code ?? 'UNKNOWN', message, rawHdrs ?? failure?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
|
|
471
478
|
recordFailureCategory(pool, code ?? 'UNKNOWN', message, activeRef, now());
|
|
472
479
|
pool.state.switches = (pool.state.switches ?? 0) + 1;
|
|
473
480
|
pool.state.lastReason = String(code ?? 'UNKNOWN');
|
|
474
481
|
pool.state.lastSwitchAt = now();
|
|
475
482
|
lastFailure = chunk;
|
|
476
|
-
logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
|
|
483
|
+
logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed before first token (${String(code)} ${String(message).slice(0, 100)}) - failover to next key`);
|
|
477
484
|
// #216: per-switch webhook (opt-in switchNotify), deduped per provider
|
|
478
485
|
if (switchNotify && activeRef) {
|
|
479
486
|
notifySwitch(runtime0, pool, {
|
|
@@ -492,6 +499,10 @@ export function createRotate(deps) {
|
|
|
492
499
|
if (kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes)) {
|
|
493
500
|
penalizeRef(activeRef, code ?? 'UNKNOWN', message, rawHdrs ?? failure?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
|
|
494
501
|
}
|
|
502
|
+
if (!yielded) {
|
|
503
|
+
for (const prev of preTokenBuffer) yield prev;
|
|
504
|
+
preTokenBuffer.length = 0;
|
|
505
|
+
}
|
|
495
506
|
yield chunk;
|
|
496
507
|
return;
|
|
497
508
|
}
|
|
@@ -539,24 +550,33 @@ export function createRotate(deps) {
|
|
|
539
550
|
});
|
|
540
551
|
}
|
|
541
552
|
recordLatency(pool, reqStore);
|
|
542
|
-
activePermit?.success();
|
|
553
|
+
if (typeof activePermit?.success === 'function') activePermit.success();
|
|
543
554
|
if (isTraceOwner) trace.finish('SUCCESS', activeRef);
|
|
555
|
+
if (!yielded) {
|
|
556
|
+
for (const prev of preTokenBuffer) yield prev;
|
|
557
|
+
preTokenBuffer.length = 0;
|
|
558
|
+
}
|
|
544
559
|
yield chunk;
|
|
545
560
|
return;
|
|
546
561
|
}
|
|
547
|
-
|
|
562
|
+
if (yielded) {
|
|
563
|
+
yield chunk;
|
|
564
|
+
} else {
|
|
565
|
+
preTokenBuffer.push(chunk);
|
|
566
|
+
}
|
|
548
567
|
}
|
|
549
568
|
} catch (e) {
|
|
550
569
|
const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
|
|
551
|
-
const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
|
|
570
|
+
const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed ?? attemptList[attempt];
|
|
552
571
|
if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
|
|
572
|
+
preTokenBuffer.length = 0; // Discard metadata from failed attempt (#527)
|
|
553
573
|
penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e), e?.headers ?? e?.response?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
|
|
554
574
|
recordFailureCategory(pool, e?.code ?? 'TRANSPORT', String(e?.message ?? e), activeRef, now());
|
|
555
575
|
pool.state.switches = (pool.state.switches ?? 0) + 1;
|
|
556
576
|
pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
|
|
557
577
|
pool.state.lastSwitchAt = now();
|
|
558
578
|
lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
|
|
559
|
-
logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
|
|
579
|
+
logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw before first token ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
|
|
560
580
|
if (switchNotify && activeRef) {
|
|
561
581
|
notifySwitch(runtime0, pool, {
|
|
562
582
|
provider: options.provider,
|
|
@@ -570,16 +590,24 @@ export function createRotate(deps) {
|
|
|
570
590
|
if (yielded && isSwitchableError(e, effectiveSwitchCodes)) {
|
|
571
591
|
penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e), e?.headers ?? e?.response?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
|
|
572
592
|
}
|
|
593
|
+
if (!yielded) {
|
|
594
|
+
for (const prev of preTokenBuffer) yield prev;
|
|
595
|
+
preTokenBuffer.length = 0;
|
|
596
|
+
}
|
|
573
597
|
yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
|
|
574
598
|
return;
|
|
575
599
|
}
|
|
576
600
|
|
|
577
601
|
if (switching) continue; // try the next key
|
|
602
|
+
if (!yielded && preTokenBuffer.length > 0) {
|
|
603
|
+
for (const prev of preTokenBuffer) yield prev;
|
|
604
|
+
preTokenBuffer.length = 0;
|
|
605
|
+
}
|
|
578
606
|
if (yielded) {
|
|
579
607
|
const activeRef = reqStore.pickedRef ?? _pickedRef ?? pool.state.lastUsed;
|
|
580
608
|
if (activeRef) recordSuccess(pool, activeRef, now());
|
|
581
609
|
recordLatency(pool, reqStore);
|
|
582
|
-
activePermit?.success();
|
|
610
|
+
if (typeof activePermit?.success === 'function') activePermit.success();
|
|
583
611
|
if (isTraceOwner) trace.finish('SUCCESS', activeRef);
|
|
584
612
|
} else if (isTraceOwner) {
|
|
585
613
|
trace.finish('COMPLETED', reqStore.pickedRef ?? _pickedRef ?? pool.state.lastUsed);
|
|
@@ -626,3 +654,25 @@ export function createRotate(deps) {
|
|
|
626
654
|
|
|
627
655
|
return rotate;
|
|
628
656
|
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* Whether a chunk carries a model-visible content delta (#527).
|
|
660
|
+
* @param {Record<string, unknown>} chunk
|
|
661
|
+
* @returns {boolean}
|
|
662
|
+
*/
|
|
663
|
+
export function isContentChunk(chunk) {
|
|
664
|
+
if (!chunk || typeof chunk !== 'object') return false;
|
|
665
|
+
if (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta') {
|
|
666
|
+
return true;
|
|
667
|
+
}
|
|
668
|
+
if (chunk.delta !== undefined || chunk.text !== undefined || chunk.content !== undefined || chunk.choices !== undefined) {
|
|
669
|
+
return true;
|
|
670
|
+
}
|
|
671
|
+
if (chunk.type === 'start' || chunk.type === 'block-start' || chunk.type === 'block-stop' ||
|
|
672
|
+
chunk.type === 'metadata' || chunk.type === 'response-metadata' || chunk.type === 'usage' ||
|
|
673
|
+
chunk.type === 'finish') {
|
|
674
|
+
return false;
|
|
675
|
+
}
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
// lib/session-router.js — Bounded SessionRouter with LRU eviction and proactive unsticking (#548).
|
|
2
|
+
// Prevents unbounded memory growth and unpins sessions from failed/cooldown keys.
|
|
3
|
+
|
|
4
|
+
export const MAX_SESSIONS = 1000;
|
|
5
|
+
export const SESSION_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
|
6
|
+
|
|
7
|
+
export class SessionRouter {
|
|
8
|
+
/**
|
|
9
|
+
* @param {{
|
|
10
|
+
* maxSessions?: number,
|
|
11
|
+
* sessionTtlMs?: number,
|
|
12
|
+
* now?: () => number,
|
|
13
|
+
* }} [options]
|
|
14
|
+
*/
|
|
15
|
+
constructor(options = {}) {
|
|
16
|
+
this.maxSessions = options.maxSessions ?? MAX_SESSIONS;
|
|
17
|
+
this.sessionTtlMs = options.sessionTtlMs ?? SESSION_TTL_MS;
|
|
18
|
+
this.now = options.now ?? (() => Date.now());
|
|
19
|
+
// sessionId -> { keyRef: string, assignedAt: number, lastUsedAt: number }
|
|
20
|
+
this.sessions = new Map();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Evict sessions older than TTL and prune LRU entries if capacity exceeded.
|
|
25
|
+
*
|
|
26
|
+
* @param {number} [currentTime]
|
|
27
|
+
* @returns {number} count of evicted sessions
|
|
28
|
+
*/
|
|
29
|
+
prune(currentTime = this.now()) {
|
|
30
|
+
let evicted = 0;
|
|
31
|
+
// 1. Evict by TTL
|
|
32
|
+
for (const [id, entry] of this.sessions.entries()) {
|
|
33
|
+
if (currentTime - entry.lastUsedAt > this.sessionTtlMs) {
|
|
34
|
+
this.sessions.delete(id);
|
|
35
|
+
evicted++;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// 2. Evict least-recently used if size exceeds maxSessions
|
|
40
|
+
while (this.sessions.size > this.maxSessions) {
|
|
41
|
+
let oldestId = null;
|
|
42
|
+
let oldestTime = Infinity;
|
|
43
|
+
for (const [id, entry] of this.sessions.entries()) {
|
|
44
|
+
if (entry.lastUsedAt < oldestTime) {
|
|
45
|
+
oldestTime = entry.lastUsedAt;
|
|
46
|
+
oldestId = id;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (oldestId !== null) {
|
|
50
|
+
this.sessions.delete(oldestId);
|
|
51
|
+
evicted++;
|
|
52
|
+
} else {
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return evicted;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Count how many active sessions are currently assigned to each key.
|
|
62
|
+
*
|
|
63
|
+
* @param {number} [currentTime]
|
|
64
|
+
* @returns {Map<string, number>}
|
|
65
|
+
*/
|
|
66
|
+
getKeyAssignmentCounts(currentTime = this.now()) {
|
|
67
|
+
const counts = new Map();
|
|
68
|
+
for (const entry of this.sessions.values()) {
|
|
69
|
+
if (currentTime - entry.lastUsedAt <= this.sessionTtlMs) {
|
|
70
|
+
counts.set(entry.keyRef, (counts.get(entry.keyRef) ?? 0) + 1);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return counts;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Select candidate key with the least active sessions assigned (Least-Used Initial Assignment).
|
|
78
|
+
*
|
|
79
|
+
* @param {string[]} availableRefs List of candidate key refs
|
|
80
|
+
* @param {number} [currentTime]
|
|
81
|
+
* @returns {string|null}
|
|
82
|
+
*/
|
|
83
|
+
selectLeastUsedKey(availableRefs, currentTime = this.now()) {
|
|
84
|
+
if (!Array.isArray(availableRefs) || availableRefs.length === 0) return null;
|
|
85
|
+
if (availableRefs.length === 1) return availableRefs[0];
|
|
86
|
+
|
|
87
|
+
const counts = this.getKeyAssignmentCounts(currentTime);
|
|
88
|
+
let minKey = availableRefs[0];
|
|
89
|
+
let minCount = counts.get(minKey) ?? 0;
|
|
90
|
+
|
|
91
|
+
for (let i = 1; i < availableRefs.length; i++) {
|
|
92
|
+
const key = availableRefs[i];
|
|
93
|
+
const count = counts.get(key) ?? 0;
|
|
94
|
+
if (count < minCount) {
|
|
95
|
+
minCount = count;
|
|
96
|
+
minKey = key;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return minKey;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Resolve pinned key for a session or allocate via Least-Used Initial Assignment.
|
|
105
|
+
*
|
|
106
|
+
* @param {string} sessionId
|
|
107
|
+
* @param {string[]} availableRefs Candidate keys currently healthy/available
|
|
108
|
+
* @param {{ preferKey?: string }} [options]
|
|
109
|
+
* @returns {{ keyRef: string, isReassigned: boolean, reason: string } | null}
|
|
110
|
+
*/
|
|
111
|
+
resolveSessionKey(sessionId, availableRefs, options = {}) {
|
|
112
|
+
if (!sessionId || !Array.isArray(availableRefs) || availableRefs.length === 0) {
|
|
113
|
+
const fallback = availableRefs?.[0] ?? null;
|
|
114
|
+
return fallback ? { keyRef: fallback, isReassigned: false, reason: 'ad_hoc' } : null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const t = this.now();
|
|
118
|
+
let existing = this.sessions.get(sessionId);
|
|
119
|
+
|
|
120
|
+
// Check TTL
|
|
121
|
+
if (existing && (t - existing.lastUsedAt > this.sessionTtlMs)) {
|
|
122
|
+
this.sessions.delete(sessionId);
|
|
123
|
+
existing = null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// If session is already pinned to an available key:
|
|
127
|
+
if (existing && availableRefs.includes(existing.keyRef)) {
|
|
128
|
+
existing.lastUsedAt = t;
|
|
129
|
+
return { keyRef: existing.keyRef, isReassigned: false, reason: 'sticky_pinned' };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Prune before adding new entry
|
|
133
|
+
if (!existing && this.sessions.size >= this.maxSessions) {
|
|
134
|
+
this.prune(t);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Select candidate: prefer explicit key or least-used
|
|
138
|
+
let chosen = options.preferKey && availableRefs.includes(options.preferKey)
|
|
139
|
+
? options.preferKey
|
|
140
|
+
: this.selectLeastUsedKey(availableRefs, t);
|
|
141
|
+
|
|
142
|
+
if (!chosen) chosen = availableRefs[0];
|
|
143
|
+
|
|
144
|
+
const wasReassigned = Boolean(existing);
|
|
145
|
+
this.sessions.set(sessionId, {
|
|
146
|
+
keyRef: chosen,
|
|
147
|
+
assignedAt: existing ? existing.assignedAt : t,
|
|
148
|
+
lastUsedAt: t,
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
keyRef: chosen,
|
|
153
|
+
isReassigned: wasReassigned,
|
|
154
|
+
reason: wasReassigned ? 'mid_session_failover' : 'new_session_assigned',
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Pin or update session sticky assignment explicitly.
|
|
160
|
+
*
|
|
161
|
+
* @param {string} sessionId
|
|
162
|
+
* @param {string} keyRef
|
|
163
|
+
*/
|
|
164
|
+
setSession(sessionId, keyRef) {
|
|
165
|
+
if (!sessionId || !keyRef) return;
|
|
166
|
+
const t = this.now();
|
|
167
|
+
const existing = this.sessions.get(sessionId);
|
|
168
|
+
this.sessions.set(sessionId, {
|
|
169
|
+
keyRef,
|
|
170
|
+
assignedAt: existing ? existing.assignedAt : t,
|
|
171
|
+
lastUsedAt: t,
|
|
172
|
+
});
|
|
173
|
+
if (this.sessions.size > this.maxSessions) {
|
|
174
|
+
this.prune(t);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Get pinned key for a session if still valid.
|
|
180
|
+
*
|
|
181
|
+
* @param {string} sessionId
|
|
182
|
+
* @returns {string|null}
|
|
183
|
+
*/
|
|
184
|
+
getSession(sessionId) {
|
|
185
|
+
if (!sessionId) return null;
|
|
186
|
+
const entry = this.sessions.get(sessionId);
|
|
187
|
+
if (!entry) return null;
|
|
188
|
+
const t = this.now();
|
|
189
|
+
if (t - entry.lastUsedAt > this.sessionTtlMs) {
|
|
190
|
+
this.sessions.delete(sessionId);
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
return entry.keyRef;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Unstick a specific session (e.g. after a failure or manual unpin).
|
|
198
|
+
*
|
|
199
|
+
* @param {string} sessionId
|
|
200
|
+
* @returns {boolean} true if session was present and removed
|
|
201
|
+
*/
|
|
202
|
+
unstick(sessionId) {
|
|
203
|
+
if (!sessionId) return false;
|
|
204
|
+
return this.sessions.delete(sessionId);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Proactively unstick all sessions pinned to a failed / cooling key (#548).
|
|
209
|
+
*
|
|
210
|
+
* @param {string} keyRef Failed key ref
|
|
211
|
+
* @returns {number} count of sessions unpinned
|
|
212
|
+
*/
|
|
213
|
+
unstickKey(keyRef) {
|
|
214
|
+
if (!keyRef) return 0;
|
|
215
|
+
let count = 0;
|
|
216
|
+
for (const [id, entry] of this.sessions.entries()) {
|
|
217
|
+
if (entry.keyRef === keyRef) {
|
|
218
|
+
this.sessions.delete(id);
|
|
219
|
+
count++;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return count;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Return router diagnostics.
|
|
227
|
+
*/
|
|
228
|
+
getStatus() {
|
|
229
|
+
const t = this.now();
|
|
230
|
+
const counts = this.getKeyAssignmentCounts(t);
|
|
231
|
+
return {
|
|
232
|
+
activeSessionsCount: this.sessions.size,
|
|
233
|
+
maxSessions: this.maxSessions,
|
|
234
|
+
sessionTtlMs: this.sessionTtlMs,
|
|
235
|
+
keyAssignments: Object.fromEntries(counts.entries()),
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Reset all sessions (useful for tests).
|
|
241
|
+
*/
|
|
242
|
+
reset() {
|
|
243
|
+
this.sessions.clear();
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Global default router instance for convenience
|
|
248
|
+
export const defaultSessionRouter = new SessionRouter();
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Helper to unstick all sessions pinned to keyRef across router or default instance (#548).
|
|
252
|
+
* @param {string|object} keyRefOrRouter
|
|
253
|
+
* @param {string} [keyRef]
|
|
254
|
+
* @returns {number}
|
|
255
|
+
*/
|
|
256
|
+
export function unstickFailedKey(keyRefOrRouter, keyRef) {
|
|
257
|
+
if (typeof keyRefOrRouter === 'string') {
|
|
258
|
+
return defaultSessionRouter.unstickKey(keyRefOrRouter);
|
|
259
|
+
}
|
|
260
|
+
if (keyRefOrRouter instanceof SessionRouter) {
|
|
261
|
+
return keyRefOrRouter.unstickKey(keyRef);
|
|
262
|
+
}
|
|
263
|
+
return defaultSessionRouter.unstickKey(keyRef);
|
|
264
|
+
}
|
|
265
|
+
|
package/package.json
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
4
|
-
"packageManager": "pnpm@10.33.2",
|
|
3
|
+
"version": "0.8.77",
|
|
5
4
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
5
|
"keywords": [
|
|
7
6
|
"deepseek-harness",
|
|
@@ -60,13 +59,6 @@
|
|
|
60
59
|
"@deepseek-ai/dsh-llm": "^0.1.7-rc.2 || >=0.2.0-rc.1 <0.2.1-0 || >=0.2.1-alpha.1 <0.2.2-0",
|
|
61
60
|
"@deepseek-ai/schemastery": "^3.18.4"
|
|
62
61
|
},
|
|
63
|
-
"scripts": {
|
|
64
|
-
"test": "node tools/test.mjs",
|
|
65
|
-
"test:core": "node tools/test.mjs --core",
|
|
66
|
-
"check": "node tools/check.mjs",
|
|
67
|
-
"build:client": "node scripts/build-client.mjs",
|
|
68
|
-
"pretest": "node scripts/build-client.mjs"
|
|
69
|
-
},
|
|
70
62
|
"devDependencies": {
|
|
71
63
|
"react": "18.3.1",
|
|
72
64
|
"react-test-renderer": "18.3.1",
|
|
@@ -75,5 +67,12 @@
|
|
|
75
67
|
"@deepseek-ai/schemastery": "^3.18.4",
|
|
76
68
|
"semver": "^7.6.3"
|
|
77
69
|
},
|
|
78
|
-
"dependencies": {}
|
|
79
|
-
|
|
70
|
+
"dependencies": {},
|
|
71
|
+
"scripts": {
|
|
72
|
+
"test": "node tools/test.mjs",
|
|
73
|
+
"test:core": "node tools/test.mjs --core",
|
|
74
|
+
"check": "node tools/check.mjs",
|
|
75
|
+
"build:client": "node scripts/build-client.mjs",
|
|
76
|
+
"pretest": "node scripts/build-client.mjs"
|
|
77
|
+
}
|
|
78
|
+
}
|