@goodandready/dsh-key-rotation 0.8.63 → 0.8.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/lib/client.js +11 -0
- package/lib/error-taxonomy.js +5 -3
- package/lib/heal.js +4 -0
- package/lib/index.js +1 -0
- package/lib/ops-status.js +2 -0
- package/lib/ops-test.js +26 -7
- package/lib/persistence.js +49 -0
- package/lib/pool-state.js +9 -2
- package/lib/resolver.js +7 -0
- package/lib/rotate.js +30 -8
- package/lib/sandbox.js +6 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.64 - 2026-10-18
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
- **Permanent Auth Pause & Granular Error Policies (#501)**:
|
|
7
|
+
- Differentiated error actions: upstream authentication failures (HTTP 401, 403, `invalid_api_key`, `unauthorized`) immediately place the failing key into a permanent auth pause state (`pausedRefs.add`, `pauseReasons.set('AUTH_FAILURE_PAUSED')`, `failedUntil: Infinity`, `KEY_AUTH_PAUSED`).
|
|
8
|
+
- Dead or unauthorized keys are stopped from waking up repeatedly every 60s cooldown, preventing repetitive broken dispatches.
|
|
9
|
+
- Granular soft error handling: transient infrastructure drops (HTTP 502, 503, 504, `TIMEOUT`, socket drops) receive a short flat cooldown (<=10s) without doubling exponential penalty counters or incrementing toward recoverable quarantine.
|
|
10
|
+
- Successful Test Matrix / manual re-test probe automatically unpauses the key.
|
|
11
|
+
- **Automatic Model Capabilities Discovery & Routing (#500)**:
|
|
12
|
+
- In `lib/sandbox.js` (`probeModels`), parse and return array of supported model IDs from upstream `/models` (`body.data.map(m => m.id)`).
|
|
13
|
+
- Discovered model capabilities are cached per key (`pool.state.modelCapabilities: Map<ref, Set<modelId>>`).
|
|
14
|
+
- In `lib/resolver.js` and `lib/rotate.js`, filter candidate keys for a requested model so prompts are never dispatched to a key that doesn't support that model (e.g., DeepSeek-R1 vs DeepSeek-Chat V3). Keys not yet probed are optimistically allowed.
|
|
15
|
+
- Atomic persistence across restarts: `pausedRefs`, `pauseReasons`, and `modelCapabilities` serialize and restore atomically via `lib/persistence.js`.
|
|
16
|
+
- Settings Card UI: Display `Auth Paused (401/403)` status badge and discovered model count badges (`🤖 Nm`) on configured keys.
|
|
17
|
+
|
|
3
18
|
## 0.8.63 - 2026-10-16
|
|
4
19
|
|
|
5
20
|
### Added
|
package/lib/client.js
CHANGED
|
@@ -98,6 +98,7 @@ window.__ModuleLoader__.load({
|
|
|
98
98
|
bulkImportSuccess: 'Imported {n} keys',
|
|
99
99
|
keyPaused: 'Paused',
|
|
100
100
|
keyRevoked: 'Revoked (401)',
|
|
101
|
+
keyAuthPaused: 'Auth Paused (401/403)',
|
|
101
102
|
pauseKey: 'Pause key from rotation',
|
|
102
103
|
resumeKey: 'Resume key rotation',
|
|
103
104
|
sparklineTraffic: 'Traffic Trend',
|
|
@@ -299,6 +300,7 @@ window.__ModuleLoader__.load({
|
|
|
299
300
|
bulkImportSuccess: '已导入 {n} 个密钥',
|
|
300
301
|
keyPaused: '已暂停',
|
|
301
302
|
keyRevoked: '已撤销 (401)',
|
|
303
|
+
keyAuthPaused: '身份验证失败已暂停 (401/403)',
|
|
302
304
|
pauseKey: '暂停此密钥轮换',
|
|
303
305
|
resumeKey: '恢复此密钥轮换',
|
|
304
306
|
sparklineTraffic: '流量趋势',
|
|
@@ -1952,6 +1954,7 @@ window.__ModuleLoader__.load({
|
|
|
1952
1954
|
if (isPaused) return { color: 'var(--dsw-alias-state-warn-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1953
1955
|
const hit = keyInfo(providerId, ref);
|
|
1954
1956
|
if (!hit) return null;
|
|
1957
|
+
if (hit.authPaused || hit.pauseReason === 'AUTH_FAILURE_PAUSED') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyAuthPaused'), ready: false, authPaused: true };
|
|
1955
1958
|
if (hit.paused) return { color: 'var(--dsw-alias-state-warn-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1956
1959
|
if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired'), ready: false, expired: true };
|
|
1957
1960
|
if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked'), ready: false, revoked: true };
|
|
@@ -2124,6 +2127,14 @@ window.__ModuleLoader__.load({
|
|
|
2124
2127
|
if (info && info.broken) {
|
|
2125
2128
|
meta.push(h('button', { key: 'rt', className: 'krot-btn', onClick: () => retestBroken(key), title: t('retestBroken') }, t('retestBroken')));
|
|
2126
2129
|
}
|
|
2130
|
+
if (info && info.modelCapabilities && info.modelCapabilities.length > 0) {
|
|
2131
|
+
meta.push(h('span', {
|
|
2132
|
+
key: 'caps',
|
|
2133
|
+
className: 'krot-tail',
|
|
2134
|
+
title: 'Discovered models (' + info.modelCapabilities.length + '):\n' + info.modelCapabilities.join('\n'),
|
|
2135
|
+
style: { fontSize: '11px', opacity: 0.85 }
|
|
2136
|
+
}, '🤖 ' + info.modelCapabilities.length + 'm'));
|
|
2137
|
+
}
|
|
2127
2138
|
meta.push(h('span', { key: 'a', className: 'krot-acts' },
|
|
2128
2139
|
btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
|
|
2129
2140
|
btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),
|
package/lib/error-taxonomy.js
CHANGED
|
@@ -36,9 +36,11 @@ export function classifyFailure(failureOrPayload) {
|
|
|
36
36
|
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA') {
|
|
37
37
|
return { action: 'switch', code: status ? '429' : (code || 'RATE_LIMIT'), soft: false, reason: 'quota' };
|
|
38
38
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
39
|
+
const numCode = Number(code);
|
|
40
|
+
if (SWITCH_STATUS.has(status) || (numCode > 0 && SWITCH_STATUS.has(numCode)) || SOFT_FAILURE_CODES.has(code)) {
|
|
41
|
+
const effCode = status ? String(status) : (numCode > 0 ? String(numCode) : code);
|
|
42
|
+
const soft = SOFT_FAILURE_CODES.has(effCode);
|
|
43
|
+
return { action: 'switch', code: effCode, soft, reason: 'http' };
|
|
42
44
|
}
|
|
43
45
|
if (SURFACE_STATUS.has(status)) {
|
|
44
46
|
return { action: 'surface', code: String(status), soft: false, reason: 'client' };
|
package/lib/heal.js
CHANGED
|
@@ -90,6 +90,10 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
|
|
|
90
90
|
pool.state.failedUntil?.delete(ref);
|
|
91
91
|
pool.state.failCounts?.delete(ref);
|
|
92
92
|
pool.state.authFailCounts?.delete(ref);
|
|
93
|
+
if (Array.isArray(res.models) && res.models.length > 0) {
|
|
94
|
+
if (!pool.state.modelCapabilities) pool.state.modelCapabilities = new Map();
|
|
95
|
+
pool.state.modelCapabilities.set(ref, new Set(res.models));
|
|
96
|
+
}
|
|
93
97
|
if (Array.isArray(pool.state.events)) {
|
|
94
98
|
pool.state.events.push({ at: now, ref, reason: 'auto-unbreak', cooldownMs: 0, type: 'heal' });
|
|
95
99
|
if (pool.state.events.length > 50) pool.state.events.shift();
|
package/lib/index.js
CHANGED
|
@@ -377,6 +377,7 @@ export function apply(ctx, config = {}) {
|
|
|
377
377
|
buildRuntime,
|
|
378
378
|
currentPool: () => dispatchStorage.getStore()?.pool,
|
|
379
379
|
currentSession: () => dispatchStorage.getStore()?.sessionId,
|
|
380
|
+
currentModel: () => dispatchStorage.getStore()?.model,
|
|
380
381
|
onPicked: (pool, candidate, meta) => {
|
|
381
382
|
const store = dispatchStorage.getStore();
|
|
382
383
|
if (store && store.pool === pool) {
|
package/lib/ops-status.js
CHANGED
|
@@ -120,7 +120,9 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
120
120
|
expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
|
|
121
121
|
broken: pool.state.brokenUntil?.has(ref) ?? false,
|
|
122
122
|
paused: isKeyPaused(pool, ref),
|
|
123
|
+
pauseReason: pool.state.pauseReasons?.get?.(ref) ?? null,
|
|
123
124
|
revoked: isKeyRevoked(pool, ref),
|
|
125
|
+
modelCapabilities: pool.state.modelCapabilities?.get?.(ref) ? [...pool.state.modelCapabilities.get(ref)] : null,
|
|
124
126
|
modelQuota,
|
|
125
127
|
modelPools: activeModelPools.map(mp => mp.base),
|
|
126
128
|
});
|
package/lib/ops-test.js
CHANGED
|
@@ -73,16 +73,24 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
73
73
|
lastTestCache.set(ref, cached);
|
|
74
74
|
if (cached.ok) {
|
|
75
75
|
for (const st of poolState.values()) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
76
|
+
st.failedUntil?.delete(ref);
|
|
77
|
+
st.failCounts?.delete(ref);
|
|
78
|
+
st.authFailCounts?.delete(ref);
|
|
79
|
+
st.brokenUntil?.delete(ref);
|
|
80
|
+
st.revokedRefs?.delete(ref);
|
|
81
|
+
if (st.pausedRefs instanceof Set) st.pausedRefs.delete(ref);
|
|
82
|
+
else if (Array.isArray(st.pausedRefs)) {
|
|
83
|
+
const pIdx = st.pausedRefs.indexOf(ref);
|
|
84
|
+
if (pIdx >= 0) st.pausedRefs.splice(pIdx, 1);
|
|
85
|
+
}
|
|
86
|
+
st.pauseReasons?.delete(ref);
|
|
87
|
+
if (Array.isArray(cached.models) && cached.models.length > 0) {
|
|
88
|
+
if (!st.modelCapabilities) st.modelCapabilities = new Map();
|
|
89
|
+
st.modelCapabilities.set(ref, new Set(cached.models));
|
|
82
90
|
}
|
|
83
91
|
}
|
|
84
92
|
}
|
|
85
|
-
json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
|
|
93
|
+
json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount, models: cached.models ?? [] });
|
|
86
94
|
return;
|
|
87
95
|
}
|
|
88
96
|
json(res, 200, { ok: true, ref, tail, source });
|
|
@@ -179,6 +187,16 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
179
187
|
st.authFailCounts?.delete(ref);
|
|
180
188
|
st.brokenUntil?.delete(ref);
|
|
181
189
|
st.revokedRefs?.delete(ref);
|
|
190
|
+
if (st.pausedRefs instanceof Set) st.pausedRefs.delete(ref);
|
|
191
|
+
else if (Array.isArray(st.pausedRefs)) {
|
|
192
|
+
const pIdx = st.pausedRefs.indexOf(ref);
|
|
193
|
+
if (pIdx >= 0) st.pausedRefs.splice(pIdx, 1);
|
|
194
|
+
}
|
|
195
|
+
st.pauseReasons?.delete(ref);
|
|
196
|
+
if (Array.isArray(cached.models) && cached.models.length > 0) {
|
|
197
|
+
if (!st.modelCapabilities) st.modelCapabilities = new Map();
|
|
198
|
+
st.modelCapabilities.set(ref, new Set(cached.models));
|
|
199
|
+
}
|
|
182
200
|
}
|
|
183
201
|
}
|
|
184
202
|
results.push({
|
|
@@ -189,6 +207,7 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
189
207
|
code: cached.code,
|
|
190
208
|
latencyMs: cached.latencyMs,
|
|
191
209
|
modelsCount: cached.modelsCount,
|
|
210
|
+
models: cached.models ?? [],
|
|
192
211
|
});
|
|
193
212
|
} catch (err) {
|
|
194
213
|
results.push({
|
package/lib/persistence.js
CHANGED
|
@@ -44,6 +44,32 @@ export class StatePersistence {
|
|
|
44
44
|
revokedRefs.push(...st.revokedRefs);
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
+
const pausedRefs = [];
|
|
48
|
+
if (st && st.pausedRefs) {
|
|
49
|
+
if (typeof st.pausedRefs.forEach === 'function') {
|
|
50
|
+
st.pausedRefs.forEach((r) => pausedRefs.push(r));
|
|
51
|
+
} else if (Array.isArray(st.pausedRefs)) {
|
|
52
|
+
pausedRefs.push(...st.pausedRefs);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const pauseReasons = {};
|
|
56
|
+
if (st && st.pauseReasons && typeof st.pauseReasons.forEach === 'function') {
|
|
57
|
+
st.pauseReasons.forEach((reason, ref) => {
|
|
58
|
+
if (typeof reason === 'string') pauseReasons[ref] = reason;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const modelCapabilities = {};
|
|
62
|
+
if (st && st.modelCapabilities && typeof st.modelCapabilities.forEach === 'function') {
|
|
63
|
+
st.modelCapabilities.forEach((caps, ref) => {
|
|
64
|
+
if (caps && typeof caps.forEach === 'function') {
|
|
65
|
+
const arr = [];
|
|
66
|
+
caps.forEach((m) => { if (typeof m === 'string') arr.push(m); });
|
|
67
|
+
if (arr.length > 0) modelCapabilities[ref] = arr;
|
|
68
|
+
} else if (Array.isArray(caps) && caps.length > 0) {
|
|
69
|
+
modelCapabilities[ref] = caps.filter((m) => typeof m === 'string');
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
47
73
|
const tokenUsage = {};
|
|
48
74
|
if (st && st.tokenUsage && typeof st.tokenUsage.forEach === 'function') {
|
|
49
75
|
st.tokenUsage.forEach((entry, ref) => {
|
|
@@ -109,6 +135,9 @@ export class StatePersistence {
|
|
|
109
135
|
lastUsed: typeof st?.lastUsed === 'string' ? st.lastUsed : null,
|
|
110
136
|
};
|
|
111
137
|
if (revokedRefs.length > 0) outPool.revokedRefs = revokedRefs;
|
|
138
|
+
if (pausedRefs.length > 0) outPool.pausedRefs = pausedRefs;
|
|
139
|
+
if (Object.keys(pauseReasons).length > 0) outPool.pauseReasons = pauseReasons;
|
|
140
|
+
if (Object.keys(modelCapabilities).length > 0) outPool.modelCapabilities = modelCapabilities;
|
|
112
141
|
if (Object.keys(tokenUsage).length > 0) outPool.tokenUsage = tokenUsage;
|
|
113
142
|
if (Object.keys(costDays).length > 0) outPool.costDays = costDays;
|
|
114
143
|
if (Object.keys(costPerKey).length > 0) outPool.costPerKey = costPerKey;
|
|
@@ -200,6 +229,26 @@ export class StatePersistence {
|
|
|
200
229
|
if (typeof r === 'string') state.revokedRefs.add(r);
|
|
201
230
|
}
|
|
202
231
|
}
|
|
232
|
+
if (Array.isArray(saved.pausedRefs)) {
|
|
233
|
+
if (!state.pausedRefs) state.pausedRefs = new Set();
|
|
234
|
+
for (const r of saved.pausedRefs) {
|
|
235
|
+
if (typeof r === 'string') state.pausedRefs.add(r);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (saved.pauseReasons && typeof saved.pauseReasons === 'object' && !Array.isArray(saved.pauseReasons)) {
|
|
239
|
+
if (!state.pauseReasons) state.pauseReasons = new Map();
|
|
240
|
+
for (const [ref, reason] of Object.entries(saved.pauseReasons)) {
|
|
241
|
+
if (typeof reason === 'string') state.pauseReasons.set(ref, reason);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (saved.modelCapabilities && typeof saved.modelCapabilities === 'object' && !Array.isArray(saved.modelCapabilities)) {
|
|
245
|
+
if (!state.modelCapabilities) state.modelCapabilities = new Map();
|
|
246
|
+
for (const [ref, list] of Object.entries(saved.modelCapabilities)) {
|
|
247
|
+
if (Array.isArray(list)) {
|
|
248
|
+
state.modelCapabilities.set(ref, new Set(list.filter((m) => typeof m === 'string')));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
203
252
|
if (saved.tokenUsage && typeof saved.tokenUsage === 'object' && !Array.isArray(saved.tokenUsage)) {
|
|
204
253
|
for (const [ref, entry] of Object.entries(saved.tokenUsage)) {
|
|
205
254
|
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
|
package/lib/pool-state.js
CHANGED
|
@@ -3,13 +3,18 @@
|
|
|
3
3
|
const MAP_FIELDS = [
|
|
4
4
|
'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
|
|
5
5
|
'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
|
|
6
|
-
'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky',
|
|
6
|
+
'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky', 'modelCapabilities', 'pauseReasons',
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
+
const SET_FIELDS = ['pausedRefs', 'revokedRefs'];
|
|
10
|
+
|
|
9
11
|
export function initializePoolState(state = {}) {
|
|
10
12
|
for (const key of MAP_FIELDS) {
|
|
11
13
|
if (!(state[key] instanceof Map)) state[key] = new Map();
|
|
12
14
|
}
|
|
15
|
+
for (const key of SET_FIELDS) {
|
|
16
|
+
if (!(state[key] instanceof Set)) state[key] = new Set(Array.isArray(state[key]) ? state[key] : []);
|
|
17
|
+
}
|
|
13
18
|
if (!Array.isArray(state.events)) state.events = [];
|
|
14
19
|
if (!Number.isSafeInteger(state.pointer) || state.pointer < 0) state.pointer = 0;
|
|
15
20
|
if (typeof state.lastUsed !== 'string') state.lastUsed = null;
|
|
@@ -21,5 +26,7 @@ export function initializePoolState(state = {}) {
|
|
|
21
26
|
export function hasPoolActivity(state) {
|
|
22
27
|
return state.pointer !== 0 || state.lastUsed !== null || state.events.length > 0 ||
|
|
23
28
|
MAP_FIELDS.some((key) => state[key].size > 0) ||
|
|
24
|
-
(state.lastSuccessAt?.size ?? 0) > 0
|
|
29
|
+
(state.lastSuccessAt?.size ?? 0) > 0 ||
|
|
30
|
+
(state.pausedRefs?.size ?? 0) > 0 ||
|
|
31
|
+
(state.revokedRefs?.size ?? 0) > 0;
|
|
25
32
|
}
|
package/lib/resolver.js
CHANGED
|
@@ -77,6 +77,7 @@ export function createResolver({
|
|
|
77
77
|
buildRuntime,
|
|
78
78
|
currentPool,
|
|
79
79
|
currentSession,
|
|
80
|
+
currentModel,
|
|
80
81
|
onPicked,
|
|
81
82
|
latencyHistogram,
|
|
82
83
|
concurrencyTracker,
|
|
@@ -88,6 +89,7 @@ export function createResolver({
|
|
|
88
89
|
// pool that rotate() actually dispatched with decides ownership. A bare
|
|
89
90
|
// resolve outside rotation falls back to the global lookup.
|
|
90
91
|
const requested = currentPool ? currentPool() ?? null : null;
|
|
92
|
+
const model = resolveOpts?.model ?? currentModel?.() ?? requested?.model ?? null;
|
|
91
93
|
const pool = resolveRefPool(runtime.index, ref, requested);
|
|
92
94
|
if (!pool) {
|
|
93
95
|
const res = await original(ref);
|
|
@@ -168,6 +170,11 @@ export function createResolver({
|
|
|
168
170
|
if (until !== undefined && until > at) continue;
|
|
169
171
|
const candidateExp = pool.expiresAt?.[candidate] ?? pool.basePool?.expiresAt?.[candidate];
|
|
170
172
|
if (candidateExp !== undefined && at >= candidateExp) continue;
|
|
173
|
+
// Issue #500: skip candidate if modelCapabilities is known and does not include requested model
|
|
174
|
+
if (model && pool.state.modelCapabilities) {
|
|
175
|
+
const caps = pool.state.modelCapabilities.get(candidate);
|
|
176
|
+
if (caps && caps.size > 0 && !caps.has(model)) continue;
|
|
177
|
+
}
|
|
171
178
|
// Local model token budget: a spent credential is simply not a candidate.
|
|
172
179
|
if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
|
|
173
180
|
const microThrottling = pool.microThrottling ?? runtime.microThrottling ?? false;
|
package/lib/rotate.js
CHANGED
|
@@ -108,12 +108,19 @@ export function createRotate(deps) {
|
|
|
108
108
|
const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
|
|
109
109
|
let activePermit = null;
|
|
110
110
|
const sessionId = options?.sessionId ?? options?.session ?? options?.metadata?.sessionId ?? options?.chatId ?? options?.conversationId;
|
|
111
|
-
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId };
|
|
111
|
+
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId, model: options.model };
|
|
112
112
|
setRotateStartMs(reqStore.startMs);
|
|
113
113
|
const wallNow = Date.now();
|
|
114
114
|
const quotaManaged = hasModelQuotaConfig(pool);
|
|
115
115
|
let attemptList = (pool.weightedRefs ?? pool.refs).slice();
|
|
116
116
|
attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow) && !isKeyDraining(r));
|
|
117
|
+
// Issue #500: filter candidate keys by confirmed model capabilities
|
|
118
|
+
if (options.model && pool.state.modelCapabilities) {
|
|
119
|
+
attemptList = attemptList.filter((ref) => {
|
|
120
|
+
const caps = pool.state.modelCapabilities.get(ref);
|
|
121
|
+
return !caps || caps.size === 0 || caps.has(options.model);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
117
124
|
if (pool.pinnedRef && attemptList.includes(pool.pinnedRef)) {
|
|
118
125
|
const pIdx = attemptList.indexOf(pool.pinnedRef);
|
|
119
126
|
if (pIdx > 0) {
|
|
@@ -196,7 +203,7 @@ export function createRotate(deps) {
|
|
|
196
203
|
const _base = pool.cooldownMs ?? cooldownMs;
|
|
197
204
|
const _max = pool.maxCooldownMs ?? maxCooldownMs;
|
|
198
205
|
const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
|
|
199
|
-
const cls = classifyFailure({ code: errCode, message: errMsg });
|
|
206
|
+
const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
|
|
200
207
|
const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
|
|
201
208
|
|
|
202
209
|
// #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
|
|
@@ -245,16 +252,26 @@ export function createRotate(deps) {
|
|
|
245
252
|
}
|
|
246
253
|
if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
|
|
247
254
|
if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
|
|
248
|
-
const _cStr = String(errCode ?? '');
|
|
249
|
-
|
|
255
|
+
const _cStr = String(errCode ?? '').toUpperCase();
|
|
256
|
+
const _isAuth = cls.reason === 'auth' || cls.code === '401' || cls.code === '403' ||
|
|
257
|
+
_cStr === 'AUTH' || _cStr === '401' || _cStr === '403' ||
|
|
258
|
+
_cStr === 'PERMISSION_DENIED' || _cStr === 'UNAUTHENTICATED' ||
|
|
259
|
+
/auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated|permission[_\s-]?denied/i.test(errMsg);
|
|
260
|
+
|
|
261
|
+
if (_isAuth) {
|
|
250
262
|
const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
|
|
251
263
|
pool.state.authFailCounts.set(targetRef, _c2);
|
|
252
264
|
if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
|
|
253
265
|
pool.state.revokedRefs.add(targetRef);
|
|
266
|
+
if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
|
|
267
|
+
if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(targetRef);
|
|
268
|
+
else if (Array.isArray(pool.state.pausedRefs) && !pool.state.pausedRefs.includes(targetRef)) pool.state.pausedRefs.push(targetRef);
|
|
269
|
+
if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
|
|
270
|
+
pool.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
|
|
254
271
|
pool.state.failedUntil.set(targetRef, Infinity);
|
|
255
|
-
pushEvent(pool, targetRef, '
|
|
256
|
-
logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently
|
|
257
|
-
// Fan-out
|
|
272
|
+
pushEvent(pool, targetRef, 'KEY_AUTH_PAUSED', 0);
|
|
273
|
+
logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently PAUSED on auth failure (${errCode})`);
|
|
274
|
+
// Fan-out permanent pause across all pools sharing this physical ref (#408):
|
|
258
275
|
const rAll = buildRuntime();
|
|
259
276
|
const candidatePools = [];
|
|
260
277
|
if (Array.isArray(rAll.pools)) candidatePools.push(...rAll.pools);
|
|
@@ -275,8 +292,13 @@ export function createRotate(deps) {
|
|
|
275
292
|
if (hasRef && other.state) {
|
|
276
293
|
if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
|
|
277
294
|
other.state.revokedRefs.add(targetRef);
|
|
295
|
+
if (!other.state.pausedRefs) other.state.pausedRefs = new Set();
|
|
296
|
+
if (other.state.pausedRefs instanceof Set) other.state.pausedRefs.add(targetRef);
|
|
297
|
+
else if (Array.isArray(other.state.pausedRefs) && !other.state.pausedRefs.includes(targetRef)) other.state.pausedRefs.push(targetRef);
|
|
298
|
+
if (!other.state.pauseReasons) other.state.pauseReasons = new Map();
|
|
299
|
+
other.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
|
|
278
300
|
if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
|
|
279
|
-
pushEvent(other, targetRef, '
|
|
301
|
+
pushEvent(other, targetRef, 'KEY_AUTH_PAUSED', 0);
|
|
280
302
|
}
|
|
281
303
|
}
|
|
282
304
|
} else {
|
package/lib/sandbox.js
CHANGED
|
@@ -107,11 +107,15 @@ export class SandboxRunner {
|
|
|
107
107
|
const status = res.status;
|
|
108
108
|
if (status >= 200 && status < 300) {
|
|
109
109
|
let modelsCount = 0;
|
|
110
|
+
let models = [];
|
|
110
111
|
try {
|
|
111
112
|
const body = await res.json();
|
|
112
|
-
|
|
113
|
+
if (Array.isArray(body && body.data)) {
|
|
114
|
+
modelsCount = body.data.length;
|
|
115
|
+
models = body.data.map((m) => (typeof m === 'string' ? m : (m && m.id))).filter(Boolean);
|
|
116
|
+
}
|
|
113
117
|
} catch (_) { /* not json */ }
|
|
114
|
-
return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount };
|
|
118
|
+
return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount, models };
|
|
115
119
|
}
|
|
116
120
|
return { ok: false, code: classifyStatus(status), latencyMs: Date.now() - started };
|
|
117
121
|
} finally {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.64",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|