@goodandready/dsh-key-rotation 0.8.62 → 0.8.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +1 -0
- package/README.ru.md +1 -0
- package/README.zh.md +1 -0
- package/lib/client.js +45 -3
- package/lib/error-taxonomy.js +5 -3
- package/lib/heal.js +4 -0
- package/lib/index.js +3 -0
- package/lib/ops-status.js +4 -1
- package/lib/ops-test.js +26 -7
- package/lib/persistence.js +49 -0
- package/lib/pool-state.js +9 -2
- package/lib/resolver.js +7 -0
- package/lib/rotate.js +30 -8
- package/lib/sandbox.js +6 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.64 - 2026-10-18
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
- **Permanent Auth Pause & Granular Error Policies (#501)**:
|
|
7
|
+
- Differentiated error actions: upstream authentication failures (HTTP 401, 403, `invalid_api_key`, `unauthorized`) immediately place the failing key into a permanent auth pause state (`pausedRefs.add`, `pauseReasons.set('AUTH_FAILURE_PAUSED')`, `failedUntil: Infinity`, `KEY_AUTH_PAUSED`).
|
|
8
|
+
- Dead or unauthorized keys are stopped from waking up repeatedly every 60s cooldown, preventing repetitive broken dispatches.
|
|
9
|
+
- Granular soft error handling: transient infrastructure drops (HTTP 502, 503, 504, `TIMEOUT`, socket drops) receive a short flat cooldown (<=10s) without doubling exponential penalty counters or incrementing toward recoverable quarantine.
|
|
10
|
+
- Successful Test Matrix / manual re-test probe automatically unpauses the key.
|
|
11
|
+
- **Automatic Model Capabilities Discovery & Routing (#500)**:
|
|
12
|
+
- In `lib/sandbox.js` (`probeModels`), parse and return array of supported model IDs from upstream `/models` (`body.data.map(m => m.id)`).
|
|
13
|
+
- Discovered model capabilities are cached per key (`pool.state.modelCapabilities: Map<ref, Set<modelId>>`).
|
|
14
|
+
- In `lib/resolver.js` and `lib/rotate.js`, filter candidate keys for a requested model so prompts are never dispatched to a key that doesn't support that model (e.g., DeepSeek-R1 vs DeepSeek-Chat V3). Keys not yet probed are optimistically allowed.
|
|
15
|
+
- Atomic persistence across restarts: `pausedRefs`, `pauseReasons`, and `modelCapabilities` serialize and restore atomically via `lib/persistence.js`.
|
|
16
|
+
- Settings Card UI: Display `Auth Paused (401/403)` status badge and discovered model count badges (`🤖 Nm`) on configured keys.
|
|
17
|
+
|
|
18
|
+
## 0.8.63 - 2026-10-16
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- **Configurable Switch to Disable Chat Header Status Chip (#493 / GitHub #25)**:
|
|
22
|
+
- Added `showChatHeaderChip` boolean setting (defaults to `true` for 100% backward compatibility) in `SettingsSchema` and runtime configuration.
|
|
23
|
+
- Added user-facing toggle in the Settings card ("Chat Header Status Chip" / "聊天头部状态按钮") under rotation settings.
|
|
24
|
+
- `KeyRotationHeaderChip` (`conversation.session.header.utilities`) dynamically subscribes to settings and checks server health state, cleanly returning `null` when disabled without taking up conversational header toolbar space.
|
|
25
|
+
- Additive field `showChatHeaderChip` exposed in `/dsh-key-rotation/status` and `/dsh-key-rotation/health`.
|
|
26
|
+
|
|
3
27
|
## 0.8.62 - 2026-10-16
|
|
4
28
|
|
|
5
29
|
### Added
|
package/README.md
CHANGED
|
@@ -378,6 +378,7 @@ dsh-key-rotation:
|
|
|
378
378
|
| `circuitBreakerOpenMs` | `number` | `30000` | How long the circuit stays open (ms). |
|
|
379
379
|
| `circuitBreakerHalfOpenProbes` | `number` | `1` | Probe requests allowed in half-open state. |
|
|
380
380
|
| `verboseLogging` | `boolean` | `false` | Per-request rotation logs (noisy; off by default). |
|
|
381
|
+
| `showChatHeaderChip` | `boolean` | `true` | Show or hide the pool status button in conversation header toolbar (#25). |
|
|
381
382
|
| `concurrencyLimit` | `number` | `0` (disabled) | Max concurrent in-flight streams per key (0 = unlimited). |
|
|
382
383
|
| `quotaResetWindow` | `object` | `null` | Calendar reset alignment for both provider-reported quota cooldowns and local per-model token budgets (`midnight_utc`, `midnight_pst`, `rolling_24h`). |
|
|
383
384
|
| `cascade` | `array` | `[]` | Fallback provider chain when primary pool is completely exhausted. |
|
package/README.ru.md
CHANGED
|
@@ -409,6 +409,7 @@ MIT © [GooDAnDReaDY](https://github.com/GooDAnDReaDY)
|
|
|
409
409
|
| `circuitBreakerOpenMs` | number | `30000` | Длительность open, мс |
|
|
410
410
|
| `circuitBreakerHalfOpenProbes` | number | `1` | Проб в half-open |
|
|
411
411
|
| `verboseLogging` | boolean | `false` | Подробные логи rotation (шумно) |
|
|
412
|
+
| `showChatHeaderChip` | boolean | `true` | Показывать кнопку статуса пула в шапке чата (#25) |
|
|
412
413
|
|
|
413
414
|
Status API (v0.8.0): у каждого провайдера `circuit`, в корне ответа `meta.expectedClones` и `meta.notifyQueue`.
|
|
414
415
|
|
package/README.zh.md
CHANGED
|
@@ -378,3 +378,4 @@ MIT © [GooDAnDReaDY](https://github.com/GooDAnDReaDY)
|
|
|
378
378
|
| `circuitBreakerThreshold` | number | `5` | 连续失败阈值 |
|
|
379
379
|
| `circuitBreakerOpenMs` | number | `30000` | 打开时长 ms |
|
|
380
380
|
| `circuitBreakerHalfOpenProbes` | number | `1` | 半开探测次数 |
|
|
381
|
+
| `showChatHeaderChip` | boolean | `true` | 在会话界面顶部工具栏显示密钥池状态按钮 (#25) |
|
package/lib/client.js
CHANGED
|
@@ -98,6 +98,7 @@ window.__ModuleLoader__.load({
|
|
|
98
98
|
bulkImportSuccess: 'Imported {n} keys',
|
|
99
99
|
keyPaused: 'Paused',
|
|
100
100
|
keyRevoked: 'Revoked (401)',
|
|
101
|
+
keyAuthPaused: 'Auth Paused (401/403)',
|
|
101
102
|
pauseKey: 'Pause key from rotation',
|
|
102
103
|
resumeKey: 'Resume key rotation',
|
|
103
104
|
sparklineTraffic: 'Traffic Trend',
|
|
@@ -224,6 +225,8 @@ window.__ModuleLoader__.load({
|
|
|
224
225
|
routingStrategySticky: 'Sticky (preserves prompt cache)',
|
|
225
226
|
proactiveGuardLabel: 'Proactive Rate-Limit Guard',
|
|
226
227
|
proactiveGuardHint: 'Pre-emptively pause keys when remaining quota or Retry-After header indicates limit before hitting 429',
|
|
228
|
+
showChatHeaderChipLabel: 'Chat Header Status Chip',
|
|
229
|
+
showChatHeaderChipDesc: 'Show pool status chip in conversation header',
|
|
227
230
|
selfHealingLabel: 'Auto-Unbreak Interval (min)',
|
|
228
231
|
selfHealingHint: 'Periodic free model check to revive broken keys (0 to disable)',
|
|
229
232
|
updateCheck: 'Check for updates',
|
|
@@ -297,6 +300,7 @@ window.__ModuleLoader__.load({
|
|
|
297
300
|
bulkImportSuccess: '已导入 {n} 个密钥',
|
|
298
301
|
keyPaused: '已暂停',
|
|
299
302
|
keyRevoked: '已撤销 (401)',
|
|
303
|
+
keyAuthPaused: '身份验证失败已暂停 (401/403)',
|
|
300
304
|
pauseKey: '暂停此密钥轮换',
|
|
301
305
|
resumeKey: '恢复此密钥轮换',
|
|
302
306
|
sparklineTraffic: '流量趋势',
|
|
@@ -422,6 +426,8 @@ window.__ModuleLoader__.load({
|
|
|
422
426
|
routingStrategySticky: '粘性模式 (优先保持提示词缓存)',
|
|
423
427
|
proactiveGuardLabel: '前瞻性限流防护',
|
|
424
428
|
proactiveGuardHint: '当响应中的剩余配额或 Retry-After 头提示即将超限时,预先冷却密钥以防触发 429',
|
|
429
|
+
showChatHeaderChipLabel: '聊天头部状态按钮',
|
|
430
|
+
showChatHeaderChipDesc: '在会话界面顶部工具栏显示密钥池状态按钮',
|
|
425
431
|
selfHealingLabel: '自愈检查间隔 (分钟)',
|
|
426
432
|
selfHealingHint: '定期轻量探测以自动恢复损坏密钥 (0 为禁用)',
|
|
427
433
|
updateCheck: '检查更新',
|
|
@@ -1948,6 +1954,7 @@ window.__ModuleLoader__.load({
|
|
|
1948
1954
|
if (isPaused) return { color: 'var(--dsw-alias-state-warn-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1949
1955
|
const hit = keyInfo(providerId, ref);
|
|
1950
1956
|
if (!hit) return null;
|
|
1957
|
+
if (hit.authPaused || hit.pauseReason === 'AUTH_FAILURE_PAUSED') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyAuthPaused'), ready: false, authPaused: true };
|
|
1951
1958
|
if (hit.paused) return { color: 'var(--dsw-alias-state-warn-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1952
1959
|
if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired'), ready: false, expired: true };
|
|
1953
1960
|
if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked'), ready: false, revoked: true };
|
|
@@ -2120,6 +2127,14 @@ window.__ModuleLoader__.load({
|
|
|
2120
2127
|
if (info && info.broken) {
|
|
2121
2128
|
meta.push(h('button', { key: 'rt', className: 'krot-btn', onClick: () => retestBroken(key), title: t('retestBroken') }, t('retestBroken')));
|
|
2122
2129
|
}
|
|
2130
|
+
if (info && info.modelCapabilities && info.modelCapabilities.length > 0) {
|
|
2131
|
+
meta.push(h('span', {
|
|
2132
|
+
key: 'caps',
|
|
2133
|
+
className: 'krot-tail',
|
|
2134
|
+
title: 'Discovered models (' + info.modelCapabilities.length + '):\n' + info.modelCapabilities.join('\n'),
|
|
2135
|
+
style: { fontSize: '11px', opacity: 0.85 }
|
|
2136
|
+
}, '🤖 ' + info.modelCapabilities.length + 'm'));
|
|
2137
|
+
}
|
|
2123
2138
|
meta.push(h('span', { key: 'a', className: 'krot-acts' },
|
|
2124
2139
|
btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
|
|
2125
2140
|
btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),
|
|
@@ -2596,6 +2611,16 @@ window.__ModuleLoader__.load({
|
|
|
2596
2611
|
}),
|
|
2597
2612
|
h('span', { key: 'label', style: { fontSize: '13px', color: 'var(--dsw-alias-label-secondary)' } }, t('proactiveGuardLabel')),
|
|
2598
2613
|
]))
|
|
2614
|
+
),
|
|
2615
|
+
h('div', { className: 'krot-grid-2', style: { marginTop: '10px' } },
|
|
2616
|
+
field(t('showChatHeaderChipLabel'), h('label', { style: { display: 'flex', alignItems: 'center', gap: '8px', height: '36px', cursor: 'pointer' } }, [
|
|
2617
|
+
h('input', {
|
|
2618
|
+
key: 'input', type: 'checkbox',
|
|
2619
|
+
checked: val.showChatHeaderChip ?? true,
|
|
2620
|
+
onChange: (e) => setField((cur) => ({ ...cur, showChatHeaderChip: e.target.checked })),
|
|
2621
|
+
}),
|
|
2622
|
+
h('span', { key: 'label', style: { fontSize: '13px', color: 'var(--dsw-alias-label-secondary)' } }, t('showChatHeaderChipDesc')),
|
|
2623
|
+
]))
|
|
2599
2624
|
)
|
|
2600
2625
|
);
|
|
2601
2626
|
|
|
@@ -2886,6 +2911,7 @@ window.__ModuleLoader__.load({
|
|
|
2886
2911
|
const t = resolveT(props);
|
|
2887
2912
|
const [snap, setSnap] = React.useState(null);
|
|
2888
2913
|
const [open, setOpen] = React.useState(false);
|
|
2914
|
+
const [, setTick] = React.useState(0);
|
|
2889
2915
|
const ref = React.useRef(null);
|
|
2890
2916
|
|
|
2891
2917
|
React.useEffect(() => {
|
|
@@ -2895,7 +2921,7 @@ window.__ModuleLoader__.load({
|
|
|
2895
2921
|
fetch('/dsh-key-rotation/health', { headers: { accept: 'application/json' }, credentials: 'same-origin', signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(8000) : undefined })
|
|
2896
2922
|
.then((r) => (r.ok ? r.json() : null))
|
|
2897
2923
|
.then((d) => { if (alive) setSnap(d); })
|
|
2898
|
-
.catch(() => {});
|
|
2924
|
+
.catch((e) => { void e; });
|
|
2899
2925
|
};
|
|
2900
2926
|
load();
|
|
2901
2927
|
const id = setInterval(load, 5000);
|
|
@@ -2908,6 +2934,13 @@ window.__ModuleLoader__.load({
|
|
|
2908
2934
|
};
|
|
2909
2935
|
}, []);
|
|
2910
2936
|
|
|
2937
|
+
React.useEffect(() => {
|
|
2938
|
+
if (!props?.settingsSource?.subscribe) return;
|
|
2939
|
+
return props.settingsSource.subscribe(() => {
|
|
2940
|
+
setTick((c) => c + 1);
|
|
2941
|
+
});
|
|
2942
|
+
}, [props?.settingsSource]);
|
|
2943
|
+
|
|
2911
2944
|
React.useEffect(() => {
|
|
2912
2945
|
if (!open) return;
|
|
2913
2946
|
const onDocClick = (e) => {
|
|
@@ -2917,6 +2950,16 @@ window.__ModuleLoader__.load({
|
|
|
2917
2950
|
return () => document.removeEventListener('click', onDocClick);
|
|
2918
2951
|
}, [open]);
|
|
2919
2952
|
|
|
2953
|
+
let settingsVal = null;
|
|
2954
|
+
try {
|
|
2955
|
+
const sSnap = props?.settingsSource?.getSnapshot?.()?.service?.getSnapshot?.();
|
|
2956
|
+
settingsVal = sSnap?.value;
|
|
2957
|
+
} catch (e) {
|
|
2958
|
+
void e;
|
|
2959
|
+
}
|
|
2960
|
+
if (settingsVal && settingsVal.showChatHeaderChip === false) return null;
|
|
2961
|
+
if (snap && snap.showChatHeaderChip === false) return null;
|
|
2962
|
+
|
|
2920
2963
|
const poolsObj = (snap && snap.pools) || {};
|
|
2921
2964
|
const poolEntries = Object.entries(poolsObj);
|
|
2922
2965
|
const total = poolEntries.reduce((a, [, p]) => a + (p.total || 0), 0);
|
|
@@ -2952,7 +2995,6 @@ window.__ModuleLoader__.load({
|
|
|
2952
2995
|
) : null
|
|
2953
2996
|
);
|
|
2954
2997
|
}
|
|
2955
|
-
|
|
2956
2998
|
function apply(ctx) {
|
|
2957
2999
|
const settingsSource = createSettingsSource(ctx);
|
|
2958
3000
|
ctx.effect(() => ctx.locale.register(NS, { en, zh }), 'dsh-key-rotation: dictionaries');
|
|
@@ -2963,7 +3005,7 @@ window.__ModuleLoader__.load({
|
|
|
2963
3005
|
try {
|
|
2964
3006
|
ctx.slots.inject('conversation.session.header.utilities', () =>
|
|
2965
3007
|
ctx.slots.register(
|
|
2966
|
-
{ name: 'conversation.session.header.utilities', id: 'dsh-key-rotation-header-chip', order: 6, locale: NS },
|
|
3008
|
+
{ name: 'conversation.session.header.utilities', id: 'dsh-key-rotation-header-chip', order: 6, locale: NS, inject: () => ({ ctx, settingsSource }) },
|
|
2967
3009
|
KeyRotationHeaderChip,
|
|
2968
3010
|
));
|
|
2969
3011
|
} catch { /* slot not available in this build */ }
|
package/lib/error-taxonomy.js
CHANGED
|
@@ -36,9 +36,11 @@ export function classifyFailure(failureOrPayload) {
|
|
|
36
36
|
if (status === 429 || code === 'RATE_LIMIT' || code === 'RESOURCE_EXHAUSTED' || code === 'QUOTA') {
|
|
37
37
|
return { action: 'switch', code: status ? '429' : (code || 'RATE_LIMIT'), soft: false, reason: 'quota' };
|
|
38
38
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
39
|
+
const numCode = Number(code);
|
|
40
|
+
if (SWITCH_STATUS.has(status) || (numCode > 0 && SWITCH_STATUS.has(numCode)) || SOFT_FAILURE_CODES.has(code)) {
|
|
41
|
+
const effCode = status ? String(status) : (numCode > 0 ? String(numCode) : code);
|
|
42
|
+
const soft = SOFT_FAILURE_CODES.has(effCode);
|
|
43
|
+
return { action: 'switch', code: effCode, soft, reason: 'http' };
|
|
42
44
|
}
|
|
43
45
|
if (SURFACE_STATUS.has(status)) {
|
|
44
46
|
return { action: 'surface', code: String(status), soft: false, reason: 'client' };
|
package/lib/heal.js
CHANGED
|
@@ -90,6 +90,10 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
|
|
|
90
90
|
pool.state.failedUntil?.delete(ref);
|
|
91
91
|
pool.state.failCounts?.delete(ref);
|
|
92
92
|
pool.state.authFailCounts?.delete(ref);
|
|
93
|
+
if (Array.isArray(res.models) && res.models.length > 0) {
|
|
94
|
+
if (!pool.state.modelCapabilities) pool.state.modelCapabilities = new Map();
|
|
95
|
+
pool.state.modelCapabilities.set(ref, new Set(res.models));
|
|
96
|
+
}
|
|
93
97
|
if (Array.isArray(pool.state.events)) {
|
|
94
98
|
pool.state.events.push({ at: now, ref, reason: 'auto-unbreak', cooldownMs: 0, type: 'heal' });
|
|
95
99
|
if (pool.state.events.length > 50) pool.state.events.shift();
|
package/lib/index.js
CHANGED
|
@@ -119,6 +119,7 @@ const SettingsSchema = Schema.object({
|
|
|
119
119
|
earlyWarningKeyThreshold: Schema.number().default(0.25),
|
|
120
120
|
warnBelowHealthy: Schema.number().default(0),
|
|
121
121
|
latencySloMs: Schema.number().default(0),
|
|
122
|
+
showChatHeaderChip: Schema.boolean().default(true),
|
|
122
123
|
providers: Schema.array(Schema.object({
|
|
123
124
|
provider: Schema.string().required(),
|
|
124
125
|
keys: Schema.array(Schema.string()).default([]),
|
|
@@ -350,6 +351,7 @@ export function apply(ctx, config = {}) {
|
|
|
350
351
|
proactiveRateLimitGuard: cfg.proactiveRateLimitGuard ?? true,
|
|
351
352
|
selfHealingIntervalMinutes: cfg.selfHealingIntervalMinutes ?? 30,
|
|
352
353
|
routingStrategy: cfg.routingStrategy ?? 'round-robin',
|
|
354
|
+
showChatHeaderChip: cfg.showChatHeaderChip !== false,
|
|
353
355
|
switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000,
|
|
354
356
|
warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0,
|
|
355
357
|
index, pools,
|
|
@@ -375,6 +377,7 @@ export function apply(ctx, config = {}) {
|
|
|
375
377
|
buildRuntime,
|
|
376
378
|
currentPool: () => dispatchStorage.getStore()?.pool,
|
|
377
379
|
currentSession: () => dispatchStorage.getStore()?.sessionId,
|
|
380
|
+
currentModel: () => dispatchStorage.getStore()?.model,
|
|
378
381
|
onPicked: (pool, candidate, meta) => {
|
|
379
382
|
const store = dispatchStorage.getStore();
|
|
380
383
|
if (store && store.pool === pool) {
|
package/lib/ops-status.js
CHANGED
|
@@ -120,7 +120,9 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
120
120
|
expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
|
|
121
121
|
broken: pool.state.brokenUntil?.has(ref) ?? false,
|
|
122
122
|
paused: isKeyPaused(pool, ref),
|
|
123
|
+
pauseReason: pool.state.pauseReasons?.get?.(ref) ?? null,
|
|
123
124
|
revoked: isKeyRevoked(pool, ref),
|
|
125
|
+
modelCapabilities: pool.state.modelCapabilities?.get?.(ref) ? [...pool.state.modelCapabilities.get(ref)] : null,
|
|
124
126
|
modelQuota,
|
|
125
127
|
modelPools: activeModelPools.map(mp => mp.base),
|
|
126
128
|
});
|
|
@@ -178,6 +180,7 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
178
180
|
}
|
|
179
181
|
json(res, 200, sanitizeSnapshot({
|
|
180
182
|
providers,
|
|
183
|
+
showChatHeaderChip: runtime.showChatHeaderChip !== false,
|
|
181
184
|
// #266/#263 operational extras (additive)
|
|
182
185
|
meta: {
|
|
183
186
|
expectedClones: [...(runtime.expectedClones ?? [])],
|
|
@@ -248,7 +251,7 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
248
251
|
}
|
|
249
252
|
}
|
|
250
253
|
const isDegraded = exhaustedAny || circuitOpenAny;
|
|
251
|
-
json(res, 200, { status: isDegraded ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
|
|
254
|
+
json(res, 200, { status: isDegraded ? 'degraded' : 'ok', showChatHeaderChip: runtimeH.showChatHeaderChip !== false, pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
|
|
252
255
|
},
|
|
253
256
|
}), 'dsh-key-rotation: health');
|
|
254
257
|
|
package/lib/ops-test.js
CHANGED
|
@@ -73,16 +73,24 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
73
73
|
lastTestCache.set(ref, cached);
|
|
74
74
|
if (cached.ok) {
|
|
75
75
|
for (const st of poolState.values()) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
76
|
+
st.failedUntil?.delete(ref);
|
|
77
|
+
st.failCounts?.delete(ref);
|
|
78
|
+
st.authFailCounts?.delete(ref);
|
|
79
|
+
st.brokenUntil?.delete(ref);
|
|
80
|
+
st.revokedRefs?.delete(ref);
|
|
81
|
+
if (st.pausedRefs instanceof Set) st.pausedRefs.delete(ref);
|
|
82
|
+
else if (Array.isArray(st.pausedRefs)) {
|
|
83
|
+
const pIdx = st.pausedRefs.indexOf(ref);
|
|
84
|
+
if (pIdx >= 0) st.pausedRefs.splice(pIdx, 1);
|
|
85
|
+
}
|
|
86
|
+
st.pauseReasons?.delete(ref);
|
|
87
|
+
if (Array.isArray(cached.models) && cached.models.length > 0) {
|
|
88
|
+
if (!st.modelCapabilities) st.modelCapabilities = new Map();
|
|
89
|
+
st.modelCapabilities.set(ref, new Set(cached.models));
|
|
82
90
|
}
|
|
83
91
|
}
|
|
84
92
|
}
|
|
85
|
-
json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
|
|
93
|
+
json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount, models: cached.models ?? [] });
|
|
86
94
|
return;
|
|
87
95
|
}
|
|
88
96
|
json(res, 200, { ok: true, ref, tail, source });
|
|
@@ -179,6 +187,16 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
179
187
|
st.authFailCounts?.delete(ref);
|
|
180
188
|
st.brokenUntil?.delete(ref);
|
|
181
189
|
st.revokedRefs?.delete(ref);
|
|
190
|
+
if (st.pausedRefs instanceof Set) st.pausedRefs.delete(ref);
|
|
191
|
+
else if (Array.isArray(st.pausedRefs)) {
|
|
192
|
+
const pIdx = st.pausedRefs.indexOf(ref);
|
|
193
|
+
if (pIdx >= 0) st.pausedRefs.splice(pIdx, 1);
|
|
194
|
+
}
|
|
195
|
+
st.pauseReasons?.delete(ref);
|
|
196
|
+
if (Array.isArray(cached.models) && cached.models.length > 0) {
|
|
197
|
+
if (!st.modelCapabilities) st.modelCapabilities = new Map();
|
|
198
|
+
st.modelCapabilities.set(ref, new Set(cached.models));
|
|
199
|
+
}
|
|
182
200
|
}
|
|
183
201
|
}
|
|
184
202
|
results.push({
|
|
@@ -189,6 +207,7 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
189
207
|
code: cached.code,
|
|
190
208
|
latencyMs: cached.latencyMs,
|
|
191
209
|
modelsCount: cached.modelsCount,
|
|
210
|
+
models: cached.models ?? [],
|
|
192
211
|
});
|
|
193
212
|
} catch (err) {
|
|
194
213
|
results.push({
|
package/lib/persistence.js
CHANGED
|
@@ -44,6 +44,32 @@ export class StatePersistence {
|
|
|
44
44
|
revokedRefs.push(...st.revokedRefs);
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
+
const pausedRefs = [];
|
|
48
|
+
if (st && st.pausedRefs) {
|
|
49
|
+
if (typeof st.pausedRefs.forEach === 'function') {
|
|
50
|
+
st.pausedRefs.forEach((r) => pausedRefs.push(r));
|
|
51
|
+
} else if (Array.isArray(st.pausedRefs)) {
|
|
52
|
+
pausedRefs.push(...st.pausedRefs);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const pauseReasons = {};
|
|
56
|
+
if (st && st.pauseReasons && typeof st.pauseReasons.forEach === 'function') {
|
|
57
|
+
st.pauseReasons.forEach((reason, ref) => {
|
|
58
|
+
if (typeof reason === 'string') pauseReasons[ref] = reason;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const modelCapabilities = {};
|
|
62
|
+
if (st && st.modelCapabilities && typeof st.modelCapabilities.forEach === 'function') {
|
|
63
|
+
st.modelCapabilities.forEach((caps, ref) => {
|
|
64
|
+
if (caps && typeof caps.forEach === 'function') {
|
|
65
|
+
const arr = [];
|
|
66
|
+
caps.forEach((m) => { if (typeof m === 'string') arr.push(m); });
|
|
67
|
+
if (arr.length > 0) modelCapabilities[ref] = arr;
|
|
68
|
+
} else if (Array.isArray(caps) && caps.length > 0) {
|
|
69
|
+
modelCapabilities[ref] = caps.filter((m) => typeof m === 'string');
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
47
73
|
const tokenUsage = {};
|
|
48
74
|
if (st && st.tokenUsage && typeof st.tokenUsage.forEach === 'function') {
|
|
49
75
|
st.tokenUsage.forEach((entry, ref) => {
|
|
@@ -109,6 +135,9 @@ export class StatePersistence {
|
|
|
109
135
|
lastUsed: typeof st?.lastUsed === 'string' ? st.lastUsed : null,
|
|
110
136
|
};
|
|
111
137
|
if (revokedRefs.length > 0) outPool.revokedRefs = revokedRefs;
|
|
138
|
+
if (pausedRefs.length > 0) outPool.pausedRefs = pausedRefs;
|
|
139
|
+
if (Object.keys(pauseReasons).length > 0) outPool.pauseReasons = pauseReasons;
|
|
140
|
+
if (Object.keys(modelCapabilities).length > 0) outPool.modelCapabilities = modelCapabilities;
|
|
112
141
|
if (Object.keys(tokenUsage).length > 0) outPool.tokenUsage = tokenUsage;
|
|
113
142
|
if (Object.keys(costDays).length > 0) outPool.costDays = costDays;
|
|
114
143
|
if (Object.keys(costPerKey).length > 0) outPool.costPerKey = costPerKey;
|
|
@@ -200,6 +229,26 @@ export class StatePersistence {
|
|
|
200
229
|
if (typeof r === 'string') state.revokedRefs.add(r);
|
|
201
230
|
}
|
|
202
231
|
}
|
|
232
|
+
if (Array.isArray(saved.pausedRefs)) {
|
|
233
|
+
if (!state.pausedRefs) state.pausedRefs = new Set();
|
|
234
|
+
for (const r of saved.pausedRefs) {
|
|
235
|
+
if (typeof r === 'string') state.pausedRefs.add(r);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (saved.pauseReasons && typeof saved.pauseReasons === 'object' && !Array.isArray(saved.pauseReasons)) {
|
|
239
|
+
if (!state.pauseReasons) state.pauseReasons = new Map();
|
|
240
|
+
for (const [ref, reason] of Object.entries(saved.pauseReasons)) {
|
|
241
|
+
if (typeof reason === 'string') state.pauseReasons.set(ref, reason);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (saved.modelCapabilities && typeof saved.modelCapabilities === 'object' && !Array.isArray(saved.modelCapabilities)) {
|
|
245
|
+
if (!state.modelCapabilities) state.modelCapabilities = new Map();
|
|
246
|
+
for (const [ref, list] of Object.entries(saved.modelCapabilities)) {
|
|
247
|
+
if (Array.isArray(list)) {
|
|
248
|
+
state.modelCapabilities.set(ref, new Set(list.filter((m) => typeof m === 'string')));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
203
252
|
if (saved.tokenUsage && typeof saved.tokenUsage === 'object' && !Array.isArray(saved.tokenUsage)) {
|
|
204
253
|
for (const [ref, entry] of Object.entries(saved.tokenUsage)) {
|
|
205
254
|
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
|
package/lib/pool-state.js
CHANGED
|
@@ -3,13 +3,18 @@
|
|
|
3
3
|
const MAP_FIELDS = [
|
|
4
4
|
'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
|
|
5
5
|
'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
|
|
6
|
-
'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky',
|
|
6
|
+
'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky', 'modelCapabilities', 'pauseReasons',
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
+
const SET_FIELDS = ['pausedRefs', 'revokedRefs'];
|
|
10
|
+
|
|
9
11
|
export function initializePoolState(state = {}) {
|
|
10
12
|
for (const key of MAP_FIELDS) {
|
|
11
13
|
if (!(state[key] instanceof Map)) state[key] = new Map();
|
|
12
14
|
}
|
|
15
|
+
for (const key of SET_FIELDS) {
|
|
16
|
+
if (!(state[key] instanceof Set)) state[key] = new Set(Array.isArray(state[key]) ? state[key] : []);
|
|
17
|
+
}
|
|
13
18
|
if (!Array.isArray(state.events)) state.events = [];
|
|
14
19
|
if (!Number.isSafeInteger(state.pointer) || state.pointer < 0) state.pointer = 0;
|
|
15
20
|
if (typeof state.lastUsed !== 'string') state.lastUsed = null;
|
|
@@ -21,5 +26,7 @@ export function initializePoolState(state = {}) {
|
|
|
21
26
|
export function hasPoolActivity(state) {
|
|
22
27
|
return state.pointer !== 0 || state.lastUsed !== null || state.events.length > 0 ||
|
|
23
28
|
MAP_FIELDS.some((key) => state[key].size > 0) ||
|
|
24
|
-
(state.lastSuccessAt?.size ?? 0) > 0
|
|
29
|
+
(state.lastSuccessAt?.size ?? 0) > 0 ||
|
|
30
|
+
(state.pausedRefs?.size ?? 0) > 0 ||
|
|
31
|
+
(state.revokedRefs?.size ?? 0) > 0;
|
|
25
32
|
}
|
package/lib/resolver.js
CHANGED
|
@@ -77,6 +77,7 @@ export function createResolver({
|
|
|
77
77
|
buildRuntime,
|
|
78
78
|
currentPool,
|
|
79
79
|
currentSession,
|
|
80
|
+
currentModel,
|
|
80
81
|
onPicked,
|
|
81
82
|
latencyHistogram,
|
|
82
83
|
concurrencyTracker,
|
|
@@ -88,6 +89,7 @@ export function createResolver({
|
|
|
88
89
|
// pool that rotate() actually dispatched with decides ownership. A bare
|
|
89
90
|
// resolve outside rotation falls back to the global lookup.
|
|
90
91
|
const requested = currentPool ? currentPool() ?? null : null;
|
|
92
|
+
const model = resolveOpts?.model ?? currentModel?.() ?? requested?.model ?? null;
|
|
91
93
|
const pool = resolveRefPool(runtime.index, ref, requested);
|
|
92
94
|
if (!pool) {
|
|
93
95
|
const res = await original(ref);
|
|
@@ -168,6 +170,11 @@ export function createResolver({
|
|
|
168
170
|
if (until !== undefined && until > at) continue;
|
|
169
171
|
const candidateExp = pool.expiresAt?.[candidate] ?? pool.basePool?.expiresAt?.[candidate];
|
|
170
172
|
if (candidateExp !== undefined && at >= candidateExp) continue;
|
|
173
|
+
// Issue #500: skip candidate if modelCapabilities is known and does not include requested model
|
|
174
|
+
if (model && pool.state.modelCapabilities) {
|
|
175
|
+
const caps = pool.state.modelCapabilities.get(candidate);
|
|
176
|
+
if (caps && caps.size > 0 && !caps.has(model)) continue;
|
|
177
|
+
}
|
|
171
178
|
// Local model token budget: a spent credential is simply not a candidate.
|
|
172
179
|
if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
|
|
173
180
|
const microThrottling = pool.microThrottling ?? runtime.microThrottling ?? false;
|
package/lib/rotate.js
CHANGED
|
@@ -108,12 +108,19 @@ export function createRotate(deps) {
|
|
|
108
108
|
const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
|
|
109
109
|
let activePermit = null;
|
|
110
110
|
const sessionId = options?.sessionId ?? options?.session ?? options?.metadata?.sessionId ?? options?.chatId ?? options?.conversationId;
|
|
111
|
-
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId };
|
|
111
|
+
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId, model: options.model };
|
|
112
112
|
setRotateStartMs(reqStore.startMs);
|
|
113
113
|
const wallNow = Date.now();
|
|
114
114
|
const quotaManaged = hasModelQuotaConfig(pool);
|
|
115
115
|
let attemptList = (pool.weightedRefs ?? pool.refs).slice();
|
|
116
116
|
attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow) && !isKeyDraining(r));
|
|
117
|
+
// Issue #500: filter candidate keys by confirmed model capabilities
|
|
118
|
+
if (options.model && pool.state.modelCapabilities) {
|
|
119
|
+
attemptList = attemptList.filter((ref) => {
|
|
120
|
+
const caps = pool.state.modelCapabilities.get(ref);
|
|
121
|
+
return !caps || caps.size === 0 || caps.has(options.model);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
117
124
|
if (pool.pinnedRef && attemptList.includes(pool.pinnedRef)) {
|
|
118
125
|
const pIdx = attemptList.indexOf(pool.pinnedRef);
|
|
119
126
|
if (pIdx > 0) {
|
|
@@ -196,7 +203,7 @@ export function createRotate(deps) {
|
|
|
196
203
|
const _base = pool.cooldownMs ?? cooldownMs;
|
|
197
204
|
const _max = pool.maxCooldownMs ?? maxCooldownMs;
|
|
198
205
|
const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
|
|
199
|
-
const cls = classifyFailure({ code: errCode, message: errMsg });
|
|
206
|
+
const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
|
|
200
207
|
const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
|
|
201
208
|
|
|
202
209
|
// #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
|
|
@@ -245,16 +252,26 @@ export function createRotate(deps) {
|
|
|
245
252
|
}
|
|
246
253
|
if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
|
|
247
254
|
if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
|
|
248
|
-
const _cStr = String(errCode ?? '');
|
|
249
|
-
|
|
255
|
+
const _cStr = String(errCode ?? '').toUpperCase();
|
|
256
|
+
const _isAuth = cls.reason === 'auth' || cls.code === '401' || cls.code === '403' ||
|
|
257
|
+
_cStr === 'AUTH' || _cStr === '401' || _cStr === '403' ||
|
|
258
|
+
_cStr === 'PERMISSION_DENIED' || _cStr === 'UNAUTHENTICATED' ||
|
|
259
|
+
/auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated|permission[_\s-]?denied/i.test(errMsg);
|
|
260
|
+
|
|
261
|
+
if (_isAuth) {
|
|
250
262
|
const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
|
|
251
263
|
pool.state.authFailCounts.set(targetRef, _c2);
|
|
252
264
|
if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
|
|
253
265
|
pool.state.revokedRefs.add(targetRef);
|
|
266
|
+
if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
|
|
267
|
+
if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(targetRef);
|
|
268
|
+
else if (Array.isArray(pool.state.pausedRefs) && !pool.state.pausedRefs.includes(targetRef)) pool.state.pausedRefs.push(targetRef);
|
|
269
|
+
if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
|
|
270
|
+
pool.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
|
|
254
271
|
pool.state.failedUntil.set(targetRef, Infinity);
|
|
255
|
-
pushEvent(pool, targetRef, '
|
|
256
|
-
logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently
|
|
257
|
-
// Fan-out
|
|
272
|
+
pushEvent(pool, targetRef, 'KEY_AUTH_PAUSED', 0);
|
|
273
|
+
logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently PAUSED on auth failure (${errCode})`);
|
|
274
|
+
// Fan-out permanent pause across all pools sharing this physical ref (#408):
|
|
258
275
|
const rAll = buildRuntime();
|
|
259
276
|
const candidatePools = [];
|
|
260
277
|
if (Array.isArray(rAll.pools)) candidatePools.push(...rAll.pools);
|
|
@@ -275,8 +292,13 @@ export function createRotate(deps) {
|
|
|
275
292
|
if (hasRef && other.state) {
|
|
276
293
|
if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
|
|
277
294
|
other.state.revokedRefs.add(targetRef);
|
|
295
|
+
if (!other.state.pausedRefs) other.state.pausedRefs = new Set();
|
|
296
|
+
if (other.state.pausedRefs instanceof Set) other.state.pausedRefs.add(targetRef);
|
|
297
|
+
else if (Array.isArray(other.state.pausedRefs) && !other.state.pausedRefs.includes(targetRef)) other.state.pausedRefs.push(targetRef);
|
|
298
|
+
if (!other.state.pauseReasons) other.state.pauseReasons = new Map();
|
|
299
|
+
other.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
|
|
278
300
|
if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
|
|
279
|
-
pushEvent(other, targetRef, '
|
|
301
|
+
pushEvent(other, targetRef, 'KEY_AUTH_PAUSED', 0);
|
|
280
302
|
}
|
|
281
303
|
}
|
|
282
304
|
} else {
|
package/lib/sandbox.js
CHANGED
|
@@ -107,11 +107,15 @@ export class SandboxRunner {
|
|
|
107
107
|
const status = res.status;
|
|
108
108
|
if (status >= 200 && status < 300) {
|
|
109
109
|
let modelsCount = 0;
|
|
110
|
+
let models = [];
|
|
110
111
|
try {
|
|
111
112
|
const body = await res.json();
|
|
112
|
-
|
|
113
|
+
if (Array.isArray(body && body.data)) {
|
|
114
|
+
modelsCount = body.data.length;
|
|
115
|
+
models = body.data.map((m) => (typeof m === 'string' ? m : (m && m.id))).filter(Boolean);
|
|
116
|
+
}
|
|
113
117
|
} catch (_) { /* not json */ }
|
|
114
|
-
return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount };
|
|
118
|
+
return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount, models };
|
|
115
119
|
}
|
|
116
120
|
return { ok: false, code: classifyStatus(status), latencyMs: Date.now() - started };
|
|
117
121
|
} finally {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.64",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|