@goodandready/dsh-key-rotation 0.8.60 → 0.8.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/lib/client.js +192 -0
- package/lib/heal.js +22 -10
- package/lib/index.js +4 -3
- package/lib/lifecycle.js +4 -1
- package/lib/ops-keys.js +239 -0
- package/lib/ops-paths.js +3 -0
- package/lib/ops-test.js +111 -0
- package/lib/pool-state.js +1 -1
- package/lib/pool.js +45 -3
- package/lib/resolver.js +35 -8
- package/lib/rotate.js +26 -7
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.62 - 2026-10-16
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- **Session-Aware Sticky Routing Affinity for Prompt Caching (#489)**:
|
|
7
|
+
- Enhanced `sticky` routing strategy with LRU session affinity (`pool.state.sessionSticky`, capped at 500 active sessions).
|
|
8
|
+
- Parallel DSH chat sessions or multi-agent subtasks retain dedicated sticky keys based on `sessionId` (`options.sessionId` / `options.session` / `options.metadata.sessionId`), preventing cross-session cache evictions and maximizing prompt prefix cache hit rates for Anthropic, DeepSeek, and OpenAI.
|
|
9
|
+
- Graceful failover to the next healthy candidate if a session's pinned key is in cooldown or quota-exhausted; automatic eviction from session map on error.
|
|
10
|
+
- **1-Click Test Matrix Dashboard (#490)**:
|
|
11
|
+
- Added `/dsh-key-rotation/test-matrix` endpoint in `lib/ops-test.js` supporting batch validation of all configured keys across all pools with bounded concurrency (5 workers) and latency tracking.
|
|
12
|
+
- Added "⚡ Test All Keys" action button in the WebUI Settings Card with an interactive results modal displaying operational key status, latency in milliseconds, and error details.
|
|
13
|
+
- **Zero-Config Pool Auto-Discovery & 1-Click Import (#491)**:
|
|
14
|
+
- Added `/dsh-key-rotation/discover` and `/dsh-key-rotation/auto-import` endpoints in `lib/ops-keys.js`.
|
|
15
|
+
- Automatically scans `~/.dsh/.credentials.yaml` and environment variables for unconfigured provider keys matching standard patterns (`DEEPSEEK_API_KEY`, `OPENCODE_GO_API_KEY_*`, `OPENAI_API_KEY`, `GROQ_API_KEY`, etc.), excluding keys already assigned to rotation pools.
|
|
16
|
+
- Added smart auto-discovery banner to the Settings Card with 1-click import into provider pools.
|
|
17
|
+
|
|
18
|
+
## 0.8.61 - 2026-10-15
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
- **Prevent Transient Rate Limit Deadlocks from Calendar Quota Locks (#486 / GitHub #24)**:
|
|
22
|
+
- Upstream errors returning transient 429, RPM/TPM exhaustion, or concurrency limits (such as SenseNova `{"message":"rpm exhausted","type":"quota_exceeded_error"}` or `{"message":"inference exceeds tpm/rpm limit","code":"insufficient_quota"}`) are now classified as `isRateThrottling` and excluded from `isQuota`.
|
|
23
|
+
- Transient rate limits receive standard short backoff (`cooldownMs` with jitter) rather than being locked out until next UTC midnight.
|
|
24
|
+
- Cleared `pool.state.quotaWindows` in `recordSuccess` upon first successful request.
|
|
25
|
+
- Extended background `autoUnbreakBrokenKeys` and proactive health probing to probe keys with calendar quota locks, restoring healthy keys immediately upon receiving 200 OK from upstream.
|
|
26
|
+
- Added `INSUFFICIENT_QUOTA` and present-tense regex matches to switchable error classification.
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- **Sticky Key Routing Strategy to Preserve Prompt Caching (#487 / GitHub #23)**:
|
|
30
|
+
- Added `'sticky'` routing strategy option in settings and provider configurations.
|
|
31
|
+
- In `'sticky'` mode, the resolver retains the same active key across consecutive requests as long as it remains healthy, maximizing prefix prompt cache hit rate on providers that cache per API key (Anthropic, DeepSeek, OpenAI).
|
|
32
|
+
- Automatically fails over to the next healthy key in the pool if the sticky key experiences an error or cooldown, then sticks to the new key.
|
|
33
|
+
- Added UI dropdown selection in the Key Rotation Settings Card with English and Chinese localizations.
|
|
34
|
+
|
|
3
35
|
## 0.8.60 - 2026-10-14
|
|
4
36
|
|
|
5
37
|
### Changed
|
package/lib/client.js
CHANGED
|
@@ -221,6 +221,7 @@ window.__ModuleLoader__.load({
|
|
|
221
221
|
routingStrategyRoundRobin: 'Round-Robin (default)',
|
|
222
222
|
routingStrategyLeastLoaded: 'Least-Loaded (concurrency)',
|
|
223
223
|
routingStrategyLowestLatency: 'Lowest-Latency (p95)',
|
|
224
|
+
routingStrategySticky: 'Sticky (preserves prompt cache)',
|
|
224
225
|
proactiveGuardLabel: 'Proactive Rate-Limit Guard',
|
|
225
226
|
proactiveGuardHint: 'Pre-emptively pause keys when remaining quota or Retry-After header indicates limit before hitting 429',
|
|
226
227
|
selfHealingLabel: 'Auto-Unbreak Interval (min)',
|
|
@@ -234,6 +235,19 @@ window.__ModuleLoader__.load({
|
|
|
234
235
|
updateFailed: 'Update failed',
|
|
235
236
|
currentVersion: 'Current version',
|
|
236
237
|
latestVersion: 'Latest version',
|
|
238
|
+
testAllKeys: 'Test All Keys',
|
|
239
|
+
testingMatrix: 'Testing Keys…',
|
|
240
|
+
matrixModalTitle: '⚡ Key Test Matrix',
|
|
241
|
+
matrixModalDesc: 'Parallel probe of all configured credentials across all provider pools.',
|
|
242
|
+
matrixSummary: '{healthy}/{total} keys operational',
|
|
243
|
+
matrixLatency: 'Latency',
|
|
244
|
+
matrixStatus: 'Status',
|
|
245
|
+
matrixClose: 'Close',
|
|
246
|
+
discoverBannerTitle: '⚡ Unassigned API Keys Discovered',
|
|
247
|
+
discoverBannerDesc: 'Found {n} credentials in your environment (~/.dsh/.credentials.yaml / env) not yet assigned to any rotation pool.',
|
|
248
|
+
autoImportBtn: '1-Click Auto-Import',
|
|
249
|
+
autoImportSuccess: 'Imported {n} keys into rotation pools',
|
|
250
|
+
autoImporting: 'Importing…',
|
|
237
251
|
};
|
|
238
252
|
|
|
239
253
|
const zh = {
|
|
@@ -405,6 +419,7 @@ window.__ModuleLoader__.load({
|
|
|
405
419
|
routingStrategyRoundRobin: '轮询 (默认)',
|
|
406
420
|
routingStrategyLeastLoaded: '最低负载 (并发)',
|
|
407
421
|
routingStrategyLowestLatency: '最低延迟 (p95)',
|
|
422
|
+
routingStrategySticky: '粘性模式 (优先保持提示词缓存)',
|
|
408
423
|
proactiveGuardLabel: '前瞻性限流防护',
|
|
409
424
|
proactiveGuardHint: '当响应中的剩余配额或 Retry-After 头提示即将超限时,预先冷却密钥以防触发 429',
|
|
410
425
|
selfHealingLabel: '自愈检查间隔 (分钟)',
|
|
@@ -418,6 +433,19 @@ window.__ModuleLoader__.load({
|
|
|
418
433
|
updateFailed: '更新失败',
|
|
419
434
|
currentVersion: '当前版本',
|
|
420
435
|
latestVersion: '最新版本',
|
|
436
|
+
testAllKeys: '批量测试所有密钥',
|
|
437
|
+
testingMatrix: '正在测试密钥…',
|
|
438
|
+
matrixModalTitle: '⚡ 密钥测试矩阵',
|
|
439
|
+
matrixModalDesc: '并行探测所有提供商密钥池中配置的凭据。',
|
|
440
|
+
matrixSummary: '{healthy}/{total} 个密钥正常可用',
|
|
441
|
+
matrixLatency: '延迟',
|
|
442
|
+
matrixStatus: '状态',
|
|
443
|
+
matrixClose: '关闭',
|
|
444
|
+
discoverBannerTitle: '⚡ 发现未分配的 API 密钥',
|
|
445
|
+
discoverBannerDesc: '在当前环境 (~/.dsh/.credentials.yaml / env) 中发现 {n} 个未分配给轮换池的密钥。',
|
|
446
|
+
autoImportBtn: '一键自动导入',
|
|
447
|
+
autoImportSuccess: '成功导入 {n} 个密钥到轮换池',
|
|
448
|
+
autoImporting: '正在导入…',
|
|
421
449
|
};
|
|
422
450
|
|
|
423
451
|
// Codes worth switching on. Host list is DEFAULT_SWITCH_CODES; config may
|
|
@@ -1125,6 +1153,73 @@ window.__ModuleLoader__.load({
|
|
|
1125
1153
|
const [simResult, setSimResult] = React.useState(null);
|
|
1126
1154
|
const [simError, setSimError] = React.useState('');
|
|
1127
1155
|
|
|
1156
|
+
// Auto-Discovery State (#491)
|
|
1157
|
+
const [discoveredKeys, setDiscoveredKeys] = React.useState([]);
|
|
1158
|
+
const [autoImporting, setAutoImporting] = React.useState(false);
|
|
1159
|
+
const [autoImportMsg, setAutoImportMsg] = React.useState('');
|
|
1160
|
+
|
|
1161
|
+
// Test Matrix State (#490)
|
|
1162
|
+
const [testMatrixRunning, setTestMatrixRunning] = React.useState(false);
|
|
1163
|
+
const [testMatrixResults, setTestMatrixResults] = React.useState(null);
|
|
1164
|
+
const [testMatrixModal, setTestMatrixModal] = React.useState(false);
|
|
1165
|
+
|
|
1166
|
+
React.useEffect(() => {
|
|
1167
|
+
let mounted = true;
|
|
1168
|
+
fetch('/dsh-key-rotation/discover', {
|
|
1169
|
+
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(5000) : undefined,
|
|
1170
|
+
})
|
|
1171
|
+
.then((r) => r.json())
|
|
1172
|
+
.then((data) => {
|
|
1173
|
+
if (mounted && data?.ok && Array.isArray(data.discovered)) {
|
|
1174
|
+
setDiscoveredKeys(data.discovered);
|
|
1175
|
+
}
|
|
1176
|
+
})
|
|
1177
|
+
.catch((e) => { void e; });
|
|
1178
|
+
return () => { mounted = false; };
|
|
1179
|
+
}, []);
|
|
1180
|
+
|
|
1181
|
+
const runTestMatrix = async () => {
|
|
1182
|
+
setTestMatrixRunning(true);
|
|
1183
|
+
setTestMatrixModal(true);
|
|
1184
|
+
try {
|
|
1185
|
+
const res = await fetch('/dsh-key-rotation/test-matrix', {
|
|
1186
|
+
method: 'POST',
|
|
1187
|
+
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(15000) : undefined,
|
|
1188
|
+
});
|
|
1189
|
+
const data = await res.json();
|
|
1190
|
+
if (data?.ok) {
|
|
1191
|
+
setTestMatrixResults(data);
|
|
1192
|
+
}
|
|
1193
|
+
} catch (err) {
|
|
1194
|
+
setTestMatrixResults({ ok: false, failed: 1, total: 1, results: [{ provider: 'all', ref: 'matrix', ok: false, code: 'error', error: String(err?.message ?? err) }] });
|
|
1195
|
+
} finally {
|
|
1196
|
+
setTestMatrixRunning(false);
|
|
1197
|
+
}
|
|
1198
|
+
};
|
|
1199
|
+
|
|
1200
|
+
const runAutoImport = async () => {
|
|
1201
|
+
setAutoImporting(true);
|
|
1202
|
+
setAutoImportMsg('');
|
|
1203
|
+
try {
|
|
1204
|
+
const res = await fetch('/dsh-key-rotation/auto-import', {
|
|
1205
|
+
method: 'POST',
|
|
1206
|
+
headers: { 'Content-Type': 'application/json' },
|
|
1207
|
+
body: JSON.stringify({}),
|
|
1208
|
+
signal: (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') ? AbortSignal.timeout(8000) : undefined,
|
|
1209
|
+
});
|
|
1210
|
+
const data = await res.json();
|
|
1211
|
+
if (data?.ok) {
|
|
1212
|
+
setAutoImportMsg(t('autoImportSuccess').replace('{n}', String(data.importedCount || 0)));
|
|
1213
|
+
setDiscoveredKeys([]);
|
|
1214
|
+
void load();
|
|
1215
|
+
}
|
|
1216
|
+
} catch (err) {
|
|
1217
|
+
setAutoImportMsg(String(err?.message ?? err));
|
|
1218
|
+
} finally {
|
|
1219
|
+
setAutoImporting(false);
|
|
1220
|
+
}
|
|
1221
|
+
};
|
|
1222
|
+
|
|
1128
1223
|
const runSimulator = async () => {
|
|
1129
1224
|
const activeProv = simProvider || (providersList[0]?.provider || '');
|
|
1130
1225
|
if (!activeProv) return;
|
|
@@ -2446,6 +2541,7 @@ window.__ModuleLoader__.load({
|
|
|
2446
2541
|
h('div', { className: 'krot-keys' }, providerRows),
|
|
2447
2542
|
h('div', { className: 'krot-foot', style: { marginTop: '10px' } },
|
|
2448
2543
|
btn(t('addProvider'), addProvider, { primary: false }),
|
|
2544
|
+
btn('⚡ ' + (testMatrixRunning ? t('testingMatrix') : t('testAllKeys')), runTestMatrix, { disabled: testMatrixRunning || totalKeysCount === 0 }),
|
|
2449
2545
|
noProviders
|
|
2450
2546
|
),
|
|
2451
2547
|
h(UpdaterSection, { t })
|
|
@@ -2489,6 +2585,7 @@ window.__ModuleLoader__.load({
|
|
|
2489
2585
|
h('option', { key: 'round-robin', value: 'round-robin' }, t('routingStrategyRoundRobin')),
|
|
2490
2586
|
h('option', { key: 'least-loaded', value: 'least-loaded' }, t('routingStrategyLeastLoaded')),
|
|
2491
2587
|
h('option', { key: 'lowest-latency', value: 'lowest-latency' }, t('routingStrategyLowestLatency')),
|
|
2588
|
+
h('option', { key: 'sticky', value: 'sticky' }, t('routingStrategySticky')),
|
|
2492
2589
|
])),
|
|
2493
2590
|
field(t('selfHealingLabel'), textInput(String(val.selfHealingIntervalMinutes ?? 30), (v) => setField((cur) => ({ ...cur, selfHealingIntervalMinutes: Number(v) || 0 })))),
|
|
2494
2591
|
field(t('proactiveGuardLabel'), h('label', { style: { display: 'flex', alignItems: 'center', gap: '8px', height: '36px', cursor: 'pointer' } }, [
|
|
@@ -2658,10 +2755,105 @@ window.__ModuleLoader__.load({
|
|
|
2658
2755
|
h('div', { className: 'krot-page-sub' }, t('header.sub'))
|
|
2659
2756
|
);
|
|
2660
2757
|
|
|
2758
|
+
const discoveryBanner = (discoveredKeys.length > 0 || autoImportMsg) ? h('div', {
|
|
2759
|
+
className: 'krot-section-card',
|
|
2760
|
+
style: {
|
|
2761
|
+
background: 'var(--dsw-alias-bg-hover)',
|
|
2762
|
+
border: '1px solid var(--dsw-alias-border-l2)',
|
|
2763
|
+
marginBottom: '12px',
|
|
2764
|
+
padding: '12px 14px',
|
|
2765
|
+
display: 'flex',
|
|
2766
|
+
flexDirection: 'column',
|
|
2767
|
+
gap: '8px',
|
|
2768
|
+
}
|
|
2769
|
+
},
|
|
2770
|
+
h('div', { style: { display: 'flex', alignItems: 'center', justifyContent: 'space-between' } },
|
|
2771
|
+
h('div', null,
|
|
2772
|
+
h('span', { style: { fontWeight: 600, fontSize: '13px' } }, t('discoverBannerTitle')),
|
|
2773
|
+
h('div', { className: 'krot-hint', style: { fontSize: '11px', marginTop: '2px' } },
|
|
2774
|
+
autoImportMsg || t('discoverBannerDesc').replace('{n}', String(discoveredKeys.length))
|
|
2775
|
+
),
|
|
2776
|
+
),
|
|
2777
|
+
discoveredKeys.length > 0 ? btn(autoImporting ? t('autoImporting') : t('autoImportBtn'), runAutoImport, {
|
|
2778
|
+
primary: true,
|
|
2779
|
+
disabled: autoImporting,
|
|
2780
|
+
}) : null
|
|
2781
|
+
),
|
|
2782
|
+
discoveredKeys.length > 0 ? h('div', { style: { display: 'flex', gap: '6px', flexWrap: 'wrap', marginTop: '4px' } },
|
|
2783
|
+
discoveredKeys.slice(0, 10).map((d) => h('span', {
|
|
2784
|
+
key: d.ref,
|
|
2785
|
+
className: 'krot-badge',
|
|
2786
|
+
style: { fontSize: '10px', background: 'var(--dsw-alias-bg-layer-3)' }
|
|
2787
|
+
}, `${d.provider}: ${d.ref} ${d.keyTail ? '(' + d.keyTail + ')' : ''}`))
|
|
2788
|
+
) : null
|
|
2789
|
+
) : null;
|
|
2790
|
+
|
|
2791
|
+
const testMatrixDialog = testMatrixModal ? h('div', {
|
|
2792
|
+
className: 'krot-modal-backdrop',
|
|
2793
|
+
onClick: () => setTestMatrixModal(false),
|
|
2794
|
+
},
|
|
2795
|
+
h('div', {
|
|
2796
|
+
className: 'krot-modal-card',
|
|
2797
|
+
role: 'dialog',
|
|
2798
|
+
'aria-modal': 'true',
|
|
2799
|
+
onClick: (e) => e.stopPropagation(),
|
|
2800
|
+
style: {
|
|
2801
|
+
maxWidth: '560px',
|
|
2802
|
+
width: '90vw',
|
|
2803
|
+
maxHeight: '80vh',
|
|
2804
|
+
display: 'flex',
|
|
2805
|
+
flexDirection: 'column',
|
|
2806
|
+
gap: '12px',
|
|
2807
|
+
}
|
|
2808
|
+
},
|
|
2809
|
+
h('div', { style: { display: 'flex', justifyContent: 'space-between', alignItems: 'center' } },
|
|
2810
|
+
h('span', { style: { fontWeight: 600, fontSize: '15px' } }, t('matrixModalTitle')),
|
|
2811
|
+
btn('✕', () => setTestMatrixModal(false), { className: 'krot-btn-sub' })
|
|
2812
|
+
),
|
|
2813
|
+
h('div', { className: 'krot-hint', style: { fontSize: '12px' } }, t('matrixModalDesc')),
|
|
2814
|
+
testMatrixRunning ? h('div', { style: { textAlign: 'center', padding: '20px' } }, t('testingMatrix')) : null,
|
|
2815
|
+
testMatrixResults ? h('div', { style: { display: 'flex', flexDirection: 'column', gap: '10px', overflowY: 'auto', maxHeight: '50vh' } },
|
|
2816
|
+
h('div', { style: { fontWeight: 600, fontSize: '13px' } },
|
|
2817
|
+
t('matrixSummary').replace('{healthy}', String(testMatrixResults.healthy)).replace('{total}', String(testMatrixResults.total))
|
|
2818
|
+
),
|
|
2819
|
+
h('div', { style: { display: 'flex', flexDirection: 'column', gap: '6px' } },
|
|
2820
|
+
(testMatrixResults.results || []).map((r, i) => h('div', {
|
|
2821
|
+
key: i,
|
|
2822
|
+
style: {
|
|
2823
|
+
display: 'flex',
|
|
2824
|
+
justifyContent: 'space-between',
|
|
2825
|
+
alignItems: 'center',
|
|
2826
|
+
padding: '6px 10px',
|
|
2827
|
+
borderRadius: '6px',
|
|
2828
|
+
background: 'var(--dsw-alias-bg-hover)',
|
|
2829
|
+
fontSize: '12px',
|
|
2830
|
+
}
|
|
2831
|
+
},
|
|
2832
|
+
h('div', { style: { display: 'flex', gap: '8px', alignItems: 'center' } },
|
|
2833
|
+
h('span', { style: { fontWeight: 500 } }, r.provider),
|
|
2834
|
+
h('span', { style: { fontFamily: 'monospace' } }, r.ref),
|
|
2835
|
+
r.keyTail ? h('span', { style: { color: 'var(--dsw-alias-text-sub)', fontSize: '11px' } }, `(${r.keyTail})`) : null
|
|
2836
|
+
),
|
|
2837
|
+
h('div', { style: { display: 'flex', gap: '8px', alignItems: 'center' } },
|
|
2838
|
+
h('span', { style: { color: 'var(--dsw-alias-text-sub)' } }, `${r.latencyMs || 0}ms`),
|
|
2839
|
+
h('span', {
|
|
2840
|
+
className: 'krot-badge ' + (r.ok ? 'krot-badge-ok' : 'krot-badge-bad')
|
|
2841
|
+
}, r.ok ? 'OK' : (r.code || 'FAIL'))
|
|
2842
|
+
)
|
|
2843
|
+
))
|
|
2844
|
+
)
|
|
2845
|
+
) : null,
|
|
2846
|
+
h('div', { style: { display: 'flex', justifyContent: 'flex-end', marginTop: '8px' } },
|
|
2847
|
+
btn(t('matrixClose'), () => setTestMatrixModal(false), { primary: true })
|
|
2848
|
+
)
|
|
2849
|
+
)
|
|
2850
|
+
) : null;
|
|
2661
2851
|
return h('div', { className: 'krot', 'aria-busy': saving || importing },
|
|
2662
2852
|
confirmModalEl,
|
|
2853
|
+
testMatrixDialog,
|
|
2663
2854
|
headerSection,
|
|
2664
2855
|
statsSection,
|
|
2856
|
+
discoveryBanner,
|
|
2665
2857
|
h('fieldset', { disabled: saving || importing || !state.writable,
|
|
2666
2858
|
style: { border: 0, padding: 0, margin: 0, minWidth: 0, display: 'flex', flexDirection: 'column', gap: '14px' } },
|
|
2667
2859
|
poolsSection,
|
package/lib/heal.js
CHANGED
|
@@ -52,21 +52,31 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
|
|
|
52
52
|
for (const pool of pools) {
|
|
53
53
|
if (!pool || !pool.state || !pool.base) continue;
|
|
54
54
|
const bu = pool.state.brokenUntil;
|
|
55
|
-
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
const qw = pool.state.quotaWindows;
|
|
56
|
+
const candidates = new Set();
|
|
57
|
+
if (bu && bu.size > 0) {
|
|
58
|
+
for (const [ref, until] of bu.entries()) {
|
|
59
|
+
if (!isKeyRevoked(pool, ref) && !isKeyPaused(pool, ref) && Number.isFinite(until) && until > now) {
|
|
60
|
+
candidates.add(ref);
|
|
61
|
+
}
|
|
61
62
|
}
|
|
62
63
|
}
|
|
63
|
-
|
|
64
|
+
if (qw && qw.size > 0) {
|
|
65
|
+
for (const [ref, until] of qw.entries()) {
|
|
66
|
+
if (!isKeyRevoked(pool, ref) && !isKeyPaused(pool, ref) && Number.isFinite(until) && until > now) {
|
|
67
|
+
candidates.add(ref);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (candidates.size === 0) continue;
|
|
72
|
+
|
|
73
|
+
for (const ref of candidates) {
|
|
64
74
|
if (!isActive()) return results;
|
|
65
75
|
const state = pool.state;
|
|
66
|
-
const fields = ['brokenUntil', 'failedUntil', 'failCounts', 'authFailCounts'];
|
|
76
|
+
const fields = ['brokenUntil', 'failedUntil', 'quotaWindows', 'failCounts', 'authFailCounts'];
|
|
67
77
|
const observed = fields.map(field => [field, state[field], state[field]?.get(ref)]);
|
|
68
78
|
// An earlier await may have removed this key from the captured worklist.
|
|
69
|
-
if (
|
|
79
|
+
if (!((bu && bu.has(ref)) || (qw && qw.has(ref)))) continue;
|
|
70
80
|
try {
|
|
71
81
|
const res = await probeFn(ref);
|
|
72
82
|
if (!isActive()) return results;
|
|
@@ -74,7 +84,9 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
|
|
|
74
84
|
if (pool.state !== state || observed.some(([field, map, value]) =>
|
|
75
85
|
state[field] !== map || map?.get(ref) !== value)) continue;
|
|
76
86
|
if (res && res.ok) {
|
|
77
|
-
bu
|
|
87
|
+
bu?.delete(ref);
|
|
88
|
+
qw?.delete(ref);
|
|
89
|
+
pool.state.quotaWindows?.delete(ref);
|
|
78
90
|
pool.state.failedUntil?.delete(ref);
|
|
79
91
|
pool.state.failCounts?.delete(ref);
|
|
80
92
|
pool.state.authFailCounts?.delete(ref);
|
package/lib/index.js
CHANGED
|
@@ -95,7 +95,7 @@ const SettingsSchema = Schema.object({
|
|
|
95
95
|
rateLimitThreshold: Schema.number().default(0.1),
|
|
96
96
|
proactiveRateLimitGuard: Schema.boolean().default(true),
|
|
97
97
|
selfHealingIntervalMinutes: Schema.number().default(30),
|
|
98
|
-
routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']).default('round-robin'),
|
|
98
|
+
routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']).default('round-robin'),
|
|
99
99
|
rpmLimit: Schema.number().default(0),
|
|
100
100
|
tpmLimit: Schema.number().default(0),
|
|
101
101
|
webhookActionToken: Schema.string().role('secret').default(''),
|
|
@@ -144,7 +144,7 @@ const SettingsSchema = Schema.object({
|
|
|
144
144
|
})).default({}),
|
|
145
145
|
cooldownMs: Schema.number(),
|
|
146
146
|
maxCooldownMs: Schema.number(),
|
|
147
|
-
routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']),
|
|
147
|
+
routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']),
|
|
148
148
|
proactiveRateLimitGuard: Schema.boolean(),
|
|
149
149
|
rpmLimit: Schema.number(),
|
|
150
150
|
tpmLimit: Schema.number(),
|
|
@@ -374,6 +374,7 @@ export function apply(ctx, config = {}) {
|
|
|
374
374
|
const resolve = createResolver({
|
|
375
375
|
buildRuntime,
|
|
376
376
|
currentPool: () => dispatchStorage.getStore()?.pool,
|
|
377
|
+
currentSession: () => dispatchStorage.getStore()?.sessionId,
|
|
377
378
|
onPicked: (pool, candidate, meta) => {
|
|
378
379
|
const store = dispatchStorage.getStore();
|
|
379
380
|
if (store && store.pool === pool) {
|
|
@@ -384,7 +385,7 @@ export function apply(ctx, config = {}) {
|
|
|
384
385
|
latencyHistogram,
|
|
385
386
|
concurrencyTracker,
|
|
386
387
|
});
|
|
387
|
-
credentials.resolve = async (ref) => resolve(ref, original);
|
|
388
|
+
credentials.resolve = async (ref, opts) => resolve(ref, original, opts);
|
|
388
389
|
credentials.__dshKeyRotationPatched = true;
|
|
389
390
|
return () => {
|
|
390
391
|
if (credentials.__dshKeyRotationPatched) {
|
package/lib/lifecycle.js
CHANGED
|
@@ -171,8 +171,10 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
|
|
|
171
171
|
running = true;
|
|
172
172
|
try {
|
|
173
173
|
const c = getConfig();
|
|
174
|
-
if (!c
|
|
174
|
+
if (!c) return;
|
|
175
175
|
const { pools } = buildRuntime();
|
|
176
|
+
const hasCalendarLocks = (pools || []).some(p => p?.state?.quotaWindows?.size > 0);
|
|
177
|
+
if (c.proactiveHealthProbing !== true && !hasCalendarLocks) return;
|
|
176
178
|
const runner = ensureSandboxRunner(ctx);
|
|
177
179
|
const now = Date.now();
|
|
178
180
|
for (const pool of (pools || [])) {
|
|
@@ -203,6 +205,7 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
|
|
|
203
205
|
if (res && res.ok) {
|
|
204
206
|
pool.state.failedUntil?.delete(ref);
|
|
205
207
|
pool.state.brokenUntil?.delete(ref);
|
|
208
|
+
pool.state.quotaWindows?.delete(ref);
|
|
206
209
|
pool.state.failCounts?.delete(ref);
|
|
207
210
|
if (Array.isArray(pool.state.events)) {
|
|
208
211
|
pool.state.events.push({ at: Date.now(), ref, reason: 'proactive-probe-heal', cooldownMs: 0, type: 'heal' });
|
package/lib/ops-keys.js
CHANGED
|
@@ -20,13 +20,138 @@ import {
|
|
|
20
20
|
RESET_PATH,
|
|
21
21
|
IMPORT_PATH,
|
|
22
22
|
DRAIN_PATH,
|
|
23
|
+
DISCOVER_PATH,
|
|
24
|
+
AUTO_IMPORT_PATH,
|
|
23
25
|
} from './ops-paths.js';
|
|
26
|
+
import fs from 'node:fs';
|
|
27
|
+
import os from 'node:os';
|
|
28
|
+
import path from 'node:path';
|
|
24
29
|
import { markKeyDraining, isKeyDraining, cancelKeyDrain, getDrainingStatus, getAllDrainingKeys } from './key-drain.js';
|
|
25
30
|
|
|
26
31
|
/**
|
|
27
32
|
* @param {object} ctx cordis context
|
|
28
33
|
* @param {object} deps live dependencies from apply()
|
|
29
34
|
*/
|
|
35
|
+
const KNOWN_PROVIDER_PATTERNS = [
|
|
36
|
+
{ provider: 'deepseek', pattern: /^DEEPSEEK_API_KEY(?:_\w+)?$/i },
|
|
37
|
+
{ provider: 'opencode', pattern: /^OPENCODE(?:_GO)?_API_KEY(?:_\w+)?$/i },
|
|
38
|
+
{ provider: 'openai', pattern: /^OPENAI_API_KEY(?:_\w+)?$/i },
|
|
39
|
+
{ provider: 'anthropic', pattern: /^ANTHROPIC_API_KEY(?:_\w+)?$/i },
|
|
40
|
+
{ provider: 'openrouter', pattern: /^OPENROUTER_API_KEY(?:_\w+)?$/i },
|
|
41
|
+
{ provider: 'siliconflow', pattern: /^SILICONFLOW_API_KEY(?:_\w+)?$/i },
|
|
42
|
+
{ provider: 'groq', pattern: /^GROQ_API_KEY(?:_\w+)?$/i },
|
|
43
|
+
{ provider: 'mistral', pattern: /^MISTRAL_API_KEY(?:_\w+)?$/i },
|
|
44
|
+
{ provider: 'gemini', pattern: /^(?:GEMINI|GOOGLE)_API_KEY(?:_\w+)?$/i },
|
|
45
|
+
{ provider: 'ollama', pattern: /^OLLAMA_API_KEY(?:_\w+)?$/i },
|
|
46
|
+
{ provider: 'sensenova', pattern: /^SENSENOVA_API_KEY(?:_\w+)?$/i },
|
|
47
|
+
{ provider: 'cohere', pattern: /^COHERE_API_KEY(?:_\w+)?$/i },
|
|
48
|
+
{ provider: 'fal', pattern: /^FAL_API_KEY(?:_\w+)?$/i },
|
|
49
|
+
{ provider: 'together', pattern: /^TOGETHER_API_KEY(?:_\w+)?$/i },
|
|
50
|
+
{ provider: 'xai', pattern: /^XAI_API_KEY(?:_\w+)?$/i },
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
export function inferProviderFromRef(ref) {
|
|
54
|
+
if (!ref || typeof ref !== 'string') return null;
|
|
55
|
+
for (const { provider, pattern } of KNOWN_PROVIDER_PATTERNS) {
|
|
56
|
+
if (pattern.test(ref)) return provider;
|
|
57
|
+
}
|
|
58
|
+
const generic = /^([A-Z0-9]+)_API_KEY(?:_\w+)?$/i.exec(ref);
|
|
59
|
+
if (generic && generic[1]) {
|
|
60
|
+
const prov = generic[1].toLowerCase();
|
|
61
|
+
if (!/TOKEN|SECRET|WEBHOOK|PASSWORD|AUTH|SESSION|OAUTH|GITEA|KANBAN/i.test(prov)) {
|
|
62
|
+
return prov;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function discoverUnconfiguredKeys(ctx, runtimePools = []) {
|
|
69
|
+
const configuredRefs = new Set();
|
|
70
|
+
for (const p of runtimePools) {
|
|
71
|
+
for (const r of (p.refs || [])) configuredRefs.add(r);
|
|
72
|
+
for (const r of (p.keys || [])) configuredRefs.add(typeof r === 'string' ? r : r?.ref);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Also check DSH settings describe if available
|
|
76
|
+
const settings = ctx.get('settings');
|
|
77
|
+
if (settings && typeof settings.describe === 'function') {
|
|
78
|
+
try {
|
|
79
|
+
const desc = settings.describe({ redactSecrets: true })?.find?.((c) => c.ns === NS);
|
|
80
|
+
const providers = desc?.value?.providers ?? [];
|
|
81
|
+
for (const p of providers) {
|
|
82
|
+
for (const k of (p.keys || [])) {
|
|
83
|
+
configuredRefs.add(typeof k === 'string' ? k : k?.ref);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
} catch (_) { /* best-effort */ }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const candidates = new Map(); // ref -> { provider, source }
|
|
90
|
+
|
|
91
|
+
// 1. Process environment
|
|
92
|
+
for (const k of Object.keys(process.env)) {
|
|
93
|
+
if (configuredRefs.has(k)) continue;
|
|
94
|
+
const prov = inferProviderFromRef(k);
|
|
95
|
+
if (prov && process.env[k]?.trim()) {
|
|
96
|
+
candidates.set(k, { provider: prov, source: 'env' });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// 2. ~/.dsh/.credentials.yaml
|
|
101
|
+
try {
|
|
102
|
+
const credPath = path.join(os.homedir(), '.dsh', '.credentials.yaml');
|
|
103
|
+
if (fs.existsSync(credPath)) {
|
|
104
|
+
const content = fs.readFileSync(credPath, 'utf8');
|
|
105
|
+
let inRefs = false;
|
|
106
|
+
for (const line of content.split('\n')) {
|
|
107
|
+
const trimmed = line.trim();
|
|
108
|
+
if (trimmed === 'refs:') {
|
|
109
|
+
inRefs = true;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (inRefs) {
|
|
113
|
+
if (line.startsWith(' ') && line.includes(':')) {
|
|
114
|
+
const refName = trimmed.split(':')[0].trim();
|
|
115
|
+
if (!configuredRefs.has(refName)) {
|
|
116
|
+
const prov = inferProviderFromRef(refName);
|
|
117
|
+
if (prov && !candidates.has(refName)) {
|
|
118
|
+
candidates.set(refName, { provider: prov, source: 'credentials' });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
} else if (!line.startsWith(' ') && trimmed.length > 0) {
|
|
122
|
+
inRefs = false;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
} catch (_) { /* best-effort */ }
|
|
128
|
+
|
|
129
|
+
const credentialsService = ctx.get('credentials');
|
|
130
|
+
const discovered = [];
|
|
131
|
+
|
|
132
|
+
for (const [ref, { provider, source }] of candidates) {
|
|
133
|
+
let tail = '';
|
|
134
|
+
try {
|
|
135
|
+
let hit = await (credentialsService?.__dshKeyRotationOriginalResolve ?? credentialsService?.resolve)?.call(credentialsService, ref);
|
|
136
|
+
let raw = hit?.value;
|
|
137
|
+
if (!raw) raw = envValue(ref);
|
|
138
|
+
if (raw) {
|
|
139
|
+
const plain = decryptSecret(raw);
|
|
140
|
+
tail = keyTail(plain);
|
|
141
|
+
}
|
|
142
|
+
} catch (_) { /* best-effort tail */ }
|
|
143
|
+
discovered.push({
|
|
144
|
+
provider,
|
|
145
|
+
ref,
|
|
146
|
+
keyTail: tail,
|
|
147
|
+
source,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return discovered;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
30
155
|
export function registerKeyRoutes(ctx, deps) {
|
|
31
156
|
|
|
32
157
|
ctx.effect(() => ctx.webServer.register({
|
|
@@ -300,4 +425,118 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
300
425
|
}), 'dsh-key-rotation: import route');
|
|
301
426
|
|
|
302
427
|
// Health for external panels (Beszel/Uptime)
|
|
428
|
+
|
|
429
|
+
// ── discover unconfigured keys route (#491) ──
|
|
430
|
+
ctx.effect(() => ctx.webServer.register({
|
|
431
|
+
kind: 'exact',
|
|
432
|
+
path: DISCOVER_PATH,
|
|
433
|
+
handler: async (req, res) => {
|
|
434
|
+
if (req.method !== 'GET') {
|
|
435
|
+
json(res, 405, { error: { code: 'method', message: 'GET only' } });
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
if (!isTrustedBridgeRequest(req)) {
|
|
439
|
+
json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: discover is local-only' } });
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
try {
|
|
443
|
+
const runtime = deps.buildRuntime?.() ?? {};
|
|
444
|
+
const discovered = await discoverUnconfiguredKeys(ctx, runtime.pools || []);
|
|
445
|
+
json(res, 200, { ok: true, discovered });
|
|
446
|
+
} catch (e) {
|
|
447
|
+
json(res, 500, { error: { code: 'discover-failed', message: String(e?.message ?? e) } });
|
|
448
|
+
}
|
|
449
|
+
},
|
|
450
|
+
}), 'dsh-key-rotation: discover route');
|
|
451
|
+
|
|
452
|
+
// ── auto-import discovered keys into pools (#491) ──
|
|
453
|
+
ctx.effect(() => ctx.webServer.register({
|
|
454
|
+
kind: 'exact',
|
|
455
|
+
path: AUTO_IMPORT_PATH,
|
|
456
|
+
handler: async (req, res) => {
|
|
457
|
+
if (req.method !== 'POST') {
|
|
458
|
+
json(res, 405, { error: { code: 'method', message: 'POST only' } });
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
if (!isTrustedBridgeRequest(req)) {
|
|
462
|
+
json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: auto-import is local-only' } });
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
let body = {};
|
|
466
|
+
try { body = await readJson(req); } catch (_) { body = {}; }
|
|
467
|
+
|
|
468
|
+
try {
|
|
469
|
+
const runtime = deps.buildRuntime?.() ?? {};
|
|
470
|
+
const discovered = await discoverUnconfiguredKeys(ctx, runtime.pools || []);
|
|
471
|
+
const targetRefs = Array.isArray(body?.refs) && body.refs.length > 0
|
|
472
|
+
? new Set(body.refs)
|
|
473
|
+
: null;
|
|
474
|
+
|
|
475
|
+
const toImport = targetRefs
|
|
476
|
+
? discovered.filter((d) => targetRefs.has(d.ref))
|
|
477
|
+
: discovered;
|
|
478
|
+
|
|
479
|
+
if (toImport.length === 0) {
|
|
480
|
+
json(res, 200, { ok: true, importedCount: 0, message: 'No unassigned keys to import' });
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const settings = ctx.get('settings');
|
|
485
|
+
if (!settings) {
|
|
486
|
+
json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } });
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
const desc = settings.describe({ redactSecrets: true })?.find?.((c) => c.ns === NS);
|
|
490
|
+
if (!desc) throw new Error('Settings namespace is unavailable');
|
|
491
|
+
|
|
492
|
+
const curProviders = (desc.value?.providers ?? []).map((p) => ({
|
|
493
|
+
...p,
|
|
494
|
+
keys: Array.isArray(p.keys) ? [...p.keys] : [],
|
|
495
|
+
}));
|
|
496
|
+
|
|
497
|
+
const providerMap = new Map();
|
|
498
|
+
for (const p of curProviders) {
|
|
499
|
+
if (p?.provider) providerMap.set(p.provider, p);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
let importedCount = 0;
|
|
503
|
+
for (const item of toImport) {
|
|
504
|
+
const { provider, ref } = item;
|
|
505
|
+
let poolConfig = providerMap.get(provider);
|
|
506
|
+
if (!poolConfig) {
|
|
507
|
+
poolConfig = {
|
|
508
|
+
provider,
|
|
509
|
+
strategy: 'round-robin',
|
|
510
|
+
cooldownMs: 30000,
|
|
511
|
+
keys: [],
|
|
512
|
+
};
|
|
513
|
+
providerMap.set(provider, poolConfig);
|
|
514
|
+
}
|
|
515
|
+
const existingKeys = poolConfig.keys.map((k) => (typeof k === 'string' ? k : k?.ref));
|
|
516
|
+
if (!existingKeys.includes(ref)) {
|
|
517
|
+
poolConfig.keys.push(ref);
|
|
518
|
+
importedCount++;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const updatedProviders = [...providerMap.values()];
|
|
523
|
+
await mutateSettings(
|
|
524
|
+
settings,
|
|
525
|
+
NS,
|
|
526
|
+
[{ op: 'set', path: ['providers'], value: updatedProviders }],
|
|
527
|
+
body.expectedRevision ?? desc.revision
|
|
528
|
+
);
|
|
529
|
+
|
|
530
|
+
json(res, 200, {
|
|
531
|
+
ok: true,
|
|
532
|
+
importedCount,
|
|
533
|
+
providersCount: updatedProviders.length,
|
|
534
|
+
providers: updatedProviders,
|
|
535
|
+
});
|
|
536
|
+
} catch (e) {
|
|
537
|
+
json(res, 400, { error: { code: 'auto-import-failed', message: String(e?.message ?? e) } });
|
|
538
|
+
}
|
|
539
|
+
},
|
|
540
|
+
}), 'dsh-key-rotation: auto-import route');
|
|
541
|
+
|
|
303
542
|
}
|
package/lib/ops-paths.js
CHANGED
|
@@ -14,3 +14,6 @@ export const DIAG_PATH = '/dsh-key-rotation/diag';
|
|
|
14
14
|
export const TRACE_PATH = '/dsh-key-rotation/trace';
|
|
15
15
|
export const SIMULATE_PATH = '/dsh-key-rotation/simulate';
|
|
16
16
|
export const DRAIN_PATH = '/dsh-key-rotation/drain';
|
|
17
|
+
export const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
|
|
18
|
+
export const DISCOVER_PATH = '/dsh-key-rotation/discover';
|
|
19
|
+
export const AUTO_IMPORT_PATH = '/dsh-key-rotation/auto-import';
|
package/lib/ops-test.js
CHANGED
|
@@ -13,6 +13,7 @@ import { bestEffort } from './best-effort.js';
|
|
|
13
13
|
import {
|
|
14
14
|
TEST_PATH,
|
|
15
15
|
SANDBOX_CACHE_PATH,
|
|
16
|
+
TEST_MATRIX_PATH,
|
|
16
17
|
} from './ops-paths.js';
|
|
17
18
|
import { decryptSecret } from './crypto-storage.js';
|
|
18
19
|
|
|
@@ -114,4 +115,114 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
114
115
|
// #199 webhook-action: interactive webhook buttons call back here.
|
|
115
116
|
// Auth: bearer token from Config (external services like Telegram/Discord
|
|
116
117
|
// cannot be same-origin, so a shared secret is the gate).
|
|
118
|
+
|
|
119
|
+
// ── test matrix route: batch probe all configured keys (#490) ──
|
|
120
|
+
ctx.effect(() => ctx.webServer.register({
|
|
121
|
+
kind: 'exact',
|
|
122
|
+
path: TEST_MATRIX_PATH,
|
|
123
|
+
handler: async (req, res) => {
|
|
124
|
+
if (req.method !== 'GET' && req.method !== 'POST') {
|
|
125
|
+
json(res, 405, { error: { code: 'method', message: 'GET or POST only' } });
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (!isTrustedBridgeRequest(req)) {
|
|
129
|
+
json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test-matrix is local-only' } });
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const base = ctx.get('credentials');
|
|
133
|
+
const runner = ensureSandboxRunner(ctx);
|
|
134
|
+
const runtime = deps.buildRuntime?.() ?? {};
|
|
135
|
+
const pools = runtime.pools || [];
|
|
136
|
+
|
|
137
|
+
const keyItems = [];
|
|
138
|
+
const seen = new Set();
|
|
139
|
+
for (const p of pools) {
|
|
140
|
+
const pName = p.provider ?? p.base ?? 'unknown';
|
|
141
|
+
for (const ref of (p.refs || [])) {
|
|
142
|
+
const id = `${pName}:${ref}`;
|
|
143
|
+
if (!seen.has(id)) {
|
|
144
|
+
seen.add(id);
|
|
145
|
+
keyItems.push({ provider: pName, ref });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const CONCURRENCY = 5;
|
|
151
|
+
const results = [];
|
|
152
|
+
let idx = 0;
|
|
153
|
+
|
|
154
|
+
async function worker() {
|
|
155
|
+
while (idx < keyItems.length) {
|
|
156
|
+
const item = keyItems[idx++];
|
|
157
|
+
if (!item) break;
|
|
158
|
+
const { provider, ref } = item;
|
|
159
|
+
try {
|
|
160
|
+
let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
|
|
161
|
+
let rawValue = hit?.value;
|
|
162
|
+
if (!rawValue) {
|
|
163
|
+
const ev = envValue(ref);
|
|
164
|
+
if (ev !== undefined) rawValue = ev;
|
|
165
|
+
}
|
|
166
|
+
let effectiveValue = rawValue ? decryptSecret(rawValue) : rawValue;
|
|
167
|
+
const tail = effectiveValue ? keyTail(effectiveValue) : '';
|
|
168
|
+
if (!effectiveValue) {
|
|
169
|
+
results.push({ provider, ref, keyTail: tail, ok: false, code: 'no-credential', latencyMs: 0 });
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const probeRes = await runner.probeModels(ref, effectiveValue);
|
|
173
|
+
const cached = { ...probeRes, at: Date.now() };
|
|
174
|
+
lastTestCache.set(ref, cached);
|
|
175
|
+
if (cached.ok) {
|
|
176
|
+
for (const st of poolState.values()) {
|
|
177
|
+
st.failedUntil?.delete(ref);
|
|
178
|
+
st.failCounts?.delete(ref);
|
|
179
|
+
st.authFailCounts?.delete(ref);
|
|
180
|
+
st.brokenUntil?.delete(ref);
|
|
181
|
+
st.revokedRefs?.delete(ref);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
results.push({
|
|
185
|
+
provider,
|
|
186
|
+
ref,
|
|
187
|
+
keyTail: tail,
|
|
188
|
+
ok: cached.ok,
|
|
189
|
+
code: cached.code,
|
|
190
|
+
latencyMs: cached.latencyMs,
|
|
191
|
+
modelsCount: cached.modelsCount,
|
|
192
|
+
});
|
|
193
|
+
} catch (err) {
|
|
194
|
+
results.push({
|
|
195
|
+
provider,
|
|
196
|
+
ref,
|
|
197
|
+
keyTail: '',
|
|
198
|
+
ok: false,
|
|
199
|
+
code: 'error',
|
|
200
|
+
error: String(err?.message ?? err),
|
|
201
|
+
latencyMs: 0,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const workers = [];
|
|
208
|
+
for (let i = 0; i < Math.min(CONCURRENCY, keyItems.length); i++) {
|
|
209
|
+
workers.push(worker());
|
|
210
|
+
}
|
|
211
|
+
await Promise.all(workers);
|
|
212
|
+
|
|
213
|
+
const total = results.length;
|
|
214
|
+
const healthy = results.filter((r) => r.ok).length;
|
|
215
|
+
const failed = total - healthy;
|
|
216
|
+
|
|
217
|
+
json(res, 200, {
|
|
218
|
+
ok: true,
|
|
219
|
+
at: Date.now(),
|
|
220
|
+
total,
|
|
221
|
+
healthy,
|
|
222
|
+
failed,
|
|
223
|
+
results,
|
|
224
|
+
});
|
|
225
|
+
},
|
|
226
|
+
}), 'dsh-key-rotation: test matrix route');
|
|
227
|
+
|
|
117
228
|
}
|
package/lib/pool-state.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const MAP_FIELDS = [
|
|
4
4
|
'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
|
|
5
5
|
'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
|
|
6
|
-
'tokenUsage', 'rpmWindows', 'tpmWindows',
|
|
6
|
+
'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky',
|
|
7
7
|
];
|
|
8
8
|
|
|
9
9
|
export function initializePoolState(state = {}) {
|
package/lib/pool.js
CHANGED
|
@@ -1,3 +1,29 @@
|
|
|
1
|
+
export const MAX_SESSION_STICKY_ENTRIES = 500;
|
|
2
|
+
|
|
3
|
+
export function setSessionSticky(pool, sessionId, candidate) {
|
|
4
|
+
if (!sessionId || !candidate || !pool?.state) return;
|
|
5
|
+
if (!(pool.state.sessionSticky instanceof Map)) {
|
|
6
|
+
pool.state.sessionSticky = new Map();
|
|
7
|
+
}
|
|
8
|
+
if (pool.state.sessionSticky.has(sessionId)) {
|
|
9
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
10
|
+
} else if (pool.state.sessionSticky.size >= MAX_SESSION_STICKY_ENTRIES) {
|
|
11
|
+
const oldest = pool.state.sessionSticky.keys().next().value;
|
|
12
|
+
if (oldest !== undefined) pool.state.sessionSticky.delete(oldest);
|
|
13
|
+
}
|
|
14
|
+
pool.state.sessionSticky.set(sessionId, candidate);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function getSessionSticky(pool, sessionId) {
|
|
18
|
+
if (!sessionId || !pool?.state?.sessionSticky) return null;
|
|
19
|
+
return pool.state.sessionSticky.get(sessionId) ?? null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function clearSessionSticky(pool, sessionId) {
|
|
23
|
+
if (!sessionId || !pool?.state?.sessionSticky) return;
|
|
24
|
+
pool.state.sessionSticky.delete(sessionId);
|
|
25
|
+
}
|
|
26
|
+
|
|
1
27
|
import { resetModelQuotaIfNeeded } from './model-quota.js';
|
|
2
28
|
import { decryptSecret } from './crypto-storage.js';
|
|
3
29
|
import { bestEffort } from './best-effort.js';
|
|
@@ -68,7 +94,8 @@ export const SWITCHABLE_MESSAGE_PATTERN = new RegExp([
|
|
|
68
94
|
/\b(?:quota|usage[\s_-]+limit|rate[\s_-]?limit)\b/i,
|
|
69
95
|
/\binsufficient[\s_-]+(?:quota|balance|credits?)\b/i,
|
|
70
96
|
/\bout[\s_-]+of[\s_-]+(?:credits?|budget)\b/i,
|
|
71
|
-
/\b(?:exceeded|exhausted)[\s_-]+(?:quota|limit|budget)\b/i,
|
|
97
|
+
/\b(?:exceeded|exceeds|exhausted|exhausts)[\s_-]+(?:quota|limit|budget|rpm|tpm|inference)\b/i,
|
|
98
|
+
/\b(?:rpm|tpm)\b/i,
|
|
72
99
|
/\bbilling\b/i,
|
|
73
100
|
/\bresource[\s_-]+exhausted\b/i,
|
|
74
101
|
/\bcapacity[\s_-]+(?:limit|exceeded|reached)\b/i,
|
|
@@ -171,7 +198,8 @@ export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jit
|
|
|
171
198
|
/** Record a success for `ref`. */
|
|
172
199
|
export function recordSuccess(pool, ref, now = Date.now()) {
|
|
173
200
|
if (pool.state.failCounts) pool.state.failCounts.delete(ref);
|
|
174
|
-
pool.state.failedUntil
|
|
201
|
+
pool.state.failedUntil?.delete(ref);
|
|
202
|
+
pool.state.quotaWindows?.delete(ref);
|
|
175
203
|
if (!pool.state.lastSuccessAt) pool.state.lastSuccessAt = new Map();
|
|
176
204
|
pool.state.lastSuccessAt.set(ref, now);
|
|
177
205
|
}
|
|
@@ -409,6 +437,20 @@ export function isRateLimited(rate, threshold = 0.1) {
|
|
|
409
437
|
export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
|
|
410
438
|
if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
|
|
411
439
|
const list = refs.slice();
|
|
440
|
+
if (strategy === 'sticky') {
|
|
441
|
+
const last = deps.sessionSticky ||
|
|
442
|
+
(deps.sessionId && (deps.pool?.state?.sessionSticky?.get?.(deps.sessionId) || deps.sessionStickyMap?.get?.(deps.sessionId))) ||
|
|
443
|
+
deps.lastUsed ||
|
|
444
|
+
(deps.pool && deps.pool.state && deps.pool.state.lastUsed);
|
|
445
|
+
if (last && list.includes(last)) {
|
|
446
|
+
const idx = list.indexOf(last);
|
|
447
|
+
if (idx > 0) {
|
|
448
|
+
list.splice(idx, 1);
|
|
449
|
+
list.unshift(last);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
return list;
|
|
453
|
+
}
|
|
412
454
|
if (strategy === 'tier-priority') {
|
|
413
455
|
const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
|
|
414
456
|
return list.sort((a, b) => {
|
|
@@ -482,7 +524,7 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
|
|
|
482
524
|
if ((status >= 500 && status <= 504) && (switchCodes.has('SERVER') || switchCodes.has(String(status)))) return true;
|
|
483
525
|
|
|
484
526
|
// 2. Standard gRPC / cloud error codes
|
|
485
|
-
if (code === 'RESOURCE_EXHAUSTED' && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT'))) return true;
|
|
527
|
+
if ((code === 'RESOURCE_EXHAUSTED' || code === 'INSUFFICIENT_QUOTA') && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT'))) return true;
|
|
486
528
|
if ((code === 'UNAVAILABLE' || code === 'INTERNAL') && switchCodes.has('SERVER')) return true;
|
|
487
529
|
if (code === 'DEADLINE_EXCEEDED' && switchCodes.has('TIMEOUT')) return true;
|
|
488
530
|
if ((code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED') && switchCodes.has('AUTH')) return true;
|
package/lib/resolver.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// delegates to `original(ref)`.
|
|
10
10
|
|
|
11
11
|
import { bucketAllow, bucketRetryMs, tpmAllow, tpmRetryMs } from './bucket.js';
|
|
12
|
-
import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, isKeyBroken, costForDay, costForWeek, budgetVerdict } from './pool.js';
|
|
12
|
+
import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, isKeyBroken, costForDay, costForWeek, budgetVerdict, getSessionSticky, setSessionSticky } from './pool.js';
|
|
13
13
|
import { decryptSecret } from './crypto-storage.js';
|
|
14
14
|
import { resolveRefPool, isQuotaManagedRequest, providerOfBase, basePoolForBase } from './pool-index.js';
|
|
15
15
|
import {
|
|
@@ -76,12 +76,13 @@ export function getProviderCost(provider, runtime, now = Date.now()) {
|
|
|
76
76
|
export function createResolver({
|
|
77
77
|
buildRuntime,
|
|
78
78
|
currentPool,
|
|
79
|
+
currentSession,
|
|
79
80
|
onPicked,
|
|
80
81
|
latencyHistogram,
|
|
81
82
|
concurrencyTracker,
|
|
82
83
|
now = Date.now,
|
|
83
84
|
}) {
|
|
84
|
-
return async function resolve(ref, original) {
|
|
85
|
+
return async function resolve(ref, original, resolveOpts) {
|
|
85
86
|
const runtime = buildRuntime();
|
|
86
87
|
// A ref may belong to several model pools at once, so the request-scoped
|
|
87
88
|
// pool that rotate() actually dispatched with decides ownership. A bare
|
|
@@ -116,9 +117,18 @@ export function createResolver({
|
|
|
116
117
|
}
|
|
117
118
|
|
|
118
119
|
const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
|
|
120
|
+
const sessionId = resolveOpts?.sessionId ?? resolveOpts?.session ?? resolveOpts?.metadata?.sessionId ?? currentSession?.();
|
|
119
121
|
let list = pool.weightedRefs ?? pool.refs;
|
|
120
|
-
if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
|
|
121
|
-
list = sortAttemptList(list, strategy, {
|
|
122
|
+
if (strategy === 'lowest-latency' || strategy === 'least-loaded' || strategy === 'sticky') {
|
|
123
|
+
list = sortAttemptList(list, strategy, {
|
|
124
|
+
latencyHistogram,
|
|
125
|
+
concurrencyTracker,
|
|
126
|
+
weights: pool.weightsMap,
|
|
127
|
+
tiers: pool.tiers,
|
|
128
|
+
sessionId,
|
|
129
|
+
sessionSticky: sessionId ? getSessionSticky(pool, sessionId) : null,
|
|
130
|
+
lastUsed: pool.state.lastUsed,
|
|
131
|
+
});
|
|
122
132
|
}
|
|
123
133
|
// Fail closed: when this request is managed by a model pool that has local
|
|
124
134
|
// token budgets, an exhausted candidate may not be dispatched and the
|
|
@@ -126,6 +136,18 @@ export function createResolver({
|
|
|
126
136
|
// the loop, so it cannot be re-evaluated mid-iteration.
|
|
127
137
|
const quotaManaged = hasModelQuotaConfig(pool) && isQuotaManagedRequest(runtime.index, ref, requested);
|
|
128
138
|
let start = strategy === 'round-robin' ? (pool.state.pointer ?? 0) : 0;
|
|
139
|
+
if (strategy === 'sticky') {
|
|
140
|
+
let preferred = null;
|
|
141
|
+
if (sessionId) {
|
|
142
|
+
const stick = getSessionSticky(pool, sessionId);
|
|
143
|
+
if (stick && list.includes(stick)) preferred = stick;
|
|
144
|
+
}
|
|
145
|
+
if (!preferred && pool.state.lastUsed && list.includes(pool.state.lastUsed)) {
|
|
146
|
+
preferred = pool.state.lastUsed;
|
|
147
|
+
}
|
|
148
|
+
const lastIdx = preferred ? list.indexOf(preferred) : -1;
|
|
149
|
+
start = lastIdx >= 0 ? lastIdx : (pool.state.pointer ?? 0);
|
|
150
|
+
}
|
|
129
151
|
if (pool.pinnedRef && list.includes(pool.pinnedRef)) {
|
|
130
152
|
const pIdx = list.indexOf(pool.pinnedRef);
|
|
131
153
|
if (pIdx > 0) {
|
|
@@ -217,18 +239,22 @@ export function createResolver({
|
|
|
217
239
|
|
|
218
240
|
// Advance the shared cursor at selection time: concurrent requests must
|
|
219
241
|
// not all pick the same slot.
|
|
220
|
-
|
|
242
|
+
if (strategy === 'sticky') {
|
|
243
|
+
pool.state.pointer = index;
|
|
244
|
+
} else {
|
|
245
|
+
pool.state.pointer = (index + 1) % list.length;
|
|
246
|
+
}
|
|
221
247
|
try {
|
|
222
248
|
const hit = await original(candidate);
|
|
223
249
|
if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
|
|
224
|
-
settle(pool, candidate, at, onPicked, acquired ? candidate : undefined);
|
|
250
|
+
settle(pool, candidate, at, onPicked, acquired ? candidate : undefined, sessionId);
|
|
225
251
|
return { ...hit, value: decryptSecret(hit.value) };
|
|
226
252
|
}
|
|
227
253
|
// No stored credential: fall back to the launching environment, which is
|
|
228
254
|
// how env-bootstrapped pools work (issue #7).
|
|
229
255
|
const envVal = envValue(candidate);
|
|
230
256
|
if (envVal !== undefined) {
|
|
231
|
-
settle(pool, candidate, at, onPicked, acquired ? candidate : undefined);
|
|
257
|
+
settle(pool, candidate, at, onPicked, acquired ? candidate : undefined, sessionId);
|
|
232
258
|
return { value: decryptSecret(envVal), source: 'env' };
|
|
233
259
|
}
|
|
234
260
|
} catch (err) {
|
|
@@ -260,8 +286,9 @@ export function createResolver({
|
|
|
260
286
|
}
|
|
261
287
|
|
|
262
288
|
/** Record a successful credential selection on the pool and request scope. */
|
|
263
|
-
function settle(pool, candidate, now, onPicked, concurrencyRef) {
|
|
289
|
+
function settle(pool, candidate, now, onPicked, concurrencyRef, sessionId) {
|
|
264
290
|
pool.state.lastUsed = candidate;
|
|
291
|
+
if (sessionId) setSessionSticky(pool, sessionId, candidate);
|
|
265
292
|
if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
|
|
266
293
|
pool.state.lastUsedAt.set(candidate, now);
|
|
267
294
|
onPicked?.(pool, candidate, { concurrencyRef });
|
package/lib/rotate.js
CHANGED
|
@@ -14,6 +14,9 @@ import {
|
|
|
14
14
|
isKeyBroken,
|
|
15
15
|
formatExhaustionMessage,
|
|
16
16
|
sortAttemptList,
|
|
17
|
+
getSessionSticky,
|
|
18
|
+
setSessionSticky,
|
|
19
|
+
clearSessionSticky,
|
|
17
20
|
} from './pool.js';
|
|
18
21
|
import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
|
|
19
22
|
import { createRequestBudget, BUDGET_EXHAUSTED_TIMEOUT, BUDGET_EXHAUSTED_ATTEMPTS } from './request-budget.js';
|
|
@@ -104,7 +107,8 @@ export function createRotate(deps) {
|
|
|
104
107
|
let lastFailure = null;
|
|
105
108
|
const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
|
|
106
109
|
let activePermit = null;
|
|
107
|
-
const
|
|
110
|
+
const sessionId = options?.sessionId ?? options?.session ?? options?.metadata?.sessionId ?? options?.chatId ?? options?.conversationId;
|
|
111
|
+
const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now(), sessionId };
|
|
108
112
|
setRotateStartMs(reqStore.startMs);
|
|
109
113
|
const wallNow = Date.now();
|
|
110
114
|
const quotaManaged = hasModelQuotaConfig(pool);
|
|
@@ -155,7 +159,15 @@ export function createRotate(deps) {
|
|
|
155
159
|
}
|
|
156
160
|
const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
|
|
157
161
|
if (strategy !== 'round-robin') {
|
|
158
|
-
attemptList = sortAttemptList(attemptList, strategy, {
|
|
162
|
+
attemptList = sortAttemptList(attemptList, strategy, {
|
|
163
|
+
latencyHistogram,
|
|
164
|
+
concurrencyTracker,
|
|
165
|
+
weights: pool.weightsMap,
|
|
166
|
+
tiers: pool.tiers,
|
|
167
|
+
sessionId,
|
|
168
|
+
sessionSticky: sessionId ? getSessionSticky(pool, sessionId) : null,
|
|
169
|
+
lastUsed: pool.state.lastUsed,
|
|
170
|
+
});
|
|
159
171
|
}
|
|
160
172
|
if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
|
|
161
173
|
// #193 / #365: prefer least-loaded key within limit with weights
|
|
@@ -178,6 +190,7 @@ export function createRotate(deps) {
|
|
|
178
190
|
|
|
179
191
|
const penalizeRef = (targetRef, errCode, errMsg) => {
|
|
180
192
|
if (!targetRef) return;
|
|
193
|
+
if (sessionId) clearSessionSticky(pool, sessionId);
|
|
181
194
|
if (errCode === LOCAL_QUOTA_CODE) return;
|
|
182
195
|
const _retry = parseRetryAfter(errMsg);
|
|
183
196
|
const _base = pool.cooldownMs ?? cooldownMs;
|
|
@@ -186,11 +199,16 @@ export function createRotate(deps) {
|
|
|
186
199
|
const cls = classifyFailure({ code: errCode, message: errMsg });
|
|
187
200
|
const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
|
|
188
201
|
|
|
189
|
-
// #337: connect quota-window calendar reset for quota exhaustion
|
|
190
|
-
//
|
|
191
|
-
//
|
|
192
|
-
const
|
|
193
|
-
/\b(?:insufficient[_\s-]+quota|
|
|
202
|
+
// #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
|
|
203
|
+
// Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
|
|
204
|
+
// must NOT be locked until midnight. Keep those on ordinary cooldowns.
|
|
205
|
+
const isRateThrottling = /\b(?:rpm|tpm|requests?\s+per\s+minute|tokens?\s+per\s+minute|per[-_\s]*minute|per[-_\s]*second|concurrency|requests?\s+limit|tokens?\s+limit|traffic)\b/i.test(errMsg) ||
|
|
206
|
+
(/\b(?:rate[-_\s]*limit)\b/i.test(errMsg) && !/\b(?:insufficient[_\s-]+quota|exceeded\s+(?:your\s+)?(?:current\s+)?quota|credit\s+balance|billing)\b/i.test(errMsg));
|
|
207
|
+
|
|
208
|
+
const isQuota = !isRateThrottling && (
|
|
209
|
+
(errCode === 'QUOTA' && !/\b(?:rpm|tpm|rate|traffic|throttle)\b/i.test(errMsg)) ||
|
|
210
|
+
/\b(?:quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota|insufficient[_\s-]+quota|credit\s+balance|billing\s+quota|out\s+of\s+credits?)\b/i.test(errMsg)
|
|
211
|
+
);
|
|
194
212
|
const qWindow = pool.quotaResetWindow ?? runtime0.quotaResetWindow;
|
|
195
213
|
if (isQuota && qWindow) {
|
|
196
214
|
const quotaResetAt = poolResetAt(pool, qWindow, now());
|
|
@@ -439,6 +457,7 @@ export function createRotate(deps) {
|
|
|
439
457
|
// #336: record success for the active key upon successful stream finish
|
|
440
458
|
if (activeRef) {
|
|
441
459
|
recordSuccess(pool, activeRef, now());
|
|
460
|
+
if (sessionId) setSessionSticky(pool, sessionId, activeRef);
|
|
442
461
|
}
|
|
443
462
|
if (activeRef) {
|
|
444
463
|
const usedTokens = extractUsageTokens(chunk?.usage ?? chunk?.metadata?.usage);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.62",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|