@goodandready/dsh-key-rotation 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.ru.md +1 -0
- package/lib/circuit-breaker.js +19 -1
- package/lib/client.js +254 -29
- package/lib/index.js +82 -1
- package/lib/persistence.js +146 -0
- package/lib/pool.js +17 -0
- package/lib/rotate.js +1 -1
- package/lib/routes-ops.js +11 -3
- package/lib/sanitize-snapshot.js +83 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
- **🕒 Accurate Midnight PST Resets**: Corrected UTC-8 timezone calculation offset sign for calendar quota reset windows.
|
|
46
46
|
- **🧹 Lifecycle Timer Cleanup**: Wrapped `canaryTimer` and `selfHealTimer` in Cordis effect scopes, eliminating background orphaned intervals on hot reload.
|
|
47
47
|
- **⚡ Stale Lock Recovery in Load Balancer**: Added expired lock detection to `pickLeastLoaded` for uninterrupted least-connections routing.
|
|
48
|
+
- **📊 Load Distribution & Modals (Changed in v0.8.5)**: Interactive segmented load distribution charts per pool, modal action confirmation, and 429/5xx backoff jitter (#283, #284).
|
|
48
49
|
- **🎨 Native Design System (Changed in v0.8.3)**: Unified with `dsh-clinebot` baseline: modular section cards, live pool telemetry stat boxes, pill badges, and complete semantic theme token styling (#281).
|
|
49
50
|
- **🌐 Localization (Changed in v0.8.2)**: Source strings are English-only. Russian/Chinese UI comes from the DSH core locale service and translation plugins (`props.t`). Active locale: host snapshot → first `navigator.languages` entry → `en` (#277).
|
|
50
51
|
|
package/README.ru.md
CHANGED
|
@@ -295,6 +295,7 @@ MIT © [GooDAnDReaDY](https://github.com/GooDAnDReaDY)
|
|
|
295
295
|
- **Очистка жизненного цикла**: Патч `credentials.resolve` и слушатели событий `ctx.on` (`llm/stream`, `agent/request-error`) переведены в скоупы `ctx.effect` с автоматическим восстановлением функций и отпиской при выгрузке плагина (#238, #239).
|
|
296
296
|
- **Роли секретов в схеме**: Полям `incidentGitHubToken` и `webhookActionToken` в схеме `Config` присвоена роль `.role('secret')` для маскирования в UI (#237).
|
|
297
297
|
- **Архитектура настроек**: Добавлена нативная интеграция со снимками `settingsScope` в карточке настроек с безопасным фоллбеком на HTTP-мост (#235).
|
|
298
|
+
- **Распределение нагрузки и модальные окна (Changed in v0.8.5)**: Диаграмма распределения нагрузки (.krot-load-chart), модальные подтверждения (.krot-modal) и джиттер кулдаунов при 429/5xx (#283, #284).
|
|
298
299
|
- **Нативная дизайн-система (Changed in v0.8.3)**: Унификация со стилем `dsh-clinebot`: модульные карточки секций, плитки оперативной телеметрии пулов, капсульные бейджи и системные CSS-токены DSH (#281).
|
|
299
300
|
- **Локализация (Changed in v0.8.2)**: Исходные строки только на английском (`en`). Русский и китайский приходят через core `props.t` / translation-плагины. Фолбек активной локали: snapshot → первый `navigator.languages` → `en`. Свой `settings.section` и встроенные `ru`/`zh`-таблицы удалены (#236, #275, #277).
|
|
300
301
|
- **Удаление мёртвого кода**: Удалена неиспользуемая функция `mountDashboard` после перехода на header chip (#240).
|
package/lib/circuit-breaker.js
CHANGED
|
@@ -90,4 +90,22 @@ export class CircuitBreaker {
|
|
|
90
90
|
if (provider) this._st.delete(provider);
|
|
91
91
|
else this._st.clear();
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
|
|
94
|
+
/** Restore entries from a persistence snapshot (#287). */
|
|
95
|
+
restore(snapshot) {
|
|
96
|
+
if (!snapshot || typeof snapshot !== 'object') return 0;
|
|
97
|
+
let n = 0;
|
|
98
|
+
for (const [provider, e] of Object.entries(snapshot)) {
|
|
99
|
+
if (!e || typeof e !== 'object') continue;
|
|
100
|
+
const state = e.state === BREAKER_OPEN || e.state === BREAKER_HALF_OPEN ? e.state : BREAKER_CLOSED;
|
|
101
|
+
this._st.set(provider, {
|
|
102
|
+
state,
|
|
103
|
+
fails: Number.isFinite(e.fails) ? e.fails : 0,
|
|
104
|
+
openedAt: Number.isFinite(e.openedAt) ? e.openedAt : 0,
|
|
105
|
+
probes: 0,
|
|
106
|
+
});
|
|
107
|
+
n += 1;
|
|
108
|
+
}
|
|
109
|
+
return n;
|
|
110
|
+
}
|
|
111
|
+
}
|
package/lib/client.js
CHANGED
|
@@ -38,6 +38,15 @@ window.__ModuleLoader__.load({
|
|
|
38
38
|
statPools: 'Configured Pools',
|
|
39
39
|
statKeys: 'Total Keys',
|
|
40
40
|
statHealthy: 'Healthy / Ready',
|
|
41
|
+
loadDistribution: 'Traffic Distribution',
|
|
42
|
+
statRequests: 'requests',
|
|
43
|
+
noTrafficYet: 'No traffic recorded yet',
|
|
44
|
+
confirm: 'Confirm',
|
|
45
|
+
cancel: 'Cancel',
|
|
46
|
+
confirmResetTitle: 'Reset cooldowns for {p}?',
|
|
47
|
+
confirmResetDesc: 'This will immediately clear all failure counts, cooldown timers, and reset the circuit breaker.',
|
|
48
|
+
confirmRemoveProvTitle: 'Remove provider {p}?',
|
|
49
|
+
confirmRemoveProvDesc: 'Are you sure you want to remove this provider and all configured keys from rotation?',
|
|
41
50
|
filterAll: 'All',
|
|
42
51
|
filterReady: 'Ready',
|
|
43
52
|
filterCooldown: 'In Cooldown',
|
|
@@ -107,6 +116,31 @@ window.__ModuleLoader__.load({
|
|
|
107
116
|
undoProvider: 'Provider removed',
|
|
108
117
|
undoKey: 'Key removed',
|
|
109
118
|
undo: 'Undo',
|
|
119
|
+
// #289 a11y / #290 states / #291 bulk / #292 locale
|
|
120
|
+
emptyTitle: 'No pools yet',
|
|
121
|
+
emptyDesc: 'Add a provider and at least one API key name to start rotating.',
|
|
122
|
+
errorTitle: 'Could not load settings',
|
|
123
|
+
retry: 'Retry',
|
|
124
|
+
unavailableTitle: 'Settings bridge unavailable',
|
|
125
|
+
unavailableDesc: 'The host did not expose the dsh-key-rotation settings namespace. Reopen Settings or check the plugin is enabled.',
|
|
126
|
+
modalClose: 'Close dialog',
|
|
127
|
+
dialogLabel: 'Confirmation',
|
|
128
|
+
loadChartAria: 'Key traffic distribution',
|
|
129
|
+
bulkCooldownPlaceholder: 'Bulk cooldown (ms)',
|
|
130
|
+
bulkApply: 'Apply to selected',
|
|
131
|
+
bulkRemove: 'Remove selected',
|
|
132
|
+
confirmBulkRemoveTitle: 'Remove {n} provider(s)?',
|
|
133
|
+
confirmBulkRemoveDesc: 'Selected providers and all their keys will be removed from rotation. You can undo once immediately after.',
|
|
134
|
+
selectProvider: 'Select provider {p}',
|
|
135
|
+
activeBadge: 'active',
|
|
136
|
+
healthScoreTitle: 'health score',
|
|
137
|
+
totalRequestsTitle: 'total requests',
|
|
138
|
+
exportProvider: 'Export this provider',
|
|
139
|
+
recentFailures: 'Recent failures ({n})',
|
|
140
|
+
noActivePools: t('noActivePools'),
|
|
141
|
+
headerPoolsTitle: 'Key rotation pools',
|
|
142
|
+
pausedBudget: 'paused',
|
|
143
|
+
sortUsage: 'Sort by usage',
|
|
110
144
|
};
|
|
111
145
|
|
|
112
146
|
// Коды, на которых имеет смысл переключать ключ. Список из хоста
|
|
@@ -229,6 +263,18 @@ window.__ModuleLoader__.load({
|
|
|
229
263
|
'.krot-stat-box{padding:12px 14px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;background:var(--dsw-alias-bg-layer-2);display:flex;flex-direction:column;gap:4px}',
|
|
230
264
|
'.krot-stat-val{font-size:20px;font-weight:700;color:var(--dsw-alias-label-primary)}',
|
|
231
265
|
'.krot-stat-lbl{font-size:12px;color:var(--dsw-alias-label-secondary)}',
|
|
266
|
+
'.krot-load-chart{margin:10px 0;padding:10px 12px;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;background:var(--dsw-alias-bg-layer-2)}',
|
|
267
|
+
'.krot-load-header{display:flex;justify-content:space-between;align-items:center;font-size:12px;color:var(--dsw-alias-label-secondary);margin-bottom:6px}',
|
|
268
|
+
'.krot-load-bar{display:flex;height:12px;border-radius:999px;overflow:hidden;background:var(--dsw-alias-bg-layer-1);border:1px solid var(--dsw-alias-border-l1)}',
|
|
269
|
+
'.krot-load-segment{height:100%;transition:width 0.2s ease}',
|
|
270
|
+
'.krot-load-legend{display:flex;flex-wrap:wrap;gap:8px;margin-top:6px;font-size:11px}',
|
|
271
|
+
'.krot-load-item{display:inline-flex;align-items:center;gap:4px}',
|
|
272
|
+
'.krot-load-dot{width:7px;height:7px;border-radius:50%}',
|
|
273
|
+
'.krot-modal-backdrop{position:fixed;top:0;left:0;right:0;bottom:0;background:rgba(0,0,0,0.55);z-index:9999;display:flex;align-items:center;justify-content:center;backdrop-filter:blur(2px)}',
|
|
274
|
+
'.krot-modal-card{width:90%;max-width:420px;background:var(--dsw-alias-bg-layer-3);border:1px solid var(--dsw-alias-border-l2);border-radius:12px;box-shadow:0 12px 36px rgba(0,0,0,0.28);padding:20px;display:flex;flex-direction:column;gap:12px}',
|
|
275
|
+
'.krot-modal-title{font-size:16px;font-weight:600;color:var(--dsw-alias-label-primary)}',
|
|
276
|
+
'.krot-modal-desc{font-size:13px;color:var(--dsw-alias-label-secondary);line-height:1.45}',
|
|
277
|
+
'.krot-modal-actions{display:flex;justify-content:flex-end;gap:10px;margin-top:8px}',
|
|
232
278
|
'.krot-badge{font-size:12px;padding:3px 10px;border-radius:999px;border:1px solid var(--dsw-alias-border-l2);display:inline-flex;align-items:center;gap:5px;font-weight:500;white-space:nowrap}',
|
|
233
279
|
'.krot-badge-ok{border-color:var(--dsw-alias-state-success-primary);color:var(--dsw-alias-state-success-primary);background:rgba(16,185,129,0.08)}',
|
|
234
280
|
'.krot-badge-warn{border-color:var(--dsw-alias-state-warning-primary);color:var(--dsw-alias-state-warning-primary);background:rgba(245,158,11,0.08)}',
|
|
@@ -388,15 +434,62 @@ window.__ModuleLoader__.load({
|
|
|
388
434
|
getScopeSnapshot
|
|
389
435
|
);
|
|
390
436
|
const [state, setState] = React.useState({ status: 'loading', value: null, revision: 0, error: '', providers: [] });
|
|
437
|
+
// #290: surface settingsScope availability as card state
|
|
438
|
+
React.useEffect(() => {
|
|
439
|
+
if (!settingsScope) {
|
|
440
|
+
setState((s) => (s.status === 'unavailable' ? s : { ...s, status: 'unavailable' }));
|
|
441
|
+
}
|
|
442
|
+
}, [settingsScope]);
|
|
391
443
|
const [draft, setDraft] = React.useState(null);
|
|
392
444
|
// ── all hooks live ABOVE any early return (React error 310 otherwise) ──
|
|
393
445
|
const [search, setSearch] = React.useState('');
|
|
394
446
|
const [statusFilter, setStatusFilter] = React.useState('all');
|
|
447
|
+
const [confirmModal, setConfirmModal] = React.useState(null);
|
|
395
448
|
const [optimisticReset, setOptimisticReset] = React.useState({});
|
|
396
449
|
const [selected, setSelected] = React.useState(new Set());
|
|
397
450
|
const [bulkCooldown, setBulkCooldown] = React.useState('');
|
|
398
451
|
const [undo, setUndo] = React.useState(null);
|
|
399
452
|
const undoTimer = React.useRef(null);
|
|
453
|
+
// #289 a11y: confirm dialog focus trap / Escape / restore (hooks above returns)
|
|
454
|
+
const modalReturnFocusRef = React.useRef(null);
|
|
455
|
+
const modalCardRef = React.useRef(null);
|
|
456
|
+
React.useEffect(() => {
|
|
457
|
+
if (!confirmModal) return undefined;
|
|
458
|
+
modalReturnFocusRef.current = typeof document !== 'undefined' ? document.activeElement : null;
|
|
459
|
+
const onKey = (e) => {
|
|
460
|
+
if (e.key === 'Escape') {
|
|
461
|
+
e.stopPropagation();
|
|
462
|
+
setConfirmModal(null);
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
if (e.key !== 'Tab' || !modalCardRef.current) return;
|
|
466
|
+
const focusables = modalCardRef.current.querySelectorAll('button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])');
|
|
467
|
+
if (!focusables.length) return;
|
|
468
|
+
const first = focusables[0];
|
|
469
|
+
const last = focusables[focusables.length - 1];
|
|
470
|
+
if (e.shiftKey && document.activeElement === first) {
|
|
471
|
+
e.preventDefault();
|
|
472
|
+
last.focus();
|
|
473
|
+
} else if (!e.shiftKey && document.activeElement === last) {
|
|
474
|
+
e.preventDefault();
|
|
475
|
+
first.focus();
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
document.addEventListener('keydown', onKey, true);
|
|
479
|
+
const id = requestAnimationFrame(() => {
|
|
480
|
+
if (!modalCardRef.current) return;
|
|
481
|
+
const cancelBtn = modalCardRef.current.querySelector('[data-krot-modal-cancel]');
|
|
482
|
+
(cancelBtn || modalCardRef.current.querySelector('button'))?.focus();
|
|
483
|
+
});
|
|
484
|
+
return () => {
|
|
485
|
+
document.removeEventListener('keydown', onKey, true);
|
|
486
|
+
cancelAnimationFrame(id);
|
|
487
|
+
const prev = modalReturnFocusRef.current;
|
|
488
|
+
if (prev && typeof prev.focus === 'function') {
|
|
489
|
+
try { prev.focus(); } catch (_) {}
|
|
490
|
+
}
|
|
491
|
+
};
|
|
492
|
+
}, [confirmModal]);
|
|
400
493
|
const [testing, setTesting] = React.useState('');
|
|
401
494
|
const [testResult, setTestResult] = React.useState({});
|
|
402
495
|
const [testAllProvider, setTestAllProvider] = React.useState('');
|
|
@@ -564,8 +657,31 @@ window.__ModuleLoader__.load({
|
|
|
564
657
|
})
|
|
565
658
|
.catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))));
|
|
566
659
|
};
|
|
567
|
-
|
|
568
|
-
|
|
660
|
+
// #290: explicit card states — loading / error / unavailable / empty / ready
|
|
661
|
+
if (state.status === 'loading' || (!val && state.status !== 'error' && state.status !== 'unavailable')) {
|
|
662
|
+
return React.createElement('div', { className: 'krot-state-block', role: 'status', 'aria-live': 'polite', style: { padding: '18px 4px', display: 'flex', flexDirection: 'column', gap: '8px' } },
|
|
663
|
+
React.createElement('p', { style: { color: 'var(--dsw-alias-label-tertiary)', fontSize: 13, margin: 0 } }, t('loading')),
|
|
664
|
+
React.createElement('div', {
|
|
665
|
+
className: 'krot-skeleton',
|
|
666
|
+
'aria-hidden': 'true',
|
|
667
|
+
style: {
|
|
668
|
+
height: 10, borderRadius: 999, background: 'linear-gradient(90deg, var(--dsw-alias-bg-layer-2), var(--dsw-alias-bg-layer-3), var(--dsw-alias-bg-layer-2))',
|
|
669
|
+
backgroundSize: '200% 100%', animation: 'krot-shimmer 1.2s ease-in-out infinite', maxWidth: 280,
|
|
670
|
+
},
|
|
671
|
+
})
|
|
672
|
+
);
|
|
673
|
+
}
|
|
674
|
+
if (state.status === 'error') {
|
|
675
|
+
return React.createElement('div', { className: 'krot-state-block', role: 'alert', style: { padding: '12px 0', display: 'flex', flexDirection: 'column', gap: '10px', alignItems: 'flex-start' } },
|
|
676
|
+
React.createElement('p', { className: 'krot-err', style: { margin: 0 } }, t('errorTitle') + (state.error ? (': ' + state.error) : '')),
|
|
677
|
+
React.createElement('button', { type: 'button', className: 'krot-btn', onClick: () => load() }, t('retry'))
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
if (state.status === 'unavailable' || !settingsScope) {
|
|
681
|
+
return React.createElement('div', { className: 'krot-state-block', role: 'status', style: { padding: '12px 0', display: 'flex', flexDirection: 'column', gap: '6px' } },
|
|
682
|
+
React.createElement('p', { style: { fontSize: 14, fontWeight: 600, color: 'var(--dsw-alias-label-primary)', margin: 0 } }, t('unavailableTitle')),
|
|
683
|
+
React.createElement('p', { className: 'krot-hint', style: { margin: 0 } }, t('unavailableDesc'))
|
|
684
|
+
);
|
|
569
685
|
}
|
|
570
686
|
|
|
571
687
|
const providers = state.providers;
|
|
@@ -892,7 +1008,7 @@ window.__ModuleLoader__.load({
|
|
|
892
1008
|
const parts = [];
|
|
893
1009
|
if (providerStatus.budgetDaily > 0) parts.push('$' + (providerStatus.todayCost ?? 0).toFixed(2) + '/' + '$' + providerStatus.budgetDaily);
|
|
894
1010
|
if (providerStatus.budgetWeekly > 0) parts.push('week $' + (providerStatus.weeklyCost ?? 0).toFixed(2) + '/' + '$' + providerStatus.budgetWeekly);
|
|
895
|
-
if (worst >= 1 && providerStatus.pauseOnBudget) parts.push('·
|
|
1011
|
+
if (worst >= 1 && providerStatus.pauseOnBudget) parts.push('· ' + t('pausedBudget'));
|
|
896
1012
|
return h('p', { className: 'krot-hint', style: { color } }, t('budgetLabel') + ' ' + parts.join(' · '));
|
|
897
1013
|
})()
|
|
898
1014
|
: null;
|
|
@@ -913,15 +1029,73 @@ window.__ModuleLoader__.load({
|
|
|
913
1029
|
document.body.appendChild(a); a.click(); a.remove();
|
|
914
1030
|
} }, t('exportCsv'));
|
|
915
1031
|
|
|
1032
|
+
const loadChart = (() => {
|
|
1033
|
+
if (keys.length === 0) return null;
|
|
1034
|
+
const ps = status[entry.provider];
|
|
1035
|
+
const keyUsages = keys.map((k, idx) => {
|
|
1036
|
+
const hit = ps && Array.isArray(ps.keys) ? ps.keys.find((x) => x.ref === k) : null;
|
|
1037
|
+
const u = hit && typeof hit.usage === 'number' ? hit.usage : 0;
|
|
1038
|
+
return { ref: k, index: idx, usage: u };
|
|
1039
|
+
});
|
|
1040
|
+
const total = keyUsages.reduce((acc, x) => acc + x.usage, 0);
|
|
1041
|
+
const palette = [
|
|
1042
|
+
'var(--dsw-alias-state-brand-primary, #6366f1)',
|
|
1043
|
+
'var(--dsw-alias-state-success-primary, #10b981)',
|
|
1044
|
+
'var(--dsw-alias-state-warning-primary, #f59e0b)',
|
|
1045
|
+
'var(--dsw-alias-state-info-primary, #3b82f6)',
|
|
1046
|
+
'#8b5cf6',
|
|
1047
|
+
'#ec4899',
|
|
1048
|
+
'#14b8a6',
|
|
1049
|
+
];
|
|
1050
|
+
return h('div', { className: 'krot-load-chart' },
|
|
1051
|
+
h('div', { className: 'krot-load-header' },
|
|
1052
|
+
h('span', { style: { fontWeight: 600 } }, '📊 ' + t('loadDistribution')),
|
|
1053
|
+
h('span', { className: 'krot-sr-only', style: { position: 'absolute', width: 1, height: 1, padding: 0, margin: -1, overflow: 'hidden', clip: 'rect(0,0,0,0)', border: 0 } },
|
|
1054
|
+
t('loadChartAria') + ': ' + (total > 0 ? keyUsages.map((x) => t('keyLabel').replace('{n}', String(x.index + 1)) + ' ' + x.usage).join(', ') : t('noTrafficYet'))),
|
|
1055
|
+
h('span', null, total > 0 ? (total + ' ' + t('statRequests')) : t('noTrafficYet'))
|
|
1056
|
+
),
|
|
1057
|
+
h('div', { className: 'krot-load-bar' },
|
|
1058
|
+
total > 0
|
|
1059
|
+
? keyUsages.map((x, i) => {
|
|
1060
|
+
if (x.usage === 0) return null;
|
|
1061
|
+
const pct = Math.max(1, Math.round((x.usage / total) * 100));
|
|
1062
|
+
const color = palette[i % palette.length];
|
|
1063
|
+
return h('div', {
|
|
1064
|
+
key: x.ref,
|
|
1065
|
+
className: 'krot-load-segment',
|
|
1066
|
+
style: { width: pct + '%', background: color },
|
|
1067
|
+
title: x.ref + ': ' + x.usage + ' (' + pct + '%)',
|
|
1068
|
+
});
|
|
1069
|
+
})
|
|
1070
|
+
: h('div', { className: 'krot-load-segment', style: { width: '100%', background: 'var(--dsw-alias-bg-layer-2)', opacity: 0.6 } })
|
|
1071
|
+
),
|
|
1072
|
+
total > 0 ? h('div', { className: 'krot-load-legend' },
|
|
1073
|
+
keyUsages.map((x, i) => {
|
|
1074
|
+
const pct = total > 0 ? Math.round((x.usage / total) * 100) : 0;
|
|
1075
|
+
const color = palette[i % palette.length];
|
|
1076
|
+
return h('span', { key: x.ref, className: 'krot-load-item' },
|
|
1077
|
+
h('span', { className: 'krot-load-dot', style: { background: color } }),
|
|
1078
|
+
h('span', { style: { color: 'var(--dsw-alias-label-secondary)' } }, t('keyLabel').replace('{n}', String(x.index + 1)) + ': ' + x.usage + ' (' + pct + '%)')
|
|
1079
|
+
);
|
|
1080
|
+
})
|
|
1081
|
+
) : null
|
|
1082
|
+
);
|
|
1083
|
+
})();
|
|
1084
|
+
|
|
916
1085
|
return h('div', { key: pIndex, className: 'krot-prov' },
|
|
917
1086
|
h('div', { className: 'krot-prov-head' },
|
|
918
|
-
h('input', {
|
|
1087
|
+
h('input', {
|
|
1088
|
+
type: 'checkbox',
|
|
1089
|
+
'aria-label': t('selectProvider').replace('{p}', entry.provider),
|
|
1090
|
+
checked: selected.has(entry.provider),
|
|
1091
|
+
onChange: (e) => { const ns = new Set(selected); if (e.target.checked) ns.add(entry.provider); else ns.delete(entry.provider); setSelected(ns); },
|
|
1092
|
+
}),
|
|
919
1093
|
btn(t('exportOne'), () => {
|
|
920
1094
|
const data = JSON.stringify([entry], null, 2);
|
|
921
1095
|
const blob = new Blob([data], { type: 'application/json' });
|
|
922
1096
|
const url = URL.createObjectURL(blob);
|
|
923
1097
|
const a = document.createElement('a'); a.href = url; a.download = entry.provider + '.json'; a.click(); URL.revokeObjectURL(url);
|
|
924
|
-
}, { title: '
|
|
1098
|
+
}, { title: t('exportProvider') }),
|
|
925
1099
|
btn('⇅', () => {
|
|
926
1100
|
const ps = status[entry.provider];
|
|
927
1101
|
if (!ps || !Array.isArray(ps.keys)) return;
|
|
@@ -933,19 +1107,27 @@ window.__ModuleLoader__.load({
|
|
|
933
1107
|
next[pIndex] = { ...next[pIndex], keys: sorted };
|
|
934
1108
|
return { ...cur, providers: next };
|
|
935
1109
|
});
|
|
936
|
-
}, { title: '
|
|
1110
|
+
}, { title: t('sortUsage') }),
|
|
937
1111
|
h('select', { className: 'krot-in', value: entry.provider, onChange: (e) => setProvider(pIndex, e.target.value) }, options),
|
|
938
1112
|
(() => {
|
|
939
1113
|
const ps = status[entry.provider];
|
|
940
1114
|
const score = ps && typeof ps.healthScore === 'number' ? ps.healthScore : null;
|
|
941
1115
|
if (score === null) return null;
|
|
942
1116
|
const color = score > 80 ? 'var(--dsw-alias-state-success-primary)' : score >= 50 ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-state-error-primary)';
|
|
943
|
-
return h('span', { className: 'krot-tail', title: '
|
|
1117
|
+
return h('span', { className: 'krot-tail', title: t('healthScoreTitle'), style: { flex: 'none', color, fontWeight: 700 } }, String(score));
|
|
944
1118
|
})(),
|
|
945
|
-
(() => { const ps = status[entry.provider]; const tot = ps && typeof ps.totalUsage === 'number' ? ps.totalUsage : null; return tot !== null ? h('span', { className: 'krot-tail', title: '
|
|
946
|
-
btn('✕', () =>
|
|
1119
|
+
(() => { const ps = status[entry.provider]; const tot = ps && typeof ps.totalUsage === 'number' ? ps.totalUsage : null; return tot !== null ? h('span', { className: 'krot-tail', title: t('totalRequestsTitle'), style: { flex: 'none' } }, String(tot)) : null; })(),
|
|
1120
|
+
btn('✕', () => setConfirmModal({
|
|
1121
|
+
title: t('confirmRemoveProvTitle').replace('{p}', entry.provider),
|
|
1122
|
+
desc: t('confirmRemoveProvDesc'),
|
|
1123
|
+
actionLabel: t('removeProvider'),
|
|
1124
|
+
danger: true,
|
|
1125
|
+
onConfirm: () => removeProvider(pIndex),
|
|
1126
|
+
}), { title: t('removeProvider') }),
|
|
947
1127
|
),
|
|
948
|
-
filterBar,
|
|
1128
|
+
filterBar,
|
|
1129
|
+
loadChart,
|
|
1130
|
+
h('div', { className: 'krot-keys' }, keyRows),
|
|
949
1131
|
h('div', { className: 'krot-foot' },
|
|
950
1132
|
btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle') }),
|
|
951
1133
|
switchesLine,
|
|
@@ -955,8 +1137,14 @@ window.__ModuleLoader__.load({
|
|
|
955
1137
|
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('div', { style: { display: 'flex', gap: '2px', alignItems: 'end', height: '24px', marginTop: '4px' } }, (() => { const now = Date.now(); const buckets = Array(24).fill(0); for (const ev of providerStatus.events) { const h = Math.floor((now - ev.at) / 3600000); if (h >= 0 && h < 24) buckets[23 - h]++; } const max = Math.max(1, ...buckets); return buckets.map((c, i) => h('div', { key: i, title: c + ' switches', style: { flex: 1, background: c ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-border-l2)', height: (c / max * 24) + 'px', minHeight: '2px', borderRadius: '2px' } })); })()) : null),
|
|
956
1138
|
// #224: 7-day switches per day (client-side, from the same events)
|
|
957
1139
|
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('div', { style: { display: 'flex', gap: '2px', alignItems: 'end', height: '16px', marginTop: '2px' } }, (() => { const now = Date.now(); const days = Array(7).fill(0); for (const ev of providerStatus.events) { const d = Math.floor((now - ev.at) / 86400000); if (d >= 0 && d < 7) days[6 - d]++; } const max = Math.max(1, ...days); return days.map((c, i) => h('div', { key: i, title: c + ' switches · day -' + (6 - i), style: { flex: 1, background: c ? 'var(--dsw-alias-state-info-primary, var(--dsw-alias-state-warning-primary))' : 'var(--dsw-alias-border-l2)', height: (c / max * 16) + 'px', minHeight: '2px', borderRadius: '2px' } })); })()) : null),
|
|
958
|
-
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('details', { style: { fontSize: '11px', marginTop: '6px' } }, h('summary', null, '
|
|
959
|
-
btn(t('resetCooldown'), () =>
|
|
1140
|
+
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('details', { style: { fontSize: '11px', marginTop: '6px' } }, h('summary', null, t('recentFailures').replace('{n}', String(providerStatus.events.length))), h('ul', { style: { margin: '4px 0 0', paddingLeft: '16px' } }, providerStatus.events.slice().reverse().map((ev, i) => h('li', { key: i, style: ev.type === 'probe' ? { opacity: .5 } : null }, new Date(ev.at).toLocaleTimeString() + ' ' + (ev.type === 'probe' ? '[probe] ' : '') + ev.ref + ' ' + ev.reason + ' cd=' + ev.cooldownMs)) )) : null),
|
|
1141
|
+
btn(t('resetCooldown'), () => setConfirmModal({
|
|
1142
|
+
title: t('confirmResetTitle').replace('{p}', entry.provider),
|
|
1143
|
+
desc: t('confirmResetDesc'),
|
|
1144
|
+
actionLabel: t('resetCooldown'),
|
|
1145
|
+
danger: false,
|
|
1146
|
+
onConfirm: () => doReset(entry.provider),
|
|
1147
|
+
}), { disabled: !(providerStatus && providerStatus.switches > 0) || resetting === entry.provider, title: t('resetCooldown') }),
|
|
960
1148
|
btn(testAllProvider === entry.provider ? t('testing') : t('testAll'), () => doTestAll(entry.provider), { disabled: testAllProvider === entry.provider, title: t('testAll') }),
|
|
961
1149
|
exportCsv,
|
|
962
1150
|
),
|
|
@@ -964,7 +1152,11 @@ window.__ModuleLoader__.load({
|
|
|
964
1152
|
});
|
|
965
1153
|
|
|
966
1154
|
const noProviders = providers.length === 0
|
|
967
|
-
? h('
|
|
1155
|
+
? h('div', { className: 'krot-empty', role: 'status', style: { display: 'flex', flexDirection: 'column', gap: '6px', padding: '16px 14px', border: '1px dashed var(--dsw-alias-border-l2)', borderRadius: 10, background: 'var(--dsw-alias-bg-layer-2)' } },
|
|
1156
|
+
h('p', { style: { margin: 0, fontWeight: 600, fontSize: 14, color: 'var(--dsw-alias-label-primary)' } }, t('emptyTitle')),
|
|
1157
|
+
h('p', { className: 'krot-hint', style: { margin: 0 } }, t('emptyDesc')),
|
|
1158
|
+
h('p', { className: 'krot-hint', style: { margin: 0 } }, t('noProviders')),
|
|
1159
|
+
)
|
|
968
1160
|
: null;
|
|
969
1161
|
|
|
970
1162
|
const allConfiguredProviders = Array.isArray(val?.providers) ? val.providers : [];
|
|
@@ -1015,19 +1207,36 @@ window.__ModuleLoader__.load({
|
|
|
1015
1207
|
h('div', { className: 'krot-foot', style: { marginBottom: '8px' } },
|
|
1016
1208
|
h('input', {
|
|
1017
1209
|
className: 'krot-in',
|
|
1018
|
-
placeholder: '
|
|
1210
|
+
placeholder: t('bulkCooldownPlaceholder'),
|
|
1019
1211
|
value: bulkCooldown,
|
|
1020
1212
|
onChange: (e) => setBulkCooldown(e.target.value),
|
|
1021
1213
|
style: { maxWidth: '160px' },
|
|
1022
1214
|
}),
|
|
1023
|
-
btn('
|
|
1215
|
+
btn(t('bulkApply'), () => {
|
|
1024
1216
|
const v = Number(bulkCooldown); if (!v) return;
|
|
1025
1217
|
setField((cur) => {
|
|
1026
1218
|
const next = [...(cur.providers ?? [])];
|
|
1027
1219
|
for (let i = 0; i < next.length; i++) if (selected.has(next[i].provider)) next[i] = { ...next[i], cooldownMs: v };
|
|
1028
1220
|
return { ...cur, providers: next };
|
|
1029
1221
|
});
|
|
1030
|
-
}, { disabled: selected.size === 0 || !bulkCooldown })
|
|
1222
|
+
}, { disabled: selected.size === 0 || !bulkCooldown }),
|
|
1223
|
+
btn(t('bulkRemove'), () => {
|
|
1224
|
+
const ids = [...selected];
|
|
1225
|
+
if (!ids.length) return;
|
|
1226
|
+
setConfirmModal({
|
|
1227
|
+
title: t('confirmBulkRemoveTitle').replace('{n}', String(ids.length)),
|
|
1228
|
+
desc: t('confirmBulkRemoveDesc'),
|
|
1229
|
+
actionLabel: t('bulkRemove'),
|
|
1230
|
+
danger: true,
|
|
1231
|
+
onConfirm: () => {
|
|
1232
|
+
setField((cur) => {
|
|
1233
|
+
const next = (cur.providers ?? []).filter((p) => !ids.has(p.provider));
|
|
1234
|
+
return { ...cur, providers: next };
|
|
1235
|
+
});
|
|
1236
|
+
setSelected(new Set());
|
|
1237
|
+
},
|
|
1238
|
+
});
|
|
1239
|
+
}, { disabled: selected.size === 0, className: 'krot-btn-danger' })
|
|
1031
1240
|
),
|
|
1032
1241
|
h('div', { className: 'krot-keys' }, providerRows),
|
|
1033
1242
|
h('div', { className: 'krot-foot', style: { marginTop: '10px' } },
|
|
@@ -1039,7 +1248,7 @@ window.__ModuleLoader__.load({
|
|
|
1039
1248
|
const failoverSection = h('div', { className: 'krot-section-card' },
|
|
1040
1249
|
h('div', { className: 'krot-section-title' },
|
|
1041
1250
|
h('span', null, '⚡ ' + t('codesTitle')),
|
|
1042
|
-
h('span', { className: 'krot-badge krot-badge-warn' }, selectedCodes.size + '
|
|
1251
|
+
h('span', { className: 'krot-badge krot-badge-warn' }, selectedCodes.size + ' ' + t('activeBadge'))
|
|
1043
1252
|
),
|
|
1044
1253
|
h('div', { className: 'krot-section-desc' }, t('sectionFailoverDesc')),
|
|
1045
1254
|
h('div', { className: 'krot-codes' }, codeList.map((code) => h('label', { key: code, className: 'krot-code' },
|
|
@@ -1124,7 +1333,34 @@ window.__ModuleLoader__.load({
|
|
|
1124
1333
|
) : null,
|
|
1125
1334
|
].filter(Boolean);
|
|
1126
1335
|
|
|
1336
|
+
const confirmModalEl = confirmModal ? h('div', {
|
|
1337
|
+
className: 'krot-modal-backdrop',
|
|
1338
|
+
onClick: () => setConfirmModal(null),
|
|
1339
|
+
},
|
|
1340
|
+
h('div', {
|
|
1341
|
+
className: 'krot-modal-card',
|
|
1342
|
+
ref: modalCardRef,
|
|
1343
|
+
role: 'dialog',
|
|
1344
|
+
'aria-modal': 'true',
|
|
1345
|
+
'aria-labelledby': 'krot-modal-title',
|
|
1346
|
+
'aria-describedby': 'krot-modal-desc',
|
|
1347
|
+
onClick: (e) => e.stopPropagation(),
|
|
1348
|
+
},
|
|
1349
|
+
h('div', { className: 'krot-modal-title', id: 'krot-modal-title' }, confirmModal.title),
|
|
1350
|
+
h('div', { className: 'krot-modal-desc', id: 'krot-modal-desc' }, confirmModal.desc),
|
|
1351
|
+
h('div', { className: 'krot-modal-actions' },
|
|
1352
|
+
btn(t('cancel'), () => setConfirmModal(null), { 'data-krot-modal-cancel': '1' }),
|
|
1353
|
+
btn(confirmModal.actionLabel || t('confirm'), () => {
|
|
1354
|
+
const fn = confirmModal.onConfirm;
|
|
1355
|
+
setConfirmModal(null);
|
|
1356
|
+
if (fn) fn();
|
|
1357
|
+
}, { primary: !confirmModal.danger, className: confirmModal.danger ? 'krot-btn-danger' : undefined })
|
|
1358
|
+
)
|
|
1359
|
+
)
|
|
1360
|
+
) : null;
|
|
1361
|
+
|
|
1127
1362
|
return h('div', { className: 'krot' },
|
|
1363
|
+
confirmModalEl,
|
|
1128
1364
|
statsSection,
|
|
1129
1365
|
poolsSection,
|
|
1130
1366
|
failoverSection,
|
|
@@ -1191,17 +1427,6 @@ window.__ModuleLoader__.load({
|
|
|
1191
1427
|
type: 'button',
|
|
1192
1428
|
className: 'krot-header-chip',
|
|
1193
1429
|
title: 'Key Rotation',
|
|
1194
|
-
sectionStats: 'Pool Health & Telemetry',
|
|
1195
|
-
sectionStatsDesc: 'Real-time telemetry of active key pools, healthy credentials, and failover status.',
|
|
1196
|
-
sectionFailover: 'Failover Triggers & Error Codes',
|
|
1197
|
-
sectionFailoverDesc: 'Select error conditions that trigger automatic switching to the next available key.',
|
|
1198
|
-
sectionTiming: 'Timing & Rotation Schedule',
|
|
1199
|
-
sectionTimingDesc: 'Configure cooldown durations and scheduled periodic rotation windows.',
|
|
1200
|
-
sectionBackup: 'Backup & Snapshots',
|
|
1201
|
-
sectionBackupDesc: 'Export or import provider pools and create full plugin configuration snapshots.',
|
|
1202
|
-
statPools: 'Configured Pools',
|
|
1203
|
-
statKeys: 'Total Keys',
|
|
1204
|
-
statHealthy: 'Healthy / Ready',
|
|
1205
1430
|
onClick: () => setOpen((v) => !v),
|
|
1206
1431
|
},
|
|
1207
1432
|
h('span', { style: { width: '8px', height: '8px', borderRadius: '50%', background: color, flex: 'none', boxShadow: `0 0 6px ${color}` } }),
|
|
@@ -1209,7 +1434,7 @@ window.__ModuleLoader__.load({
|
|
|
1209
1434
|
h('span', { style: { fontSize: '9px', opacity: 0.6 } }, open ? '▲' : '▼')
|
|
1210
1435
|
),
|
|
1211
1436
|
open ? h('div', { className: 'krot-popover' },
|
|
1212
|
-
h('div', { className: 'krot-pop-title' }, '
|
|
1437
|
+
h('div', { className: 'krot-pop-title' }, t('headerPoolsTitle')),
|
|
1213
1438
|
!poolEntries.length ? h('div', { style: { fontSize: '12px', color: 'var(--dsw-alias-label-tertiary)' } }, 'No active pools') : null,
|
|
1214
1439
|
poolEntries.map(([name, p]) => {
|
|
1215
1440
|
const c = p.exhausted ? 'var(--dsw-alias-state-error-primary)' : (p.healthy < p.total ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-state-success-primary)');
|
package/lib/index.js
CHANGED
|
@@ -49,6 +49,8 @@ import { BoundedMap } from './bounded-map.js';
|
|
|
49
49
|
import { classifyFailure } from './error-taxonomy.js';
|
|
50
50
|
import { safeParseJson } from './atomic-io.js';
|
|
51
51
|
import { registerOpsRoutes } from './routes-ops.js';
|
|
52
|
+
import { StatePersistence, resolveStatePath } from './persistence.js';
|
|
53
|
+
import path from 'node:path';
|
|
52
54
|
|
|
53
55
|
/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
|
|
54
56
|
const PIAI_NS = 'llm-pi-ai';
|
|
@@ -113,6 +115,9 @@ let moduleBreaker = null;
|
|
|
113
115
|
let moduleNotifyQueue = null;
|
|
114
116
|
// Global config accessor safe against early initialization
|
|
115
117
|
let getConfig = () => null;
|
|
118
|
+
function verboseLoggingOn() {
|
|
119
|
+
try { return Boolean(getConfig()?.verboseLogging); } catch (_) { return false; }
|
|
120
|
+
}
|
|
116
121
|
let getRuntime = () => null;
|
|
117
122
|
let sandboxRunner = null;
|
|
118
123
|
const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
|
|
@@ -204,6 +209,9 @@ export const Config = Schema.object({
|
|
|
204
209
|
circuitBreakerThreshold: Schema.number().default(5),
|
|
205
210
|
circuitBreakerOpenMs: Schema.number().default(30000),
|
|
206
211
|
circuitBreakerHalfOpenProbes: Schema.number().default(1),
|
|
212
|
+
// #287 persistence across restarts
|
|
213
|
+
persistenceEnabled: Schema.boolean().default(true),
|
|
214
|
+
persistencePath: Schema.string().default(''),
|
|
207
215
|
switchNotifyThrottleMs: Schema.number().default(60000),
|
|
208
216
|
warnBelowHealthy: Schema.number().default(0),
|
|
209
217
|
latencySloMs: Schema.number().default(0),
|
|
@@ -278,6 +286,77 @@ export function apply(ctx, config = {}) {
|
|
|
278
286
|
// ── key-pool state, persisted across config reloads ──
|
|
279
287
|
// base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
|
|
280
288
|
const poolState = new Map();
|
|
289
|
+
|
|
290
|
+
// #287: persist cooldowns / circuit / quota across DSH reloads.
|
|
291
|
+
// Path resolution: explicit config -> host data dir -> DSH_HOME -> cwd.
|
|
292
|
+
// Disabled (with a single warn) when no writable directory can be found.
|
|
293
|
+
let statePersistence = null;
|
|
294
|
+
try {
|
|
295
|
+
const cfg0 = getConfig() ?? config ?? {};
|
|
296
|
+
// Cordis ctx properties require inject; bare getters throw.
|
|
297
|
+
// Use only host env/cwd so apply() never trips the injector.
|
|
298
|
+
const hostDirs = [
|
|
299
|
+
process.env.DSH_HOME,
|
|
300
|
+
process.cwd(),
|
|
301
|
+
].filter((d) => typeof d === 'string' && d.length > 0);
|
|
302
|
+
const resolvedPath = resolveStatePath({
|
|
303
|
+
configuredPath: cfg0.persistencePath,
|
|
304
|
+
dataDir: hostDirs[0],
|
|
305
|
+
});
|
|
306
|
+
if (cfg0.persistenceEnabled !== false && resolvedPath) {
|
|
307
|
+
statePersistence = new StatePersistence({ filePath: resolvedPath });
|
|
308
|
+
statePersistence.load().then((snap) => {
|
|
309
|
+
if (!snap) return;
|
|
310
|
+
try {
|
|
311
|
+
StatePersistence.restorePools(poolState, snap);
|
|
312
|
+
if (moduleBreaker && snap.circuit) moduleBreaker.restore(snap.circuit);
|
|
313
|
+
if (verboseLoggingOn()) {
|
|
314
|
+
console.warn(`[dsh-key-rotation] restored ${Object.keys(snap.pools ?? {}).length} pool state(s) from ${path.basename(resolvedPath)}`);
|
|
315
|
+
}
|
|
316
|
+
} catch (e) {
|
|
317
|
+
console.warn('[dsh-key-rotation] persistence restore failed', e?.message ?? e);
|
|
318
|
+
}
|
|
319
|
+
}).catch(() => {});
|
|
320
|
+
} else if (cfg0.persistenceEnabled !== false && !resolvedPath) {
|
|
321
|
+
console.warn('[dsh-key-rotation] persistence disabled: no data directory');
|
|
322
|
+
}
|
|
323
|
+
} catch (e) {
|
|
324
|
+
console.warn('[dsh-key-rotation] persistence init failed', e?.message ?? e);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function persistenceSnapshot() {
|
|
328
|
+
if (!statePersistence) return null;
|
|
329
|
+
return StatePersistence.serialize({
|
|
330
|
+
poolState,
|
|
331
|
+
circuitSnapshot: moduleBreaker ? moduleBreaker.snapshot() : {},
|
|
332
|
+
quotaSnapshot: {},
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function schedulePersist() {
|
|
337
|
+
if (!statePersistence) return;
|
|
338
|
+
const snap = persistenceSnapshot();
|
|
339
|
+
if (snap) statePersistence.save(snap);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
ctx.effect(() => {
|
|
343
|
+
const timer = setInterval(() => { try { schedulePersist(); } catch (_) {} }, 15000);
|
|
344
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
345
|
+
return () => {
|
|
346
|
+
clearInterval(timer);
|
|
347
|
+
try {
|
|
348
|
+
if (statePersistence) {
|
|
349
|
+
const snap = persistenceSnapshot();
|
|
350
|
+
if (snap) {
|
|
351
|
+
statePersistence.save(snap);
|
|
352
|
+
// best-effort flush; dispose clears the debounce timer
|
|
353
|
+
void statePersistence.flush();
|
|
354
|
+
}
|
|
355
|
+
statePersistence.dispose();
|
|
356
|
+
}
|
|
357
|
+
} catch (_) {}
|
|
358
|
+
};
|
|
359
|
+
}, 'dsh-key-rotation: state persistence');
|
|
281
360
|
// Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
|
|
282
361
|
ctx.effect(() => {
|
|
283
362
|
const id = setInterval(() => {
|
|
@@ -747,6 +826,7 @@ export function apply(ctx, config = {}) {
|
|
|
747
826
|
poolState,
|
|
748
827
|
getRotationDisabled: () => rotationDisabled,
|
|
749
828
|
setRotationDisabled: (v) => { rotationDisabled = v; },
|
|
829
|
+
circuitBreaker: moduleBreaker,
|
|
750
830
|
});
|
|
751
831
|
|
|
752
832
|
ctx.effect(() => ctx.on('llm/stream', (options, next) => {
|
|
@@ -783,8 +863,9 @@ export function apply(ctx, config = {}) {
|
|
|
783
863
|
if (moduleBreaker) moduleBreaker.onFailure(provider);
|
|
784
864
|
const ref = pool.state.lastUsed;
|
|
785
865
|
if (ref) {
|
|
786
|
-
const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000, undefined, cls.soft);
|
|
866
|
+
const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000, undefined, cls.soft, true);
|
|
787
867
|
pushEvent(pool, ref, code || 'UNKNOWN', backoff);
|
|
868
|
+
schedulePersist();
|
|
788
869
|
pool.state.switches = (pool.state.switches ?? 0) + 1;
|
|
789
870
|
pool.state.lastReason = code || 'UNKNOWN';
|
|
790
871
|
pool.state.lastSwitchAt = Date.now();
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// lib/persistence.js — atomic snapshot of rotation state across restarts (#287).
|
|
2
|
+
// Cooldowns, circuit-breaker entries and quota counters live in memory; a DSH
|
|
3
|
+
// reload would otherwise forget cooldowns and stampede recovered keys.
|
|
4
|
+
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { atomicWriteJson, safeReadJson } from './atomic-io.js';
|
|
7
|
+
|
|
8
|
+
const DEFAULT_FILE = 'dsh-key-rotation-state.json';
|
|
9
|
+
const SAVE_DEBOUNCE_MS = 400;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {{ filePath: string, now?: () => number }} opts
|
|
13
|
+
*/
|
|
14
|
+
export class StatePersistence {
|
|
15
|
+
constructor({ filePath, now = Date.now } = {}) {
|
|
16
|
+
if (!filePath || typeof filePath !== 'string') {
|
|
17
|
+
throw new Error('StatePersistence: filePath is required');
|
|
18
|
+
}
|
|
19
|
+
this.filePath = filePath;
|
|
20
|
+
this._now = now;
|
|
21
|
+
this._timer = null;
|
|
22
|
+
this._dirty = false;
|
|
23
|
+
this._lastWrite = 0;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Build a plain snapshot from live maps/objects. */
|
|
27
|
+
static serialize({ poolState, circuitSnapshot, quotaSnapshot }) {
|
|
28
|
+
const pools = {};
|
|
29
|
+
if (poolState && typeof poolState.forEach === 'function') {
|
|
30
|
+
poolState.forEach((st, base) => {
|
|
31
|
+
const failedUntil = {};
|
|
32
|
+
if (st && st.failedUntil && typeof st.failedUntil.forEach === 'function') {
|
|
33
|
+
st.failedUntil.forEach((until, ref) => {
|
|
34
|
+
if (Number.isFinite(until)) failedUntil[ref] = until;
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
pools[base] = {
|
|
38
|
+
failedUntil,
|
|
39
|
+
pointer: Number.isFinite(st?.pointer) ? st.pointer : 0,
|
|
40
|
+
lastUsed: typeof st?.lastUsed === 'string' ? st.lastUsed : null,
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
version: 1,
|
|
46
|
+
savedAt: Date.now(),
|
|
47
|
+
pools,
|
|
48
|
+
circuit: circuitSnapshot && typeof circuitSnapshot === 'object' ? circuitSnapshot : {},
|
|
49
|
+
quota: quotaSnapshot && typeof quotaSnapshot === 'object' ? quotaSnapshot : {},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
_nowSafe() {
|
|
54
|
+
const n = Number(this._now?.() ?? Date.now());
|
|
55
|
+
return Number.isFinite(n) ? n : Date.now();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Schedule a debounced atomic write. Never throws to the caller.
|
|
60
|
+
* @param {object} payload result of StatePersistence.serialize
|
|
61
|
+
*/
|
|
62
|
+
save(payload) {
|
|
63
|
+
this._pending = payload;
|
|
64
|
+
this._dirty = true;
|
|
65
|
+
if (this._timer) return;
|
|
66
|
+
this._timer = setTimeout(() => {
|
|
67
|
+
this._timer = null;
|
|
68
|
+
void this.flush();
|
|
69
|
+
}, SAVE_DEBOUNCE_MS);
|
|
70
|
+
if (typeof this._timer.unref === 'function') this._timer.unref();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async flush() {
|
|
74
|
+
if (!this._dirty || !this._pending) return false;
|
|
75
|
+
const payload = this._pending;
|
|
76
|
+
this._dirty = false;
|
|
77
|
+
this._pending = null;
|
|
78
|
+
try {
|
|
79
|
+
await atomicWriteJson(this.filePath, payload);
|
|
80
|
+
this._lastWrite = this._nowSafe();
|
|
81
|
+
return true;
|
|
82
|
+
} catch {
|
|
83
|
+
// Keep previous good file; mark dirty so a later save retries.
|
|
84
|
+
this._dirty = true;
|
|
85
|
+
this._pending = this._pending || payload;
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Load a snapshot. Corrupt/missing file → null (never wipes memory).
|
|
92
|
+
* @returns {Promise<object|null>}
|
|
93
|
+
*/
|
|
94
|
+
async load() {
|
|
95
|
+
const raw = await safeReadJson(this.filePath, null);
|
|
96
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
|
|
97
|
+
if (raw.version !== 1) return null;
|
|
98
|
+
return raw;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Restore poolState Map from a loaded snapshot. */
|
|
102
|
+
static restorePools(poolState, snapshot) {
|
|
103
|
+
if (!poolState || !snapshot || !snapshot.pools || typeof snapshot.pools !== 'object') return 0;
|
|
104
|
+
let n = 0;
|
|
105
|
+
for (const [base, st] of Object.entries(snapshot.pools)) {
|
|
106
|
+
if (!st || typeof st !== 'object') continue;
|
|
107
|
+
const failedUntil = new Map();
|
|
108
|
+
if (st.failedUntil && typeof st.failedUntil === 'object') {
|
|
109
|
+
for (const [ref, until] of Object.entries(st.failedUntil)) {
|
|
110
|
+
if (Number.isFinite(until)) failedUntil.set(ref, until);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
poolState.set(base, {
|
|
114
|
+
failedUntil,
|
|
115
|
+
pointer: Number.isFinite(st.pointer) ? st.pointer : 0,
|
|
116
|
+
lastUsed: typeof st.lastUsed === 'string' ? st.lastUsed : undefined,
|
|
117
|
+
});
|
|
118
|
+
n += 1;
|
|
119
|
+
}
|
|
120
|
+
return n;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
dispose() {
|
|
124
|
+
if (this._timer) {
|
|
125
|
+
clearTimeout(this._timer);
|
|
126
|
+
this._timer = null;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Resolve the on-disk state path. Prefer an explicit config path; otherwise
|
|
133
|
+
* place the file under `dataDir` when the host provides one. Returns null when
|
|
134
|
+
* persistence cannot be enabled safely (no writable data directory).
|
|
135
|
+
*/
|
|
136
|
+
export function resolveStatePath({ dataDir, configuredPath } = {}) {
|
|
137
|
+
if (configuredPath && typeof configuredPath === 'string' && configuredPath.trim()) {
|
|
138
|
+
return configuredPath.trim();
|
|
139
|
+
}
|
|
140
|
+
if (dataDir && typeof dataDir === 'string' && dataDir.trim()) {
|
|
141
|
+
return path.join(dataDir.trim(), DEFAULT_FILE);
|
|
142
|
+
}
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export default StatePersistence;
|
package/lib/pool.js
CHANGED
|
@@ -381,3 +381,20 @@ export function budgetVerdict(spend, budget) {
|
|
|
381
381
|
const ratio = spend / budget;
|
|
382
382
|
return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
|
|
383
383
|
}
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
/** Safely reset a provider's circuit breaker to closed state. */
|
|
387
|
+
export function resetCircuitForProvider(circuitBreaker, provider) {
|
|
388
|
+
if (!circuitBreaker || !provider) return false;
|
|
389
|
+
try {
|
|
390
|
+
if (typeof circuitBreaker.reset === 'function') {
|
|
391
|
+
circuitBreaker.reset(provider);
|
|
392
|
+
return true;
|
|
393
|
+
}
|
|
394
|
+
if (typeof circuitBreaker.onSuccess === 'function') {
|
|
395
|
+
circuitBreaker.onSuccess(provider);
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
} catch (_) {}
|
|
399
|
+
return false;
|
|
400
|
+
}
|
package/lib/rotate.js
CHANGED
|
@@ -67,7 +67,7 @@ export function createRotate(deps) {
|
|
|
67
67
|
const _max = pool.maxCooldownMs ?? maxCooldownMs;
|
|
68
68
|
const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
|
|
69
69
|
const cls = classifyFailure({ code: errCode, message: errMsg });
|
|
70
|
-
const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft);
|
|
70
|
+
const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
|
|
71
71
|
if (circuitBreaker) circuitBreaker.onFailure(pool.base ?? options?.provider);
|
|
72
72
|
pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
|
|
73
73
|
if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
|
package/lib/routes-ops.js
CHANGED
|
@@ -22,6 +22,7 @@ import { usageRows, usageCsv } from './usage-report.js';
|
|
|
22
22
|
import { findSecrets, looksLikeApiSecret } from './keycheck.js';
|
|
23
23
|
import { nextQuotaReset } from './quota-window.js';
|
|
24
24
|
import { classifyFailure } from './error-taxonomy.js';
|
|
25
|
+
import { sanitizeSnapshot } from './sanitize-snapshot.js';
|
|
25
26
|
|
|
26
27
|
const STATUS_PATH = '/dsh-key-rotation/status';
|
|
27
28
|
const SNAPSHOT_PATH = '/dsh-key-rotation/snapshot';
|
|
@@ -46,6 +47,7 @@ export function registerOpsRoutes(ctx, deps) {
|
|
|
46
47
|
poolState,
|
|
47
48
|
getRotationDisabled,
|
|
48
49
|
setRotationDisabled,
|
|
50
|
+
circuitBreaker,
|
|
49
51
|
} = deps;
|
|
50
52
|
|
|
51
53
|
// ── status route: what the settings card cannot know on its own ──
|
|
@@ -165,7 +167,7 @@ export function registerOpsRoutes(ctx, deps) {
|
|
|
165
167
|
providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
|
|
166
168
|
}
|
|
167
169
|
}
|
|
168
|
-
json(res, 200, {
|
|
170
|
+
json(res, 200, sanitizeSnapshot({
|
|
169
171
|
providers,
|
|
170
172
|
// #266/#263 operational extras (additive)
|
|
171
173
|
meta: {
|
|
@@ -174,7 +176,7 @@ export function registerOpsRoutes(ctx, deps) {
|
|
|
174
176
|
breakerEnabled: runtime.circuitBreakerEnabled !== false,
|
|
175
177
|
at: now,
|
|
176
178
|
},
|
|
177
|
-
});
|
|
179
|
+
}, now));
|
|
178
180
|
},
|
|
179
181
|
}), 'dsh-key-rotation: status route');
|
|
180
182
|
|
|
@@ -352,7 +354,13 @@ export function registerOpsRoutes(ctx, deps) {
|
|
|
352
354
|
st.authFailCounts?.clear();
|
|
353
355
|
st.brokenUntil?.clear();
|
|
354
356
|
st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
|
|
355
|
-
|
|
357
|
+
let circuitReset = false;
|
|
358
|
+
const br = circuitBreaker ?? buildRuntime().breaker;
|
|
359
|
+
if (br) {
|
|
360
|
+
if (typeof br.reset === 'function') { br.reset(provider); circuitReset = true; }
|
|
361
|
+
else if (typeof br.onSuccess === 'function') { br.onSuccess(provider); circuitReset = true; }
|
|
362
|
+
}
|
|
363
|
+
json(res, 200, { ok: true, provider, cleared, circuitReset });
|
|
356
364
|
return;
|
|
357
365
|
}
|
|
358
366
|
if (ref) {
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// lib/sanitize-snapshot.js — clamp/repair runtime snapshots before they leave the host (#288).
|
|
2
|
+
// Guards the settings card and ops APIs against NaN, negative remaining times,
|
|
3
|
+
// and inconsistent counters after clock jumps or empty-pool edge cases.
|
|
4
|
+
|
|
5
|
+
function toFiniteNumber(value, fallback = 0) {
|
|
6
|
+
const n = Number(value);
|
|
7
|
+
return Number.isFinite(n) ? n : fallback;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function clampNonNegative(value) {
|
|
11
|
+
const n = toFiniteNumber(value, 0);
|
|
12
|
+
return n < 0 ? 0 : n;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function clampRemaining(value, nowMs) {
|
|
16
|
+
const n = toFiniteNumber(value, 0);
|
|
17
|
+
if (n <= 0) return 0;
|
|
18
|
+
// If the absolute timestamp is in the past relative to now, remaining is 0.
|
|
19
|
+
if (n > 1e12 && n <= nowMs) return 0;
|
|
20
|
+
// Cooldown remaining should not exceed ~7 days.
|
|
21
|
+
if (n < 1e12 && n > 7 * 86400000) return 7 * 86400000;
|
|
22
|
+
return n;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Sanitize a per-key status entry.
|
|
27
|
+
* @param {object} key
|
|
28
|
+
* @param {number} now
|
|
29
|
+
*/
|
|
30
|
+
export function sanitizeKeyStatus(key, now = Date.now()) {
|
|
31
|
+
if (!key || typeof key !== 'object') return null;
|
|
32
|
+
return {
|
|
33
|
+
...key,
|
|
34
|
+
present: Boolean(key.present),
|
|
35
|
+
active: Boolean(key.active),
|
|
36
|
+
cooldownMsLeft: clampRemaining(key.cooldownMsLeft, now),
|
|
37
|
+
usage: clampNonNegative(key.usage),
|
|
38
|
+
failures: clampNonNegative(key.failures),
|
|
39
|
+
weight: clampNonNegative(key.weight || 1) || 1,
|
|
40
|
+
rpm: key.rpm && typeof key.rpm === 'object'
|
|
41
|
+
? {
|
|
42
|
+
...key.rpm,
|
|
43
|
+
used: clampNonNegative(key.rpm.used),
|
|
44
|
+
remaining: clampNonNegative(key.rpm.remaining),
|
|
45
|
+
limit: clampNonNegative(key.rpm.limit),
|
|
46
|
+
}
|
|
47
|
+
: key.rpm,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Sanitize a full status snapshot ({ providers: [...], ... }).
|
|
53
|
+
* Never mutates the input.
|
|
54
|
+
*/
|
|
55
|
+
export function sanitizeSnapshot(snapshot, now = Date.now()) {
|
|
56
|
+
if (!snapshot || typeof snapshot !== 'object') return { providers: [] };
|
|
57
|
+
const providers = Array.isArray(snapshot.providers) ? snapshot.providers : [];
|
|
58
|
+
return {
|
|
59
|
+
...snapshot,
|
|
60
|
+
providers: providers.map((p) => {
|
|
61
|
+
if (!p || typeof p !== 'object') return p;
|
|
62
|
+
const keys = Array.isArray(p.keys) ? p.keys : [];
|
|
63
|
+
return {
|
|
64
|
+
...p,
|
|
65
|
+
switches: clampNonNegative(p.switches),
|
|
66
|
+
totalUsage: p.totalUsage == null ? p.totalUsage : clampNonNegative(p.totalUsage),
|
|
67
|
+
healthScore: p.healthScore == null
|
|
68
|
+
? p.healthScore
|
|
69
|
+
: Math.max(0, Math.min(100, toFiniteNumber(p.healthScore, 0))),
|
|
70
|
+
lastSwitchAt: p.lastSwitchAt == null ? null : (Number.isFinite(Number(p.lastSwitchAt)) ? Number(p.lastSwitchAt) : null),
|
|
71
|
+
lastExhaustionAt: p.lastExhaustionAt == null ? null : (Number.isFinite(Number(p.lastExhaustionAt)) ? Number(p.lastExhaustionAt) : null),
|
|
72
|
+
todayCost: p.todayCost == null ? p.todayCost : clampNonNegative(p.todayCost),
|
|
73
|
+
weeklyCost: p.weeklyCost == null ? p.weeklyCost : clampNonNegative(p.weeklyCost),
|
|
74
|
+
budgetDaily: p.budgetDaily == null ? p.budgetDaily : clampNonNegative(p.budgetDaily),
|
|
75
|
+
budgetWeekly: p.budgetWeekly == null ? p.budgetWeekly : clampNonNegative(p.budgetWeekly),
|
|
76
|
+
p95: p.p95 == null ? null : clampNonNegative(p.p95),
|
|
77
|
+
keys: keys.map((k) => sanitizeKeyStatus(k, now)).filter(Boolean),
|
|
78
|
+
};
|
|
79
|
+
}),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export default sanitizeSnapshot;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.6",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|