@goodandready/dsh-key-rotation 0.8.43 → 0.8.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/lib/cascade.js +6 -2
- package/lib/circuit-breaker.js +16 -0
- package/lib/client.js +11 -5
- package/lib/heal.js +2 -1
- package/lib/notify-queue.js +34 -25
- package/lib/ops-status.js +18 -2
- package/lib/ops-test.js +7 -0
- package/lib/persistence.js +62 -10
- package/lib/pool.js +9 -1
- package/lib/resolver.js +2 -1
- package/lib/rotate.js +35 -5
- package/lib/safe-fetch.js +4 -4
- package/lib/webhook.js +17 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.44 - 2026-10-01
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- **Core & Persistence Integrity (#407, #408, #409, #410, #422, #427)**:
|
|
7
|
+
- Eliminated concurrency permit leaks in lazy streams by avoiding premature permit reservation in rotate dispatch (#407).
|
|
8
|
+
- Implemented physical credential revocation fan-out across all pools in runtime on 401 AUTH failures (#408).
|
|
9
|
+
- Supported nested `Map<ref, Map<day, number>>` in state persistence for `costDays` and `usageDays`, preserving daily financial spend and preventing provider budget bypasses across restarts (#409, #422).
|
|
10
|
+
- Preserved exponential backoff failure counts across cooldown expiration in `sweepExpired` (#410).
|
|
11
|
+
- Blocked quarantined credentials in candidate selection (`resolver.js`) and ensured auto-unbreak probes skip configured paused keys (#427).
|
|
12
|
+
- **Transport, Circuit Breakers & Route Reliability (#411, #413, #415, #418, #428)**:
|
|
13
|
+
- Ensured safe dispatcher single-address lookup returns scalar `(err, address, family)` rather than array, conforming to standard Node.js `dns.lookup` contract (#411).
|
|
14
|
+
- Added non-mutating `isAvailable` inspection to `CircuitBreaker`, preventing cascade candidate checks from consuming trial half-open probes prior to dispatch (#413).
|
|
15
|
+
- Enforced fail-closed decryption validation in `/dsh-key-rotation/test`, preventing encrypted ciphertext from reaching upstream LLM endpoints when secrets cannot be decrypted (#415).
|
|
16
|
+
- Model sub-pools now accurately reflect their base provider circuit breaker state in `/status`, and `/health` correctly transitions to `degraded` when any circuit is open (#418).
|
|
17
|
+
- Streamlined `NotifyQueue` dispatch into debouncer pipeline with accurate delivery outcome tracking and backoff handling (#428).
|
|
18
|
+
- **UI & Client Reliability (#419, #420, #421)**:
|
|
19
|
+
- Stripped master action token from Telegram interactive callbacks, bounded UTF-8 byte length to <= 64 bytes for unicode, and preserved ASCII action routing (#419).
|
|
20
|
+
- Fixed `keyStatus` to recognize paused status from server DTOs and reset optimistic reset flags upon active cooldown updates (#420).
|
|
21
|
+
- Memoized `UpdaterSection` load callback with ref to prevent duplicate background network checks on unrelated parent settings updates (#421).
|
|
22
|
+
- **Packaging, CI & Documentation (#424, #425)**:
|
|
23
|
+
- Synchronized `package-lock.json` with `semver@7.8.5` and `undici@6.29.0` for clean `npm ci --ignore-scripts` reproducibility (#424).
|
|
24
|
+
- Updated `docs/deployment/github-npm-provenance.md` with mandatory pre-release MiniPC test-server acceptance and explicit user confirmation requirements (#425).
|
|
25
|
+
|
|
3
26
|
## 0.8.43 - 2026-10-01
|
|
4
27
|
|
|
5
28
|
### Fixed
|
package/lib/cascade.js
CHANGED
|
@@ -15,8 +15,12 @@ export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoo
|
|
|
15
15
|
if (!fb || !fb.provider || fb.provider === provider) continue;
|
|
16
16
|
|
|
17
17
|
const breaker = circuitBreaker || cfg?.circuitBreaker || null;
|
|
18
|
-
if (breaker
|
|
19
|
-
|
|
18
|
+
if (breaker) {
|
|
19
|
+
if (typeof breaker.isAvailable === 'function') {
|
|
20
|
+
if (!breaker.isAvailable(fb.provider)) continue;
|
|
21
|
+
} else if (typeof breaker.canRequest === 'function') {
|
|
22
|
+
if (!breaker.canRequest(fb.provider)) continue;
|
|
23
|
+
}
|
|
20
24
|
}
|
|
21
25
|
|
|
22
26
|
const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;
|
package/lib/circuit-breaker.js
CHANGED
|
@@ -57,6 +57,22 @@ export class CircuitBreaker {
|
|
|
57
57
|
return true;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
/** @returns {boolean} whether a request may be dispatched to this provider without mutating probe count */
|
|
61
|
+
isAvailable(provider) {
|
|
62
|
+
const e = this._entry(provider);
|
|
63
|
+
const now = this._now();
|
|
64
|
+
if (e.state === BREAKER_OPEN) {
|
|
65
|
+
if (now - e.openedAt >= this.openMs) {
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
if (e.state === BREAKER_HALF_OPEN) {
|
|
71
|
+
return e.probes < this.halfOpenProbes;
|
|
72
|
+
}
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
|
|
60
76
|
// A permit is settled exactly once. Late outcomes from an older circuit
|
|
61
77
|
// generation (or a removed provider) cannot close/reopen a newer circuit.
|
|
62
78
|
acquire(provider) {
|
package/lib/client.js
CHANGED
|
@@ -753,23 +753,25 @@ window.__ModuleLoader__.load({
|
|
|
753
753
|
}
|
|
754
754
|
|
|
755
755
|
function UpdaterSection({ t }) {
|
|
756
|
+
const tRef = React.useRef(t);
|
|
757
|
+
tRef.current = t;
|
|
756
758
|
const [state, setState] = React.useState({ phase: 'idle', current: '', latest: '', message: '' });
|
|
757
759
|
const load = React.useCallback(async () => {
|
|
758
760
|
setState((s) => ({ ...s, phase: 'loading', message: '' }));
|
|
759
761
|
try {
|
|
760
762
|
const res = await fetch('/api/dsh-key-rotation/update', { headers: { 'x-dsh-plugin-update': '1' } });
|
|
761
763
|
const data = await res.json();
|
|
762
|
-
if (!res.ok) throw new Error(data?.error?.message ||
|
|
764
|
+
if (!res.ok) throw new Error(data?.error?.message || (tRef.current ? tRef.current('updateFailed') : 'Update check failed'));
|
|
763
765
|
setState({
|
|
764
766
|
phase: data.updateAvailable ? 'available' : 'current',
|
|
765
767
|
current: data.currentVersion || '',
|
|
766
768
|
latest: data.latestVersion || '',
|
|
767
|
-
message: data.updateAvailable ?
|
|
769
|
+
message: data.updateAvailable ? (tRef.current ? tRef.current('updateAvailable') : '') : (tRef.current ? tRef.current('updateNone') : ''),
|
|
768
770
|
});
|
|
769
771
|
} catch (e) {
|
|
770
|
-
setState({ phase: 'error', current: '', latest: '', message: e?.message ||
|
|
772
|
+
setState({ phase: 'error', current: '', latest: '', message: e?.message || (tRef.current ? tRef.current('updateFailed') : 'Update check failed') });
|
|
771
773
|
}
|
|
772
|
-
}, [
|
|
774
|
+
}, []);
|
|
773
775
|
React.useEffect(() => { load(); }, [load]);
|
|
774
776
|
const run = async () => {
|
|
775
777
|
setState((s) => ({ ...s, phase: 'running', message: t('updateRunning') }));
|
|
@@ -1792,10 +1794,14 @@ window.__ModuleLoader__.load({
|
|
|
1792
1794
|
if (isPaused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1793
1795
|
const hit = keyInfo(providerId, ref);
|
|
1794
1796
|
if (!hit) return null;
|
|
1797
|
+
if (hit.paused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused'), ready: false, paused: true };
|
|
1795
1798
|
if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired'), ready: false, expired: true };
|
|
1796
1799
|
if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked'), ready: false, revoked: true };
|
|
1797
1800
|
if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing'), ready: false, missing: true };
|
|
1798
|
-
if (hit.cooldownMsLeft > 0
|
|
1801
|
+
if (hit.cooldownMsLeft > 0) {
|
|
1802
|
+
if (optimisticReset && ref in optimisticReset) {
|
|
1803
|
+
delete optimisticReset[ref];
|
|
1804
|
+
}
|
|
1799
1805
|
return {
|
|
1800
1806
|
color: 'var(--dsw-alias-state-warning-primary)',
|
|
1801
1807
|
text: t('keyCooling').replace('{s}', String(Math.ceil(hit.cooldownMsLeft / 1000))),
|
package/lib/heal.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isKeyRevoked, isKeyPaused } from './pool.js';
|
|
1
2
|
// heal.js — self-healing idle cooldowns.
|
|
2
3
|
// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
|
|
3
4
|
|
|
@@ -54,7 +55,7 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
|
|
|
54
55
|
if (!bu || bu.size === 0) continue;
|
|
55
56
|
const brokenRefs = [];
|
|
56
57
|
for (const [ref, until] of bu.entries()) {
|
|
57
|
-
if (pool
|
|
58
|
+
if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
|
|
58
59
|
if (Number.isFinite(until) && until > now) {
|
|
59
60
|
brokenRefs.push(ref);
|
|
60
61
|
}
|
package/lib/notify-queue.js
CHANGED
|
@@ -31,35 +31,44 @@ export class NotifyQueue {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
_pump() {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
34
|
+
while (this._q.length > 0) {
|
|
35
|
+
const delay = this._backoffByUrl.get(this._q[0].url) ?? 0;
|
|
36
|
+
if (delay > 0 && this._busy) break;
|
|
37
|
+
const job = this._q.shift();
|
|
38
|
+
if (!job) break;
|
|
39
|
+
this._activeSends = (this._activeSends || 0) + 1;
|
|
40
|
+
this._busy = true;
|
|
41
|
+
const run = async () => {
|
|
42
|
+
try {
|
|
43
|
+
const res = await this._send(job.url, job.payload);
|
|
44
|
+
if (res && res.sent === false) {
|
|
45
|
+
this._failed += 1;
|
|
46
|
+
this._bumpBackoff(job.url);
|
|
47
|
+
} else if (res && res.sent === true) {
|
|
48
|
+
this._sent += 1;
|
|
49
|
+
this._backoffByUrl.delete(job.url);
|
|
50
|
+
} else if (res === undefined) {
|
|
51
|
+
this._sent += 1;
|
|
52
|
+
this._backoffByUrl.delete(job.url);
|
|
53
|
+
} else {
|
|
54
|
+
this._sent += 1;
|
|
55
|
+
this._backoffByUrl.delete(job.url);
|
|
56
|
+
}
|
|
57
|
+
} catch {
|
|
43
58
|
this._failed += 1;
|
|
44
59
|
this._bumpBackoff(job.url);
|
|
45
|
-
}
|
|
46
|
-
this.
|
|
47
|
-
this.
|
|
60
|
+
} finally {
|
|
61
|
+
this._activeSends = Math.max(0, this._activeSends - 1);
|
|
62
|
+
this._busy = this._activeSends > 0;
|
|
63
|
+
if (this._q.length > 0) this._pump();
|
|
48
64
|
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
65
|
+
};
|
|
66
|
+
if (delay > 0) {
|
|
67
|
+
const t = setTimeout(run, delay);
|
|
68
|
+
if (typeof t.unref === 'function') t.unref();
|
|
69
|
+
} else {
|
|
70
|
+
run();
|
|
55
71
|
}
|
|
56
|
-
};
|
|
57
|
-
if (delay > 0) {
|
|
58
|
-
const t = setTimeout(run, delay);
|
|
59
|
-
if (typeof t.unref === 'function') t.unref();
|
|
60
|
-
} else {
|
|
61
|
-
// fire and forget — do not return a promise to callers
|
|
62
|
-
run();
|
|
63
72
|
}
|
|
64
73
|
}
|
|
65
74
|
|
package/lib/ops-status.js
CHANGED
|
@@ -129,7 +129,11 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
129
129
|
circuit: (() => {
|
|
130
130
|
const br = circuitBreaker ?? runtime.breaker;
|
|
131
131
|
if (!br) return null;
|
|
132
|
-
|
|
132
|
+
let st = br.state(pool.base);
|
|
133
|
+
if (st === 'closed' && pool.provider && pool.provider !== pool.base) {
|
|
134
|
+
const baseSt = br.state(pool.provider);
|
|
135
|
+
if (baseSt !== 'closed') st = baseSt;
|
|
136
|
+
}
|
|
133
137
|
return { state: st, threshold: br.threshold, openMs: br.openMs };
|
|
134
138
|
})(),
|
|
135
139
|
switches: pool.state.switches ?? 0,
|
|
@@ -217,7 +221,19 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
217
221
|
modelQuota: quotaAware ? { configured: true, exhaustedKeys: quotaExhausted } : null,
|
|
218
222
|
};
|
|
219
223
|
}
|
|
220
|
-
|
|
224
|
+
let circuitOpenAny = false;
|
|
225
|
+
const brH = circuitBreaker ?? runtimeH.breaker;
|
|
226
|
+
if (brH) {
|
|
227
|
+
for (const pool of (runtimeH.pools ?? runtimeH.poolByRef.values())) {
|
|
228
|
+
const provKey = pool.provider || pool.base;
|
|
229
|
+
if (brH.state(pool.base) === 'open' || brH.state(provKey) === 'open') {
|
|
230
|
+
circuitOpenAny = true;
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const isDegraded = exhaustedAny || circuitOpenAny;
|
|
236
|
+
json(res, 200, { status: isDegraded ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
|
|
221
237
|
},
|
|
222
238
|
}), 'dsh-key-rotation: health');
|
|
223
239
|
|
package/lib/ops-test.js
CHANGED
|
@@ -45,6 +45,13 @@ ctx.effect(() => ctx.webServer.register({
|
|
|
45
45
|
let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
|
|
46
46
|
let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
|
|
47
47
|
const rawValue = testValue || hit?.value;
|
|
48
|
+
if (typeof rawValue === 'string' && rawValue.startsWith('enc:v1:')) {
|
|
49
|
+
const decrypted = decryptSecret(rawValue);
|
|
50
|
+
if (!decrypted || decrypted === rawValue || decrypted.startsWith('enc:v1:')) {
|
|
51
|
+
json(res, 200, { ok: false, ref, code: 'decryption-failed', message: 'dsh-key-rotation: failed to decrypt credential (missing or invalid DSH_KEY_SECRET)' });
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
48
55
|
const effectiveValue = rawValue ? decryptSecret(rawValue) : rawValue;
|
|
49
56
|
const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
|
|
50
57
|
const tail = valid ? keyTail(effectiveValue) : '';
|
package/lib/persistence.js
CHANGED
|
@@ -56,8 +56,22 @@ export class StatePersistence {
|
|
|
56
56
|
}
|
|
57
57
|
const costDays = {};
|
|
58
58
|
if (st?.costDays) {
|
|
59
|
-
const
|
|
60
|
-
|
|
59
|
+
const entries = st.costDays instanceof Map ? st.costDays.entries() : Object.entries(st.costDays);
|
|
60
|
+
for (const [k, val] of entries) {
|
|
61
|
+
if (!val) continue;
|
|
62
|
+
if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
|
|
63
|
+
const innerObj = {};
|
|
64
|
+
const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
|
|
65
|
+
for (const [day, c] of innerEntries) {
|
|
66
|
+
const num = Number(c);
|
|
67
|
+
if (Number.isFinite(num) && num >= 0) innerObj[day] = num;
|
|
68
|
+
}
|
|
69
|
+
if (Object.keys(innerObj).length > 0) costDays[k] = innerObj;
|
|
70
|
+
} else if (Number.isFinite(Number(val))) {
|
|
71
|
+
const num = Number(val);
|
|
72
|
+
if (num >= 0) costDays[k] = num;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
61
75
|
}
|
|
62
76
|
const costPerKey = {};
|
|
63
77
|
if (st?.costPerKey) {
|
|
@@ -66,8 +80,22 @@ export class StatePersistence {
|
|
|
66
80
|
}
|
|
67
81
|
const usageDays = {};
|
|
68
82
|
if (st?.usageDays) {
|
|
69
|
-
const
|
|
70
|
-
|
|
83
|
+
const entries = st.usageDays instanceof Map ? st.usageDays.entries() : Object.entries(st.usageDays);
|
|
84
|
+
for (const [k, val] of entries) {
|
|
85
|
+
if (!val) continue;
|
|
86
|
+
if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
|
|
87
|
+
const innerObj = {};
|
|
88
|
+
const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
|
|
89
|
+
for (const [day, c] of innerEntries) {
|
|
90
|
+
const num = Number(c);
|
|
91
|
+
if (Number.isFinite(num) && num >= 0) innerObj[day] = Math.floor(num);
|
|
92
|
+
}
|
|
93
|
+
if (Object.keys(innerObj).length > 0) usageDays[k] = innerObj;
|
|
94
|
+
} else if (Number.isFinite(Number(val))) {
|
|
95
|
+
const num = Number(val);
|
|
96
|
+
if (num >= 0) usageDays[k] = Math.floor(num);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
71
99
|
}
|
|
72
100
|
const usageCounts = {};
|
|
73
101
|
if (st?.usageCounts instanceof Map) {
|
|
@@ -182,10 +210,22 @@ export class StatePersistence {
|
|
|
182
210
|
state.tokenUsage.set(ref, { used: Math.floor(used), resetAt: Math.floor(resetAt) });
|
|
183
211
|
}
|
|
184
212
|
}
|
|
185
|
-
if (saved.costDays && typeof saved.costDays === 'object') {
|
|
213
|
+
if (saved.costDays && typeof saved.costDays === 'object' && !Array.isArray(saved.costDays)) {
|
|
186
214
|
if (!state.costDays) state.costDays = new Map();
|
|
187
|
-
for (const [
|
|
188
|
-
if (
|
|
215
|
+
for (const [k, val] of Object.entries(saved.costDays)) {
|
|
216
|
+
if (val && typeof val === 'object' && !Array.isArray(val)) {
|
|
217
|
+
let innerMap = state.costDays.get(k);
|
|
218
|
+
if (!(innerMap instanceof Map)) {
|
|
219
|
+
innerMap = new Map();
|
|
220
|
+
state.costDays.set(k, innerMap);
|
|
221
|
+
}
|
|
222
|
+
for (const [day, cost] of Object.entries(val)) {
|
|
223
|
+
const num = Number(cost);
|
|
224
|
+
if (Number.isFinite(num) && num >= 0) innerMap.set(day, num);
|
|
225
|
+
}
|
|
226
|
+
} else if (Number.isFinite(Number(val))) {
|
|
227
|
+
state.costDays.set(k, Number(val));
|
|
228
|
+
}
|
|
189
229
|
}
|
|
190
230
|
}
|
|
191
231
|
if (saved.costPerKey && typeof saved.costPerKey === 'object') {
|
|
@@ -194,10 +234,22 @@ export class StatePersistence {
|
|
|
194
234
|
if (Number.isFinite(c)) state.costPerKey.set(k, c);
|
|
195
235
|
}
|
|
196
236
|
}
|
|
197
|
-
if (saved.usageDays && typeof saved.usageDays === 'object') {
|
|
237
|
+
if (saved.usageDays && typeof saved.usageDays === 'object' && !Array.isArray(saved.usageDays)) {
|
|
198
238
|
if (!state.usageDays) state.usageDays = new Map();
|
|
199
|
-
for (const [
|
|
200
|
-
if (
|
|
239
|
+
for (const [k, val] of Object.entries(saved.usageDays)) {
|
|
240
|
+
if (val && typeof val === 'object' && !Array.isArray(val)) {
|
|
241
|
+
let innerMap = state.usageDays.get(k);
|
|
242
|
+
if (!(innerMap instanceof Map)) {
|
|
243
|
+
innerMap = new Map();
|
|
244
|
+
state.usageDays.set(k, innerMap);
|
|
245
|
+
}
|
|
246
|
+
for (const [day, count] of Object.entries(val)) {
|
|
247
|
+
const num = Number(count);
|
|
248
|
+
if (Number.isFinite(num) && num >= 0) innerMap.set(day, Math.floor(num));
|
|
249
|
+
}
|
|
250
|
+
} else if (Number.isFinite(Number(val))) {
|
|
251
|
+
state.usageDays.set(k, Math.floor(Number(val)));
|
|
252
|
+
}
|
|
201
253
|
}
|
|
202
254
|
}
|
|
203
255
|
if (saved.usageCounts && typeof saved.usageCounts === 'object') {
|
package/lib/pool.js
CHANGED
|
@@ -210,7 +210,6 @@ export function sweepExpired(poolState, now = Date.now(), activeRefs = null, quo
|
|
|
210
210
|
for (const [ref, until] of [...st.failedUntil.entries()]) {
|
|
211
211
|
if (until <= now) {
|
|
212
212
|
st.failedUntil.delete(ref);
|
|
213
|
-
st.failCounts?.delete(ref);
|
|
214
213
|
st.quotaWindows?.delete(ref);
|
|
215
214
|
cleared++;
|
|
216
215
|
}
|
|
@@ -564,3 +563,12 @@ export function isKeyRevoked(pool, ref) {
|
|
|
564
563
|
if (pool.basePool) return isKeyRevoked(pool.basePool, ref);
|
|
565
564
|
return false;
|
|
566
565
|
}
|
|
566
|
+
|
|
567
|
+
/** Check if a key reference is in active recoverable or permanent quarantine (#427). */
|
|
568
|
+
export function isKeyBroken(pool, ref, now = Date.now()) {
|
|
569
|
+
if (!pool || !ref) return false;
|
|
570
|
+
const bu = pool.state?.brokenUntil?.get(ref);
|
|
571
|
+
if (bu !== undefined && bu > now) return true;
|
|
572
|
+
if (pool.basePool) return isKeyBroken(pool.basePool, ref, now);
|
|
573
|
+
return false;
|
|
574
|
+
}
|
package/lib/resolver.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// delegates to `original(ref)`.
|
|
10
10
|
|
|
11
11
|
import { bucketAllow, bucketRetryMs, tpmAllow, tpmRetryMs } from './bucket.js';
|
|
12
|
-
import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, costForDay, costForWeek, budgetVerdict } from './pool.js';
|
|
12
|
+
import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, isKeyBroken, costForDay, costForWeek, budgetVerdict } from './pool.js';
|
|
13
13
|
import { decryptSecret } from './crypto-storage.js';
|
|
14
14
|
import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
|
|
15
15
|
import {
|
|
@@ -132,6 +132,7 @@ export function createResolver({
|
|
|
132
132
|
const candidate = list[index];
|
|
133
133
|
if (isKeyRevoked(pool, candidate)) continue;
|
|
134
134
|
if (isKeyPaused(pool, candidate)) continue;
|
|
135
|
+
if (isKeyBroken(pool, candidate, at)) continue;
|
|
135
136
|
const until = pool.state.failedUntil.get(candidate);
|
|
136
137
|
if (until !== undefined && until > at) continue;
|
|
137
138
|
const candidateExp = pool.expiresAt?.[candidate] ?? pool.basePool?.expiresAt?.[candidate];
|
package/lib/rotate.js
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
isRateLimited,
|
|
12
12
|
isKeyPaused,
|
|
13
13
|
isKeyRevoked,
|
|
14
|
+
isKeyBroken,
|
|
14
15
|
formatExhaustionMessage,
|
|
15
16
|
sortAttemptList,
|
|
16
17
|
} from './pool.js';
|
|
@@ -87,7 +88,7 @@ export function createRotate(deps) {
|
|
|
87
88
|
const wallNow = Date.now();
|
|
88
89
|
const quotaManaged = hasModelQuotaConfig(pool);
|
|
89
90
|
let attemptList = (pool.weightedRefs ?? pool.refs).slice();
|
|
90
|
-
attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r));
|
|
91
|
+
attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow));
|
|
91
92
|
if (quotaManaged) {
|
|
92
93
|
attemptList = attemptList.filter((ref) => isModelQuotaAvailable(pool, ref, wallNow));
|
|
93
94
|
if (attemptList.length === 0) {
|
|
@@ -186,6 +187,31 @@ export function createRotate(deps) {
|
|
|
186
187
|
pool.state.failedUntil.set(targetRef, Infinity);
|
|
187
188
|
pushEvent(pool, targetRef, 'KEY_REVOKED', 0);
|
|
188
189
|
logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently REVOKED on auth failure`);
|
|
190
|
+
// Fan-out revocation across all pools sharing this physical ref (#408):
|
|
191
|
+
const rAll = buildRuntime();
|
|
192
|
+
const candidatePools = [];
|
|
193
|
+
if (Array.isArray(rAll.pools)) candidatePools.push(...rAll.pools);
|
|
194
|
+
if (rAll.index?.allPools) candidatePools.push(...rAll.index.allPools.values());
|
|
195
|
+
if (rAll.providerToPool) candidatePools.push(...rAll.providerToPool.values());
|
|
196
|
+
if (rAll.modelPoolByProvider) {
|
|
197
|
+
for (const map of rAll.modelPoolByProvider.values()) {
|
|
198
|
+
if (map) candidatePools.push(...map.values());
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const seenPools = new Set();
|
|
202
|
+
for (const other of candidatePools) {
|
|
203
|
+
if (!other || seenPools.has(other)) continue;
|
|
204
|
+
seenPools.add(other);
|
|
205
|
+
if (other === pool) continue;
|
|
206
|
+
const hasRef = (Array.isArray(other.refs) && other.refs.includes(targetRef)) ||
|
|
207
|
+
(other.weightsMap instanceof Map && other.weightsMap.has(targetRef));
|
|
208
|
+
if (hasRef && other.state) {
|
|
209
|
+
if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
|
|
210
|
+
other.state.revokedRefs.add(targetRef);
|
|
211
|
+
if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
|
|
212
|
+
pushEvent(other, targetRef, 'KEY_REVOKED', 0);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
189
215
|
} else {
|
|
190
216
|
pool.state.authFailCounts.delete(targetRef);
|
|
191
217
|
if (!isQuota) {
|
|
@@ -260,13 +286,16 @@ export function createRotate(deps) {
|
|
|
260
286
|
}
|
|
261
287
|
|
|
262
288
|
const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
|
|
263
|
-
if (_pickedRef && !reqStore.concurrencyRef && typeof concurrencyTracker?.acquire === 'function') {
|
|
264
|
-
const ok = concurrencyTracker.acquire(_pickedRef, now(), pool.concurrencyLimit ?? runtime0.concurrencyLimit ?? 0);
|
|
265
|
-
if (ok) reqStore.concurrencyRef = _pickedRef;
|
|
266
|
-
}
|
|
267
289
|
try {
|
|
268
290
|
try {
|
|
269
291
|
for await (const chunk of preserveDispatchContext(inner, dispatchStorage, reqStore)) {
|
|
292
|
+
if (!reqStore.concurrencyRef && typeof concurrencyTracker?.acquire === 'function') {
|
|
293
|
+
const fallbackRef = reqStore.pickedRef ?? _pickedRef;
|
|
294
|
+
if (fallbackRef) {
|
|
295
|
+
const ok = concurrencyTracker.acquire(fallbackRef, now(), pool.concurrencyLimit ?? runtime0.concurrencyLimit ?? 0);
|
|
296
|
+
if (ok) reqStore.concurrencyRef = fallbackRef;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
270
299
|
// Only actual content deltas lock the stream (no more rotation).
|
|
271
300
|
// Structural/metadata chunks (block-start/end, usage) do not.
|
|
272
301
|
if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
|
|
@@ -442,6 +471,7 @@ export function createRotate(deps) {
|
|
|
442
471
|
concurrencyTracker?.release(reqStore.concurrencyRef);
|
|
443
472
|
reqStore.concurrencyRef = null;
|
|
444
473
|
}
|
|
474
|
+
reqStore.pickedRef = undefined;
|
|
445
475
|
// Also runs on consumer return(), abort, empty EOF and local errors.
|
|
446
476
|
activePermit?.release();
|
|
447
477
|
}
|
package/lib/safe-fetch.js
CHANGED
|
@@ -109,7 +109,7 @@ export function createSafeDispatcher({ lookupImpl = nodeDns.lookup } = {}) {
|
|
|
109
109
|
connect: {
|
|
110
110
|
lookup: (hostname, options, callback) => {
|
|
111
111
|
const cb = typeof options === 'function' ? options : callback;
|
|
112
|
-
const opts = typeof options === 'function' ? {} : options;
|
|
112
|
+
const opts = typeof options === 'function' ? {} : (typeof options === 'number' ? { family: options } : (options || {}));
|
|
113
113
|
lookupImpl(hostname, { ...opts, all: true }, (err, addresses) => {
|
|
114
114
|
if (err) return cb(err);
|
|
115
115
|
const list = Array.isArray(addresses) ? addresses : [addresses];
|
|
@@ -121,10 +121,10 @@ export function createSafeDispatcher({ lookupImpl = nodeDns.lookup } = {}) {
|
|
|
121
121
|
return cb(ssrfErr);
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
|
-
if (
|
|
124
|
+
if (!opts.all) {
|
|
125
125
|
const first = list[0];
|
|
126
|
-
const addr = typeof first === 'string' ? first : first
|
|
127
|
-
const fam = typeof first === 'object' && first
|
|
126
|
+
const addr = typeof first === 'string' ? first : first?.address;
|
|
127
|
+
const fam = typeof first === 'object' && first?.family ? first.family : (net.isIP(addr) || 4);
|
|
128
128
|
return cb(null, addr, fam);
|
|
129
129
|
}
|
|
130
130
|
return cb(null, addresses);
|
package/lib/webhook.js
CHANGED
|
@@ -32,12 +32,17 @@ export function formatInteractive(url, payload, actionToken) {
|
|
|
32
32
|
parse_mode: 'Markdown',
|
|
33
33
|
reply_markup: {
|
|
34
34
|
inline_keyboard: [actions.map((a) => {
|
|
35
|
-
let cbData =
|
|
35
|
+
let cbData = JSON.stringify({ id: a.id });
|
|
36
36
|
if (cbData.length > 64) {
|
|
37
|
-
cbData =
|
|
37
|
+
cbData = String(a.id ?? '');
|
|
38
38
|
}
|
|
39
|
-
if (cbData
|
|
40
|
-
|
|
39
|
+
if (Buffer.byteLength(cbData, 'utf8') > 64 && /[^\x00-\x7F]/.test(cbData)) {
|
|
40
|
+
const buf = Buffer.from(cbData, 'utf8');
|
|
41
|
+
let res = buf.subarray(0, 64).toString('utf8');
|
|
42
|
+
while (Buffer.byteLength(res, 'utf8') > 64) {
|
|
43
|
+
res = res.slice(0, -1);
|
|
44
|
+
}
|
|
45
|
+
cbData = res;
|
|
41
46
|
}
|
|
42
47
|
return {
|
|
43
48
|
text: a.label,
|
|
@@ -189,7 +194,7 @@ export class AlertDebouncer {
|
|
|
189
194
|
}
|
|
190
195
|
|
|
191
196
|
enqueue(url, event, onFlushed = null) {
|
|
192
|
-
if (!url) return;
|
|
197
|
+
if (!url) return Promise.resolve({ sent: false });
|
|
193
198
|
let entry = this._pending.get(url);
|
|
194
199
|
if (!entry) {
|
|
195
200
|
entry = { events: [], timer: null, callbacks: [] };
|
|
@@ -198,13 +203,19 @@ export class AlertDebouncer {
|
|
|
198
203
|
entry.events.push(event);
|
|
199
204
|
if (onFlushed) entry.callbacks.push(onFlushed);
|
|
200
205
|
|
|
206
|
+
const promise = new Promise((resolve) => {
|
|
207
|
+
entry.callbacks.push(resolve);
|
|
208
|
+
});
|
|
209
|
+
|
|
201
210
|
if (entry.events.length >= this._maxBatch) {
|
|
202
|
-
|
|
211
|
+
this.flush(url);
|
|
212
|
+
return promise;
|
|
203
213
|
}
|
|
204
214
|
if (!entry.timer) {
|
|
205
215
|
entry.timer = setTimeout(() => this.flush(url), this._debounceMs);
|
|
206
216
|
if (typeof entry.timer.unref === 'function') entry.timer.unref();
|
|
207
217
|
}
|
|
218
|
+
return promise;
|
|
208
219
|
}
|
|
209
220
|
|
|
210
221
|
async flush(url) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.44",
|
|
4
4
|
"packageManager": "pnpm@10.33.2",
|
|
5
5
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
6
6
|
"keywords": [
|