@goodandready/dsh-key-rotation 0.7.26 → 0.7.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +140 -118
- package/cordis.patch.yml +6 -6
- package/lib/agent-budget.js +68 -68
- package/lib/bucket.js +41 -0
- package/lib/canary.js +56 -56
- package/lib/cascade.js +39 -39
- package/lib/client-helpers.js +21 -21
- package/lib/client.js +52 -1
- package/lib/concurrency.js +72 -72
- package/lib/heal.js +35 -35
- package/lib/histogram.js +66 -66
- package/lib/incident.js +76 -76
- package/lib/index.js +1536 -1402
- package/lib/keycheck.js +31 -0
- package/lib/pool.js +237 -227
- package/lib/quota-window.js +45 -45
- package/lib/quota.js +39 -39
- package/lib/region.js +50 -50
- package/lib/sandbox.js +117 -117
- package/lib/shadow.js +81 -81
- package/lib/webhook.js +133 -64
- package/package.json +58 -58
package/lib/cascade.js
CHANGED
|
@@ -1,39 +1,39 @@
|
|
|
1
|
-
// cascade.js — cross-provider failover cascade (issue #194).
|
|
2
|
-
// ponytail: minimal — pick fallback provider from a RegionMap-like config.
|
|
3
|
-
|
|
4
|
-
export const CASCADE_MAX_DEPTH = 1;
|
|
5
|
-
|
|
6
|
-
export function pickCascadeFallback(provider, cfg, pools) {
|
|
7
|
-
const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
|
|
8
|
-
for (const entry of list) {
|
|
9
|
-
const fb = typeof entry === 'string' ? { provider: entry } : entry;
|
|
10
|
-
if (!fb || !fb.provider || fb.provider === provider) continue;
|
|
11
|
-
const pool = pools instanceof Map ? pools.get(fb.provider) : (pools ? pools[fb.provider] : null);
|
|
12
|
-
if (!pool) continue;
|
|
13
|
-
const now = Date.now();
|
|
14
|
-
let healthy = 0;
|
|
15
|
-
for (const ref of pool.refs) {
|
|
16
|
-
const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
|
|
17
|
-
if (failedUntil > now) continue;
|
|
18
|
-
const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
|
|
19
|
-
if (exp !== undefined && now >= exp) continue;
|
|
20
|
-
healthy += 1;
|
|
21
|
-
}
|
|
22
|
-
if (healthy === 0) continue;
|
|
23
|
-
return { provider: fb.provider, pool, model: fb.model || null };
|
|
24
|
-
}
|
|
25
|
-
return null;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export function hasHealthyKey(pool, now) {
|
|
29
|
-
now = now || Date.now();
|
|
30
|
-
if (!pool || !Array.isArray(pool.refs)) return false;
|
|
31
|
-
for (const ref of pool.refs) {
|
|
32
|
-
const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
|
|
33
|
-
if (failedUntil > now) continue;
|
|
34
|
-
const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
|
|
35
|
-
if (exp !== undefined && now >= exp) continue;
|
|
36
|
-
return true;
|
|
37
|
-
}
|
|
38
|
-
return false;
|
|
39
|
-
}
|
|
1
|
+
// cascade.js — cross-provider failover cascade (issue #194).
|
|
2
|
+
// ponytail: minimal — pick fallback provider from a RegionMap-like config.
|
|
3
|
+
|
|
4
|
+
export const CASCADE_MAX_DEPTH = 1;
|
|
5
|
+
|
|
6
|
+
export function pickCascadeFallback(provider, cfg, pools) {
|
|
7
|
+
const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
|
|
8
|
+
for (const entry of list) {
|
|
9
|
+
const fb = typeof entry === 'string' ? { provider: entry } : entry;
|
|
10
|
+
if (!fb || !fb.provider || fb.provider === provider) continue;
|
|
11
|
+
const pool = pools instanceof Map ? pools.get(fb.provider) : (pools ? pools[fb.provider] : null);
|
|
12
|
+
if (!pool) continue;
|
|
13
|
+
const now = Date.now();
|
|
14
|
+
let healthy = 0;
|
|
15
|
+
for (const ref of pool.refs) {
|
|
16
|
+
const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
|
|
17
|
+
if (failedUntil > now) continue;
|
|
18
|
+
const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
|
|
19
|
+
if (exp !== undefined && now >= exp) continue;
|
|
20
|
+
healthy += 1;
|
|
21
|
+
}
|
|
22
|
+
if (healthy === 0) continue;
|
|
23
|
+
return { provider: fb.provider, pool, model: fb.model || null };
|
|
24
|
+
}
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function hasHealthyKey(pool, now) {
|
|
29
|
+
now = now || Date.now();
|
|
30
|
+
if (!pool || !Array.isArray(pool.refs)) return false;
|
|
31
|
+
for (const ref of pool.refs) {
|
|
32
|
+
const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
|
|
33
|
+
if (failedUntil > now) continue;
|
|
34
|
+
const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
|
|
35
|
+
if (exp !== undefined && now >= exp) continue;
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
return false;
|
|
39
|
+
}
|
package/lib/client-helpers.js
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
export function formatAgo(t, at) {
|
|
2
|
-
if (!at) return '';
|
|
3
|
-
const sec = Math.max(0, Math.round((Date.now() - at) / 1000));
|
|
4
|
-
if (sec < 60) return t('justNow');
|
|
5
|
-
if (sec < 3600) return t('minutesAgo').replace('{n}', String(Math.round(sec / 60)));
|
|
6
|
-
return t('hoursAgo').replace('{n}', String(Math.round(sec / 3600)));
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export function nextKeyRef(providerId, existingKeys, allRefs) {
|
|
10
|
-
const fromExisting = (existingKeys || []).find((k) => typeof k === 'string' && k.length > 0);
|
|
11
|
-
const base = fromExisting
|
|
12
|
-
? fromExisting.replace(/_\d+$/, '')
|
|
13
|
-
: String(providerId || 'provider').toUpperCase().replace(/[^A-Z0-9]+/g, '_').replace(/^_+|_+$/g, '') + '_API_KEY';
|
|
14
|
-
const taken = new Set(allRefs);
|
|
15
|
-
if (!taken.has(base)) return base;
|
|
16
|
-
for (let n = 2; n < 1000; n++) {
|
|
17
|
-
const candidate = base + '_' + n;
|
|
18
|
-
if (!taken.has(candidate)) return candidate;
|
|
19
|
-
}
|
|
20
|
-
return base + '_' + Date.now();
|
|
21
|
-
}
|
|
1
|
+
export function formatAgo(t, at) {
|
|
2
|
+
if (!at) return '';
|
|
3
|
+
const sec = Math.max(0, Math.round((Date.now() - at) / 1000));
|
|
4
|
+
if (sec < 60) return t('justNow');
|
|
5
|
+
if (sec < 3600) return t('minutesAgo').replace('{n}', String(Math.round(sec / 60)));
|
|
6
|
+
return t('hoursAgo').replace('{n}', String(Math.round(sec / 3600)));
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function nextKeyRef(providerId, existingKeys, allRefs) {
|
|
10
|
+
const fromExisting = (existingKeys || []).find((k) => typeof k === 'string' && k.length > 0);
|
|
11
|
+
const base = fromExisting
|
|
12
|
+
? fromExisting.replace(/_\d+$/, '')
|
|
13
|
+
: String(providerId || 'provider').toUpperCase().replace(/[^A-Z0-9]+/g, '_').replace(/^_+|_+$/g, '') + '_API_KEY';
|
|
14
|
+
const taken = new Set(allRefs);
|
|
15
|
+
if (!taken.has(base)) return base;
|
|
16
|
+
for (let n = 2; n < 1000; n++) {
|
|
17
|
+
const candidate = base + '_' + n;
|
|
18
|
+
if (!taken.has(candidate)) return candidate;
|
|
19
|
+
}
|
|
20
|
+
return base + '_' + Date.now();
|
|
21
|
+
}
|
package/lib/client.js
CHANGED
|
@@ -63,6 +63,7 @@ window.__ModuleLoader__.load({
|
|
|
63
63
|
keySaved: 'saved',
|
|
64
64
|
keyFromEnv: 'from the environment, read-only here',
|
|
65
65
|
keyWriteFailed: 'could not store the key: {msg}',
|
|
66
|
+
notSecretShape: 'saved, but the value does not look like an API key - check for a typo',
|
|
66
67
|
keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
|
|
67
68
|
brokenKey: 'broken (3× AUTH)',
|
|
68
69
|
keyExpired: 'expired',
|
|
@@ -119,6 +120,7 @@ window.__ModuleLoader__.load({
|
|
|
119
120
|
keySaved: 'сохранён',
|
|
120
121
|
keyFromEnv: 'задан в окружении, отсюда не меняется',
|
|
121
122
|
keyWriteFailed: 'не удалось сохранить ключ: {msg}',
|
|
123
|
+
notSecretShape: 'сохранено, но значение не похоже на API-ключ — проверьте опечатки',
|
|
122
124
|
keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
|
|
123
125
|
brokenKey: 'сломан (3× AUTH)',
|
|
124
126
|
keyExpired: 'истёк',
|
|
@@ -390,6 +392,11 @@ window.__ModuleLoader__.load({
|
|
|
390
392
|
.then((r) => r.json().then((data) => ({ ok: r.ok, data })))
|
|
391
393
|
.then(({ ok, data }) => {
|
|
392
394
|
if (!ok) throw new Error(data?.error?.message ?? 'unknown error');
|
|
395
|
+
// #200 leak-detector hint: stored value does not match any known
|
|
396
|
+
// API-key shape - probably a placeholder or a typo.
|
|
397
|
+
if (data?.looksLikeSecret === false) {
|
|
398
|
+
setSecretError(t('notSecretShape'));
|
|
399
|
+
}
|
|
393
400
|
setSecretDraft((cur) => {
|
|
394
401
|
const next = { ...cur };
|
|
395
402
|
delete next[rowKey];
|
|
@@ -803,10 +810,54 @@ window.__ModuleLoader__.load({
|
|
|
803
810
|
if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', init); else init();
|
|
804
811
|
}
|
|
805
812
|
|
|
806
|
-
|
|
813
|
+
// #201 header chip: one dot + counts for all pools. Green = all healthy,
|
|
814
|
+
// amber = some keys cooling, red = a pool fully exhausted. Click opens the
|
|
815
|
+
// same summary the floating dashboard shows.
|
|
816
|
+
function KeyRotationHeaderChip() {
|
|
817
|
+
const [snap, setSnap] = React.useState(null);
|
|
818
|
+
React.useEffect(() => {
|
|
819
|
+
let alive = true;
|
|
820
|
+
const load = () => {
|
|
821
|
+
fetch('/dsh-key-rotation/health', { headers: { accept: 'application/json' }, credentials: 'same-origin' })
|
|
822
|
+
.then((r) => (r.ok ? r.json() : null))
|
|
823
|
+
.then((d) => { if (alive) setSnap(d); })
|
|
824
|
+
.catch(() => {});
|
|
825
|
+
};
|
|
826
|
+
load();
|
|
827
|
+
const id = setInterval(load, 4000);
|
|
828
|
+
return () => { alive = false; clearInterval(id); };
|
|
829
|
+
}, []);
|
|
830
|
+
const pools = Object.values((snap && snap.pools) || {});
|
|
831
|
+
const total = pools.reduce((a, p) => a + (p.total || 0), 0);
|
|
832
|
+
const healthy = pools.reduce((a, p) => a + (p.healthy || 0), 0);
|
|
833
|
+
const anyExhausted = pools.some((p) => p.exhausted);
|
|
834
|
+
const color = !pools.length ? 'var(--dsw-alias-label-tertiary)' : anyExhausted ? '#e5484d' : healthy < total ? '#f5a623' : '#30a46c';
|
|
835
|
+
const label = pools.length ? `${healthy}/${total}` : 'rot';
|
|
836
|
+
return h('span', {
|
|
837
|
+
className: 'krot-chip',
|
|
838
|
+
title: 'Key Rotation',
|
|
839
|
+
style: { display: 'inline-flex', alignItems: 'center', gap: '5px', fontSize: '11px', color: 'var(--dsw-alias-label-secondary)', cursor: 'default' },
|
|
840
|
+
},
|
|
841
|
+
h('span', { style: { width: '8px', height: '8px', borderRadius: '50%', background: color, flex: 'none' } }),
|
|
842
|
+
label);
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
function apply(ctx) {
|
|
807
846
|
ctx.effect(() => ctx.locale.register(NS, { en, ru }), 'dsh-key-rotation: dictionaries');
|
|
808
847
|
// Dashboard widget: lives on every page, polls /health (#152).
|
|
809
848
|
ctx.effect(() => mountDashboard(), 'dsh-key-rotation: dashboard widget');
|
|
849
|
+
// Header chip (#201): status dot in the session header utilities slot,
|
|
850
|
+
// same slot dsh-gitea / dsh-subscriptions use for their header widgets.
|
|
851
|
+
ctx.effect(() => {
|
|
852
|
+
if (!ctx.slots) return;
|
|
853
|
+
try {
|
|
854
|
+
ctx.slots.inject('conversation.session.header.utilities', () =>
|
|
855
|
+
ctx.slots.register(
|
|
856
|
+
{ name: 'conversation.session.header.utilities', id: 'dsh-key-rotation-header-chip', order: 6 },
|
|
857
|
+
KeyRotationHeaderChip,
|
|
858
|
+
));
|
|
859
|
+
} catch { /* slot not available in this build */ }
|
|
860
|
+
}, 'dsh-key-rotation: header chip');
|
|
810
861
|
function useLocale() {
|
|
811
862
|
return useActiveLocale(ctx);
|
|
812
863
|
}
|
package/lib/concurrency.js
CHANGED
|
@@ -1,72 +1,72 @@
|
|
|
1
|
-
// concurrency.js — per-key in-flight counter + least-connections picking (issue #193).
|
|
2
|
-
|
|
3
|
-
export const CONCURRENCY_DEFAULT_LIMIT = 0;
|
|
4
|
-
export const CONCURRENCY_STALE_LOCK_MS = 5 * 60 * 1000;
|
|
5
|
-
|
|
6
|
-
export class ConcurrencyTracker {
|
|
7
|
-
constructor(opts) {
|
|
8
|
-
opts = opts || {};
|
|
9
|
-
const limit = opts.limit !== undefined ? opts.limit : CONCURRENCY_DEFAULT_LIMIT;
|
|
10
|
-
this._limit = (Number.isFinite(limit) && limit >= 0) ? Math.floor(limit) : 0;
|
|
11
|
-
this._staleMs = opts.staleMs || CONCURRENCY_STALE_LOCK_MS;
|
|
12
|
-
this._inFlight = new Map();
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
isEnabled() { return this._limit > 0; }
|
|
16
|
-
get limit() { return this._limit; }
|
|
17
|
-
|
|
18
|
-
acquire(ref, now) {
|
|
19
|
-
now = now || Date.now();
|
|
20
|
-
if (!this.isEnabled()) return true;
|
|
21
|
-
let e = this._inFlight.get(ref);
|
|
22
|
-
if (!e) {
|
|
23
|
-
e = { count: 0, lastAcquired: now };
|
|
24
|
-
this._inFlight.set(ref, e);
|
|
25
|
-
}
|
|
26
|
-
if (now - e.lastAcquired > this._staleMs) {
|
|
27
|
-
e.count = 0;
|
|
28
|
-
}
|
|
29
|
-
if (e.count >= this._limit) return false;
|
|
30
|
-
e.count += 1;
|
|
31
|
-
e.lastAcquired = now;
|
|
32
|
-
return true;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
release(ref, now) {
|
|
36
|
-
now = now || Date.now();
|
|
37
|
-
const e = this._inFlight.get(ref);
|
|
38
|
-
if (!e) return;
|
|
39
|
-
e.count = Math.max(0, e.count - 1);
|
|
40
|
-
e.lastAcquired = now;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
snapshot() {
|
|
44
|
-
const out = {};
|
|
45
|
-
for (const [k, v] of this._inFlight) out[k] = { count: v.count };
|
|
46
|
-
return out;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
pickLeastLoaded(candidates, now) {
|
|
50
|
-
now = now || Date.now();
|
|
51
|
-
if (!Array.isArray(candidates) || candidates.length === 0) return null;
|
|
52
|
-
let best = null;
|
|
53
|
-
let bestCount = Infinity;
|
|
54
|
-
for (const ref of candidates) {
|
|
55
|
-
const e = this._inFlight.get(ref);
|
|
56
|
-
const count = e ? e.count : 0;
|
|
57
|
-
if (this.isEnabled() && count >= this._limit) continue;
|
|
58
|
-
if (count < bestCount) {
|
|
59
|
-
best = ref;
|
|
60
|
-
bestCount = count;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
return best;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
clear(ref) {
|
|
67
|
-
if (ref) this._inFlight.delete(ref);
|
|
68
|
-
else this._inFlight.clear();
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
get size() { return this._inFlight.size; }
|
|
72
|
-
}
|
|
1
|
+
// concurrency.js — per-key in-flight counter + least-connections picking (issue #193).
|
|
2
|
+
|
|
3
|
+
export const CONCURRENCY_DEFAULT_LIMIT = 0;
|
|
4
|
+
export const CONCURRENCY_STALE_LOCK_MS = 5 * 60 * 1000;
|
|
5
|
+
|
|
6
|
+
export class ConcurrencyTracker {
|
|
7
|
+
constructor(opts) {
|
|
8
|
+
opts = opts || {};
|
|
9
|
+
const limit = opts.limit !== undefined ? opts.limit : CONCURRENCY_DEFAULT_LIMIT;
|
|
10
|
+
this._limit = (Number.isFinite(limit) && limit >= 0) ? Math.floor(limit) : 0;
|
|
11
|
+
this._staleMs = opts.staleMs || CONCURRENCY_STALE_LOCK_MS;
|
|
12
|
+
this._inFlight = new Map();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
isEnabled() { return this._limit > 0; }
|
|
16
|
+
get limit() { return this._limit; }
|
|
17
|
+
|
|
18
|
+
acquire(ref, now) {
|
|
19
|
+
now = now || Date.now();
|
|
20
|
+
if (!this.isEnabled()) return true;
|
|
21
|
+
let e = this._inFlight.get(ref);
|
|
22
|
+
if (!e) {
|
|
23
|
+
e = { count: 0, lastAcquired: now };
|
|
24
|
+
this._inFlight.set(ref, e);
|
|
25
|
+
}
|
|
26
|
+
if (now - e.lastAcquired > this._staleMs) {
|
|
27
|
+
e.count = 0;
|
|
28
|
+
}
|
|
29
|
+
if (e.count >= this._limit) return false;
|
|
30
|
+
e.count += 1;
|
|
31
|
+
e.lastAcquired = now;
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
release(ref, now) {
|
|
36
|
+
now = now || Date.now();
|
|
37
|
+
const e = this._inFlight.get(ref);
|
|
38
|
+
if (!e) return;
|
|
39
|
+
e.count = Math.max(0, e.count - 1);
|
|
40
|
+
e.lastAcquired = now;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
snapshot() {
|
|
44
|
+
const out = {};
|
|
45
|
+
for (const [k, v] of this._inFlight) out[k] = { count: v.count };
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
pickLeastLoaded(candidates, now) {
|
|
50
|
+
now = now || Date.now();
|
|
51
|
+
if (!Array.isArray(candidates) || candidates.length === 0) return null;
|
|
52
|
+
let best = null;
|
|
53
|
+
let bestCount = Infinity;
|
|
54
|
+
for (const ref of candidates) {
|
|
55
|
+
const e = this._inFlight.get(ref);
|
|
56
|
+
const count = e ? e.count : 0;
|
|
57
|
+
if (this.isEnabled() && count >= this._limit) continue;
|
|
58
|
+
if (count < bestCount) {
|
|
59
|
+
best = ref;
|
|
60
|
+
bestCount = count;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return best;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
clear(ref) {
|
|
67
|
+
if (ref) this._inFlight.delete(ref);
|
|
68
|
+
else this._inFlight.clear();
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
get size() { return this._inFlight.size; }
|
|
72
|
+
}
|
package/lib/heal.js
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
// heal.js — self-healing idle cooldowns.
|
|
2
|
-
// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
|
|
3
|
-
|
|
4
|
-
// Returns array of { ref, poolBase } entries that were healed in this tick.
|
|
5
|
-
// Mutates `pools` (removes from failedUntil, pushes heal event into events).
|
|
6
|
-
// `now` parameter is injectable for tests.
|
|
7
|
-
export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
|
|
8
|
-
if (!Array.isArray(pools) || pools.length === 0) return [];
|
|
9
|
-
if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
|
|
10
|
-
const healed = [];
|
|
11
|
-
for (const pool of pools) {
|
|
12
|
-
if (!pool || !pool.state || !pool.base) continue;
|
|
13
|
-
const fu = pool.state.failedUntil;
|
|
14
|
-
const lu = pool.state.lastUsed;
|
|
15
|
-
if (!fu || fu.size === 0) continue;
|
|
16
|
-
const expiredRefs = [];
|
|
17
|
-
for (const [ref, until] of fu.entries()) {
|
|
18
|
-
if (!Number.isFinite(until)) continue;
|
|
19
|
-
if (until > now) continue; // cooldown still active
|
|
20
|
-
const last = lu ? lu.get(ref) : undefined;
|
|
21
|
-
if (!Number.isFinite(last)) continue; // never used → no signal, skip
|
|
22
|
-
if (now - last < idleMs) continue; // used recently → don't heal
|
|
23
|
-
expiredRefs.push(ref);
|
|
24
|
-
}
|
|
25
|
-
for (const ref of expiredRefs) {
|
|
26
|
-
fu.delete(ref);
|
|
27
|
-
if (Array.isArray(pool.state.events)) {
|
|
28
|
-
pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
|
|
29
|
-
if (pool.state.events.length > 50) pool.state.events.shift();
|
|
30
|
-
}
|
|
31
|
-
healed.push({ ref, poolBase: pool.base });
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
return healed;
|
|
35
|
-
}
|
|
1
|
+
// heal.js — self-healing idle cooldowns.
|
|
2
|
+
// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
|
|
3
|
+
|
|
4
|
+
// Returns array of { ref, poolBase } entries that were healed in this tick.
|
|
5
|
+
// Mutates `pools` (removes from failedUntil, pushes heal event into events).
|
|
6
|
+
// `now` parameter is injectable for tests.
|
|
7
|
+
export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
|
|
8
|
+
if (!Array.isArray(pools) || pools.length === 0) return [];
|
|
9
|
+
if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
|
|
10
|
+
const healed = [];
|
|
11
|
+
for (const pool of pools) {
|
|
12
|
+
if (!pool || !pool.state || !pool.base) continue;
|
|
13
|
+
const fu = pool.state.failedUntil;
|
|
14
|
+
const lu = pool.state.lastUsed;
|
|
15
|
+
if (!fu || fu.size === 0) continue;
|
|
16
|
+
const expiredRefs = [];
|
|
17
|
+
for (const [ref, until] of fu.entries()) {
|
|
18
|
+
if (!Number.isFinite(until)) continue;
|
|
19
|
+
if (until > now) continue; // cooldown still active
|
|
20
|
+
const last = lu ? lu.get(ref) : undefined;
|
|
21
|
+
if (!Number.isFinite(last)) continue; // never used → no signal, skip
|
|
22
|
+
if (now - last < idleMs) continue; // used recently → don't heal
|
|
23
|
+
expiredRefs.push(ref);
|
|
24
|
+
}
|
|
25
|
+
for (const ref of expiredRefs) {
|
|
26
|
+
fu.delete(ref);
|
|
27
|
+
if (Array.isArray(pool.state.events)) {
|
|
28
|
+
pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
|
|
29
|
+
if (pool.state.events.length > 50) pool.state.events.shift();
|
|
30
|
+
}
|
|
31
|
+
healed.push({ ref, poolBase: pool.base });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return healed;
|
|
35
|
+
}
|
package/lib/histogram.js
CHANGED
|
@@ -1,66 +1,66 @@
|
|
|
1
|
-
// histogram.js — per-ref latency ring buffer + percentile.
|
|
2
|
-
// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
|
|
3
|
-
|
|
4
|
-
export const LATENCY_DEFAULT_WINDOW = 200;
|
|
5
|
-
|
|
6
|
-
export class LatencyHistogram {
|
|
7
|
-
constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
|
|
8
|
-
const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
|
|
9
|
-
this._window = w;
|
|
10
|
-
this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
|
|
11
|
-
this._lastAt = new Map(); // ref -> epochMs of last sample
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
record(ref, ms) {
|
|
15
|
-
if (!ref || !Number.isFinite(ms) || ms < 0) return;
|
|
16
|
-
let entry = this._buffers.get(ref);
|
|
17
|
-
if (!entry) {
|
|
18
|
-
entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
|
|
19
|
-
this._buffers.set(ref, entry);
|
|
20
|
-
}
|
|
21
|
-
entry.buf[entry.head] = ms;
|
|
22
|
-
entry.head = (entry.head + 1) % this._window;
|
|
23
|
-
if (entry.count < this._window) entry.count += 1;
|
|
24
|
-
this._lastAt.set(ref, Date.now());
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
// Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
|
|
28
|
-
snapshot(ref) {
|
|
29
|
-
const entry = this._buffers.get(ref);
|
|
30
|
-
const lastAt = this._lastAt.get(ref);
|
|
31
|
-
if (!entry || entry.count === 0) {
|
|
32
|
-
return { count: 0, lastAt: lastAt || null };
|
|
33
|
-
}
|
|
34
|
-
const arr = entry.buf.subarray(0, entry.count);
|
|
35
|
-
const sorted = Array.from(arr).sort((a, b) => a - b);
|
|
36
|
-
const n = sorted.length;
|
|
37
|
-
return {
|
|
38
|
-
count: n,
|
|
39
|
-
lastAt: lastAt || null,
|
|
40
|
-
p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
|
|
41
|
-
p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
|
|
42
|
-
p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
// Returns { [ref]: snapshot }
|
|
47
|
-
snapshotAll() {
|
|
48
|
-
const out = {};
|
|
49
|
-
for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
|
|
50
|
-
return out;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
clear(ref) {
|
|
54
|
-
if (ref) {
|
|
55
|
-
this._buffers.delete(ref);
|
|
56
|
-
this._lastAt.delete(ref);
|
|
57
|
-
} else {
|
|
58
|
-
this._buffers.clear();
|
|
59
|
-
this._lastAt.clear();
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
get size() {
|
|
64
|
-
return this._buffers.size;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
1
|
+
// histogram.js — per-ref latency ring buffer + percentile.
|
|
2
|
+
// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
|
|
3
|
+
|
|
4
|
+
export const LATENCY_DEFAULT_WINDOW = 200;
|
|
5
|
+
|
|
6
|
+
export class LatencyHistogram {
|
|
7
|
+
constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
|
|
8
|
+
const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
|
|
9
|
+
this._window = w;
|
|
10
|
+
this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
|
|
11
|
+
this._lastAt = new Map(); // ref -> epochMs of last sample
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
record(ref, ms) {
|
|
15
|
+
if (!ref || !Number.isFinite(ms) || ms < 0) return;
|
|
16
|
+
let entry = this._buffers.get(ref);
|
|
17
|
+
if (!entry) {
|
|
18
|
+
entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
|
|
19
|
+
this._buffers.set(ref, entry);
|
|
20
|
+
}
|
|
21
|
+
entry.buf[entry.head] = ms;
|
|
22
|
+
entry.head = (entry.head + 1) % this._window;
|
|
23
|
+
if (entry.count < this._window) entry.count += 1;
|
|
24
|
+
this._lastAt.set(ref, Date.now());
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
|
|
28
|
+
snapshot(ref) {
|
|
29
|
+
const entry = this._buffers.get(ref);
|
|
30
|
+
const lastAt = this._lastAt.get(ref);
|
|
31
|
+
if (!entry || entry.count === 0) {
|
|
32
|
+
return { count: 0, lastAt: lastAt || null };
|
|
33
|
+
}
|
|
34
|
+
const arr = entry.buf.subarray(0, entry.count);
|
|
35
|
+
const sorted = Array.from(arr).sort((a, b) => a - b);
|
|
36
|
+
const n = sorted.length;
|
|
37
|
+
return {
|
|
38
|
+
count: n,
|
|
39
|
+
lastAt: lastAt || null,
|
|
40
|
+
p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
|
|
41
|
+
p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
|
|
42
|
+
p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Returns { [ref]: snapshot }
|
|
47
|
+
snapshotAll() {
|
|
48
|
+
const out = {};
|
|
49
|
+
for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
clear(ref) {
|
|
54
|
+
if (ref) {
|
|
55
|
+
this._buffers.delete(ref);
|
|
56
|
+
this._lastAt.delete(ref);
|
|
57
|
+
} else {
|
|
58
|
+
this._buffers.clear();
|
|
59
|
+
this._lastAt.clear();
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
get size() {
|
|
64
|
+
return this._buffers.size;
|
|
65
|
+
}
|
|
66
|
+
}
|