@goodandready/dsh-key-rotation 0.7.27 → 0.7.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/lib/bucket.js +11 -0
- package/lib/client.js +43 -4
- package/lib/index.js +125 -3
- package/lib/maintenance.js +64 -0
- package/lib/usage-report.js +49 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -39,6 +39,11 @@
|
|
|
39
39
|
- **Interactive webhooks** (added in 0.7.27) — set `webhookActionToken` and exhaustion notifications gain action buttons on Telegram (`inline_keyboard`), Discord (buttons) and Slack (actions block): *Reset cooldown* and *Pause 1h* per pool. Buttons call back to `POST /dsh-key-rotation/webhook-action` with a bearer token; actions `reset-<provider>`, `pause-<provider>` (1 h), `disable-rotation`, `enable-rotation`.
|
|
40
40
|
- **Token leak detector** (added in 0.7.27) — recognizes live key shapes (OpenAI `sk-`, Anthropic `sk-ant-`, Google `AIza…`, GitHub `ghp_…`, AWS `AKIA…`, Slack `xox…`, Telegram bot tokens, Stripe, PEM blocks). Saving the config section with a real key pasted into a wrong field is rejected (`400 secret-in-config`); the key-save response carries `looksLikeSecret`, and the card hints when the value does not look like an API key.
|
|
41
41
|
- **Header status chip** (added in 0.7.27) — a small chip in the session header shows one dot for all pools (green = all keys healthy, amber = some cooling, red = a pool fully exhausted) plus a `healthy/total` counter; polls every 4 s.
|
|
42
|
+
- **Key expiry pre-warning** (added in 0.7.28) — `expiryWarnDays` (default 7): keys expiring within that horizon send one webhook notification per key per day and show a yellow *expires in Nd* badge in the card.
|
|
43
|
+
- **Provider cost budget** (added in 0.7.28) — per-provider `costBudgetDaily` / `costBudgetWeekly` (0 = off): a warn webhook fires from 80% spend, and with `pauseOnBudget: true` the whole pool is paused for 24 h at 100%. The card shows a live budget line with warn/red coloring.
|
|
44
|
+
- **Usage report + CSV export** (added in 0.7.28) — `GET /dsh-key-rotation/usage?days=N[&provider=…][&format=csv]` returns per-key requests/cost over the window; an *Export CSV* button in the card downloads the same data for one provider.
|
|
45
|
+
- **RPM capacity indicator** (added in 0.7.28) — `/status` carries `rpm: {used, remaining, resetMs}` per key; the card shows a ⏱ counter next to the active key.
|
|
46
|
+
- **Real key test probe** (added in 0.7.28) — the per-key *Test* button now sends `probe=models` through the existing `/test` route, so it validates the key against the live API (models list + latency), not just credential presence.
|
|
42
47
|
|
|
43
48
|
## Install
|
|
44
49
|
|
|
@@ -83,6 +88,9 @@ dsh-key-rotation:
|
|
|
83
88
|
| `providers` | — | `[{ provider, keys: [envName, ...] }]`. `keys` are credential/env **names**, not the key values themselves. |
|
|
84
89
|
| `rpmLimit` | `0` | Requests-per-minute cap **per key** (0 = off). A capped key is skipped pre-emptively until its 60 s window frees up. |
|
|
85
90
|
| `webhookActionToken` | `''` | Bearer token for the interactive webhook callback route. When set, exhaustion webhooks carry action buttons; empty disables them. |
|
|
91
|
+
| `expiryWarnDays` | `7` | Pre-warning horizon (days) for keys with `expiresAt`: webhook + card badge. |
|
|
92
|
+
| `providers[].costBudgetDaily` / `.costBudgetWeekly` | `0` | Daily / weekly spend budget per provider (0 = off). Warn webhook from 80%. |
|
|
93
|
+
| `providers[].pauseOnBudget` | `false` | Pause the whole pool for 24 h when a budget is exceeded. |
|
|
86
94
|
| `providers[].tags` | `[]` | Free-form labels for a provider pool, surfaced in `GET /status`. |
|
|
87
95
|
|
|
88
96
|
### Interactive webhook actions (added in 0.7.27)
|
|
@@ -98,6 +106,15 @@ curl -X POST http://127.0.0.1:3080/dsh-key-rotation/webhook-action \
|
|
|
98
106
|
|
|
99
107
|
Actions: `reset-<provider>` (clear cooldowns), `pause-<provider>` (pause the whole pool for 1 hour), `disable-rotation` / `enable-rotation` (global). Platform callback payloads (`callback_data`, Discord `custom_id`, Slack button `value`) are accepted too. Without the correct bearer token the route answers `401`; without a configured token — `503`.
|
|
100
108
|
|
|
109
|
+
### Usage report (added in 0.7.28)
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
curl "http://127.0.0.1:3080/dsh-key-rotation/usage?days=7" # JSON
|
|
113
|
+
curl "http://127.0.0.1:3080/dsh-key-rotation/usage?format=csv&days=30&provider=my-provider" > usage.csv
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Per-key rows: `requests`, `cost`, `active`, and per-day counts over the window (1–90 days, default 7).
|
|
117
|
+
|
|
101
118
|
### How keys are stored
|
|
102
119
|
|
|
103
120
|
The plugin config only ever references keys by **name** (e.g. `MY_PROVIDER_API_KEY`). The values live in the dsh **Credentials** service or `$DSH_HOME/.credentials.yaml` — never in the plugin config.
|
package/lib/bucket.js
CHANGED
|
@@ -39,3 +39,14 @@ export function bucketSweep(windows, liveRefs) {
|
|
|
39
39
|
if (!liveRefs.has(ref)) windows.delete(ref);
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
|
+
|
|
43
|
+
/** #210: snapshot for /status - used/remaining/resetMs for one ref. */
|
|
44
|
+
export function bucketInfo(windows, ref, limit, now = Date.now()) {
|
|
45
|
+
if (!limit || limit <= 0) return null;
|
|
46
|
+
const hits = (windows?.get(ref) ?? []).filter((t) => t > now - WINDOW_MS);
|
|
47
|
+
return {
|
|
48
|
+
used: hits.length,
|
|
49
|
+
remaining: Math.max(0, limit - hits.length),
|
|
50
|
+
resetMs: hits.length ? Math.max(0, hits[0] + WINDOW_MS - now) : 0,
|
|
51
|
+
};
|
|
52
|
+
}
|
package/lib/client.js
CHANGED
|
@@ -64,6 +64,9 @@ window.__ModuleLoader__.load({
|
|
|
64
64
|
keyFromEnv: 'from the environment, read-only here',
|
|
65
65
|
keyWriteFailed: 'could not store the key: {msg}',
|
|
66
66
|
notSecretShape: 'saved, but the value does not look like an API key - check for a typo',
|
|
67
|
+
rpmTitle: 'requests/min: {u} used, {r} remaining',
|
|
68
|
+
budgetLabel: 'budget:',
|
|
69
|
+
exportCsv: 'Export CSV',
|
|
67
70
|
keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
|
|
68
71
|
brokenKey: 'broken (3× AUTH)',
|
|
69
72
|
keyExpired: 'expired',
|
|
@@ -121,6 +124,9 @@ window.__ModuleLoader__.load({
|
|
|
121
124
|
keyFromEnv: 'задан в окружении, отсюда не меняется',
|
|
122
125
|
keyWriteFailed: 'не удалось сохранить ключ: {msg}',
|
|
123
126
|
notSecretShape: 'сохранено, но значение не похоже на API-ключ — проверьте опечатки',
|
|
127
|
+
rpmTitle: 'запросов/мин: {u} использовано, {r} осталось',
|
|
128
|
+
budgetLabel: 'бюджет:',
|
|
129
|
+
exportCsv: 'CSV',
|
|
124
130
|
keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
|
|
125
131
|
brokenKey: 'сломан (3× AUTH)',
|
|
126
132
|
keyExpired: 'истёк',
|
|
@@ -294,7 +300,8 @@ window.__ModuleLoader__.load({
|
|
|
294
300
|
}); };
|
|
295
301
|
const doTest = (ref) => {
|
|
296
302
|
setTesting(ref); setTestResult((m) => ({ ...m, [ref]: null }));
|
|
297
|
-
|
|
303
|
+
// #212: real API probe (models is free on most providers), not just presence
|
|
304
|
+
fetch('/dsh-key-rotation/test', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref, probe: 'models' }) })
|
|
298
305
|
.then((r) => r.json())
|
|
299
306
|
.then((data) => setTestResult((m) => ({ ...m, [ref]: data })))
|
|
300
307
|
.catch((e) => setTestResult((m) => ({ ...m, [ref]: { ok: false, message: String(e?.message ?? e) } })))
|
|
@@ -517,7 +524,8 @@ window.__ModuleLoader__.load({
|
|
|
517
524
|
if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired') };
|
|
518
525
|
if (hit.expiresAt && !hit.expired) {
|
|
519
526
|
const days = Math.ceil((hit.expiresAt - Date.now()) / 86400000);
|
|
520
|
-
|
|
527
|
+
const warnDays = Number(val?.expiryWarnDays) || 7; // #207: configurable horizon
|
|
528
|
+
if (days <= warnDays) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)) };
|
|
521
529
|
}
|
|
522
530
|
if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
|
|
523
531
|
if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
|
|
@@ -593,12 +601,19 @@ window.__ModuleLoader__.load({
|
|
|
593
601
|
}
|
|
594
602
|
}
|
|
595
603
|
if (info && info.lastUsedAt) meta.push(h('span', { key: 'lu', className: 'krot-tail', title: 'last used' }, formatAgo((k)=>t(k), info.lastUsedAt)));
|
|
604
|
+
// #210: RPM capacity indicator (only when rpmLimit is active)
|
|
605
|
+
if (info && info.rpm) meta.push(h('span', { key: 'rpm', className: 'krot-tail',
|
|
606
|
+
title: t('rpmTitle').replace('{u}', String(info.rpm.used)).replace('{r}', String(info.rpm.remaining)),
|
|
607
|
+
style: info.rpm.remaining === 0 ? { color: 'var(--dsw-alias-state-error-primary)', fontWeight: 700 } : undefined },
|
|
608
|
+
'⏱' + info.rpm.remaining));
|
|
596
609
|
if (info && typeof info.cost === 'number' && info.cost > 0) meta.push(h('span', { key: 'c', className: 'krot-tail', title: 'cost' }, '$' + info.cost.toFixed(2)));
|
|
597
610
|
const tr = testResult[key];
|
|
598
611
|
if (tr) meta.push(h('span', { key: 'tr', className: 'krot-tail',
|
|
599
|
-
title: tr.message || (tr.ok ? t('testOk') : t('testFail'))
|
|
612
|
+
title: (tr.message || (tr.ok ? t('testOk') : t('testFail')))
|
|
613
|
+
+ (tr.ok && tr.modelsCount ? ' · ' + tr.modelsCount + ' models' : '')
|
|
614
|
+
+ (tr.ok && tr.latencyMs ? ' · ' + tr.latencyMs + 'ms' : ''),
|
|
600
615
|
style: { color: tr.ok ? 'var(--dsw-alias-state-success-primary)' : 'var(--dsw-alias-state-error-primary)', fontWeight: 700 } },
|
|
601
|
-
tr.ok ? '✓' : '✕'));
|
|
616
|
+
tr.ok ? (tr.modelsCount ? tr.modelsCount + 'm' : '✓') : '✕'));
|
|
602
617
|
meta.push(h('button', { key: 't', className: 'krot-btn', onClick: () => doTest(key), disabled: testing === key, title: t('testKey') }, testing === key ? '…' : t('testKey')));
|
|
603
618
|
meta.push(h('span', { key: 'a', className: 'krot-acts' },
|
|
604
619
|
btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
|
|
@@ -623,6 +638,28 @@ window.__ModuleLoader__.load({
|
|
|
623
638
|
const exhaustionWarning = providerStatus && providerStatus.lastExhaustionAt && (Date.now() - providerStatus.lastExhaustionAt) < 3600000
|
|
624
639
|
? h('p', { className: 'krot-err' }, t('poolExhausted') + ' (' + formatAgo(t, providerStatus.lastExhaustionAt) + ')')
|
|
625
640
|
: null;
|
|
641
|
+
// #208: budget line (warn color at >=80%, red at 100%)
|
|
642
|
+
const budgetLine = providerStatus && (providerStatus.budgetDaily > 0 || providerStatus.budgetWeekly > 0)
|
|
643
|
+
? (() => {
|
|
644
|
+
const dayRatio = providerStatus.budgetDaily > 0 ? providerStatus.todayCost / providerStatus.budgetDaily : 0;
|
|
645
|
+
const weekRatio = providerStatus.budgetWeekly > 0 ? providerStatus.weeklyCost / providerStatus.budgetWeekly : 0;
|
|
646
|
+
const worst = Math.max(dayRatio, weekRatio);
|
|
647
|
+
const color = worst >= 1 ? 'var(--dsw-alias-state-error-primary)' : worst >= 0.8 ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-label-tertiary)';
|
|
648
|
+
const parts = [];
|
|
649
|
+
if (providerStatus.budgetDaily > 0) parts.push('$' + (providerStatus.todayCost ?? 0).toFixed(2) + '/' + '$' + providerStatus.budgetDaily);
|
|
650
|
+
if (providerStatus.budgetWeekly > 0) parts.push('week $' + (providerStatus.weeklyCost ?? 0).toFixed(2) + '/' + '$' + providerStatus.budgetWeekly);
|
|
651
|
+
if (worst >= 1 && providerStatus.pauseOnBudget) parts.push('· paused');
|
|
652
|
+
return h('p', { className: 'krot-hint', style: { color } }, t('budgetLabel') + ' ' + parts.join(' · '));
|
|
653
|
+
})()
|
|
654
|
+
: null;
|
|
655
|
+
// #209: CSV export for this provider's usage (last 7 days)
|
|
656
|
+
const exportCsv = h('button', { className: 'krot-btn', title: t('exportCsv'),
|
|
657
|
+
onClick: () => {
|
|
658
|
+
const url = '/dsh-key-rotation/usage?format=csv&days=7&provider=' + encodeURIComponent(entry.provider);
|
|
659
|
+
const a = document.createElement('a');
|
|
660
|
+
a.href = url; a.download = 'usage-' + entry.provider + '.csv';
|
|
661
|
+
document.body.appendChild(a); a.click(); a.remove();
|
|
662
|
+
} }, t('exportCsv'));
|
|
626
663
|
|
|
627
664
|
return h('div', { key: pIndex, className: 'krot-prov' },
|
|
628
665
|
h('div', { className: 'krot-prov-head' },
|
|
@@ -660,11 +697,13 @@ window.__ModuleLoader__.load({
|
|
|
660
697
|
h('div', { className: 'krot-foot' },
|
|
661
698
|
btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle') }),
|
|
662
699
|
switchesLine,
|
|
700
|
+
budgetLine,
|
|
663
701
|
exhaustionWarning,
|
|
664
702
|
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('div', { style: { display: 'flex', gap: '2px', alignItems: 'end', height: '24px', marginTop: '4px' } }, (() => { const now = Date.now(); const buckets = Array(24).fill(0); for (const ev of providerStatus.events) { const h = Math.floor((now - ev.at) / 3600000); if (h >= 0 && h < 24) buckets[23 - h]++; } const max = Math.max(1, ...buckets); return buckets.map((c, i) => h('div', { key: i, title: c + ' switches', style: { flex: 1, background: c ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-border-l2)', height: (c / max * 24) + 'px', minHeight: '2px', borderRadius: '2px' } })); })()) : null),
|
|
665
703
|
(providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('details', { style: { fontSize: '11px', marginTop: '6px' } }, h('summary', null, 'Recent failures ('+providerStatus.events.length+')'), h('ul', { style: { margin: '4px 0 0', paddingLeft: '16px' } }, providerStatus.events.slice().reverse().map((ev, i) => h('li', { key: i, style: ev.type === 'probe' ? { opacity: .5 } : null }, new Date(ev.at).toLocaleTimeString() + ' ' + (ev.type === 'probe' ? '[probe] ' : '') + ev.ref + ' ' + ev.reason + ' cd=' + ev.cooldownMs)) )) : null),
|
|
666
704
|
btn(t('resetCooldown'), () => doReset(entry.provider), { disabled: !(providerStatus && providerStatus.switches > 0) || resetting === entry.provider, title: t('resetCooldown') }),
|
|
667
705
|
btn(testAllProvider === entry.provider ? t('testing') : t('testAll'), () => doTestAll(entry.provider), { disabled: testAllProvider === entry.provider, title: t('testAll') }),
|
|
706
|
+
exportCsv,
|
|
668
707
|
),
|
|
669
708
|
);
|
|
670
709
|
});
|
package/lib/index.js
CHANGED
|
@@ -48,6 +48,7 @@ const KEY_PATH = '/dsh-key-rotation/key';
|
|
|
48
48
|
const RESET_PATH = '/dsh-key-rotation/reset';
|
|
49
49
|
const IMPORT_PATH = '/dsh-key-rotation/import';
|
|
50
50
|
const HEALTH_PATH = '/dsh-key-rotation/health';
|
|
51
|
+
const USAGE_PATH = '/dsh-key-rotation/usage';
|
|
51
52
|
const TEST_PATH = '/dsh-key-rotation/test';
|
|
52
53
|
const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
|
|
53
54
|
const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
|
|
@@ -69,7 +70,9 @@ import { RegionMap } from './region.js';
|
|
|
69
70
|
import { IncidentReporter } from './incident.js';
|
|
70
71
|
import { ShadowRouter } from './shadow.js';
|
|
71
72
|
import { WebhookSender } from './webhook.js';
|
|
72
|
-
import { bucketAllow, bucketRetryMs, bucketSweep } from './bucket.js';
|
|
73
|
+
import { bucketAllow, bucketRetryMs, bucketSweep, bucketInfo } from './bucket.js';
|
|
74
|
+
import { expiringSoon, shouldNotifyDaily, costForDay, budgetVerdict, costForWeek } from './maintenance.js';
|
|
75
|
+
import { usageRows, usageCsv } from './usage-report.js';
|
|
73
76
|
import { findSecrets, looksLikeApiSecret } from './keycheck.js';
|
|
74
77
|
|
|
75
78
|
/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
|
|
@@ -78,6 +81,10 @@ const PIAI_NS = 'llm-pi-ai';
|
|
|
78
81
|
const MARKER = '__dshKeyRotation';
|
|
79
82
|
/** #199: set true via webhook action; checked in the llm/stream interceptor. */
|
|
80
83
|
let rotationDisabled = false;
|
|
84
|
+
// #207/#208 dedupe maps: one notification per key/window per day.
|
|
85
|
+
const expiryNotifiedAt = new Map();
|
|
86
|
+
const budgetNotifiedAt = new Map();
|
|
87
|
+
const DAY_MS = 86400000;
|
|
81
88
|
const MAX_EVENTS = 50;
|
|
82
89
|
function pushEvent(pool, ref, reason, cooldownMs, type) {
|
|
83
90
|
const ev = { at: Date.now(), ref, reason: String(reason ?? 'UNKNOWN'), cooldownMs, type: type ?? 'fail' };
|
|
@@ -183,12 +190,16 @@ export const Config = Schema.object({
|
|
|
183
190
|
rateLimitThreshold: Schema.number().default(0.1),
|
|
184
191
|
rpmLimit: Schema.number().default(0),
|
|
185
192
|
webhookActionToken: Schema.string().default(''),
|
|
193
|
+
expiryWarnDays: Schema.number().default(7),
|
|
186
194
|
providers: Schema.array(Schema.object({
|
|
187
195
|
provider: Schema.string().required(),
|
|
188
196
|
keys: Schema.array(Schema.string()).default([]),
|
|
189
197
|
weights: Schema.array(Schema.number()).default([]),
|
|
190
198
|
expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
|
|
191
199
|
tags: Schema.array(Schema.string()).default([]),
|
|
200
|
+
costBudgetDaily: Schema.number(),
|
|
201
|
+
costBudgetWeekly: Schema.number(),
|
|
202
|
+
pauseOnBudget: Schema.boolean().default(false),
|
|
192
203
|
models: Schema.dict(Schema.object({
|
|
193
204
|
keys: Schema.array(Schema.string()).default([]),
|
|
194
205
|
weights: Schema.array(Schema.number()).default([]),
|
|
@@ -542,6 +553,56 @@ export function apply(ctx, config = {}) {
|
|
|
542
553
|
}
|
|
543
554
|
const n = sweepExpired(poolState, now);
|
|
544
555
|
if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
|
|
556
|
+
// #207 expiry pre-warning + #208 cost budget - piggybacked on this timer,
|
|
557
|
+
// deduped to one notification per key/window per day (shouldNotifyDaily).
|
|
558
|
+
try {
|
|
559
|
+
const runtime = buildRuntime();
|
|
560
|
+
const seen = new Set();
|
|
561
|
+
for (const pool of runtime.poolByRef.values()) {
|
|
562
|
+
if (seen.has(pool.base)) continue;
|
|
563
|
+
seen.add(pool.base);
|
|
564
|
+
// #207: keys expiring within expiryWarnDays -> one webhook per key/day
|
|
565
|
+
for (const { ref, expiresInDays } of expiringSoon(pool, runtime.expiryWarnDays, now)) {
|
|
566
|
+
if (!shouldNotifyDaily(expiryNotifiedAt, pool.base + ':' + ref, now)) continue;
|
|
567
|
+
console.warn(`[dsh-key-rotation] ${pool.base}: key ${ref} expires in ~${expiresInDays}d`);
|
|
568
|
+
if (runtime.notifyWebhook) {
|
|
569
|
+
webhookSender.send(runtime.notifyWebhook, {
|
|
570
|
+
title: `Key expiring soon: ${pool.base}`,
|
|
571
|
+
text: `${ref} expires in ~${expiresInDays} day(s)`,
|
|
572
|
+
provider: pool.base,
|
|
573
|
+
kind: 'expiry',
|
|
574
|
+
keys: [ref],
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
// #208: daily/weekly budget -> warn webhook, optional 1-day pause at 100%
|
|
579
|
+
const budget = runtime.providerBudgets.get(pool.base);
|
|
580
|
+
if (!budget) continue;
|
|
581
|
+
const daily = costForDay(pool.state.costDays);
|
|
582
|
+
const weekly = costForWeek(pool.state.costDays, now);
|
|
583
|
+
const verdict = budgetVerdict(daily, budget.costBudgetDaily);
|
|
584
|
+
const wVerdict = budgetVerdict(weekly, budget.costBudgetWeekly);
|
|
585
|
+
const hit = verdict.warn || wVerdict.warn;
|
|
586
|
+
if (hit && shouldNotifyDaily(budgetNotifiedAt, pool.base + ':budget', now)) {
|
|
587
|
+
console.warn(`[dsh-key-rotation] ${pool.base}: cost budget - day $${daily.toFixed(2)}/$${budget.costBudgetDaily} week $${weekly.toFixed(2)}/$${budget.costBudgetWeekly}`);
|
|
588
|
+
if (runtime.notifyWebhook) {
|
|
589
|
+
webhookSender.send(runtime.notifyWebhook, {
|
|
590
|
+
title: `Cost budget: ${pool.base}`,
|
|
591
|
+
text: `day $${daily.toFixed(2)} of $${budget.costBudgetDaily} · week $${weekly.toFixed(2)} of $${budget.costBudgetWeekly}` + (verdict.exceeded || wVerdict.exceeded ? ' · EXCEEDED' : ''),
|
|
592
|
+
provider: pool.base,
|
|
593
|
+
kind: 'budget',
|
|
594
|
+
spend: { daily, weekly },
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
if ((verdict.exceeded || wVerdict.exceeded) && budget.pauseOnBudget) {
|
|
599
|
+
const until = now + DAY_MS;
|
|
600
|
+
for (const ref of pool.refs) {
|
|
601
|
+
if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
} catch (_) { /* maintenance must never crash the sweep */ }
|
|
545
606
|
}, 30000);
|
|
546
607
|
return () => clearInterval(id);
|
|
547
608
|
}, 'dsh-key-rotation: sweep expired cooldowns');
|
|
@@ -674,10 +735,15 @@ export function apply(ctx, config = {}) {
|
|
|
674
735
|
}
|
|
675
736
|
// #195: provider -> tags (metadata, surfaced in status)
|
|
676
737
|
const providerTags = new Map();
|
|
738
|
+
// #208: provider -> { costBudgetDaily, costBudgetWeekly, pauseOnBudget }
|
|
739
|
+
const providerBudgets = new Map();
|
|
677
740
|
for (const p of cfg.providers ?? []) {
|
|
678
741
|
if (Array.isArray(p.tags) && p.tags.length > 0) providerTags.set(p.provider, p.tags);
|
|
742
|
+
const daily = typeof p.costBudgetDaily === 'number' ? p.costBudgetDaily : 0;
|
|
743
|
+
const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
|
|
744
|
+
if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
|
|
679
745
|
}
|
|
680
|
-
return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, providerTags, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
|
|
746
|
+
return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
|
|
681
747
|
}
|
|
682
748
|
|
|
683
749
|
// ── patch credentials.resolve: pool refs resolve to the next healthy key ──
|
|
@@ -888,6 +954,12 @@ export function apply(ctx, config = {}) {
|
|
|
888
954
|
if (!isNaN(c)) {
|
|
889
955
|
if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
|
|
890
956
|
pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
|
|
957
|
+
// #208: cost per day per key (mirrors usageDays) for budget checks
|
|
958
|
+
if (!pool.state.costDays) pool.state.costDays = new Map();
|
|
959
|
+
const cday = new Date().toISOString().slice(0, 10);
|
|
960
|
+
const cMap = pool.state.costDays.get(pool.state.lastUsed) || new Map();
|
|
961
|
+
cMap.set(cday, (cMap.get(cday) ?? 0) + c);
|
|
962
|
+
pool.state.costDays.set(pool.state.lastUsed, cMap);
|
|
891
963
|
}
|
|
892
964
|
}
|
|
893
965
|
// Usage by day (#119)
|
|
@@ -987,7 +1059,7 @@ export function apply(ctx, config = {}) {
|
|
|
987
1059
|
json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: status is local-only' } });
|
|
988
1060
|
return;
|
|
989
1061
|
}
|
|
990
|
-
const { poolByRef, providerTags } = buildRuntime();
|
|
1062
|
+
const { poolByRef, providerTags, providerBudgets } = buildRuntime();
|
|
991
1063
|
const base = ctx.get('credentials');
|
|
992
1064
|
const now = Date.now();
|
|
993
1065
|
const seen = new Set();
|
|
@@ -995,6 +1067,7 @@ export function apply(ctx, config = {}) {
|
|
|
995
1067
|
for (const pool of poolByRef.values()) {
|
|
996
1068
|
if (seen.has(pool.base)) continue;
|
|
997
1069
|
seen.add(pool.base);
|
|
1070
|
+
try {
|
|
998
1071
|
const keys = [];
|
|
999
1072
|
for (const ref of pool.refs) {
|
|
1000
1073
|
let present = false;
|
|
@@ -1032,6 +1105,8 @@ export function apply(ctx, config = {}) {
|
|
|
1032
1105
|
writable,
|
|
1033
1106
|
active: pool.state.lastUsed === ref,
|
|
1034
1107
|
cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
|
|
1108
|
+
// #210: RPM capacity snapshot (null when rpmLimit is off)
|
|
1109
|
+
rpm: bucketInfo(pool.state.rpmWindows, ref, pool.rpmLimit, now),
|
|
1035
1110
|
usage: pool.state.usageCounts?.get(ref) ?? 0,
|
|
1036
1111
|
byModel: pool.state.byModel?.get(ref) ? Object.fromEntries(pool.state.byModel.get(ref)) : {},
|
|
1037
1112
|
usageDays: pool.state.usageDays?.get(ref) ? Object.fromEntries(pool.state.usageDays.get(ref)) : {},
|
|
@@ -1054,12 +1129,59 @@ export function apply(ctx, config = {}) {
|
|
|
1054
1129
|
totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
|
|
1055
1130
|
events: (pool.state.events ?? []).slice(-50),
|
|
1056
1131
|
healthScore: computeHealthScore(pool.state),
|
|
1132
|
+
// #208: today/week spend + configured budget for the card
|
|
1133
|
+
todayCost: costForDay(pool.state.costDays),
|
|
1134
|
+
weeklyCost: costForWeek(pool.state.costDays, now),
|
|
1135
|
+
budgetDaily: providerBudgets.get(pool.base)?.costBudgetDaily ?? 0,
|
|
1136
|
+
budgetWeekly: providerBudgets.get(pool.base)?.costBudgetWeekly ?? 0,
|
|
1137
|
+
pauseOnBudget: providerBudgets.get(pool.base)?.pauseOnBudget ?? false,
|
|
1057
1138
|
});
|
|
1139
|
+
} catch (e) {
|
|
1140
|
+
console.warn(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
|
|
1141
|
+
providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
|
|
1142
|
+
}
|
|
1058
1143
|
}
|
|
1059
1144
|
json(res, 200, { providers });
|
|
1060
1145
|
},
|
|
1061
1146
|
}), 'dsh-key-rotation: status route');
|
|
1062
1147
|
|
|
1148
|
+
// #209: usage report - per-key requests/cost over the last N days.
|
|
1149
|
+
// ?format=csv returns text/csv; ?days=N window (1..90, default 7).
|
|
1150
|
+
ctx.effect(() => ctx.webServer.register({
|
|
1151
|
+
kind: 'exact',
|
|
1152
|
+
path: USAGE_PATH,
|
|
1153
|
+
handler: (req, res) => {
|
|
1154
|
+
if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
|
|
1155
|
+
if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: usage is local-only' } }); return; }
|
|
1156
|
+
const url = new URL(req.url ?? USAGE_PATH, 'http://localhost');
|
|
1157
|
+
const days = Math.min(90, Math.max(1, Number(url.searchParams.get('days')) || 7));
|
|
1158
|
+
const csv = url.searchParams.get('format') === 'csv';
|
|
1159
|
+
const provider = url.searchParams.get('provider') ?? '';
|
|
1160
|
+
const runtime = buildRuntime();
|
|
1161
|
+
const now = Date.now();
|
|
1162
|
+
const seen = new Set();
|
|
1163
|
+
const report = [];
|
|
1164
|
+
for (const pool of runtime.poolByRef.values()) {
|
|
1165
|
+
if (seen.has(pool.base)) continue;
|
|
1166
|
+
seen.add(pool.base);
|
|
1167
|
+
if (provider && pool.base !== provider) continue;
|
|
1168
|
+
report.push({ provider: pool.base, rows: usageRows(pool, days, now) });
|
|
1169
|
+
}
|
|
1170
|
+
if (csv) {
|
|
1171
|
+
res.writeHead(200, { 'content-type': 'text/csv; charset=utf-8', 'content-disposition': 'attachment; filename="dsh-key-rotation-usage.csv"' });
|
|
1172
|
+
const parts = [];
|
|
1173
|
+
for (const p of report) {
|
|
1174
|
+
if (parts.length > 0) parts.push('');
|
|
1175
|
+
parts.push('# ' + p.provider);
|
|
1176
|
+
parts.push(usageCsv(p.rows));
|
|
1177
|
+
}
|
|
1178
|
+
res.end(parts.join('\n') + '\n');
|
|
1179
|
+
return;
|
|
1180
|
+
}
|
|
1181
|
+
json(res, 200, { at: now, days, providers: report });
|
|
1182
|
+
},
|
|
1183
|
+
}), 'dsh-key-rotation: usage route');
|
|
1184
|
+
|
|
1063
1185
|
// ── key route: store a key value without leaving the rotation card ──
|
|
1064
1186
|
//
|
|
1065
1187
|
// Adding a key used to mean two screens: create the credential elsewhere,
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// lib/maintenance.js - pure helpers for #207 (expiry pre-warning) and
|
|
2
|
+
// #208 (cost budget). No I/O; the timers in index.js call these and decide
|
|
3
|
+
// whether to send webhooks.
|
|
4
|
+
|
|
5
|
+
const DAY_MS = 86400000;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* #207: keys of `pool` whose expiresAt falls within the next `warnDays`.
|
|
9
|
+
* Returns [{ ref, expiresInDays, expiresAt }], soonest first.
|
|
10
|
+
*/
|
|
11
|
+
export function expiringSoon(pool, warnDays, now = Date.now()) {
|
|
12
|
+
if (!pool || !pool.expiresAt) return [];
|
|
13
|
+
const horizon = now + Math.max(1, warnDays ?? 7) * DAY_MS;
|
|
14
|
+
return Object.entries(pool.expiresAt)
|
|
15
|
+
.filter(([, at]) => at > now && at <= horizon)
|
|
16
|
+
.map(([ref, at]) => ({ ref, expiresAt: at, expiresInDays: Math.max(0, Math.floor((at - now) / DAY_MS)) }))
|
|
17
|
+
.sort((a, b) => a.expiresAt - b.expiresAt);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** #207 dedupe: true when a day-level notification for `key` is due. */
|
|
21
|
+
export function shouldNotifyDaily(lastNotified, key, now = Date.now()) {
|
|
22
|
+
if (!lastNotified.has(key)) {
|
|
23
|
+
lastNotified.set(key, now);
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
const last = lastNotified.get(key);
|
|
27
|
+
if (now - last < DAY_MS) return false;
|
|
28
|
+
lastNotified.set(key, now);
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* #208: total spend of a pool on ISO day `day` (defaults to today)
|
|
34
|
+
* across costDays Map<ref, Map<day, cost>>.
|
|
35
|
+
*/
|
|
36
|
+
export function costForDay(costDays, day) {
|
|
37
|
+
const d = day ?? new Date().toISOString().slice(0, 10);
|
|
38
|
+
let total = 0;
|
|
39
|
+
for (const perRef of (costDays?.values() ?? [])) {
|
|
40
|
+
total += perRef.get(d) ?? 0;
|
|
41
|
+
}
|
|
42
|
+
return total;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** #208: total spend over the last 7 ISO days ending today. */
|
|
46
|
+
export function costForWeek(costDays, now = Date.now()) {
|
|
47
|
+
let total = 0;
|
|
48
|
+
for (let i = 0; i < 7; i++) {
|
|
49
|
+
const d = new Date(now - i * 86400000).toISOString().slice(0, 10);
|
|
50
|
+
total += costForDay(costDays, d);
|
|
51
|
+
}
|
|
52
|
+
return total;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* #208 budget verdict for a pool: { spend, budget, ratio, warn, exceeded }.
|
|
57
|
+
* budget <= 0 -> never warn.
|
|
58
|
+
*/
|
|
59
|
+
export function budgetVerdict(spend, budget) {
|
|
60
|
+
if (!budget || budget <= 0) return { spend, budget: 0, ratio: 0, warn: false, exceeded: false };
|
|
61
|
+
const ratio = spend / budget;
|
|
62
|
+
// warn from 80%, exceeded at 100%
|
|
63
|
+
return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
|
|
64
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// lib/usage-report.js - #209 usage report rows from pool state. Pure helpers.
|
|
2
|
+
const DAY_MS = 86400000;
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Build per-key usage rows for the last `days` ISO days (default 7).
|
|
6
|
+
* Returns [{ ref, requests, cost, active, usageByDay: {day: n} }].
|
|
7
|
+
*/
|
|
8
|
+
export function usageRows(pool, days = 7, now = Date.now()) {
|
|
9
|
+
const out = [];
|
|
10
|
+
const dayKeys = [];
|
|
11
|
+
for (let i = 0; i < Math.max(1, days); i++) dayKeys.push(new Date(now - i * DAY_MS).toISOString().slice(0, 10));
|
|
12
|
+
const refs = pool?.refs ?? [];
|
|
13
|
+
for (const ref of refs) {
|
|
14
|
+
const daysMap = pool.state.usageDays?.get(ref) ?? new Map();
|
|
15
|
+
const costMap = pool.state.costDays?.get(ref) ?? new Map();
|
|
16
|
+
let requests = 0, cost = 0;
|
|
17
|
+
const usageByDay = {};
|
|
18
|
+
for (const d of dayKeys) {
|
|
19
|
+
const r = daysMap.get(d) ?? 0;
|
|
20
|
+
const c = costMap.get(d) ?? 0;
|
|
21
|
+
requests += r;
|
|
22
|
+
cost += c;
|
|
23
|
+
usageByDay[d] = r;
|
|
24
|
+
}
|
|
25
|
+
out.push({
|
|
26
|
+
ref,
|
|
27
|
+
requests,
|
|
28
|
+
cost: Math.round(cost * 100) / 100,
|
|
29
|
+
active: pool.state.lastUsed === ref,
|
|
30
|
+
usageByDay,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** CSV of usage rows: ref,requests,cost,active + per-day columns. */
|
|
37
|
+
export function usageCsv(rows) {
|
|
38
|
+
const dayCols = [...new Set(rows.flatMap((r) => Object.keys(r.usageByDay)))].sort();
|
|
39
|
+
const head = ['ref', 'requests', 'cost', 'active', ...dayCols];
|
|
40
|
+
const esc = (v) => {
|
|
41
|
+
const s = String(v ?? '');
|
|
42
|
+
return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
|
|
43
|
+
};
|
|
44
|
+
const lines = [head.join(',')];
|
|
45
|
+
for (const r of rows) {
|
|
46
|
+
lines.push([esc(r.ref), r.requests, r.cost, r.active ? 'yes' : 'no', ...dayCols.map((d) => r.usageByDay[d] ?? 0)].join(','));
|
|
47
|
+
}
|
|
48
|
+
return lines.join('\n');
|
|
49
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.28",
|
|
4
4
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"deepseek-harness",
|