@goodandready/dsh-key-rotation 0.7.29 → 0.7.30

package/README.md

@@ -49,6 +49,11 @@
   - **Budget action buttons** (added in 0.7.29) — when `webhookActionToken` is set, budget notifications carry *Pause 1h* / *Reset cooldown* buttons (same callback route as exhaustion alerts).
   - **Config snapshot export/restore** (added in 0.7.29) — *Snapshot ⬇* downloads the whole config as one JSON (token fields exported empty, keys are credential names only); *Restore* imports it back — empty token fields never wipe existing secrets, and a live-looking credential in the file is rejected.
   - **Last probe result per key** (added in 0.7.29) — the card polls `/sandbox-cache` and shows the most recent probe outcome (✓/✕ + latency) next to each key, greyed out when older than 24 h.
+  - **Pre-exhaustion alert** (added in 0.7.30) — `warnBelowHealthy` (0 = off): while fewer keys than the threshold are healthy, a webhook fires (once per day per pool) with a *Reset cooldown* button, so the pool never silently dries up.
+  - **Telegram-native callbacks** (added in 0.7.30) — `/webhook-action` parses Telegram update envelopes (`callback_query.data`), so interactive buttons work natively; `POST {"setWebhook": {"botToken": …}}` registers the bot webhook in one call (the token is not stored).
+  - **Broken-key review** (added in 0.7.30) — keys quarantined for repeated AUTH failures get a *Re-test* button: a live probe (models list) that automatically lifts the 30-day quarantine when the key answers again.
+  - **7-day switches chart** (added in 0.7.30) — a second bar chart under the 24 h sparkline shows switches per day over the last week (client-side, no new server state).
+  - **Latency SLO alert** (added in 0.7.30) — `latencySloMs` (0 = off): when a key's p95 latency exceeds the threshold, a webhook fires (once per day per key); the card shows `p95 / SLO` per provider.
 
 ## Install
 
@@ -96,6 +101,8 @@ dsh-key-rotation:
 | `expiryWarnDays` | `7` | Pre-warning horizon (days) for keys with `expiresAt`: webhook + card badge. |
 | `switchNotify` | `false` | Send a webhook on every key switch (opt-in — can be chatty). |
 | `switchNotifyThrottleMs` | `60000` | Minimum gap between switch notifications for the same provider. |
+| `warnBelowHealthy` | `0` | Webhook "pool running low" while healthy keys < N (0 = off). |
+| `latencySloMs` | `0` | Latency SLO per key: webhook when p95 exceeds it (0 = off). |
 | `providers[].weights` | `[]` | Positional round-robin weights per key (editable in the card, 0.7.29). |
 | `providers[].costBudgetDaily` / `.costBudgetWeekly` | `0` | Daily / weekly spend budget per provider (0 = off). Warn webhook from 80%. |
 | `providers[].pauseOnBudget` | `false` | Pause the whole pool for 24 h when a budget is exceeded. |

package/lib/client.js

@@ -68,6 +68,8 @@ window.__ModuleLoader__.load({
       budgetLabel: 'budget:',
       exportCsv: 'Export CSV',
       weightHint: 'round-robin weight: how many times this key joins the cycle (1 = equal share)',
+      retestBroken: 'Re-test',
+      retestFail: 're-test failed - key still down',
       snapshotExport: 'Snapshot ⬇',
       snapshotImport: 'Restore',
       keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
@@ -131,6 +133,8 @@ window.__ModuleLoader__.load({
       budgetLabel: 'бюджет:',
       exportCsv: 'CSV',
       weightHint: 'вес в круге: сколько раз ключ участвует в ротации (1 = поровну)',
+      retestBroken: 'Re-test',
+      retestFail: 'перепроверка не прошла — ключ всё ещё недоступен',
       snapshotExport: 'Снапшот ⬇',
       snapshotImport: 'Восстановить',
       keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
@@ -380,9 +384,21 @@ window.__ModuleLoader__.load({
           .catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))))
           .finally(() => setResetting(''));
       };
-
-
-      const keyInfo = (providerId, ref) => {
+      // #223: re-test a broken key; a successful live probe lifts the 30-day broken quarantine
+      const retestBroken = (ref) => {
+        setSecretError('');
+        fetch('/dsh-key-rotation/test', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref, probe: 'models' }) })
+          .then((r) => r.json())
+          .then((data) => {
+            if (data && data.ok) {
+              return fetch('/dsh-key-rotation/reset', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref }) });
+            }
+            setSecretError(t('retestFail'));
+            return null;
+          })
+          .catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))));
+      };
+      const keyInfo = (providerId, ref) => {
         const entryStatus = status[providerId];
         if (!entryStatus || !ref) return null;
         return (entryStatus.keys ?? []).find((k) => k.ref === ref) ?? null;
@@ -682,6 +698,10 @@ window.__ModuleLoader__.load({
               (pc.ok ? '✓' : '✕') + (pc.latencyMs ? ' ' + pc.latencyMs + 'ms' : '')));
           }
           meta.push(h('button', { key: 't', className: 'krot-btn', onClick: () => doTest(key), disabled: testing === key, title: t('testKey') }, testing === key ? '…' : t('testKey')));
+          // #223: broken keys get a one-click live re-test + auto-unbreak
+          if (info && info.broken) {
+            meta.push(h('button', { key: 'rt', className: 'krot-btn', onClick: () => retestBroken(key), title: t('retestBroken') }, t('retestBroken')));
+          }
           meta.push(h('span', { key: 'a', className: 'krot-acts' },
             btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
             btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),
@@ -719,6 +739,14 @@ window.__ModuleLoader__.load({
               return h('p', { className: 'krot-hint', style: { color } }, t('budgetLabel') + ' ' + parts.join(' · '));
             })()
           : null;
+        // #225: provider p95 latency + SLO marker
+        const sloLine = providerStatus && providerStatus.p95 != null
+          ? (() => {
+              const over = providerStatus.latencySloMs && providerStatus.p95 > providerStatus.latencySloMs;
+              return h('p', { className: 'krot-hint', style: over ? { color: 'var(--dsw-alias-state-warning-primary)' } : undefined },
+                'p95 ' + providerStatus.p95 + 'ms' + (providerStatus.latencySloMs ? ' / ' + providerStatus.latencySloMs + 'ms SLO' : ''));
+            })()
+          : null;
         // #209: CSV export for this provider's usage (last 7 days)
         const exportCsv = h('button', { className: 'krot-btn', title: t('exportCsv'),
           onClick: () => {
@@ -765,8 +793,11 @@ window.__ModuleLoader__.load({
             btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle') }),
             switchesLine,
             budgetLine,
+            sloLine,
             exhaustionWarning,
             (providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('div', { style: { display: 'flex', gap: '2px', alignItems: 'end', height: '24px', marginTop: '4px' } }, (() => { const now = Date.now(); const buckets = Array(24).fill(0); for (const ev of providerStatus.events) { const h = Math.floor((now - ev.at) / 3600000); if (h >= 0 && h < 24) buckets[23 - h]++; } const max = Math.max(1, ...buckets); return buckets.map((c, i) => h('div', { key: i, title: c + ' switches', style: { flex: 1, background: c ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-border-l2)', height: (c / max * 24) + 'px', minHeight: '2px', borderRadius: '2px' } })); })()) : null),
+            // #224: 7-day switches per day (client-side, from the same events)
+            (providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('div', { style: { display: 'flex', gap: '2px', alignItems: 'end', height: '16px', marginTop: '2px' } }, (() => { const now = Date.now(); const days = Array(7).fill(0); for (const ev of providerStatus.events) { const d = Math.floor((now - ev.at) / 86400000); if (d >= 0 && d < 7) days[6 - d]++; } const max = Math.max(1, ...days); return days.map((c, i) => h('div', { key: i, title: c + ' switches · day -' + (6 - i), style: { flex: 1, background: c ? 'var(--dsw-alias-state-info-primary, var(--dsw-alias-state-warning-primary))' : 'var(--dsw-alias-border-l2)', height: (c / max * 16) + 'px', minHeight: '2px', borderRadius: '2px' } })); })()) : null),
             (providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('details', { style: { fontSize: '11px', marginTop: '6px' } }, h('summary', null, 'Recent failures ('+providerStatus.events.length+')'), h('ul', { style: { margin: '4px 0 0', paddingLeft: '16px' } }, providerStatus.events.slice().reverse().map((ev, i) => h('li', { key: i, style: ev.type === 'probe' ? { opacity: .5 } : null }, new Date(ev.at).toLocaleTimeString() + ' ' + (ev.type === 'probe' ? '[probe] ' : '') + ev.ref + ' ' + ev.reason + ' cd=' + ev.cooldownMs)) )) : null),
             btn(t('resetCooldown'), () => doReset(entry.provider), { disabled: !(providerStatus && providerStatus.switches > 0) || resetting === entry.provider, title: t('resetCooldown') }),
             btn(testAllProvider === entry.provider ? t('testing') : t('testAll'), () => doTestAll(entry.provider), { disabled: testAllProvider === entry.provider, title: t('testAll') }),

package/lib/index.js

@@ -86,6 +86,8 @@ let rotationDisabled = false;
 const expiryNotifiedAt = new Map();
 const budgetNotifiedAt = new Map();
 const switchNotifiedAt = new Map();
+const lowHealthNotifiedAt = new Map();
+const sloNotifiedAt = new Map();
 const DAY_MS = 86400000;
 
 // #216: one webhook per switch, deduped to at most one message per provider
@@ -215,6 +217,8 @@ export const Config = Schema.object({
   expiryWarnDays: Schema.number().default(7),
   switchNotify: Schema.boolean().default(false),
   switchNotifyThrottleMs: Schema.number().default(60000),
+  warnBelowHealthy: Schema.number().default(0),
+  latencySloMs: Schema.number().default(0),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
@@ -633,6 +637,55 @@ export function apply(ctx, config = {}) {
               if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
             }
           }
+          // #221: pool running low - webhook while healthy < warnBelowHealthy
+          const warnBelow = runtime.warnBelowHealthy ?? 0;
+          if (warnBelow > 0) {
+            let healthy = 0;
+            for (const ref of pool.refs) {
+              const fu = pool.state.failedUntil.get(ref);
+              if (fu !== undefined && fu > now) continue;
+              const exp = pool.expiresAt?.[ref];
+              if (exp !== undefined && now >= exp) continue;
+              healthy++;
+            }
+            if (healthy < warnBelow && shouldNotifyDaily(lowHealthNotifiedAt, pool.base, now)) {
+              console.warn(`[dsh-key-rotation] ${pool.base}: pool running low - ${healthy}/${pool.refs.length} healthy`);
+              if (runtime.notifyWebhook) {
+                const token = runtime.webhookActionToken ?? '';
+                webhookSender.send(runtime.notifyWebhook, {
+                  title: `Pool running low: ${pool.base}`,
+                  text: `${healthy}/${pool.refs.length} keys healthy (alert below ${warnBelow})`,
+                  provider: pool.base,
+                  kind: 'low-health',
+                  healthy,
+                  total: pool.refs.length,
+                  actionToken: token || undefined,
+                  actions: token ? [{ id: `reset-${pool.base}`, label: 'Reset cooldown' }] : undefined,
+                });
+              }
+            }
+          }
+          // #225: latency SLO - webhook when a key's p95 exceeds the threshold
+          const slo = runtime.latencySloMs ?? 0;
+          if (slo > 0) {
+            for (const ref of pool.refs) {
+              const snap = latencyHistogram.snapshot(ref);
+              if (!snap.p95 || snap.p95 <= slo) continue;
+              if (!shouldNotifyDaily(sloNotifiedAt, pool.base + ':' + ref + ':slo', now)) continue;
+              console.warn(`[dsh-key-rotation] ${pool.base}: ${ref} p95 ${Math.round(snap.p95)}ms > SLO ${slo}ms`);
+              if (runtime.notifyWebhook) {
+                webhookSender.send(runtime.notifyWebhook, {
+                  title: `Latency SLO exceeded: ${pool.base}`,
+                  text: `${ref} p95 ${Math.round(snap.p95)}ms > ${slo}ms (${snap.count} samples)`,
+                  provider: pool.base,
+                  kind: 'latency-slo',
+                  ref,
+                  p95: Math.round(snap.p95),
+                  slo,
+                });
+              }
+            }
+          }
         }
       } catch (_) { /* maintenance must never crash the sweep */ }
     }, 30000);
@@ -776,7 +829,7 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -1104,6 +1157,7 @@ export function apply(ctx, config = {}) {
       const { poolByRef, providerTags, providerBudgets } = buildRuntime();
       const base = ctx.get('credentials');
       const now = Date.now();
+      const latencySloMs = buildRuntime().latencySloMs;
       const seen = new Set();
       const providers = [];
       for (const pool of poolByRef.values()) {
@@ -1171,6 +1225,12 @@ export function apply(ctx, config = {}) {
           lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
           exhaustionCount: pool.state.exhaustionCount ?? 0,
           totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
+          // #225: aggregate p95 across the pool's keys
+          p95: (() => {
+            const vals = (pool.refs ?? []).map((r) => latencyHistogram.snapshot(r)).filter((s) => s && s.p95 != null).map((s) => s.p95);
+            return vals.length ? Math.round(Math.max(...vals)) : null;
+          })(),
+          latencySloMs,
           events: (pool.state.events ?? []).slice(-50),
           healthScore: computeHealthScore(pool.state),
           // #208: today/week spend + configured budget for the card
@@ -1592,6 +1652,28 @@ export function apply(ctx, config = {}) {
       if (!action && typeof body?.callback_data === 'string') {
         try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
       }
+      // #222: Telegram update envelope {update_id, callback_query:{data}}
+      if (!action && typeof body?.callback_query?.data === 'string') {
+        try { action = String(JSON.parse(body.callback_query.data)?.id ?? ''); } catch { action = ''; }
+      }
+      // #222: Telegram setWebhook registration helper
+      if (typeof body?.setWebhook === 'object' && body.setWebhook) {
+        const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
+        if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
+        // derive the public URL from request headers; explicit URL wins
+        const url = typeof body.setWebhook.url === 'string' && body.setWebhook.url ? body.setWebhook.url : `https://${String(req.headers.host ?? '')}/dsh-key-rotation/webhook-action`;
+        try {
+          const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
+            method: 'POST', headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ url, allowed_updates: ['callback_query'] }),
+          });
+          const hookData = await hookRes.json().catch(() => ({}));
+          json(res, 200, { ok: hookRes.ok, url, telegram: hookData });
+        } catch (e) {
+          json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
+        }
+        return;
+      }
       if (!action) { json(res, 400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }); return; }
       const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.29",
+  "version": "0.7.30",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",