@goodandready/dsh-key-rotation 0.8.78 → 0.8.80

package/lib/ops-webhook.js

@@ -1,218 +1,2 @@
-import crypto from 'node:crypto';
-// lib/ops-webhook.js — webhook interactive action route (#312 split from routes-ops.js, #388 telegram secret-token & answerCallbackQuery).
-import {
-  json,
-  readJson,
-} from './http-bridge.js';
-import { bestEffort } from './best-effort.js';
-import { WEBHOOK_ACTION_PATH } from './ops-paths.js';
-import { resolveAction } from './webhook.js';
-
-function constantTimeEqual(a, b) {
-  if (typeof a !== 'string' || typeof b !== 'string') return false;
-  const aBuf = Buffer.from(a);
-  const bBuf = Buffer.from(b);
-  return aBuf.length === bBuf.length && crypto.timingSafeEqual(aBuf, bBuf);
-}
-
-function getHeader(headers, name) {
-  if (!headers || typeof headers !== 'object') return '';
-  const target = name.toLowerCase();
-  for (const [k, v] of Object.entries(headers)) {
-    if (k.toLowerCase() === target) return String(v ?? '');
-  }
-  return '';
-}
-
-/**
- * @param {object} ctx cordis context
- * @param {object} deps live dependencies from apply()
- */
-export function registerWebhookActionRoute(ctx, deps) {
-  const {
-    buildRuntime,
-    poolState,
-    setRotationDisabled,
-    circuitBreaker,
-  } = deps;
-
-ctx.effect(() => ctx.webServer.register({
-  kind: 'exact',
-  path: WEBHOOK_ACTION_PATH,
-  handler: async (req, res) => {
-    if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-    const runtime = buildRuntime();
-    const expected = runtime.webhookActionToken;
-    if (!expected) { json(res, 503, { error: { code: 'no-token', message: 'dsh-key-rotation: webhookActionToken is not configured' } }); return; }
-
-    const authHeader = getHeader(req.headers, 'authorization');
-    const secretHeader = getHeader(req.headers, 'x-telegram-bot-api-secret-token');
-    const bearerMatch = constantTimeEqual(authHeader, `Bearer ${expected}`);
-    const secretMatch = constantTimeEqual(secretHeader, expected);
-    if (!bearerMatch && !secretMatch) {
-      json(res, 401, { error: { code: 'unauthorized', message: 'dsh-key-rotation: bad webhook action token' } });
-      return;
-    }
-
-    let body;
-    try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-
-    // #222 / #388: Telegram setWebhook registration helper
-    if (typeof body?.setWebhook === 'object' && body.setWebhook) {
-      const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
-      if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
-      const TELEGRAM_BOT_TOKEN_RE = /^[0-9]{5,16}:[a-zA-Z0-9_-]{20,50}$/;
-      if (!TELEGRAM_BOT_TOKEN_RE.test(botToken)) {
-        json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: invalid telegram bot token format' } });
-        return;
-      }
-      const rawUrl = typeof body.setWebhook.url === 'string' ? body.setWebhook.url.trim() : '';
-      if (!rawUrl || !rawUrl.startsWith('https://')) {
-        json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: explicit public https url required in setWebhook.url' } });
-        return;
-      }
-      try {
-        const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({
-            url: rawUrl,
-            secret_token: expected,
-            allowed_updates: ['callback_query'],
-          }),
-          signal: AbortSignal.timeout(10000),
-        });
-        const hookData = await hookRes.json().catch(() => ({}));
-        json(res, 200, { ok: hookRes.ok, url: rawUrl, secret_token_set: true, telegram: hookData });
-      } catch (e) {
-        json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
-      }
-      return;
-    }
-
-    const callbackQueryId = typeof body?.callback_query?.id === 'string'
-      ? body.callback_query.id
-      : (typeof body?.callback_query?.id === 'number' ? String(body.callback_query.id) : '');
-    const botToken = typeof body?.botToken === 'string' ? body.botToken : '';
-
-    const respond = async (statusCode, payload, answerText) => {
-      if (callbackQueryId) {
-        payload.method = 'answerCallbackQuery';
-        payload.callback_query_id = callbackQueryId;
-        payload.text = answerText || payload.error?.message || 'OK';
-        if (botToken) {
-          await bestEffort('telegram-answerCallbackQuery', () => fetch(`https://api.telegram.org/bot${botToken}/answerCallbackQuery`, {
-            method: 'POST',
-            headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ callback_query_id: callbackQueryId, text: payload.text }),
-            signal: AbortSignal.timeout(5000),
-          }), ctx?.logger);
-        }
-      }
-      json(res, statusCode, payload);
-    };
-
-    // Accept callback payloads from formatInteractive (Telegram/Discord/Slack) or plain {action}
-    let action = typeof body?.action === 'string' ? body.action : '';
-    if (!action && typeof body?.data === 'string') {
-      try { action = String(JSON.parse(body.data)?.id ?? ''); } catch { action = ''; }
-    }
-    if (!action && typeof body?.callback_data === 'string') {
-      try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
-    }
-    // #222 / #388: Telegram update envelope {update_id, callback_query:{id, data}}
-    if (!action && typeof body?.callback_query?.data === 'string') {
-      try {
-        const parsed = JSON.parse(body.callback_query.data);
-        action = typeof parsed?.id === 'string' ? parsed.id : String(parsed ?? '');
-      } catch {
-        action = body.callback_query.data;
-      }
-    }
-    action = resolveAction(action);
-    if (!action) {
-      await respond(400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }, 'No action in payload');
-      return;
-    }
-    const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
-    try {
-      if (action === 'disable-rotation') {
-        setRotationDisabled(true);
-        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.('[dsh-key-rotation] rotation DISABLED via webhook action');
-        await respond(200, { ok: true, action }, 'Rotation disabled');
-        return;
-      }
-      if (action === 'enable-rotation') {
-        setRotationDisabled(false);
-        await respond(200, { ok: true, action }, 'Rotation enabled');
-        return;
-      }
-      if (action.startsWith('pause-') || action.startsWith('reset-')) {
-        const st = poolState.get(provider);
-        if (!st) {
-          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-          return;
-        }
-        if (action.startsWith('pause-')) {
-          const until = Date.now() + 3600000; // 1h pause
-          let found = false;
-          for (const [key, s] of poolState.entries()) {
-            if (key === provider || key.startsWith(provider + '::')) {
-              found = true;
-              for (const ref of (s.failedUntil ? [...s.failedUntil.keys()] : [])) {
-                s.failedUntil.set(ref, Math.max(s.failedUntil.get(ref) ?? 0, until));
-              }
-            }
-          }
-          for (const p of buildRuntime().poolByRef.values()) {
-            if (p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) {
-              found = true;
-              for (const ref of p.refs) p.state?.failedUntil?.set(ref, Math.max(p.state?.failedUntil?.get(ref) ?? 0, until));
-            }
-          }
-          if (!found) {
-            await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-            return;
-          }
-          (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
-          await respond(200, { ok: true, action, provider, until }, `Pool ${provider} paused 1h`);
-          return;
-        }
-        let found = false;
-        let cleared = 0;
-        for (const [key, s] of poolState.entries()) {
-          if (key === provider || key.startsWith(provider + '::')) {
-            found = true;
-            cleared += s.failedUntil ? s.failedUntil.size : 0;
-            s.failedUntil?.clear();
-            s.failCounts?.clear();
-            s.authFailCounts?.clear();
-            s.brokenUntil?.clear();
-            s.revokedRefs?.clear();
-            s.tokenUsage?.clear();
-            s.switches = 0;
-            s.lastReason = undefined;
-            s.lastSwitchAt = undefined;
-          }
-        }
-        if (!found) {
-          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-          return;
-        }
-        let circuitReset = false;
-        const br = circuitBreaker ?? buildRuntime().breaker;
-        if (br) {
-          if (typeof br.reset === 'function') { br.reset(provider); circuitReset = true; }
-          else if (typeof br.onSuccess === 'function') { br.onSuccess(provider); circuitReset = true; }
-        }
-        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} RESET via webhook action (circuitReset=${circuitReset})`);
-        await respond(200, { ok: true, action, provider, cleared, circuitReset }, `Pool ${provider} reset`);
-        return;
-      }
-      await respond(400, { error: { code: 'unknown-action', message: `dsh-key-rotation: unknown action '${action}'` } }, `Unknown action: ${action}`);
-    } catch (e) {
-      await respond(500, { error: { code: 'action-failed', message: String(e?.message ?? e) } }, 'Action failed');
-    }
-  },
-}), 'dsh-key-rotation: webhook-action');
-}
+// lib/ops-webhook.js — Re-export facade for lib/routes/ (#522).
+export * from "./routes/webhook.js";

package/lib/persistence.js

@@ -1,8 +1,9 @@
-// lib/persistence.js — atomic snapshot of rotation state across restarts (#287).
-// Cooldowns, circuit-breaker entries and quota counters live in memory; a DSH
-// reload would otherwise forget cooldowns and stampede recovered keys.
+// lib/persistence.js — atomic snapshot and append-only event journal (#287, #525).
+// Cooldowns, circuit-breaker entries and quota counters live in memory; an append-only
+// journal records incremental events without rewriting the entire state tree on disk.
 
 import path from 'node:path';
+import fs from 'node:fs';
 import { initializePoolState, hasPoolActivity } from './pool-state.js';
 import { atomicWriteJson, safeReadJson } from './atomic-io.js';
 
@@ -18,6 +19,9 @@ export class StatePersistence {
       throw new Error('StatePersistence: filePath is required');
     }
     this.filePath = filePath;
+    this.journalPath = filePath.endsWith('.json')
+      ? filePath.slice(0, -5) + '-journal.jsonl'
+      : `${filePath}.jsonl`;
     this._now = now;
     this._timer = null;
     this._dirty = false;
@@ -112,8 +116,8 @@ export class StatePersistence {
             if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
               const innerObj = {};
               const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
-              for (const [day, c] of innerEntries) {
-                const num = Number(c);
+              for (const [day, count] of innerEntries) {
+                const num = Number(count);
                 if (Number.isFinite(num) && num >= 0) innerObj[day] = Math.floor(num);
               }
               if (Object.keys(innerObj).length > 0) usageDays[k] = innerObj;
@@ -168,7 +172,22 @@ export class StatePersistence {
   }
 
   /**
-   * Schedule a debounced atomic write. Never throws to the caller.
+   * Append-only event journaling (#525). Records incremental event without full state rewrite.
+   * @param {object} event
+   */
+  async appendEvent(event) {
+    if (!event || typeof event !== 'object') return false;
+    const line = JSON.stringify({ ...event, at: event.at ?? this._nowSafe() }) + '\n';
+    try {
+      await fs.promises.appendFile(this.journalPath, line, 'utf-8');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Schedule a debounced atomic snapshot write. Never throws to the caller.
    * @param {object} payload result of StatePersistence.serialize
    */
   save(payload) {
@@ -190,9 +209,11 @@ export class StatePersistence {
     try {
       await atomicWriteJson(this.filePath, payload);
       this._lastWrite = this._nowSafe();
+      try {
+        await fs.promises.writeFile(this.journalPath, '', 'utf-8');
+      } catch {}
       return true;
     } catch {
-      // Keep previous good file; mark dirty so a later save retries.
       this._dirty = true;
       this._pending = this._pending || payload;
       return false;
@@ -200,16 +221,56 @@ export class StatePersistence {
   }
 
   /**
-   * Load a snapshot. Corrupt/missing file → null (never wipes memory).
+   * Load snapshot and replay any journaled events (#525).
    * @returns {Promise<object|null>}
    */
   async load() {
     const raw = await safeReadJson(this.filePath, null);
     if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
     if (raw.version !== 1 && raw.version !== 2) return null;
+
+    if (fs.existsSync(this.journalPath)) {
+      try {
+        const content = await fs.promises.readFile(this.journalPath, 'utf-8');
+        const lines = content.split('\n');
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed) {
+            try { this.replayEvent(raw, JSON.parse(trimmed)); } catch {}
+          }
+        }
+      } catch {}
+    }
+
     return raw;
   }
 
+  /** Replay individual event into a snapshot object (#525). */
+  replayEvent(snapshot, event) {
+    if (!snapshot || !event?.pool || !event?.ref) return;
+    if (!snapshot.pools) snapshot.pools = {};
+    if (!snapshot.pools[event.pool]) {
+      snapshot.pools[event.pool] = { failedUntil: {}, pausedRefs: [], revokedRefs: [] };
+    }
+    const p = snapshot.pools[event.pool];
+    if (event.type === 'cooldown' || event.type === 'failure') {
+      if (!p.failedUntil) p.failedUntil = {};
+      p.failedUntil[event.ref] = event.until ?? (event.at + (event.cooldownMs || 60000));
+    } else if (event.type === 'success' || event.type === 'reset') {
+      if (p.failedUntil) delete p.failedUntil[event.ref];
+    } else if (event.type === 'pause') {
+      if (!Array.isArray(p.pausedRefs)) p.pausedRefs = [];
+      if (!p.pausedRefs.includes(event.ref)) p.pausedRefs.push(event.ref);
+    } else if (event.type === 'unpause') {
+      if (Array.isArray(p.pausedRefs)) {
+        p.pausedRefs = p.pausedRefs.filter((r) => r !== event.ref);
+      }
+    } else if (event.type === 'revoke') {
+      if (!Array.isArray(p.revokedRefs)) p.revokedRefs = [];
+      if (!p.revokedRefs.includes(event.ref)) p.revokedRefs.push(event.ref);
+    }
+  }
+
   /** Restore poolState Map from a loaded snapshot. */
   static restorePools(poolState, snapshot) {
     if (!poolState || !snapshot?.pools || typeof snapshot.pools !== 'object' || Array.isArray(snapshot.pools)) return 0;
@@ -218,8 +279,6 @@ export class StatePersistence {
       if (!saved || typeof saved !== 'object' || Array.isArray(saved)) continue;
       const existing = poolState.get(base);
       const state = initializePoolState(existing);
-      // Never replace objects retained by buildRuntime(), nor roll back a pool
-      // already used while the asynchronous load was in flight.
       if (!existing || !hasPoolActivity(state)) {
         if (saved.failedUntil && typeof saved.failedUntil === 'object' && !Array.isArray(saved.failedUntil)) {
           for (const [ref, until] of Object.entries(saved.failedUntil)) {

package/lib/pool-state.js

@@ -1,5 +1,7 @@
 // Shared runtime shape for fresh and persisted pools. Hydrate in place so cached
 // runtime pools, persistence and timers continue to observe the same object.
+import { KeyStateMachine } from './key-state-machine.js';
+
 const MAP_FIELDS = [
   'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
   'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
@@ -18,6 +20,7 @@ export function initializePoolState(state = {}) {
   if (!Array.isArray(state.events)) state.events = [];
   if (!Number.isSafeInteger(state.pointer) || state.pointer < 0) state.pointer = 0;
   if (typeof state.lastUsed !== 'string') state.lastUsed = null;
+  if (!state.keyStateMachine) state.keyStateMachine = new KeyStateMachine();
   return state;
 }
 

package/lib/pool.js

@@ -195,6 +195,7 @@ export function pickNext(pool, now, refsCount = pool.refs.length) {
 /** Apply a failed-key cooldown to pool state. Pure: returns next state shape. */
 export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
   unstickFailedKey(pool, ref);
+  pool?.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs }, now);
   return {
     ...pool,
     state: {
@@ -231,10 +232,9 @@ export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jit
     pool.state.failCounts.set(ref, next);
   }
   let backoff = computeBackoff(baseMs, next || 1, maxMs, isSoft);
-  if (jitter) {
-    backoff = applyJitter(backoff);
-  }
+  if (jitter) backoff = applyJitter(backoff);
   pool.state.failedUntil.set(ref, now + backoff);
+  pool.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs: backoff, reason: isSoft ? 'SOFT_FAILURE' : 'FAILURE' }, now);
   return backoff;
 }
 
@@ -245,6 +245,7 @@ export function recordSuccess(pool, ref, now = Date.now()) {
   pool.state.quotaWindows?.delete(ref);
   if (!pool.state.lastSuccessAt) pool.state.lastSuccessAt = new Map();
   pool.state.lastSuccessAt.set(ref, now);
+  pool.state?.keyStateMachine?.activate(ref, { reason: 'SUCCESS' }, now);
 }
 
 /** Decay penalty failCounts for stable keys that haven't failed in decayIntervalMs. */
@@ -376,12 +377,9 @@ export function parseDurationToSeconds(v) {
 /** Check if a key reference is in maintenance/paused mode. */
 export function isKeyPaused(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
-  if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
-  if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
-  if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
-  if (pool.basePool) return isKeyPaused(pool.basePool, ref);
-  return false;
+  if (pool.pausedRefs?.has?.(ref) || pool.pausedRefs?.includes?.(ref)) return true;
+  if (pool.state?.pausedRefs?.has?.(ref) || pool.state?.pausedRefs?.includes?.(ref)) return true;
+  return pool.basePool ? isKeyPaused(pool.basePool, ref) : false;
 }
 
 /** Extract rate-limit info from an object that may carry response headers. */
@@ -578,12 +576,9 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
 /** Check if a key reference is permanently revoked due to auth/401 failure (#369). */
 export function isKeyRevoked(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.revokedRefs instanceof Set && pool.revokedRefs.has(ref)) return true;
-  if (Array.isArray(pool.revokedRefs) && pool.revokedRefs.includes(ref)) return true;
-  if (pool.state && pool.state.revokedRefs instanceof Set && pool.state.revokedRefs.has(ref)) return true;
-  if (pool.state && Array.isArray(pool.state.revokedRefs) && pool.state.revokedRefs.includes(ref)) return true;
-  if (pool.basePool) return isKeyRevoked(pool.basePool, ref);
-  return false;
+  if (pool.revokedRefs?.has?.(ref) || pool.revokedRefs?.includes?.(ref)) return true;
+  if (pool.state?.revokedRefs?.has?.(ref) || pool.state?.revokedRefs?.includes?.(ref)) return true;
+  return pool.basePool ? isKeyRevoked(pool.basePool, ref) : false;
 }
 
 /** Check if a key reference is in active recoverable or permanent quarantine (#427). */

package/lib/quota.js

@@ -1,39 +1,2 @@
-// lib/quota.js — quota-remaining persistence per ref.
-// ponytail: pure helpers, snapshot() returns shallow copies.
-
-export class QuotaStore {
-  constructor() {
-    this._data = new Map(); // ref -> { remaining, limit, reset, at }
-  }
-
-  set(ref, info) {
-    if (!ref) return;
-    if (!info || typeof info !== 'object') return;
-    const next = {
-      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
-      limit: Number.isFinite(info.limit) ? info.limit : null,
-      reset: Number.isFinite(info.reset) ? info.reset : null,
-      at: Number.isFinite(info.at) ? info.at : Date.now(),
-    };
-    this._data.set(ref, next);
-  }
-
-  get(ref) {
-    return this._data.get(ref);
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._data) out[k] = v;
-    return out;
-  }
-
-  clear(ref) {
-    if (ref) this._data.delete(ref);
-    else this._data.clear();
-  }
-
-  get size() {
-    return this._data.size;
-  }
-}
+// lib/quota.js — Backwards-compatible facade (#530).
+export * from './core/quota.js';

package/lib/routes/balance.js

@@ -0,0 +1,147 @@
+// lib/ops-balance.js — operational HTTP routes for balance inspection (#495).
+
+import {
+  json,
+  readJson,
+} from '../http-bridge.js';
+import {
+  isTrustedBridgeRequest,
+  isValidRef,
+} from '../pool.js';
+import { BALANCE_PATH } from './paths.js';
+import { inspectBalanceForRef } from '../balance-inspector.js';
+import { pushEvent } from '../notify-events.js';
+
+export function registerBalanceRoutes(ctx, deps) {
+  const {
+    poolState,
+    buildRuntime,
+  } = deps;
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: BALANCE_PATH,
+    handler: async (req, res) => {
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { ok: false, error: 'Forbidden' });
+        return;
+      }
+
+      const urlObj = new URL(req.url, 'http://127.0.0.1');
+
+      // GET: Query single key or provider cached balances
+      if (req.method === 'GET') {
+        const ref = urlObj.searchParams.get('ref');
+        const provider = urlObj.searchParams.get('provider');
+
+        if (ref) {
+          const runtime = buildRuntime();
+          const targetPool = provider ? runtime.pools?.find((p) => p.provider === provider) : null;
+          const pool = targetPool ?? runtime.poolByRef?.get(ref);
+          const cached = pool?.state?.balances?.get(ref) ?? null;
+
+          if (urlObj.searchParams.get('refresh') === 'true' && provider) {
+            const inspectRes = await inspectBalanceForRef(ctx, ref, provider);
+            if (inspectRes.ok) {
+              if (pool && pool.state) {
+                if (!pool.state.balances) pool.state.balances = new Map();
+                pool.state.balances.set(ref, { ...inspectRes, at: Date.now() });
+
+                // Check soft pause
+                applyBalanceState(pool, ref, inspectRes);
+              }
+              json(res, 200, { ok: true, ref, balance: inspectRes });
+              return;
+            }
+            json(res, 200, { ok: false, ref, error: inspectRes.error, cached });
+            return;
+          }
+
+          json(res, 200, { ok: true, ref, balance: cached });
+          return;
+        }
+
+        // Return all cached balances
+        const out = {};
+        if (poolState && typeof poolState.forEach === 'function') {
+          poolState.forEach((st, provName) => {
+            if (st && st.balances instanceof Map) {
+              for (const [k, v] of st.balances) {
+                out[k] = v;
+              }
+            }
+          });
+        }
+        json(res, 200, { ok: true, balances: out });
+        return;
+      }
+
+      // POST: Trigger refresh of balances for a provider or key list
+      if (req.method === 'POST') {
+        const body = await readJson(req);
+        const provider = body?.provider;
+        const ref = body?.ref;
+        const testValue = body?.value;
+
+        if (ref && provider) {
+          const inspectRes = await inspectBalanceForRef(ctx, ref, provider, { testValue });
+          const runtime = buildRuntime();
+          const pool = runtime.pools?.find((p) => p.provider === provider) ?? runtime.poolByRef?.get(ref);
+
+          if (inspectRes.ok && pool && pool.state) {
+            if (!pool.state.balances) pool.state.balances = new Map();
+            pool.state.balances.set(ref, { ...inspectRes, at: Date.now() });
+            applyBalanceState(pool, ref, inspectRes);
+          }
+
+          json(res, inspectRes.ok ? 200 : 400, { ok: inspectRes.ok, ref, balance: inspectRes });
+          return;
+        }
+
+        if (provider) {
+          const runtime = buildRuntime();
+          const pool = runtime.pools?.find((p) => p.provider === provider);
+          if (!pool || !Array.isArray(pool.refs) || pool.refs.length === 0) {
+            json(res, 404, { ok: false, error: `Provider '${provider}' not found or has no keys` });
+            return;
+          }
+
+          const results = {};
+          for (const kRef of pool.refs) {
+            const r = await inspectBalanceForRef(ctx, kRef, provider);
+            results[kRef] = r;
+            if (r.ok && pool.state) {
+              if (!pool.state.balances) pool.state.balances = new Map();
+              pool.state.balances.set(kRef, { ...r, at: Date.now() });
+              applyBalanceState(pool, kRef, r);
+            }
+          }
+          json(res, 200, { ok: true, provider, balances: results });
+          return;
+        }
+
+        json(res, 400, { ok: false, error: 'ref or provider required' });
+        return;
+      }
+
+      json(res, 405, { ok: false, error: 'Method Not Allowed' });
+    },
+  }), 'dsh-key-rotation: balance routes');
+}
+
+function applyBalanceState(pool, ref, balanceRes) {
+  if (!pool || !pool.state) return;
+  const isZero = balanceRes.isAvailable === false || (typeof balanceRes.total === 'number' && balanceRes.total <= 0);
+
+  if (isZero) {
+    if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(ref);
+    if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
+    pool.state.pauseReasons.set(ref, 'EMPTY_BALANCE_PAUSED');
+    pushEvent(pool, ref, 'BALANCE_EMPTY', 0);
+  } else if (pool.state.pauseReasons?.get?.(ref) === 'EMPTY_BALANCE_PAUSED') {
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.delete(ref);
+    pool.state.pauseReasons.delete(ref);
+    pushEvent(pool, ref, 'BALANCE_RESTORED', 0);
+  }
+}