@goodandready/dsh-key-rotation 0.8.78 → 0.8.79

package/lib/ops-webhook.js

@@ -1,218 +1,2 @@
-import crypto from 'node:crypto';
-// lib/ops-webhook.js — webhook interactive action route (#312 split from routes-ops.js, #388 telegram secret-token & answerCallbackQuery).
-import {
-  json,
-  readJson,
-} from './http-bridge.js';
-import { bestEffort } from './best-effort.js';
-import { WEBHOOK_ACTION_PATH } from './ops-paths.js';
-import { resolveAction } from './webhook.js';
-
-function constantTimeEqual(a, b) {
-  if (typeof a !== 'string' || typeof b !== 'string') return false;
-  const aBuf = Buffer.from(a);
-  const bBuf = Buffer.from(b);
-  return aBuf.length === bBuf.length && crypto.timingSafeEqual(aBuf, bBuf);
-}
-
-function getHeader(headers, name) {
-  if (!headers || typeof headers !== 'object') return '';
-  const target = name.toLowerCase();
-  for (const [k, v] of Object.entries(headers)) {
-    if (k.toLowerCase() === target) return String(v ?? '');
-  }
-  return '';
-}
-
-/**
- * @param {object} ctx cordis context
- * @param {object} deps live dependencies from apply()
- */
-export function registerWebhookActionRoute(ctx, deps) {
-  const {
-    buildRuntime,
-    poolState,
-    setRotationDisabled,
-    circuitBreaker,
-  } = deps;
-
-ctx.effect(() => ctx.webServer.register({
-  kind: 'exact',
-  path: WEBHOOK_ACTION_PATH,
-  handler: async (req, res) => {
-    if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-    const runtime = buildRuntime();
-    const expected = runtime.webhookActionToken;
-    if (!expected) { json(res, 503, { error: { code: 'no-token', message: 'dsh-key-rotation: webhookActionToken is not configured' } }); return; }
-
-    const authHeader = getHeader(req.headers, 'authorization');
-    const secretHeader = getHeader(req.headers, 'x-telegram-bot-api-secret-token');
-    const bearerMatch = constantTimeEqual(authHeader, `Bearer ${expected}`);
-    const secretMatch = constantTimeEqual(secretHeader, expected);
-    if (!bearerMatch && !secretMatch) {
-      json(res, 401, { error: { code: 'unauthorized', message: 'dsh-key-rotation: bad webhook action token' } });
-      return;
-    }
-
-    let body;
-    try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-
-    // #222 / #388: Telegram setWebhook registration helper
-    if (typeof body?.setWebhook === 'object' && body.setWebhook) {
-      const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
-      if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
-      const TELEGRAM_BOT_TOKEN_RE = /^[0-9]{5,16}:[a-zA-Z0-9_-]{20,50}$/;
-      if (!TELEGRAM_BOT_TOKEN_RE.test(botToken)) {
-        json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: invalid telegram bot token format' } });
-        return;
-      }
-      const rawUrl = typeof body.setWebhook.url === 'string' ? body.setWebhook.url.trim() : '';
-      if (!rawUrl || !rawUrl.startsWith('https://')) {
-        json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: explicit public https url required in setWebhook.url' } });
-        return;
-      }
-      try {
-        const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({
-            url: rawUrl,
-            secret_token: expected,
-            allowed_updates: ['callback_query'],
-          }),
-          signal: AbortSignal.timeout(10000),
-        });
-        const hookData = await hookRes.json().catch(() => ({}));
-        json(res, 200, { ok: hookRes.ok, url: rawUrl, secret_token_set: true, telegram: hookData });
-      } catch (e) {
-        json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
-      }
-      return;
-    }
-
-    const callbackQueryId = typeof body?.callback_query?.id === 'string'
-      ? body.callback_query.id
-      : (typeof body?.callback_query?.id === 'number' ? String(body.callback_query.id) : '');
-    const botToken = typeof body?.botToken === 'string' ? body.botToken : '';
-
-    const respond = async (statusCode, payload, answerText) => {
-      if (callbackQueryId) {
-        payload.method = 'answerCallbackQuery';
-        payload.callback_query_id = callbackQueryId;
-        payload.text = answerText || payload.error?.message || 'OK';
-        if (botToken) {
-          await bestEffort('telegram-answerCallbackQuery', () => fetch(`https://api.telegram.org/bot${botToken}/answerCallbackQuery`, {
-            method: 'POST',
-            headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ callback_query_id: callbackQueryId, text: payload.text }),
-            signal: AbortSignal.timeout(5000),
-          }), ctx?.logger);
-        }
-      }
-      json(res, statusCode, payload);
-    };
-
-    // Accept callback payloads from formatInteractive (Telegram/Discord/Slack) or plain {action}
-    let action = typeof body?.action === 'string' ? body.action : '';
-    if (!action && typeof body?.data === 'string') {
-      try { action = String(JSON.parse(body.data)?.id ?? ''); } catch { action = ''; }
-    }
-    if (!action && typeof body?.callback_data === 'string') {
-      try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
-    }
-    // #222 / #388: Telegram update envelope {update_id, callback_query:{id, data}}
-    if (!action && typeof body?.callback_query?.data === 'string') {
-      try {
-        const parsed = JSON.parse(body.callback_query.data);
-        action = typeof parsed?.id === 'string' ? parsed.id : String(parsed ?? '');
-      } catch {
-        action = body.callback_query.data;
-      }
-    }
-    action = resolveAction(action);
-    if (!action) {
-      await respond(400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }, 'No action in payload');
-      return;
-    }
-    const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
-    try {
-      if (action === 'disable-rotation') {
-        setRotationDisabled(true);
-        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.('[dsh-key-rotation] rotation DISABLED via webhook action');
-        await respond(200, { ok: true, action }, 'Rotation disabled');
-        return;
-      }
-      if (action === 'enable-rotation') {
-        setRotationDisabled(false);
-        await respond(200, { ok: true, action }, 'Rotation enabled');
-        return;
-      }
-      if (action.startsWith('pause-') || action.startsWith('reset-')) {
-        const st = poolState.get(provider);
-        if (!st) {
-          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-          return;
-        }
-        if (action.startsWith('pause-')) {
-          const until = Date.now() + 3600000; // 1h pause
-          let found = false;
-          for (const [key, s] of poolState.entries()) {
-            if (key === provider || key.startsWith(provider + '::')) {
-              found = true;
-              for (const ref of (s.failedUntil ? [...s.failedUntil.keys()] : [])) {
-                s.failedUntil.set(ref, Math.max(s.failedUntil.get(ref) ?? 0, until));
-              }
-            }
-          }
-          for (const p of buildRuntime().poolByRef.values()) {
-            if (p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) {
-              found = true;
-              for (const ref of p.refs) p.state?.failedUntil?.set(ref, Math.max(p.state?.failedUntil?.get(ref) ?? 0, until));
-            }
-          }
-          if (!found) {
-            await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-            return;
-          }
-          (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
-          await respond(200, { ok: true, action, provider, until }, `Pool ${provider} paused 1h`);
-          return;
-        }
-        let found = false;
-        let cleared = 0;
-        for (const [key, s] of poolState.entries()) {
-          if (key === provider || key.startsWith(provider + '::')) {
-            found = true;
-            cleared += s.failedUntil ? s.failedUntil.size : 0;
-            s.failedUntil?.clear();
-            s.failCounts?.clear();
-            s.authFailCounts?.clear();
-            s.brokenUntil?.clear();
-            s.revokedRefs?.clear();
-            s.tokenUsage?.clear();
-            s.switches = 0;
-            s.lastReason = undefined;
-            s.lastSwitchAt = undefined;
-          }
-        }
-        if (!found) {
-          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
-          return;
-        }
-        let circuitReset = false;
-        const br = circuitBreaker ?? buildRuntime().breaker;
-        if (br) {
-          if (typeof br.reset === 'function') { br.reset(provider); circuitReset = true; }
-          else if (typeof br.onSuccess === 'function') { br.onSuccess(provider); circuitReset = true; }
-        }
-        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} RESET via webhook action (circuitReset=${circuitReset})`);
-        await respond(200, { ok: true, action, provider, cleared, circuitReset }, `Pool ${provider} reset`);
-        return;
-      }
-      await respond(400, { error: { code: 'unknown-action', message: `dsh-key-rotation: unknown action '${action}'` } }, `Unknown action: ${action}`);
-    } catch (e) {
-      await respond(500, { error: { code: 'action-failed', message: String(e?.message ?? e) } }, 'Action failed');
-    }
-  },
-}), 'dsh-key-rotation: webhook-action');
-}
+// lib/ops-webhook.js — Re-export facade for lib/routes/ (#522).
+export * from "./routes/webhook.js";

package/lib/pool-state.js

@@ -1,5 +1,7 @@
 // Shared runtime shape for fresh and persisted pools. Hydrate in place so cached
 // runtime pools, persistence and timers continue to observe the same object.
+import { KeyStateMachine } from './key-state-machine.js';
+
 const MAP_FIELDS = [
   'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
   'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
@@ -18,6 +20,7 @@ export function initializePoolState(state = {}) {
   if (!Array.isArray(state.events)) state.events = [];
   if (!Number.isSafeInteger(state.pointer) || state.pointer < 0) state.pointer = 0;
   if (typeof state.lastUsed !== 'string') state.lastUsed = null;
+  if (!state.keyStateMachine) state.keyStateMachine = new KeyStateMachine();
   return state;
 }
 

package/lib/pool.js

@@ -195,6 +195,7 @@ export function pickNext(pool, now, refsCount = pool.refs.length) {
 /** Apply a failed-key cooldown to pool state. Pure: returns next state shape. */
 export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
   unstickFailedKey(pool, ref);
+  pool?.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs }, now);
   return {
     ...pool,
     state: {
@@ -231,10 +232,9 @@ export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jit
     pool.state.failCounts.set(ref, next);
   }
   let backoff = computeBackoff(baseMs, next || 1, maxMs, isSoft);
-  if (jitter) {
-    backoff = applyJitter(backoff);
-  }
+  if (jitter) backoff = applyJitter(backoff);
   pool.state.failedUntil.set(ref, now + backoff);
+  pool.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs: backoff, reason: isSoft ? 'SOFT_FAILURE' : 'FAILURE' }, now);
   return backoff;
 }
 
@@ -245,6 +245,7 @@ export function recordSuccess(pool, ref, now = Date.now()) {
   pool.state.quotaWindows?.delete(ref);
   if (!pool.state.lastSuccessAt) pool.state.lastSuccessAt = new Map();
   pool.state.lastSuccessAt.set(ref, now);
+  pool.state?.keyStateMachine?.activate(ref, { reason: 'SUCCESS' }, now);
 }
 
 /** Decay penalty failCounts for stable keys that haven't failed in decayIntervalMs. */
@@ -376,12 +377,9 @@ export function parseDurationToSeconds(v) {
 /** Check if a key reference is in maintenance/paused mode. */
 export function isKeyPaused(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
-  if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
-  if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
-  if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
-  if (pool.basePool) return isKeyPaused(pool.basePool, ref);
-  return false;
+  if (pool.pausedRefs?.has?.(ref) || pool.pausedRefs?.includes?.(ref)) return true;
+  if (pool.state?.pausedRefs?.has?.(ref) || pool.state?.pausedRefs?.includes?.(ref)) return true;
+  return pool.basePool ? isKeyPaused(pool.basePool, ref) : false;
 }
 
 /** Extract rate-limit info from an object that may carry response headers. */
@@ -578,12 +576,9 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
 /** Check if a key reference is permanently revoked due to auth/401 failure (#369). */
 export function isKeyRevoked(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.revokedRefs instanceof Set && pool.revokedRefs.has(ref)) return true;
-  if (Array.isArray(pool.revokedRefs) && pool.revokedRefs.includes(ref)) return true;
-  if (pool.state && pool.state.revokedRefs instanceof Set && pool.state.revokedRefs.has(ref)) return true;
-  if (pool.state && Array.isArray(pool.state.revokedRefs) && pool.state.revokedRefs.includes(ref)) return true;
-  if (pool.basePool) return isKeyRevoked(pool.basePool, ref);
-  return false;
+  if (pool.revokedRefs?.has?.(ref) || pool.revokedRefs?.includes?.(ref)) return true;
+  if (pool.state?.revokedRefs?.has?.(ref) || pool.state?.revokedRefs?.includes?.(ref)) return true;
+  return pool.basePool ? isKeyRevoked(pool.basePool, ref) : false;
 }
 
 /** Check if a key reference is in active recoverable or permanent quarantine (#427). */

package/lib/routes/balance.js

@@ -0,0 +1,147 @@
+// lib/ops-balance.js — operational HTTP routes for balance inspection (#495).
+
+import {
+  json,
+  readJson,
+} from '../http-bridge.js';
+import {
+  isTrustedBridgeRequest,
+  isValidRef,
+} from '../pool.js';
+import { BALANCE_PATH } from './paths.js';
+import { inspectBalanceForRef } from '../balance-inspector.js';
+import { pushEvent } from '../notify-events.js';
+
+export function registerBalanceRoutes(ctx, deps) {
+  const {
+    poolState,
+    buildRuntime,
+  } = deps;
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: BALANCE_PATH,
+    handler: async (req, res) => {
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { ok: false, error: 'Forbidden' });
+        return;
+      }
+
+      const urlObj = new URL(req.url, 'http://127.0.0.1');
+
+      // GET: Query single key or provider cached balances
+      if (req.method === 'GET') {
+        const ref = urlObj.searchParams.get('ref');
+        const provider = urlObj.searchParams.get('provider');
+
+        if (ref) {
+          const runtime = buildRuntime();
+          const targetPool = provider ? runtime.pools?.find((p) => p.provider === provider) : null;
+          const pool = targetPool ?? runtime.poolByRef?.get(ref);
+          const cached = pool?.state?.balances?.get(ref) ?? null;
+
+          if (urlObj.searchParams.get('refresh') === 'true' && provider) {
+            const inspectRes = await inspectBalanceForRef(ctx, ref, provider);
+            if (inspectRes.ok) {
+              if (pool && pool.state) {
+                if (!pool.state.balances) pool.state.balances = new Map();
+                pool.state.balances.set(ref, { ...inspectRes, at: Date.now() });
+
+                // Check soft pause
+                applyBalanceState(pool, ref, inspectRes);
+              }
+              json(res, 200, { ok: true, ref, balance: inspectRes });
+              return;
+            }
+            json(res, 200, { ok: false, ref, error: inspectRes.error, cached });
+            return;
+          }
+
+          json(res, 200, { ok: true, ref, balance: cached });
+          return;
+        }
+
+        // Return all cached balances
+        const out = {};
+        if (poolState && typeof poolState.forEach === 'function') {
+          poolState.forEach((st, provName) => {
+            if (st && st.balances instanceof Map) {
+              for (const [k, v] of st.balances) {
+                out[k] = v;
+              }
+            }
+          });
+        }
+        json(res, 200, { ok: true, balances: out });
+        return;
+      }
+
+      // POST: Trigger refresh of balances for a provider or key list
+      if (req.method === 'POST') {
+        const body = await readJson(req);
+        const provider = body?.provider;
+        const ref = body?.ref;
+        const testValue = body?.value;
+
+        if (ref && provider) {
+          const inspectRes = await inspectBalanceForRef(ctx, ref, provider, { testValue });
+          const runtime = buildRuntime();
+          const pool = runtime.pools?.find((p) => p.provider === provider) ?? runtime.poolByRef?.get(ref);
+
+          if (inspectRes.ok && pool && pool.state) {
+            if (!pool.state.balances) pool.state.balances = new Map();
+            pool.state.balances.set(ref, { ...inspectRes, at: Date.now() });
+            applyBalanceState(pool, ref, inspectRes);
+          }
+
+          json(res, inspectRes.ok ? 200 : 400, { ok: inspectRes.ok, ref, balance: inspectRes });
+          return;
+        }
+
+        if (provider) {
+          const runtime = buildRuntime();
+          const pool = runtime.pools?.find((p) => p.provider === provider);
+          if (!pool || !Array.isArray(pool.refs) || pool.refs.length === 0) {
+            json(res, 404, { ok: false, error: `Provider '${provider}' not found or has no keys` });
+            return;
+          }
+
+          const results = {};
+          for (const kRef of pool.refs) {
+            const r = await inspectBalanceForRef(ctx, kRef, provider);
+            results[kRef] = r;
+            if (r.ok && pool.state) {
+              if (!pool.state.balances) pool.state.balances = new Map();
+              pool.state.balances.set(kRef, { ...r, at: Date.now() });
+              applyBalanceState(pool, kRef, r);
+            }
+          }
+          json(res, 200, { ok: true, provider, balances: results });
+          return;
+        }
+
+        json(res, 400, { ok: false, error: 'ref or provider required' });
+        return;
+      }
+
+      json(res, 405, { ok: false, error: 'Method Not Allowed' });
+    },
+  }), 'dsh-key-rotation: balance routes');
+}
+
+function applyBalanceState(pool, ref, balanceRes) {
+  if (!pool || !pool.state) return;
+  const isZero = balanceRes.isAvailable === false || (typeof balanceRes.total === 'number' && balanceRes.total <= 0);
+
+  if (isZero) {
+    if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(ref);
+    if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
+    pool.state.pauseReasons.set(ref, 'EMPTY_BALANCE_PAUSED');
+    pushEvent(pool, ref, 'BALANCE_EMPTY', 0);
+  } else if (pool.state.pauseReasons?.get?.(ref) === 'EMPTY_BALANCE_PAUSED') {
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.delete(ref);
+    pool.state.pauseReasons.delete(ref);
+    pushEvent(pool, ref, 'BALANCE_RESTORED', 0);
+  }
+}