@goodandready/dsh-key-rotation 0.8.12 → 0.8.13

package/lib/lifecycle.js

@@ -0,0 +1,122 @@
+// lib/lifecycle.js — background timers and lifecycle effects
+import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
+import { StatePersistence, resolveStatePath } from './persistence.js';
+import path from 'node:path';
+
+export function setupIdleHealEffect(ctx, getConfig, buildRuntime) {
+  return ctx.effect(() => {
+    const cfg = getConfig();
+    if (!cfg || cfg.selfHealCooldown === false) return () => {};
+    const timer = setInterval(() => {
+      try {
+        const c = getConfig();
+        if (!c || c.selfHealCooldown === false) return;
+        const idle = Number.isFinite(c.selfHealIdleMs) && c.selfHealIdleMs > 0 ? c.selfHealIdleMs : 3600000;
+        const providers = Array.isArray(c.providers) ? c.providers : [];
+        const pools = providers
+          .map((p) => buildRuntime().providerToPool.get(p.provider))
+          .filter(Boolean);
+        healIdleCooldowns(pools, idle);
+      } catch (_) { /* ponytail: never crash the timer */ }
+    }, 60000);
+    if (typeof timer.unref === 'function') timer.unref();
+    return () => clearInterval(timer);
+  }, 'dsh-key-rotation: self-healing idle');
+}
+
+export function setupAutoUnbreakEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger) {
+  return ctx.effect(() => {
+    const cfg = getConfig();
+    const intervalMin = cfg?.selfHealingIntervalMinutes ?? 30;
+    if (!intervalMin || intervalMin <= 0) return () => {};
+    const intervalMs = intervalMin * 60 * 1000;
+    const timer = setInterval(async () => {
+      try {
+        const c = getConfig();
+        if (!c || !c.selfHealingIntervalMinutes || c.selfHealingIntervalMinutes <= 0) return;
+        const { pools } = buildRuntime();
+        const runner = ensureSandboxRunner(ctx);
+        await autoUnbreakBrokenKeys(pools, async (ref) => {
+          let val = (await ctx.credentials?.resolve?.(ref))?.value;
+          if (!val) return { ok: false };
+          return runner.probeModels(ref, val);
+        });
+      } catch (e) { logger?.warn?.('[dsh-key-rotation] auto-unbreak failed', e); }
+    }, intervalMs);
+    if (typeof timer.unref === 'function') timer.unref();
+    return () => clearInterval(timer);
+  }, 'dsh-key-rotation: auto-unbreak');
+}
+
+export function setupPersistence(ctx, { cfg0, poolState, moduleBreaker, verboseLoggingOn, logger }) {
+  let statePersistence = null;
+  try {
+    const hostDirs = [
+      process.env.DSH_HOME,
+      process.cwd(),
+    ].filter((d) => typeof d === 'string' && d.length > 0);
+    const resolvedPath = resolveStatePath({
+      configuredPath: cfg0.persistencePath,
+      dataDir: hostDirs[0],
+    });
+    if (cfg0.persistenceEnabled !== false && resolvedPath) {
+      statePersistence = new StatePersistence({ filePath: resolvedPath });
+      statePersistence.load().then((snap) => {
+        if (!snap) return;
+        try {
+          StatePersistence.restorePools(poolState, snap);
+          if (moduleBreaker && snap.circuit) moduleBreaker.restore(snap.circuit);
+          if (verboseLoggingOn?.()) {
+            logger?.warn?.(`[dsh-key-rotation] restored ${Object.keys(snap.pools ?? {}).length} pool state(s) from ${path.basename(resolvedPath)}`);
+          }
+        } catch (e) {
+          logger?.warn?.('[dsh-key-rotation] persistence restore failed', e?.message ?? e);
+        }
+      }).catch(() => {});
+    } else if (cfg0.persistenceEnabled !== false && !resolvedPath) {
+      logger?.warn?.('[dsh-key-rotation] persistence disabled: no data directory');
+    }
+  } catch (e) {
+    logger?.warn?.('[dsh-key-rotation] persistence init failed', e?.message ?? e);
+  }
+
+  function persistenceSnapshot() {
+    if (!statePersistence) return null;
+    return StatePersistence.serialize({
+      poolState,
+      circuitSnapshot: moduleBreaker ? moduleBreaker.snapshot() : {},
+      quotaSnapshot: {},
+    });
+  }
+
+  function schedulePersist() {
+    if (!statePersistence) return;
+    const snap = persistenceSnapshot();
+    if (snap) statePersistence.save(snap);
+  }
+
+  ctx.effect(() => {
+    const timer = setInterval(() => {
+      try { schedulePersist(); }
+      catch (e) { logger?.warn?.('[dsh-key-rotation] periodic persist failed', e); }
+    }, 15000);
+    if (typeof timer.unref === 'function') timer.unref();
+    return () => {
+      clearInterval(timer);
+      try {
+        if (statePersistence) {
+          const snap = persistenceSnapshot();
+          if (snap) {
+            statePersistence.save(snap);
+            void statePersistence.flush();
+          }
+          statePersistence.dispose();
+        }
+      } catch (e) {
+        logger?.warn?.('[dsh-key-rotation] dispose persist failed', e);
+      }
+    };
+  }, 'dsh-key-rotation: state persistence');
+
+  return { schedulePersist, persistenceSnapshot };
+}

package/lib/logger.js

@@ -0,0 +1,13 @@
+// lib/logger.js — safe Cordis logger wrapper
+const noop = () => {};
+const noopLogger = { warn: noop, info: noop, error: noop, debug: noop, log: noop };
+
+export function getLogger(ctx, scope = 'dsh-key-rotation') {
+  if (typeof ctx?.logger === 'function') {
+    return ctx.logger(scope);
+  }
+  if (ctx?.logger && typeof ctx.logger.warn === 'function') {
+    return ctx.logger;
+  }
+  return noopLogger;
+}

package/lib/ops-status.js

@@ -143,7 +143,7 @@ ctx.effect(() => ctx.webServer.register({
         proactiveRateLimitGuard: pool.proactiveRateLimitGuard ?? runtime.proactiveRateLimitGuard ?? true,
       });
       } catch (e) {
-        console.warn(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
+        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
         providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
       }
     }

package/lib/ops-webhook.js

@@ -64,7 +64,7 @@ ctx.effect(() => ctx.webServer.register({
     try {
       if (action === 'disable-rotation') {
         setRotationDisabled(true);
-        console.warn('[dsh-key-rotation] rotation DISABLED via webhook action');
+        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.('[dsh-key-rotation] rotation DISABLED via webhook action');
         json(res, 200, { ok: true, action });
         return;
       }
@@ -84,7 +84,7 @@ ctx.effect(() => ctx.webServer.register({
             if (p.base !== provider) continue;
             for (const ref of p.refs) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
           }
-          console.warn(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
+          (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
           json(res, 200, { ok: true, action, provider, until: Date.now() + 3600000 });
           return;
         }
@@ -102,7 +102,7 @@ ctx.effect(() => ctx.webServer.register({
           if (typeof br.reset === 'function') { br.reset(provider); circuitReset = true; }
           else if (typeof br.onSuccess === 'function') { br.onSuccess(provider); circuitReset = true; }
         }
-        console.warn(`[dsh-key-rotation] pool ${provider} RESET via webhook action (circuitReset=${circuitReset})`);
+        (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} RESET via webhook action (circuitReset=${circuitReset})`);
         json(res, 200, { ok: true, action, provider, cleared, circuitReset });
         return;
       }

package/lib/pool-builder.js

@@ -0,0 +1,86 @@
+// lib/pool-builder.js — provider & per-model pool assembly and cleanup
+import { bucketSweep } from './bucket.js';
+
+export function parseExpiry(v) {
+  if (typeof v === 'number' && Number.isFinite(v) && v > 0) return v;
+  if (typeof v === 'string' && v.length > 0) {
+    const t = Date.parse(v);
+    if (!Number.isNaN(t)) return t;
+  }
+  return undefined;
+}
+
+export function buildPoolItem({
+  base,
+  keys,
+  weights,
+  poolCooldown,
+  poolMax,
+  expiresAt,
+  poolStrategy,
+  poolGuard,
+  rpmLimit,
+  makeState,
+}) {
+  const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
+  if (refs.length === 0) return null;
+  const w = Array.isArray(weights) ? weights : [];
+  const weightedRefs = [];
+  for (let i = 0; i < refs.length; i++) {
+    const ww = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
+    for (let k = 0; k < ww; k++) weightedRefs.push(refs[i]);
+  }
+  const parsedExpiry = {};
+  if (Array.isArray(expiresAt)) {
+    for (let i = 0; i < refs.length; i++) {
+      const exp = parseExpiry(expiresAt[i]);
+      if (exp !== undefined) parsedExpiry[refs[i]] = exp;
+    }
+  }
+  return {
+    base,
+    refs,
+    weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
+    weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
+    state: makeState(base),
+    cooldownMs: poolCooldown,
+    maxCooldownMs: poolMax,
+    expiresAt: parsedExpiry,
+    rpmLimit,
+    routingStrategy: poolStrategy,
+    proactiveRateLimitGuard: poolGuard,
+  };
+}
+
+export function cleanupRemovedProviders({
+  cfg,
+  poolState,
+  poolByRef,
+  providerToPool,
+  expectedClones,
+  moduleBreaker,
+  lowHealthNotifiedAt,
+  budgetNotifiedAt,
+}) {
+  // auto-cleanup: remove poolState for providers that are now empty or removed
+  for (const key of [...poolState.keys()]) {
+    if (![...poolByRef.values()].some((p) => p.base === key)) {
+      poolState.delete(key);
+      lowHealthNotifiedAt?.delete?.(key);
+      budgetNotifiedAt?.delete?.(key + ':budget');
+    }
+  }
+  // #192: drop RPM windows for refs that no longer belong to any pool
+  for (const st of poolState.values()) {
+    if (st.rpmWindows) bucketSweep(st.rpmWindows, new Set(poolByRef.keys()));
+  }
+  // drop breaker entries for removed providers
+  if (moduleBreaker) {
+    for (const key of Object.keys(moduleBreaker.snapshot())) {
+      if (![...providerToPool.keys()].includes(key) && !expectedClones.has(key)) {
+        const still = (cfg.providers ?? []).some((p) => p.provider === key);
+        if (!still) moduleBreaker.reset(key);
+      }
+    }
+  }
+}

package/lib/rotate.js

@@ -33,9 +33,12 @@ export function createRotate(deps) {
     setRotateStartMs,
     quotaStore,
     circuitBreaker,
+    logger,
     now = nowMono,
   } = deps;
 
+  const logWarn = (msg) => (logger?.warn ? logger.warn(msg) : null);
+
   function rotate(options, pool) {
     return (async function* () {
       const runtime0 = buildRuntime();
@@ -93,7 +96,7 @@ export function createRotate(deps) {
 
       // #260: fail fast when provider circuit is open
       if (circuitBreaker && !circuitBreaker.canRequest(options.provider)) {
-        console.warn(`[dsh-key-rotation] ${options.provider}: circuit open — skipping dispatch`);
+        logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — skipping dispatch`);
         yield finishError('CIRCUIT_OPEN', `[dsh-key-rotation] provider '${options.provider}' circuit is open`);
         return;
       }
@@ -116,7 +119,7 @@ export function createRotate(deps) {
           penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
           lastFailure = finishError(e?.code ?? 'TRANSPORT',
             `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
-          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
+          logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
           continue;
         }
 
@@ -146,7 +149,7 @@ export function createRotate(deps) {
                   pool.state.lastReason = String(code ?? 'UNKNOWN');
                   pool.state.lastSwitchAt = now();
                   lastFailure = chunk;
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
+                  logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
                   // #216: per-switch webhook (opt-in switchNotify), deduped per provider
                   if (switchNotify && activeRef) {
                     notifySwitch(runtime0, pool, {
@@ -206,7 +209,7 @@ export function createRotate(deps) {
                     const effCool = Math.min(cool, maxCool ?? cool);
                     recordFailure(pool, activeRef, now(), effCool, maxCool);
                     pushEvent(pool, activeRef, 'RATE_LIMIT', effCool);
-                    console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} proactive pause (remaining ${String(rate.remaining ?? '?')}/${String(rate.limit ?? '?')}, cool ${Math.round(effCool / 1000)}s) — next request will rotate`);
+                    logWarn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} proactive pause (remaining ${String(rate.remaining ?? '?')}/${String(rate.limit ?? '?')}, cool ${Math.round(effCool / 1000)}s) — next request will rotate`);
                   }
                 }
                 // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
@@ -229,7 +232,7 @@ export function createRotate(deps) {
               pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
               pool.state.lastSwitchAt = now();
               lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-              console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
+              logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
               if (switchNotify && activeRef) {
                 notifySwitch(runtime0, pool, {
                   provider: options.provider,
@@ -256,7 +259,7 @@ export function createRotate(deps) {
       // pool exhausted — all keys cooling or missing
       pool.state.lastExhaustionAt = now();
       pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
-      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
+      logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
       const runtime = buildRuntime();
       // notify via extracted helper (see notifyExhaustion above)
       notifyExhaustion(runtime, pool, { provider: options.provider });
@@ -266,7 +269,7 @@ export function createRotate(deps) {
         const pools = runtime.providerToPool;
         const fb = pickCascadeFallback(options.provider, runtime, pools);
         if (fb && fb.pool && fb.pool !== pool) {
-          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
+          logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
           pool.state.lastReason = 'CASCADE';
           pool.state.lastSwitchAt = now();
           // Re-dispatch on the fallback pool (depth-1 via __isCascade guard)

package/lib/sandbox-service.js

@@ -0,0 +1,55 @@
+// lib/sandbox-service.js — sandbox runner and cache service
+import { LastTestCache, SandboxRunner } from './sandbox.js';
+
+export function createSandboxService({ getRuntime }) {
+  let sandboxRunner = null;
+  let defaultCtx = null;
+  const lastTestCache = new LastTestCache();
+
+  function ensureSandboxRunner(ctx) {
+    if (ctx) defaultCtx = ctx;
+    if (sandboxRunner) return sandboxRunner;
+    function resolveBaseUrl(providerOrRef) {
+      try {
+        let provider = providerOrRef;
+        const rt = typeof getRuntime === 'function' ? getRuntime() : null;
+        const pool = rt?.poolByRef?.get(providerOrRef);
+        if (pool?.base) provider = pool.base;
+        else if (pool?.provider) provider = pool.provider;
+
+        const c = ctx || defaultCtx;
+        const pInfo = c?.llm?.getProvider?.(provider);
+        if (pInfo && (pInfo.baseUrl || pInfo.endpoint || pInfo.url)) {
+          return String(pInfo.baseUrl || pInfo.endpoint || pInfo.url);
+        }
+        for (const info of (c?.llm?.listProviders?.() || [])) {
+          if (info && (info.id === provider || info.name === provider)) {
+            const u = info.baseUrl || info.endpoint || info.url;
+            if (u) return String(u);
+          }
+        }
+        const ns = c?.get ? c.get('llm-pi-ai') : null;
+        const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
+        if (Array.isArray(list)) {
+          const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
+          const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
+          if (base) return String(base);
+        }
+        return null;
+      } catch (_) {
+        return null;
+      }
+    }
+    sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
+    return sandboxRunner;
+  }
+
+  async function probeRef(ref, key) {
+    const runner = ensureSandboxRunner();
+    const result = await runner.probeModels(ref, key);
+    lastTestCache.set(ref, { ...result, at: Date.now() });
+    return result;
+  }
+
+  return { ensureSandboxRunner, probeRef, lastTestCache };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.12",
+  "version": "0.8.13",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
@@ -33,6 +33,9 @@
     "lib",
     "cordis.patch.yml",
     "README.md",
+    "README.zh.md",
+    "README.ru.md",
+    "CHANGELOG.md",
     "LICENSE"
   ],
   "dsh": {