@goodandready/dsh-key-rotation 0.7.20 → 0.7.21

package/lib/heal.js

@@ -0,0 +1,35 @@
+// heal.js — self-healing idle cooldowns.
+// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
+
+// Returns array of { ref, poolBase } entries that were healed in this tick.
+// Mutates `pools` (removes from failedUntil, pushes heal event into events).
+// `now` parameter is injectable for tests.
+export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
+  if (!Array.isArray(pools) || pools.length === 0) return [];
+  if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
+  const healed = [];
+  for (const pool of pools) {
+    if (!pool || !pool.state || !pool.base) continue;
+    const fu = pool.state.failedUntil;
+    const lu = pool.state.lastUsed;
+    if (!fu || fu.size === 0) continue;
+    const expiredRefs = [];
+    for (const [ref, until] of fu.entries()) {
+      if (!Number.isFinite(until)) continue;
+      if (until > now) continue; // cooldown still active
+      const last = lu ? lu.get(ref) : undefined;
+      if (!Number.isFinite(last)) continue; // never used → no signal, skip
+      if (now - last < idleMs) continue; // used recently → don't heal
+      expiredRefs.push(ref);
+    }
+    for (const ref of expiredRefs) {
+      fu.delete(ref);
+      if (Array.isArray(pool.state.events)) {
+        pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
+        if (pool.state.events.length > 50) pool.state.events.shift();
+      }
+      healed.push({ ref, poolBase: pool.base });
+    }
+  }
+  return healed;
+}

package/lib/histogram.js

@@ -0,0 +1,66 @@
+// histogram.js — per-ref latency ring buffer + percentile.
+// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
+
+export const LATENCY_DEFAULT_WINDOW = 200;
+
+export class LatencyHistogram {
+  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
+    const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
+    this._window = w;
+    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
+    this._lastAt = new Map();  // ref -> epochMs of last sample
+  }
+
+  record(ref, ms) {
+    if (!ref || !Number.isFinite(ms) || ms < 0) return;
+    let entry = this._buffers.get(ref);
+    if (!entry) {
+      entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
+      this._buffers.set(ref, entry);
+    }
+    entry.buf[entry.head] = ms;
+    entry.head = (entry.head + 1) % this._window;
+    if (entry.count < this._window) entry.count += 1;
+    this._lastAt.set(ref, Date.now());
+  }
+
+  // Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
+  snapshot(ref) {
+    const entry = this._buffers.get(ref);
+    const lastAt = this._lastAt.get(ref);
+    if (!entry || entry.count === 0) {
+      return { count: 0, lastAt: lastAt || null };
+    }
+    const arr = entry.buf.subarray(0, entry.count);
+    const sorted = Array.from(arr).sort((a, b) => a - b);
+    const n = sorted.length;
+    return {
+      count: n,
+      lastAt: lastAt || null,
+      p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
+      p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
+      p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
+    };
+  }
+
+  // Returns { [ref]: snapshot }
+  snapshotAll() {
+    const out = {};
+    for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) {
+      this._buffers.delete(ref);
+      this._lastAt.delete(ref);
+    } else {
+      this._buffers.clear();
+      this._lastAt.clear();
+    }
+  }
+
+  get size() {
+    return this._buffers.size;
+  }
+}

package/lib/index.js

@@ -49,6 +49,10 @@ const RESET_PATH = '/dsh-key-rotation/reset';
 const IMPORT_PATH = '/dsh-key-rotation/import';
 const HEALTH_PATH = '/dsh-key-rotation/health';
 const TEST_PATH = '/dsh-key-rotation/test';
+const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
+import { LastTestCache, SandboxRunner } from './sandbox.js';
+import { healIdleCooldowns } from './heal.js';
+import { LatencyHistogram } from './histogram.js';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
@@ -70,7 +74,40 @@ function pushEvent(pool, ref, reason, cooldownMs, type) {
 // treat pre-content failures whose message matches these patterns as
 // switchable even when the code is not in `switchCodes`.
 
-// Bootstrap key pools. The user configures them in the Settings GUI or via
+// Sandbox-test infrastructure (sandbox.js): in-memory cache + runner.
+let lastTestCacheRunnerCtx = null;
+const lastTestCache = new LastTestCache();
+const latencyHistogram = new LatencyHistogram();
+let sandboxRunner = null;
+function ensureSandboxRunner(ctx) {
+  if (sandboxRunner) return sandboxRunner;
+  // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
+  function resolveBaseUrl(provider) {
+    try {
+      const ns = ctx.get(PIAI_NS);
+      const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
+      if (!Array.isArray(list)) return null;
+      // ponytail: match by id OR name OR alias; pick first hit
+      const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
+      const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
+      return base ? String(base) : null;
+    } catch (e) {
+      return null;
+    }
+  }
+  sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
+  return sandboxRunner;
+}
+async function probeRef(ref, key) {
+  // ref may be like "PROVIDER/KEY_NAME" — for sandbox we only care about the credential ref
+  // (the resolveBaseUrl uses the full provider id; ref can carry any string)
+  const runner = ensureSandboxRunner(lastTestCacheRunnerCtx);
+  const result = await runner.probeModels(ref, key);
+  lastTestCache.set(ref, { ...result, at: Date.now() });
+  return result;
+}
+
+// // Bootstrap key pools. The user configures them in the Settings GUI or via
 // the dsh profile bundle config; the plugin itself ships no provider defaults
 // so it does not bind to any specific installation. Empty array means: until
 // the user adds a pool, no rotation happens, and every provider falls back to
@@ -87,6 +124,10 @@ export const Config = Schema.object({
   backupIntervalMs: Schema.number().default(86400000),
   backupKeep: Schema.number().default(7),
   rotationScheduleDays: Schema.number().default(0),
+  selfHealCooldown: Schema.boolean().default(true),
+  selfHealIdleMs: Schema.number().default(3600000),
+  latencyEnabled: Schema.boolean().default(true),
+  latencyWindow: Schema.number().default(200),
   rateLimitThreshold: Schema.number().default(0.1),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
@@ -255,6 +296,30 @@ export function apply(ctx, config = {}) {
   // profile does not need a second copy of that package.)
   let getConfig = () => config;
   registerConfigBridge(ctx, () => buildRuntime().cloneIds);
+  lastTestCacheRunnerCtx = ctx;
+  // Cache should not survive profile restarts (apply is called per reload).
+  // We deliberately do NOT clear on every apply — that would wipe badges when
+  // the user is just typing in the settings card. Re-init only on true reload.
+  ensureSandboxRunner(ctx);
+
+  // Self-healing idle cooldowns: every 60s, lift expired cooldowns for keys
+  // that have been idle for selfHealIdleMs (default 1h). ponytail: small
+  // interval, low cost; skipped when selfHealCooldown is disabled in config.
+  // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
+  // fresh config on every tick. Naive but correct: 60s cadence is cheap.
+  const selfHealTimer = setInterval(() => {
+    const cfg = getConfig();
+    if (!cfg || cfg.selfHealCooldown === false) return;
+    try {
+      const idle = Number.isFinite(cfg.selfHealIdleMs) && cfg.selfHealIdleMs > 0 ? cfg.selfHealIdleMs : 3600000;
+      const providers = Array.isArray(cfg.providers) ? cfg.providers : [];
+      const pools = providers
+        .map((p) => buildRuntime().providerToPool.get(p.provider))
+        .filter(Boolean);
+      healIdleCooldowns(pools, idle);
+    } catch (_) { /* ponytail: never crash the timer */ }
+  }, 60000);
+  if (typeof selfHealTimer.unref === 'function') selfHealTimer.unref();
 
   // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
   const DASH_HTML = '';
@@ -928,7 +993,7 @@ export function apply(ctx, config = {}) {
         if (exhausted) exhaustedAny = true;
         pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
       }
-      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny });
+      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll() });
     },
   }), 'dsh-key-rotation: health');
 
@@ -944,11 +1009,11 @@ export function apply(ctx, config = {}) {
       if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
       // Optional value for pre-save validation (issue #118)
       const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
+      const probe = body?.probe === 'models' || body?.probe === 'chat' ? body.probe : undefined;
       const base = ctx.get('credentials');
       try {
         let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
         let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-        // Pre-save validation: check the provided value directly (issue #118)
         const effectiveValue = testValue || hit?.value;
         const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
         const tail = valid ? keyTail(effectiveValue) : '';
@@ -960,6 +1025,16 @@ export function apply(ctx, config = {}) {
           const ev = envValue(ref);
           if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
         }
+        // sandbox probe (models is free; chat is hook-only, see sandbox.js)
+        if (probe) {
+          const keyForProbe = effectiveValue;
+          const runner = ensureSandboxRunner(ctx);
+          const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
+          const cached = { ...result, at: Date.now() };
+          lastTestCache.set(ref, cached);
+          json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
+          return;
+        }
         json(res, 200, { ok: true, ref, tail, source });
       } catch (e) {
         json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
@@ -970,6 +1045,16 @@ export function apply(ctx, config = {}) {
   // Intercept the llm/stream waterfall: rotate any request whose provider maps
   // to a configured key pool; pass everything else (and internal dispatches)
   // straight through.
+  // Read-only cache snapshot for clients (badge polling).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SANDBOX_CACHE_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: cache is local-only' } }); return; }
+      json(res, 200, lastTestCache.snapshot());
+    },
+  }), 'dsh-key-rotation: sandbox cache');
+
   ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
     const { providerToPool, modelPoolByProvider } = buildRuntime();

package/lib/sandbox.js

@@ -0,0 +1,117 @@
+// sandbox.js — probe sandbox-test runner + last-test cache.
+// Ponytail-mode (full): simplest correct path.
+// YAGNI: chat completions is a hook (not-implemented).
+// In-memory only; restart dsh-web = clear cache.
+
+export const PROBE_MODELS_TIMEOUT_MS = 5000;
+export const PROBE_RETRY_DELAY_MS = 1000;
+export const LAST_TEST_MAX = 200;
+
+export class LastTestCache {
+  constructor(max = LAST_TEST_MAX) {
+    this._max = max;
+    this._data = new Map();
+  }
+
+  set(ref, result) {
+    if (!ref || !result) return;
+    if (this._data.has(ref)) this._data.delete(ref);
+    this._data.set(ref, result);
+    while (this._data.size > this._max) {
+      const first = this._data.keys().next().value;
+      if (first === undefined) break;
+      this._data.delete(first);
+    }
+  }
+
+  get(ref) {
+    return this._data.get(ref);
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._data) out[k] = v;
+    return out;
+  }
+
+  clear() {
+    this._data.clear();
+  }
+
+  get size() {
+    return this._data.size;
+  }
+}
+
+function classifyStatus(status) {
+  if (status === 401 || status === 403) return 'auth';
+  if (status === 404) return 'not-found';
+  if (status === 429) return 'rate-limit';
+  if (status >= 500 && status < 600) return 'server';
+  return `http-${status}`;
+}
+
+export class SandboxRunner {
+  constructor({ fetchImpl, resolveBaseUrl, log = () => {} } = {}) {
+    if (typeof fetchImpl !== 'function') throw new Error('sandbox: fetchImpl required');
+    if (typeof resolveBaseUrl !== 'function') throw new Error('sandbox: resolveBaseUrl required');
+    this._fetch = fetchImpl;
+    this._resolveBaseUrl = resolveBaseUrl;
+    this._log = log;
+  }
+
+  async probeModels(ref, key) {
+    if (!ref || typeof key !== 'string' || key.length === 0) {
+      return { ok: false, code: 'no-credential', latencyMs: 0 };
+    }
+    const baseUrl = await this._resolveBaseUrl(ref);
+    if (!baseUrl) {
+      return { ok: false, code: 'no-baseurl', latencyMs: 0 };
+    }
+    const url = `${baseUrl.replace(/\/+$/, '')}/models`;
+    const started = Date.now();
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), PROBE_MODELS_TIMEOUT_MS);
+    const doFetch = () => this._fetch(url, {
+      method: 'GET',
+      headers: { authorization: `Bearer ${key}`, accept: 'application/json' },
+      signal: ctrl.signal,
+    });
+    try {
+      let res;
+      try {
+        res = await doFetch();
+      } catch (e) {
+        if (e && e.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+        return { ok: false, code: 'network', latencyMs: Date.now() - started };
+      }
+      // ponytail: 1 retry on 5xx — naive; classifier refines if needed
+      if (res.status >= 500 && res.status < 600) {
+        await new Promise((r) => setTimeout(r, PROBE_RETRY_DELAY_MS));
+        if (ctrl.signal.aborted) return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+        try {
+          res = await doFetch();
+        } catch (e) {
+          if (e && e.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+          return { ok: false, code: 'network', latencyMs: Date.now() - started };
+        }
+      }
+      const status = res.status;
+      if (status >= 200 && status < 300) {
+        let modelsCount = 0;
+        try {
+          const body = await res.json();
+          modelsCount = Array.isArray(body && body.data) ? body.data.length : 0;
+        } catch (_) { /* not json */ }
+        return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount };
+      }
+      return { ok: false, code: classifyStatus(status), latencyMs: Date.now() - started };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async probeChat(_ref, _key) {
+    return { ok: false, code: 'not-implemented', latencyMs: 0 };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.20",
+  "version": "0.7.21",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",
@@ -53,6 +53,6 @@
     "@deepseek-ai/dsh-llm": "^0.1.0-rc.6"
   },
   "scripts": {
-    "test": "node --test test/*.test.js"
+    "test": "node --test test/*.test.js test/*.test.mjs"
   }
 }