@goodandready/dsh-key-rotation 0.8.79 → 0.8.80

package/CHANGELOG.md

@@ -55,6 +55,13 @@
 - **Fail-Closed Balance & Probe Inspection**: Prevent sending ciphertext credentials (`enc:v1:...`) upstream if decryption fails (#539).
 - **PBKDF2 Iteration Bounds & Async KDF**: Enforce strict iteration limits (`10000..200000`) and async derivation to eliminate event-loop blocking DoS (#542).
 
+## [0.8.80] - 2026-10-30
+
+### Changed
+- **#530 (M) Domain Core Layer (`lib/core/`)**: Structured domain core modules under `lib/core/` (`state-machine.js`, `concurrency.js`, `quota.js`, `pool.js`, `rotate.js`, `index.js`) while maintaining transparent backwards-compatible root facades.
+- **#528 (M) Cordis Dynamic Dependency Injection**: Reduced required static `inject` array to minimal `['llm']`. Wrapped HTTP routes and WebServer lifecycle in `ctx.inject(['webServer'], ...)`, enabling fully headless/CLI operation without crashing when WebServer or settings are unmounted.
+- **#525 (M) Append-Only Event Journal Persistence**: Introduced incremental append-only event journaling (`*-journal.jsonl`) alongside debounced periodic atomic snapshots. Eliminates heavy O(N) full state tree disk rewrites on routine key switches and cooldowns, automatically compacting and rotating upon snapshot flush.
+
 ## [0.8.79] - 2026-10-30
 
 ### Changed

package/lib/concurrency.js

@@ -1,115 +1,2 @@
-// concurrency.js — per-key in-flight counter + least-connections picking (issues #193, #407, #463).
-
-const CONCURRENCY_DEFAULT_LIMIT = 0;
-// Default stale timeout: 30 minutes to support prolonged streaming generation (#463)
-const CONCURRENCY_STALE_LOCK_MS = 30 * 60 * 1000;
-
-export class ConcurrencyTracker {
-  constructor(opts) {
-    opts = opts || {};
-    const limit = opts.limit !== undefined ? opts.limit : CONCURRENCY_DEFAULT_LIMIT;
-    this._limit = (Number.isFinite(limit) && limit >= 0) ? Math.floor(limit) : 0;
-    this._staleMs = opts.staleMs || CONCURRENCY_STALE_LOCK_MS;
-    this._inFlight = new Map();
-  }
-
-  configure(opts) {
-    if (!opts) return;
-    if (opts.limit !== undefined) {
-      this._limit = (Number.isFinite(opts.limit) && opts.limit >= 0) ? Math.floor(opts.limit) : 0;
-    }
-    if (opts.staleMs !== undefined && Number.isFinite(opts.staleMs) && opts.staleMs > 0) {
-      this._staleMs = opts.staleMs;
-    }
-  }
-
-  isEnabled() { return this._limit > 0; }
-  get limit() { return this._limit; }
-
-  acquire(ref, now, limit) {
-    now = now || Date.now();
-    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
-    let e = this._inFlight.get(ref);
-    if (!e) {
-      e = { count: 0, lastAcquired: now };
-      this._inFlight.set(ref, e);
-    }
-    if (now - e.lastAcquired > this._staleMs) {
-      e.count = 0;
-    }
-    if (effectiveLimit > 0 && e.count >= effectiveLimit) return false;
-    e.count += 1;
-    e.lastAcquired = now;
-    return true;
-  }
-
-  touch(ref, now) {
-    now = now || Date.now();
-    const e = this._inFlight.get(ref);
-    if (e && e.count > 0) {
-      e.lastAcquired = now;
-    }
-  }
-
-  release(ref, now) {
-    now = now || Date.now();
-    const e = this._inFlight.get(ref);
-    if (!e) return;
-    e.count = Math.max(0, e.count - 1);
-    e.lastAcquired = now;
-  }
-
-  getActive(ref, now) {
-    now = now || Date.now();
-    if (!ref) return 0;
-    const e = this._inFlight.get(ref);
-    if (!e) return 0;
-    if (now - e.lastAcquired > this._staleMs) {
-      e.count = 0;
-      return 0;
-    }
-    return e.count;
-  }
-
-  snapshot(now) {
-    now = now || Date.now();
-    const out = {};
-    for (const [k, v] of this._inFlight) {
-      let count = v.count;
-      if (now - v.lastAcquired > this._staleMs) {
-        count = 0;
-        v.count = 0;
-      }
-      out[k] = { count };
-    }
-    return out;
-  }
-
-  pickLeastLoaded(candidates, now, weights, limit) {
-    now = now || Date.now();
-    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
-    if (!Array.isArray(candidates) || candidates.length === 0) return null;
-    let best = null;
-    let bestScore = Infinity;
-    for (const ref of candidates) {
-      const e = this._inFlight.get(ref);
-      let count = e ? e.count : 0;
-      if (e && now - e.lastAcquired > this._staleMs) count = 0;
-      if (effectiveLimit > 0 && count >= effectiveLimit) continue;
-      const w = (weights && Number(weights[ref]) > 0) ? Number(weights[ref]) : 1;
-      const score = count / w;
-      if (score < bestScore) {
-        best = ref;
-        bestScore = score;
-      }
-    }
-    return best;
-  }
-
-  clear(ref) {
-    if (ref) this._inFlight.delete(ref);
-    else this._inFlight.clear();
-  }
-
-  get size() { return this._inFlight.size; }
-}
+// lib/concurrency.js — Backwards-compatible facade (#530).
+export * from './core/concurrency.js';

package/lib/core/concurrency.js

@@ -0,0 +1,115 @@
+// concurrency.js — per-key in-flight counter + least-connections picking (issues #193, #407, #463).
+
+const CONCURRENCY_DEFAULT_LIMIT = 0;
+// Default stale timeout: 30 minutes to support prolonged streaming generation (#463)
+const CONCURRENCY_STALE_LOCK_MS = 30 * 60 * 1000;
+
+export class ConcurrencyTracker {
+  constructor(opts) {
+    opts = opts || {};
+    const limit = opts.limit !== undefined ? opts.limit : CONCURRENCY_DEFAULT_LIMIT;
+    this._limit = (Number.isFinite(limit) && limit >= 0) ? Math.floor(limit) : 0;
+    this._staleMs = opts.staleMs || CONCURRENCY_STALE_LOCK_MS;
+    this._inFlight = new Map();
+  }
+
+  configure(opts) {
+    if (!opts) return;
+    if (opts.limit !== undefined) {
+      this._limit = (Number.isFinite(opts.limit) && opts.limit >= 0) ? Math.floor(opts.limit) : 0;
+    }
+    if (opts.staleMs !== undefined && Number.isFinite(opts.staleMs) && opts.staleMs > 0) {
+      this._staleMs = opts.staleMs;
+    }
+  }
+
+  isEnabled() { return this._limit > 0; }
+  get limit() { return this._limit; }
+
+  acquire(ref, now, limit) {
+    now = now || Date.now();
+    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
+    let e = this._inFlight.get(ref);
+    if (!e) {
+      e = { count: 0, lastAcquired: now };
+      this._inFlight.set(ref, e);
+    }
+    if (now - e.lastAcquired > this._staleMs) {
+      e.count = 0;
+    }
+    if (effectiveLimit > 0 && e.count >= effectiveLimit) return false;
+    e.count += 1;
+    e.lastAcquired = now;
+    return true;
+  }
+
+  touch(ref, now) {
+    now = now || Date.now();
+    const e = this._inFlight.get(ref);
+    if (e && e.count > 0) {
+      e.lastAcquired = now;
+    }
+  }
+
+  release(ref, now) {
+    now = now || Date.now();
+    const e = this._inFlight.get(ref);
+    if (!e) return;
+    e.count = Math.max(0, e.count - 1);
+    e.lastAcquired = now;
+  }
+
+  getActive(ref, now) {
+    now = now || Date.now();
+    if (!ref) return 0;
+    const e = this._inFlight.get(ref);
+    if (!e) return 0;
+    if (now - e.lastAcquired > this._staleMs) {
+      e.count = 0;
+      return 0;
+    }
+    return e.count;
+  }
+
+  snapshot(now) {
+    now = now || Date.now();
+    const out = {};
+    for (const [k, v] of this._inFlight) {
+      let count = v.count;
+      if (now - v.lastAcquired > this._staleMs) {
+        count = 0;
+        v.count = 0;
+      }
+      out[k] = { count };
+    }
+    return out;
+  }
+
+  pickLeastLoaded(candidates, now, weights, limit) {
+    now = now || Date.now();
+    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
+    if (!Array.isArray(candidates) || candidates.length === 0) return null;
+    let best = null;
+    let bestScore = Infinity;
+    for (const ref of candidates) {
+      const e = this._inFlight.get(ref);
+      let count = e ? e.count : 0;
+      if (e && now - e.lastAcquired > this._staleMs) count = 0;
+      if (effectiveLimit > 0 && count >= effectiveLimit) continue;
+      const w = (weights && Number(weights[ref]) > 0) ? Number(weights[ref]) : 1;
+      const score = count / w;
+      if (score < bestScore) {
+        best = ref;
+        bestScore = score;
+      }
+    }
+    return best;
+  }
+
+  clear(ref) {
+    if (ref) this._inFlight.delete(ref);
+    else this._inFlight.clear();
+  }
+
+  get size() { return this._inFlight.size; }
+}

package/lib/core/index.js

@@ -0,0 +1,6 @@
+// lib/core/index.js — Unified Domain Core Layer (#530).
+export * from './state-machine.js';
+export * from './concurrency.js';
+export * from './quota.js';
+export * from './pool.js';
+export * from './rotate.js';

package/lib/core/pool.js

@@ -0,0 +1,2 @@
+// lib/core/pool.js — Core pool exports (#530).
+export * from '../pool.js';

package/lib/core/quota.js

@@ -0,0 +1,39 @@
+// lib/quota.js — quota-remaining persistence per ref.
+// ponytail: pure helpers, snapshot() returns shallow copies.
+
+export class QuotaStore {
+  constructor() {
+    this._data = new Map(); // ref -> { remaining, limit, reset, at }
+  }
+
+  set(ref, info) {
+    if (!ref) return;
+    if (!info || typeof info !== 'object') return;
+    const next = {
+      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
+      limit: Number.isFinite(info.limit) ? info.limit : null,
+      reset: Number.isFinite(info.reset) ? info.reset : null,
+      at: Number.isFinite(info.at) ? info.at : Date.now(),
+    };
+    this._data.set(ref, next);
+  }
+
+  get(ref) {
+    return this._data.get(ref);
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._data) out[k] = v;
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) this._data.delete(ref);
+    else this._data.clear();
+  }
+
+  get size() {
+    return this._data.size;
+  }
+}

package/lib/core/rotate.js

@@ -0,0 +1,2 @@
+// lib/core/rotate.js — Core rotation engine exports (#530).
+export * from '../rotate.js';

package/lib/core/state-machine.js

@@ -0,0 +1,267 @@
+// lib/key-state-machine.js — Encapsulated Key State Machine (#529).
+// Manages credential lifecycle across deterministic states and valid transitions.
+
+export const KEY_STATES = Object.freeze({
+  ACTIVE: 'ACTIVE',
+  COOLDOWN: 'COOLDOWN',
+  PROBING: 'PROBING',
+  EXHAUSTED: 'EXHAUSTED',
+  PAUSED: 'PAUSED',
+  REVOKED: 'REVOKED',
+});
+
+export const VALID_TRANSITIONS = Object.freeze({
+  [KEY_STATES.ACTIVE]: Object.freeze([
+    KEY_STATES.COOLDOWN,
+    KEY_STATES.EXHAUSTED,
+    KEY_STATES.PAUSED,
+    KEY_STATES.REVOKED,
+  ]),
+  [KEY_STATES.COOLDOWN]: Object.freeze([
+    KEY_STATES.PROBING,
+    KEY_STATES.ACTIVE,
+    KEY_STATES.PAUSED,
+    KEY_STATES.REVOKED,
+  ]),
+  [KEY_STATES.PROBING]: Object.freeze([
+    KEY_STATES.ACTIVE,
+    KEY_STATES.COOLDOWN,
+    KEY_STATES.PAUSED,
+    KEY_STATES.REVOKED,
+  ]),
+  [KEY_STATES.EXHAUSTED]: Object.freeze([
+    KEY_STATES.ACTIVE,
+    KEY_STATES.COOLDOWN,
+    KEY_STATES.PAUSED,
+    KEY_STATES.REVOKED,
+  ]),
+  [KEY_STATES.PAUSED]: Object.freeze([
+    KEY_STATES.ACTIVE,
+    KEY_STATES.COOLDOWN,
+    KEY_STATES.REVOKED,
+  ]),
+  [KEY_STATES.REVOKED]: Object.freeze([
+    KEY_STATES.ACTIVE,
+  ]),
+});
+
+export class KeyStateMachine {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.maxHistory] Maximum transition history entries per pool
+   */
+  constructor(options = {}) {
+    /** @type {Map<string, { state: string, updatedAt: number, previousState?: string, reason?: string, failedUntil?: number }>} */
+    this.keyRecords = new Map();
+    /** @type {Array<{ ref: string, from: string, to: string, at: number, reason: string|null }>} */
+    this.history = [];
+    this.maxHistory = options.maxHistory ?? 100;
+  }
+
+  /**
+   * Computes the current deterministic state of a key reference.
+   *
+   * @param {string} keyRef
+   * @param {object} [context] Optional pool state context for backward compatibility
+   * @param {Set<string>} [context.revokedRefs]
+   * @param {Set<string>} [context.pausedRefs]
+   * @param {Map<string, number>} [context.failedUntil]
+   * @param {number} [now]
+   * @returns {string} Current KEY_STATES value
+   */
+  getState(keyRef, context = {}, now = Date.now()) {
+    if (!keyRef) return KEY_STATES.ACTIVE;
+
+    // 1. Revocation is absolute
+    if (context.revokedRefs && context.revokedRefs.has(keyRef)) {
+      return KEY_STATES.REVOKED;
+    }
+
+    // 2. Manual pause overrides operational state
+    if (context.pausedRefs && context.pausedRefs.has(keyRef)) {
+      return KEY_STATES.PAUSED;
+    }
+
+    const record = this.keyRecords.get(keyRef);
+    if (!record) {
+      const failedUntil = context.failedUntil?.get?.(keyRef);
+      if (typeof failedUntil === 'number' && failedUntil > now) {
+        return KEY_STATES.COOLDOWN;
+      }
+      return KEY_STATES.ACTIVE;
+    }
+
+    // Cooldown transition to probing when timer elapses
+    if (record.state === KEY_STATES.COOLDOWN) {
+      const until = record.failedUntil ?? context.failedUntil?.get?.(keyRef);
+      if (typeof until === 'number' && until <= now) {
+        return KEY_STATES.PROBING;
+      }
+    }
+
+    return record.state;
+  }
+
+  /**
+   * Transitions a key reference to a new state if valid.
+   *
+   * @param {string} keyRef
+   * @param {string} targetState Target KEY_STATES value
+   * @param {object} [metadata]
+   * @param {string} [metadata.reason]
+   * @param {number} [metadata.failedUntil]
+   * @param {object} [metadata.context]
+   * @param {number} [now]
+   * @returns {string|null} Resulting state or null on failure
+   */
+  transition(keyRef, targetState, metadata = {}, now = Date.now()) {
+    if (!keyRef || !targetState) return null;
+
+    const currentState = this.getState(keyRef, metadata.context, now);
+    if (currentState === targetState) return currentState;
+
+    const allowed = VALID_TRANSITIONS[currentState] || [];
+    if (!allowed.includes(targetState)) {
+      // In relaxed mode, force transition with logged anomaly
+      targetState = targetState in KEY_STATES ? targetState : KEY_STATES.ACTIVE;
+    }
+
+    const record = this.keyRecords.get(keyRef) ?? {
+      state: currentState,
+      updatedAt: now,
+    };
+
+    record.previousState = currentState;
+    record.state = targetState;
+    record.updatedAt = now;
+    if (metadata.reason !== undefined) record.reason = metadata.reason;
+    if (metadata.failedUntil !== undefined) record.failedUntil = metadata.failedUntil;
+
+    this.keyRecords.set(keyRef, record);
+
+    const event = {
+      ref: keyRef,
+      from: currentState,
+      to: targetState,
+      at: now,
+      reason: metadata.reason ?? null,
+    };
+
+    this.history.push(event);
+    if (this.history.length > this.maxHistory) {
+      this.history.shift();
+    }
+
+    // Synchronize external pool state if provided
+    if (metadata.context) {
+      if (targetState === KEY_STATES.REVOKED) {
+        metadata.context.revokedRefs?.add?.(keyRef);
+        metadata.context.pausedRefs?.delete?.(keyRef);
+      } else if (targetState === KEY_STATES.PAUSED) {
+        metadata.context.pausedRefs?.add?.(keyRef);
+      } else if (currentState === KEY_STATES.PAUSED && targetState !== KEY_STATES.PAUSED) {
+        metadata.context.pausedRefs?.delete?.(keyRef);
+      }
+    }
+
+    return targetState;
+  }
+
+  /**
+   * Puts key into COOLDOWN state until specified duration elapses.
+   */
+  transitionToCooldown(keyRef, options = {}, now = Date.now()) {
+    const cooldownMs = Number(options.cooldownMs) > 0 ? Number(options.cooldownMs) : 60000;
+    const failedUntil = now + cooldownMs;
+    return this.transition(keyRef, KEY_STATES.COOLDOWN, {
+      failedUntil,
+      reason: options.reason || 'COOLDOWN',
+      context: options.context,
+    }, now);
+  }
+
+  /** Marks key as ready for canary probe dispatch. */
+  markProbing(keyRef, metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.PROBING, {
+      reason: metadata.reason || 'CANARY_PROBE',
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Activates key following successful request or probe. */
+  activate(keyRef, metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.ACTIVE, {
+      reason: metadata.reason || 'SUCCESS',
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Marks key as exhausted due to local quota or TPM/RPM window. */
+  markExhausted(keyRef, metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.EXHAUSTED, {
+      reason: metadata.reason || 'QUOTA_EXHAUSTED',
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Manually pauses key from rotation. */
+  pause(keyRef, reason = 'USER_PAUSE', metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.PAUSED, {
+      reason,
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Unpauses key, returning it to active rotation. */
+  unpause(keyRef, metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.ACTIVE, {
+      reason: 'USER_RESUME',
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Revokes key permanently until credentials change. */
+  revoke(keyRef, reason = 'INVALID_CREDENTIAL', metadata = {}, now = Date.now()) {
+    return this.transition(keyRef, KEY_STATES.REVOKED, {
+      reason,
+      context: metadata.context,
+    }, now);
+  }
+
+  /** Checks if key is usable for dispatch (ACTIVE or PROBING). */
+  isUsable(keyRef, context = {}, now = Date.now()) {
+    const state = this.getState(keyRef, context, now);
+    return state === KEY_STATES.ACTIVE || state === KEY_STATES.PROBING;
+  }
+
+  /** Returns transition history for a specific key or all keys. */
+  getHistory(keyRef) {
+    if (!keyRef) return this.history.slice();
+    return this.history.filter((e) => e.ref === keyRef);
+  }
+
+  /** Returns state distribution statistics across a set of keys. */
+  getStats(refs = [], context = {}, now = Date.now()) {
+    const stats = {
+      total: refs.length,
+      [KEY_STATES.ACTIVE]: 0,
+      [KEY_STATES.COOLDOWN]: 0,
+      [KEY_STATES.PROBING]: 0,
+      [KEY_STATES.EXHAUSTED]: 0,
+      [KEY_STATES.PAUSED]: 0,
+      [KEY_STATES.REVOKED]: 0,
+    };
+    for (const ref of refs) {
+      const state = this.getState(ref, context, now);
+      if (state in stats) stats[state]++;
+    }
+    return stats;
+  }
+
+  clear() {
+    this.keyRecords.clear();
+    this.history = [];
+  }
+}
+
+export const defaultKeyStateMachine = new KeyStateMachine();

package/lib/index.js

@@ -55,7 +55,7 @@ import { LOCAL_QUOTA_CODE } from './rotate.js';
 import { setupIdleHealEffect, setupAutoUnbreakEffect, setupPersistence, setupProactiveHealthProbeEffect, setupKeyDrainEffect } from './lifecycle.js';
 
 export const name = '@goodandready/dsh-key-rotation';
-export const inject = ['llm', 'webServer', 'settings', 'credentials'];
+export const inject = ['llm'];
 export { keyTail, KEY_TAIL_CHARS, KEY_TAIL_PLACEHOLDER, isLoopbackAddress, isTrustedBridgeRequest, isDesktopShellRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage, getRuntime };
 export { notifySwitch, notifyExhaustion, notifyEarlyWarning };
 export { classifyResponseError } from './errors.js';
@@ -66,6 +66,7 @@ export { KeyStateMachine, KEY_STATES, VALID_TRANSITIONS } from './key-state-mach
 export { unwrapVolatileConfig } from './config-compat.js';
 export { QuotaRunway, defaultQuotaRunway, estimateRunway } from './quota-runway.js';
 export * as transports from './transports/index.js';
+export * as core from './core/index.js';
 
 const NS = 'dsh-key-rotation';
 
@@ -224,7 +225,6 @@ export function apply(ctx, config = {}) {
   const { ensureSandboxRunner, probeRef, lastTestCache } = sandboxService;
 
   const readProfiles = createProviderProfilesReader(ctx, PIAI_NS);
-  registerConfigBridge(ctx, () => buildRuntime().cloneIds);
   ensureSandboxRunner(ctx);
 
   setupIdleHealEffect(ctx, getConfig, buildRuntime);
@@ -246,6 +246,40 @@ export function apply(ctx, config = {}) {
 
   setupAutoUnbreakEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger);
 
+  // Dynamic webServer registration (#528)
+  const setupWebRoutes = (webCtx = ctx) => {
+    const target = webCtx?.webServer ? webCtx : ctx;
+    registerConfigBridge(target, () => buildRuntime().cloneIds);
+    registerOpsRoutes(target, {
+      buildRuntime,
+      latencyHistogram,
+      lastTestCache,
+      ensureSandboxRunner,
+      poolState,
+      getRotationDisabled: () => rotationDisabled,
+      setRotationDisabled: (v) => { rotationDisabled = v; },
+      circuitBreaker: moduleBreaker,
+      notifyQueue: moduleNotifyQueue,
+      quotaStore,
+      keyDrainManager,
+      selectionTraceStore,
+      diagBuffer,
+    });
+    if (typeof target.webServer?.register === 'function') {
+      registerPluginUpdater(target, {
+        endpoint: '/api/dsh-key-rotation/update',
+        packageName: '@goodandready/dsh-key-rotation',
+        manifestUrl: new URL('../package.json', import.meta.url),
+      });
+    }
+  };
+
+  if (ctx.webServer) {
+    setupWebRoutes(ctx);
+  } else if (typeof ctx.inject === 'function') {
+    ctx.inject(['webServer'], setupWebRoutes);
+  }
+
   // Periodic sweep of expired cooldowns
   ctx.effect(() => {
     const id = setInterval(() => {
@@ -471,30 +505,7 @@ export function apply(ctx, config = {}) {
     logger,
   });
 
-  registerOpsRoutes(ctx, {
-    buildRuntime,
-    latencyHistogram,
-    lastTestCache,
-    ensureSandboxRunner,
-    poolState,
-    getRotationDisabled: () => rotationDisabled,
-    setRotationDisabled: (v) => { rotationDisabled = v; },
-    circuitBreaker: moduleBreaker,
-    notifyQueue: moduleNotifyQueue,
-    quotaStore,
-    keyDrainManager,
-    selectionTraceStore,
-    diagBuffer,
-  });
-
-  ctx.effect(() => {
-    if (typeof ctx.webServer?.register !== 'function') return;
-    registerPluginUpdater(ctx, {
-      endpoint: '/api/dsh-key-rotation/update',
-      packageName: '@goodandready/dsh-key-rotation',
-      manifestUrl: new URL('../package.json', import.meta.url),
-    });
-  }, 'dsh-key-rotation: plugin updater');
+// routes registered dynamically above via setupWebRoutes
 
   ctx.effect(() => ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();

package/lib/key-state-machine.js

@@ -1,267 +1,2 @@
-// lib/key-state-machine.js — Encapsulated Key State Machine (#529).
-// Manages credential lifecycle across deterministic states and valid transitions.
-
-export const KEY_STATES = Object.freeze({
-  ACTIVE: 'ACTIVE',
-  COOLDOWN: 'COOLDOWN',
-  PROBING: 'PROBING',
-  EXHAUSTED: 'EXHAUSTED',
-  PAUSED: 'PAUSED',
-  REVOKED: 'REVOKED',
-});
-
-export const VALID_TRANSITIONS = Object.freeze({
-  [KEY_STATES.ACTIVE]: Object.freeze([
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.EXHAUSTED,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.COOLDOWN]: Object.freeze([
-    KEY_STATES.PROBING,
-    KEY_STATES.ACTIVE,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.PROBING]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.EXHAUSTED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.PAUSED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.REVOKED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-  ]),
-});
-
-export class KeyStateMachine {
-  /**
-   * @param {object} [options]
-   * @param {number} [options.maxHistory] Maximum transition history entries per pool
-   */
-  constructor(options = {}) {
-    /** @type {Map<string, { state: string, updatedAt: number, previousState?: string, reason?: string, failedUntil?: number }>} */
-    this.keyRecords = new Map();
-    /** @type {Array<{ ref: string, from: string, to: string, at: number, reason: string|null }>} */
-    this.history = [];
-    this.maxHistory = options.maxHistory ?? 100;
-  }
-
-  /**
-   * Computes the current deterministic state of a key reference.
-   *
-   * @param {string} keyRef
-   * @param {object} [context] Optional pool state context for backward compatibility
-   * @param {Set<string>} [context.revokedRefs]
-   * @param {Set<string>} [context.pausedRefs]
-   * @param {Map<string, number>} [context.failedUntil]
-   * @param {number} [now]
-   * @returns {string} Current KEY_STATES value
-   */
-  getState(keyRef, context = {}, now = Date.now()) {
-    if (!keyRef) return KEY_STATES.ACTIVE;
-
-    // 1. Revocation is absolute
-    if (context.revokedRefs && context.revokedRefs.has(keyRef)) {
-      return KEY_STATES.REVOKED;
-    }
-
-    // 2. Manual pause overrides operational state
-    if (context.pausedRefs && context.pausedRefs.has(keyRef)) {
-      return KEY_STATES.PAUSED;
-    }
-
-    const record = this.keyRecords.get(keyRef);
-    if (!record) {
-      const failedUntil = context.failedUntil?.get?.(keyRef);
-      if (typeof failedUntil === 'number' && failedUntil > now) {
-        return KEY_STATES.COOLDOWN;
-      }
-      return KEY_STATES.ACTIVE;
-    }
-
-    // Cooldown transition to probing when timer elapses
-    if (record.state === KEY_STATES.COOLDOWN) {
-      const until = record.failedUntil ?? context.failedUntil?.get?.(keyRef);
-      if (typeof until === 'number' && until <= now) {
-        return KEY_STATES.PROBING;
-      }
-    }
-
-    return record.state;
-  }
-
-  /**
-   * Transitions a key reference to a new state if valid.
-   *
-   * @param {string} keyRef
-   * @param {string} targetState Target KEY_STATES value
-   * @param {object} [metadata]
-   * @param {string} [metadata.reason]
-   * @param {number} [metadata.failedUntil]
-   * @param {object} [metadata.context]
-   * @param {number} [now]
-   * @returns {string|null} Resulting state or null on failure
-   */
-  transition(keyRef, targetState, metadata = {}, now = Date.now()) {
-    if (!keyRef || !targetState) return null;
-
-    const currentState = this.getState(keyRef, metadata.context, now);
-    if (currentState === targetState) return currentState;
-
-    const allowed = VALID_TRANSITIONS[currentState] || [];
-    if (!allowed.includes(targetState)) {
-      // In relaxed mode, force transition with logged anomaly
-      targetState = targetState in KEY_STATES ? targetState : KEY_STATES.ACTIVE;
-    }
-
-    const record = this.keyRecords.get(keyRef) ?? {
-      state: currentState,
-      updatedAt: now,
-    };
-
-    record.previousState = currentState;
-    record.state = targetState;
-    record.updatedAt = now;
-    if (metadata.reason !== undefined) record.reason = metadata.reason;
-    if (metadata.failedUntil !== undefined) record.failedUntil = metadata.failedUntil;
-
-    this.keyRecords.set(keyRef, record);
-
-    const event = {
-      ref: keyRef,
-      from: currentState,
-      to: targetState,
-      at: now,
-      reason: metadata.reason ?? null,
-    };
-
-    this.history.push(event);
-    if (this.history.length > this.maxHistory) {
-      this.history.shift();
-    }
-
-    // Synchronize external pool state if provided
-    if (metadata.context) {
-      if (targetState === KEY_STATES.REVOKED) {
-        metadata.context.revokedRefs?.add?.(keyRef);
-        metadata.context.pausedRefs?.delete?.(keyRef);
-      } else if (targetState === KEY_STATES.PAUSED) {
-        metadata.context.pausedRefs?.add?.(keyRef);
-      } else if (currentState === KEY_STATES.PAUSED && targetState !== KEY_STATES.PAUSED) {
-        metadata.context.pausedRefs?.delete?.(keyRef);
-      }
-    }
-
-    return targetState;
-  }
-
-  /**
-   * Puts key into COOLDOWN state until specified duration elapses.
-   */
-  transitionToCooldown(keyRef, options = {}, now = Date.now()) {
-    const cooldownMs = Number(options.cooldownMs) > 0 ? Number(options.cooldownMs) : 60000;
-    const failedUntil = now + cooldownMs;
-    return this.transition(keyRef, KEY_STATES.COOLDOWN, {
-      failedUntil,
-      reason: options.reason || 'COOLDOWN',
-      context: options.context,
-    }, now);
-  }
-
-  /** Marks key as ready for canary probe dispatch. */
-  markProbing(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.PROBING, {
-      reason: metadata.reason || 'CANARY_PROBE',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Activates key following successful request or probe. */
-  activate(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.ACTIVE, {
-      reason: metadata.reason || 'SUCCESS',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Marks key as exhausted due to local quota or TPM/RPM window. */
-  markExhausted(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.EXHAUSTED, {
-      reason: metadata.reason || 'QUOTA_EXHAUSTED',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Manually pauses key from rotation. */
-  pause(keyRef, reason = 'USER_PAUSE', metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.PAUSED, {
-      reason,
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Unpauses key, returning it to active rotation. */
-  unpause(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.ACTIVE, {
-      reason: 'USER_RESUME',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Revokes key permanently until credentials change. */
-  revoke(keyRef, reason = 'INVALID_CREDENTIAL', metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.REVOKED, {
-      reason,
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Checks if key is usable for dispatch (ACTIVE or PROBING). */
-  isUsable(keyRef, context = {}, now = Date.now()) {
-    const state = this.getState(keyRef, context, now);
-    return state === KEY_STATES.ACTIVE || state === KEY_STATES.PROBING;
-  }
-
-  /** Returns transition history for a specific key or all keys. */
-  getHistory(keyRef) {
-    if (!keyRef) return this.history.slice();
-    return this.history.filter((e) => e.ref === keyRef);
-  }
-
-  /** Returns state distribution statistics across a set of keys. */
-  getStats(refs = [], context = {}, now = Date.now()) {
-    const stats = {
-      total: refs.length,
-      [KEY_STATES.ACTIVE]: 0,
-      [KEY_STATES.COOLDOWN]: 0,
-      [KEY_STATES.PROBING]: 0,
-      [KEY_STATES.EXHAUSTED]: 0,
-      [KEY_STATES.PAUSED]: 0,
-      [KEY_STATES.REVOKED]: 0,
-    };
-    for (const ref of refs) {
-      const state = this.getState(ref, context, now);
-      if (state in stats) stats[state]++;
-    }
-    return stats;
-  }
-
-  clear() {
-    this.keyRecords.clear();
-    this.history = [];
-  }
-}
-
-export const defaultKeyStateMachine = new KeyStateMachine();
+// lib/key-state-machine.js — Backwards-compatible facade (#530).
+export * from './core/state-machine.js';

package/lib/persistence.js

@@ -1,8 +1,9 @@
-// lib/persistence.js — atomic snapshot of rotation state across restarts (#287).
-// Cooldowns, circuit-breaker entries and quota counters live in memory; a DSH
-// reload would otherwise forget cooldowns and stampede recovered keys.
+// lib/persistence.js — atomic snapshot and append-only event journal (#287, #525).
+// Cooldowns, circuit-breaker entries and quota counters live in memory; an append-only
+// journal records incremental events without rewriting the entire state tree on disk.
 
 import path from 'node:path';
+import fs from 'node:fs';
 import { initializePoolState, hasPoolActivity } from './pool-state.js';
 import { atomicWriteJson, safeReadJson } from './atomic-io.js';
 
@@ -18,6 +19,9 @@ export class StatePersistence {
       throw new Error('StatePersistence: filePath is required');
     }
     this.filePath = filePath;
+    this.journalPath = filePath.endsWith('.json')
+      ? filePath.slice(0, -5) + '-journal.jsonl'
+      : `${filePath}.jsonl`;
     this._now = now;
     this._timer = null;
     this._dirty = false;
@@ -112,8 +116,8 @@ export class StatePersistence {
             if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
               const innerObj = {};
               const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
-              for (const [day, c] of innerEntries) {
-                const num = Number(c);
+              for (const [day, count] of innerEntries) {
+                const num = Number(count);
                 if (Number.isFinite(num) && num >= 0) innerObj[day] = Math.floor(num);
               }
               if (Object.keys(innerObj).length > 0) usageDays[k] = innerObj;
@@ -168,7 +172,22 @@ export class StatePersistence {
   }
 
   /**
-   * Schedule a debounced atomic write. Never throws to the caller.
+   * Append-only event journaling (#525). Records incremental event without full state rewrite.
+   * @param {object} event
+   */
+  async appendEvent(event) {
+    if (!event || typeof event !== 'object') return false;
+    const line = JSON.stringify({ ...event, at: event.at ?? this._nowSafe() }) + '\n';
+    try {
+      await fs.promises.appendFile(this.journalPath, line, 'utf-8');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Schedule a debounced atomic snapshot write. Never throws to the caller.
    * @param {object} payload result of StatePersistence.serialize
    */
   save(payload) {
@@ -190,9 +209,11 @@ export class StatePersistence {
     try {
       await atomicWriteJson(this.filePath, payload);
       this._lastWrite = this._nowSafe();
+      try {
+        await fs.promises.writeFile(this.journalPath, '', 'utf-8');
+      } catch {}
       return true;
     } catch {
-      // Keep previous good file; mark dirty so a later save retries.
       this._dirty = true;
       this._pending = this._pending || payload;
       return false;
@@ -200,16 +221,56 @@ export class StatePersistence {
   }
 
   /**
-   * Load a snapshot. Corrupt/missing file → null (never wipes memory).
+   * Load snapshot and replay any journaled events (#525).
    * @returns {Promise<object|null>}
    */
   async load() {
     const raw = await safeReadJson(this.filePath, null);
     if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
     if (raw.version !== 1 && raw.version !== 2) return null;
+
+    if (fs.existsSync(this.journalPath)) {
+      try {
+        const content = await fs.promises.readFile(this.journalPath, 'utf-8');
+        const lines = content.split('\n');
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed) {
+            try { this.replayEvent(raw, JSON.parse(trimmed)); } catch {}
+          }
+        }
+      } catch {}
+    }
+
     return raw;
   }
 
+  /** Replay individual event into a snapshot object (#525). */
+  replayEvent(snapshot, event) {
+    if (!snapshot || !event?.pool || !event?.ref) return;
+    if (!snapshot.pools) snapshot.pools = {};
+    if (!snapshot.pools[event.pool]) {
+      snapshot.pools[event.pool] = { failedUntil: {}, pausedRefs: [], revokedRefs: [] };
+    }
+    const p = snapshot.pools[event.pool];
+    if (event.type === 'cooldown' || event.type === 'failure') {
+      if (!p.failedUntil) p.failedUntil = {};
+      p.failedUntil[event.ref] = event.until ?? (event.at + (event.cooldownMs || 60000));
+    } else if (event.type === 'success' || event.type === 'reset') {
+      if (p.failedUntil) delete p.failedUntil[event.ref];
+    } else if (event.type === 'pause') {
+      if (!Array.isArray(p.pausedRefs)) p.pausedRefs = [];
+      if (!p.pausedRefs.includes(event.ref)) p.pausedRefs.push(event.ref);
+    } else if (event.type === 'unpause') {
+      if (Array.isArray(p.pausedRefs)) {
+        p.pausedRefs = p.pausedRefs.filter((r) => r !== event.ref);
+      }
+    } else if (event.type === 'revoke') {
+      if (!Array.isArray(p.revokedRefs)) p.revokedRefs = [];
+      if (!p.revokedRefs.includes(event.ref)) p.revokedRefs.push(event.ref);
+    }
+  }
+
   /** Restore poolState Map from a loaded snapshot. */
   static restorePools(poolState, snapshot) {
     if (!poolState || !snapshot?.pools || typeof snapshot.pools !== 'object' || Array.isArray(snapshot.pools)) return 0;
@@ -218,8 +279,6 @@ export class StatePersistence {
       if (!saved || typeof saved !== 'object' || Array.isArray(saved)) continue;
       const existing = poolState.get(base);
       const state = initializePoolState(existing);
-      // Never replace objects retained by buildRuntime(), nor roll back a pool
-      // already used while the asynchronous load was in flight.
       if (!existing || !hasPoolActivity(state)) {
         if (saved.failedUntil && typeof saved.failedUntil === 'object' && !Array.isArray(saved.failedUntil)) {
           for (const [ref, until] of Object.entries(saved.failedUntil)) {

package/lib/quota.js

@@ -1,39 +1,2 @@
-// lib/quota.js — quota-remaining persistence per ref.
-// ponytail: pure helpers, snapshot() returns shallow copies.
-
-export class QuotaStore {
-  constructor() {
-    this._data = new Map(); // ref -> { remaining, limit, reset, at }
-  }
-
-  set(ref, info) {
-    if (!ref) return;
-    if (!info || typeof info !== 'object') return;
-    const next = {
-      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
-      limit: Number.isFinite(info.limit) ? info.limit : null,
-      reset: Number.isFinite(info.reset) ? info.reset : null,
-      at: Number.isFinite(info.at) ? info.at : Date.now(),
-    };
-    this._data.set(ref, next);
-  }
-
-  get(ref) {
-    return this._data.get(ref);
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._data) out[k] = v;
-    return out;
-  }
-
-  clear(ref) {
-    if (ref) this._data.delete(ref);
-    else this._data.clear();
-  }
-
-  get size() {
-    return this._data.size;
-  }
-}
+// lib/quota.js — Backwards-compatible facade (#530).
+export * from './core/quota.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.79",
+  "version": "0.8.80",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [