@bevel-software/platform-core-backend 0.14.0 → 0.19.0

package/src/modules/mcp/__tests__/surface-log-throttle.test.ts

@@ -0,0 +1,169 @@
+import { describe, expect, it } from 'vitest';
+import { DEFAULT_SURFACE_LOG_INTERVAL_MS, SurfaceLogThrottle } from '../surface-log-throttle.js';
+
+/**
+ * The surface line must keep appearing (it is the measurement of the
+ * stateless design's per-request cost) without appearing per request. What
+ * is pinned: first rebuild logs; identical rebuilds inside the interval do
+ * not and are counted; a shape change logs at once; the interval elapsing
+ * logs again carrying the count; users are independent; quiet users are
+ * forgotten.
+ */
+describe('SurfaceLogThrottle', () => {
+  function make(intervalMs = DEFAULT_SURFACE_LOG_INTERVAL_MS) {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(intervalMs, () => now);
+    return { throttle, advance: (ms: number) => (now += ms) };
+  }
+
+  it('logs the first rebuild for a user, then suppresses identical rebuilds inside the interval', () => {
+    const { throttle } = make();
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 })).toEqual({ log: true, suppressed: 0 });
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 })).toEqual({ log: false, suppressed: 1 });
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 })).toEqual({ log: false, suppressed: 2 });
+  });
+
+  it('logs at once when the shape changes, with the count of what it suppressed', () => {
+    const { throttle } = make();
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    // A manual failed (or was fixed): the count moved.
+    expect(throttle.decide('u1', { tools: 3, manuals: 2 })).toEqual({ log: true, suppressed: 1 });
+    // The new shape is now the baseline.
+    expect(throttle.decide('u1', { tools: 3, manuals: 2 })).toEqual({ log: false, suppressed: 1 });
+  });
+
+  it('logs again once the interval has elapsed, carrying the suppressed count', () => {
+    const { throttle, advance } = make(60_000);
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    advance(59_999);
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 }).log).toBe(false);
+    advance(1);
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 })).toEqual({ log: true, suppressed: 3 });
+  });
+
+  it('keeps users independent', () => {
+    const { throttle } = make();
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 }).log).toBe(true);
+    expect(throttle.decide('u2', { tools: 5, manuals: 2 }).log).toBe(true);
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 }).log).toBe(false);
+    expect(throttle.decide('u2', { tools: 5, manuals: 2 }).log).toBe(false);
+  });
+
+  it('forgets a user who has been quiet for the interval — the next rebuild logs fresh', () => {
+    const { throttle, advance } = make(60_000);
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    throttle.decide('u1', { tools: 5, manuals: 2 });
+    advance(60_000);
+    // Another user's rebuild is what runs the prune; u1's stale entry goes.
+    throttle.decide('u2', { tools: 1, manuals: 1 });
+    expect(throttle.decide('u1', { tools: 5, manuals: 2 })).toEqual({ log: true, suppressed: 0 });
+  });
+});
+
+describe('SurfaceLogThrottle — pruning cost', () => {
+  const shape = { tools: 5, manuals: 2 };
+
+  it('sweeps once per interval, not on every rebuild — and every quiet user is still gone by the next sweep', () => {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(60_000, () => now);
+
+    throttle.decide('a', shape); // t0 — the first sweep runs here (nothing to drop)
+    now += 30_000;
+    throttle.decide('b', shape); // t0+30s — no sweep owed yet
+    now += 30_000;
+    throttle.decide('c', shape); // t0+60s — sweep: a (quiet 60s) dropped, b kept
+    expect(throttle.size()).toBe(2);
+
+    now += 30_001;
+    throttle.decide('d', shape); // t0+90.001s — b is quiet ≥ interval, but the
+    // sweep ran 30s ago, so no walk happens on this request: b waits.
+    expect(throttle.size()).toBe(3);
+
+    now += 29_999;
+    throttle.decide('e', shape); // t0+120s — sweep: b and c dropped, d kept
+    expect(throttle.size()).toBe(2);
+  });
+});
+
+describe('SurfaceLogThrottle — a wall clock that steps backwards', () => {
+  const shape = { tools: 5, manuals: 2 };
+
+  it('does not stall pruning until the clock climbs back past the last sweep', () => {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(60_000, () => now);
+
+    throttle.decide('a', shape); // t0 — first sweep
+    now += 60_000;
+    throttle.decide('b', shape); // t0+60s — sweep (a dropped); last sweep is now t0+60s
+
+    // The clock steps back 45s (NTP correction / VM restore). A cadence that
+    // waited for `now - lastSweep >= interval` would not sweep again until
+    // t0+120s in the OLD domain — 105s from here, not 60s.
+    now -= 45_000; // t0+15s
+    throttle.decide('c', shape);
+    now += 60_000; // t0+75s — c has been quiet a full interval in the new domain
+    throttle.decide('d', shape);
+
+    // c is gone: the backwards step reset the cadence instead of stalling it.
+    // So is b: its stamp (t0+60s) was in the future at the reset, so it was
+    // restamped as seen then, and has now been quiet an interval too.
+    expect(throttle.size()).toBe(1);
+  });
+});
+
+describe('SurfaceLogThrottle — a user rebuilding under suppression is not quiet', () => {
+  const shape = { tools: 5, manuals: 2 };
+
+  it('survives another user\'s sweep, so the count its next line owes is not lost', () => {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(60_000, () => now);
+
+    throttle.decide('a', shape); // t0: logged
+    now += 30_000;
+    throttle.decide('a', shape); // t0+30s: suppressed (1) — a is active
+    now += 30_000;
+    // t0+60s: another user's request runs the sweep. a's last LOGGED line is
+    // an interval old, but a rebuilt thirty seconds ago; it stays.
+    throttle.decide('b', shape);
+    expect(throttle.size()).toBe(2);
+    // a's next line carries the rebuild that was suppressed, not zero.
+    expect(throttle.decide('a', shape)).toEqual({ log: true, suppressed: 1 });
+  });
+
+  it('is still forgotten once it has not rebuilt at all for an interval', () => {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(60_000, () => now);
+    throttle.decide('a', shape);
+    now += 30_000;
+    throttle.decide('a', shape); // suppressed
+    now += 60_000; // a has been quiet a whole interval since that rebuild
+    throttle.decide('b', shape); // sweep
+    expect(throttle.size()).toBe(1);
+  });
+});
+
+describe('SurfaceLogThrottle — a large clock rollback', () => {
+  const shape = { tools: 5, manuals: 2 };
+
+  it('does not keep pre-rollback users for the length of the step — they age from the reset', () => {
+    let now = 1_000_000;
+    const throttle = new SurfaceLogThrottle(60_000, () => now);
+
+    throttle.decide('a', shape); // t0
+    now += 60_000;
+    throttle.decide('b', shape); // t0+60s — sweep; b stamped t0+60s
+
+    // The clock rolls back ten intervals. b's stamp is now 10 minutes in the
+    // future; waiting for the clock to pass it by an interval would keep b
+    // for eleven intervals — far past the two-interval bound.
+    now -= 600_000; // t0-540s
+    throttle.decide('c', shape); // sweep (cadence reset): b restamped as seen now
+    now += 60_000; // one interval in the new domain
+    throttle.decide('d', shape); // sweep: b and c have been quiet an interval — gone
+
+    expect(throttle.size()).toBe(1);
+  });
+});

package/src/modules/mcp/downstream-pool.ts

@@ -0,0 +1,256 @@
+/**
+ * The ONE piece of state the stateless MCP endpoint keeps between requests: a
+ * pool of connections to proxied third-party MCP servers (`mcp.json` manuals),
+ * keyed by (user, server).
+ *
+ * Everything else a request needs is re-derived from its own bearer. These
+ * connections are the exception because a downstream MCP server is session-ful
+ * by nature — dialing a fresh handshake on every tool call would add a round
+ * trip per call and hammer the provider. So they are cached, and the pool is
+ * built so that losing any of it costs a reconnect, never correctness:
+ *
+ *   - created LAZILY, on the first request that needs the server;
+ *   - SINGLE-FLIGHT per key: concurrent first calls share one creation, so a
+ *     burst produces one connection instead of N with all but one orphaned;
+ *   - IDLE-EVICTED and SIZE-BOUNDED, with the retired session store's proven
+ *     defaults (4 hours idle, 5000 entries);
+ *   - every eviction CLOSES what it evicts (`dispose`) — the pool is the owner
+ *     of each connection it opened, and nothing else closes them;
+ *   - LEASED: `acquire` hands out a lease the caller releases when its
+ *     operation ends. A leased entry is in use, so it is never idle and never
+ *     the LRU victim; an entry invalidated while leased leaves the pool at once
+ *     but is closed only when its last lease is released. A call in progress
+ *     never has its connection closed underneath it.
+ *
+ * Eviction is lazy, like the store it replaces: `acquire` sweeps idle entries
+ * first and enforces the cap on insert. No background timer, so constructing a
+ * pool never leaks an interval handle (in tests or anywhere else). When every
+ * entry is leased the cap cannot evict anything, and the pool holds more than
+ * `maxEntries` until leases are released — and the release that ends an
+ * entry's last lease re-applies the cap, so the overflow ends with the leases,
+ * not with whenever the next acquire happens to run.
+ *
+ * Healing a connection whose server restarted is NOT this class's job: the
+ * pooled value carries the existing session-recovery wrapper, which
+ * re-registers on the spec's 404 in place. The pool only decides how long a
+ * value lives.
+ */
+
+import { logger } from '../../shared/logging.js';
+
+const log = logger('mcp');
+
+export interface DownstreamPoolOptions<V> {
+  /** Close a value the pool is dropping. Errors are logged, never thrown. */
+  dispose: (value: V) => Promise<void>;
+  /** Drop entries unused for longer than this. Default: 4 hours. */
+  idleTtlMs?: number;
+  /** Hard cap on pooled entries; the least-recently-used is evicted first. Default: 5000. */
+  maxEntries?: number;
+  /** Override `Date.now` — injected only by tests. */
+  now?: () => number;
+}
+
+/** A pooled value in use. `release` ends the use; calling it again is a no-op. */
+export interface Lease<V> {
+  readonly value: V;
+  release(): void;
+}
+
+export const DEFAULT_DOWNSTREAM_IDLE_TTL_MS = 4 * 60 * 60 * 1000;
+export const DEFAULT_DOWNSTREAM_MAX_ENTRIES = 5000;
+
+interface Entry<V> {
+  key: string;
+  value: V;
+  lastUsedAt: number;
+  /** Outstanding leases; an entry with any is in use and not evictable. */
+  leases: number;
+  /** Removed from the pool while leased: close it when the last lease ends. */
+  retired: boolean;
+}
+
+interface Pending<V> {
+  promise: Promise<V>;
+  /** Set by `evictWhere` when the key is invalidated while creation is in flight. */
+  stale: boolean;
+}
+
+export class DownstreamPool<V> {
+  private readonly entries = new Map<string, Entry<V>>();
+  private readonly pending = new Map<string, Pending<V>>();
+  private readonly dispose: (value: V) => Promise<void>;
+  private readonly idleTtlMs: number;
+  private readonly maxEntries: number;
+  private readonly now: () => number;
+
+  constructor(opts: DownstreamPoolOptions<V>) {
+    this.dispose = opts.dispose;
+    this.idleTtlMs = opts.idleTtlMs ?? DEFAULT_DOWNSTREAM_IDLE_TTL_MS;
+    this.maxEntries = Math.max(1, opts.maxEntries ?? DEFAULT_DOWNSTREAM_MAX_ENTRIES);
+    this.now = opts.now ?? Date.now;
+  }
+
+  /**
+   * Lease the pooled value for `key`, creating it with `create` when absent.
+   * The caller MUST release the lease when its operation ends (try/finally).
+   * A creation that rejects is NOT cached — every waiter sees the rejection and
+   * the next `acquire` tries again (retry pacing is the caller's policy, see
+   * ManualFailureMemo).
+   */
+  async acquire(key: string, create: () => Promise<V>): Promise<Lease<V>> {
+    this.sweep();
+    const hit = this.entries.get(key);
+    if (hit) return this.lease(hit);
+    const inflight = this.pending.get(key);
+    if (inflight) return this.awaitPending(key, inflight, create);
+
+    const record: Pending<V> = { promise: create(), stale: false };
+    this.pending.set(key, record);
+    const forget = () => {
+      if (this.pending.get(key) === record) this.pending.delete(key);
+    };
+    let value: V;
+    try {
+      value = await record.promise;
+    } finally {
+      forget();
+    }
+    if (record.stale) {
+      // Invalidated mid-creation (the user's credentials changed): the value
+      // was built from the old state. Close it and build a current one.
+      this.disposeQuietly(key, value);
+      return this.acquire(key, create);
+    }
+    return this.lease(this.insert(key, value));
+  }
+
+  /**
+   * Evict every entry whose key matches — the hook for "this user's
+   * credentials changed". Idle entries are closed now; leased ones leave the
+   * pool now and are closed when released. A creation in flight for a matching
+   * key is marked stale, so its result is closed and rebuilt instead of cached.
+   */
+  evictWhere(predicate: (key: string) => boolean): void {
+    for (const [key, entry] of [...this.entries]) {
+      if (predicate(key)) this.retire(entry);
+    }
+    for (const [key, record] of this.pending) {
+      if (predicate(key)) record.stale = true;
+    }
+  }
+
+  /** Evict and close everything (leased entries close on release). */
+  closeAll(): void {
+    this.evictWhere(() => true);
+  }
+
+  /** Pooled (resolved) entries; diagnostics and tests. Does not sweep. */
+  size(): number {
+    return this.entries.size;
+  }
+
+  private async awaitPending(key: string, record: Pending<V>, create: () => Promise<V>): Promise<Lease<V>> {
+    const value = await record.promise;
+    // The creator owns closing a stale value; a joiner just asks again.
+    if (record.stale) return this.acquire(key, create);
+    // The creator pooled the value before this continuation ran. If it has
+    // already left the pool again, lease whatever the pool holds now.
+    const entry = this.entries.get(key);
+    if (!entry || entry.value !== value) return this.acquire(key, create);
+    return this.lease(entry);
+  }
+
+  private lease(entry: Entry<V>): Lease<V> {
+    entry.leases += 1;
+    entry.lastUsedAt = this.now();
+    let released = false;
+    return {
+      value: entry.value,
+      release: () => {
+        if (released) return;
+        released = true;
+        entry.leases -= 1;
+        // Idleness is measured from the end of the last use.
+        entry.lastUsedAt = this.now();
+        if (entry.retired && entry.leases === 0) this.disposeQuietly(entry.key, entry.value);
+        // An insert that found every entry leased could not evict and let the
+        // pool overflow the cap. The overflow has to end with the leases, not
+        // with whenever the next acquire happens to run — so the release that
+        // ends an entry's last lease re-applies the cap. The entry just
+        // released is the newest by `lastUsedAt`, so older idle entries go
+        // first.
+        if (entry.leases === 0) this.enforceCap();
+      },
+    };
+  }
+
+  private insert(key: string, value: V): Entry<V> {
+    this.sweep();
+    this.enforceCap(1);
+    const entry: Entry<V> = { key, value, lastUsedAt: this.now(), leases: 0, retired: false };
+    this.entries.set(key, entry);
+    return entry;
+  }
+
+  /**
+   * Evict idle entries, least recently used first, until at most
+   * `maxEntries - reserve` remain — or nothing idle is left, in which case
+   * the pool stays over the cap until a lease is released (which calls this
+   * again). `reserve` is the room an insert needs for the entry it is about
+   * to add.
+   */
+  private enforceCap(reserve = 0): void {
+    while (this.entries.size + reserve > this.maxEntries && this.evictLeastRecentlyUsed()) {
+      // each pass evicted one idle entry
+    }
+  }
+
+  private sweep(): void {
+    const cutoff = this.now() - this.idleTtlMs;
+    for (const entry of [...this.entries.values()]) {
+      if (entry.leases === 0 && entry.lastUsedAt < cutoff) this.retire(entry);
+    }
+  }
+
+  /** Evict the least-recently-used entry not in use; false when every entry is leased. */
+  private evictLeastRecentlyUsed(): boolean {
+    let oldest: Entry<V> | undefined;
+    for (const entry of this.entries.values()) {
+      if (entry.leases > 0) continue;
+      if (!oldest || entry.lastUsedAt < oldest.lastUsedAt) oldest = entry;
+    }
+    if (!oldest) return false;
+    this.retire(oldest);
+    return true;
+  }
+
+  /** Take an entry out of the pool; close it now, or on its last release when leased. */
+  private retire(entry: Entry<V>): void {
+    if (this.entries.get(entry.key) === entry) this.entries.delete(entry.key);
+    entry.retired = true;
+    if (entry.leases === 0) this.disposeQuietly(entry.key, entry.value);
+  }
+
+  /**
+   * Fire-and-forget close: a slow downstream teardown must not block the
+   * request that happened to trigger the sweep, and a failing one must not
+   * stop the rest of the sweep.
+   */
+  private disposeQuietly(key: string, value: V): void {
+    void Promise.resolve()
+      .then(() => this.dispose(value))
+      .catch((err) => {
+        log.warn(`closing pooled downstream connection failed (${poolKeyLabel(key)}):`, { err });
+      });
+  }
+}
+
+/** A log-safe rendering of a pool key: user and manual, never the template fingerprint. */
+function poolKeyLabel(key: string): string {
+  const [userId, manual] = key.split(POOL_KEY_SEPARATOR);
+  return `user=${userId ?? '?'} manual=${manual ?? '?'}`;
+}
+
+/** Separator for (user, manual, fingerprint) keys — cannot occur in any of the parts. */
+export const POOL_KEY_SEPARATOR = '\u0000';

package/src/modules/mcp/downstream-token-refresh.ts

@@ -0,0 +1,170 @@
+/**
+ * Refresh-and-retry for proxied third-party MCP servers (`mcp.json` manuals)
+ * that reject the caller's OAuth token.
+ *
+ * The vault refreshes a token when its STORED expiry says so. A provider can
+ * disagree — a token revoked early, an expiry the provider never stated, a
+ * clock that drifted — and then the downstream answers 401 / `invalid_token`
+ * while the vault still believes the token is good. The proxy reacts to that
+ * rejection: one forced refresh for (user, manual), one retry of the call, and
+ * the caller sees only the final outcome.
+ *
+ * Two pieces live here, both pure of the proxy so they are testable alone: the
+ * rejection classifier, and the guard that allows one refresh per (user,
+ * manual) per minute so a broken grant can never loop the provider.
+ */
+
+/** How far up a `cause` chain to look before giving up. */
+const MAX_CAUSE_DEPTH = 8;
+
+/** The minimum spacing between two forced refreshes of one (user, manual). */
+export const DOWNSTREAM_REFRESH_WINDOW_MS = 60_000;
+
+/**
+ * Did the downstream reject the credential itself — 401 or an `invalid_token`
+ * error — as opposed to anything else that can go wrong with a call?
+ *
+ * Two shapes reach here. A failed CALL keeps the MCP SDK's
+ * `StreamableHTTPError`, whose `code` is the HTTP status. A failed
+ * REGISTRATION is flattened to prose by `@utcp/mcp`
+ * (`Server 'srv': Streamable HTTP error: Error POSTing to endpoint: <body>`),
+ * where the status is gone and only the response body is left to read.
+ *
+ * 403 is deliberately NOT a rejection: it means the token is valid but not
+ * permitted, and a refreshed token carries the same permissions. An error that
+ * STATES its HTTP status is believed over its prose, so a 403 whose body says
+ * `invalid_token` (or any other non-401 status that mentions one) is not
+ * refreshed: the status is the server's verdict, the message only its wording.
+ */
+export function isDownstreamTokenRejection(err: unknown): boolean {
+  const seen = new Set<unknown>();
+  let current = err;
+  for (let depth = 0; depth < MAX_CAUSE_DEPTH && current != null && !seen.has(current); depth += 1) {
+    seen.add(current);
+    if (typeof current === 'string') return saysTokenRejected(current);
+    if (typeof current !== 'object') return false;
+    const e = current as Record<string, unknown>;
+    // A known status decides on its own — and ends the walk, so a 403's own
+    // body can't be re-read as a rejection one link further up the chain.
+    const status = httpStatusOf(e);
+    if (status !== undefined) return status === 401;
+    if (typeof e.message === 'string' && saysTokenRejected(e.message)) return true;
+    current = e.cause;
+  }
+  return false;
+}
+
+/**
+ * The HTTP status this error carries, if it carries one at all — the MCP SDK's
+ * `StreamableHTTPError.code`, a `status`/`statusCode`, or a nested response's.
+ *
+ * `code` doubles as a JSON-RPC code in the SDK (negative, e.g. `-32001`) and
+ * as a syscall string in Node (`'ECONNRESET'`), so only a number inside the
+ * HTTP range counts as a status; anything else leaves the status unknown and
+ * the message is read instead.
+ */
+function httpStatusOf(e: Record<string, unknown>): number | undefined {
+  const asStatus = (v: unknown): number | undefined =>
+    typeof v === 'number' && Number.isInteger(v) && v >= 100 && v <= 599 ? v : undefined;
+  for (const field of ['status', 'statusCode', 'code'] as const) {
+    const status = asStatus(e[field]);
+    if (status !== undefined) return status;
+  }
+  return asStatus((e.response as { status?: unknown } | undefined)?.status);
+}
+
+function saysTokenRejected(message: string): boolean {
+  const m = message.toLowerCase();
+  // A status number inside a URL (`/v1/401/…`, a port) proves nothing.
+  const scrubbed = m.replace(/\bhttps?:\/\/\S+/g, ' ');
+  // `unauthorized` as a word (the 401 reason phrase, an `error` value), not
+  // as a substring: `unauthorized_client` is an OAuth error about the CLIENT
+  // registration, which a refreshed token would not change.
+  return (
+    /\b401\b/.test(scrubbed) ||
+    m.includes('invalid_token') ||
+    m.includes('invalid token') ||
+    /\bunauthorized\b/.test(m)
+  );
+}
+
+/**
+ * At most one refresh per key per window, single-flight within it.
+ *
+ * The first rejection in a window runs the refresh; rejections that arrive
+ * WHILE it runs (a burst of calls failing against the same dead token) share
+ * its outcome instead of being refused, so a burst doesn't turn into one
+ * success and N-1 failures. A rejection after the refresh settled but inside
+ * the window gets `undefined` — no refresh, the call's own error stands —
+ * which is what bounds a grant the provider keeps refreshing but the
+ * downstream keeps refusing to one refresh a minute.
+ */
+export class DownstreamRefreshGuard<T> {
+  private readonly attempts = new Map<
+    string,
+    { startedAt: number; outcome: Promise<T>; settled: boolean; value?: T }
+  >();
+
+  constructor(
+    private readonly windowMs: number = DOWNSTREAM_REFRESH_WINDOW_MS,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  /** Run `refresh` for `key` unless the window forbids it; `undefined` when it does. */
+  run(key: string, refresh: () => Promise<T>): Promise<T> | undefined {
+    const at = this.now();
+    this.prune(at);
+    const existing = this.attempts.get(key);
+    if (existing) return existing.settled ? undefined : existing.outcome;
+    const record: { startedAt: number; outcome: Promise<T>; settled: boolean; value?: T } = {
+      startedAt: at,
+      outcome: Promise.resolve() as Promise<unknown> as Promise<T>,
+      settled: false,
+    };
+    record.outcome = (async () => {
+      try {
+        const value = await refresh();
+        record.value = value;
+        return value;
+      } finally {
+        record.settled = true;
+      }
+    })();
+    this.attempts.set(key, record);
+    return record.outcome;
+  }
+
+  /**
+   * Forget the attempts `predicate` selects — the key, and the outcome it
+   * settled on (`undefined` while it is still running).
+   *
+   * For when something OUTSIDE this guard changed the credential the window was
+   * protecting, so the window no longer describes anything real.
+   */
+  clearWhere(predicate: (key: string, outcome: T | undefined) => boolean): void {
+    for (const [key, record] of this.attempts) {
+      if (predicate(key, record.value)) this.attempts.delete(key);
+    }
+  }
+
+  /**
+   * Forget attempts whose window is over, so the map is bounded by recent
+   * (user, manual) pairs.
+   *
+   * Age alone decides, NOT settlement: a refresh that never settles — a token
+   * request that hangs with no timeout — would otherwise pin its key forever,
+   * so no later rejection for that (user, manual) could ever refresh again and
+   * the entry could never be evicted. Once the window has passed, a new attempt
+   * is within policy whether the previous one finished or not.
+   *
+   * A clock that moved BACKWARD (an NTP step) makes every age negative; such a
+   * window is forgotten too, since otherwise it would hold for the step plus a
+   * minute rather than the minute the policy promises.
+   */
+  private prune(at: number): void {
+    for (const [key, record] of this.attempts) {
+      const age = at - record.startedAt;
+      if (age < 0 || age >= this.windowMs) this.attempts.delete(key);
+    }
+  }
+}