@oh-my-pi/pi-ai 18.13.0 → 18.13.2

package/src/auth/refresh.ts

@@ -1,4 +1,4 @@
-import { logger, postmortem } from "@oh-my-pi/pi-utils";
+import { isSqliteBusyError, logger, postmortem } from "@oh-my-pi/pi-utils";
 import * as AIError from "../error";
 import { getOAuthProvider, normalizeOAuthCredentialExpiry, refreshOAuthToken } from "../registry/oauth";
 import type { OAuthCredentials, OAuthProvider } from "../registry/oauth/types";
@@ -6,7 +6,12 @@ import type { Provider } from "../types";
 import { raceSignal } from "./abort";
 import { authCredentialEquals, type CredentialPool, credentialDisabledEvent } from "./pool";
 import { resolveCredentialIdentityKey, serializeCredential } from "./sqlite-credential-store";
-import { type AuthCredentialStore, type CredentialRefreshLeaseStore, hasRefreshLeases } from "./store";
+import {
+	type AuthCredentialStore,
+	type CredentialRefreshLeaseHolder,
+	type CredentialRefreshLeaseStore,
+	hasRefreshLeases,
+} from "./store";
 import {
 	REMOTE_REFRESH_SENTINEL,
 	type AuthCredentialSnapshotEntry,
@@ -33,13 +38,16 @@ import {
  */
 export const OAUTH_REFRESH_SKEW_MS = 60_000;
 /**
- * How long auth recovery may reuse a token this refresher actually minted.
+ * How long auth recovery may reuse a token minted moments ago — by this
+ * refresher or, through the shared store, by another process.
  * Re-minting a token that a provider rejected moments ago only rotates the
- * refresh token and churns broker snapshots during provider-wide outages.
+ * refresh token, revokes the access token every peer holds, and churns broker
+ * snapshots during provider-wide outages.
  */
 const OAUTH_REMINT_COOLDOWN_MS = 5 * 60_000;
 const OAUTH_REFRESH_LEASE_TTL_MS = 15_000;
 const OAUTH_REFRESH_LEASE_POLL_MS = 50;
+const OAUTH_REFRESH_LEASE_WAIT_MAX_MS = 250;
 const OAUTH_REFRESH_LEASE_RENEW_MS = 5_000;
 /** How long a caller waits for an OAuth token exchange before giving up. */
 const OAUTH_REFRESH_OPERATION_TIMEOUT_MS = 10_000;
@@ -53,6 +61,31 @@ const DEFAULT_OAUTH_REFRESH_TIMEOUT_MS = 10_000;
  * peers waiting on the lease still recover within a minute.
  */
 const OAUTH_REFRESH_EXCHANGE_BACKSTOP_MS = 60_000;
+/**
+ * How long a refresher keeps retrying a busy store to save a token it already
+ * exchanged. The provider has consumed the old refresh token by then, so
+ * giving up loses the login for every process sharing the store.
+ */
+const OAUTH_REFRESH_PERSIST_TIMEOUT_MS = 60_000;
+const OAUTH_REFRESH_PERSIST_RETRY_MAX_MS = 1_000;
+/**
+ * How long after another refresher's lease lapsed that refresher may still be
+ * saving a token it exchanged: its exchange backstop plus its save window.
+ */
+const OAUTH_REFRESH_STALLED_OWNER_MS = OAUTH_REFRESH_EXCHANGE_BACKSTOP_MS + OAUTH_REFRESH_PERSIST_TIMEOUT_MS;
+/**
+ * How long process exit waits for refreshes still in flight: an exchange up to
+ * its backstop, then the save window, plus slack for the last save attempt. A
+ * shorter wait drops a token the provider already rotated, which leaves the
+ * shared store with a spent refresh token.
+ */
+const OAUTH_REFRESH_SHUTDOWN_WAIT_MS = OAUTH_REFRESH_STALLED_OWNER_MS + 5_000;
+
+/** Short non-secret fingerprint so logs can tell which process minted which token. */
+function tokenFingerprint(token: unknown): string {
+	if (typeof token !== "string" || token.length === 0) return "none";
+	return new Bun.CryptoHasher("sha256").update(token).digest("hex").slice(0, 8);
+}
 
 type LeasedExchangeOutcome<T extends OAuthCredential> = { result: StoredOAuthRefreshResult<T> } | { failure: unknown };
 
@@ -114,6 +147,11 @@ export class OAuthRefresher {
 	#oauthCredentialRefreshInFlight: Map<number, Promise<OAuthCredentials>> = new Map();
 	/** Access token this process last minted per credential id. */
 	#recentMints = new Map<number, { provider: string; access: string; at: number }>();
+	/**
+	 * Refresh tokens rejected after taking over a stalled refresher's lease, per
+	 * credential id: that refresher may still save their successor until `until`.
+	 */
+	#stalledRefreshes = new Map<number, { refresh: string; until: number }>();
 	#pendingRefreshes = new Set<Promise<unknown>>();
 	#cancelRefreshCleanup?: () => void;
 	#shuttingDown = false;
@@ -161,6 +199,154 @@ export class OAuthRefresher {
 			: this.#deps.store.refreshOAuthCredential === undefined;
 	}
 
+	/**
+	 * Whether the stored row holds a token the caller has not seen rejected, so
+	 * it adopts that token instead of exchanging again. A forced refresh
+	 * compares tokens only (callers pass an `expires: 0` copy), and compares
+	 * against the access token a provider 401 rejected when the caller names
+	 * it: this process may already hold the row a peer rotated, while the token
+	 * that got the 401 is the one before it. Callers name the credential's own
+	 * access token, never the API key built from it, which a provider with a
+	 * structured key format makes a different string.
+	 */
+	#storedIsNewer<T extends OAuthCredential>(current: T, options: StoredOAuthRefreshOptions<T>): boolean {
+		if (options.rejectedAccess !== undefined) return current.access !== options.rejectedAccess;
+		const observed = options.observedCredential;
+		if (!observed) return false;
+		return options.forceRefresh
+			? current.access !== observed.access || current.refresh !== observed.refresh
+			: !authCredentialEquals(current, observed);
+	}
+
+	/**
+	 * Hand a taken-over lease back to the refresher that stalled holding it. A busy
+	 * store is retried within the save window: if this process released the lease
+	 * instead, the next refresher would find no owner and disable a login that
+	 * owner may still save. Returns false when the hand-back never landed; the
+	 * caller then keeps its own lease to expire, which still names a recent owner.
+	 */
+	async #handBackLease(
+		leaseStore: CredentialRefreshLeaseStore,
+		credentialId: number,
+		owner: string,
+		holder: CredentialRefreshLeaseHolder,
+	): Promise<boolean> {
+		const deadline = Date.now() + OAUTH_REFRESH_PERSIST_TIMEOUT_MS;
+		for (let attempt = 0; ; attempt++) {
+			try {
+				return leaseStore.handBackCredentialRefreshLease?.(credentialId, owner, holder) ?? false;
+			} catch (error) {
+				if (!isSqliteBusyError(error)) throw error;
+				if (Date.now() >= deadline) return false;
+			}
+			await Bun.sleep(Math.min(OAUTH_REFRESH_PERSIST_RETRY_MAX_MS, OAUTH_REFRESH_LEASE_POLL_MS * 2 ** attempt));
+		}
+	}
+
+	/**
+	 * Give up a lease this refresher holds without having saved a token. A lease
+	 * taken over from a stalled refresher goes back to it, because that refresher
+	 * may still be saving: releasing it would leave the next refresher no owner,
+	 * and it would disable a login that may live. Any other lease is released.
+	 */
+	async #leaveLease(
+		leaseStore: CredentialRefreshLeaseStore,
+		credentialId: number,
+		owner: string,
+		takenOverFrom: CredentialRefreshLeaseHolder | undefined,
+	): Promise<void> {
+		if (!takenOverFrom) {
+			leaseStore.releaseCredentialRefreshLease(credentialId, owner);
+			return;
+		}
+		try {
+			await this.#handBackLease(leaseStore, credentialId, owner, takenOverFrom);
+		} catch (error) {
+			// The lease then expires still naming this refresher, a recent holder.
+			logger.warn("OAuth refresh could not hand a taken-over lease back", { credentialId, error: String(error) });
+		}
+	}
+
+	/**
+	 * Save a token this process already exchanged. The provider consumed the old
+	 * refresh token, so a busy store is retried (re-arming the lease each time:
+	 * a blocked event loop can outlive its TTL, and an expired lease no peer
+	 * took is still ours) instead of dropping the only valid token. A refused
+	 * save is retried too, within the same 60 s window, while the row still
+	 * holds the spent token: a peer that took over the lapsed lease hands it
+	 * back once the provider rejects that token, but the next peer can hold it
+	 * at the moment this save lands.
+	 */
+	async #persistExchangedCredential(
+		provider: string,
+		credentialId: number,
+		expectedData: string,
+		credential: OAuthCredential,
+		lease: { store: CredentialRefreshLeaseStore; owner: string } | undefined,
+		fingerprints: Record<string, unknown>,
+	): Promise<boolean> {
+		const deadline = Date.now() + OAUTH_REFRESH_PERSIST_TIMEOUT_MS;
+		let refused = false;
+		for (let attempt = 0; ; attempt++) {
+			if (attempt > 0 && Date.now() >= deadline) {
+				if (refused) {
+					logger.warn("OAuth exchanged token not saved before the save deadline", fingerprints);
+					return false;
+				}
+				throw new AIError.ConfigurationError("OAuth exchanged token could not be saved: store stayed busy");
+			}
+			let renewed = true;
+			try {
+				if (lease) {
+					renewed = lease.store.renewCredentialRefreshLease(
+						credentialId,
+						lease.owner,
+						Date.now() + OAUTH_REFRESH_LEASE_TTL_MS,
+					);
+					// An expired lease no peer took is still ours: re-acquire it rather
+					// than fail the fence and drop a token the provider already minted.
+					if (!renewed) {
+						renewed = lease.store.tryAcquireCredentialRefreshLease(
+							credentialId,
+							lease.owner,
+							Date.now() + OAUTH_REFRESH_LEASE_TTL_MS,
+						);
+					}
+				}
+				const saved =
+					this.#deps.store.tryUpdateAuthCredentialIfMatches?.(
+						credentialId,
+						expectedData,
+						credential,
+						lease ? { owner: lease.owner, nowMs: Date.now() } : undefined,
+					) ?? false;
+				if (saved) return true;
+				// A peer that saved first changed the row; otherwise no peer can hold a
+				// usable successor of the spent token, so wait for the lease instead.
+				const stored = this.#deps.store.listAuthCredentials(provider).find(entry => entry.id === credentialId);
+				if (!stored || serializeCredential(provider, stored.credential)?.data !== expectedData) {
+					logger.warn("OAuth exchanged token not saved", { ...fingerprints, leaseHeld: renewed });
+					return false;
+				}
+				if (!refused) {
+					logger.warn("OAuth exchanged token save refused while a peer holds the lease; retrying", {
+						...fingerprints,
+						leaseHeld: renewed,
+					});
+				}
+				refused = true;
+			} catch (error) {
+				if (!isSqliteBusyError(error)) {
+					logger.error("OAuth exchanged token could not be saved", { ...fingerprints, error: String(error) });
+					throw error;
+				}
+				refused = false;
+				logger.warn("OAuth exchanged token save hit a busy store; retrying", { ...fingerprints, attempt });
+			}
+			await Bun.sleep(Math.min(OAUTH_REFRESH_PERSIST_RETRY_MAX_MS, OAUTH_REFRESH_LEASE_POLL_MS * 2 ** attempt));
+		}
+	}
+
 	/**
 	 * Refresh one stored OAuth credential under durable row ownership.
 	 */
@@ -174,10 +360,10 @@ export class OAuthRefresher {
 
 	/** Keeps shutdown waiting until `pending` settles, so rotated tokens are persisted before exit. */
 	#track<R>(pending: Promise<R>): Promise<R> {
-		this.#cancelRefreshCleanup ??= postmortem.register("oauth-refresh", () => this.#drainRefreshes(), {
-			exitOnly: true,
-		});
+		// Add before registering: a registration made while exit cleanup runs drains at
+		// once, and must find this work pending or the process exits without it.
 		this.#pendingRefreshes.add(pending);
+		this.#armShutdownDrain();
 		const settled = () => {
 			this.#pendingRefreshes.delete(pending);
 			if (this.#pendingRefreshes.size === 0 && !this.#shuttingDown) {
@@ -189,9 +375,23 @@ export class OAuthRefresher {
 		return pending;
 	}
 
+	/**
+	 * Registers the shutdown drain if it is not registered yet. A registration made
+	 * once exit has begun drains at once, which sets `#shuttingDown`.
+	 */
+	#armShutdownDrain(): void {
+		this.#cancelRefreshCleanup ??= postmortem.register("oauth-refresh", () => this.#drainRefreshes(), {
+			exitOnly: true,
+			deadlineMs: OAUTH_REFRESH_SHUTDOWN_WAIT_MS,
+			// A quit that waits in silence looks hung, and killing it loses the token anyway.
+			waitMessage: "Waiting for the login refresh to finish saving",
+		});
+	}
+
 	async #drainRefreshes(): Promise<void> {
 		this.#shuttingDown = true;
-		await Promise.allSettled(this.#pendingRefreshes);
+		// Work tracked while draining (a lease hand-back a returning caller left behind) is waited for too.
+		while (this.#pendingRefreshes.size > 0) await Promise.allSettled([...this.#pendingRefreshes]);
 	}
 
 	async #refreshStored<T extends OAuthCredential = OAuthCredential>(
@@ -202,6 +402,7 @@ export class OAuthRefresher {
 		const leaseStore = hasRefreshLeases(this.#deps.store) ? this.#deps.store : undefined;
 		const owner = crypto.randomUUID();
 		let leasedCredentialId: number | undefined;
+		let takenOverFrom: CredentialRefreshLeaseHolder | undefined;
 		const budgetMs = options.refreshTimeoutMs ?? OAUTH_REFRESH_OPERATION_TIMEOUT_MS;
 		// Waiting on a peer's lease counts against the same budget as the exchange.
 		const waitDeadline = Date.now() + budgetMs;
@@ -231,11 +432,7 @@ export class OAuthRefresher {
 			// Adopt the stored copy only when it is still usable; a stored copy that
 			// is itself expired must fall through to a refresh rather than be handed
 			// back and fail the downstream `getOAuthApiKey` expiry precondition.
-			if (
-				options.observedCredential &&
-				!authCredentialEquals(current, options.observedCredential) &&
-				currentIsFresh
-			) {
+			if (currentIsFresh && this.#storedIsNewer(current, options)) {
 				return { credential: current, refreshed: false, removed: false };
 			}
 			if (!options.forceRefresh && currentIsFresh) {
@@ -246,15 +443,46 @@ export class OAuthRefresher {
 			}
 			// A caller whose budget ran out must not start an exchange it will not wait for.
 			if (Date.now() >= waitDeadline) return this.#budgetExceeded(provider, options, current);
+			// This process already had this refresh token rejected after taking over
+			// a stalled refresher, which may still save its successor: wait for that
+			// save instead of replaying the token at the provider.
+			const stalled = this.#stalledRefreshes.get(row.id);
+			if (stalled) {
+				const stalledMs = stalled.until - Date.now();
+				if (stalled.refresh === current.refresh && stalledMs > 0) {
+					await raceSignal(
+						Bun.sleep(Math.min(stalledMs, waitDeadline - Date.now(), OAUTH_REFRESH_LEASE_WAIT_MAX_MS)),
+						options.signal,
+						"OAuth refresh wait for a stalled refresher aborted by caller",
+					);
+					continue;
+				}
+				this.#stalledRefreshes.delete(row.id);
+			}
+			// A lease row still naming another owner means that owner stalled or died
+			// holding it. It may have spent the stored refresh token already and still
+			// be retrying the save of its successor, so remember it: a rejection of
+			// that token is then not proof the login is dead.
+			const priorHolder = leaseStore.getCredentialRefreshLeaseHolder?.(row.id);
 			if (leaseStore.tryAcquireCredentialRefreshLease(row.id, owner, Date.now() + OAUTH_REFRESH_LEASE_TTL_MS)) {
 				leasedCredentialId = row.id;
+				if (
+					priorHolder &&
+					priorHolder.owner !== owner &&
+					Date.now() - priorHolder.expiresAtMs < OAUTH_REFRESH_STALLED_OWNER_MS
+				) {
+					takenOverFrom = priorHolder;
+				}
 				break;
 			}
 			const leaseExpiresAt = leaseStore.getCredentialRefreshLeaseExpiresAt(row.id);
 			const waitMs =
 				leaseExpiresAt === undefined
 					? OAUTH_REFRESH_LEASE_POLL_MS
-					: Math.min(Math.max(leaseExpiresAt - Date.now(), OAUTH_REFRESH_LEASE_POLL_MS), 250);
+					: Math.min(
+							Math.max(leaseExpiresAt - Date.now(), OAUTH_REFRESH_LEASE_POLL_MS),
+							OAUTH_REFRESH_LEASE_WAIT_MAX_MS,
+						);
 			const remainingMs = waitDeadline - Date.now();
 			if (remainingMs <= 0) return this.#budgetExceeded(provider, options, current);
 			await raceSignal(
@@ -283,13 +511,9 @@ export class OAuthRefresher {
 				return { credential: undefined, refreshed: false, removed: false };
 			}
 			const currentIsFresh = Date.now() + refreshSkewMs < current.expires;
-			// Re-check after acquiring the lease: only adopt the stored copy on an
-			// observed mismatch when it is still usable, mirroring the pre-lease guard.
-			if (
-				options.observedCredential &&
-				!authCredentialEquals(current, options.observedCredential) &&
-				currentIsFresh
-			) {
+			// Re-check after acquiring the lease: a peer that held it may have just
+			// rotated the row, mirroring the pre-lease guard.
+			if (currentIsFresh && this.#storedIsNewer(current, options)) {
 				return { credential: current, refreshed: false, removed: false };
 			}
 			if (!options.forceRefresh && currentIsFresh) {
@@ -298,6 +522,9 @@ export class OAuthRefresher {
 			if (options.canRefresh && !options.canRefresh(current)) {
 				return { credential: current, refreshed: false, removed: false };
 			}
+			// Arm the drain first: once exit has begun, arming it sets `#shuttingDown`, so a
+			// refresh that starts then stops here instead of spending a token nobody saves.
+			this.#armShutdownDrain();
 			if (this.#shuttingDown || options.signal?.aborted) {
 				throw new AIError.AbortError("OAuth refresh stopped before token exchange");
 			}
@@ -319,6 +546,7 @@ export class OAuthRefresher {
 					leaseStore,
 					leasedCredentialId,
 					owner,
+					takenOverFrom,
 				),
 			);
 			// Lease release now belongs to the exchange.
@@ -336,7 +564,9 @@ export class OAuthRefresher {
 			return this.#refreshFailed(options, current, outcome.failure);
 		} finally {
 			if (leasedCredentialId !== undefined && leaseStore) {
-				leaseStore.releaseCredentialRefreshLease(leasedCredentialId, owner);
+				// A hand-back retries a busy store for the save window; the caller, which is
+				// leaving with its budget spent or its signal aborted, does not wait for it.
+				void this.#track(this.#leaveLease(leaseStore, leasedCredentialId, owner, takenOverFrom));
 			}
 		}
 	}
@@ -372,8 +602,9 @@ export class OAuthRefresher {
 
 	/**
 	 * Runs one token exchange under the row lease and persists the rotated
-	 * credential by CAS, then releases the lease. A non-definitive exchange error
-	 * resolves as `failure` so the waiting caller applies its failure policy.
+	 * credential by CAS, then releases the lease. An exchange error that does not
+	 * disable the login resolves as `failure` so the waiting caller applies its
+	 * failure policy.
 	 */
 	async #exchangeLeased<T extends OAuthCredential>(
 		provider: string,
@@ -384,7 +615,10 @@ export class OAuthRefresher {
 		leaseStore: CredentialRefreshLeaseStore | undefined,
 		leasedCredentialId: number | undefined,
 		owner: string,
+		takenOverFrom: CredentialRefreshLeaseHolder | undefined,
 	): Promise<LeasedExchangeOutcome<T>> {
+		let keepLease = false;
+		let saved = false;
 		try {
 			let stopLeaseRenewal = false;
 			let leaseRenewalError: unknown;
@@ -392,14 +626,35 @@ export class OAuthRefresher {
 			const leaseRenewal =
 				leasedCredentialId !== undefined && leaseStore
 					? (async () => {
+							// Renew every interval; after a busy store, retry almost at once,
+							// since another full interval would outlive the lease.
+							let delayMs = OAUTH_REFRESH_LEASE_RENEW_MS;
 							while (!stopLeaseRenewal) {
-								await Promise.race([Bun.sleep(OAUTH_REFRESH_LEASE_RENEW_MS), leaseRenewalStopped.promise]);
+								await Promise.race([Bun.sleep(delayMs), leaseRenewalStopped.promise]);
 								if (stopLeaseRenewal) return;
-								const renewed = leaseStore.renewCredentialRefreshLease(
-									leasedCredentialId,
-									owner,
-									Date.now() + OAUTH_REFRESH_LEASE_TTL_MS,
-								);
+								let renewed: boolean;
+								try {
+									renewed = leaseStore.renewCredentialRefreshLease(
+										leasedCredentialId,
+										owner,
+										Date.now() + OAUTH_REFRESH_LEASE_TTL_MS,
+									);
+									// An expired lease no peer took is still ours.
+									if (!renewed) {
+										renewed = leaseStore.tryAcquireCredentialRefreshLease(
+											leasedCredentialId,
+											owner,
+											Date.now() + OAUTH_REFRESH_LEASE_TTL_MS,
+										);
+									}
+								} catch (error) {
+									// A busy store only delays this renewal; ownership is lost
+									// only once another owner holds the lease row.
+									if (!isSqliteBusyError(error)) throw error;
+									delayMs = OAUTH_REFRESH_LEASE_POLL_MS;
+									continue;
+								}
+								delayMs = OAUTH_REFRESH_LEASE_RENEW_MS;
 								if (!renewed) {
 									throw new AIError.ConfigurationError("OAuth refresh ownership was lost before persistence");
 								}
@@ -420,10 +675,9 @@ export class OAuthRefresher {
 					// Stop waiting even when the refresh ignores its signal, so the lease is released.
 					backstopReached.reject(error);
 				},
-				Math.max(
-					options.refreshTimeoutMs ?? OAUTH_REFRESH_OPERATION_TIMEOUT_MS,
-					OAUTH_REFRESH_EXCHANGE_BACKSTOP_MS,
-				),
+				// A fixed ceiling, whatever the caller's budget: a peer that takes over a
+				// lapsed lease, and shutdown, both assume no exchange runs longer.
+				OAUTH_REFRESH_EXCHANGE_BACKSTOP_MS,
 			);
 
 			let refreshed: OAuthCredentials;
@@ -431,6 +685,45 @@ export class OAuthRefresher {
 				refreshed = await Promise.race([options.refresh(current, refreshAbort.signal), backstopReached.promise]);
 			} catch (error) {
 				if (!options.isDefinitiveFailure?.(error)) return { failure: error };
+				if (takenOverFrom && leaseStore && leasedCredentialId !== undefined) {
+					// The owner whose lease lapsed may have spent this token and still be
+					// saving its successor. Give the lease back so it can, and fail this
+					// refresh as transient instead of disabling a login that may live.
+					this.#stalledRefreshes.set(row.id, {
+						refresh: current.refresh,
+						until: takenOverFrom.expiresAtMs + OAUTH_REFRESH_STALLED_OWNER_MS,
+					});
+					// Stop renewing first: a renewal after the hand-back would take it back.
+					stopLeaseRenewal = true;
+					leaseRenewalStopped.resolve();
+					await leaseRenewal;
+					// Never release this lease afterwards: a hand-back that landed left
+					// nothing to release, and one that did not must leave a recent owner.
+					keepLease = true;
+					const handedBack = await this.#handBackLease(leaseStore, leasedCredentialId, owner, takenOverFrom).catch(
+						(error: unknown) => {
+							// The lease then expires still naming this refresher, a recent holder.
+							logger.warn("OAuth refresh could not hand a taken-over lease back", {
+								credentialId: row.id,
+								error: String(error),
+							});
+							return false;
+						},
+					);
+					logger.warn("OAuth refresh token rejected after taking over a stalled refresher; not disabling", {
+						provider,
+						credentialId: row.id,
+						spentRefresh: tokenFingerprint(current.refresh),
+						handedBack,
+					});
+					// A transient failure, so the caller's failure policy applies to it.
+					return {
+						failure: new AIError.OAuthError(
+							`OAuth refresh for ${provider} waits for a stalled refresher that may still save its token`,
+							{ kind: "token-refresh", provider },
+						),
+					};
+				}
 				const disabledCause = options.disabledCause?.(error) ?? `oauth refresh failed: ${String(error)}`;
 				const disabled = this.#deps.store.tryDisableAuthCredentialIfMatches(
 					row.id,
@@ -466,20 +759,36 @@ export class OAuthRefresher {
 				await leaseRenewal;
 				clearTimeout(backstop);
 			}
-			if (leaseRenewalError) throw leaseRenewalError;
-
 			const merged: T = options.mergeRefreshedCredential
 				? options.mergeRefreshedCredential(current, refreshed)
 				: mergeRefreshedCredential(current, refreshed);
+			const fingerprints = {
+				provider,
+				credentialId: row.id,
+				spentRefresh: tokenFingerprint(current.refresh),
+				access: tokenFingerprint(merged.access),
+				refresh: tokenFingerprint(merged.refresh),
+				expires: merged.expires,
+			};
+			logger.info("OAuth token exchange succeeded", fingerprints);
+			if (leaseRenewalError) {
+				logger.warn("OAuth refresh lost its lease before saving the exchanged token", fingerprints);
+				// The provider already consumed the old refresh token. The fenced save
+				// below writes only once the lease is ours again and the row is unchanged.
+				if (!this.#deps.store.tryUpdateAuthCredentialIfMatches) throw leaseRenewalError;
+			}
 			if (this.#deps.store.tryUpdateAuthCredentialIfMatches) {
 				if (
-					!this.#deps.store.tryUpdateAuthCredentialIfMatches(
+					!(await this.#persistExchangedCredential(
+						provider,
 						row.id,
 						serializedData,
 						merged,
-						leasedCredentialId !== undefined ? { owner, nowMs: Date.now() } : undefined,
-					)
+						leasedCredentialId !== undefined && leaseStore ? { store: leaseStore, owner } : undefined,
+						fingerprints,
+					))
 				) {
+					logger.warn("OAuth refresh lost its compare-and-set; exchanged token not saved", fingerprints);
 					await this.#deps.pool.reload();
 					const latest = this.#deps.pool.entries(provider).find(entry => entry.id === row.id)?.credential;
 					return {
@@ -493,6 +802,7 @@ export class OAuthRefresher {
 			} else {
 				this.#deps.store.updateAuthCredential(row.id, merged);
 			}
+			saved = true;
 			if (this.#mintsLocally()) {
 				this.#rememberMint(provider, row.id, merged);
 			}
@@ -505,8 +815,9 @@ export class OAuthRefresher {
 			);
 			return { result: { credential: merged, refreshed: true, removed: false } };
 		} finally {
-			if (leasedCredentialId !== undefined && leaseStore) {
-				leaseStore.releaseCredentialRefreshLease(leasedCredentialId, owner);
+			if (leasedCredentialId !== undefined && leaseStore && !keepLease) {
+				if (saved) leaseStore.releaseCredentialRefreshLease(leasedCredentialId, owner);
+				else await this.#leaveLease(leaseStore, leasedCredentialId, owner, takenOverFrom);
 			}
 		}
 	}
@@ -579,13 +890,15 @@ export class OAuthRefresher {
 		credentialId: number | undefined,
 		signal?: AbortSignal,
 		reason?: OAuthRefreshReason,
+		rejectedAccess?: string,
 	): Promise<OAuthCredentials> {
 		const authRecovery = reason === "auth-recovery";
 		if (authRecovery && credentialId !== undefined && !this.#oauthCredentialRefreshInFlight.has(credentialId)) {
 			const recent = this.#recentMint(credentialId);
-			if (recent) return recent.credential;
+			// A token the provider has just rejected is not a usable recent mint.
+			if (recent && recent.credential.access !== rejectedAccess) return recent.credential;
 		}
-		return this.#refreshSingleFlight(provider, credential, credentialId, signal, reason);
+		return this.#refreshSingleFlight(provider, credential, credentialId, signal, reason, rejectedAccess);
 	}
 
 	/** Refresh without recent-mint reuse; still shares the per-credential in-flight request. */
@@ -595,17 +908,31 @@ export class OAuthRefresher {
 		credentialId: number | undefined,
 		signal?: AbortSignal,
 		reason?: OAuthRefreshReason,
+		rejectedAccess?: string,
 	): Promise<OAuthCredentials> {
 		credential = normalizeOAuthCredentialExpiry(provider, credential);
 		if (credentialId !== undefined) {
 			const existing = this.#oauthCredentialRefreshInFlight.get(credentialId);
-			if (existing) return raceSignal(existing, signal, "credential refresh aborted");
+			if (existing) {
+				const joined = await raceSignal(existing, signal, "credential refresh aborted");
+				// A refresh started for another request may adopt the very token this
+				// request was refused with; refresh again instead of returning it.
+				if (rejectedAccess === undefined || joined.access !== rejectedAccess) return joined;
+				return this.#refreshSingleFlight(provider, credential, credentialId, signal, reason, rejectedAccess);
+			}
 		}
 		if (Date.now() + OAUTH_REFRESH_SKEW_MS < credential.expires) return credential;
 		if (credentialId === undefined) {
-			return this.#refreshOAuthCredentialUnshared(provider, credential, undefined, signal, reason);
+			return this.#refreshOAuthCredentialUnshared(provider, credential, undefined, signal, reason, rejectedAccess);
 		}
-		const promise = this.#refreshOAuthCredentialUnshared(provider, credential, credentialId, signal, reason)
+		const promise = this.#refreshOAuthCredentialUnshared(
+			provider,
+			credential,
+			credentialId,
+			signal,
+			reason,
+			rejectedAccess,
+		)
 			.then(refreshed => {
 				// A delegated refresh may have returned a broker-cached token rather
 				// than minting one. Only direct local requests can establish mint time.
@@ -627,14 +954,18 @@ export class OAuthRefresher {
 		credentialId: number | undefined,
 		signal?: AbortSignal,
 		reason?: OAuthRefreshReason,
+		rejectedAccess?: string,
 	): Promise<OAuthCredentials> {
 		if (credentialId !== undefined && hasRefreshLeases(this.#deps.store)) {
 			const forceRefresh = credential.expires === 0;
 			const result = await this.refreshStored(provider, {
 				credentialId,
-				observedCredential: forceRefresh ? undefined : credential,
+				// A forced refresh still names the tokens it wants replaced: when the
+				// row already holds newer ones, another process refreshed first.
+				observedCredential: credential,
 				credentialFromRow: row => row,
 				forceRefresh,
+				rejectedAccess: reason === "auth-recovery" ? rejectedAccess : undefined,
 				signal,
 				// `refreshStored` owns the caller budget and the exchange backstop.
 				refresh: (current, refreshSignal) =>
@@ -755,14 +1086,30 @@ export class OAuthRefresher {
 		options?: OAuthRefreshByIdOptions,
 	): Promise<AuthCredentialSnapshotEntry> {
 		const existing = this.#oauthRefreshInFlight.get(id);
-		if (existing) return raceSignal(existing, signal, "credential refresh aborted");
+		if (existing) {
+			const joined = await raceSignal(existing, signal, "credential refresh aborted");
+			// A refresh that started earlier can land on the very token this caller's
+			// request was refused with; that is no recovery, so refresh again.
+			if (
+				options?.rejectedAccess === undefined ||
+				joined.credential.type !== "oauth" ||
+				joined.credential.access !== options.rejectedAccess
+			) {
+				return joined;
+			}
+			// Whatever refresh runs now is checked against the rejected token the same way.
+			return this.refreshById(id, signal, options);
+		}
 		const recent = options?.reuseRecentMint ? this.#recentMint(id) : undefined;
-		if (recent) return snapshotEntry(id, recent.provider, recent.credential);
+		// A token the provider has just rejected is not a usable recent mint.
+		if (recent && recent.credential.access !== options?.rejectedAccess) {
+			return snapshotEntry(id, recent.provider, recent.credential);
+		}
 
 		const promise = (async () => {
 			this.#deps.pool.bump("credential-refresh-start");
 			try {
-				return await this.#forceRefreshCredentialByIdUnshared(id, signal, options?.reason);
+				return await this.#forceRefreshCredentialByIdUnshared(id, signal, options?.reason, options?.rejectedAccess);
 			} catch (error) {
 				this.#deps.pool.bump("credential-refresh-failure");
 				throw error;
@@ -778,6 +1125,7 @@ export class OAuthRefresher {
 		id: number,
 		signal?: AbortSignal,
 		reason?: OAuthRefreshReason,
+		rejectedAccess?: string,
 	): Promise<AuthCredentialSnapshotEntry> {
 		for (const provider of this.#deps.pool.providers()) {
 			const entries = this.#deps.pool.entries(provider);
@@ -794,11 +1142,19 @@ export class OAuthRefresher {
 			// value we actually attempted (NOT the expires:0 clone below).
 			const attempted = target.credential;
 			// Pass a clone with expires=0 and bypass recent-mint reuse so scheduled
-			// refreshes and direct force-refresh callers always mint.
+			// refreshes and direct force-refresh callers mint, unless the stored row
+			// already holds a token the provider has not rejected.
 			const stale: OAuthCredential = { ...attempted, expires: 0 };
 			let refreshed: OAuthCredentials;
 			try {
-				refreshed = await this.#refreshSingleFlight(provider as Provider, stale, id, signal, reason);
+				refreshed = await this.#refreshSingleFlight(
+					provider as Provider,
+					stale,
+					id,
+					signal,
+					reason,
+					rejectedAccess,
+				);
 			} catch (error) {
 				// A definitively-dead grant tears the row down here, where the
 				// attempted credential is known. CAS on the persisted credential so a