deepline 0.3.184 → 0.3.186

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -2,12 +2,16 @@ import {
   acquireRateStateViaAppRuntime,
   penalizeRateStateViaAppRuntime,
   releaseRateStateViaAppRuntime,
+  RateStateValidationUnavailableError,
+  validateRateStateViaAppRuntime,
   type WorkerRuntimeApiContext,
 } from '../app-runtime-api';
 import { randomUUID } from 'node:crypto';
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
   noopPacingPermit,
+  PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingPermit,
   type ProviderConcurrencyGrant,
   type ProviderPacingGrant,
@@ -19,19 +23,18 @@ import {
   ProviderExhaustedError,
   providerNameFromBucketId,
 } from '../run-failure';
+import type { PlayRunnerRateStateValidateResult } from '../protocol';
 
 /**
- * Longest provider pacer hold this substrate will sleep toward before failing
- * the call fast with a typed {@link ProviderExhaustedError}. Duration alone
- * decides — there is no quota-vs-rate classification. A hold at or under this is
- * slept off (today's behavior); a hold above it is skipped with no spend and no
- * provider dispatch.
+ * Longest provider pacer hold one executor will wait through in-process.
+ * Longer holds throw a typed handoff before another provider dispatch. Managed
+ * durable Plays catch that signal and park on a persisted sleep; callers without
+ * durable boundaries receive the typed provider-exhausted outcome.
  *
- * Enforced ONLY on the `app_runtime_postgres` (Absurd/Node DB-authoritative)
- * backend; the in-memory and coordinator backends never construct this class and
- * so keep their sleep-toward-it behavior byte-for-byte.
+ * Enforced only by the `app_runtime_postgres` backend. In-memory and coordinator
+ * backends do not construct this class and retain their existing wait behavior.
  */
-export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 60_000;
+export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 300_000;
 /**
  * Total time one acquirer may spend cycling through window-exhausted refills.
  * Each refill's sleep is bounded, but nothing bounded how many of them a row
@@ -39,6 +42,7 @@ export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 60_000;
  * starter parked here for twenty minutes with its core idle.
  */
 export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
+const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
 
 /**
  * Permits leased per app-runtime round trip for the direct Node substrate.
@@ -62,6 +66,9 @@ export interface AppRuntimeRatePort {
   penalize(
     input: Parameters<typeof penalizeRateStateViaAppRuntime>[1],
   ): ReturnType<typeof penalizeRateStateViaAppRuntime>;
+  validate?(
+    input: Parameters<typeof validateRateStateViaAppRuntime>[1],
+  ): ReturnType<typeof validateRateStateViaAppRuntime>;
 }
 
 function contextRatePort(context: WorkerRuntimeApiContext): AppRuntimeRatePort {
@@ -69,6 +76,7 @@ function contextRatePort(context: WorkerRuntimeApiContext): AppRuntimeRatePort {
     acquire: (input) => acquireRateStateViaAppRuntime(context, input),
     release: (input) => releaseRateStateViaAppRuntime(context, input),
     penalize: (input) => penalizeRateStateViaAppRuntime(context, input),
+    validate: (input) => validateRateStateViaAppRuntime(context, input),
   };
 }
 
@@ -113,6 +121,7 @@ interface LeasedBlock {
       reservationId: string | null,
       concurrencyGrant: ProviderConcurrencyGrant | null,
       pacingGrant: ProviderPacingGrant | null,
+      scopeRetryAtMsAtGrant: number,
     ]
   >;
   rateScopeToken: string | null;
@@ -130,6 +139,12 @@ type PendingConcurrencyObservations = {
   observations: Array<{ kind: 'success'; latencyMs: number }>;
 };
 
+type SuccessFeedback = {
+  observedSuccesses: number;
+  concurrencyObservations: PendingConcurrencyObservations['observations'];
+  concurrencyObservationBatch?: PendingConcurrencyObservations['batch'];
+};
+
 interface PendingRelease {
   bucketId: string;
   rateScopeToken: string | null;
@@ -171,6 +186,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly blocks = new Map<string, LeasedBlock>();
   private readonly refills = new Map<string, Promise<void>>();
   private readonly pendingReleases = new Map<string, PendingRelease>();
+  private readonly validationFlights = new Map<
+    string,
+    { generation: number; promise: Promise<PlayRunnerRateStateValidateResult> }
+  >();
+  private readonly validationGenerations = new Map<string, number>();
+  private readonly validationScopes = new Set<string>();
+  private readonly pendingPenaltyWrites = new Map<string, Promise<void>>();
+  private readonly serverClockOffsetsMs = new Map<string, number>();
+  private readonly localProviderRetryAtMs = new Map<string, number>();
   /**
    * Last locally synthesized dispatch slot per lane. During a rolling deploy,
    * a gateway that omits its global schedule can return another block while
@@ -186,8 +210,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly waiters = new Map<string, number>();
   /** Last-known server cooldown per bucket (surfaced for PROVIDER_EXHAUSTED). */
   private readonly cooldowns = new Map<string, RateStateCooldown>();
+  private readonly providerStalls = new Map<
+    string,
+    { startedAt: number; warned: boolean }
+  >();
   private readonly effectiveRps = new Map<string, number>();
   private readonly pendingSuccesses = new Map<string, number>();
+  private readonly successFeedbackFlights = new Map<string, SuccessFeedback>();
   /**
    * Completion samples travel with the next lease refill. This keeps the
    * Gradient2 checkpoint current without adding one gateway request per row.
@@ -230,6 +259,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     return this.cooldowns.get(bucketId) ?? null;
   }
 
+  private serverNow(bucketId: string): number {
+    return this.now() + (this.serverClockOffsetsMs.get(bucketId) ?? 0);
+  }
+
   /**
    * Pre-check the last-known server cooldown for a bucket. If the pacer is
    * already holding this provider past PROVIDER_EXHAUSTED_MAX_WAIT_MS, throw a
@@ -238,35 +271,77 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    * later than the internally-capped `coolUntilMs` and the cooldown is active;
    * otherwise reports `coolUntilMs`. No-op when the bucket has no live hold.
    */
-  private throwIfPacerExhausted(bucketId: string): void {
-    const exhausted = this.pacerExhaustion(bucketId);
+  private throwIfPacerExhausted(
+    bucketId: string,
+    yieldProviderHolds = false,
+  ): void {
+    const exhausted = this.pacerExhaustion(bucketId, yieldProviderHolds);
     if (exhausted) throw exhausted;
+    const stall = this.providerStalls.get(bucketId);
+    if (!stall) return;
+    const elapsedMs = this.now() - stall.startedAt;
+    const cooldown = this.cooldowns.get(bucketId);
+    const serverNowMs = this.serverNow(bucketId);
+    const retryAtMs = Math.max(
+      cooldown?.coolUntilMs ?? 0,
+      cooldown?.claimedRetryAtMs ?? 0,
+    );
+    const retryAtActive = retryAtMs > serverNowMs;
+    if (!retryAtActive && elapsedMs >= RATE_ACQUIRE_MAX_WAIT_MS) {
+      // Keep the no-progress timer across short Retry-After cycles so its
+      // warning remains useful. Once its five-minute guard has been spent and
+      // the last hold is over, however, reset it before re-admitting; otherwise
+      // this process would turn every later run into retry-now exhaustion.
+      this.providerStalls.delete(bucketId);
+      return;
+    }
+    if (
+      retryAtActive &&
+      (elapsedMs >= RATE_ACQUIRE_MAX_WAIT_MS ||
+        retryAtMs - serverNowMs > RATE_ACQUIRE_MAX_WAIT_MS - elapsedMs)
+    ) {
+      throw new ProviderExhaustedError({
+        provider: providerNameFromBucketId(bucketId),
+        retryAtMs,
+      });
+    }
+    if (!stall.warned && elapsedMs >= PROVIDER_NO_PROGRESS_WARNING_MS) {
+      stall.warned = true;
+      console.warn('[runtime.provider.no_progress]', {
+        provider: providerNameFromBucketId(bucketId),
+        elapsedMs,
+        retryAt: retryAtActive ? new Date(retryAtMs).toISOString() : null,
+      });
+    }
   }
 
-  private pacerExhaustion(bucketId: string): ProviderExhaustedError | null {
+  private pacerExhaustion(
+    bucketId: string,
+    yieldProviderHolds = false,
+  ): ProviderExhaustedError | null {
     const cooldown = this.cooldowns.get(bucketId);
     if (!cooldown) return null;
-    const now = this.now();
+    const now = this.serverNow(bucketId);
     const coolActive = cooldown.coolUntilMs - now > 0;
-    if (!coolActive) return null;
-    const coolWait = cooldown.coolUntilMs - now;
     const claimActive = cooldown.claimedRetryAtMs - now > 0;
-    // Duration alone decides. Neither hold is over threshold → let it sleep.
+    if (!coolActive && !claimActive) return null;
+    const coolWait = coolActive ? cooldown.coolUntilMs - now : 0;
+    const claimWait = claimActive ? cooldown.claimedRetryAtMs - now : 0;
+    // Duration alone decides. Either active hold can be the effective
+    // deadline; in particular, Retry-After can outlive the capped AIMD hold.
     if (
+      !yieldProviderHolds &&
       coolWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS &&
-      !(
-        claimActive &&
-        cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS
-      )
+      claimWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS
     ) {
       return null;
     }
-    // Retry deadline: the server Retry-After verbatim when present, else the
-    // pacer's own cooldown deadline.
-    const retryAtMs =
-      cooldown.claimedRetryAtMs > 0
-        ? cooldown.claimedRetryAtMs
-        : cooldown.coolUntilMs;
+    // Report the furthest active hold, even after the internally capped AIMD
+    // deadline has elapsed.
+    const retryAtMs = Math.max(
+      coolActive ? cooldown.coolUntilMs : 0,
+      claimActive ? cooldown.claimedRetryAtMs : 0,
+    );
     return new ProviderExhaustedError({
       provider: providerNameFromBucketId(bucketId),
       retryAtMs,
@@ -287,7 +362,9 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     }
     const laneKey = rateLaneKey(
       bucketId,
-      rulesSignature([...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId))),
+      rulesSignature(
+        [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId)),
+      ),
     );
     const block = this.liveBlock(bucketId, laneKey);
     if (!block) return null;
@@ -302,6 +379,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
+    yieldProviderHolds?: boolean;
   }): Promise<PacingPermit> {
     const { bucketId, rules, signal } = input;
     if (rules.length === 0) return noopPacingPermit();
@@ -309,7 +387,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     // Pre-check: zero round trips when the pacer is already holding past the
     // threshold. Fails fast with a typed PROVIDER_EXHAUSTED before touching the
     // store.
-    this.throwIfPacerExhausted(bucketId);
+    this.throwIfPacerExhausted(bucketId, input.yieldProviderHolds === true);
 
     const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
     const rulesKey = rulesSignature(ordered);
@@ -325,6 +403,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     // serialized every map row behind one sleeping acquirer.
     const acquireStartedAt = this.now();
     while (true) {
+      this.throwIfPacerExhausted(bucketId, input.yieldProviderHolds === true);
       if (signal?.aborted) {
         throw signal.reason instanceof Error
           ? signal.reason
@@ -361,6 +440,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           rulesKey,
           laneKey,
           signal,
+          input.yieldProviderHolds === true,
         );
       } finally {
         const remaining = (this.waiters.get(laneKey) ?? 1) - 1;
@@ -417,7 +497,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         Math.min(waiters, Math.trunc(rateBudget), ...concurrencyLimits),
       );
     }
-    return Math.max(floor, Math.min(waiters, Math.trunc(rateBudget)));
+    const learnedFloor = this.effectiveRps.has(laneKey)
+      ? Math.min(floor, rateBudget)
+      : floor;
+    return Math.max(learnedFloor, Math.min(waiters, Math.trunc(rateBudget)));
   }
 
   /**
@@ -437,21 +520,119 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   /** Store the server's advisory cooldown for a bucket if the response carried one. */
   private recordCooldown(
     bucketId: string,
-    response: { coolUntilMs?: number; claimedRetryAtMs?: number },
+    response: {
+      coolUntilMs?: number;
+      claimedRetryAtMs?: number;
+      serverNowMs?: number;
+    },
   ): void {
+    if (response.serverNowMs != null && Number.isFinite(response.serverNowMs)) {
+      this.serverClockOffsetsMs.set(
+        bucketId,
+        response.serverNowMs - this.now(),
+      );
+    }
     const coolUntilMs = Number(response.coolUntilMs);
     const claimedRetryAtMs = Number(response.claimedRetryAtMs);
+    const now = this.serverNow(bucketId);
     if (!Number.isFinite(coolUntilMs) && !Number.isFinite(claimedRetryAtMs)) {
       return;
     }
+    const existing = this.cooldowns.get(bucketId);
     this.cooldowns.set(bucketId, {
-      coolUntilMs: Number.isFinite(coolUntilMs) ? Math.max(0, coolUntilMs) : 0,
-      claimedRetryAtMs: Number.isFinite(claimedRetryAtMs)
-        ? Math.max(0, claimedRetryAtMs)
-        : 0,
+      coolUntilMs: Math.max(
+        existing && existing.coolUntilMs > now ? existing.coolUntilMs : 0,
+        Number.isFinite(coolUntilMs) ? Math.max(0, coolUntilMs) : 0,
+      ),
+      claimedRetryAtMs: Math.max(
+        existing && existing.claimedRetryAtMs > now
+          ? existing.claimedRetryAtMs
+          : 0,
+        Number.isFinite(claimedRetryAtMs) && claimedRetryAtMs > now
+          ? Math.max(0, claimedRetryAtMs)
+          : 0,
+      ),
     });
   }
 
+  /**
+   * Revalidate a cached ticket against the shared provider hold. Concurrent
+   * rows in this runner share one in-flight read per provider bucket; completed
+   * reads are never cached. A local 429 invalidates that flight synchronously,
+   * and its write is awaited before any stale validator can accept a ticket.
+   */
+  private async validateProviderPermit(
+    bucketId: string,
+    rateScopeToken: string | null,
+    scopeRetryAtMsAtGrant: number,
+  ): Promise<boolean> {
+    const validate = this.port.validate;
+    if (!validate || !this.validationScopes.has(bucketId)) return true;
+
+    while (true) {
+      const pendingPenalty = this.pendingPenaltyWrites.get(bucketId);
+      if (pendingPenalty) await pendingPenalty;
+
+      // This is a database-clock deadline, not a worker-clock deadline.
+      // Retain the local fence even if persisting the 429 failed.
+      const localRetryAtMs = this.localProviderRetryAtMs.get(bucketId) ?? 0;
+      if (localRetryAtMs > this.serverNow(bucketId)) {
+        this.drainBlocksForBucket(bucketId);
+        return false;
+      }
+
+      const generation = this.validationGenerations.get(bucketId) ?? 0;
+      let flight = this.validationFlights.get(bucketId);
+      if (!flight || flight.generation !== generation) {
+        const promise = Promise.resolve()
+          .then(() =>
+            this.port.validate!({
+              bucketId,
+              rateScopeToken,
+              schedulerSchema: this.schedulerSchema,
+            }),
+          )
+          .finally(() => {
+            if (this.validationFlights.get(bucketId)?.promise === promise) {
+              this.validationFlights.delete(bucketId);
+            }
+          });
+        flight = { generation, promise };
+        this.validationFlights.set(bucketId, flight);
+      }
+
+      const result = await flight.promise;
+      if ((this.validationGenerations.get(bucketId) ?? 0) !== generation) {
+        continue;
+      }
+      if (!result.allowed) {
+        const retryAtMs = result.retryAtMs;
+        this.localProviderRetryAtMs.set(bucketId, retryAtMs);
+        this.drainBlocksForBucket(bucketId);
+        this.recordCooldown(bucketId, {
+          coolUntilMs: Math.min(
+            retryAtMs,
+            result.serverNowMs + PROVIDER_AIMD_COOLDOWN_CAP_MS,
+          ),
+          claimedRetryAtMs: retryAtMs,
+          serverNowMs: result.serverNowMs,
+        });
+        return false;
+      }
+      if (localRetryAtMs <= result.serverNowMs) {
+        this.localProviderRetryAtMs.delete(bucketId);
+      }
+      // The hold's deadline is monotonic. Even after it expires, a ticket
+      // minted before that hold belongs to the old pace and must be replaced.
+      // Preserve newer tickets another waiter may already have acquired.
+      if (result.retryAtMs > scopeRetryAtMsAtGrant) {
+        this.drainBlocksForBucket(bucketId, result.retryAtMs);
+        return false;
+      }
+      return true;
+    }
+  }
+
   /**
    * Build a PROVIDER_EXHAUSTED error for a granted=0 refill response. Retry
    * deadline: the server Retry-After (`claimedRetryAtMs`) verbatim when present,
@@ -465,14 +646,16 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       claimedRetryAtMs?: number;
     },
   ): ProviderExhaustedError {
+    const now = this.serverNow(bucketId);
     const claimedRetryAtMs = Number(response.claimedRetryAtMs);
     const coolUntilMs = Number(response.coolUntilMs);
     const retryAtMs =
-      Number.isFinite(claimedRetryAtMs) && claimedRetryAtMs > 0
-        ? claimedRetryAtMs
-        : Number.isFinite(coolUntilMs) && coolUntilMs > 0
-          ? coolUntilMs
-          : this.now() + Math.max(0, response.waitMs);
+      Math.max(
+        Number.isFinite(claimedRetryAtMs) && claimedRetryAtMs > now
+          ? claimedRetryAtMs
+          : 0,
+        Number.isFinite(coolUntilMs) && coolUntilMs > now ? coolUntilMs : 0,
+      ) || now + Math.max(0, response.waitMs);
     return new ProviderExhaustedError({
       provider: providerNameFromBucketId(bucketId),
       retryAtMs,
@@ -493,8 +676,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rulesKey: string,
     laneKey: string,
     signal?: AbortSignal,
+    yieldProviderHolds = false,
   ): Promise<void> {
-    const existing = this.refills.get(laneKey);
+    const refillKey = yieldProviderHolds ? `${laneKey}:yield` : laneKey;
+    const existing = this.refills.get(refillKey);
     if (existing) return existing;
     const promise = this.performRefill(
       bucketId,
@@ -503,12 +688,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       rulesKey,
       laneKey,
       signal,
+      yieldProviderHolds,
     ).finally(() => {
-      if (this.refills.get(laneKey) === promise) {
-        this.refills.delete(laneKey);
+      if (this.refills.get(refillKey) === promise) {
+        this.refills.delete(refillKey);
       }
     });
-    this.refills.set(laneKey, promise);
+    this.refills.set(refillKey, promise);
     return promise;
   }
 
@@ -519,6 +705,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rulesKey: string,
     laneKey: string,
     signal?: AbortSignal,
+    yieldProviderHolds = false,
   ): Promise<void> {
     let consecutiveFailures = 0;
     // Yield once before sizing the first request so the whole synchronously-
@@ -539,71 +726,30 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       }
       try {
         const successKey = pendingSuccessKey(bucketId, ordered);
-        const observedSuccesses = this.pendingSuccesses.get(successKey) ?? 0;
-        const pendingConcurrency =
-          this.pendingConcurrencyObservations.get(successKey);
-        const concurrencyObservations = [
-          ...(pendingConcurrency?.observations ?? []),
-        ];
+        const feedback = this.snapshotSuccessFeedback(successKey);
         const response = await this.port.acquire({
           bucketId,
           rateScopeToken,
-          // Stamp adaptiveMaxRps on default-pacing rules so the DB row's ceiling
-          // matches the old in-memory AIMD range.
           rules: withAdaptiveMaxRps(ordered),
           requested: this.requestedForBucket(laneKey, ordered),
-          observedSuccesses,
-          ...(pendingConcurrency
-            ? {
-                concurrencyObservations,
-                concurrencyObservationBatch: pendingConcurrency.batch,
-              }
-            : {}),
+          ...feedback,
           schedulerSchema: this.schedulerSchema,
         });
-        if (observedSuccesses > 0) {
-          const remaining = Math.max(
-            0,
-            (this.pendingSuccesses.get(successKey) ?? 0) - observedSuccesses,
-          );
-          if (remaining > 0) this.pendingSuccesses.set(successKey, remaining);
-          else this.pendingSuccesses.delete(successKey);
-        }
-        if (pendingConcurrency) {
-          const pending = this.pendingConcurrencyObservations.get(successKey);
-          const remainingSuccesses = this.pendingSuccesses.get(successKey) ?? 0;
-          if (
-            pending &&
-            pending.batch.sequence === pendingConcurrency.batch.sequence
-          ) {
-            const remaining = pending.observations.slice(
-              concurrencyObservations.length,
-            );
-            if (remainingSuccesses > 0 || remaining.length > 0) {
-              // Observations that arrived while this acquire was in flight need
-              // a new identity. Reusing the acknowledged sequence would make
-              // the gateway correctly discard their rate and Gradient2 input.
-              this.pendingConcurrencyObservations.set(
-                successKey,
-                this.newPendingConcurrencyObservations(successKey, remaining),
-              );
-            } else {
-              this.pendingConcurrencyObservations.delete(successKey);
-            }
-          } else if (
-            pending &&
-            pending.observations.length > concurrencyObservations.length
-          ) {
-            const remaining = pending.observations.slice(
-              concurrencyObservations.length,
-            );
-            this.pendingConcurrencyObservations.set(
-              successKey,
-              this.newPendingConcurrencyObservations(successKey, remaining),
-            );
-          }
-        }
+        this.acknowledgeSuccessFeedback(successKey, feedback);
         this.recordCooldown(bucketId, response);
+        if (response.scopeValidationSupported === true)
+          this.validationScopes.add(bucketId);
+        if (
+          response.granted === 0 &&
+          (response.claimedRetryAtMs ?? 0) >
+            (response.serverNowMs ?? this.now()) &&
+          !this.providerStalls.has(bucketId)
+        ) {
+          this.providerStalls.set(bucketId, {
+            startedAt: this.now(),
+            warned: false,
+          });
+        }
         if (
           Number.isFinite(response.effectiveRequestsPerSecond) &&
           Number(response.effectiveRequestsPerSecond) > 0
@@ -626,6 +772,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           );
           return;
         }
+        if (yieldProviderHolds) {
+          const activeHold = this.pacerExhaustion(bucketId, true);
+          if (activeHold) throw activeHold;
+        }
         // Zero grants without a finite positive wait cannot make progress.
         // Older gateways return this for an empty/invalid normalized rule set;
         // fail exhausted instead of spinning once per millisecond forever.
@@ -635,10 +785,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
             waitMs: 0,
           });
         }
-        // Live path: the server told us to wait past the threshold. Fail fast
-        // with a typed PROVIDER_EXHAUSTED instead of sleeping toward it. The
-        // rejection propagates to every waiter parked on this single-flight
-        // refill (they all skip with no spend). Duration alone decides.
+        // Do not hold a sandbox for a multi-minute provider deadline. Managed
+        // Plays turn this typed hold into a durable sleep; non-durable callers
+        // still receive the typed error. Every waiter on this single-flight
+        // refill sees the same deadline without dispatching provider work.
         if (response.waitMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS) {
           throw this.providerExhaustedFromResponse(bucketId, response);
         }
@@ -650,9 +800,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         await this.sleep(this.jitteredWaitMs(response.waitMs));
         return;
       } catch (error) {
-        // PROVIDER_EXHAUSTED is a deliberate skip, not a store failure: it must
-        // escape this fail-closed retry ladder verbatim so it reaches the
-        // acquire caller (and, via step-miss handling, the row).
+        // A provider hold is not a store failure. Preserve its deadline for
+        // the caller's durable suspension or non-durable error path.
         if (error instanceof ProviderExhaustedError) throw error;
         // Fail closed. A degraded rate-state store must not grant unlimited
         // provider calls; after this bounded retry ladder the run fails loudly.
@@ -691,6 +840,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       effectiveRequestsPerSecond?: unknown;
       serverNowMs?: unknown;
       scheduledAtMs?: unknown;
+      claimedRetryAtMs?: number;
     },
   ): void {
     const hasConcurrency = ordered.some((rule) => rule.maxConcurrency != null);
@@ -767,9 +917,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     );
     const receivedAtMs = this.now();
     const hasGatewaySchedule = Array.isArray(response.scheduledAtMs);
-    const gatewayScheduledAtMs: number[] = Array.isArray(
-      response.scheduledAtMs,
-    )
+    const gatewayScheduledAtMs: number[] = Array.isArray(response.scheduledAtMs)
       ? response.scheduledAtMs.map((scheduledAtMs) => Number(scheduledAtMs))
       : Array.from(
           { length: response.granted },
@@ -790,7 +938,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
               receivedAtMs +
               Math.max(0, firstGatewayScheduledAtMs - serverNowMs) +
               (scheduledAtMs - firstGatewayScheduledAtMs),
-        )
+          )
         : gatewayScheduledAtMs;
     const previousFallbackScheduledAtMs =
       this.nextFallbackScheduledAtMs.get(laneKey);
@@ -891,21 +1039,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       bucketId,
       rateScopeToken,
       scheduledAtMs.map(
-        (
-          scheduledAtMs,
-          index,
-        ): [
-          number,
-          string | null,
-          string | null,
-          ProviderConcurrencyGrant | null,
-          ProviderPacingGrant | null,
-        ] => [
+        (scheduledAtMs, index): LeasedBlock['permits'][number] => [
           scheduledAtMs,
           hasConcurrency ? (leaseIds[index] ?? null) : null,
           reservationIds[index] ?? null,
           concurrencyGrants[index] ?? null,
           pacingGrant,
+          Math.max(0, response.claimedRetryAtMs ?? 0),
         ],
       ),
       rulesKey,
@@ -918,24 +1058,78 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     bucketId: string;
     rateScopeToken?: string | null;
     cooldownMs: number;
+    retryAtMs?: number;
     rules?: readonly PacingRule[];
     latencyMs?: number | null;
     concurrencyGrant?: ProviderConcurrencyGrant | null;
   }): Promise<void> {
-    if (input.cooldownMs <= 0) return;
+    const cooldownMs =
+      Number.isFinite(input.cooldownMs) && input.cooldownMs > 0
+        ? Math.ceil(input.cooldownMs)
+        : 0;
+    const explicitRetryAtMs =
+      typeof input.retryAtMs === 'number' && Number.isFinite(input.retryAtMs)
+        ? Math.max(0, Math.ceil(input.retryAtMs))
+        : null;
+    const hasExplicitRetryAtMs = explicitRetryAtMs !== null;
+    const retryAtMs =
+      explicitRetryAtMs ?? this.serverNow(input.bucketId) + cooldownMs;
+    // Even an HTTP-date that expired during response delivery is fresh rate
+    // feedback. The durable store deduplicates it against the last absolute
+    // deadline while still learning from the 429; only synthesized, already
+    // elapsed cooldowns have no new provider evidence to persist.
+    if (retryAtMs <= this.serverNow(input.bucketId) && !hasExplicitRetryAtMs) {
+      return;
+    }
+    if (!this.providerStalls.has(input.bucketId)) {
+      this.providerStalls.set(input.bucketId, {
+        startedAt: this.now(),
+        warned: false,
+      });
+    }
+
+    this.validationGenerations.set(
+      input.bucketId,
+      (this.validationGenerations.get(input.bucketId) ?? 0) + 1,
+    );
+    this.validationFlights.delete(input.bucketId);
+    this.localProviderRetryAtMs.set(
+      input.bucketId,
+      Math.max(this.localProviderRetryAtMs.get(input.bucketId) ?? 0, retryAtMs),
+    );
+    this.recordCooldown(input.bucketId, {
+      coolUntilMs: Math.min(
+        retryAtMs,
+        this.serverNow(input.bucketId) + PROVIDER_AIMD_COOLDOWN_CAP_MS,
+      ),
+      claimedRetryAtMs: retryAtMs,
+    });
     this.drainBlocksForBucket(input.bucketId);
+    const successKey = pendingSuccessKey(input.bucketId, input.rules ?? []);
+    const feedback = this.snapshotSuccessFeedback(successKey);
+    const previousPenalty = this.pendingPenaltyWrites.get(input.bucketId);
+    const penaltyWrite = Promise.all([
+      previousPenalty,
+      Promise.resolve().then(async () => {
+        await this.port.penalize({
+          bucketId: input.bucketId,
+          rateScopeToken: input.rateScopeToken?.trim() || null,
+          cooldownMs,
+          retryAtMs,
+          ...feedback,
+          ...(input.rules ? { rules: [...input.rules] } : {}),
+          ...(input.latencyMs != null ? { latencyMs: input.latencyMs } : {}),
+          ...(input.concurrencyGrant
+            ? { concurrencyGrant: input.concurrencyGrant }
+            : {}),
+          schedulerSchema: this.schedulerSchema,
+        });
+        this.acknowledgeSuccessFeedback(successKey, feedback);
+      }),
+    ]).then(() => undefined);
+    this.pendingPenaltyWrites.set(input.bucketId, penaltyWrite);
     try {
-      await this.port.penalize({
-        bucketId: input.bucketId,
-        rateScopeToken: input.rateScopeToken?.trim() || null,
-        cooldownMs: input.cooldownMs,
-        ...(input.rules ? { rules: [...input.rules] } : {}),
-        ...(input.latencyMs != null ? { latencyMs: input.latencyMs } : {}),
-        ...(input.concurrencyGrant
-          ? { concurrencyGrant: input.concurrencyGrant }
-          : {}),
-        schedulerSchema: this.schedulerSchema,
-      });
+      await penaltyWrite;
     } catch (error) {
       const normalized = normalizeError(error);
       this.onStoreFailure({
@@ -945,6 +1139,38 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       throw new Error(
         `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
       );
+    } finally {
+      if (this.pendingPenaltyWrites.get(input.bucketId) === penaltyWrite) {
+        this.pendingPenaltyWrites.delete(input.bucketId);
+      }
+    }
+  }
+
+  private snapshotSuccessFeedback(key: string): SuccessFeedback {
+    const inFlight = this.successFeedbackFlights.get(key);
+    if (inFlight) return inFlight;
+    const pending = this.pendingConcurrencyObservations.get(key);
+    const feedback: SuccessFeedback = {
+      observedSuccesses: this.pendingSuccesses.get(key) ?? 0,
+      concurrencyObservations: [...(pending?.observations ?? [])],
+      ...(pending ? { concurrencyObservationBatch: pending.batch } : {}),
+    };
+    if (pending) {
+      // Freeze the payload until acknowledged. A concurrent acquire and 429
+      // may send it, but cannot send different counts under one identity.
+      this.pendingSuccesses.delete(key);
+      this.pendingConcurrencyObservations.delete(key);
+      this.successFeedbackFlights.set(key, feedback);
+    }
+    return feedback;
+  }
+
+  private acknowledgeSuccessFeedback(
+    key: string,
+    feedback: SuccessFeedback,
+  ): void {
+    if (this.successFeedbackFlights.get(key) === feedback) {
+      this.successFeedbackFlights.delete(key);
     }
   }
 
@@ -953,6 +1179,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rules: readonly PacingRule[];
     latencyMs?: number | null;
   }): void {
+    this.providerStalls.delete(input.bucketId);
     const key = pendingSuccessKey(input.bucketId, input.rules);
     this.pendingSuccesses.set(
       key,
@@ -1014,7 +1241,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     scheduled: LeasedBlock['permits'][number],
   ): PacingPermit {
     const { rateScopeToken, rules } = block;
-    const [, leaseId, reservationId, concurrencyGrant, pacingGrant] = scheduled;
+    const [
+      ,
+      leaseId,
+      reservationId,
+      concurrencyGrant,
+      pacingGrant,
+      scopeRetryAtMsAtGrant,
+    ] = scheduled;
     let settled = false;
     const refund = async (): Promise<void> => {
       if (!reservationId) {
@@ -1033,6 +1267,31 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     return {
       concurrencyGrant,
       pacingGrant,
+      gatewayDispatchProof: rateScopeToken
+        ? {
+            bucketId,
+            rateScopeToken,
+            retryAtMsAtGrant: scopeRetryAtMsAtGrant,
+          }
+        : null,
+      validate: async () => {
+        try {
+          return await this.validateProviderPermit(
+            bucketId,
+            rateScopeToken,
+            scopeRetryAtMsAtGrant,
+          );
+        } catch (error) {
+          if (error instanceof RateStateValidationUnavailableError) {
+            throw error;
+          }
+          const normalized = normalizeError(error);
+          this.onStoreFailure({ bucketId, error: normalized.message });
+          throw new Error(
+            `Rate-state permit validation failed for ${bucketId}: ${normalized.message}`,
+          );
+        }
+      },
       release: () => {
         if (settled) return;
         settled = true;
@@ -1054,7 +1313,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         // Only the block this permit was drawn from may take it back: a lane
         // whose block expired, was drained by a penalty, or was replaced has
         // already given up its grant, and the permit must be refunded instead.
-        if (this.blocks.get(laneKey) === block && block.expiresAt > this.now()) {
+        if (
+          this.blocks.get(laneKey) === block &&
+          block.expiresAt > this.now()
+        ) {
           block.permits.push(scheduled);
           block.permits.sort((a, b) => a[0] - b[0]);
           return;
@@ -1140,17 +1402,31 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     };
   }
 
-  private drainBlocksForBucket(bucketId: string): void {
+  private drainBlocksForBucket(
+    bucketId: string,
+    beforeRetryAtMs?: number,
+  ): void {
     const prefix = `${bucketId}\0`;
     for (const [laneKey, block] of this.blocks) {
       if (!laneKey.startsWith(prefix)) continue;
-      this.blocks.delete(laneKey);
-      this.nextFallbackScheduledAtMs.delete(laneKey);
+      const drained =
+        beforeRetryAtMs === undefined
+          ? block.permits
+          : block.permits.filter((permit) => permit[5] < beforeRetryAtMs);
+      if (drained.length === 0 && beforeRetryAtMs !== undefined) continue;
+      block.permits =
+        beforeRetryAtMs === undefined
+          ? []
+          : block.permits.filter((permit) => permit[5] >= beforeRetryAtMs);
+      if (block.permits.length === 0) {
+        this.blocks.delete(laneKey);
+        this.nextFallbackScheduledAtMs.delete(laneKey);
+      }
       this.releaseReserved(
         bucketId,
         block.rateScopeToken,
         block.rules,
-        block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
+        drained.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
       );
     }
   }
@@ -1266,17 +1542,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
  * advisory maximum instead of treating a recommendation as a hard ceiling.
  */
 function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
-  return rules.map((rule) =>
-    rule.adaptiveMaxRps != null
-      ? rule
-      : ({
-          ...rule,
-          adaptiveMaxRps: Math.max(
-            (rule.requestsPerWindow / rule.windowMs) * 1_000,
-            DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
-          ),
-        } as PacingRule),
-  );
+  return rules.map((rule) => {
+    // This one shared default rule is synthesized for unknown providers whose
+    // credential bucket spans operations. Keep the generic policy unchanged
+    // for local/coordinator backends, but let Postgres start a new provider
+    // scope with one probe/sec before learning its sustainable rate.
+    const seed =
+      rule.ruleId === 'default:provider-scope'
+        ? {
+            ...rule,
+            requestsPerWindow: DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+            windowMs: 1_000,
+          }
+        : rule;
+    if (seed.adaptiveMaxRps != null) return seed;
+    return {
+      ...seed,
+      adaptiveMaxRps: Math.max(
+        (rule.requestsPerWindow / rule.windowMs) * 1_000,
+        DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
+      ),
+    } as PacingRule;
+  });
 }
 
 function rulesSignature(rules: readonly PacingRule[]): string {