deepline 0.3.323 → 0.3.325

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -10,11 +10,13 @@ import { randomUUID } from 'node:crypto';
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
   DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS,
   noopPacingPermit,
   PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingPermit,
   type ProviderConcurrencyGrant,
   type ProviderPacingGrant,
+  type ProviderPermitValidationOptions,
   type PacingRule,
   type RateStateBackend,
   type RateStateBackendKind,
@@ -105,7 +107,11 @@ function abortReason(signal: AbortSignal): Error {
     : new Error('Rate-state acquire aborted.');
 }
 
-function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
+function sleepWithAbort(
+  ms: number,
+  signal?: AbortSignal,
+  interrupted?: Promise<void>,
+): Promise<void> {
   if (signal?.aborted) return Promise.reject(abortReason(signal));
   return new Promise((resolve, reject) => {
     const cleanup = () => signal?.removeEventListener('abort', onAbort);
@@ -120,6 +126,11 @@ function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
     };
     signal?.addEventListener('abort', onAbort, { once: true });
     if (signal?.aborted) onAbort();
+    void interrupted?.then(() => {
+      clearTimeout(timer);
+      cleanup();
+      resolve();
+    });
   });
 }
 
@@ -129,6 +140,7 @@ function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
  * hold more than a moment of a shared provider window.
  */
 const RATE_BLOCK_HORIZON_MS = 1_000;
+const UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS = 500;
 const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
 /**
  * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
@@ -154,6 +166,7 @@ interface LeasedBlock {
       concurrencyGrant: ProviderConcurrencyGrant | null,
       pacingGrant: ProviderPacingGrant | null,
       scopeRetryAtMsAtGrant: number,
+      dispatchValidationSupported: boolean,
     ]
   >;
   rateScopeToken: string | null;
@@ -209,7 +222,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly port: AppRuntimeRatePort;
   private readonly schedulerSchema: string | null;
   private readonly now: () => number;
-  private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+  private readonly sleep: (
+    ms: number,
+    signal?: AbortSignal,
+    interrupted?: Promise<void>,
+  ) => Promise<void>;
   private readonly jitter: () => number;
   private readonly onStoreFailure: (info: {
     bucketId: string;
@@ -218,15 +235,33 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly blocks = new Map<string, LeasedBlock>();
   private readonly refills = new Map<string, Promise<void>>();
   private readonly pendingReleases = new Map<string, PendingRelease>();
+  private readonly leaseReleaseSignals = new Map<
+    string,
+    { generation: number; waiters: Set<() => void> }
+  >();
   private readonly validationFlights = new Map<
     string,
     { generation: number; promise: Promise<PlayRunnerRateStateValidateResult> }
   >();
   private readonly validationGenerations = new Map<string, number>();
+  private readonly validatedDispatchCadences = new Map<
+    string,
+    {
+      atMs: number;
+      lastAtMs: number;
+      used: number;
+      burstSize: number;
+      spacingMs: number;
+    }
+  >();
   private readonly validationScopes = new Set<string>();
   private readonly pendingPenaltyWrites = new Map<string, Promise<void>>();
   private readonly serverClockOffsetsMs = new Map<string, number>();
   private readonly localProviderRetryAtMs = new Map<string, number>();
+  private readonly scheduledPermitWaits = new Map<
+    string,
+    Set<{ retryAtMsAtGrant: number; invalidate: () => void }>
+  >();
   /**
    * Last locally synthesized dispatch slot per lane. During a rolling deploy,
    * a gateway that omits its global schedule can return another block while
@@ -240,6 +275,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    * instead of a fixed floor, so a 200-row map fans into ~one round trip.
    */
   private readonly waiters = new Map<string, number>();
+  /** Due retries precede fresh draws; already assigned tickets keep ownership. */
+  private readonly retryAdmissions = new Map<string, Set<Promise<void>>>();
   /** Last-known server cooldown per bucket (surfaced for PROVIDER_EXHAUSTED). */
   private readonly cooldowns = new Map<string, RateStateCooldown>();
   private readonly providerStalls = new Map<
@@ -396,6 +433,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId)),
       ),
     );
+    if (this.retryAdmissions.get(laneKey)?.size) return null;
     const block = this.liveBlock(bucketId, laneKey);
     if (!block) return null;
     const next = block.permits[0];
@@ -410,6 +448,75 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rules: readonly PacingRule[];
     signal?: AbortSignal;
     yieldProviderHolds?: boolean;
+    retryPriority?: boolean;
+  }): Promise<PacingPermit> {
+    if (!input.retryPriority || input.rules.length === 0)
+      return this.acquireFromLane(input);
+    const ordered = [...input.rules].sort((a, b) =>
+      a.ruleId.localeCompare(b.ruleId),
+    );
+    const laneKey = rateLaneKey(input.bucketId, rulesSignature(ordered));
+    let complete!: () => void;
+    const pending = new Promise<void>((resolve) => {
+      complete = resolve;
+    });
+    const retries =
+      this.retryAdmissions.get(laneKey) ?? new Set<Promise<void>>();
+    retries.add(pending);
+    this.retryAdmissions.set(laneKey, retries);
+    try {
+      return await this.acquireFromLane(input);
+    } finally {
+      retries.delete(pending);
+      if (retries.size === 0) this.retryAdmissions.delete(laneKey);
+      complete();
+    }
+  }
+
+  private async waitForRetryAdmissions(
+    laneKey: string,
+    bucketId: string,
+    acquireStartedAt: number,
+    signal?: AbortSignal,
+  ): Promise<void> {
+    const retries = this.retryAdmissions.get(laneKey);
+    if (!retries?.size) return;
+    const remainingMs = Math.max(
+      0,
+      PROVIDER_ADMISSION_WAIT_BUDGET_MS - (this.now() - acquireStartedAt),
+    );
+    await new Promise<void>((resolve, reject) => {
+      const onAbort = () => finish(abortReason(signal!));
+      const timer = setTimeout(
+        () =>
+          finish(
+            new ProviderAdmissionWaitTimeoutError({
+              provider: providerNameFromBucketId(bucketId),
+              waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+            }),
+          ),
+        remainingMs,
+      );
+      timer.unref();
+      const finish = (error?: unknown) => {
+        clearTimeout(timer);
+        signal?.removeEventListener('abort', onAbort);
+        if (error !== undefined) reject(error);
+        else resolve();
+      };
+      signal?.addEventListener('abort', onAbort, { once: true });
+      if (signal?.aborted) onAbort();
+      else void Promise.all([...retries]).then(() => finish());
+    });
+  }
+
+  private async acquireFromLane(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+    rules: readonly PacingRule[];
+    signal?: AbortSignal;
+    yieldProviderHolds?: boolean;
+    retryPriority?: boolean;
   }): Promise<PacingPermit> {
     const { bucketId, rules, signal } = input;
     if (rules.length === 0) return noopPacingPermit();
@@ -439,6 +546,19 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           ? signal.reason
           : new Error('Rate-state acquire aborted.');
       }
+      if (!input.retryPriority && this.retryAdmissions.get(laneKey)?.size) {
+        await this.waitForRetryAdmissions(
+          laneKey,
+          bucketId,
+          acquireStartedAt,
+          signal,
+        );
+        // A new retry may have joined while the previous preferred call drew
+        // its permit. Never let a fresh continuation consume that next grant.
+        if (this.retryAdmissions.get(laneKey)?.size) continue;
+        this.throwIfPacerExhausted(bucketId, input.yieldProviderHolds === true);
+        signal?.throwIfAborted();
+      }
       const localPermit = this.drawFromBlock(bucketId, laneKey);
       if (localPermit) {
         const waitMs = localPermit.scheduledAtMs - this.now();
@@ -458,7 +578,20 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           });
         }
         try {
-          if (waitMs > 0) await this.sleep(waitMs, signal);
+          if (
+            waitMs > 0 &&
+            !(await this.waitForScheduledPermit(
+              bucketId,
+              localPermit.retryAtMsAtGrant,
+              waitMs,
+              signal,
+            ))
+          ) {
+            await this.cancelUndispatchedPermit(localPermit);
+            const pendingPenalty = this.pendingPenaltyWrites.get(bucketId);
+            if (pendingPenalty) await pendingPenalty;
+            continue;
+          }
         } catch (error) {
           await this.cancelUndispatchedPermit(localPermit);
           throw error;
@@ -468,7 +601,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           throw abortReason(signal);
         }
         if (
-          this.now() - acquireStartedAt >= PROVIDER_ADMISSION_WAIT_BUDGET_MS
+          this.now() - acquireStartedAt >=
+          PROVIDER_ADMISSION_WAIT_BUDGET_MS
         ) {
           await this.cancelUndispatchedPermit(localPermit);
           throw new ProviderAdmissionWaitTimeoutError({
@@ -532,12 +666,24 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     // (10 rps declared, 100 rps learned) to ten permits per round trip, which on
     // a hot map is a round trip per permit. The store's own token floor, not
     // this number, is what bounds the grant.
-    const horizonMs = Math.min(RATE_BLOCK_HORIZON_MS, leasedBlockTtlMs(rules));
+    const learnedRps = this.effectiveRps.get(laneKey);
+    const recoveringUnknownProvider =
+      learnedRps !== undefined &&
+      learnedRps < DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS &&
+      rules.some((rule) => rule.ruleId === 'default:provider-scope') &&
+      rules.every((rule) => rule.ruleId.startsWith('default:'));
+    // Below recovered capacity, the next clean observation can double the
+    // pace after 500ms. Do not reserve a full second at the old slower pace:
+    // that schedule debt would delay grants even after learning succeeds.
+    const horizonMs = Math.min(
+      recoveringUnknownProvider
+        ? UNKNOWN_PROVIDER_RECOVERY_BLOCK_HORIZON_MS
+        : RATE_BLOCK_HORIZON_MS,
+      leasedBlockTtlMs(rules),
+    );
     const rateBudget = Math.max(
       1,
-      Math.floor(
-        ((this.effectiveRps.get(laneKey) ?? configuredRps) * horizonMs) / 1_000,
-      ),
+      Math.floor(((learnedRps ?? configuredRps) * horizonMs) / 1_000),
     );
     if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
     const waiters = this.waiters.get(laneKey) ?? 1;
@@ -596,6 +742,16 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       return;
     }
     const existing = this.cooldowns.get(bucketId);
+    // Expiry ends the wait, not the hold's fence. An acquire may discover a
+    // remote throttle after its deadline; wake tickets minted before it now.
+    if (claimedRetryAtMs > (this.localProviderRetryAtMs.get(bucketId) ?? 0)) {
+      this.recordLocalProviderRetryAt(bucketId, claimedRetryAtMs);
+      this.validationGenerations.set(
+        bucketId,
+        (this.validationGenerations.get(bucketId) ?? 0) + 1,
+      );
+      this.validationFlights.delete(bucketId);
+    }
     this.cooldowns.set(bucketId, {
       coolUntilMs: Math.max(
         existing && existing.coolUntilMs > now ? existing.coolUntilMs : 0,
@@ -612,6 +768,50 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     });
   }
 
+  private recordLocalProviderRetryAt(
+    bucketId: string,
+    retryAtMs: number,
+  ): void {
+    if (retryAtMs <= (this.localProviderRetryAtMs.get(bucketId) ?? 0)) return;
+    this.localProviderRetryAtMs.set(bucketId, retryAtMs);
+    for (const waiting of this.scheduledPermitWaits.get(bucketId) ?? []) {
+      if (waiting.retryAtMsAtGrant < retryAtMs) waiting.invalidate();
+    }
+  }
+
+  /** A newer hold invalidates old tickets, including their obsolete sleep. */
+  private async waitForScheduledPermit(
+    bucketId: string,
+    retryAtMsAtGrant: number,
+    waitMs: number,
+    signal?: AbortSignal,
+  ): Promise<boolean> {
+    let invalidated = false;
+    let invalidate!: () => void;
+    const interrupted = new Promise<void>((resolve) => {
+      invalidate = () => {
+        invalidated = true;
+        resolve();
+      };
+    });
+    const waiting = { retryAtMsAtGrant, invalidate };
+    const waiters = this.scheduledPermitWaits.get(bucketId) ?? new Set();
+    waiters.add(waiting);
+    this.scheduledPermitWaits.set(bucketId, waiters);
+    try {
+      if ((this.localProviderRetryAtMs.get(bucketId) ?? 0) > retryAtMsAtGrant)
+        return false;
+      await Promise.race([
+        this.sleep(waitMs, signal, interrupted),
+        interrupted,
+      ]);
+      return !invalidated;
+    } finally {
+      waiters.delete(waiting);
+      if (waiters.size === 0) this.scheduledPermitWaits.delete(bucketId);
+    }
+  }
+
   /**
    * Revalidate a cached ticket against the shared provider hold. Concurrent
    * rows in this runner share one in-flight read per provider bucket; completed
@@ -622,6 +822,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     bucketId: string,
     rateScopeToken: string | null,
     scopeRetryAtMsAtGrant: number,
+    dispatchValidationAtGateway = false,
   ): Promise<boolean> {
     const validate = this.port.validate;
     if (!validate || !this.validationScopes.has(bucketId)) return true;
@@ -638,6 +839,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         return false;
       }
 
+      // Integration execute carries the signed scope and grant snapshot to
+      // this receiver's existing post-queue validator. A preceding worker RPC
+      // only adds another admission round trip; it cannot make that later
+      // check fresher. Direct HTTP and pre-expansion receivers retain the read.
+      if (dispatchValidationAtGateway && rateScopeToken) {
+        if (localRetryAtMs > scopeRetryAtMsAtGrant) {
+          this.drainBlocksForBucket(bucketId, localRetryAtMs);
+          return false;
+        }
+        return true;
+      }
+
       const generation = this.validationGenerations.get(bucketId) ?? 0;
       let flight = this.validationFlights.get(bucketId);
       if (!flight || flight.generation !== generation) {
@@ -664,7 +877,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       }
       if (!result.allowed) {
         const retryAtMs = result.retryAtMs;
-        this.localProviderRetryAtMs.set(bucketId, retryAtMs);
+        if (retryAtMs > (this.localProviderRetryAtMs.get(bucketId) ?? 0)) {
+          this.validationGenerations.set(bucketId, generation + 1);
+          this.validationFlights.delete(bucketId);
+        }
+        this.recordLocalProviderRetryAt(bucketId, retryAtMs);
         this.drainBlocksForBucket(bucketId);
         this.recordCooldown(bucketId, {
           coolUntilMs: Math.min(
@@ -676,13 +893,12 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         });
         return false;
       }
-      if (localRetryAtMs <= result.serverNowMs) {
-        this.localProviderRetryAtMs.delete(bucketId);
-      }
       // The hold's deadline is monotonic. Even after it expires, a ticket
       // minted before that hold belongs to the old pace and must be replaced.
       // Preserve newer tickets another waiter may already have acquired.
       if (result.retryAtMs > scopeRetryAtMsAtGrant) {
+        this.validationGenerations.set(bucketId, generation + 1);
+        this.validationFlights.delete(bucketId);
         this.drainBlocksForBucket(bucketId, result.retryAtMs);
         return false;
       }
@@ -781,7 +997,25 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         await Promise.resolve();
         if (signal?.aborted) return;
       }
+      // A legacy receiver cannot revalidate a grant at dispatch. Persist the
+      // known hold before asking the store for any replacement, including a
+      // newer penalty registered while an earlier one was completing.
+      const pendingPenalty = this.pendingPenaltyWrites.get(bucketId);
+      if (pendingPenalty) {
+        await pendingPenalty;
+        continue;
+      }
       try {
+        let releaseSignal = this.leaseReleaseSignals.get(bucketId);
+        if (
+          !releaseSignal &&
+          ordered.some((rule) => rule.maxConcurrency != null)
+        ) {
+          releaseSignal = { generation: 0, waiters: new Set() };
+          this.leaseReleaseSignals.set(bucketId, releaseSignal);
+        }
+        // A completed release can make an in-flight zero-grant response stale.
+        const releaseGeneration = releaseSignal?.generation;
         const successKey = pendingSuccessKey(bucketId, ordered);
         const feedback = this.snapshotSuccessFeedback(successKey);
         const response = await this.port.acquire({
@@ -854,7 +1088,32 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         // and single-flight another refill if still empty. ±10% jitter so a
         // synchronized wave re-probes at spread-out times; capped by
         // MAX_ACQUIRE_SLEEP_MS.
-        await this.sleep(this.jitteredWaitMs(response.waitMs));
+        const waitMs = this.jitteredWaitMs(response.waitMs);
+        const serverNowMs = response.serverNowMs ?? this.serverNow(bucketId);
+        const authoritativeHold =
+          (response.claimedRetryAtMs ?? 0) > serverNowMs ||
+          (response.coolUntilMs ?? 0) > serverNowMs;
+        if (!releaseSignal || authoritativeHold) {
+          await this.sleep(waitMs);
+        } else {
+          // Lease expiry predicts when a saturated scope can admit again;
+          // our own confirmed release is fresher evidence. Re-query the store,
+          // which still owns rate/concurrency admission and provider holds.
+          let wake!: () => void;
+          const released = new Promise<void>((resolve) => {
+            wake = resolve;
+          });
+          releaseSignal.waiters.add(wake);
+          try {
+            if (releaseSignal.generation !== releaseGeneration) return;
+            await Promise.race([
+              this.sleep(waitMs, undefined, released),
+              released,
+            ]);
+          } finally {
+            releaseSignal.waiters.delete(wake);
+          }
+        }
         return;
       } catch (error) {
         // A provider hold is not a store failure. Preserve its deadline for
@@ -898,6 +1157,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       serverNowMs?: unknown;
       scheduledAtMs?: unknown;
       claimedRetryAtMs?: number;
+      dispatchValidationSupported?: boolean;
     },
   ): void {
     const hasConcurrency = ordered.some((rule) => rule.maxConcurrency != null);
@@ -1103,6 +1363,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           concurrencyGrants[index] ?? null,
           pacingGrant,
           Math.max(0, response.claimedRetryAtMs ?? 0),
+          response.dispatchValidationSupported === true,
         ],
       ),
       rulesKey,
@@ -1145,23 +1406,6 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       });
     }
 
-    this.validationGenerations.set(
-      input.bucketId,
-      (this.validationGenerations.get(input.bucketId) ?? 0) + 1,
-    );
-    this.validationFlights.delete(input.bucketId);
-    this.localProviderRetryAtMs.set(
-      input.bucketId,
-      Math.max(this.localProviderRetryAtMs.get(input.bucketId) ?? 0, retryAtMs),
-    );
-    this.recordCooldown(input.bucketId, {
-      coolUntilMs: Math.min(
-        retryAtMs,
-        this.serverNow(input.bucketId) + PROVIDER_AIMD_COOLDOWN_CAP_MS,
-      ),
-      claimedRetryAtMs: retryAtMs,
-    });
-    this.drainBlocksForBucket(input.bucketId);
     const successKey = pendingSuccessKey(input.bucketId, input.rules ?? []);
     const feedback = this.snapshotSuccessFeedback(successKey);
     const previousPenalty = this.pendingPenaltyWrites.get(input.bucketId);
@@ -1185,6 +1429,21 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       }),
     ]).then(() => undefined);
     this.pendingPenaltyWrites.set(input.bucketId, penaltyWrite);
+    // Register the durable intent before invalidation wakes assigned callers.
+    this.validationGenerations.set(
+      input.bucketId,
+      (this.validationGenerations.get(input.bucketId) ?? 0) + 1,
+    );
+    this.validationFlights.delete(input.bucketId);
+    this.recordLocalProviderRetryAt(input.bucketId, retryAtMs);
+    this.recordCooldown(input.bucketId, {
+      coolUntilMs: Math.min(
+        retryAtMs,
+        this.serverNow(input.bucketId) + PROVIDER_AIMD_COOLDOWN_CAP_MS,
+      ),
+      claimedRetryAtMs: retryAtMs,
+    });
+    this.drainBlocksForBucket(input.bucketId);
     try {
       await penaltyWrite;
     } catch (error) {
@@ -1305,6 +1564,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       concurrencyGrant,
       pacingGrant,
       scopeRetryAtMsAtGrant,
+      dispatchValidationSupported,
     ] = scheduled;
     let settled = false;
     const refund = async (): Promise<void> => {
@@ -1329,17 +1589,108 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
             bucketId,
             rateScopeToken,
             retryAtMsAtGrant: scopeRetryAtMsAtGrant,
+            ...(dispatchValidationSupported
+              ? { dispatchValidationSupported: true }
+              : {}),
           }
         : null,
-      validate: async () => {
+      validate: async (
+        signal?: AbortSignal,
+        options?: ProviderPermitValidationOptions,
+      ) => {
         try {
-          return await this.validateProviderPermit(
-            bucketId,
-            rateScopeToken,
-            scopeRetryAtMsAtGrant,
-          );
+          const startedAtMs = this.now();
+          while (true) {
+            signal?.throwIfAborted();
+            if (settled) return false;
+            if (
+              !(await this.validateProviderPermit(
+                bucketId,
+                rateScopeToken,
+                scopeRetryAtMsAtGrant,
+                dispatchValidationSupported &&
+                  options?.dispatchValidationAtGateway === true,
+              ))
+            )
+              return false;
+            const generation = this.validationGenerations.get(bucketId) ?? 0;
+            while (true) {
+              signal?.throwIfAborted();
+              if (settled) return false;
+              if (
+                (this.validationGenerations.get(bucketId) ?? 0) !== generation
+              )
+                break;
+              if (
+                (this.localProviderRetryAtMs.get(bucketId) ?? 0) >
+                this.serverNow(bucketId)
+              )
+                return false;
+              const targetRequestsPerSecond =
+                this.effectiveRps.get(laneKey) ??
+                pacingGrant?.targetRequestsPerSecond;
+              if (targetRequestsPerSecond == null) return true;
+              // A delayed shared validation must not collapse reserved slots
+              // into a burst. Advance this cursor only when a send is ready;
+              // sleeping/cancelled callers never reserve future positions.
+              // The database owns admission and the app rechecks provider
+              // holds at arrival. This local fence only delays spent permits.
+              const spacingMs = Math.max(
+                1,
+                Math.ceil(1_000 / targetRequestsPerSecond),
+              );
+              const burstSize = Math.max(
+                1,
+                Math.floor(targetRequestsPerSecond / 100),
+              );
+              const previous = this.validatedDispatchCadences.get(bucketId);
+              const nowMs = this.now();
+              const sameCadence =
+                previous?.spacingMs === spacingMs &&
+                previous.burstSize === burstSize;
+              // A fresh authoritative rate also governs already prepared
+              // permits. Carry debt from the last send when the rate changes;
+              // a staggered burst's first send can precede it by an interval.
+              const previousBoundaryMs = previous
+                ? (sameCadence ? previous.atMs : previous.lastAtMs) +
+                  spacingMs * burstSize
+                : nowMs;
+              const inBurst =
+                sameCadence &&
+                previous.used < burstSize &&
+                nowMs < previousBoundaryMs;
+              const waitMs = inBurst
+                ? 0
+                : Math.max(0, previousBoundaryMs - nowMs);
+              if (
+                nowMs - startedAtMs + waitMs >=
+                PROVIDER_ADMISSION_WAIT_BUDGET_MS
+              ) {
+                throw new ProviderAdmissionWaitTimeoutError({
+                  provider: providerNameFromBucketId(bucketId),
+                  waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+                });
+              }
+              if (waitMs > 0) {
+                await this.sleep(waitMs, signal);
+                continue;
+              }
+              this.validatedDispatchCadences.set(bucketId, {
+                atMs: inBurst ? previous.atMs : nowMs,
+                lastAtMs: nowMs,
+                used: inBurst ? previous.used + 1 : 1,
+                burstSize,
+                spacingMs,
+              });
+              return true;
+            }
+          }
         } catch (error) {
-          if (error instanceof RateStateValidationUnavailableError) {
+          if (
+            signal?.aborted ||
+            error instanceof ProviderAdmissionWaitTimeoutError ||
+            error instanceof RateStateValidationUnavailableError
+          ) {
             throw error;
           }
           const normalized = normalizeError(error);
@@ -1450,6 +1801,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     permit: PacingPermit;
     scheduledAtMs: number;
     reservationId: string | null;
+    retryAtMsAtGrant: number;
   } | null {
     const block = this.liveBlock(bucketId, laneKey);
     if (!block) return null;
@@ -1461,6 +1813,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       permit: this.permitFor(bucketId, laneKey, block, scheduled),
       scheduledAtMs: scheduled[0],
       reservationId: scheduled[2],
+      // Legacy gateways do not attest a hold snapshot. Their newly assigned
+      // tickets retain existing behavior after an expired local hold; a later
+      // local 429 still wakes them. Modern snapshots invalidate stale grants.
+      retryAtMsAtGrant: this.validationScopes.has(bucketId)
+        ? scheduled[5]
+        : Math.max(
+            scheduled[5],
+            this.localProviderRetryAtMs.get(bucketId) ?? 0,
+          ),
     };
   }
 
@@ -1529,6 +1890,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         refundReservationIds: [input.reservationId],
         schedulerSchema: this.schedulerSchema,
       });
+      if (input.leaseIds.length > 0) this.signalLeaseRelease(input.bucketId);
     } catch (error) {
       const normalized = normalizeError(error);
       this.onStoreFailure({
@@ -1582,6 +1944,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           leaseIds,
           schedulerSchema: this.schedulerSchema,
         });
+        this.signalLeaseRelease(pending.bucketId);
       } catch (error) {
         const normalized = normalizeError(error);
         this.pendingReleaseFailure = new Error(
@@ -1610,6 +1973,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     this.pendingReleaseFailure = null;
     throw error;
   }
+
+  private signalLeaseRelease(bucketId: string): void {
+    const signal = this.leaseReleaseSignals.get(bucketId);
+    if (!signal) return;
+    signal.generation += 1;
+    for (const wake of signal.waiters) wake();
+  }
 }
 
 /**