deepline 0.1.208 → 0.1.210

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -9,7 +9,25 @@ import {
   type PacingPermit,
   type PacingRule,
   type RateStateBackend,
+  type RateStateBackendKind,
 } from './rate-state-backend';
+import {
+  ProviderExhaustedError,
+  providerNameFromBucketId,
+} from '../run-failure';
+
+/**
+ * Longest provider pacer hold this substrate will sleep toward before failing
+ * the call fast with a typed {@link ProviderExhaustedError}. Duration alone
+ * decides — there is no quota-vs-rate classification. A hold at or under this is
+ * slept off (today's behavior); a hold above it is skipped with no spend and no
+ * provider dispatch.
+ *
+ * Enforced ONLY on the `app_runtime_postgres` (Absurd/Node DB-authoritative)
+ * backend; the in-memory and coordinator backends never construct this class and
+ * so keep their sleep-toward-it behavior byte-for-byte.
+ */
+export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 60_000;
 
 /**
  * Permits leased per app-runtime round trip for the direct Node substrate.
@@ -17,11 +35,58 @@ import {
  * Eight keeps DB/API round trips amortized for batch-heavy maps while limiting
  * stale-block over-issue to 8 x live runner processes per bucket/window.
  */
+/**
+ * Transport seam for the app-runtime rate-state store. Mirrors the three module
+ * calls the backend makes, minus the `context` argument (the default port binds
+ * it). Aligns this backend with {@link CoordinatorRateStateBackend}'s injectable
+ * port so both are driven — and tested — the same way.
+ */
+export interface AppRuntimeRatePort {
+  acquire(
+    input: Parameters<typeof acquireRateStateViaAppRuntime>[1],
+  ): ReturnType<typeof acquireRateStateViaAppRuntime>;
+  release(
+    input: Parameters<typeof releaseRateStateViaAppRuntime>[1],
+  ): ReturnType<typeof releaseRateStateViaAppRuntime>;
+  penalize(
+    input: Parameters<typeof penalizeRateStateViaAppRuntime>[1],
+  ): ReturnType<typeof penalizeRateStateViaAppRuntime>;
+}
+
+function contextRatePort(context: WorkerRuntimeApiContext): AppRuntimeRatePort {
+  return {
+    acquire: (input) => acquireRateStateViaAppRuntime(context, input),
+    release: (input) => releaseRateStateViaAppRuntime(context, input),
+    penalize: (input) => penalizeRateStateViaAppRuntime(context, input),
+  };
+}
+
+/**
+ * Permits leased per app-runtime round trip. Eight keeps DB/API round trips
+ * amortized for batch-heavy maps while bounding stale-block over-issue to 8 x
+ * live runner processes per bucket/window. Deliberately conservative: absurd
+ * runs many concurrent sandboxes against the same org+provider bucket, so a
+ * larger block would let one runner reserve a whole window and starve peers.
+ */
 export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
 const LEASE_BLOCK_MIN_TTL_MS = 250;
 const LEASE_BLOCK_MAX_TTL_MS = 5_000;
 const MAX_ACQUIRE_SLEEP_MS = 5_000;
 const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
+/**
+ * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
+ * of buckets that all hit granted=0 at the same instant re-probe at slightly
+ * different times instead of thundering the store in lockstep. ±10%.
+ */
+const ACQUIRE_WAIT_JITTER_FRACTION = 0.1;
+
+/** Last-known server cooldown surface for a bucket (consumed by PROVIDER_EXHAUSTED). */
+export interface RateStateCooldown {
+  /** Absolute epoch-ms until which the pacer's own (internally-capped) cooldown holds. */
+  coolUntilMs: number;
+  /** Most recent server Retry-After, stored verbatim as an absolute epoch-ms deadline. */
+  claimedRetryAtMs: number;
+}
 
 interface LeasedBlock {
   /**
@@ -55,29 +120,52 @@ interface Options {
   now?: () => number;
   sleep?: (ms: number) => Promise<void>;
   onStoreFailure?: (info: { bucketId: string; error: string }) => void;
+  /** Override the transport (tests inject a counting/faulting port). */
+  port?: AppRuntimeRatePort;
+  /**
+   * Deterministic jitter source in [0,1) for the ±10% wait jitter. Defaults to
+   * Math.random; tests pin it to assert the jittered sleep stays in bounds.
+   */
+  jitter?: () => number;
 }
 
 export class AppRuntimeRateStateBackend implements RateStateBackend {
-  private readonly context: WorkerRuntimeApiContext;
+  /**
+   * DB-authoritative: the whole token bucket + AIMD runs in Postgres, so the
+   * Governor defers pacing to this backend (see `resolveAdaptivePacing` /
+   * `reportProviderBackpressure` in governor.ts).
+   */
+  readonly kind: RateStateBackendKind = 'app_runtime_postgres';
+  private readonly port: AppRuntimeRatePort;
   private readonly schedulerSchema: string | null;
   private readonly now: () => number;
   private readonly sleep: (ms: number) => Promise<void>;
+  private readonly jitter: () => number;
   private readonly onStoreFailure: (info: {
     bucketId: string;
     error: string;
   }) => void;
   private readonly blocks = new Map<string, LeasedBlock>();
-  private readonly acquisitionTails = new Map<string, Promise<void>>();
+  private readonly refills = new Map<string, Promise<void>>();
   private readonly pendingReleases = new Map<string, PendingRelease>();
+  /**
+   * Acquirers currently parked on each bucket's refill (empty block, awaiting a
+   * grant). Demand-sizing reads this to request a block sized to the live wave
+   * instead of a fixed floor, so a 200-row map fans into ~one round trip.
+   */
+  private readonly waiters = new Map<string, number>();
+  /** Last-known server cooldown per bucket (surfaced for PROVIDER_EXHAUSTED). */
+  private readonly cooldowns = new Map<string, RateStateCooldown>();
   private pendingReleaseFailure: Error | null = null;
 
   constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
-    this.context = context;
+    this.port = options.port ?? contextRatePort(context);
     this.schedulerSchema = options.schedulerSchema?.trim() || null;
     this.now = options.now ?? (() => Date.now());
     this.sleep =
       options.sleep ??
       ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.jitter = options.jitter ?? Math.random;
     this.onStoreFailure =
       options.onStoreFailure ??
       ((info) => {
@@ -85,6 +173,51 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       });
   }
 
+  /**
+   * Last-known server cooldown for a bucket, or null if the bucket has never
+   * reported one. Minimal read surface for the PROVIDER_EXHAUSTED task: it lets
+   * a caller learn how long the pacer is holding a provider off without issuing
+   * its own acquire.
+   */
+  cooldownFor(bucketId: string): RateStateCooldown | null {
+    return this.cooldowns.get(bucketId) ?? null;
+  }
+
+  /**
+   * Pre-check the last-known server cooldown for a bucket. If the pacer is
+   * already holding this provider past PROVIDER_EXHAUSTED_MAX_WAIT_MS, throw a
+   * typed {@link ProviderExhaustedError} before any store round trip. Prefers
+   * the server Retry-After (`claimedRetryAtMs`) as the retry deadline when it is
+   * later than the internally-capped `coolUntilMs` and the cooldown is active;
+   * otherwise reports `coolUntilMs`. No-op when the bucket has no live hold.
+   */
+  private throwIfPacerExhausted(bucketId: string): void {
+    const cooldown = this.cooldowns.get(bucketId);
+    if (!cooldown) return;
+    const now = this.now();
+    const coolActive = cooldown.coolUntilMs - now > 0;
+    if (!coolActive) return;
+    const coolWait = cooldown.coolUntilMs - now;
+    const claimActive = cooldown.claimedRetryAtMs - now > 0;
+    // Duration alone decides. Neither hold is over threshold → let it sleep.
+    if (
+      coolWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS &&
+      !(claimActive && cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS)
+    ) {
+      return;
+    }
+    // Retry deadline: the server Retry-After verbatim when present, else the
+    // pacer's own cooldown deadline.
+    const retryAtMs =
+      cooldown.claimedRetryAtMs > 0
+        ? cooldown.claimedRetryAtMs
+        : cooldown.coolUntilMs;
+    throw new ProviderExhaustedError({
+      provider: providerNameFromBucketId(bucketId),
+      retryAtMs,
+    });
+  }
+
   async acquire(input: {
     bucketId: string;
     rules: readonly PacingRule[];
@@ -93,125 +226,285 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     const { bucketId, rules, signal } = input;
     if (rules.length === 0) return noopPacingPermit();
     this.throwPendingReleaseFailure();
+    // Pre-check: zero round trips when the pacer is already holding past the
+    // threshold. Fails fast with a typed PROVIDER_EXHAUSTED before touching the
+    // store.
+    this.throwIfPacerExhausted(bucketId);
 
     const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
     const rulesKey = rulesSignature(ordered);
-    return await this.withBucketAcquisitionLock(bucketId, async () => {
+
+    // Drawing a cached permit is a synchronous, single-threaded critical
+    // section (`drawFromBlock` has no `await`), so concurrent acquires can never
+    // hand out the same permit — no lock is needed for the fast path. The ONLY
+    // thing that needs coordination is the block refill, which we single-flight
+    // per bucket: one network round-trip (and any window-exhausted sleep) serves
+    // the whole waiting wave, then everyone re-draws. This replaces the old
+    // per-bucket lock that held across the RPC + up-to-5s sleep and thereby
+    // serialized every map row behind one sleeping acquirer.
+    while (true) {
+      if (signal?.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Rate-state acquire aborted.');
+      }
       const localPermit = this.drawFromBlock(bucketId, rulesKey);
       if (localPermit) return localPermit;
+      // Count this acquirer as waiting on the refill so the single-flight refill
+      // can size its block to the live wave. Decremented once a block lands (or
+      // the refill errors) — see the finally in `refillBlock`.
+      this.waiters.set(bucketId, (this.waiters.get(bucketId) ?? 0) + 1);
+      try {
+        await this.refillBlock(bucketId, ordered, rulesKey, signal);
+      } finally {
+        const remaining = (this.waiters.get(bucketId) ?? 1) - 1;
+        if (remaining > 0) this.waiters.set(bucketId, remaining);
+        else this.waiters.delete(bucketId);
+      }
+    }
+  }
+
+  /**
+   * How many permits to request on the next refill for a bucket. Buckets with any
+   * `maxConcurrency` rule stay at the fixed floor block: each permit pins a
+   * durable concurrency slot, so over-reserving would strand real slots. Pure-rate
+   * buckets demand-size: clamp the live waiter count to
+   * [block floor, window budget] where the window budget is the smallest
+   * `requestsPerWindow` across the pure-rate rules (never reserve more than one
+   * window can hold). A 200-row wave then lands in ~one round trip.
+   */
+  private requestedForBucket(
+    bucketId: string,
+    rules: readonly PacingRule[],
+  ): number {
+    const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
+    if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
+    const windowBudget = Math.min(
+      ...rules
+        .map((rule) => rule.requestsPerWindow)
+        .filter((value) => Number.isFinite(value) && value > 0),
+    );
+    if (!Number.isFinite(windowBudget) || windowBudget <= 0) return floor;
+    const waiters = this.waiters.get(bucketId) ?? 1;
+    return Math.max(floor, Math.min(waiters, Math.trunc(windowBudget)));
+  }
+
+  /**
+   * Sleep duration for a granted=0 refill: the server's `waitMs`, spread by ±10%
+   * jitter and capped by MAX_ACQUIRE_SLEEP_MS. Always at least 1ms.
+   */
+  private jitteredWaitMs(serverWaitMs: number): number {
+    const base = Math.max(1, Math.min(serverWaitMs, MAX_ACQUIRE_SLEEP_MS));
+    // jitter() in [0,1) → factor in [1-0.1, 1+0.1).
+    const factor = 1 + (this.jitter() * 2 - 1) * ACQUIRE_WAIT_JITTER_FRACTION;
+    return Math.max(
+      1,
+      Math.min(Math.round(base * factor), MAX_ACQUIRE_SLEEP_MS),
+    );
+  }
+
+  /** Store the server's advisory cooldown for a bucket if the response carried one. */
+  private recordCooldown(
+    bucketId: string,
+    response: { coolUntilMs?: number; claimedRetryAtMs?: number },
+  ): void {
+    const coolUntilMs = Number(response.coolUntilMs);
+    const claimedRetryAtMs = Number(response.claimedRetryAtMs);
+    if (!Number.isFinite(coolUntilMs) && !Number.isFinite(claimedRetryAtMs)) {
+      return;
+    }
+    this.cooldowns.set(bucketId, {
+      coolUntilMs: Number.isFinite(coolUntilMs) ? Math.max(0, coolUntilMs) : 0,
+      claimedRetryAtMs: Number.isFinite(claimedRetryAtMs)
+        ? Math.max(0, claimedRetryAtMs)
+        : 0,
+    });
+  }
+
+  /**
+   * Build a PROVIDER_EXHAUSTED error for a granted=0 refill response. Retry
+   * deadline: the server Retry-After (`claimedRetryAtMs`) verbatim when present,
+   * else the pacer cooldown deadline (`coolUntilMs`), else `now + waitMs`.
+   */
+  private providerExhaustedFromResponse(
+    bucketId: string,
+    response: { waitMs: number; coolUntilMs?: number; claimedRetryAtMs?: number },
+  ): ProviderExhaustedError {
+    const claimedRetryAtMs = Number(response.claimedRetryAtMs);
+    const coolUntilMs = Number(response.coolUntilMs);
+    const retryAtMs =
+      Number.isFinite(claimedRetryAtMs) && claimedRetryAtMs > 0
+        ? claimedRetryAtMs
+        : Number.isFinite(coolUntilMs) && coolUntilMs > 0
+          ? coolUntilMs
+          : this.now() + Math.max(0, response.waitMs);
+    return new ProviderExhaustedError({
+      provider: providerNameFromBucketId(bucketId),
+      retryAtMs,
+    });
+  }
+
+  /**
+   * Single-flight a block refill for a bucket. Concurrent acquires that find the
+   * block empty all await the SAME in-flight refill instead of each issuing its
+   * own serialized round-trip, so the RPC and any window-exhausted sleep happen
+   * once per wave. Rejection (fail-closed store failure) propagates to every
+   * waiter.
+   */
+  private refillBlock(
+    bucketId: string,
+    ordered: PacingRule[],
+    rulesKey: string,
+    signal?: AbortSignal,
+  ): Promise<void> {
+    const existing = this.refills.get(bucketId);
+    if (existing) return existing;
+    const promise = this.performRefill(
+      bucketId,
+      ordered,
+      rulesKey,
+      signal,
+    ).finally(() => {
+      if (this.refills.get(bucketId) === promise) {
+        this.refills.delete(bucketId);
+      }
+    });
+    this.refills.set(bucketId, promise);
+    return promise;
+  }
 
-      let consecutiveFailures = 0;
-      while (true) {
-        if (signal?.aborted) {
-          throw signal.reason instanceof Error
-            ? signal.reason
-            : new Error('Rate-state acquire aborted.');
+  private async performRefill(
+    bucketId: string,
+    ordered: PacingRule[],
+    rulesKey: string,
+    signal?: AbortSignal,
+  ): Promise<void> {
+    let consecutiveFailures = 0;
+    // Yield once before sizing the first request so the whole synchronously-
+    // scheduled acquire wave finishes parking (incrementing `waiters`) before
+    // `requestedForBucket` reads the count. Without this, the single-flight
+    // refill fires while only the first acquirer has registered, and demand
+    // sizing collapses to the floor. Single-flight still holds — this is the
+    // one and only refill for the bucket; peers just enqueue behind it.
+    let firstPass = true;
+    while (true) {
+      // Abort is enforced per-acquirer in the acquire loop; a shared refill just
+      // stops early so waiters re-check their own signal.
+      if (signal?.aborted) return;
+      if (firstPass) {
+        firstPass = false;
+        await Promise.resolve();
+        if (signal?.aborted) return;
+      }
+      try {
+        const response = await this.port.acquire({
+          bucketId,
+          // Stamp adaptiveMaxRps on default-pacing rules so the DB row's ceiling
+          // matches the old in-memory AIMD range.
+          rules: withAdaptiveMaxRps(ordered),
+          requested: this.requestedForBucket(bucketId, ordered),
+          schedulerSchema: this.schedulerSchema,
+        });
+        this.recordCooldown(bucketId, response);
+        if (response.granted > 0) {
+          // Merge the whole grant into the block and let acquirers re-draw. No
+          // permit is handed out here, so nothing is double-drawn.
+          this.mergeGrantedBlock(bucketId, ordered, rulesKey, response);
+          return;
         }
-        try {
-          const response = await acquireRateStateViaAppRuntime(this.context, {
-            bucketId,
-            rules: ordered,
-            requested: APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE,
-            schedulerSchema: this.schedulerSchema,
-          });
-          consecutiveFailures = 0;
-          if (response.granted > 0) {
-            const hasConcurrency = ordered.some(
-              (rule) => rule.maxConcurrency != null,
-            );
-            const leaseIds = Array.isArray(response.leaseIds)
-              ? response.leaseIds.filter(
-                  (leaseId): leaseId is string =>
-                    typeof leaseId === 'string' && leaseId.trim().length > 0,
-                )
-              : [];
-            if (hasConcurrency && leaseIds.length !== response.granted) {
-              throw new Error(
-                `Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
-              );
-            }
-            if (!hasConcurrency && leaseIds.length > 0) {
-              throw new Error(
-                `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
-              );
-            }
-            const currentLeaseId = hasConcurrency ? leaseIds[0]! : null;
-            const remainingLeaseIds = hasConcurrency ? leaseIds.slice(1) : [];
-            const remainingWindowPermits = hasConcurrency
-              ? 0
-              : response.granted - 1;
-            if (remainingLeaseIds.length > 0 || remainingWindowPermits > 0) {
-              this.mergeBlock(
-                bucketId,
-                remainingLeaseIds,
-                remainingWindowPermits,
-                rulesKey,
-                ordered,
-              );
-            }
-            return this.permitFor(bucketId, ordered, currentLeaseId);
-          }
-          await this.sleep(
-            Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
-          );
-        } catch (error) {
-          // Fail closed. A degraded rate-state store must not grant unlimited
-          // provider calls; after this bounded retry ladder the run fails loudly.
-          consecutiveFailures += 1;
-          const normalized = normalizeError(error);
-          this.onStoreFailure({ bucketId, error: normalized.message });
-          if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
-            throw new Error(
-              `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
-            );
-          }
-          await this.sleep(
-            STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
+        // Live path: the server told us to wait past the threshold. Fail fast
+        // with a typed PROVIDER_EXHAUSTED instead of sleeping toward it. The
+        // rejection propagates to every waiter parked on this single-flight
+        // refill (they all skip with no spend). Duration alone decides.
+        if (response.waitMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS) {
+          throw this.providerExhaustedFromResponse(bucketId, response);
+        }
+        // Window exhausted: sleep the SERVER's waitMs once for the whole waiting
+        // wave (not per row, and not a guess), then return so acquirers re-draw
+        // and single-flight another refill if still empty. ±10% jitter so a
+        // synchronized wave re-probes at spread-out times; capped by
+        // MAX_ACQUIRE_SLEEP_MS.
+        await this.sleep(this.jitteredWaitMs(response.waitMs));
+        return;
+      } catch (error) {
+        // PROVIDER_EXHAUSTED is a deliberate skip, not a store failure: it must
+        // escape this fail-closed retry ladder verbatim so it reaches the
+        // acquire caller (and, via step-miss handling, the row).
+        if (error instanceof ProviderExhaustedError) throw error;
+        // Fail closed. A degraded rate-state store must not grant unlimited
+        // provider calls; after this bounded retry ladder the run fails loudly.
+        consecutiveFailures += 1;
+        const normalized = normalizeError(error);
+        this.onStoreFailure({ bucketId, error: normalized.message });
+        if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
+          throw new Error(
+            `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
           );
         }
+        await this.sleep(
+          STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
+        );
       }
-    });
+    }
   }
 
-  private async withBucketAcquisitionLock<T>(
+  /**
+   * Validate a granted acquire response and merge its permits into the bucket's
+   * local block. Concurrency rules carry one lease token per permit (release
+   * semantics); pure-rate rules carry untracked window permits the store already
+   * decremented.
+   */
+  private mergeGrantedBlock(
     bucketId: string,
-    operation: () => Promise<T>,
-  ): Promise<T> {
-    const previous = this.acquisitionTails.get(bucketId) ?? Promise.resolve();
-    let unlock!: () => void;
-    const gate = new Promise<void>((resolve) => {
-      unlock = resolve;
-    });
-    const tail = previous.then(() => gate);
-    this.acquisitionTails.set(bucketId, tail);
-    await previous;
-    try {
-      return await operation();
-    } finally {
-      unlock();
-      if (this.acquisitionTails.get(bucketId) === tail) {
-        this.acquisitionTails.delete(bucketId);
-      }
+    ordered: PacingRule[],
+    rulesKey: string,
+    response: { granted: number; leaseIds?: unknown },
+  ): void {
+    const hasConcurrency = ordered.some((rule) => rule.maxConcurrency != null);
+    const leaseIds = Array.isArray(response.leaseIds)
+      ? response.leaseIds.filter(
+          (leaseId): leaseId is string =>
+            typeof leaseId === 'string' && leaseId.trim().length > 0,
+        )
+      : [];
+    if (hasConcurrency && leaseIds.length !== response.granted) {
+      throw new Error(
+        `Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
+      );
+    }
+    if (!hasConcurrency && leaseIds.length > 0) {
+      throw new Error(
+        `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
+      );
     }
+    this.mergeBlock(
+      bucketId,
+      hasConcurrency ? leaseIds : [],
+      hasConcurrency ? 0 : response.granted,
+      rulesKey,
+      ordered,
+    );
   }
 
   penalize(input: { bucketId: string; cooldownMs: number }): void {
     if (input.cooldownMs <= 0) return;
     this.drainBlock(input.bucketId);
-    void penalizeRateStateViaAppRuntime(this.context, {
-      bucketId: input.bucketId,
-      cooldownMs: input.cooldownMs,
-      schedulerSchema: this.schedulerSchema,
-    }).catch((error) => {
-      const normalized = normalizeError(error);
-      this.pendingReleaseFailure = new Error(
-        `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
-      );
-      this.onStoreFailure({
+    void this.port
+      .penalize({
         bucketId: input.bucketId,
-        error: normalized.message,
+        cooldownMs: input.cooldownMs,
+        schedulerSchema: this.schedulerSchema,
+      })
+      .catch((error) => {
+        const normalized = normalizeError(error);
+        this.pendingReleaseFailure = new Error(
+          `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
+        );
+        this.onStoreFailure({
+          bucketId: input.bucketId,
+          error: normalized.message,
+        });
       });
-    });
   }
 
   private permitFor(
@@ -328,7 +621,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     while (pending.leaseIds.length > 0) {
       const leaseIds = pending.leaseIds.splice(0);
       try {
-        await releaseRateStateViaAppRuntime(this.context, {
+        await this.port.release({
           bucketId: pending.bucketId,
           rules: pending.rules,
           leaseIds,
@@ -364,6 +657,40 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   }
 }
 
+/**
+ * Multiplier applied to a default-pacing rule's `requestsPerWindow` to derive the
+ * `adaptiveMaxRps` ceiling. Matches the DB seed's `DEFAULT_PACING_MAX_RPS_MULTIPLIER`
+ * and the old in-memory AIMD range (default RPS ramps up to base * 25).
+ */
+const DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER = 25;
+
+/**
+ * A rule id shaped `default:<toolId>` is the undeclared-tool default-pacing rule
+ * minted by `defaultPacingForTool` in governor.ts. Only those ramp above their
+ * base rate; declared providers pin at their contractual rate.
+ */
+function isDefaultPacingRuleId(ruleId: string): boolean {
+  return ruleId.startsWith('default:');
+}
+
+/**
+ * Stamp `adaptiveMaxRps` onto default-pacing rules so the DB row's ceiling matches
+ * the old in-memory AIMD range (base * 25). Declared rules pass through untouched
+ * (the DB pins them at base). The field is extra on the wire; the server's
+ * `normalizeRules` reads it, everyone else ignores it.
+ */
+function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
+  return rules.map((rule) =>
+    isDefaultPacingRuleId(rule.ruleId)
+      ? ({
+          ...rule,
+          adaptiveMaxRps:
+            rule.requestsPerWindow * DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER,
+        } as PacingRule)
+      : rule,
+  );
+}
+
 function rulesSignature(rules: readonly PacingRule[]): string {
   return [...rules]
     .map(

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -282,6 +282,15 @@ export function createPlayExecutionGovernor(
   const budgetState = input.budgetState ?? new InMemoryBudgetStateBackend();
   const providerByTool = new Map<string, string>();
 
+  // When the rate-state backend owns pacing authoritatively (the Absurd/Node
+  // app-runtime Postgres pacer runs the whole token bucket + AIMD in the row),
+  // the Governor's in-memory adaptive admission becomes a passthrough: it only
+  // SEEDS declared/default rules and never halves/ramps them here. Backpressure
+  // is routed solely to `rateState.penalize` so the DB row is the single source
+  // of pacing truth. The coordinator and in-memory backends leave `kind`
+  // undefined and keep the in-memory adaptive admission (byte-identical).
+  const dbAuthoritativePacing = input.rateState.kind === 'app_runtime_postgres';
+
   const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
 
   async function resolveAdaptivePacing(toolId: string): Promise<{
@@ -291,6 +300,14 @@ export function createPlayExecutionGovernor(
     const resolvedPacing = await input.resolvePacing(toolId);
     const declaredPacing =
       resolvedPacing && resolvedPacing.rules.length > 0 ? resolvedPacing : null;
+    if (dbAuthoritativePacing) {
+      // Passthrough: hand the DB-backed pacer the declared rules verbatim (or
+      // the default-pacing shape when undeclared). No in-memory admission — the
+      // row is authoritative.
+      const pacing = declaredPacing ?? defaultPacingForTool(toolId, policy);
+      providerByTool.set(toolId, pacing.provider);
+      return pacing;
+    }
     const pacing = adaptiveAdmission.resolvePacing({
       orgId: input.scope.orgId,
       toolId,
@@ -531,11 +548,16 @@ export function createPlayExecutionGovernor(
       resolveRowConcurrency(policy, requested),
 
     reportProviderBackpressure(bp) {
-      adaptiveAdmission.observeProviderBackpressure({
-        orgId: input.scope.orgId,
-        provider: bp.provider,
-        retryAfterMs: bp.retryAfterMs,
-      });
+      // DB-authoritative pacer: route backpressure ONLY to the row via
+      // penalize. The in-memory adaptive admission must not also halve, or the
+      // provider would be throttled twice (once in the row, once in memory).
+      if (!dbAuthoritativePacing) {
+        adaptiveAdmission.observeProviderBackpressure({
+          orgId: input.scope.orgId,
+          provider: bp.provider,
+          retryAfterMs: bp.retryAfterMs,
+        });
+      }
       input.rateState.penalize({
         bucketId: bucketId(bp.provider),
         cooldownMs: bp.retryAfterMs,