deepline 0.3.184 → 0.3.185

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -4,9 +4,13 @@ import type {
   PlayRunnerRateStateConcurrencyGrant,
   PlayRunnerRateStatePenalizeInput,
   PlayRunnerRateStateReleaseInput,
+  PlayRunnerRateStateValidateInput,
+  PlayRunnerRateStateValidateResult,
 } from '../protocol';
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+  PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingRule,
 } from '../governor/rate-state-backend';
 import { type PostgresSchedulerQueryClient } from './postgres';
@@ -18,6 +22,7 @@ import {
 } from '../provider-capacity/gradient2-concurrency-controller';
 import { providerConcurrencyControllerOptions } from '../provider-capacity/provider-concurrency-policy';
 import { classifyProviderThrottle } from '../provider-capacity/provider-throttle-classifier';
+import { acquireRateScopeSql, updateRateScopeSql } from './postgres-rate-scope';
 
 const DEFAULT_SCHEMA = 'runtime_scheduler';
 const RATE_BUCKET_TABLE = 'rate_bucket';
@@ -27,6 +32,8 @@ const RATE_RESERVATION_TABLE = 'rate_reservations';
 const RATE_RESERVATION_BUCKET_EXPIRY_INDEX =
   'runtime_scheduler_rate_reservation_bucket_expiry_idx';
 const RATE_REQUESTER_TABLE = 'rate_requesters';
+const RATE_SCOPE_HOLD_TABLE = 'rate_scope_holds';
+
 const MIN_CONCURRENCY_WAIT_MS = 10;
 /**
  * Longest a requester held back by the fair-share cap sleeps before it re-asks.
@@ -53,7 +60,9 @@ const CONCURRENCY_OBSERVATION_DEDUP_RETENTION_MS = 24 * 60 * 60 * 1000;
 const CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE = 128;
 const ACTIVE_LEASE_TTL_MS = 5 * 60 * 1000;
 const RATE_RESERVATION_TTL_MS = 5 * 60 * 1000;
-const MIN_ADAPTIVE_RPS_FLOOR = 0.01;
+// A guessed initial rate must not prevent learning an hourly quota. This is a
+// numerical floor (one request/day), not a fraction of the catalog seed.
+const MIN_ADAPTIVE_RPS_FLOOR = 1 / 86_400;
 /**
  * Horizon a granted=0 caller is allowed to sleep toward a WHOLE block instead of
  * a single token. Waiting for one token means one permit per store round trip,
@@ -66,6 +75,32 @@ const MIN_ADAPTIVE_RPS_FLOOR = 0.01;
  */
 const BLOCK_WAIT_HORIZON_MS = 1_000;
 
+/**
+ * Bounded summary of one persisted provider-throttle decision. This is emitted
+ * as telemetry; it deliberately contains no rule ids, credentials, or row data.
+ */
+export type PostgresRateStatePenaltyTelemetry = {
+  retryAtMs: number | null;
+  coolUntilMs: number | null;
+  rateRuleCount: number;
+  rateDecreasedRuleCount: number;
+  refillMinBeforeRps: number | null;
+  refillMaxBeforeRps: number | null;
+  refillMinAfterRps: number | null;
+  refillMaxAfterRps: number | null;
+  observationCycleMaxMs: number | null;
+  confirmedSuccessesMax: number | null;
+  configuredConcurrencyRuleCount: number;
+  gradient2Attribution: 'not_observed' | 'rate' | 'concurrency' | 'ambiguous';
+  gradient2Disposition: 'not_observed' | 'applied' | 'already_applied';
+  gradient2RuleCount: number;
+  gradient2ChangedRuleCount: number;
+  gradient2LimitMinBefore: number | null;
+  gradient2LimitMaxBefore: number | null;
+  gradient2LimitMinAfter: number | null;
+  gradient2LimitMaxAfter: number | null;
+};
+
 /**
  * Burst horizon of the adaptive token bucket, in milliseconds of the rate
  * ACTUALLY in force.
@@ -74,8 +109,11 @@ const BLOCK_WAIT_HORIZON_MS = 1_000;
  * its seed: config-drift detection compares it against the seed verbatim, so
  * persisting a learned capacity would make every warm acquire look like drift
  * and re-seed the row. The bucket the grant actually spends is therefore
- * derived, not stored: one second of `refill_per_sec`, floored at the declared
- * burst.
+ * derived, not stored: one second of `refill_per_sec` before a throttle and
+ * 100ms of it afterwards. Before a throttle the declared burst is a floor;
+ * after a throttle only a single probe is the floor.
+ * Retaining the catalog burst after learning a slow rate lets a fresh runtime
+ * reserve minutes of future slots before another run can register demand.
  *
  * Without this the AIMD ramp is invisible to throughput. A grant is bounded by
  * the bucket's tokens, tokens are clamped to `capacity`, and every grant costs
@@ -84,29 +122,50 @@ const BLOCK_WAIT_HORIZON_MS = 1_000;
  * (capacity 10) stayed at ~7.7 calls/s and a declared 25 rps lane at ~13
  * calls/s no matter how high `refill_per_sec` climbed.
  *
- * The burst stays bounded: one second of the learned rate, and the learned rate
- * is itself capped at `max_rps`. The dispatch cursor still spaces every permit
- * at 1/`refill_per_sec`, so provider arrivals never exceed the rate in force.
+ * The burst stays bounded by the learned rate and `max_rps`. The dispatch
+ * cursor still spaces every permit at 1/`refill_per_sec`, so provider arrivals
+ * never exceed the rate in force. The shorter post-throttle horizon limits
+ * fixed-window boundary overshoot while retaining small batches on fast lanes.
  */
 const ADAPTIVE_BURST_HORIZON_MS = 1_000;
+const BACKPRESSURED_BURST_HORIZON_MS = 100;
 
 /**
  * Effective (spendable) bucket capacity for a rate-row alias. Derived per
- * statement from the learned rate; see {@link ADAPTIVE_BURST_HORIZON_MS}.
+ * statement from the learned rate; see the adaptive burst horizon constants.
  */
-function effectiveCapacitySql(alias: string): string {
-  return `GREATEST(${alias}.capacity, ${alias}.refill_per_sec * ${ADAPTIVE_BURST_HORIZON_MS} / 1000.0)`;
+function effectiveCapacitySql(
+  alias: string,
+  refillColumn = 'refill_per_sec',
+): string {
+  // After a provider 429, Retry-After marks the earliest safe probe time; it
+  // does not authorize banking a full rate-sized burst while the hold runs.
+  // rate_observation_started_at_ms stays NULL until the first post-hold grant,
+  // so that grant is exactly one probe even when the preceding successful
+  // throughput estimate was high. Once the probe starts a fresh observation
+  // epoch, only a short micro-burst resumes. A full second of tokens can straddle
+  // a fixed provider quota boundary as adjacent bursts and double the allowed
+  // starts in one window. A learned rate below the declared base is also
+  // backpressure evidence and must not restore a stale (possibly 1,000-wide)
+  // configured burst.
+  return `CASE
+    WHEN ${alias}.seen_backpressure AND ${alias}.rate_observation_started_at_ms IS NULL THEN 1.0
+    ELSE GREATEST(
+      CASE WHEN ${alias}.seen_backpressure OR ${alias}.${refillColumn} < ${alias}.base_refill_per_sec THEN 1 ELSE ${alias}.capacity END,
+      ${alias}.${refillColumn} * CASE
+        WHEN ${alias}.seen_backpressure THEN ${BACKPRESSURED_BURST_HORIZON_MS}
+        ELSE ${ADAPTIVE_BURST_HORIZON_MS}
+      END / 1000.0
+    )
+  END`;
 }
 
 /**
- * Internal cooldown cap. A 429 report (penalize) can carry an arbitrarily large
- * server Retry-After; we store that verbatim in `claimed_retry_at_ms` for
- * observability but cap the pacer's own cooldown at 45 minutes so a stuck
- * provider is re-probed at least this often instead of freezing the bucket for
- * the whole (possibly hours-long) Retry-After.
+ * AIMD cooldown cap. A 429's absolute Retry-After is separately persisted in
+ * `rate_scope_holds` and gates every rule/run until that deadline. This cap only
+ * bounds how long the per-rule AIMD state remains in its cooldown phase; it
+ * must never shorten or bypass the provider's authoritative hold.
  */
-const MAX_INTERNAL_COOLDOWN_MS = 45 * 60 * 1000;
-
 /**
  * Buckets this process has already seeded, keyed by schema + bucket id + the
  * exact seed payload. Seeding is a pure upsert of declared configuration, so
@@ -181,6 +240,33 @@ function requesterTableName(options?: {
   return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_REQUESTER_TABLE)}`;
 }
 
+function scopeHoldTableName(options?: {
+  schema?: string | null;
+  schedulerSchema?: string | null;
+}): string {
+  return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_SCOPE_HOLD_TABLE)}`;
+}
+
+function isMissingRateScopeHoldTableError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  return (
+    candidate.code === '42P01' &&
+    typeof candidate.message === 'string' &&
+    new RegExp(`\\b${RATE_SCOPE_HOLD_TABLE}\\b`).test(candidate.message)
+  );
+}
+
+function isMissingRateThrottleCeilingColumnError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  return (
+    candidate.code === '42703' &&
+    typeof candidate.message === 'string' &&
+    /\brate_throttle_ceiling_rps\b/.test(candidate.message)
+  );
+}
+
 /**
  * v43 added `rate_requesters` and the lease `requester_id` column. Only an
  * acquire that carries a requester identity touches them, so during the
@@ -271,7 +357,9 @@ async function withRateStateAvailabilityGuard<T>(
       !isMissingRateReservationTableError(error) &&
       !isMissingConcurrencyCheckpointError(error) &&
       !isMissingConcurrencyObserverTableError(error) &&
-      !isMissingRateRequesterSchemaError(error)
+      !isMissingRateRequesterSchemaError(error) &&
+      !isMissingRateScopeHoldTableError(error) &&
+      !isMissingRateThrottleCeilingColumnError(error)
     ) {
       throw error;
     }
@@ -298,11 +386,19 @@ function normalizeRules(
   return [...rules]
     .map((rule) => {
       const isDefaultPacing = rule.ruleId.startsWith('default:');
+      const isUnknownProviderScope = rule.ruleId === 'default:provider-scope';
       return {
         ruleId: rule.ruleId,
         // Rates are fractional; only dispatched requests must be whole.
-        requestsPerWindow: rule.requestsPerWindow,
-        windowMs: Math.max(0, Math.trunc(rule.windowMs)),
+        // Keep the shared, unhinted credential scope conservative even when
+        // an older runner submits the generic 10-RPS policy seed. The shared
+        // row's AIMD controller learns upward from successful observations.
+        requestsPerWindow: isUnknownProviderScope
+          ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
+          : rule.requestsPerWindow,
+        windowMs: isUnknownProviderScope
+          ? 1_000
+          : Math.max(0, Math.trunc(rule.windowMs)),
         maxConcurrency:
           rule.maxConcurrency == null
             ? null
@@ -331,7 +427,7 @@ function normalizeRules(
  * per second; `capacity` is one window of burst. The adaptive `[min,max]` range
  * is derived value-first so the same wire shape drives both declared and
  * default-pacing buckets:
- *   - min_rps = min(base, max(0.01, base/8))
+ *   - min_rps = min(base, 1/86400)
  *     (low-RPM rules must still be able to back down below their base)
  *   - max_rps = max(base, 100)     (catalogued-rate recommendation)
  *   - max_rps = explicit adaptiveMaxRps when supplied
@@ -350,7 +446,7 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   // Preserve its fractional refill rate, with dispatch times resolved in ms.
   const capacity = Math.max(1, rule.requestsPerWindow);
   const base = (rule.requestsPerWindow / rule.windowMs) * 1000;
-  const minRps = Math.min(base, Math.max(MIN_ADAPTIVE_RPS_FLOOR, base / 8));
+  const minRps = Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
   const maxRps =
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
@@ -401,6 +497,40 @@ export async function ensurePostgresRateStateSchema(
       PRIMARY KEY (bucket_id, rule_id)
     )
   `);
+  // Provider Retry-After belongs to the credential/provider bucket, not a
+  // particular tool-rule row. This durable row survives worker/run restarts
+  // and newly introduced rules without growing with request volume.
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${scopeHoldTableName(options)} (
+      bucket_id text PRIMARY KEY,
+      retry_at_ms bigint NOT NULL DEFAULT 0
+    )
+  `);
+  // Null refill marks an old hold, not an unusable schema. The first seed
+  // initializes its learner without trusting pre-scope per-rule estimates.
+  // No runtime wire version or migration-ledger admission check is involved.
+  await client.query(`
+    ALTER TABLE ${scopeHoldTableName(options)}
+      ADD COLUMN IF NOT EXISTS refill_per_sec float8,
+      ADD COLUMN IF NOT EXISTS capacity float8 NOT NULL DEFAULT 1,
+      ADD COLUMN IF NOT EXISTS max_rps float8 NOT NULL DEFAULT 100,
+      ADD COLUMN IF NOT EXISTS tokens float8 NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS refilled_at_ms bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS next_dispatch_at_ms bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS seen_backpressure boolean NOT NULL DEFAULT false,
+      ADD COLUMN IF NOT EXISTS cool_until_ms bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS last_ramp_at_ms bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS rate_observation_started_at_ms bigint,
+      ADD COLUMN IF NOT EXISTS rate_successes bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS rate_recovery_interval_ms bigint NOT NULL DEFAULT 1000,
+      ADD COLUMN IF NOT EXISTS rate_throttle_ceiling_rps float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS rate_observation_started_at_ms bigint,
+      ADD COLUMN IF NOT EXISTS rate_successes bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS rate_recovery_interval_ms bigint NOT NULL DEFAULT 1000
+  `);
   await client.query(`
     ALTER TABLE ${bucketTableName(options)}
       ADD COLUMN IF NOT EXISTS next_dispatch_at_ms bigint NOT NULL DEFAULT 0
@@ -523,11 +653,12 @@ export async function ensurePostgresRateStateSchema(
     CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_bucket_bucket_idx
       ON ${bucketTableName(options)} (bucket_id)
   `);
-  // v43: one row per (bucket, run) remembers how much of the current
-  // fair-share window that run has drawn, so several runs contending for one
-  // org's provider window each keep a max-min share instead of first-come
-  // draining every window. Rows expire after a short idle; a lone requester
-  // never has peers and is granted exactly as before.
+  // v43: one row per (bucket, requester) remembers how much of the current
+  // fair-share window it has drawn. New gateway callers identify the
+  // authenticated org, so multiple runs from one customer cannot multiply its
+  // share of a fleet-managed provider bucket. Older run-keyed rows expire after
+  // their normal idle horizon during rollout. A lone requester never has peers
+  // and is granted exactly as before.
   await client.query(`
     CREATE TABLE IF NOT EXISTS ${requesterTableName(options)} (
       bucket_id text NOT NULL,
@@ -545,7 +676,7 @@ export async function ensurePostgresRateStateSchema(
       ON ${requesterTableName(options)} (expires_at_ms)
   `);
   // The same share applies to in-flight concurrency slots, which needs each
-  // live lease attributed to the run that holds it.
+  // live lease attributed to its requester.
   await client.query(`
     ALTER TABLE ${concurrencyLeaseTableName(options)}
       ADD COLUMN IF NOT EXISTS requester_id text
@@ -573,6 +704,90 @@ async function seedBucketRows(
           $3::bigint,
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
+      ), seed_config AS (
+        SELECT * FROM jsonb_to_recordset($2::jsonb) AS seed(
+          rule_id text, capacity float8, base_refill_per_sec float8,
+          min_rps float8, max_rps float8, adaptive_max_configured boolean
+        )
+      ), legacy AS (
+        SELECT COALESCE(MAX(claimed_retry_at_ms), 0) AS retry_at_ms,
+          COALESCE(bool_or(seen_backpressure), false) AS throttled,
+          COALESCE(
+            bool_and(
+              refill_per_sec = base_refill_per_sec AND NOT seen_backpressure
+            ) FILTER (WHERE rule_id LIKE 'default:%'),
+            false
+          ) AND NOT EXISTS (
+            SELECT 1
+            FROM ${bucketTableName(input)} existing
+            JOIN seed_config sc ON sc.rule_id = existing.rule_id
+            WHERE existing.bucket_id = $1
+              AND existing.base_refill_per_sec = sc.base_refill_per_sec
+              AND existing.capacity = sc.capacity
+          ) AS unlearned_unknown_provider
+        FROM ${bucketTableName(input)} WHERE bucket_id = $1
+      ), seed_scope_hold AS (
+        -- Preserve actual holds, but never import poisoned per-rule learning.
+        -- A legacy throttled scope restarts at at most one probe/second.
+        INSERT INTO ${scopeHoldTableName(input)} AS hold (
+          bucket_id, retry_at_ms, refill_per_sec, capacity, max_rps, tokens,
+          refilled_at_ms, next_dispatch_at_ms, seen_backpressure
+        )
+        SELECT $1, legacy.retry_at_ms,
+          LEAST(MIN(sc.base_refill_per_sec), CASE WHEN legacy.throttled THEN 1 ELSE 'Infinity'::float8 END),
+          MIN(sc.capacity), MAX(sc.max_rps),
+          CASE WHEN legacy.throttled THEN 0 ELSE MIN(sc.capacity) END,
+          sn.now_ms, GREATEST(sn.now_ms, legacy.retry_at_ms), legacy.throttled
+        FROM seed_config sc CROSS JOIN seed_now sn CROSS JOIN legacy
+        GROUP BY sn.now_ms, legacy.retry_at_ms, legacy.throttled
+        ON CONFLICT (bucket_id) DO UPDATE SET
+          retry_at_ms = GREATEST(hold.retry_at_ms, EXCLUDED.retry_at_ms),
+          refill_per_sec = CASE
+            -- Older runners seeded this shared credential row at the generic
+            -- default (10 RPS). If that old value never moved from its seed,
+            -- adopt the conservative shared-scope seed on the first acquire
+            -- after upgrade. Preserve any rate with actual ramp or throttle
+            -- evidence, and never erase a provider Retry-After hold.
+            WHEN EXISTS (
+              SELECT 1 FROM seed_config
+              WHERE rule_id = 'default:provider-scope'
+            )
+              AND (SELECT unlearned_unknown_provider FROM legacy)
+              AND NOT hold.seen_backpressure
+              AND hold.retry_at_ms <= EXCLUDED.refilled_at_ms
+            THEN EXCLUDED.refill_per_sec
+            ELSE COALESCE(hold.refill_per_sec,
+              LEAST(EXCLUDED.refill_per_sec, CASE WHEN hold.retry_at_ms > 0 THEN 1 ELSE 'Infinity'::float8 END))
+          END,
+          capacity = CASE
+            WHEN EXISTS (
+              SELECT 1 FROM seed_config
+              WHERE rule_id = 'default:provider-scope'
+            )
+              AND (SELECT unlearned_unknown_provider FROM legacy)
+              AND NOT hold.seen_backpressure
+              AND hold.retry_at_ms <= EXCLUDED.refilled_at_ms
+            THEN EXCLUDED.capacity
+            ELSE GREATEST(hold.capacity, EXCLUDED.capacity)
+          END,
+          max_rps = GREATEST(hold.max_rps, EXCLUDED.max_rps),
+          tokens = CASE
+            WHEN EXISTS (
+              SELECT 1 FROM seed_config
+              WHERE rule_id = 'default:provider-scope'
+            )
+              AND (SELECT unlearned_unknown_provider FROM legacy)
+              AND NOT hold.seen_backpressure
+              AND hold.retry_at_ms <= EXCLUDED.refilled_at_ms
+            THEN LEAST(hold.tokens, EXCLUDED.capacity)
+            WHEN hold.refill_per_sec IS NULL THEN
+              CASE WHEN hold.retry_at_ms > 0 THEN 0 ELSE EXCLUDED.tokens END
+            ELSE hold.tokens
+          END,
+          refilled_at_ms = CASE WHEN hold.refill_per_sec IS NULL THEN EXCLUDED.refilled_at_ms ELSE hold.refilled_at_ms END,
+          next_dispatch_at_ms = CASE WHEN hold.refill_per_sec IS NULL THEN GREATEST(hold.retry_at_ms, EXCLUDED.next_dispatch_at_ms) ELSE hold.next_dispatch_at_ms END,
+          seen_backpressure = hold.seen_backpressure OR EXCLUDED.seen_backpressure OR hold.retry_at_ms > 0
+        RETURNING *
       )
       INSERT INTO ${bucketTableName(input)} AS existing (
         bucket_id,
@@ -589,24 +804,26 @@ async function seedBucketRows(
         claimed_retry_at_ms,
         last_ramp_at_ms,
         refilled_at_ms,
-        next_dispatch_at_ms
+        next_dispatch_at_ms,
+        rate_recovery_interval_ms
       )
       SELECT
         $1,
         seed.rule_id,
         seed.capacity,
         seed.capacity,
-        seed.base_refill_per_sec,
+        LEAST(seed.base_refill_per_sec, COALESCE(learned.refill_per_sec, seed.base_refill_per_sec)),
         seed.base_refill_per_sec,
         seed.min_rps,
         seed.max_rps,
         seed.adaptive_max_configured,
-        false,
+        learned.seen_backpressure,
         0,
         0,
         0,
         sn.now_ms,
-        sn.now_ms
+        sn.now_ms,
+        learned.rate_recovery_interval_ms
       FROM jsonb_to_recordset($2::jsonb) AS seed(
         rule_id text,
         capacity float8,
@@ -616,6 +833,9 @@ async function seedBucketRows(
         adaptive_max_configured boolean
       )
       CROSS JOIN seed_now sn
+      CROSS JOIN seed_scope_hold learned
+      -- Force the same hold -> bucket lock order as acquire and penalty.
+      -- An unreferenced data-modifying CTE may run after the bucket write.
       ON CONFLICT (bucket_id, rule_id) DO UPDATE SET
         tokens = LEAST(existing.tokens, EXCLUDED.capacity),
         capacity = EXCLUDED.capacity,
@@ -679,14 +899,18 @@ type RateStateAcquireInput = PlayRunnerRateStateAcquireInput & {
 };
 
 /**
- * Fair-share identity for a rate-state acquire: the root (billing) run when the
- * executor token carries one, else the run itself, so a parent and the child
- * plays it launches draw from one slice instead of multiplying their count.
+ * Fair-share identity for a rate-state acquire. A provider bucket can be shared
+ * by many runs (and, for a fleet-managed credential, many organizations), so
+ * runs must not multiply one customer's share. Prefer the authenticated org;
+ * retain a run fallback for older internal callers without org attribution.
  */
 export function rateStateRequesterId(auth: {
+  orgId?: string | null;
   internalExecutorRunId?: string | null;
   internalExecutorBillingRunId?: string | null;
 }): string | null {
+  const orgId = auth.orgId?.trim();
+  if (orgId) return `org:${orgId}`;
   const requesterId =
     auth.internalExecutorBillingRunId?.trim() ||
     auth.internalExecutorRunId?.trim() ||
@@ -714,14 +938,15 @@ type FairShareSql = {
 };
 
 /**
- * Max-min fair share across the runs contending for one bucket, folded into
- * the single grant statement.
+ * Max-min fair share across requesters contending for one bucket, folded into
+ * the single grant statement. Authenticated gateway callers use organization
+ * identity, so independent runs by one organization consume one share.
  *
  * A fair-share window is the time the binding rule (slowest refill) takes to
  * refill its spendable burst: one second for an adaptive lane, the declared
  * window for a slow rule. Windows are aligned to the clock so every requester
  * agrees on the current one. Each requester's grant inside a window is capped
- * at `ceil(capacity / N)` (N = requesters seen within the last
+ * at `ceil(window permits / N)` (N = requesters seen within the last
  * {@link FAIR_SHARE_REQUESTER_IDLE_WINDOWS} windows) plus whatever the bucket
  * holds beyond the slices still reserved for peers with unmet demand. Peers
  * whose last ask was fully granted reserve nothing, and an idle peer drops
@@ -733,7 +958,7 @@ type FairShareSql = {
  * On the concurrency path the same share bounds in-flight leases per
  * requester, otherwise a run holding every slot re-takes each one it releases.
  *
- * A run alone on a bucket writes its ledger row only when its window rolls or
+ * A requester alone on a bucket writes its ledger row only when its window rolls or
  * its lease nears expiry, not on every acquire, so the solo path pays reads
  * and no extra writes. The cost of that is one window: a peer arriving
  * mid-window sees the incumbent's draw from when it last wrote, so the
@@ -752,8 +977,8 @@ function fairShareSql(
   const requester = `${options.requesterParam}::text`;
   const requested = `${options.requestedParam}::int`;
   const rawAvailable = options.concurrency
-    ? `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill))::int, (SELECT free FROM free_slots)))`
-    : `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill))::int))`;
+    ? `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill) + 1e-9)::int, (SELECT free FROM free_slots)))`
+    : `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill) + 1e-9)::int))`;
   const liveLeasesHeldBy = (holder: string): string =>
     options.concurrency
       ? `(
@@ -813,10 +1038,11 @@ function fairShareSql(
       fair_window AS (
         SELECT
           r.now_ms,
-          r.effective_capacity AS capacity,
+          r.effective_capacity AS burst_capacity,
+          r.refill_per_sec,
           GREATEST(
             ${ADAPTIVE_BURST_HORIZON_MS},
-            CEIL(r.capacity / GREATEST(r.refill_per_sec, 0.0001) * 1000)
+            CEIL(r.effective_capacity / GREATEST(r.refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000)
           )::bigint AS window_ms
         FROM refreshed r
         ORDER BY r.refill_per_sec ASC, r.effective_capacity ASC
@@ -825,12 +1051,19 @@ function fairShareSql(
       fair_epoch AS (
         SELECT
           w.now_ms,
-          w.capacity,
+          -- Fairness allocates the permits replenished during this window,
+          -- not just the instantaneous token-bucket burst. After a 429 the
+          -- burst intentionally shrinks to one probe, while a learned 5 RPS
+          -- pace still has five permits to share over a one-second window.
+          GREATEST(
+            w.burst_capacity,
+            CEIL(w.refill_per_sec * w.window_ms / 1000.0)
+          ) AS capacity,
           w.window_ms,
           w.now_ms - (w.now_ms % w.window_ms) AS started_at_ms
         FROM fair_window w
       ),
-      -- What this acquire has to write, if anything. A run asking many times
+      -- What this acquire has to write, if anything. A requester asking many times
       -- a second must not pay a sweep and an upsert on every one of them: a
       -- lone run's ledger row only has to exist and stay live, so it is
       -- rewritten when its window rolls or its lease nears expiry, and on
@@ -866,26 +1099,6 @@ function fairShareSql(
           EXISTS (SELECT 1 FROM peers) AS contended
         FROM fair_epoch e
       ),
-      -- Reclaims the oldest expired rows across every bucket, not just this
-      -- one: a bucket whose last run ended is never acquired again, so a
-      -- bucket-scoped sweep would leave its rows behind forever. Bounded per
-      -- acquire and driven by the expiry index, as the concurrency-observer
-      -- reclaim is. The row for this acquire's own (bucket, requester) is
-      -- never swept: it is about to be rewritten, and the peer snapshot below
-      -- reads this same statement snapshot.
-      expire_requesters AS (
-        DELETE FROM ${table} AS stale
-        WHERE (SELECT refresh OR contended FROM fair_write)
-          AND (stale.bucket_id, stale.requester_id) IN (
-          SELECT q.bucket_id, q.requester_id
-          FROM ${table} q
-          WHERE q.expires_at_ms <= (SELECT now_ms FROM fair_epoch)
-            AND NOT (q.bucket_id = $1 AND q.requester_id = ${requester})
-          ORDER BY q.expires_at_ms
-          LIMIT ${FAIR_SHARE_REQUESTER_CLEANUP_BATCH_SIZE}
-        )
-        RETURNING 1
-      ),
       fair_self AS (
         SELECT COALESCE((
           SELECT CASE
@@ -994,6 +1207,28 @@ function fairShareSql(
           last_seen_ms = EXCLUDED.last_seen_ms,
           expires_at_ms = EXCLUDED.expires_at_ms
         RETURNING 1
+      ),
+      -- Reclaims the oldest expired rows across every bucket, not just this
+      -- one: a bucket whose last run ended is never acquired again, so a
+      -- bucket-scoped sweep would leave its rows behind forever. Bounded per
+      -- acquire and driven by the expiry index, as the concurrency-observer
+      -- reclaim is. The row for this acquire's own (bucket, requester) is
+      -- never swept: it is about to be rewritten, and the peer snapshot below
+      -- reads this same statement snapshot.
+      expire_requesters AS (
+        DELETE FROM ${table} AS stale
+        WHERE (SELECT refresh OR contended FROM fair_write)
+          AND (SELECT COUNT(*) FROM remember_requester) >= 0
+          AND (stale.bucket_id, stale.requester_id) IN (
+          SELECT q.bucket_id, q.requester_id
+          FROM ${table} q
+          WHERE q.expires_at_ms <= (SELECT now_ms FROM fair_epoch)
+            AND NOT (q.bucket_id = $1 AND q.requester_id = ${requester})
+          ORDER BY q.expires_at_ms
+          LIMIT ${FAIR_SHARE_REQUESTER_CLEANUP_BATCH_SIZE}
+          FOR UPDATE OF q SKIP LOCKED
+        )
+        RETURNING 1
       )`,
   };
 }
@@ -1060,9 +1295,11 @@ async function acquirePureRate(
           adaptive_max_configured boolean
         )
       ),
+      ${acquireRateScopeSql(scopeHoldTableName(input), '$5')},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
+          AND EXISTS (SELECT 1 FROM provider_hold_rows)
           AND expires_at_ms <= (SELECT now_ms FROM now_value)
         RETURNING 1
       ),
@@ -1076,18 +1313,21 @@ async function acquirePureRate(
             WHEN sc.adaptive_max_configured THEN sc.max_rps
             ELSE b.max_rps
           END AS max_rps,
-          b.seen_backpressure,
+          ph.seen_backpressure,
           sc.base_refill_per_sec,
           LEAST(
             CASE
               WHEN sc.adaptive_max_configured THEN sc.max_rps
               ELSE b.max_rps
             END,
-            GREATEST(sc.min_rps, b.refill_per_sec)
+            GREATEST(sc.min_rps, ph.refill_per_sec)
           ) AS refill_per_sec,
-          b.cool_until_ms,
-          b.last_ramp_at_ms,
-          b.next_dispatch_at_ms,
+          normalized.next_refill_per_sec,
+          ph.cool_until_ms,
+          ph.last_ramp_at_ms,
+          ph.rate_recovery_interval_ms,
+          ph.observation_was_idle AS observation_was_idle,
+          GREATEST(b.next_dispatch_at_ms, ph.next_dispatch_at_ms) AS next_dispatch_at_ms,
           nv.now_ms AS now_ms,
           -- A warm row can have been widened by an older adaptive ceiling or
           -- edited out of band. Normalize it to the current seed before this
@@ -1100,12 +1340,15 @@ async function acquirePureRate(
           b.max_rps AS stored_max_rps,
           b.adaptive_max_configured OR sc.adaptive_max_configured
             AS adaptive_max_configured,
+          ph.retry_at_ms AS scope_retry_at_ms,
           -- Spendable burst: one second of the rate in force, floored at the
           -- declared window burst. The normalized row is what this acquire
           -- spends; the stored row remains available for drift reporting.
           ${effectiveCapacitySql('normalized')} AS effective_capacity,
+          ${effectiveCapacitySql('normalized', 'next_refill_per_sec')} AS scheduled_capacity,
           LEAST(
             ${effectiveCapacitySql('normalized')},
+            ph.tokens_after_refill,
             LEAST(b.tokens, sc.capacity)
               + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
                 * LEAST(
@@ -1113,22 +1356,33 @@ async function acquirePureRate(
                       WHEN sc.adaptive_max_configured THEN sc.max_rps
                       ELSE b.max_rps
                     END,
-                    GREATEST(sc.min_rps, b.refill_per_sec)
+                    GREATEST(sc.min_rps, ph.refill_per_sec)
                   )
           ) AS tokens_after_refill
         FROM ${bucketTableName(input)} b
         JOIN seed_config sc ON sc.rule_id = b.rule_id
         CROSS JOIN now_value nv
+        CROSS JOIN provider_hold ph
         CROSS JOIN LATERAL (
           SELECT
             sc.capacity AS capacity,
+            sc.base_refill_per_sec AS base_refill_per_sec,
+            ph.seen_backpressure AS seen_backpressure,
+            ph.rate_observation_started_at_ms AS rate_observation_started_at_ms,
             LEAST(
               CASE
                 WHEN sc.adaptive_max_configured THEN sc.max_rps
                 ELSE b.max_rps
               END,
-              GREATEST(sc.min_rps, b.refill_per_sec)
-            ) AS refill_per_sec
+              GREATEST(sc.min_rps, ph.refill_per_sec)
+            ) AS refill_per_sec,
+            LEAST(
+              CASE
+                WHEN sc.adaptive_max_configured THEN sc.max_rps
+                ELSE b.max_rps
+              END,
+              GREATEST(sc.min_rps, ph.next_refill_per_sec)
+            ) AS next_refill_per_sec
         ) normalized
         WHERE b.bucket_id = $1
           AND b.rule_id = ANY($2::text[])
@@ -1136,7 +1390,9 @@ async function acquirePureRate(
         FOR UPDATE OF b
       ),
       cooling AS (
-        SELECT COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms
+        SELECT
+          COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms,
+          COALESCE(MAX(scope_retry_at_ms), 0) AS retry_at_ms
         FROM refreshed
       ),${fair?.ctes ?? ''}
       grant_calc AS (
@@ -1147,10 +1403,14 @@ async function acquirePureRate(
             -- No rule rows means the bucket was never seeded (or was deleted).
             -- LEAST() ignores NULLs, so without this guard an unseeded bucket
             -- would "grant" the full request against a NULL schedule.
-            WHEN (SELECT COUNT(*) FROM refreshed) = 0 THEN 0
-            WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
+            WHEN (SELECT COUNT(*) FROM refreshed) = 0
+              OR (SELECT refill_per_sec FROM provider_hold) IS NULL THEN 0
+            WHEN GREATEST(
+              (SELECT cool_until_ms FROM cooling),
+              (SELECT retry_at_ms FROM cooling)
+            ) > (SELECT now_ms FROM refreshed LIMIT 1)
               THEN 0
-            ELSE GREATEST(0, LEAST($4::int, FLOOR(MIN(tokens_after_refill))::int${fair?.capTerm ?? ''}))
+            ELSE GREATEST(0, LEAST($4::int, FLOOR(MIN(tokens_after_refill) + 1e-9)::int${fair?.capTerm ?? ''}))
           END AS granted
         FROM refreshed
       ),
@@ -1160,25 +1420,36 @@ async function acquirePureRate(
           -- at the spacing in force when its permits were granted, so a lane
           -- that started at 0.25 rps carries a minute-long tail that a later
           -- ramp to 100 rps could never work off (the column is monotone).
-          -- Clamp the carried debt to one full bucket at the CURRENT refill.
+          -- Clamp the carried debt to one full bucket at the refill this
+          -- transaction is about to persist and report. Otherwise the grant
+          -- is rate-limited by the previous AIMD step for one extra batch.
           -- cool_until_ms stays inside the GREATEST, and penalize pushes
           -- next_dispatch past the cooldown, so a real 429 hold is never
           -- shortened by this clamp.
           GREATEST(
             MAX(now_ms),
             MAX(cool_until_ms),
+            MAX(scope_retry_at_ms),
             LEAST(
               MAX(next_dispatch_at_ms),
               MAX(now_ms)
                 + CEIL(
-                    MAX(effective_capacity / GREATEST(refill_per_sec, 0.0001) * 1000)
+                    MAX(
+                      scheduled_capacity
+                        / GREATEST(
+                          next_refill_per_sec,
+                          ${MIN_ADAPTIVE_RPS_FLOOR}
+                        )
+                        * 1000
+                    )
                   )::bigint
             )
           )::bigint AS first_at_ms,
-          CEIL(MAX(1000.0 / GREATEST(refill_per_sec, 0.0001)))::bigint AS spacing_ms,
-          GREATEST(1, FLOOR(MIN(refill_per_sec) / 100.0))::bigint AS burst_size
+          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR})))::bigint AS spacing_ms,
+          GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
+      ${updateRateScopeSql(scopeHoldTableName(input))},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -1189,6 +1460,10 @@ async function acquirePureRate(
           adaptive_max_configured = r.adaptive_max_configured,
           tokens = r.tokens_after_refill - (SELECT granted FROM grant_calc),
           refilled_at_ms = r.now_ms,
+          rate_observation_started_at_ms = (SELECT rate_observation_started_at_ms FROM updated_scope),
+          rate_successes = (SELECT rate_successes FROM updated_scope),
+          rate_recovery_interval_ms = (SELECT rate_recovery_interval_ms FROM updated_scope),
+          seen_backpressure = (SELECT seen_backpressure FROM updated_scope),
           next_dispatch_at_ms = CASE
             WHEN (SELECT granted FROM grant_calc) > 0
               THEN (SELECT first_at_ms FROM schedule_calc)
@@ -1203,29 +1478,10 @@ async function acquirePureRate(
           -- AIMD ramp, folded in. Reaching this refill proves a local lease
           -- block was exhausted; only a confirmed provider success can turn
           -- that demand into a rate increase. Slow-start doubles until the
-          -- first 429, then adds
-          -- max(1, base/10) per step. Never exceeds max_rps.
-          refill_per_sec = CASE
-            WHEN $5::int > 0
-              AND r.now_ms - r.last_ramp_at_ms >= 1000
-              AND r.cool_until_ms <= r.now_ms
-            THEN LEAST(
-              r.max_rps,
-              CASE
-                WHEN r.seen_backpressure
-                  THEN r.refill_per_sec + GREATEST(1, r.base_refill_per_sec / 10)
-                ELSE r.refill_per_sec * 2
-              END
-            )
-            ELSE r.refill_per_sec
-          END,
-          last_ramp_at_ms = CASE
-            WHEN $5::int > 0
-              AND r.now_ms - r.last_ramp_at_ms >= 1000
-              AND r.cool_until_ms <= r.now_ms
-            THEN r.now_ms
-            ELSE r.last_ramp_at_ms
-          END
+          -- first 429, then increases by at most 25% per successful recovery
+          -- step. Never exceeds max_rps.
+          refill_per_sec = LEAST(r.max_rps, (SELECT next_refill_per_sec FROM provider_hold)),
+          last_ramp_at_ms = (SELECT next_ramp_at_ms FROM provider_hold)
         FROM refreshed r
         WHERE b.bucket_id = r.bucket_id
           AND b.rule_id = r.rule_id
@@ -1264,7 +1520,8 @@ async function acquirePureRate(
       )${fair?.rememberCte ?? ''}
       SELECT
         (SELECT granted FROM grant_calc) AS granted,
-        (SELECT COUNT(*)::int FROM refreshed) AS observed_rule_rows,
+        CASE WHEN (SELECT refill_per_sec FROM provider_hold) IS NULL THEN 0
+          ELSE (SELECT COUNT(*)::int FROM refreshed) END AS observed_rule_rows,
         (
           SELECT COALESCE(
             bool_or(
@@ -1283,8 +1540,11 @@ async function acquirePureRate(
         ) AS config_drift,
         (SELECT now_ms FROM refreshed LIMIT 1) AS server_now_ms,
         (SELECT cool_until_ms FROM cooling) AS cool_until_ms,
-        (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
-           WHERE bucket_id = $1 AND rule_id = ANY($2::text[])) AS claimed_retry_at_ms,
+        GREATEST(
+          (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
+             WHERE bucket_id = $1 AND rule_id = ANY($2::text[])),
+          (SELECT retry_at_ms FROM provider_hold)
+        ) AS claimed_retry_at_ms,
         (SELECT COALESCE(MIN(new_refill_per_sec), 0) FROM updated) AS effective_rps,
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN ARRAY(
@@ -1319,8 +1579,14 @@ async function acquirePureRate(
         -- round trip per permit. The cooldown branch above still wins outright.
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN 0
-          WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
-            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
+          WHEN GREATEST(
+            (SELECT cool_until_ms FROM cooling),
+            (SELECT retry_at_ms FROM cooling)
+          ) > (SELECT now_ms FROM refreshed LIMIT 1)
+            THEN GREATEST(
+              (SELECT cool_until_ms FROM cooling),
+              (SELECT retry_at_ms FROM cooling)
+            ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
           ELSE (
             SELECT CEIL(
               MAX(
@@ -1333,7 +1599,7 @@ async function acquirePureRate(
                       FLOOR(new_refill_per_sec * ${BLOCK_WAIT_HORIZON_MS} / 1000.0)
                     )
                   ) - tokens_after_refill
-                ) / GREATEST(new_refill_per_sec, 0.0001) * 1000
+                ) / GREATEST(new_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000
               )
             )::bigint
             FROM updated
@@ -1469,15 +1735,18 @@ async function acquireWithConcurrency(
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
       ),
+      ${acquireRateScopeSql(scopeHoldTableName(input), '$10')},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
+          AND EXISTS (SELECT 1 FROM provider_hold_rows)
           AND expires_at_ms <= (SELECT now_ms FROM now_value)
         RETURNING 1
       ),
       reclaim AS (
         DELETE FROM ${concurrencyLeaseTableName(input)}
         WHERE bucket_id = $1
+          AND EXISTS (SELECT 1 FROM provider_hold_rows)
           AND rule_id = ANY($5::text[])
           AND expires_at_ms <= (SELECT now_ms FROM now_value)
         RETURNING 1
@@ -1506,20 +1775,23 @@ async function acquireWithConcurrency(
             WHEN sc.adaptive_max_configured THEN sc.max_rps
             ELSE b.max_rps
           END AS max_rps,
-          b.seen_backpressure,
+          ph.seen_backpressure,
           sc.base_refill_per_sec,
           LEAST(
             CASE
               WHEN sc.adaptive_max_configured THEN sc.max_rps
               ELSE b.max_rps
             END,
-            GREATEST(sc.min_rps, b.refill_per_sec)
+            GREATEST(sc.min_rps, ph.refill_per_sec)
           ) AS refill_per_sec,
+          normalized.next_refill_per_sec,
           b.learned_concurrency_limit,
           b.concurrency_minimum_rtt_ms,
-          b.cool_until_ms,
-          b.last_ramp_at_ms,
-          b.next_dispatch_at_ms,
+          ph.cool_until_ms,
+          ph.last_ramp_at_ms,
+          GREATEST(b.next_dispatch_at_ms, ph.next_dispatch_at_ms) AS next_dispatch_at_ms,
+          ph.rate_recovery_interval_ms,
+          ph.observation_was_idle AS observation_was_idle,
           nv.now_ms AS now_ms,
           -- Normalize a warm row before deriving its grant. The stored values
           -- are retained for the config_drift diagnostic below.
@@ -1529,11 +1801,14 @@ async function acquireWithConcurrency(
           b.max_rps AS stored_max_rps,
           b.adaptive_max_configured OR sc.adaptive_max_configured
             AS adaptive_max_configured,
+          ph.retry_at_ms AS scope_retry_at_ms,
           -- Same derived burst as the pure-rate path. The declared
           -- maxConcurrency is untouched: free_slots still bounds the grant.
           ${effectiveCapacitySql('normalized')} AS effective_capacity,
+          ${effectiveCapacitySql('normalized', 'next_refill_per_sec')} AS scheduled_capacity,
           LEAST(
             ${effectiveCapacitySql('normalized')},
+            ph.tokens_after_refill,
             LEAST(b.tokens, sc.capacity)
               + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
                 * LEAST(
@@ -1541,22 +1816,33 @@ async function acquireWithConcurrency(
                       WHEN sc.adaptive_max_configured THEN sc.max_rps
                       ELSE b.max_rps
                     END,
-                    GREATEST(sc.min_rps, b.refill_per_sec)
+                    GREATEST(sc.min_rps, ph.refill_per_sec)
                   )
           ) AS tokens_after_refill
         FROM ${bucketTableName(input)} b
         JOIN seed_config sc ON sc.rule_id = b.rule_id
         CROSS JOIN now_value nv
+        CROSS JOIN provider_hold ph
         CROSS JOIN LATERAL (
           SELECT
             sc.capacity AS capacity,
+            sc.base_refill_per_sec AS base_refill_per_sec,
+            ph.seen_backpressure AS seen_backpressure,
+            ph.rate_observation_started_at_ms AS rate_observation_started_at_ms,
             LEAST(
               CASE
                 WHEN sc.adaptive_max_configured THEN sc.max_rps
                 ELSE b.max_rps
               END,
-              GREATEST(sc.min_rps, b.refill_per_sec)
-            ) AS refill_per_sec
+              GREATEST(sc.min_rps, ph.refill_per_sec)
+            ) AS refill_per_sec,
+            LEAST(
+              CASE
+                WHEN sc.adaptive_max_configured THEN sc.max_rps
+                ELSE b.max_rps
+              END,
+              GREATEST(sc.min_rps, ph.next_refill_per_sec)
+            ) AS next_refill_per_sec
         ) normalized
         WHERE b.bucket_id = $1
           AND b.rule_id = ANY($2::text[])
@@ -1615,7 +1901,10 @@ async function acquireWithConcurrency(
         JOIN refreshed USING (rule_id)
       ),
       cooling AS (
-        SELECT COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms FROM refreshed
+        SELECT
+          COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms,
+          COALESCE(MAX(scope_retry_at_ms), 0) AS retry_at_ms
+        FROM refreshed
       ),${fair?.ctes ?? ''}
       grant_calc AS (
         SELECT
@@ -1623,14 +1912,18 @@ async function acquireWithConcurrency(
             -- No rule rows means the bucket was never seeded (or was deleted).
             -- LEAST() ignores NULLs, so without this guard an unseeded bucket
             -- would "grant" the full request against a NULL schedule.
-            WHEN (SELECT COUNT(*) FROM refreshed) = 0 THEN 0
-            WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
+            WHEN (SELECT COUNT(*) FROM refreshed) = 0
+              OR (SELECT refill_per_sec FROM provider_hold) IS NULL THEN 0
+            WHEN GREATEST(
+              (SELECT cool_until_ms FROM cooling),
+              (SELECT retry_at_ms FROM cooling)
+            ) > (SELECT now_ms FROM refreshed LIMIT 1)
               THEN 0
             ELSE GREATEST(
               0,
               LEAST(
                 $4::int,
-                FLOOR(MIN(tokens_after_refill))::int,
+                FLOOR(MIN(tokens_after_refill) + 1e-9)::int,
                 (SELECT free FROM free_slots)${fair?.capTerm ?? ''}
               )
             )
@@ -1643,25 +1936,36 @@ async function acquireWithConcurrency(
           -- at the spacing in force when its permits were granted, so a lane
           -- that started at 0.25 rps carries a minute-long tail that a later
           -- ramp to 100 rps could never work off (the column is monotone).
-          -- Clamp the carried debt to one full bucket at the CURRENT refill.
+          -- Clamp the carried debt to one full bucket at the refill this
+          -- transaction is about to persist and report. Otherwise the grant
+          -- is rate-limited by the previous AIMD step for one extra batch.
           -- cool_until_ms stays inside the GREATEST, and penalize pushes
           -- next_dispatch past the cooldown, so a real 429 hold is never
           -- shortened by this clamp.
           GREATEST(
             MAX(now_ms),
             MAX(cool_until_ms),
+            MAX(scope_retry_at_ms),
             LEAST(
               MAX(next_dispatch_at_ms),
               MAX(now_ms)
                 + CEIL(
-                    MAX(effective_capacity / GREATEST(refill_per_sec, 0.0001) * 1000)
+                    MAX(
+                      scheduled_capacity
+                        / GREATEST(
+                          next_refill_per_sec,
+                          ${MIN_ADAPTIVE_RPS_FLOOR}
+                        )
+                        * 1000
+                    )
                   )::bigint
             )
           )::bigint AS first_at_ms,
-          CEIL(MAX(1000.0 / GREATEST(refill_per_sec, 0.0001)))::bigint AS spacing_ms,
-          GREATEST(1, FLOOR(MIN(refill_per_sec) / 100.0))::bigint AS burst_size
+          CEIL(MAX(1000.0 / GREATEST(next_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR})))::bigint AS spacing_ms,
+          GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
+      ${updateRateScopeSql(scopeHoldTableName(input))},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -1683,30 +1987,15 @@ async function acquireWithConcurrency(
                   * (SELECT burst_size FROM schedule_calc)
             ELSE b.next_dispatch_at_ms
           END,
+          rate_observation_started_at_ms = (SELECT rate_observation_started_at_ms FROM updated_scope),
+          rate_successes = (SELECT rate_successes FROM updated_scope),
+          rate_recovery_interval_ms = (SELECT rate_recovery_interval_ms FROM updated_scope),
+          seen_backpressure = (SELECT seen_backpressure FROM updated_scope),
           -- Match the pure-rate path: a depleted local lease block plus a
           -- confirmed success is sufficient demand; concurrency leases change
           -- admission accounting, not the AIMD learning contract.
-          refill_per_sec = CASE
-            WHEN $10::int > 0
-              AND r.now_ms - r.last_ramp_at_ms >= 1000
-              AND r.cool_until_ms <= r.now_ms
-            THEN LEAST(
-              r.max_rps,
-              CASE
-                WHEN r.seen_backpressure
-                  THEN r.refill_per_sec + GREATEST(1, r.base_refill_per_sec / 10)
-                ELSE r.refill_per_sec * 2
-              END
-            )
-            ELSE r.refill_per_sec
-          END,
-          last_ramp_at_ms = CASE
-            WHEN $10::int > 0
-              AND r.now_ms - r.last_ramp_at_ms >= 1000
-              AND r.cool_until_ms <= r.now_ms
-            THEN r.now_ms
-            ELSE r.last_ramp_at_ms
-          END
+          refill_per_sec = LEAST(r.max_rps, (SELECT next_refill_per_sec FROM provider_hold)),
+          last_ramp_at_ms = (SELECT next_ramp_at_ms FROM provider_hold)
         FROM refreshed r
         WHERE b.bucket_id = $1
           AND b.rule_id = r.rule_id
@@ -1773,7 +2062,8 @@ async function acquireWithConcurrency(
       )${fair?.rememberCte ?? ''}
       SELECT
         (SELECT granted FROM grant_calc) AS granted,
-        (SELECT COUNT(*)::int FROM refreshed) AS observed_rule_rows,
+        CASE WHEN (SELECT refill_per_sec FROM provider_hold) IS NULL THEN 0
+          ELSE (SELECT COUNT(*)::int FROM refreshed) END AS observed_rule_rows,
         (
           SELECT COALESCE(
             bool_or(
@@ -1792,8 +2082,11 @@ async function acquireWithConcurrency(
         ) AS config_drift,
         (SELECT now_ms FROM refreshed LIMIT 1) AS server_now_ms,
         (SELECT cool_until_ms FROM cooling) AS cool_until_ms,
-        (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
-           WHERE bucket_id = $1 AND rule_id = ANY($2::text[])) AS claimed_retry_at_ms,
+        GREATEST(
+          (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
+             WHERE bucket_id = $1 AND rule_id = ANY($2::text[])),
+          (SELECT retry_at_ms FROM provider_hold)
+        ) AS claimed_retry_at_ms,
         (SELECT COALESCE(MIN(new_refill_per_sec), 0) FROM updated) AS effective_rps,
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN ARRAY(
@@ -1839,12 +2132,18 @@ async function acquireWithConcurrency(
         END AS concurrency_grants,
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN 0
-          WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
-            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
+          WHEN GREATEST(
+            (SELECT cool_until_ms FROM cooling),
+            (SELECT retry_at_ms FROM cooling)
+          ) > (SELECT now_ms FROM refreshed LIMIT 1)
+            THEN GREATEST(
+              (SELECT cool_until_ms FROM cooling),
+              (SELECT retry_at_ms FROM cooling)
+            ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
           WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
           ELSE (
             SELECT CEIL(
-              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, 0.0001) * 1000)
+              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, ${MIN_ADAPTIVE_RPS_FLOOR}) * 1000)
             )::bigint
             FROM updated
           )
@@ -1933,6 +2232,60 @@ export async function acquirePostgresRateState(
   );
 }
 
+/**
+ * Validate a cached permit at the physical dispatch boundary. The shared hold
+ * row lock linearizes this check with a concurrent Retry-After write; callers
+ * that lose the race return their local slots before reacquiring.
+ */
+export async function validatePostgresRateStatePermit(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerRateStateValidateInput & {
+    nowMs?: number;
+  },
+): Promise<PlayRunnerRateStateValidateResult> {
+  return await withRateStateAvailabilityGuard(input, async () => {
+    const result = await client.query<{
+      server_now_ms: string | number;
+      retry_at_ms: string | number;
+    }>(
+      `
+        WITH now_value AS (
+          SELECT COALESCE(
+            $2::bigint,
+            FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
+          ) AS now_ms
+        ),
+        hold_rows AS (
+          SELECT retry_at_ms
+          FROM ${scopeHoldTableName(input)}
+          WHERE bucket_id = $1
+          FOR SHARE
+        )
+        SELECT
+          now_value.now_ms AS server_now_ms,
+          COALESCE(MAX(hold_rows.retry_at_ms), 0) AS retry_at_ms
+        FROM now_value
+        LEFT JOIN hold_rows ON true
+        GROUP BY now_value.now_ms
+      `,
+      [input.bucketId, input.nowMs ?? null],
+    );
+    const row = result.rows[0];
+    if (!row) {
+      throw new Error(
+        `Rate-state hold validation returned no row for ${input.bucketId}.`,
+      );
+    }
+    const serverNowMs = Math.max(0, Number(row.server_now_ms));
+    const retryAtMs = Math.max(0, Number(row.retry_at_ms));
+    return {
+      allowed: retryAtMs <= serverNowMs,
+      serverNowMs,
+      retryAtMs,
+    };
+  });
+}
+
 type ConcurrencyObservation = Gradient2Observation & {
   kind: 'success' | 'throttled';
 };
@@ -2045,6 +2398,7 @@ async function concurrencyThrottleFrom(
   input: PlayRunnerRateStatePenalizeInput,
   rules: readonly AcquirePacingRule[],
 ): Promise<{
+  attribution: PostgresRateStatePenaltyTelemetry['gradient2Attribution'];
   observations: ConcurrencyObservation[];
   batch: ConcurrencyObservationBatch | null;
 }> {
@@ -2053,7 +2407,7 @@ async function concurrencyThrottleFrom(
     !Number.isFinite(input.latencyMs) ||
     input.latencyMs <= 0
   ) {
-    return { observations: [], batch: null };
+    return { attribution: 'not_observed', observations: [], batch: null };
   }
   const grant = input.concurrencyGrant;
   if (
@@ -2062,13 +2416,13 @@ async function concurrencyThrottleFrom(
     !grant.admissionId.trim() ||
     grant.admissionId.trim().length > 64
   ) {
-    return { observations: [], batch: null };
+    return { attribution: 'not_observed', observations: [], batch: null };
   }
   const concurrencyRuleIds = rules
     .filter((rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0)
     .map((rule) => rule.ruleId);
   if (concurrencyRuleIds.length === 0) {
-    return { observations: [], batch: null };
+    return { attribution: 'not_observed', observations: [], batch: null };
   }
   const attributedLease = await client.query<{
     in_flight_at_start: number | null;
@@ -2104,7 +2458,7 @@ async function concurrencyThrottleFrom(
       minimumRttMs !== undefined &&
       (!Number.isFinite(minimumRttMs) || minimumRttMs <= 0))
   ) {
-    return { observations: [], batch: null };
+    return { attribution: 'not_observed', observations: [], batch: null };
   }
   const attribution = classifyProviderThrottle({
     grant: {
@@ -2113,8 +2467,11 @@ async function concurrencyThrottleFrom(
     },
     inFlightAtStart,
   });
-  if (attribution === 'rate') return { observations: [], batch: null };
+  if (attribution === 'rate') {
+    return { attribution, observations: [], batch: null };
+  }
   return {
+    attribution,
     observations: [
       {
         kind: 'throttled',
@@ -2142,15 +2499,65 @@ async function applyConcurrencyObservations(
   input: Pick<PlayRunnerRateStateAcquireInput, 'bucketId'> & {
     schema?: string | null;
     schedulerSchema?: string | null;
+    observedSuccesses?: number;
   },
   rules: readonly AcquirePacingRule[],
   observations: readonly ConcurrencyObservation[],
   batch: ConcurrencyObservationBatch | null,
-): Promise<'unbatched' | 'applied' | 'already_applied'> {
+): Promise<{
+  disposition: 'unbatched' | 'applied' | 'already_applied';
+  ruleCount: number;
+  changedRuleCount: number;
+  limitMinBefore: number | null;
+  limitMaxBefore: number | null;
+  limitMinAfter: number | null;
+  limitMaxAfter: number | null;
+}> {
   const concurrencyRules = rules.filter(
     (rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0,
   );
-  if (batch === null) return 'unbatched';
+  const countSuccesses = async () => {
+    const successes = Math.max(
+      0,
+      Math.min(1_000_000, Math.trunc(input.observedSuccesses ?? 0)),
+    );
+    if (successes <= 0) return;
+    await client.query(
+      `
+      UPDATE ${scopeHoldTableName(input)}
+      SET rate_successes = rate_successes + $2::bigint,
+          refill_per_sec = CASE
+            -- A 429 can beat other requests already in flight. Until the
+            -- first post-hold grant starts a new observation epoch, their
+            -- confirmed successes refine the same estimate, never past the
+            -- first throttle's multiplicative-decrease ceiling.
+            WHEN rate_observation_started_at_ms IS NULL
+              AND rate_throttle_ceiling_rps IS NOT NULL
+              AND rate_recovery_interval_ms > 0
+            THEN GREATEST(refill_per_sec, LEAST(
+              rate_throttle_ceiling_rps,
+              (rate_successes + $2::bigint) * 1000.0 / rate_recovery_interval_ms
+            ))
+            ELSE refill_per_sec END
+      WHERE bucket_id = $1
+    `,
+      [input.bucketId, successes],
+    );
+  };
+  if (batch === null) {
+    // Legacy writers have no observation identity. Preserve their success
+    // input; new writers always supply the replay-safe batch below.
+    await countSuccesses();
+    return {
+      disposition: 'unbatched',
+      ruleCount: 0,
+      changedRuleCount: 0,
+      limitMinBefore: null,
+      limitMaxBefore: null,
+      limitMinAfter: null,
+      limitMaxAfter: null,
+    };
+  }
 
   await client.query('BEGIN');
   try {
@@ -2159,19 +2566,7 @@ async function applyConcurrencyObservations(
         WITH now_value AS (
           SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint AS now_ms
         ),
-        expired AS (
-          DELETE FROM ${concurrencyObserverTableName(input)} AS observer
-          WHERE (observer.bucket_id, observer.observer_id, observer.lane_id) IN (
-            SELECT
-              stale.bucket_id,
-              stale.observer_id,
-              stale.lane_id
-            FROM ${concurrencyObserverTableName(input)} AS stale
-            WHERE stale.expires_at_ms <= (SELECT now_ms FROM now_value)
-            ORDER BY stale.expires_at_ms
-            LIMIT ${CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE}
-          )
-        )
+        accepted AS (
         INSERT INTO ${concurrencyObserverTableName(input)} AS observer (
           bucket_id,
           observer_id,
@@ -2195,6 +2590,23 @@ async function applyConcurrencyObservations(
           expires_at_ms = EXCLUDED.expires_at_ms
         WHERE observer.last_sequence < EXCLUDED.last_sequence
         RETURNING last_sequence
+        ), expired AS (
+          -- Own this observation before opportunistically reclaiming others.
+          -- Never wait for another feedback writer or delete our renewed row
+          -- through this statement's pre-update snapshot.
+          DELETE FROM ${concurrencyObserverTableName(input)} AS observer
+          WHERE (SELECT COUNT(*) FROM accepted) >= 0
+            AND (observer.bucket_id, observer.observer_id, observer.lane_id) IN (
+              SELECT stale.bucket_id, stale.observer_id, stale.lane_id
+              FROM ${concurrencyObserverTableName(input)} AS stale
+              WHERE stale.expires_at_ms <= (SELECT now_ms FROM now_value)
+                AND NOT (stale.bucket_id = $1 AND stale.observer_id = $2 AND stale.lane_id = $3)
+              ORDER BY stale.expires_at_ms
+              LIMIT ${CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE}
+              FOR UPDATE OF stale SKIP LOCKED
+            )
+        )
+        SELECT last_sequence FROM accepted
       `,
       [
         input.bucketId,
@@ -2209,11 +2621,30 @@ async function applyConcurrencyObservations(
     // than replaying its samples.
     if (accepted.rows.length === 0) {
       await client.query('COMMIT');
-      return 'already_applied';
+      return {
+        disposition: 'already_applied',
+        ruleCount: 0,
+        changedRuleCount: 0,
+        limitMinBefore: null,
+        limitMaxBefore: null,
+        limitMinAfter: null,
+        limitMaxAfter: null,
+      };
     }
+    // Count real responses, not leased tickets. Commit the count with its
+    // observation identity so a lost response cannot inflate learned goodput.
+    await countSuccesses();
     if (concurrencyRules.length === 0 || observations.length === 0) {
       await client.query('COMMIT');
-      return 'applied';
+      return {
+        disposition: 'applied',
+        ruleCount: 0,
+        changedRuleCount: 0,
+        limitMinBefore: null,
+        limitMaxBefore: null,
+        limitMinAfter: null,
+        limitMaxAfter: null,
+      };
     }
     const rows = await client.query<{
       rule_id: string;
@@ -2247,6 +2678,9 @@ async function applyConcurrencyObservations(
     const ruleById = new Map(
       concurrencyRules.map((rule) => [rule.ruleId, rule]),
     );
+    const limitsBefore: number[] = [];
+    const limitsAfter: number[] = [];
+    let changedRuleCount = 0;
     for (const row of rows.rows) {
       const rule = ruleById.get(row.rule_id);
       if (!rule?.maxConcurrency) continue;
@@ -2258,6 +2692,7 @@ async function applyConcurrencyObservations(
           row.learned_concurrency_limit ?? INITIAL_CONCURRENCY_DISCOVERY_LIMIT,
         ),
       );
+      limitsBefore.push(initialLimit);
       const controller = new Gradient2ConcurrencyController(
         providerConcurrencyControllerOptions({
           initialLimit,
@@ -2284,6 +2719,8 @@ async function applyConcurrencyObservations(
       );
       for (const observation of observations) controller.observe(observation);
       const checkpoint = controller.checkpoint();
+      limitsAfter.push(checkpoint.limit);
+      if (checkpoint.limit !== initialLimit) changedRuleCount += 1;
       await client.query(
         `
           UPDATE ${bucketTableName(input)}
@@ -2310,8 +2747,18 @@ async function applyConcurrencyObservations(
         ],
       );
     }
+    const beforeRange = numberRange(limitsBefore);
+    const afterRange = numberRange(limitsAfter);
     await client.query('COMMIT');
-    return 'applied';
+    return {
+      disposition: 'applied',
+      ruleCount: limitsBefore.length,
+      changedRuleCount,
+      limitMinBefore: beforeRange.min,
+      limitMaxBefore: beforeRange.max,
+      limitMinAfter: afterRange.min,
+      limitMaxAfter: afterRange.max,
+    };
   } catch (error) {
     try {
       await client.query('ROLLBACK');
@@ -2351,7 +2798,7 @@ async function acquireRateStateStatements(
     await seedBucketRows(client, input, seeds, nowMs);
     seeded = true;
   }
-  const observationBatchDisposition = await applyConcurrencyObservations(
+  const observationBatch = await applyConcurrencyObservations(
     client,
     input,
     rules,
@@ -2362,7 +2809,7 @@ async function acquireRateStateStatements(
   // runner lost the prior acquire response, its retry still needs a fresh
   // permit, but it must not ramp AIMD or advance Gradient2 a second time.
   grantInput =
-    observationBatchDisposition === 'already_applied'
+    observationBatch.disposition === 'already_applied'
       ? { ...input, observedSuccesses: 0 }
       : input;
   let outcome = await grant();
@@ -2426,6 +2873,21 @@ async function releaseRateStateStatements(
     ),
   ];
   if (refundReservationIds.length > 0) {
+    // A pre-expansion gateway can leave a valid reservation and rule row but
+    // no scope owner. Create its uninitialized hold before taking the normal
+    // hold -> reservation -> rule lock path; seedBucketRows will initialize its
+    // learner on the next admission and preserve this inherited deadline.
+    await client.query(
+      `
+        INSERT INTO ${scopeHoldTableName(input)} (bucket_id, retry_at_ms)
+        SELECT bucket_id, MAX(claimed_retry_at_ms)
+        FROM ${bucketTableName(input)}
+        WHERE bucket_id = $1
+        GROUP BY bucket_id
+        ON CONFLICT (bucket_id) DO NOTHING
+      `,
+      [input.bucketId],
+    );
     // Delete only a server-minted, unexpired reservation. That one mutation is
     // both authorization and idempotency: arbitrary runner input cannot credit
     // a bucket, and an HTTP retry sees no reservation left to refund. The shared
@@ -2439,9 +2901,13 @@ async function releaseRateStateStatements(
             FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
           ) AS now_ms
         ),
+        held AS (
+          SELECT * FROM ${scopeHoldTableName(input)} WHERE bucket_id = $1 FOR UPDATE
+        ),
         refunded AS (
           DELETE FROM ${reservationTableName(input)}
           WHERE bucket_id = $1
+            AND EXISTS (SELECT 1 FROM held)
             AND reservation_id = ANY($4::text[])
             AND expires_at_ms > (SELECT now_ms FROM now_value)
           RETURNING reservation_id
@@ -2449,6 +2915,19 @@ async function releaseRateStateStatements(
         refund_count AS (
           SELECT COUNT(*)::float8 AS value FROM refunded
         ),
+        refunded_scope AS (
+          UPDATE ${scopeHoldTableName(input)} h SET
+            tokens = CASE WHEN h.refill_per_sec IS NULL THEN h.tokens ELSE LEAST(
+              GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, h.refill_per_sec),
+              h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) / 1000.0 * h.refill_per_sec
+                + (SELECT value FROM refund_count)
+            ) END,
+            refilled_at_ms = CASE WHEN h.refill_per_sec IS NULL
+              THEN h.refilled_at_ms ELSE nv.now_ms END
+          FROM now_value nv
+          WHERE h.bucket_id = $1 AND (SELECT value FROM refund_count) > 0
+          RETURNING h.bucket_id
+        ),
         refreshed AS (
           SELECT
             b.bucket_id,
@@ -2460,6 +2939,7 @@ async function releaseRateStateStatements(
             nv.now_ms
           FROM ${bucketTableName(input)} b
           CROSS JOIN now_value nv
+          JOIN refunded_scope ON refunded_scope.bucket_id = b.bucket_id
           WHERE b.bucket_id = $1
             AND b.rule_id = ANY($2::text[])
           ORDER BY b.rule_id
@@ -2513,7 +2993,7 @@ async function releaseRateStateStatements(
 export async function penalizePostgresRateState(
   client: PostgresSchedulerQueryClient,
   input: PlayRunnerRateStatePenalizeInput & { nowMs?: number },
-): Promise<void> {
+): Promise<PostgresRateStatePenaltyTelemetry> {
   return await withRateStateAvailabilityGuard(input, () =>
     penalizeRateStateStatements(client, input),
   );
@@ -2522,58 +3002,268 @@ export async function penalizePostgresRateState(
 async function penalizeRateStateStatements(
   client: PostgresSchedulerQueryClient,
   input: PlayRunnerRateStatePenalizeInput & { nowMs?: number },
-): Promise<void> {
-  if (!Number.isFinite(input.cooldownMs) || input.cooldownMs <= 0) return;
-  // Single autocommit statement applied to every rule row of the bucket:
-  //  - claimed_retry_at_ms: server Retry-After stored verbatim (observability).
-  //  - cool_until_ms: extended by the cooldown, internally capped at 45min.
+): Promise<PostgresRateStatePenaltyTelemetry> {
+  const cooldownMs =
+    Number.isFinite(input.cooldownMs) && input.cooldownMs > 0
+      ? Math.ceil(input.cooldownMs)
+      : 0;
+  const retryAtMs =
+    input.retryAtMs != null && Number.isFinite(input.retryAtMs)
+      ? Math.max(0, Math.ceil(input.retryAtMs))
+      : null;
+  if (cooldownMs <= 0 && retryAtMs == null) {
+    return emptyRateStatePenaltyTelemetry();
+  }
+  const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
+  await applyConcurrencyObservations(
+    client,
+    input,
+    rules,
+    concurrencyObservationsFrom(input.concurrencyObservations),
+    concurrencyObservationBatchFrom(input.concurrencyObservationBatch),
+  );
+  // Lock/update the provider-scope hold before touching rate-rule rows, matching
+  // acquire's lock order. The absolute deadline is idempotent across transport
+  // retries and gates admission independently of the capped AIMD cooldown.
   //  - refill_per_sec: multiplicative decrease, but ONLY once per active
   //    cooldown window (halve-once guard: skip if already cooling), floored at
   //    min_rps.
   //  - seen_backpressure flips the ramp from slow-start (x2) to additive.
   //  - tokens drained: no more grants until refill re-accrues past the cooldown.
   await client.query(
+    `INSERT INTO ${scopeHoldTableName(input)} (bucket_id) VALUES ($1) ON CONFLICT DO NOTHING`,
+    [input.bucketId],
+  );
+  const rateStateResult = await client.query<{
+    retry_at_ms: string | number | null;
+    cool_until_ms: string | number | null;
+    rate_rule_count: string | number;
+    rate_decreased_rule_count: string | number;
+    refill_min_before_rps: string | number | null;
+    refill_max_before_rps: string | number | null;
+    refill_min_after_rps: string | number | null;
+    refill_max_after_rps: string | number | null;
+    observation_cycle_max_ms: string | number | null;
+    confirmed_successes_max: string | number | null;
+  }>(
     `
       WITH now_value AS (
         SELECT COALESCE(
           $3::bigint,
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
+      ), held AS (
+        SELECT * FROM ${scopeHoldTableName(input)} WHERE bucket_id = $1 FOR UPDATE
+      ), evidence AS (
+        SELECT h.*,
+          COALESCE(h.refill_per_sec, 1) AS refill_before,
+          GREATEST(h.retry_at_ms, COALESCE($5::bigint, nv.now_ms + $2::bigint)) AS deadline,
+          (
+            COALESCE($5::bigint, nv.now_ms + $2::bigint) > nv.now_ms
+            -- A short HTTP-date may expire between the provider response and
+            -- this durable feedback write. A strictly newer absolute deadline
+            -- is still fresh throttle evidence and must update AIMD even when
+            -- it no longer requires a wait. Equal/older deadlines are replayed
+            -- feedback and must not start another decrease episode.
+            OR ($5::bigint IS NOT NULL AND $5::bigint > h.retry_at_ms)
+          )
+            AND nv.now_ms >= h.retry_at_ms
+            AND (h.rate_observation_started_at_ms IS NOT NULL OR h.retry_at_ms = 0)
+              AS new_episode,
+          GREATEST(1000,
+            CASE
+              WHEN $5::bigint IS NOT NULL AND h.rate_successes <= 0 THEN 1000
+              WHEN $5::bigint IS NOT NULL THEN
+                GREATEST(h.retry_at_ms, $5::bigint)
+                  - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
+              ELSE
+                GREATEST(h.retry_at_ms, COALESCE($5::bigint, nv.now_ms + $2::bigint))
+                  - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
+            END) AS cycle_ms,
+          nv.now_ms
+        FROM held h CROSS JOIN now_value nv
+      ), throttle_ceiling AS (
+        SELECT e.*,
+          GREATEST(${MIN_ADAPTIVE_RPS_FLOOR},
+            CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
+              CASE WHEN $5::bigint IS NOT NULL
+                -- An explicit Retry-After is only the provider's next safe
+                -- dispatch deadline. It is not a rate measurement; with no
+                -- successful-call evidence, use one conservative probe per
+                -- second after that deadline instead of deriving a permanent
+                -- RPM limit from the length of the hold.
+                THEN GREATEST(e.refill_before, 1000.0 / e.cycle_ms)
+                ELSE e.refill_before END
+              ELSE e.refill_before / 2 END
+          ) AS next_ceiling_rps
+        FROM evidence e
+      ), learned AS (
+        SELECT e.*,
+          CASE WHEN e.new_episode THEN GREATEST(${MIN_ADAPTIVE_RPS_FLOOR}, LEAST(
+            e.next_ceiling_rps,
+            GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
+          )) ELSE e.refill_before END AS refill_after
+        FROM throttle_ceiling e
+      ), provider_hold AS (
+        UPDATE ${scopeHoldTableName(input)} h SET
+          retry_at_ms = e.deadline,
+          refill_per_sec = e.refill_after,
+          seen_backpressure = e.seen_backpressure OR e.new_episode,
+          cool_until_ms = GREATEST(e.cool_until_ms, LEAST(e.deadline, e.now_ms + LEAST($2::bigint, $4::bigint))),
+          tokens = CASE WHEN e.new_episode THEN 0 ELSE e.tokens END,
+          rate_successes = e.rate_successes,
+          rate_throttle_ceiling_rps = CASE WHEN e.new_episode
+            THEN e.next_ceiling_rps ELSE e.rate_throttle_ceiling_rps END,
+          rate_observation_started_at_ms = CASE WHEN e.new_episode THEN NULL ELSE e.rate_observation_started_at_ms END,
+          rate_recovery_interval_ms = CASE WHEN e.new_episode THEN e.cycle_ms ELSE e.rate_recovery_interval_ms END,
+          last_ramp_at_ms = CASE WHEN e.new_episode THEN e.deadline ELSE e.last_ramp_at_ms END,
+          -- All pre-429 permits are revalidated against this durable hold
+          -- before dispatch. Their obsolete schedule debt must not add a
+          -- second wait after the upstream's explicit retry deadline.
+          next_dispatch_at_ms = CASE WHEN e.new_episode
+            THEN e.deadline ELSE GREATEST(e.next_dispatch_at_ms, e.deadline) END
+        FROM learned e WHERE h.bucket_id = e.bucket_id
+        RETURNING h.*, e.new_episode, e.cycle_ms, e.rate_successes AS successes_before
+      ), locked_bucket_rows AS (
+        SELECT b.bucket_id, b.rule_id, b.refill_per_sec AS refill_before
+        FROM ${bucketTableName(input)} b
+        JOIN provider_hold h ON h.bucket_id = b.bucket_id
+        ORDER BY b.rule_id FOR UPDATE OF b
+      ), updated AS (
+        -- Compatibility projections only: no per-rule learning history is
+        -- consulted. Inactive operations inherit the same scope evidence.
+        UPDATE ${bucketTableName(input)} b SET
+          claimed_retry_at_ms = GREATEST(b.claimed_retry_at_ms, h.retry_at_ms),
+          refill_per_sec = LEAST(b.max_rps, GREATEST(b.min_rps, h.refill_per_sec)),
+          rate_successes = h.rate_successes,
+          rate_observation_started_at_ms = h.rate_observation_started_at_ms,
+          rate_recovery_interval_ms = h.rate_recovery_interval_ms,
+          last_ramp_at_ms = h.last_ramp_at_ms,
+          cool_until_ms = h.cool_until_ms,
+          seen_backpressure = h.seen_backpressure,
+          tokens = CASE WHEN h.new_episode THEN 0 ELSE b.tokens END,
+          next_dispatch_at_ms = CASE WHEN h.new_episode
+            THEN h.next_dispatch_at_ms
+            ELSE GREATEST(b.next_dispatch_at_ms, h.next_dispatch_at_ms) END
+        FROM locked_bucket_rows locked CROSS JOIN provider_hold h
+        WHERE b.bucket_id = locked.bucket_id AND b.rule_id = locked.rule_id
+        RETURNING locked.refill_before, h.cycle_ms, h.successes_before,
+          b.refill_per_sec AS refill_after, b.cool_until_ms
       )
-      UPDATE ${bucketTableName(input)} b
-      SET
-        claimed_retry_at_ms = (SELECT now_ms FROM now_value) + $2::bigint,
-        refill_per_sec = CASE
-          WHEN (SELECT now_ms FROM now_value) >= b.cool_until_ms
-            THEN GREATEST(b.min_rps, b.refill_per_sec / 2)
-          ELSE b.refill_per_sec
-        END,
-        cool_until_ms = GREATEST(
-          b.cool_until_ms,
-          (SELECT now_ms FROM now_value) + LEAST($2::bigint, $4::bigint)
-        ),
-        seen_backpressure = true,
-        tokens = 0,
-        next_dispatch_at_ms = GREATEST(
-          b.next_dispatch_at_ms,
-          (SELECT now_ms FROM now_value) + LEAST($2::bigint, $4::bigint)
-        )
-      WHERE b.bucket_id = $1
+      SELECT
+        (SELECT MAX(retry_at_ms) FROM provider_hold) AS retry_at_ms,
+        MAX(updated.cool_until_ms) AS cool_until_ms,
+        COUNT(*)::int AS rate_rule_count,
+        COUNT(*) FILTER (WHERE updated.refill_after < updated.refill_before)::int
+          AS rate_decreased_rule_count,
+        MIN(updated.refill_before) AS refill_min_before_rps,
+        MAX(updated.refill_before) AS refill_max_before_rps,
+        MIN(updated.refill_after) AS refill_min_after_rps,
+        MAX(updated.refill_after) AS refill_max_after_rps,
+        MAX(updated.cycle_ms) AS observation_cycle_max_ms,
+        MAX(updated.successes_before) AS confirmed_successes_max
+      FROM updated
     `,
     [
       input.bucketId,
-      Math.ceil(input.cooldownMs),
+      cooldownMs,
       input.nowMs ?? null,
-      MAX_INTERNAL_COOLDOWN_MS,
+      PROVIDER_AIMD_COOLDOWN_CAP_MS,
+      retryAtMs,
     ],
   );
-  const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
+  const rateRow = rateStateResult.rows[0];
+  const aimd = {
+    retryAtMs: finiteNumberOrNull(rateRow?.retry_at_ms),
+    coolUntilMs: finiteNumberOrNull(rateRow?.cool_until_ms),
+    rateRuleCount: Math.max(
+      0,
+      Math.trunc(Number(rateRow?.rate_rule_count ?? 0)),
+    ),
+    rateDecreasedRuleCount: Math.max(
+      0,
+      Math.trunc(Number(rateRow?.rate_decreased_rule_count ?? 0)),
+    ),
+    refillMinBeforeRps: finiteNumberOrNull(rateRow?.refill_min_before_rps),
+    refillMaxBeforeRps: finiteNumberOrNull(rateRow?.refill_max_before_rps),
+    refillMinAfterRps: finiteNumberOrNull(rateRow?.refill_min_after_rps),
+    refillMaxAfterRps: finiteNumberOrNull(rateRow?.refill_max_after_rps),
+    observationCycleMaxMs: finiteNumberOrNull(
+      rateRow?.observation_cycle_max_ms,
+    ),
+    confirmedSuccessesMax: finiteNumberOrNull(rateRow?.confirmed_successes_max),
+  };
+  const configuredConcurrencyRuleCount = rules.filter(
+    (rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0,
+  ).length;
   const throttle = await concurrencyThrottleFrom(client, input, rules);
-  await applyConcurrencyObservations(
+  // The first observation fold consumed buffered successes before AIMD used
+  // them. This second fold owns Gradient2's throttle sample only.
+  const gradient2 = await applyConcurrencyObservations(
     client,
-    input,
+    { ...input, observedSuccesses: 0 },
     rules,
     throttle.observations,
     throttle.batch,
   );
+  return {
+    ...aimd,
+    configuredConcurrencyRuleCount,
+    gradient2Attribution: throttle.attribution,
+    gradient2Disposition:
+      throttle.attribution === 'rate' ||
+      throttle.attribution === 'not_observed' ||
+      gradient2.disposition === 'unbatched'
+        ? 'not_observed'
+        : gradient2.disposition,
+    gradient2RuleCount: gradient2.ruleCount,
+    gradient2ChangedRuleCount: gradient2.changedRuleCount,
+    gradient2LimitMinBefore: gradient2.limitMinBefore,
+    gradient2LimitMaxBefore: gradient2.limitMaxBefore,
+    gradient2LimitMinAfter: gradient2.limitMinAfter,
+    gradient2LimitMaxAfter: gradient2.limitMaxAfter,
+  };
+}
+
+function emptyRateStatePenaltyTelemetry(): PostgresRateStatePenaltyTelemetry {
+  return {
+    retryAtMs: null,
+    coolUntilMs: null,
+    rateRuleCount: 0,
+    rateDecreasedRuleCount: 0,
+    refillMinBeforeRps: null,
+    refillMaxBeforeRps: null,
+    refillMinAfterRps: null,
+    refillMaxAfterRps: null,
+    observationCycleMaxMs: null,
+    confirmedSuccessesMax: null,
+    configuredConcurrencyRuleCount: 0,
+    gradient2Attribution: 'not_observed',
+    gradient2Disposition: 'not_observed',
+    gradient2RuleCount: 0,
+    gradient2ChangedRuleCount: 0,
+    gradient2LimitMinBefore: null,
+    gradient2LimitMaxBefore: null,
+    gradient2LimitMinAfter: null,
+    gradient2LimitMaxAfter: null,
+  };
+}
+
+function finiteNumberOrNull(value: unknown): number | null {
+  const parsed = Number(value);
+  return value != null && Number.isFinite(parsed) ? parsed : null;
+}
+
+function numberRange(values: readonly number[]): {
+  min: number | null;
+  max: number | null;
+} {
+  let min: number | null = null;
+  let max: number | null = null;
+  for (const value of values) {
+    if (!Number.isFinite(value)) continue;
+    min = min === null ? value : Math.min(min, value);
+    max = max === null ? value : Math.max(max, value);
+  }
+  return { min, max };
 }