deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -1,6 +1,7 @@
 import type {
   PlayRunnerRateStateAcquireInput,
   PlayRunnerRateStateAcquireResult,
+  PlayRunnerRateStateConcurrencyGrant,
   PlayRunnerRateStatePenalizeInput,
   PlayRunnerRateStateReleaseInput,
 } from '../protocol';
@@ -8,16 +9,90 @@ import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
   type PacingRule,
 } from '../governor/rate-state-backend';
-import type { PostgresSchedulerQueryClient } from './postgres';
+import { type PostgresSchedulerQueryClient } from './postgres';
 import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
 import { stringifyPostgresJson } from '../postgres-json';
+import {
+  Gradient2ConcurrencyController,
+  type Gradient2Observation,
+} from '../provider-capacity/gradient2-concurrency-controller';
+import { providerConcurrencyControllerOptions } from '../provider-capacity/provider-concurrency-policy';
+import { classifyProviderThrottle } from '../provider-capacity/provider-throttle-classifier';
 
 const DEFAULT_SCHEMA = 'runtime_scheduler';
 const RATE_BUCKET_TABLE = 'rate_bucket';
 const RATE_CONCURRENCY_LEASE_TABLE = 'rate_concurrency_leases';
+const RATE_CONCURRENCY_OBSERVER_TABLE = 'rate_concurrency_observers';
+const RATE_RESERVATION_TABLE = 'rate_reservations';
+const RATE_REQUESTER_TABLE = 'rate_requesters';
 const MIN_CONCURRENCY_WAIT_MS = 10;
+/**
+ * Longest a requester held back by the fair-share cap sleeps before it re-asks.
+ * The natural wait is the rest of the current fair-share window (a sibling's
+ * reserved slice frees at the boundary), but a slow rule's window can be a
+ * minute or more, and the runtime treats any hold above its 60 s exhaustion
+ * threshold as a provider outage. A capped requester therefore re-asks at most
+ * this often; a healthy-but-busy provider must never read as exhausted.
+ */
+export const FAIR_SHARE_MAX_WAIT_MS = 30_000;
+/** Re-ask cadence when a sibling's reserved concurrency slots are the limit. */
+const FAIR_SHARE_SLOT_WAIT_MS = 100;
+/** Idle fair-share windows after which a requester stops counting as active. */
+const FAIR_SHARE_REQUESTER_IDLE_WINDOWS = 2;
+/** Opportunistically reclaim idle requester rows without a whole-table lock. */
+const FAIR_SHARE_REQUESTER_CLEANUP_BATCH_SIZE = 64;
+/** Conservative warm-start width for a previously unobserved provider scope. */
+const INITIAL_CONCURRENCY_DISCOVERY_LIMIT = 8;
+/** A transport batch must stay bounded even when a runner completes huge maps. */
+const MAX_CONCURRENCY_OBSERVATIONS_PER_ACQUIRE = 4_096;
+/** Keep retry identities long enough for failed runtime work to recover. */
+const CONCURRENCY_OBSERVATION_DEDUP_RETENTION_MS = 24 * 60 * 60 * 1000;
+/** Opportunistically reclaim stale runner identities without a whole-table lock. */
+const CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE = 128;
 const ACTIVE_LEASE_TTL_MS = 5 * 60 * 1000;
+const RATE_RESERVATION_TTL_MS = 5 * 60 * 1000;
 const MIN_ADAPTIVE_RPS_FLOOR = 0.01;
+/**
+ * Horizon a granted=0 caller is allowed to sleep toward a WHOLE block instead of
+ * a single token. Waiting for one token means one permit per store round trip,
+ * so a lane's real ceiling becomes 1/(1000/rps + RTT) — at 10 rps that is ~6/s
+ * against a 10/s budget, and raising the rate does not help because the round
+ * trip dominates. Waiting for a block amortizes the round trip instead. Bounded
+ * so a slow lane (a 15 RPM rule, or one floored at min_rps after a 429) still
+ * reports the same single-token wait it does today and can never be mistaken for
+ * PROVIDER_EXHAUSTED.
+ */
+const BLOCK_WAIT_HORIZON_MS = 1_000;
+
+/**
+ * Burst horizon of the adaptive token bucket, in milliseconds of the rate
+ * ACTUALLY in force.
+ *
+ * The stored `capacity` column is the DECLARED window burst and must stay at
+ * its seed: config-drift detection compares it against the seed verbatim, so
+ * persisting a learned capacity would make every warm acquire look like drift
+ * and re-seed the row. The bucket the grant actually spends is therefore
+ * derived, not stored: one second of `refill_per_sec`, floored at the declared
+ * burst.
+ *
+ * Without this the AIMD ramp is invisible to throughput. A grant is bounded by
+ * the bucket's tokens, tokens are clamped to `capacity`, and every grant costs
+ * one app-runtime round trip — so a lane's real ceiling is
+ * `capacity / round-trip`, not `refill_per_sec`. A default-pacing lane
+ * (capacity 10) stayed at ~7.7 calls/s and a declared 25 rps lane at ~13
+ * calls/s no matter how high `refill_per_sec` climbed.
+ *
+ * The burst stays bounded: one second of the learned rate, and the learned rate
+ * is itself capped at `max_rps`. The dispatch cursor still spaces every permit
+ * at 1/`refill_per_sec`, so provider arrivals never exceed the rate in force.
+ */
+const ADAPTIVE_BURST_HORIZON_MS = 1_000;
+
+/**
+ * Effective (spendable) bucket capacity for rate rows aliased `b`. Derived per
+ * statement from the learned rate; see {@link ADAPTIVE_BURST_HORIZON_MS}.
+ */
+const EFFECTIVE_CAPACITY_SQL = `GREATEST(b.capacity, b.refill_per_sec * ${ADAPTIVE_BURST_HORIZON_MS} / 1000.0)`;
 
 /**
  * Internal cooldown cap. A 429 report (penalize) can carry an arbitrarily large
@@ -28,6 +103,29 @@ const MIN_ADAPTIVE_RPS_FLOOR = 0.01;
  */
 const MAX_INTERNAL_COOLDOWN_MS = 45 * 60 * 1000;
 
+/**
+ * Buckets this process has already seeded, keyed by schema + bucket id + the
+ * exact seed payload. Seeding is a pure upsert of declared configuration, so
+ * once this process has written a key the rule rows are known present AND
+ * current; replaying that upsert on every acquire buys nothing but a round
+ * trip. Changed configuration hashes to a different key, so a newly deployed
+ * adaptive ceiling still reaches an existing bucket on its next acquire.
+ * Bounded FIFO: eviction costs one extra seed, never correctness, and a row
+ * that disappears behind a warm mark is detected by the grant statement itself
+ * (see `acquirePostgresRateState`) rather than silently granting nothing.
+ */
+const WARM_RATE_BUCKET_LIMIT = 4096;
+const warmRateBuckets = new Set<string>();
+
+function markRateBucketWarm(key: string): void {
+  if (warmRateBuckets.has(key)) return;
+  if (warmRateBuckets.size >= WARM_RATE_BUCKET_LIMIT) {
+    const oldest = warmRateBuckets.values().next();
+    if (!oldest.done) warmRateBuckets.delete(oldest.value);
+  }
+  warmRateBuckets.add(key);
+}
+
 function quoteIdent(value: string): string {
   return `"${value.replace(/"/g, '""')}"`;
 }
@@ -58,6 +156,127 @@ function concurrencyLeaseTableName(options?: {
   return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_CONCURRENCY_LEASE_TABLE)}`;
 }
 
+function concurrencyObserverTableName(options?: {
+  schema?: string | null;
+  schedulerSchema?: string | null;
+}): string {
+  return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_CONCURRENCY_OBSERVER_TABLE)}`;
+}
+
+function reservationTableName(options?: {
+  schema?: string | null;
+  schedulerSchema?: string | null;
+}): string {
+  return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_RESERVATION_TABLE)}`;
+}
+
+function requesterTableName(options?: {
+  schema?: string | null;
+  schedulerSchema?: string | null;
+}): string {
+  return `${quoteIdent(schemaName(options))}.${quoteIdent(RATE_REQUESTER_TABLE)}`;
+}
+
+/**
+ * v43 added `rate_requesters` and the lease `requester_id` column. Only an
+ * acquire that carries a requester identity touches them, so during the
+ * expansion window a v43 process serving a v42 database converts exactly that
+ * missing relation/column into the retryable deployment-skew error, mirroring
+ * {@link isMissingRateReservationTableError}.
+ */
+function isMissingRateRequesterSchemaError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  if (typeof candidate.message !== 'string') return false;
+  return (
+    (candidate.code === '42P01' &&
+      new RegExp(`\\b${RATE_REQUESTER_TABLE}\\b`).test(candidate.message)) ||
+    (candidate.code === '42703' && /\brequester_id\b/.test(candidate.message))
+  );
+}
+
+/**
+ * `rate_reservations` arrived with schema v35. The deploy-owned migration owns
+ * its creation (`ensurePostgresRateStateSchema`); no hot path runs DDL. During
+ * the expansion window a v35 process can therefore be serving a v34 database,
+ * and every reservation statement below fails with 42P01 `undefined_table`.
+ *
+ * Raw, that surfaces to the runner as an ordinary query error and fails the
+ * call. Recognizing exactly this relation — and only this relation — converts
+ * it into the typed, retryable deployment-skew error the runtime already
+ * understands, so the runner waits out the migration instead. Any other
+ * missing relation is a real defect and propagates untouched.
+ */
+function isMissingRateReservationTableError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  if (candidate.code !== '42P01') return false;
+  if (typeof candidate.message !== 'string') return false;
+  return new RegExp(`\\b${RATE_RESERVATION_TABLE}\\b`).test(candidate.message);
+}
+
+function isMissingConcurrencyCheckpointError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  return (
+    candidate.code === '42703' &&
+    typeof candidate.message === 'string' &&
+    /\b(learned_concurrency_limit|in_flight_at_start|target_requests_per_second|minimum_rtt_ms)\b/.test(
+      candidate.message,
+    )
+  );
+}
+
+function isMissingConcurrencyObserverTableError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { code?: unknown; message?: unknown };
+  if (typeof candidate.message !== 'string') return false;
+  return (
+    (candidate.code === '42P01' &&
+      new RegExp(`\\b${RATE_CONCURRENCY_OBSERVER_TABLE}\\b`).test(
+        candidate.message,
+      )) ||
+    (candidate.code === '42703' && /\bexpires_at_ms\b/.test(candidate.message))
+  );
+}
+
+/**
+ * An incomplete deploy can leave the provider-rate tables absent temporarily.
+ * This is an operation-local, retryable store failure—not a scheduler-version
+ * admission gate—so unrelated runtime work continues through its own stores.
+ */
+export class RuntimeRateStateUnavailableError extends Error {
+  readonly code = 'RUNTIME_RATE_STATE_UNAVAILABLE';
+
+  constructor(input: { schema: string }) {
+    super(
+      `Runtime rate state in schema ${input.schema} is unavailable; retry after the deploy migration completes.`,
+    );
+    this.name = 'RuntimeRateStateUnavailableError';
+  }
+}
+
+async function withRateStateAvailabilityGuard<T>(
+  options: { schema?: string | null; schedulerSchema?: string | null },
+  run: () => Promise<T>,
+): Promise<T> {
+  try {
+    return await run();
+  } catch (error) {
+    if (
+      !isMissingRateReservationTableError(error) &&
+      !isMissingConcurrencyCheckpointError(error) &&
+      !isMissingConcurrencyObserverTableError(error) &&
+      !isMissingRateRequesterSchemaError(error)
+    ) {
+      throw error;
+    }
+    throw new RuntimeRateStateUnavailableError({
+      schema: schemaName(options),
+    });
+  }
+}
+
 /**
  * Acquire input rules may carry an optional `adaptiveMaxRps` the client can send
  * to override a bucket's recovery ceiling. When absent, a catalogued provider
@@ -107,20 +326,22 @@ function normalizeRules(
  * Seed parameters for a rule row. `base_refill` is the declared rate in tokens
  * per second; `capacity` is one window of burst. The adaptive `[min,max]` range
  * is derived value-first so the same wire shape drives both declared and
- * catalogued-rate buckets:
+ * default-pacing buckets:
  *   - min_rps = min(base, max(0.01, base/8))
  *     (low-RPM rules must still be able to back down below their base)
  *   - max_rps = max(base, 100)     (catalogued-rate recommendation)
  *   - max_rps = explicit adaptiveMaxRps when supplied
  */
-function seedForRule(rule: AcquirePacingRule): {
+type RateBucketSeed = {
   rule_id: string;
   capacity: number;
   base_refill_per_sec: number;
   min_rps: number;
   max_rps: number;
   adaptive_max_configured: boolean;
-} {
+};
+
+function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   // Even a sub-one-request window must accumulate one complete permit.
   // Preserve its fractional refill rate, with dispatch times resolved in ms.
   const capacity = Math.max(1, rule.requestsPerWindow);
@@ -181,6 +402,37 @@ export async function ensurePostgresRateStateSchema(
     ALTER TABLE ${bucketTableName(options)}
       ADD COLUMN IF NOT EXISTS adaptive_max_configured boolean NOT NULL DEFAULT false
   `);
+  // v37: durable Gradient2 checkpoint. Null limit marks a pre-v37 bucket and
+  // starts its next concurrency-bearing acquire at the conservative discovery
+  // width; rate-only rules leave these fields unused.
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS learned_concurrency_limit float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_short_rtt_ms float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_long_rtt_ms float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_minimum_rtt_ms float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_successful_samples integer NOT NULL DEFAULT 0
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_window_limit float8
+  `);
+  await client.query(`
+    ALTER TABLE ${bucketTableName(options)}
+      ADD COLUMN IF NOT EXISTS concurrency_has_observed_throttle boolean NOT NULL DEFAULT false
+  `);
   await client.query(`
     CREATE TABLE IF NOT EXISTS ${concurrencyLeaseTableName(options)} (
       bucket_id text NOT NULL,
@@ -197,10 +449,84 @@ export async function ensurePostgresRateStateSchema(
     CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_concurrency_expiry_idx
       ON ${concurrencyLeaseTableName(options)} (expires_at_ms)
   `);
+  // v38: each server-minted lease also retains the admission facts required to
+  // classify a later 429. The runner returns the lease id, but the gateway
+  // reloads these values instead of trusting request-body attribution.
+  await client.query(`
+    ALTER TABLE ${concurrencyLeaseTableName(options)}
+      ADD COLUMN IF NOT EXISTS in_flight_at_start integer
+  `);
+  await client.query(`
+    ALTER TABLE ${concurrencyLeaseTableName(options)}
+      ADD COLUMN IF NOT EXISTS target_requests_per_second float8
+  `);
+  await client.query(`
+    ALTER TABLE ${concurrencyLeaseTableName(options)}
+      ADD COLUMN IF NOT EXISTS minimum_rtt_ms float8
+  `);
+  // One row per runner/rule lane remembers the latest locally buffered
+  // observation sequence. It prevents an acquire whose response was lost from
+  // applying the same Gradient2 samples twice, without persisting each result.
+  // The bounded retry identity expires so completed sandbox runs cannot grow
+  // the durable control plane forever.
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${concurrencyObserverTableName(options)} (
+      bucket_id text NOT NULL,
+      observer_id text NOT NULL,
+      lane_id text NOT NULL,
+      last_sequence bigint NOT NULL CHECK (last_sequence > 0),
+      updated_at_ms bigint NOT NULL,
+      expires_at_ms bigint NOT NULL,
+      PRIMARY KEY (bucket_id, observer_id, lane_id)
+    )
+  `);
+  await client.query(`
+    CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_concurrency_observer_expiry_idx
+      ON ${concurrencyObserverTableName(options)} (expires_at_ms)
+  `);
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${reservationTableName(options)} (
+      bucket_id text NOT NULL,
+      reservation_id text NOT NULL,
+      expires_at_ms bigint NOT NULL,
+      PRIMARY KEY (bucket_id, reservation_id)
+    )
+  `);
+  await client.query(`
+    CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_reservation_expiry_idx
+      ON ${reservationTableName(options)} (expires_at_ms)
+  `);
   await client.query(`
     CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_bucket_bucket_idx
       ON ${bucketTableName(options)} (bucket_id)
   `);
+  // v43: one row per (bucket, run) remembers how much of the current
+  // fair-share window that run has drawn, so several runs contending for one
+  // org's provider window each keep a max-min share instead of first-come
+  // draining every window. Rows expire after a short idle; a lone requester
+  // never has peers and is granted exactly as before.
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${requesterTableName(options)} (
+      bucket_id text NOT NULL,
+      requester_id text NOT NULL,
+      window_started_at_ms bigint NOT NULL,
+      granted_in_window integer NOT NULL DEFAULT 0,
+      demand_unmet boolean NOT NULL DEFAULT false,
+      last_seen_ms bigint NOT NULL,
+      expires_at_ms bigint NOT NULL,
+      PRIMARY KEY (bucket_id, requester_id)
+    )
+  `);
+  await client.query(`
+    CREATE INDEX IF NOT EXISTS runtime_scheduler_rate_requester_expiry_idx
+      ON ${requesterTableName(options)} (expires_at_ms)
+  `);
+  // The same share applies to in-flight concurrency slots, which needs each
+  // live lease attributed to the run that holds it.
+  await client.query(`
+    ALTER TABLE ${concurrencyLeaseTableName(options)}
+      ADD COLUMN IF NOT EXISTS requester_id text
+  `);
 }
 
 /**
@@ -214,12 +540,17 @@ async function seedBucketRows(
     schema?: string | null;
     schedulerSchema?: string | null;
   },
-  rules: AcquirePacingRule[],
-  nowMs: number,
+  seeds: RateBucketSeed[],
+  nowMs: number | null,
 ): Promise<void> {
-  const seeds = rules.map(seedForRule);
   await client.query(
     `
+      WITH seed_now AS (
+        SELECT COALESCE(
+          $3::bigint,
+          FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
+        ) AS now_ms
+      )
       INSERT INTO ${bucketTableName(input)} AS existing (
         bucket_id,
         rule_id,
@@ -251,8 +582,8 @@ async function seedBucketRows(
         0,
         0,
         0,
-        $3::bigint,
-        $3::bigint
+        sn.now_ms,
+        sn.now_ms
       FROM jsonb_to_recordset($2::jsonb) AS seed(
         rule_id text,
         capacity float8,
@@ -261,6 +592,7 @@ async function seedBucketRows(
         max_rps float8,
         adaptive_max_configured boolean
       )
+      CROSS JOIN seed_now sn
       ON CONFLICT (bucket_id, rule_id) DO UPDATE SET
         tokens = LEAST(existing.tokens, EXCLUDED.capacity),
         capacity = EXCLUDED.capacity,
@@ -288,30 +620,399 @@ async function seedBucketRows(
   );
 }
 
+/**
+ * Grant result plus the number of rule rows the statement actually locked.
+ * Fewer rows than declared rules means the bucket rows are missing; the caller
+ * turns that into a re-seed (and then a loud failure), never a silent
+ * granted=0 spin.
+ */
+type InternalAcquireOutcome = PlayRunnerRateStateAcquireResult & {
+  observedRuleRows: number;
+  /** True when a seed upsert would still change a stored row (config drift). */
+  configDrift: boolean;
+};
+
+function acquireResultOf(
+  outcome: InternalAcquireOutcome,
+): PlayRunnerRateStateAcquireResult {
+  const result: PlayRunnerRateStateAcquireResult & {
+    observedRuleRows?: number;
+    configDrift?: boolean;
+  } = { ...outcome };
+  delete result.observedRuleRows;
+  delete result.configDrift;
+  return result;
+}
+
+type RateStateAcquireInput = PlayRunnerRateStateAcquireInput & {
+  nowMs?: number;
+  /**
+   * Server-resolved identity of the run asking (see
+   * {@link rateStateRequesterId}); never taken from the request body. Without
+   * it the acquire is not fair-share governed and touches none of the v43
+   * requester objects.
+   */
+  requesterId?: string | null;
+};
+
+/**
+ * Fair-share identity for a rate-state acquire: the root (billing) run when the
+ * executor token carries one, else the run itself, so a parent and the child
+ * plays it launches draw from one slice instead of multiplying their count.
+ */
+export function rateStateRequesterId(auth: {
+  internalExecutorRunId?: string | null;
+  internalExecutorBillingRunId?: string | null;
+}): string | null {
+  const requesterId =
+    auth.internalExecutorBillingRunId?.trim() ||
+    auth.internalExecutorRunId?.trim() ||
+    '';
+  return requesterId || null;
+}
+
+function normalizeRequesterId(input: {
+  requesterId?: string | null;
+}): string | null {
+  const requesterId =
+    typeof input.requesterId === 'string' ? input.requesterId.trim() : '';
+  return requesterId || null;
+}
+
+type FairShareSql = {
+  /** CTEs (each comma-terminated) that need `refreshed`, `cooling`, and — on the concurrency path — `limits` and `free_slots`. */
+  ctes: string;
+  /** Extra `LEAST()` term for the grant. */
+  capTerm: string;
+  /** `wait_ms` branch for a requester held back while capacity remains. */
+  waitBranch: string;
+  /** Comma-prefixed data-modifying CTE recording this acquire; needs `grant_calc`. */
+  rememberCte: string;
+};
+
+/**
+ * Max-min fair share across the runs contending for one bucket, folded into
+ * the single grant statement.
+ *
+ * A fair-share window is the time the binding rule (slowest refill) takes to
+ * refill its spendable burst: one second for an adaptive lane, the declared
+ * window for a slow rule. Windows are aligned to the clock so every requester
+ * agrees on the current one. Each requester's grant inside a window is capped
+ * at `ceil(capacity / N)` (N = requesters seen within the last
+ * {@link FAIR_SHARE_REQUESTER_IDLE_WINDOWS} windows) plus whatever the bucket
+ * holds beyond the slices still reserved for peers with unmet demand. Peers
+ * whose last ask was fully granted reserve nothing, and an idle peer drops
+ * out of N, so the window stays fully usable; a lone requester is never
+ * capped. The token bucket remains the source of truth for what exists, so a
+ * late starter that finds tokens while its greedy sibling is over share is
+ * granted immediately, and the sibling's wait is the rest of the window.
+ *
+ * On the concurrency path the same share bounds in-flight leases per
+ * requester, otherwise a run holding every slot re-takes each one it releases.
+ *
+ * A run alone on a bucket writes its ledger row only when its window rolls or
+ * its lease nears expiry, not on every acquire, so the solo path pays reads
+ * and no extra writes. The cost of that is one window: a peer arriving
+ * mid-window sees the incumbent's draw from when it last wrote, so the
+ * incumbent can exceed its share until the window rolls, after which both
+ * write on every acquire and the share is exact.
+ */
+function fairShareSql(
+  input: { schema?: string | null; schedulerSchema?: string | null },
+  options: {
+    requesterParam: string;
+    requestedParam: string;
+    concurrency: { ruleIdsParam: string } | null;
+  },
+): FairShareSql {
+  const table = requesterTableName(input);
+  const requester = `${options.requesterParam}::text`;
+  const requested = `${options.requestedParam}::int`;
+  const rawAvailable = options.concurrency
+    ? `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill))::int, (SELECT free FROM free_slots)))`
+    : `GREATEST(0, LEAST(${requested}, FLOOR(MIN(tokens_after_refill))::int))`;
+  const liveLeasesHeldBy = (holder: string): string =>
+    options.concurrency
+      ? `(
+          SELECT COUNT(DISTINCT cl.lease_id)
+          FROM ${concurrencyLeaseTableName(input)} cl
+          WHERE cl.bucket_id = $1
+            AND cl.rule_id = ANY(${options.concurrency.ruleIdsParam}::text[])
+            AND cl.requester_id = ${holder}
+            AND cl.expires_at_ms > (SELECT now_ms FROM fair_epoch)
+        )`
+      : '0';
+  const slotCtes = options.concurrency
+    ? `
+      fair_slots AS (
+        SELECT
+          CEIL(
+            (SELECT MIN(max_concurrency) FROM limits)::float8
+              / (SELECT requesters FROM fair_share)
+          ) AS slot_share,
+          ${liveLeasesHeldBy(requester)} AS own_active
+      ),
+      fair_peer_slots AS (
+        SELECT
+          GREATEST(
+            0,
+            (SELECT slot_share FROM fair_slots) - ${liveLeasesHeldBy('p.requester_id')}
+          ) AS reserved
+        FROM peers p
+        WHERE p.unmet
+      ),
+      fair_slot_grant AS (
+        SELECT
+          CASE
+            WHEN (SELECT requesters FROM fair_share) <= 1
+              THEN (SELECT free FROM free_slots)
+            ELSE FLOOR(LEAST(
+              (SELECT free FROM free_slots),
+              own.remaining
+                + GREATEST(
+                    0,
+                    (SELECT free FROM free_slots)
+                      - own.remaining
+                      - COALESCE((SELECT SUM(reserved) FROM fair_peer_slots), 0)
+                  )
+            ))::int
+          END AS slot_cap
+        FROM (
+          SELECT GREATEST(
+            0,
+            (SELECT slot_share FROM fair_slots) - (SELECT own_active FROM fair_slots)
+          ) AS remaining
+        ) own
+      ),`
+    : '';
+  return {
+    ctes: `
+      fair_window AS (
+        SELECT
+          r.now_ms,
+          r.effective_capacity AS capacity,
+          GREATEST(
+            ${ADAPTIVE_BURST_HORIZON_MS},
+            CEIL(r.capacity / GREATEST(r.refill_per_sec, 0.0001) * 1000)
+          )::bigint AS window_ms
+        FROM refreshed r
+        ORDER BY r.refill_per_sec ASC, r.effective_capacity ASC
+        LIMIT 1
+      ),
+      fair_epoch AS (
+        SELECT
+          w.now_ms,
+          w.capacity,
+          w.window_ms,
+          w.now_ms - (w.now_ms % w.window_ms) AS started_at_ms
+        FROM fair_window w
+      ),
+      -- What this acquire has to write, if anything. A run asking many times
+      -- a second must not pay a sweep and an upsert on every one of them: a
+      -- lone run's ledger row only has to exist and stay live, so it is
+      -- rewritten when its window rolls or its lease nears expiry, and on
+      -- every acquire only while peers are actually present. That keeps the
+      -- solo path at roughly one write per window instead of one per acquire.
+      peers AS (
+        SELECT
+          q.requester_id,
+          CASE
+            WHEN q.window_started_at_ms = e.started_at_ms THEN q.granted_in_window
+            ELSE 0
+          END AS taken,
+          CASE
+            WHEN q.window_started_at_ms = e.started_at_ms THEN q.demand_unmet
+            ELSE true
+          END AS unmet
+        FROM ${table} q
+        CROSS JOIN fair_epoch e
+        WHERE q.bucket_id = $1
+          AND q.requester_id <> ${requester}
+          AND q.expires_at_ms > e.now_ms
+      ),
+      fair_write AS (
+        SELECT
+          COALESCE((
+            SELECT
+              q.window_started_at_ms <> e.started_at_ms
+              OR q.expires_at_ms - e.now_ms < e.window_ms
+            FROM ${table} q
+            WHERE q.bucket_id = $1
+              AND q.requester_id = ${requester}
+          ), true) AS refresh,
+          EXISTS (SELECT 1 FROM peers) AS contended
+        FROM fair_epoch e
+      ),
+      -- Reclaims the oldest expired rows across every bucket, not just this
+      -- one: a bucket whose last run ended is never acquired again, so a
+      -- bucket-scoped sweep would leave its rows behind forever. Bounded per
+      -- acquire and driven by the expiry index, as the concurrency-observer
+      -- reclaim is. The row for this acquire's own (bucket, requester) is
+      -- never swept: it is about to be rewritten, and the peer snapshot below
+      -- reads this same statement snapshot.
+      expire_requesters AS (
+        DELETE FROM ${table} AS stale
+        WHERE (SELECT refresh OR contended FROM fair_write)
+          AND (stale.bucket_id, stale.requester_id) IN (
+          SELECT q.bucket_id, q.requester_id
+          FROM ${table} q
+          WHERE q.expires_at_ms <= (SELECT now_ms FROM fair_epoch)
+            AND NOT (q.bucket_id = $1 AND q.requester_id = ${requester})
+          ORDER BY q.expires_at_ms
+          LIMIT ${FAIR_SHARE_REQUESTER_CLEANUP_BATCH_SIZE}
+        )
+        RETURNING 1
+      ),
+      fair_self AS (
+        SELECT COALESCE((
+          SELECT CASE
+            WHEN q.window_started_at_ms = e.started_at_ms THEN q.granted_in_window
+            ELSE 0
+          END
+          FROM ${table} q
+          WHERE q.bucket_id = $1
+            AND q.requester_id = ${requester}
+        ), 0) AS taken
+        FROM fair_epoch e
+      ),
+      fair_share AS (
+        SELECT
+          (SELECT COUNT(*) FROM peers) + 1 AS requesters,
+          CEIL(e.capacity / ((SELECT COUNT(*) FROM peers) + 1)) AS share
+        FROM fair_epoch e
+      ),
+      fair_grant AS (
+        SELECT
+          raw.available,
+          CASE
+            WHEN (SELECT requesters FROM fair_share) <= 1 THEN raw.available
+            ELSE FLOOR(LEAST(
+              raw.available,
+              own.remaining
+                + GREATEST(
+                    0,
+                    raw.available
+                      - own.remaining
+                      - COALESCE((
+                          SELECT SUM(GREATEST(0, (SELECT share FROM fair_share) - p.taken))
+                          FROM peers p
+                          WHERE p.unmet
+                        ), 0)
+                  )
+            ))::int
+          END AS token_cap
+        FROM (SELECT ${rawAvailable} AS available FROM refreshed) raw
+        CROSS JOIN (
+          SELECT GREATEST(
+            0,
+            (SELECT share FROM fair_share) - (SELECT taken FROM fair_self)
+          ) AS remaining
+        ) own
+      ),${slotCtes}
+      fair_cap AS (
+        SELECT
+          (SELECT available FROM fair_grant) AS available,
+          (SELECT token_cap FROM fair_grant) AS token_cap,
+          LEAST(
+            (SELECT token_cap FROM fair_grant)${
+              options.concurrency
+                ? ',\n            (SELECT slot_cap FROM fair_slot_grant)'
+                : ''
+            }
+          ) AS cap
+      ),`,
+    capTerm: ', (SELECT cap FROM fair_cap)',
+    waitBranch: options.concurrency
+      ? `
+          WHEN (SELECT available FROM fair_cap) > 0
+            THEN CASE
+              WHEN (SELECT token_cap FROM fair_cap) <= 0
+                THEN LEAST(
+                  (SELECT started_at_ms + window_ms - now_ms FROM fair_epoch),
+                  ${FAIR_SHARE_MAX_WAIT_MS}
+                )
+              ELSE ${FAIR_SHARE_SLOT_WAIT_MS}
+            END`
+      : `
+          WHEN (SELECT available FROM fair_cap) > 0
+            THEN LEAST(
+              (SELECT started_at_ms + window_ms - now_ms FROM fair_epoch),
+              ${FAIR_SHARE_MAX_WAIT_MS}
+            )`,
+    rememberCte: `,
+      remember_requester AS (
+        INSERT INTO ${table} AS requester (
+          bucket_id,
+          requester_id,
+          window_started_at_ms,
+          granted_in_window,
+          demand_unmet,
+          last_seen_ms,
+          expires_at_ms
+        )
+        SELECT
+          $1,
+          ${requester},
+          e.started_at_ms,
+          (SELECT granted FROM grant_calc),
+          (SELECT granted FROM grant_calc) < ${requested},
+          e.now_ms,
+          e.now_ms + ${FAIR_SHARE_REQUESTER_IDLE_WINDOWS} * e.window_ms
+        FROM fair_epoch e
+        WHERE (SELECT refresh OR contended FROM fair_write)
+        ON CONFLICT (bucket_id, requester_id) DO UPDATE SET
+          granted_in_window = CASE
+            WHEN requester.window_started_at_ms = EXCLUDED.window_started_at_ms
+              THEN requester.granted_in_window
+            ELSE 0
+          END + EXCLUDED.granted_in_window,
+          window_started_at_ms = EXCLUDED.window_started_at_ms,
+          demand_unmet = EXCLUDED.demand_unmet,
+          last_seen_ms = EXCLUDED.last_seen_ms,
+          expires_at_ms = EXCLUDED.expires_at_ms
+        RETURNING 1
+      )`,
+  };
+}
+
 /**
  * Pure-rate (no maxConcurrency) acquire. A deterministic row lock inside one
  * autocommit statement makes every contender reserve from the latest committed
  * token and dispatch cursor instead of reusing a stale CTE snapshot. It refills
  * every rule row against clock_timestamp(), grants the integer floor of the min
  * available tokens across the bucket's rules (multi-rule = most constrained),
- * deducts, and folds the AIMD ramp in. Returns the granted count and, when
- * granted is 0, the wait until one more token accrues on the binding rule (or
- * the cooldown clears).
+ * bounded by the requester's fair share when other runs contend (see
+ * {@link fairShareSql}), deducts, and folds the AIMD ramp in. Returns the
+ * granted count and, when granted is 0, the wait until one more token accrues
+ * on the binding rule, the rest of the fair-share window when the share was
+ * the limit, or the cooldown remainder.
  */
 async function acquirePureRate(
   client: PostgresSchedulerQueryClient,
-  input: PlayRunnerRateStateAcquireInput & { nowMs?: number },
+  input: RateStateAcquireInput,
   rules: AcquirePacingRule[],
+  seeds: RateBucketSeed[],
   requested: number,
-): Promise<PlayRunnerRateStateAcquireResult> {
+): Promise<InternalAcquireOutcome> {
   const ruleIds = rules.map((rule) => rule.ruleId);
+  const requesterId = normalizeRequesterId(input);
+  const fair = requesterId
+    ? fairShareSql(input, {
+        requesterParam: '$8',
+        requestedParam: '$4',
+        concurrency: null,
+      })
+    : null;
   const result = await client.query<{
     granted: string | number;
     wait_ms: string | number;
+    observed_rule_rows: string | number;
+    config_drift: boolean | null;
     server_now_ms: string | number;
     cool_until_ms: string | number;
     claimed_retry_at_ms: string | number;
     scheduled_at_ms: Array<string | number> | null;
+    reservation_ids: string[] | null;
     effective_rps: string | number;
   }>(
     `
@@ -321,6 +1022,27 @@ async function acquirePureRate(
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
       ),
+      seed_config AS (
+        -- The declared configuration for this acquire, compared below against
+        -- the stored rows. This is the same predicate that gates the seed
+        -- upsert's DO UPDATE, so "drift" means exactly "a seed would change
+        -- this row" — which is how a warm process still repairs a row that was
+        -- widened or edited out of band.
+        SELECT * FROM jsonb_to_recordset($7::jsonb) AS seed(
+          rule_id text,
+          capacity float8,
+          base_refill_per_sec float8,
+          min_rps float8,
+          max_rps float8,
+          adaptive_max_configured boolean
+        )
+      ),
+      reclaim_reservations AS (
+        DELETE FROM ${reservationTableName(input)}
+        WHERE bucket_id = $1
+          AND expires_at_ms <= (SELECT now_ms FROM now_value)
+        RETURNING 1
+      ),
       refreshed AS (
         SELECT
           b.bucket_id,
@@ -335,8 +1057,12 @@ async function acquirePureRate(
           b.last_ramp_at_ms,
           b.next_dispatch_at_ms,
           nv.now_ms AS now_ms,
+          -- Spendable burst: one second of the rate in force, floored at the
+          -- declared window burst. b.capacity stays the seeded declared value
+          -- so config-drift detection keeps comparing like with like.
+          ${EFFECTIVE_CAPACITY_SQL} AS effective_capacity,
           LEAST(
-            b.capacity,
+            ${EFFECTIVE_CAPACITY_SQL},
             b.tokens + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0 * b.refill_per_sec
           ) AS tokens_after_refill
         FROM ${bucketTableName(input)} b
@@ -349,24 +1075,42 @@ async function acquirePureRate(
       cooling AS (
         SELECT COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms
         FROM refreshed
-      ),
+      ),${fair?.ctes ?? ''}
       grant_calc AS (
         SELECT
           -- Cooling buckets grant nothing; otherwise the integer floor of the
           -- minimum post-refill tokens across every rule row (most constrained).
           CASE
+            -- No rule rows means the bucket was never seeded (or was deleted).
+            -- LEAST() ignores NULLs, so without this guard an unseeded bucket
+            -- would "grant" the full request against a NULL schedule.
+            WHEN (SELECT COUNT(*) FROM refreshed) = 0 THEN 0
             WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
               THEN 0
-            ELSE GREATEST(0, LEAST($4::int, FLOOR(MIN(tokens_after_refill))::int))
+            ELSE GREATEST(0, LEAST($4::int, FLOOR(MIN(tokens_after_refill))::int${fair?.capTerm ?? ''}))
           END AS granted
         FROM refreshed
       ),
       schedule_calc AS (
         SELECT
+          -- Dispatch debt decays after a ramp. next_dispatch_at_ms is booked
+          -- at the spacing in force when its permits were granted, so a lane
+          -- that started at 0.25 rps carries a minute-long tail that a later
+          -- ramp to 100 rps could never work off (the column is monotone).
+          -- Clamp the carried debt to one full bucket at the CURRENT refill.
+          -- cool_until_ms stays inside the GREATEST, and penalize pushes
+          -- next_dispatch past the cooldown, so a real 429 hold is never
+          -- shortened by this clamp.
           GREATEST(
             MAX(now_ms),
             MAX(cool_until_ms),
-            MAX(next_dispatch_at_ms)
+            LEAST(
+              MAX(next_dispatch_at_ms),
+              MAX(now_ms)
+                + CEIL(
+                    MAX(effective_capacity / GREATEST(refill_per_sec, 0.0001) * 1000)
+                  )::bigint
+            )
           )::bigint AS first_at_ms,
           CEIL(MAX(1000.0 / GREATEST(refill_per_sec, 0.0001)))::bigint AS spacing_ms,
           GREATEST(1, FLOOR(MIN(refill_per_sec) / 100.0))::bigint AS burst_size
@@ -388,15 +1132,15 @@ async function acquirePureRate(
                   * (SELECT burst_size FROM schedule_calc)
             ELSE b.next_dispatch_at_ms
           END,
-          -- AIMD ramp, folded in. Only after confirmed provider success,
-          -- under real demand, not cooling, and at least 1s since the last
-          -- ramp. Slow-start doubles until the first 429, then adds
+          -- AIMD ramp, folded in. Reaching this refill proves a local lease
+          -- block was exhausted; only a confirmed provider success can turn
+          -- that demand into a rate increase. Slow-start doubles until the
+          -- first 429, then adds
           -- max(1, base/10) per step. Never exceeds max_rps.
           refill_per_sec = CASE
             WHEN $5::int > 0
               AND r.now_ms - r.last_ramp_at_ms >= 1000
               AND r.cool_until_ms <= r.now_ms
-              AND r.tokens_after_refill < r.capacity / 2
             THEN LEAST(
               r.max_rps,
               CASE
@@ -411,7 +1155,6 @@ async function acquirePureRate(
             WHEN $5::int > 0
               AND r.now_ms - r.last_ramp_at_ms >= 1000
               AND r.cool_until_ms <= r.now_ms
-              AND r.tokens_after_refill < r.capacity / 2
             THEN r.now_ms
             ELSE r.last_ramp_at_ms
           END
@@ -421,11 +1164,55 @@ async function acquirePureRate(
         RETURNING
           b.rule_id,
           r.tokens_after_refill,
+          r.effective_capacity AS capacity,
           b.refill_per_sec AS new_refill_per_sec,
           r.now_ms AS now_ms
-      )
+      ),
+      reservation_tokens AS (
+        SELECT
+          permit_index,
+          gen_random_uuid()::text AS reservation_id
+        FROM generate_series(1, (SELECT granted FROM grant_calc)) AS permit_index
+      ),
+      minted_reservations AS (
+        INSERT INTO ${reservationTableName(input)} (
+          bucket_id,
+          reservation_id,
+          expires_at_ms
+        )
+        SELECT
+          $1,
+          reservation_id,
+          (SELECT first_at_ms FROM schedule_calc)
+            + FLOOR(
+                (permit_index - 1)::numeric
+                / (SELECT burst_size FROM schedule_calc)
+              )::bigint
+              * (SELECT spacing_ms FROM schedule_calc)
+              * (SELECT burst_size FROM schedule_calc)
+            + $6::bigint
+        FROM reservation_tokens
+        RETURNING reservation_id
+      )${fair?.rememberCte ?? ''}
       SELECT
         (SELECT granted FROM grant_calc) AS granted,
+        (SELECT COUNT(*)::int FROM refreshed) AS observed_rule_rows,
+        (
+          SELECT COALESCE(
+            bool_or(
+              r.capacity IS DISTINCT FROM sc.capacity
+              OR r.base_refill_per_sec IS DISTINCT FROM sc.base_refill_per_sec
+              OR r.min_rps IS DISTINCT FROM sc.min_rps
+              OR (
+                sc.adaptive_max_configured
+                AND r.max_rps IS DISTINCT FROM sc.max_rps
+              )
+            ),
+            false
+          )
+          FROM refreshed r
+          JOIN seed_config sc ON sc.rule_id = r.rule_id
+        ) AS config_drift,
         (SELECT now_ms FROM refreshed LIMIT 1) AS server_now_ms,
         (SELECT cool_until_ms FROM cooling) AS cool_until_ms,
         (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
@@ -445,15 +1232,41 @@ async function acquirePureRate(
           )
           ELSE '{}'::bigint[]
         END AS scheduled_at_ms,
-        -- Wait until either the cooldown clears or one token accrues on the
+        CASE
+          WHEN (SELECT COUNT(*) FROM minted_reservations)
+            = (SELECT granted FROM grant_calc)
+          THEN ARRAY(
+            SELECT reservation_id
+            FROM reservation_tokens
+            ORDER BY permit_index
+          )
+          ELSE NULL
+        END AS reservation_ids,
+        -- Wait until either the cooldown clears or a whole BLOCK accrues on the
         -- most-constrained rule (whichever a granted=0 caller must sleep for).
+        -- The block target is the caller's own request, bounded by the bucket's
+        -- burst capacity and by what this rule can actually refill inside
+        -- BLOCK_WAIT_HORIZON_MS, and floored at one token — so a slow rule keeps
+        -- exactly today's single-token wait while a hot lane stops paying a
+        -- round trip per permit. The cooldown branch above still wins outright.
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN 0
           WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
-            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)
+            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
           ELSE (
             SELECT CEIL(
-              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, 0.0001) * 1000)
+              MAX(
+                (
+                  GREATEST(
+                    1.0,
+                    LEAST(
+                      $4::numeric,
+                      FLOOR(capacity),
+                      FLOOR(new_refill_per_sec * ${BLOCK_WAIT_HORIZON_MS} / 1000.0)
+                    )
+                  ) - tokens_after_refill
+                ) / GREATEST(new_refill_per_sec, 0.0001) * 1000
+              )
             )::bigint
             FROM updated
           )
@@ -465,13 +1278,27 @@ async function acquirePureRate(
       input.nowMs ?? null,
       requested,
       Math.max(0, Math.trunc(input.observedSuccesses ?? 0)),
+      RATE_RESERVATION_TTL_MS,
+      stringifyPostgresJson(seeds),
+      ...(requesterId ? [requesterId] : []),
     ],
   );
   const row = result.rows[0];
   if (!row) {
-    // Seeding raced away or rows vanished; treat as a short retryable wait.
-    return { granted: 0, waitMs: MIN_CONCURRENCY_WAIT_MS, leaseIds: [] };
+    // Seeding raced away or rows vanished; the caller re-seeds and retries.
+    return {
+      granted: 0,
+      waitMs: MIN_CONCURRENCY_WAIT_MS,
+      leaseIds: [],
+      observedRuleRows: 0,
+      configDrift: false,
+    };
   }
+  const observedRuleRows = Math.max(
+    0,
+    Math.trunc(Number(row.observed_rule_rows)),
+  );
+  const configDrift = row.config_drift === true;
   const granted = Math.max(0, Math.trunc(Number(row.granted)));
   const serverNowMs = Math.max(0, Number(row.server_now_ms));
   const coolUntilMs = Math.max(0, Number(row.cool_until_ms));
@@ -479,13 +1306,27 @@ async function acquirePureRate(
   const scheduledAtMs = Array.isArray(row.scheduled_at_ms)
     ? row.scheduled_at_ms.map(Number)
     : [];
+  const reservationIds = Array.isArray(row.reservation_ids)
+    ? row.reservation_ids.filter(
+        (reservationId): reservationId is string =>
+          typeof reservationId === 'string' && reservationId.length > 0,
+      )
+    : [];
+  if (granted > 0 && reservationIds.length !== granted) {
+    throw new Error(
+      `Rate-state reservation minting drift for ${input.bucketId}: granted ${granted} with ${reservationIds.length} cancellation tickets.`,
+    );
+  }
   const effectiveRequestsPerSecond = Math.max(0, Number(row.effective_rps));
   const waitMs = granted > 0 ? 0 : Math.max(1, Math.trunc(Number(row.wait_ms)));
   return {
     granted,
     waitMs,
     serverNowMs,
+    observedRuleRows,
+    configDrift,
     leaseIds: [],
+    reservationIds: granted > 0 ? reservationIds : [],
     scheduledAtMs,
     effectiveRequestsPerSecond,
     coolUntilMs,
@@ -500,17 +1341,29 @@ async function acquirePureRate(
  * lease minting all execute in ONE multi-CTE statement (one implicit
  * transaction), so a crash can never deduct tokens without minting the matching
  * lease rows. Lease ids are generated in SQL (gen_random_uuid) and returned to
- * the client, one per granted permit, releasable independently.
+ * the client, one per granted permit, releasable independently. With a
+ * requester identity the grant is also bounded by that run's fair share of
+ * both the window and the concurrency slots (see {@link fairShareSql}), and
+ * each minted lease records who holds it.
  */
 async function acquireWithConcurrency(
   client: PostgresSchedulerQueryClient,
-  input: PlayRunnerRateStateAcquireInput & { nowMs?: number },
+  input: RateStateAcquireInput,
   rules: AcquirePacingRule[],
+  seeds: RateBucketSeed[],
   requested: number,
-): Promise<PlayRunnerRateStateAcquireResult> {
+): Promise<InternalAcquireOutcome> {
   const ruleIds = rules.map((rule) => rule.ruleId);
   const concurrencyRules = rules.filter((rule) => rule.maxConcurrency != null);
   const concurrencyRuleIds = concurrencyRules.map((rule) => rule.ruleId);
+  const requesterId = normalizeRequesterId(input);
+  const fair = requesterId
+    ? fairShareSql(input, {
+        requesterParam: '$13',
+        requestedParam: '$4',
+        concurrency: { ruleIdsParam: '$5' },
+      })
+    : null;
   // Per-rule limits, NOT collapsed to a single min. When a bucket carries more
   // than one concurrency rule, each rule mints its own lease row per permit, so
   // SUM(permits) over all rules double-counts a single in-flight permit. The
@@ -531,10 +1384,13 @@ async function acquireWithConcurrency(
   const result = await client.query<{
     granted: string | number;
     wait_ms: string | number;
+    observed_rule_rows: string | number;
+    config_drift: boolean | null;
     server_now_ms: string | number;
     cool_until_ms: string | number;
     claimed_retry_at_ms: string | number;
     lease_ids: string[] | null;
+    concurrency_grants: unknown[] | null;
     scheduled_at_ms: Array<string | number> | null;
     effective_rps: string | number;
   }>(
@@ -545,6 +1401,12 @@ async function acquireWithConcurrency(
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
       ),
+      reclaim_reservations AS (
+        DELETE FROM ${reservationTableName(input)}
+        WHERE bucket_id = $1
+          AND expires_at_ms <= (SELECT now_ms FROM now_value)
+        RETURNING 1
+      ),
       reclaim AS (
         DELETE FROM ${concurrencyLeaseTableName(input)}
         WHERE bucket_id = $1
@@ -552,22 +1414,77 @@ async function acquireWithConcurrency(
           AND expires_at_ms <= (SELECT now_ms FROM now_value)
         RETURNING 1
       ),
+      seed_config AS (
+        -- The declared configuration for this acquire, compared below against
+        -- the stored rows. This is the same predicate that gates the seed
+        -- upsert's DO UPDATE, so "drift" means exactly "a seed would change
+        -- this row" — which is how a warm process still repairs a row that was
+        -- widened or edited out of band.
+        SELECT * FROM jsonb_to_recordset($12::jsonb) AS seed(
+          rule_id text,
+          capacity float8,
+          base_refill_per_sec float8,
+          min_rps float8,
+          max_rps float8,
+          adaptive_max_configured boolean
+        )
+      ),
+      refreshed AS (
+        SELECT
+          b.rule_id,
+          b.capacity,
+          b.min_rps,
+          b.max_rps,
+          b.seen_backpressure,
+          b.base_refill_per_sec,
+          b.learned_concurrency_limit,
+          b.concurrency_minimum_rtt_ms,
+          b.cool_until_ms,
+          b.last_ramp_at_ms,
+          b.next_dispatch_at_ms,
+          nv.now_ms AS now_ms,
+          -- Same derived burst as the pure-rate path. The declared
+          -- maxConcurrency is untouched: free_slots still bounds the grant.
+          ${EFFECTIVE_CAPACITY_SQL} AS effective_capacity,
+          LEAST(
+            ${EFFECTIVE_CAPACITY_SQL},
+            b.tokens + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0 * b.refill_per_sec
+          ) AS tokens_after_refill,
+          b.refill_per_sec
+        FROM ${bucketTableName(input)} b
+        CROSS JOIN now_value nv
+        WHERE b.bucket_id = $1
+          AND b.rule_id = ANY($2::text[])
+        ORDER BY b.rule_id
+        FOR UPDATE OF b
+      ),
       limits AS (
-        -- One row per concurrency rule with its declared cap.
-        SELECT rule_id, max_concurrency
+        -- Depend on the refreshed rows: a learned-capacity transition locks these
+        -- same rows, so a grant that waits behind it must read the new limit.
+        -- The catalogue is still the hard ceiling.
+        SELECT
+          lim.rule_id,
+          LEAST(
+            lim.max_concurrency,
+            CEIL(
+              COALESCE(
+                bucket.learned_concurrency_limit,
+                LEAST(lim.max_concurrency, ${INITIAL_CONCURRENCY_DISCOVERY_LIMIT})
+              )
+            )
+          )::int AS max_concurrency
         FROM jsonb_to_recordset($9::jsonb) AS lim(
           rule_id text,
           max_concurrency int
         )
+        JOIN refreshed bucket ON bucket.rule_id = lim.rule_id
       ),
       active_by_rule AS (
-        -- Live in-flight permits PER RULE. A CTE SELECT sees the pre-statement
-        -- snapshot, so expired rows the reclaim DELETE removes are still visible
-        -- here; filter them out explicitly by expiry so the free-slot count
-        -- matches the post-reclaim table. Grouping by rule_id (not SUM across
-        -- all rules) is what keeps a single permit — which mints one lease row
-        -- per concurrency rule — from being counted once per rule.
-        SELECT l.rule_id,
+        -- Live in-flight permits per rule. One physical permit has one lease
+        -- row per rule, so each rule is counted independently before we take
+        -- the tightest free-slot value.
+        SELECT
+          l.rule_id,
           COALESCE((
             SELECT SUM(cl.permits)
             FROM ${concurrencyLeaseTableName(input)} cl
@@ -578,7 +1495,6 @@ async function acquireWithConcurrency(
         FROM limits l
       ),
       free_slots AS (
-        -- Most-constrained free concurrency across all rules of the bucket.
         SELECT COALESCE(
           MIN(l.max_concurrency - a.active_count),
           $6::int
@@ -586,35 +1502,23 @@ async function acquireWithConcurrency(
         FROM limits l
         JOIN active_by_rule a ON a.rule_id = l.rule_id
       ),
-      refreshed AS (
+      concurrency_context AS (
         SELECT
-          b.rule_id,
-          b.capacity,
-          b.max_rps,
-          b.seen_backpressure,
-          b.base_refill_per_sec,
-          b.cool_until_ms,
-          b.last_ramp_at_ms,
-          b.next_dispatch_at_ms,
-          nv.now_ms AS now_ms,
-          LEAST(
-            b.capacity,
-            b.tokens + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0 * b.refill_per_sec
-          ) AS tokens_after_refill,
-          b.refill_per_sec
-        FROM ${bucketTableName(input)} b
-        CROSS JOIN now_value nv
-        WHERE b.bucket_id = $1
-          AND b.rule_id = ANY($2::text[])
-        ORDER BY b.rule_id
-        FOR UPDATE OF b
+          COALESCE(MAX(active_count), 0)::int AS active_count,
+          MIN(concurrency_minimum_rtt_ms) AS minimum_rtt_ms
+        FROM active_by_rule
+        JOIN refreshed USING (rule_id)
       ),
       cooling AS (
         SELECT COALESCE(MAX(cool_until_ms), 0) AS cool_until_ms FROM refreshed
-      ),
+      ),${fair?.ctes ?? ''}
       grant_calc AS (
         SELECT
           CASE
+            -- No rule rows means the bucket was never seeded (or was deleted).
+            -- LEAST() ignores NULLs, so without this guard an unseeded bucket
+            -- would "grant" the full request against a NULL schedule.
+            WHEN (SELECT COUNT(*) FROM refreshed) = 0 THEN 0
             WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
               THEN 0
             ELSE GREATEST(
@@ -622,7 +1526,7 @@ async function acquireWithConcurrency(
               LEAST(
                 $4::int,
                 FLOOR(MIN(tokens_after_refill))::int,
-                (SELECT free FROM free_slots)
+                (SELECT free FROM free_slots)${fair?.capTerm ?? ''}
               )
             )
           END AS granted
@@ -630,10 +1534,24 @@ async function acquireWithConcurrency(
       ),
       schedule_calc AS (
         SELECT
+          -- Dispatch debt decays after a ramp. next_dispatch_at_ms is booked
+          -- at the spacing in force when its permits were granted, so a lane
+          -- that started at 0.25 rps carries a minute-long tail that a later
+          -- ramp to 100 rps could never work off (the column is monotone).
+          -- Clamp the carried debt to one full bucket at the CURRENT refill.
+          -- cool_until_ms stays inside the GREATEST, and penalize pushes
+          -- next_dispatch past the cooldown, so a real 429 hold is never
+          -- shortened by this clamp.
           GREATEST(
             MAX(now_ms),
             MAX(cool_until_ms),
-            MAX(next_dispatch_at_ms)
+            LEAST(
+              MAX(next_dispatch_at_ms),
+              MAX(now_ms)
+                + CEIL(
+                    MAX(effective_capacity / GREATEST(refill_per_sec, 0.0001) * 1000)
+                  )::bigint
+            )
           )::bigint AS first_at_ms,
           CEIL(MAX(1000.0 / GREATEST(refill_per_sec, 0.0001)))::bigint AS spacing_ms,
           GREATEST(1, FLOOR(MIN(refill_per_sec) / 100.0))::bigint AS burst_size
@@ -655,13 +1573,13 @@ async function acquireWithConcurrency(
                   * (SELECT burst_size FROM schedule_calc)
             ELSE b.next_dispatch_at_ms
           END,
-          -- Match the pure-rate path: concurrency leases change admission
-          -- accounting, not the AIMD learning contract.
+          -- Match the pure-rate path: a depleted local lease block plus a
+          -- confirmed success is sufficient demand; concurrency leases change
+          -- admission accounting, not the AIMD learning contract.
           refill_per_sec = CASE
             WHEN $10::int > 0
               AND r.now_ms - r.last_ramp_at_ms >= 1000
               AND r.cool_until_ms <= r.now_ms
-              AND r.tokens_after_refill < r.capacity / 2
             THEN LEAST(
               r.max_rps,
               CASE
@@ -676,7 +1594,6 @@ async function acquireWithConcurrency(
             WHEN $10::int > 0
               AND r.now_ms - r.last_ramp_at_ms >= 1000
               AND r.cool_until_ms <= r.now_ms
-              AND r.tokens_after_refill < r.capacity / 2
             THEN r.now_ms
             ELSE r.last_ramp_at_ms
           END
@@ -692,8 +1609,10 @@ async function acquireWithConcurrency(
         -- functions included), so each lease_id below stays stable across the
         -- per-rule cross join in minted — every concurrency rule row shares the
         -- same lease_id for a given permit, matching release semantics.
-        SELECT gen_random_uuid()::text AS lease_id
-        FROM generate_series(1, (SELECT granted FROM grant_calc))
+        SELECT
+          permit_index,
+          gen_random_uuid()::text AS lease_id
+        FROM generate_series(1, (SELECT granted FROM grant_calc)) AS permit_index
       ),
       minted AS (
         -- Mint the lease rows from grant_calc (NOT by re-reading the lease
@@ -704,20 +1623,63 @@ async function acquireWithConcurrency(
           rule_id,
           lease_id,
           permits,
-          expires_at_ms
+          expires_at_ms,
+          in_flight_at_start,
+          target_requests_per_second,
+          minimum_rtt_ms${fair ? ',\n          requester_id' : ''}
         )
         SELECT
           $1,
           rule_id,
           lt.lease_id,
           1,
-          (SELECT now_ms FROM now_value) + $8::bigint
+          (SELECT now_ms FROM now_value) + $8::bigint,
+          (SELECT active_count FROM concurrency_context) + lt.permit_index,
+          (SELECT COALESCE(MIN(new_refill_per_sec), 0) FROM updated),
+          (SELECT minimum_rtt_ms FROM concurrency_context)${fair ? ',\n          $13::text' : ''}
         FROM unnest($5::text[]) AS rule_id
         CROSS JOIN lease_tokens lt
         RETURNING lease_id
-      )
+      ),
+      minted_reservations AS (
+        INSERT INTO ${reservationTableName(input)} (
+          bucket_id,
+          reservation_id,
+          expires_at_ms
+        )
+        SELECT
+          $1,
+          lease_id,
+          (SELECT first_at_ms FROM schedule_calc)
+            + FLOOR(
+                (permit_index - 1)::numeric
+                / (SELECT burst_size FROM schedule_calc)
+              )::bigint
+              * (SELECT spacing_ms FROM schedule_calc)
+              * (SELECT burst_size FROM schedule_calc)
+            + $11::bigint
+        FROM lease_tokens
+        RETURNING reservation_id
+      )${fair?.rememberCte ?? ''}
       SELECT
         (SELECT granted FROM grant_calc) AS granted,
+        (SELECT COUNT(*)::int FROM refreshed) AS observed_rule_rows,
+        (
+          SELECT COALESCE(
+            bool_or(
+              r.capacity IS DISTINCT FROM sc.capacity
+              OR r.base_refill_per_sec IS DISTINCT FROM sc.base_refill_per_sec
+              OR r.min_rps IS DISTINCT FROM sc.min_rps
+              OR (
+                sc.adaptive_max_configured
+                AND r.max_rps IS DISTINCT FROM sc.max_rps
+              )
+            ),
+            false
+          )
+          FROM refreshed r
+          JOIN seed_config sc ON sc.rule_id = r.rule_id
+        ) AS config_drift,
         (SELECT now_ms FROM refreshed LIMIT 1) AS server_now_ms,
         (SELECT cool_until_ms FROM cooling) AS cool_until_ms,
         (SELECT COALESCE(MAX(claimed_retry_at_ms), 0) FROM ${bucketTableName(input)}
@@ -737,11 +1699,38 @@ async function acquireWithConcurrency(
           )
           ELSE '{}'::bigint[]
         END AS scheduled_at_ms,
-        (SELECT COALESCE(ARRAY_AGG(DISTINCT lease_id), '{}') FROM minted) AS lease_ids,
+        CASE
+          WHEN (SELECT COUNT(DISTINCT lease_id) FROM minted)
+            = (SELECT granted FROM grant_calc)
+            AND (SELECT COUNT(*) FROM minted_reservations)
+              = (SELECT granted FROM grant_calc)
+          THEN ARRAY(
+            SELECT lease_id
+            FROM lease_tokens
+            ORDER BY permit_index
+          )
+          ELSE NULL
+        END AS lease_ids,
+        CASE
+          WHEN (SELECT granted FROM grant_calc) > 0 THEN ARRAY(
+            SELECT jsonb_build_object(
+              'admissionId', lease_id,
+              'inFlightAtStart',
+                (SELECT active_count FROM concurrency_context) + permit_index,
+              'targetRequestsPerSecond',
+                (SELECT COALESCE(MIN(new_refill_per_sec), 0) FROM updated),
+              'minimumRttMs',
+                (SELECT minimum_rtt_ms FROM concurrency_context)
+            )
+            FROM lease_tokens
+            ORDER BY permit_index
+          )
+          ELSE '{}'::jsonb[]
+        END AS concurrency_grants,
         CASE
           WHEN (SELECT granted FROM grant_calc) > 0 THEN 0
           WHEN (SELECT cool_until_ms FROM cooling) > (SELECT now_ms FROM refreshed LIMIT 1)
-            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)
+            THEN (SELECT cool_until_ms FROM cooling) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
           WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
           ELSE (
             SELECT CEIL(
@@ -764,12 +1753,26 @@ async function acquireWithConcurrency(
       ACTIVE_LEASE_TTL_MS,
       stringifyPostgresJson(concurrencyLimits),
       Math.max(0, Math.trunc(input.observedSuccesses ?? 0)),
+      RATE_RESERVATION_TTL_MS,
+      stringifyPostgresJson(seeds),
+      ...(requesterId ? [requesterId] : []),
     ],
   );
   const row = result.rows[0];
   if (!row) {
-    return { granted: 0, waitMs: MIN_CONCURRENCY_WAIT_MS, leaseIds: [] };
+    return {
+      granted: 0,
+      waitMs: MIN_CONCURRENCY_WAIT_MS,
+      leaseIds: [],
+      observedRuleRows: 0,
+      configDrift: false,
+    };
   }
+  const observedRuleRows = Math.max(
+    0,
+    Math.trunc(Number(row.observed_rule_rows)),
+  );
+  const configDrift = row.config_drift === true;
   const granted = Math.max(0, Math.trunc(Number(row.granted)));
   const serverNowMs = Math.max(0, Number(row.server_now_ms));
   const coolUntilMs = Math.max(0, Number(row.cool_until_ms));
@@ -790,11 +1793,20 @@ async function acquireWithConcurrency(
       `Rate-state lease minting drift for ${input.bucketId}: granted ${granted} with ${leaseIds.length} lease tokens.`,
     );
   }
+  const concurrencyGrants = parseConcurrencyGrants(
+    row.concurrency_grants,
+    granted,
+    input.bucketId,
+  );
   return {
     granted,
     waitMs,
     serverNowMs,
+    observedRuleRows,
+    configDrift,
     leaseIds: granted > 0 ? leaseIds : [],
+    concurrencyGrants,
+    reservationIds: granted > 0 ? leaseIds : [],
     scheduledAtMs,
     effectiveRequestsPerSecond,
     coolUntilMs,
@@ -804,36 +1816,568 @@ async function acquireWithConcurrency(
 
 export async function acquirePostgresRateState(
   client: PostgresSchedulerQueryClient,
-  input: PlayRunnerRateStateAcquireInput & { nowMs?: number },
+  input: RateStateAcquireInput,
+): Promise<PlayRunnerRateStateAcquireResult> {
+  return await withRateStateAvailabilityGuard(input, () =>
+    acquireRateStateStatements(client, input),
+  );
+}
+
+type ConcurrencyObservation = Gradient2Observation & {
+  kind: 'success' | 'throttled';
+};
+
+type ConcurrencyObservationBatch = {
+  observerId: string;
+  laneId: string;
+  sequence: number;
+};
+
+function parseConcurrencyGrants(
+  grants: unknown,
+  granted: number,
+  bucketId: string,
+): PlayRunnerRateStateConcurrencyGrant[] {
+  if (granted === 0) return [];
+  if (!Array.isArray(grants) || grants.length !== granted) {
+    throw new Error(
+      `Rate-state concurrency attribution drift for ${bucketId}: granted ${granted} with invalid attribution tokens.`,
+    );
+  }
+  return grants.map((value) => {
+    let candidate: unknown = value;
+    if (typeof candidate === 'string') {
+      try {
+        candidate = JSON.parse(candidate) as unknown;
+      } catch {
+        // The validation below reports the durable-boundary error.
+      }
+    }
+    if (
+      !candidate ||
+      typeof candidate !== 'object' ||
+      Array.isArray(candidate)
+    ) {
+      throw new Error(
+        `Rate-state concurrency attribution drift for ${bucketId}: invalid token.`,
+      );
+    }
+    const record = candidate as Record<string, unknown>;
+    const admissionId =
+      typeof record.admissionId === 'string' ? record.admissionId.trim() : '';
+    const inFlightAtStart = Number(record.inFlightAtStart);
+    const targetRequestsPerSecond = record.targetRequestsPerSecond;
+    const minimumRttMs = record.minimumRttMs;
+    if (
+      !admissionId ||
+      admissionId.length > 64 ||
+      !Number.isSafeInteger(inFlightAtStart) ||
+      inFlightAtStart < 1 ||
+      (targetRequestsPerSecond !== null &&
+        (!Number.isFinite(targetRequestsPerSecond) ||
+          Number(targetRequestsPerSecond) <= 0)) ||
+      (minimumRttMs !== null &&
+        (!Number.isFinite(minimumRttMs) || Number(minimumRttMs) <= 0))
+    ) {
+      throw new Error(
+        `Rate-state concurrency attribution drift for ${bucketId}: invalid token.`,
+      );
+    }
+    return {
+      admissionId,
+      inFlightAtStart,
+      targetRequestsPerSecond:
+        targetRequestsPerSecond === null
+          ? null
+          : Number(targetRequestsPerSecond),
+      minimumRttMs: minimumRttMs === null ? null : Number(minimumRttMs),
+    };
+  });
+}
+
+function concurrencyObservationBatchFrom(
+  batch: PlayRunnerRateStateAcquireInput['concurrencyObservationBatch'],
+): ConcurrencyObservationBatch | null {
+  if (!batch || typeof batch !== 'object') return null;
+  const observerId = batch.observerId?.trim();
+  const laneId = batch.laneId?.trim();
+  const sequence = Number(batch.sequence);
+  if (
+    !observerId ||
+    observerId.length > 64 ||
+    !laneId ||
+    laneId.length > 64 ||
+    !Number.isSafeInteger(sequence) ||
+    sequence <= 0
+  ) {
+    return null;
+  }
+  return { observerId, laneId, sequence };
+}
+
+function concurrencyObservationsFrom(
+  observations: PlayRunnerRateStateAcquireInput['concurrencyObservations'],
+): ConcurrencyObservation[] {
+  if (!Array.isArray(observations)) return [];
+  return observations
+    .slice(0, MAX_CONCURRENCY_OBSERVATIONS_PER_ACQUIRE)
+    .flatMap((observation) =>
+      (observation.kind === 'success' || observation.kind === 'throttled') &&
+      Number.isFinite(observation.latencyMs) &&
+      observation.latencyMs > 0
+        ? [{ kind: observation.kind, latencyMs: observation.latencyMs }]
+        : [],
+    );
+}
+
+async function concurrencyThrottleFrom(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerRateStatePenalizeInput,
+  rules: readonly AcquirePacingRule[],
+): Promise<{
+  observations: ConcurrencyObservation[];
+  batch: ConcurrencyObservationBatch | null;
+}> {
+  if (
+    input.latencyMs == null ||
+    !Number.isFinite(input.latencyMs) ||
+    input.latencyMs <= 0
+  ) {
+    return { observations: [], batch: null };
+  }
+  const grant = input.concurrencyGrant;
+  if (
+    !grant ||
+    typeof grant.admissionId !== 'string' ||
+    !grant.admissionId.trim() ||
+    grant.admissionId.trim().length > 64
+  ) {
+    return { observations: [], batch: null };
+  }
+  const concurrencyRuleIds = rules
+    .filter((rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0)
+    .map((rule) => rule.ruleId);
+  if (concurrencyRuleIds.length === 0) {
+    return { observations: [], batch: null };
+  }
+  const attributedLease = await client.query<{
+    in_flight_at_start: number | null;
+    target_requests_per_second: number | null;
+    minimum_rtt_ms: number | null;
+  }>(
+    `
+      SELECT
+        in_flight_at_start,
+        target_requests_per_second,
+        minimum_rtt_ms
+      FROM ${concurrencyLeaseTableName(input)}
+      WHERE bucket_id = $1
+        AND lease_id = $2
+        AND rule_id = ANY($3::text[])
+      ORDER BY rule_id
+      LIMIT 1
+    `,
+    [input.bucketId, grant.admissionId.trim(), concurrencyRuleIds],
+  );
+  const lease = attributedLease.rows[0];
+  const inFlightAtStart = Number(lease?.in_flight_at_start);
+  const targetRequestsPerSecond = lease?.target_requests_per_second;
+  const minimumRttMs = lease?.minimum_rtt_ms;
+  if (
+    !Number.isSafeInteger(inFlightAtStart) ||
+    inFlightAtStart < 1 ||
+    (targetRequestsPerSecond !== null &&
+      targetRequestsPerSecond !== undefined &&
+      (!Number.isFinite(targetRequestsPerSecond) ||
+        targetRequestsPerSecond <= 0)) ||
+    (minimumRttMs !== null &&
+      minimumRttMs !== undefined &&
+      (!Number.isFinite(minimumRttMs) || minimumRttMs <= 0))
+  ) {
+    return { observations: [], batch: null };
+  }
+  const attribution = classifyProviderThrottle({
+    grant: {
+      targetRequestsPerSecond: targetRequestsPerSecond ?? undefined,
+      minimumRttMs: minimumRttMs ?? null,
+    },
+    inFlightAtStart,
+  });
+  if (attribution === 'rate') return { observations: [], batch: null };
+  return {
+    observations: [
+      {
+        kind: 'throttled',
+        latencyMs: input.latencyMs,
+        inFlightAtStart,
+      },
+    ],
+    // Provider permits are server-minted. Reusing this unique admission id
+    // makes a retried 429 report an idempotent checkpoint transition.
+    batch: {
+      observerId: 'provider-throttle',
+      laneId: grant.admissionId.trim(),
+      sequence: 1,
+    },
+  };
+}
+
+/**
+ * Fold completed provider attempts into the same Postgres rows that enforce
+ * their permits. `FOR UPDATE` makes a checkpoint transition linearizable across
+ * runners: no sandbox-local controller can overwrite another's RTT history.
+ */
+async function applyConcurrencyObservations(
+  client: PostgresSchedulerQueryClient,
+  input: Pick<PlayRunnerRateStateAcquireInput, 'bucketId'> & {
+    schema?: string | null;
+    schedulerSchema?: string | null;
+  },
+  rules: readonly AcquirePacingRule[],
+  observations: readonly ConcurrencyObservation[],
+  batch: ConcurrencyObservationBatch | null,
+): Promise<'unbatched' | 'applied' | 'already_applied'> {
+  const concurrencyRules = rules.filter(
+    (rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0,
+  );
+  if (batch === null) return 'unbatched';
+
+  await client.query('BEGIN');
+  try {
+    const accepted = await client.query<{ last_sequence: number }>(
+      `
+        WITH now_value AS (
+          SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint AS now_ms
+        ),
+        expired AS (
+          DELETE FROM ${concurrencyObserverTableName(input)} AS observer
+          WHERE (observer.bucket_id, observer.observer_id, observer.lane_id) IN (
+            SELECT
+              stale.bucket_id,
+              stale.observer_id,
+              stale.lane_id
+            FROM ${concurrencyObserverTableName(input)} AS stale
+            WHERE stale.expires_at_ms <= (SELECT now_ms FROM now_value)
+            ORDER BY stale.expires_at_ms
+            LIMIT ${CONCURRENCY_OBSERVER_CLEANUP_BATCH_SIZE}
+          )
+        )
+        INSERT INTO ${concurrencyObserverTableName(input)} AS observer (
+          bucket_id,
+          observer_id,
+          lane_id,
+          last_sequence,
+          updated_at_ms,
+          expires_at_ms
+        )
+        SELECT
+          $1,
+          $2,
+          $3,
+          $4,
+          now_ms,
+          now_ms + $5::bigint
+        FROM now_value
+        ON CONFLICT (bucket_id, observer_id, lane_id) DO UPDATE
+        SET
+          last_sequence = EXCLUDED.last_sequence,
+          updated_at_ms = EXCLUDED.updated_at_ms,
+          expires_at_ms = EXCLUDED.expires_at_ms
+        WHERE observer.last_sequence < EXCLUDED.last_sequence
+        RETURNING last_sequence
+      `,
+      [
+        input.bucketId,
+        batch.observerId,
+        batch.laneId,
+        batch.sequence,
+        CONCURRENCY_OBSERVATION_DEDUP_RETENTION_MS,
+      ],
+    );
+    // Same batch after an uncertain response, or a delayed older one. The
+    // checkpoint already contains its transition, so commit a no-op rather
+    // than replaying its samples.
+    if (accepted.rows.length === 0) {
+      await client.query('COMMIT');
+      return 'already_applied';
+    }
+    if (concurrencyRules.length === 0 || observations.length === 0) {
+      await client.query('COMMIT');
+      return 'applied';
+    }
+    const rows = await client.query<{
+      rule_id: string;
+      learned_concurrency_limit: number | null;
+      concurrency_short_rtt_ms: number | null;
+      concurrency_long_rtt_ms: number | null;
+      concurrency_minimum_rtt_ms: number | null;
+      concurrency_successful_samples: number;
+      concurrency_window_limit: number | null;
+      concurrency_has_observed_throttle: boolean;
+    }>(
+      `
+        SELECT
+          rule_id,
+          learned_concurrency_limit,
+          concurrency_short_rtt_ms,
+          concurrency_long_rtt_ms,
+          concurrency_minimum_rtt_ms,
+          concurrency_successful_samples,
+          concurrency_window_limit,
+          concurrency_has_observed_throttle
+        FROM ${bucketTableName(input)}
+        WHERE bucket_id = $1
+          AND rule_id = ANY($2::text[])
+        ORDER BY rule_id
+        FOR UPDATE
+      `,
+      [input.bucketId, concurrencyRules.map((rule) => rule.ruleId)],
+    );
+
+    const ruleById = new Map(
+      concurrencyRules.map((rule) => [rule.ruleId, rule]),
+    );
+    for (const row of rows.rows) {
+      const rule = ruleById.get(row.rule_id);
+      if (!rule?.maxConcurrency) continue;
+      const maximumLimit = Math.max(1, Math.trunc(rule.maxConcurrency));
+      const initialLimit = Math.min(
+        maximumLimit,
+        Math.max(
+          1,
+          row.learned_concurrency_limit ?? INITIAL_CONCURRENCY_DISCOVERY_LIMIT,
+        ),
+      );
+      const controller = new Gradient2ConcurrencyController(
+        providerConcurrencyControllerOptions({
+          initialLimit,
+          maximumLimit,
+        }),
+        row.learned_concurrency_limit == null
+          ? undefined
+          : {
+              limit: initialLimit,
+              shortRttMs: row.concurrency_short_rtt_ms,
+              longRttMs: row.concurrency_long_rtt_ms,
+              minimumRttMs: row.concurrency_minimum_rtt_ms,
+              successfulSamplesInWindow: Math.max(
+                0,
+                Math.trunc(row.concurrency_successful_samples ?? 0),
+              ),
+              windowStartedAtLimit: Math.max(
+                1,
+                Math.trunc(row.concurrency_window_limit ?? initialLimit),
+              ),
+              hasObservedThrottle:
+                row.concurrency_has_observed_throttle === true,
+            },
+      );
+      for (const observation of observations) controller.observe(observation);
+      const checkpoint = controller.checkpoint();
+      await client.query(
+        `
+          UPDATE ${bucketTableName(input)}
+          SET
+            learned_concurrency_limit = $3,
+            concurrency_short_rtt_ms = $4,
+            concurrency_long_rtt_ms = $5,
+            concurrency_minimum_rtt_ms = $6,
+            concurrency_successful_samples = $7,
+            concurrency_window_limit = $8,
+            concurrency_has_observed_throttle = $9
+          WHERE bucket_id = $1 AND rule_id = $2
+        `,
+        [
+          input.bucketId,
+          row.rule_id,
+          checkpoint.limit,
+          checkpoint.shortRttMs,
+          checkpoint.longRttMs,
+          checkpoint.minimumRttMs,
+          checkpoint.successfulSamplesInWindow,
+          checkpoint.windowStartedAtLimit,
+          checkpoint.hasObservedThrottle === true,
+        ],
+      );
+    }
+    await client.query('COMMIT');
+    return 'applied';
+  } catch (error) {
+    try {
+      await client.query('ROLLBACK');
+    } catch {
+      // Preserve the original controller/storage error.
+    }
+    throw error;
+  }
+}
+
+async function acquireRateStateStatements(
+  client: PostgresSchedulerQueryClient,
+  input: RateStateAcquireInput,
 ): Promise<PlayRunnerRateStateAcquireResult> {
   const rules = normalizeRules(input.rules as AcquirePacingRule[]);
   if (rules.length === 0) return { granted: 0, waitMs: 0, leaseIds: [] };
   const requested = positiveInt(input.requested, 'requested permits');
-  const nowMs =
-    input.nowMs ??
-    Number(
-      (
-        await client.query<{ now_ms: string | number }>(
-          `SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint AS now_ms`,
-        )
-      ).rows[0]?.now_ms ?? Date.now(),
-    );
-  // Cold path: seed missing rows (ON CONFLICT DO NOTHING, no-op when warm).
-  await seedBucketRows(client, input, rules, nowMs);
+  const seeds = rules.map(seedForRule);
+  // Warm identity = schema + bucket + the exact declared configuration, so a
+  // changed rule shape is never mistaken for an already-seeded one.
+  const warmKey = `${schemaName(input)}\u0000${input.bucketId}\u0000${stringifyPostgresJson(seeds)}`;
+  // Undefined nowMs stays undefined all the way into SQL: the seed and the
+  // grant each read clock_timestamp() inside their own statement, so an
+  // acquire never spends a round trip asking the database what time it is.
+  const nowMs = input.nowMs ?? null;
   const hasConcurrency = rules.some((rule) => rule.maxConcurrency != null);
-  if (!hasConcurrency) {
-    return acquirePureRate(client, { ...input, nowMs }, rules, requested);
+  let grantInput = input;
+  const grant = (): Promise<InternalAcquireOutcome> =>
+    hasConcurrency
+      ? acquireWithConcurrency(client, grantInput, rules, seeds, requested)
+      : acquirePureRate(client, grantInput, rules, seeds, requested);
+
+  // Cold path only: seed missing rows / refresh changed configuration. A warm
+  // acquire is exactly one statement.
+  let seeded = false;
+  if (!warmRateBuckets.has(warmKey)) {
+    await seedBucketRows(client, input, seeds, nowMs);
+    seeded = true;
+  }
+  const observationBatchDisposition = await applyConcurrencyObservations(
+    client,
+    input,
+    rules,
+    concurrencyObservationsFrom(input.concurrencyObservations),
+    concurrencyObservationBatchFrom(input.concurrencyObservationBatch),
+  );
+  // The buffered success count and RTT observations share one identity. If a
+  // runner lost the prior acquire response, its retry still needs a fresh
+  // permit, but it must not ramp AIMD or advance Gradient2 a second time.
+  grantInput =
+    observationBatchDisposition === 'already_applied'
+      ? { ...input, observedSuccesses: 0 }
+      : input;
+  let outcome = await grant();
+  if (outcome.observedRuleRows < rules.length) {
+    // The rows this process believed warm are gone (schema reset, manual
+    // delete, lost seed). Never let that decay into a silent granted=0 spin.
+    warmRateBuckets.delete(warmKey);
+    if (seeded || outcome.granted > 0) {
+      throw new Error(
+        `Rate-state bucket ${input.bucketId} locked ${outcome.observedRuleRows} of ${rules.length} rule rows (${rules
+          .map((rule) => rule.ruleId)
+          .join(', ')}) after seeding.`,
+      );
+    }
+    await seedBucketRows(client, input, seeds, nowMs);
+    outcome = await grant();
+    if (outcome.observedRuleRows < rules.length) {
+      throw new Error(
+        `Rate-state bucket ${input.bucketId} still missing rule rows after re-seeding: locked ${outcome.observedRuleRows} of ${rules.length} (${rules
+          .map((rule) => rule.ruleId)
+          .join(', ')}).`,
+      );
+    }
+  }
+  if (outcome.configDrift && !seeded) {
+    // A warm mark is not a promise that nobody edited the rows. The grant
+    // statement compares the stored configuration against this acquire's seed,
+    // so a row widened out of band is repaired here instead of waiting for a
+    // process restart. Repairing after the grant keeps the acquire itself to a
+    // single granting statement, so no permit is ever issued twice.
+    await seedBucketRows(client, input, seeds, nowMs);
   }
-  return acquireWithConcurrency(client, { ...input, nowMs }, rules, requested);
+  markRateBucketWarm(warmKey);
+  return acquireResultOf(outcome);
 }
 
 export async function releasePostgresRateState(
   client: PostgresSchedulerQueryClient,
   input: PlayRunnerRateStateReleaseInput & { nowMs?: number },
 ): Promise<void> {
-  const rules = normalizeRules(input.rules as AcquirePacingRule[]).filter(
-    (rule) => rule.maxConcurrency != null,
+  return await withRateStateAvailabilityGuard(input, () =>
+    releaseRateStateStatements(client, input),
   );
+}
+
+async function releaseRateStateStatements(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerRateStateReleaseInput & { nowMs?: number },
+): Promise<void> {
+  const allRules = normalizeRules(input.rules as AcquirePacingRule[]);
+  if (allRules.length === 0) return;
+  const refundReservationIds = [
+    ...new Set(
+      (input.refundReservationIds ?? [])
+        .filter(
+          (reservationId): reservationId is string =>
+            typeof reservationId === 'string',
+        )
+        .map((reservationId) => reservationId.trim())
+        .filter(Boolean),
+    ),
+  ];
+  if (refundReservationIds.length > 0) {
+    // Delete only a server-minted, unexpired reservation. That one mutation is
+    // both authorization and idempotency: arbitrary runner input cannot credit
+    // a bucket, and an HTTP retry sees no reservation left to refund. The shared
+    // dispatch cursor deliberately stays monotonic, so cancellation can never
+    // compress provider starts into a burst.
+    await client.query(
+      `
+        WITH now_value AS (
+          SELECT COALESCE(
+            $3::bigint,
+            FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
+          ) AS now_ms
+        ),
+        refunded AS (
+          DELETE FROM ${reservationTableName(input)}
+          WHERE bucket_id = $1
+            AND reservation_id = ANY($4::text[])
+            AND expires_at_ms > (SELECT now_ms FROM now_value)
+          RETURNING reservation_id
+        ),
+        refund_count AS (
+          SELECT COUNT(*)::float8 AS value FROM refunded
+        ),
+        refreshed AS (
+          SELECT
+            b.bucket_id,
+            b.rule_id,
+            b.capacity,
+            b.refill_per_sec,
+            b.refilled_at_ms,
+            b.tokens,
+            nv.now_ms
+          FROM ${bucketTableName(input)} b
+          CROSS JOIN now_value nv
+          WHERE b.bucket_id = $1
+            AND b.rule_id = ANY($2::text[])
+          ORDER BY b.rule_id
+          FOR UPDATE OF b
+        )
+        UPDATE ${bucketTableName(input)} b
+        SET
+          tokens = LEAST(
+            r.capacity,
+            r.tokens
+              + GREATEST(0, r.now_ms - r.refilled_at_ms) / 1000.0 * r.refill_per_sec
+              + (SELECT value FROM refund_count)
+          ),
+          refilled_at_ms = r.now_ms
+        FROM refreshed r
+        WHERE b.bucket_id = r.bucket_id
+          AND b.rule_id = r.rule_id
+          AND (SELECT value FROM refund_count) > 0
+      `,
+      [
+        input.bucketId,
+        allRules.map((rule) => rule.ruleId),
+        input.nowMs ?? null,
+        refundReservationIds,
+      ],
+    );
+  }
+  const rules = allRules.filter((rule) => rule.maxConcurrency != null);
   if (rules.length === 0) return;
   const leaseIds = [
     ...new Set(
@@ -859,6 +2403,15 @@ export async function releasePostgresRateState(
 export async function penalizePostgresRateState(
   client: PostgresSchedulerQueryClient,
   input: PlayRunnerRateStatePenalizeInput & { nowMs?: number },
+): Promise<void> {
+  return await withRateStateAvailabilityGuard(input, () =>
+    penalizeRateStateStatements(client, input),
+  );
+}
+
+async function penalizeRateStateStatements(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerRateStatePenalizeInput & { nowMs?: number },
 ): Promise<void> {
   if (!Number.isFinite(input.cooldownMs) || input.cooldownMs <= 0) return;
   // Single autocommit statement applied to every rule row of the bucket:
@@ -904,4 +2457,13 @@ export async function penalizePostgresRateState(
       MAX_INTERNAL_COOLDOWN_MS,
     ],
   );
+  const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
+  const throttle = await concurrencyThrottleFrom(client, input, rules);
+  await applyConcurrencyObservations(
+    client,
+    input,
+    rules,
+    throttle.observations,
+    throttle.batch,
+  );
 }