deepline 0.3.323 → 0.3.325

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -10,6 +10,7 @@ import type {
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
   DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS,
   DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
   PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingRule,
@@ -34,6 +35,7 @@ const RATE_RESERVATION_BUCKET_EXPIRY_INDEX =
   'runtime_scheduler_rate_reservation_bucket_expiry_idx';
 const RATE_REQUESTER_TABLE = 'rate_requesters';
 const RATE_SCOPE_HOLD_TABLE = 'rate_scope_holds';
+const PHYSICAL_DISPATCH_RULE_ID = 'runtime:physical-dispatch';
 
 const MIN_CONCURRENCY_WAIT_MS = 10;
 /**
@@ -53,6 +55,10 @@ const FAIR_SHARE_REQUESTER_IDLE_WINDOWS = 2;
 const FAIR_SHARE_REQUESTER_CLEANUP_BATCH_SIZE = 64;
 /** Conservative warm-start width for a previously unobserved provider scope. */
 const INITIAL_CONCURRENCY_DISCOVERY_LIMIT = 8;
+// Rate spacing cannot bound requests delayed between admission and the
+// physical provider. Unknown scopes discover with at most two shared leases
+// until a productive rate epoch establishes that the provider can sustain more.
+const UNKNOWN_PROVIDER_CONCURRENCY_DISCOVERY_LIMIT = 2;
 /** A transport batch must stay bounded even when a runner completes huge maps. */
 const MAX_CONCURRENCY_OBSERVATIONS_PER_ACQUIRE = 4_096;
 /** Keep retry identities long enough for failed runtime work to recover. */
@@ -68,6 +74,7 @@ const MIN_ADAPTIVE_RPS_FLOOR = 5 / 60;
 const MIN_NO_PROGRESS_PROBE_RPS = 1 / 60;
 // Used only to keep wait-time division finite for persisted legacy state.
 const MIN_SCHEDULE_RPS = 1 / 86_400;
+const PRODUCTIVE_UNKNOWN_PROVIDER_DECREASE_FACTOR = 0.9;
 /**
  * Horizon a granted=0 caller is allowed to sleep toward a WHOLE block instead of
  * a single token. Waiting for one token means one permit per store round trip,
@@ -115,8 +122,8 @@ export type PostgresRateStatePenaltyTelemetry = {
  * persisting a learned capacity would make every warm acquire look like drift
  * and re-seed the row. The bucket the grant actually spends is therefore
  * derived, not stored: one second of `refill_per_sec` before a throttle and
- * 100ms of it afterwards. Before a throttle the declared burst is a floor;
- * after a throttle only a single probe is the floor.
+ * a short bounded slice afterwards. Before a throttle the declared burst is a
+ * floor; after a throttle only a single probe is the floor.
  * Retaining the catalog burst after learning a slow rate lets a fresh runtime
  * reserve minutes of future slots before another run can register demand.
  *
@@ -128,12 +135,17 @@ export type PostgresRateStatePenaltyTelemetry = {
  * calls/s no matter how high `refill_per_sec` climbed.
  *
  * The burst stays bounded by the learned rate and `max_rps`. The dispatch
- * cursor still spaces every permit at 1/`refill_per_sec`, so provider arrivals
- * never exceed the rate in force. The shorter post-throttle horizon limits
- * fixed-window boundary overshoot while retaining small batches on fast lanes.
+ * cursor still spaces every permit at 1/`refill_per_sec`. The shorter
+ * post-throttle horizon limits fixed-window boundary overshoot while retaining
+ * small batches on fast lanes.
  */
 const ADAPTIVE_BURST_HORIZON_MS = 1_000;
 const BACKPRESSURED_BURST_HORIZON_MS = 100;
+// Unknown providers recover near 10 RPS, where a 100ms slice leases only one
+// permit. A shared admission round trip over 100ms would then cap actual work
+// below the recovered target. Lease half a second of scheduled permits after
+// the first probe; their dispatch times and the final shared hold still apply.
+const UNKNOWN_PROVIDER_BACKPRESSURED_BURST_HORIZON_MS = 500;
 
 /**
  * Effective (spendable) bucket capacity for a rate-row alias. Derived per
@@ -142,7 +154,11 @@ const BACKPRESSURED_BURST_HORIZON_MS = 100;
 function effectiveCapacitySql(
   alias: string,
   refillColumn = 'refill_per_sec',
+  unknownProvider = false,
 ): string {
+  const backpressuredBurstHorizonMs = unknownProvider
+    ? UNKNOWN_PROVIDER_BACKPRESSURED_BURST_HORIZON_MS
+    : BACKPRESSURED_BURST_HORIZON_MS;
   // After a provider 429, Retry-After marks the earliest safe probe time; it
   // does not authorize banking a full rate-sized burst while the hold runs.
   // rate_observation_started_at_ms stays NULL until the first post-hold grant,
@@ -158,7 +174,7 @@ function effectiveCapacitySql(
     ELSE GREATEST(
       CASE WHEN ${alias}.seen_backpressure OR ${alias}.${refillColumn} < ${alias}.base_refill_per_sec THEN 1 ELSE ${alias}.capacity END,
       ${alias}.${refillColumn} * CASE
-        WHEN ${alias}.seen_backpressure THEN ${BACKPRESSURED_BURST_HORIZON_MS}
+        WHEN ${alias}.seen_backpressure THEN ${backpressuredBurstHorizonMs}
         ELSE ${ADAPTIVE_BURST_HORIZON_MS}
       END / 1000.0
     )
@@ -753,6 +769,7 @@ async function seedBucketRows(
               AND existing.capacity = sc.capacity
           ) AS unlearned_unknown_provider
         FROM ${bucketTableName(input)} WHERE bucket_id = $1
+          AND rule_id <> '${PHYSICAL_DISPATCH_RULE_ID}'
       ), seed_scope_hold AS (
         -- Preserve actual holds, but never import poisoned per-rule learning.
         -- A legacy throttled scope restarts at at most one probe/second.
@@ -1279,6 +1296,9 @@ async function acquirePureRate(
   requested: number,
 ): Promise<InternalAcquireOutcome> {
   const ruleIds = rules.map((rule) => rule.ruleId);
+  const unknownProvider = rules.some(
+    (rule) => rule.ruleId === 'default:provider-scope',
+  );
   const requesterId = normalizeRequesterId(input);
   const fair = requesterId
     ? fairShareSql(input, {
@@ -1325,6 +1345,7 @@ async function acquirePureRate(
         scopeHoldTableName(input),
         '$5',
         minimumRateSqlForRules(rules),
+        unknownProvider,
       )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
@@ -1374,10 +1395,10 @@ async function acquirePureRate(
           -- Spendable burst: one second of the rate in force, floored at the
           -- declared window burst. The normalized row is what this acquire
           -- spends; the stored row remains available for drift reporting.
-          ${effectiveCapacitySql('normalized')} AS effective_capacity,
-          ${effectiveCapacitySql('normalized', 'next_refill_per_sec')} AS scheduled_capacity,
+          ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)} AS effective_capacity,
+          ${effectiveCapacitySql('normalized', 'next_refill_per_sec', unknownProvider)} AS scheduled_capacity,
           LEAST(
-            ${effectiveCapacitySql('normalized')},
+            ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
             ph.tokens_after_refill,
             LEAST(b.tokens, sc.capacity)
               + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
@@ -1479,7 +1500,7 @@ async function acquirePureRate(
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
-      ${updateRateScopeSql(scopeHoldTableName(input))},
+      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -1508,8 +1529,9 @@ async function acquirePureRate(
           -- AIMD ramp, folded in. Reaching this refill proves a local lease
           -- block was exhausted; only a confirmed provider success can turn
           -- that demand into a rate increase. Slow-start doubles until the
-          -- first 429, then increases by at most 25% per successful recovery
-          -- step. Never exceeds max_rps.
+          -- first 429. Known-provider recovery uses 25% steps; unknown scopes
+          -- rediscover below their seed and probe gently above it. Never
+          -- exceeds max_rps.
           refill_per_sec = LEAST(r.max_rps, (SELECT next_refill_per_sec FROM provider_hold)),
           last_ramp_at_ms = (SELECT next_ramp_at_ms FROM provider_hold)
         FROM refreshed r
@@ -1702,8 +1724,10 @@ async function acquirePureRate(
  * Mixed / concurrency-bearing acquire. The token bucket still governs the
  * request rate; the durable lease table governs the simultaneous-in-flight cap.
  * Expired-lease reclaim + active count + rate refill + grant + token deduction +
- * lease minting all execute in ONE multi-CTE statement (one implicit
- * transaction), so a crash can never deduct tokens without minting the matching
+ * lease minting all execute in one multi-CTE statement after taking the scope
+ * lock in a separate READ COMMITTED statement. Waiting for that lock must finish
+ * before the lease count takes its snapshot, or concurrent acquires miss leases
+ * committed during their wait. A crash cannot deduct tokens without minting the matching
  * lease rows. Lease ids are generated in SQL (gen_random_uuid) and returned to
  * the client, one per granted permit, releasable independently. With a
  * requester identity the grant is also bounded by that run's fair share of
@@ -1716,8 +1740,44 @@ async function acquireWithConcurrency(
   rules: AcquirePacingRule[],
   seeds: RateBucketSeed[],
   requested: number,
+): Promise<InternalAcquireOutcome> {
+  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+  try {
+    await client.query(
+      `SELECT bucket_id FROM ${scopeHoldTableName(input)}
+       WHERE bucket_id = $1 FOR UPDATE`,
+      [input.bucketId],
+    );
+    const outcome = await grantWithConcurrency(
+      client,
+      input,
+      rules,
+      seeds,
+      requested,
+    );
+    await client.query('COMMIT');
+    return outcome;
+  } catch (error) {
+    try {
+      await client.query('ROLLBACK');
+    } catch {
+      // Preserve the original admission/storage error.
+    }
+    throw error;
+  }
+}
+
+async function grantWithConcurrency(
+  client: PostgresSchedulerQueryClient,
+  input: RateStateAcquireInput,
+  rules: AcquirePacingRule[],
+  seeds: RateBucketSeed[],
+  requested: number,
 ): Promise<InternalAcquireOutcome> {
   const ruleIds = rules.map((rule) => rule.ruleId);
+  const unknownProvider = rules.some(
+    (rule) => rule.ruleId === 'default:provider-scope',
+  );
   const concurrencyRules = rules.filter((rule) => rule.maxConcurrency != null);
   const concurrencyRuleIds = concurrencyRules.map((rule) => rule.ruleId);
   const requesterId = normalizeRequesterId(input);
@@ -1741,10 +1801,10 @@ async function acquireWithConcurrency(
   // One multi-CTE statement: reclaim expired leases, refill the rate rows,
   // count live leases, compute the grant bounded by rate AND free concurrency
   // slots, deduct the tokens, AND mint the lease rows — atomically. Grant +
-  // deduction + lease minting share one implicit transaction, so a crash can
+  // deduction + lease minting share the scope-lock transaction, so a crash can
   // never deduct tokens without the matching lease rows (which would otherwise
-  // over-admit the provider until the phantom leases "expired"). PgBouncer-safe:
-  // autocommit, deterministic row locking, no lock_timeout, no session state.
+  // over-admit the provider until the phantom leases "expired"). PgBouncer's
+  // transaction pooling keeps the scope lock through this fresh snapshot.
   const result = await client.query<{
     granted: string | number;
     wait_ms: string | number;
@@ -1769,6 +1829,7 @@ async function acquireWithConcurrency(
         scopeHoldTableName(input),
         '$10',
         minimumRateSqlForRules(rules),
+        unknownProvider,
       )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
@@ -1823,6 +1884,7 @@ async function acquireWithConcurrency(
           b.concurrency_minimum_rtt_ms,
           ph.cool_until_ms,
           ph.last_ramp_at_ms,
+          ph.rate_observation_started_at_ms,
           GREATEST(b.next_dispatch_at_ms, ph.next_dispatch_at_ms) AS next_dispatch_at_ms,
           ph.rate_recovery_interval_ms,
           ph.observation_was_idle AS observation_was_idle,
@@ -1838,10 +1900,10 @@ async function acquireWithConcurrency(
           ph.retry_at_ms AS scope_retry_at_ms,
           -- Same derived burst as the pure-rate path. The declared
           -- maxConcurrency is untouched: free_slots still bounds the grant.
-          ${effectiveCapacitySql('normalized')} AS effective_capacity,
-          ${effectiveCapacitySql('normalized', 'next_refill_per_sec')} AS scheduled_capacity,
+          ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)} AS effective_capacity,
+          ${effectiveCapacitySql('normalized', 'next_refill_per_sec', unknownProvider)} AS scheduled_capacity,
           LEAST(
-            ${effectiveCapacitySql('normalized')},
+            ${effectiveCapacitySql('normalized', 'refill_per_sec', unknownProvider)},
             ph.tokens_after_refill,
             LEAST(b.tokens, sc.capacity)
               + GREATEST(0, nv.now_ms - b.refilled_at_ms) / 1000.0
@@ -1891,10 +1953,17 @@ async function acquireWithConcurrency(
           lim.rule_id,
           LEAST(
             lim.max_concurrency,
+            CASE WHEN bucket.rule_id = 'default:provider-scope'
+                AND NOT bucket.seen_backpressure
+                AND bucket.last_ramp_at_ms <= COALESCE(bucket.rate_observation_started_at_ms, bucket.now_ms)
+              THEN ${UNKNOWN_PROVIDER_CONCURRENCY_DISCOVERY_LIMIT}
+              ELSE lim.max_concurrency END,
             CEIL(
               COALESCE(
                 bucket.learned_concurrency_limit,
-                LEAST(lim.max_concurrency, ${INITIAL_CONCURRENCY_DISCOVERY_LIMIT})
+                LEAST(lim.max_concurrency, CASE WHEN bucket.rule_id = 'default:provider-scope'
+                  THEN ${UNKNOWN_PROVIDER_CONCURRENCY_DISCOVERY_LIMIT}
+                  ELSE ${INITIAL_CONCURRENCY_DISCOVERY_LIMIT} END)
               )
             )
           )::int AS max_concurrency
@@ -1999,7 +2068,7 @@ async function acquireWithConcurrency(
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
-      ${updateRateScopeSql(scopeHoldTableName(input))},
+      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -2035,6 +2104,7 @@ async function acquireWithConcurrency(
           AND b.rule_id = r.rule_id
         RETURNING
           r.tokens_after_refill,
+          r.effective_capacity AS capacity,
           b.refill_per_sec AS new_refill_per_sec
       ),
       lease_tokens AS (
@@ -2177,7 +2247,19 @@ async function acquireWithConcurrency(
           WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
           ELSE (
             SELECT CEIL(
-              MAX((1.0 - tokens_after_refill) / GREATEST(new_refill_per_sec, ${MIN_SCHEDULE_RPS}) * 1000)
+              MAX(
+                (
+                  GREATEST(
+                    1.0,
+                    LEAST(
+                      $4::numeric,
+                      FLOOR(capacity),
+                      FLOOR(new_refill_per_sec * ${BLOCK_WAIT_HORIZON_MS} / 1000.0),
+                      (SELECT free FROM free_slots)
+                    )
+                  ) - tokens_after_refill
+                ) / GREATEST(new_refill_per_sec, ${MIN_SCHEDULE_RPS}) * 1000
+              )
             )::bigint
             FROM updated
           )
@@ -2320,6 +2402,124 @@ export async function validatePostgresRateStatePermit(
   });
 }
 
+/**
+ * Space unknown-provider sends after app preparation, where network/charge
+ * delays can collapse previously spaced permits. The existing scope remains
+ * the only learner; this internal rule row stores only the physical cadence.
+ * Legacy/uninitialized and declared scopes retain hold-only validation.
+ */
+export async function claimPostgresProviderDispatch(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerRateStateValidateInput & { nowMs?: number },
+): Promise<
+  PlayRunnerRateStateValidateResult & { physicalCadenceWait: boolean }
+> {
+  return withRateStateAvailabilityGuard(input, async () => {
+    const result = await client.query<{
+      server_now_ms: string | number;
+      retry_at_ms: string | number;
+      allowed: boolean;
+      cadence_wait: boolean;
+    }>(
+      `
+        WITH now_value AS (
+          SELECT COALESCE($2::bigint,
+            FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint) AS now_ms
+        ), scope_rows AS (
+          SELECT h.retry_at_ms, h.cool_until_ms, h.seen_backpressure,
+            CASE WHEN EXISTS (
+              SELECT 1 FROM ${bucketTableName(input)} b
+              WHERE b.bucket_id = h.bucket_id AND b.rule_id = 'default:provider-scope'
+            ) THEN h.refill_per_sec ELSE NULL END AS physical_rps
+          FROM ${scopeHoldTableName(input)} h WHERE h.bucket_id = $1
+          FOR SHARE OF h
+        ), cadence AS (
+          SELECT nv.now_ms,
+            GREATEST(COALESCE(s.retry_at_ms, 0), COALESCE(s.cool_until_ms, 0)) AS hold_until_ms,
+            s.physical_rps,
+            COALESCE(s.seen_backpressure, false) AS preserve_cadence_phase,
+            GREATEST(1, FLOOR(s.physical_rps / 100))::int AS burst_size,
+            GREATEST(1, CEIL(1000.0 / s.physical_rps))::bigint AS spacing_ms
+          FROM now_value nv LEFT JOIN scope_rows s ON true
+        ), cursor AS (
+          INSERT INTO ${bucketTableName(input)} AS existing (
+            bucket_id, rule_id, tokens, capacity, refill_per_sec,
+            base_refill_per_sec, min_rps, max_rps, refilled_at_ms,
+            next_dispatch_at_ms, cool_until_ms
+          )
+          SELECT $1, '${PHYSICAL_DISPATCH_RULE_ID}', c.burst_size - 1,
+            c.burst_size, c.physical_rps, c.physical_rps,
+            ${MIN_SCHEDULE_RPS}, c.physical_rps, c.now_ms,
+            c.now_ms + c.spacing_ms * c.burst_size, 0
+          FROM cadence c
+          WHERE c.hold_until_ms <= c.now_ms AND c.physical_rps > 0
+          ON CONFLICT (bucket_id, rule_id) DO UPDATE SET
+            -- Always return the locked current cursor, including a denial.
+            -- A snapshot read after a blocked upsert could miss a concurrent
+            -- insert and report the wrong retry deadline.
+            tokens = CASE
+              WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
+                THEN EXCLUDED.capacity - 1
+              WHEN LEAST(existing.tokens, EXCLUDED.capacity) >= 1
+                THEN LEAST(existing.tokens, EXCLUDED.capacity) - 1
+              ELSE existing.tokens END,
+            cool_until_ms = CASE
+              WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
+                OR LEAST(existing.tokens, EXCLUDED.capacity) >= 1 THEN 0
+              ELSE LEAST(existing.next_dispatch_at_ms, EXCLUDED.next_dispatch_at_ms) END,
+            -- Persist debt decay at the new pace so repeated probes converge
+            -- to this deadline instead of moving it forward on every denial.
+            next_dispatch_at_ms = CASE
+              WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
+                -- Keep the cadence phase when a request reaches its slot
+                -- late. Reanchoring every grant to now adds query/network
+                -- jitter to every interval and lowers sustained throughput.
+                -- Expire missed slots and retain only the first future slot,
+                -- so idle time never creates a catch-up burst.
+                THEN CASE WHEN (SELECT preserve_cadence_phase FROM cadence)
+                  THEN (existing.next_dispatch_at_ms::numeric
+                    + (
+                      FLOOR(
+                        (EXCLUDED.refilled_at_ms::numeric - existing.next_dispatch_at_ms::numeric)
+                          / (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms)
+                      ) + 1
+                    ) * (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms))::bigint
+                  -- Cold discovery retains strict spacing until provider
+                  -- backpressure establishes an adaptive cadence.
+                  ELSE EXCLUDED.next_dispatch_at_ms END
+              ELSE LEAST(existing.next_dispatch_at_ms, EXCLUDED.next_dispatch_at_ms) END,
+            capacity = EXCLUDED.capacity,
+            refill_per_sec = EXCLUDED.refill_per_sec,
+            base_refill_per_sec = EXCLUDED.base_refill_per_sec,
+            max_rps = EXCLUDED.max_rps,
+            refilled_at_ms = EXCLUDED.refilled_at_ms
+          RETURNING cool_until_ms
+        )
+        SELECT c.now_ms AS server_now_ms,
+          GREATEST(c.hold_until_ms, COALESCE(cursor.cool_until_ms, 0)) AS retry_at_ms,
+          c.hold_until_ms <= c.now_ms AND (
+            c.physical_rps IS NULL OR cursor.cool_until_ms = 0
+          ) AS allowed,
+          c.hold_until_ms <= c.now_ms AND COALESCE(cursor.cool_until_ms > c.now_ms, false) AS cadence_wait
+        FROM cadence c LEFT JOIN cursor ON true
+      `,
+      [input.bucketId, input.nowMs ?? null],
+    );
+    const row = result.rows[0];
+    if (!row || typeof row.allowed !== 'boolean') {
+      throw new Error(
+        `Physical provider admission returned no decision for ${input.bucketId}.`,
+      );
+    }
+    return {
+      allowed: row.allowed,
+      physicalCadenceWait: row.cadence_wait,
+      serverNowMs: Number(row.server_now_ms),
+      retryAtMs: Number(row.retry_at_ms),
+    };
+  });
+}
+
 type ConcurrencyObservation = Gradient2Observation & {
   kind: 'success' | 'throttled';
 };
@@ -2723,7 +2923,10 @@ async function applyConcurrencyObservations(
         maximumLimit,
         Math.max(
           1,
-          row.learned_concurrency_limit ?? INITIAL_CONCURRENCY_DISCOVERY_LIMIT,
+          row.learned_concurrency_limit ??
+            (rule.ruleId === 'default:provider-scope'
+              ? UNKNOWN_PROVIDER_CONCURRENCY_DISCOVERY_LIMIT
+              : INITIAL_CONCURRENCY_DISCOVERY_LIMIT),
         ),
       );
       limitsBefore.push(initialLimit);
@@ -2731,6 +2934,7 @@ async function applyConcurrencyObservations(
         providerConcurrencyControllerOptions({
           initialLimit,
           maximumLimit,
+          unknownProvider: rule.ruleId === 'default:provider-scope',
         }),
         row.learned_concurrency_limit == null
           ? undefined
@@ -2745,7 +2949,7 @@ async function applyConcurrencyObservations(
               ),
               windowStartedAtLimit: Math.max(
                 1,
-                Math.trunc(row.concurrency_window_limit ?? initialLimit),
+                row.concurrency_window_limit ?? initialLimit,
               ),
               hasObservedThrottle:
                 row.concurrency_has_observed_throttle === true,
@@ -2826,7 +3030,8 @@ async function acquireRateStateStatements(
       : acquirePureRate(client, grantInput, rules, seeds, requested);
 
   // Cold path only: seed missing rows / refresh changed configuration. A warm
-  // acquire is exactly one statement.
+  // pure-rate acquire is exactly one statement. Concurrency-bearing acquires
+  // take the scope lock before the lease-count statement gets its snapshot.
   let seeded = false;
   if (!warmRateBuckets.has(warmKey)) {
     await seedBucketRows(client, input, seeds, nowMs);
@@ -3067,7 +3272,7 @@ async function penalizeRateStateStatements(
   // one-minute probe interval so the five-RPM floor cannot turn into a retry
   // loop.
   //  - refill_per_sec: multiplicative decrease, but ONLY once per active
-  //    cooldown window (halve-once guard: skip if already cooling), floored at
+  //    cooldown window (skip if already cooling), floored at
   //    min_rps.
   //  - seen_backpressure flips the ramp from slow-start (x2) to additive.
   //  - tokens drained: no more grants until refill re-accrues past the cooldown.
@@ -3095,9 +3300,37 @@ async function penalizeRateStateStatements(
         ) AS now_ms
       ), held AS (
         SELECT * FROM ${scopeHoldTableName(input)} WHERE bucket_id = $1 FOR UPDATE
+      ), seeded_hold AS (
+        SELECT h.*, CASE WHEN cardinality($6::text[]) > 0
+          THEN 'default:provider-scope' = ANY($6::text[])
+            AND NOT EXISTS (
+              SELECT 1 FROM unnest($6::text[]) AS active(rule_id)
+              WHERE active.rule_id NOT LIKE 'default:%'
+            )
+          ELSE EXISTS (
+          SELECT 1 FROM ${bucketTableName(input)} b
+          WHERE b.bucket_id = h.bucket_id
+            AND b.rule_id = 'default:provider-scope'
+          ) AND NOT EXISTS (
+            SELECT 1 FROM ${bucketTableName(input)} b
+            WHERE b.bucket_id = h.bucket_id
+              AND b.rule_id NOT LIKE 'default:%'
+              AND b.rule_id <> '${PHYSICAL_DISPATCH_RULE_ID}'
+          ) END AS unknown_provider
+        FROM held h
       ), evidence AS (
         SELECT h.*,
           COALESCE(h.refill_per_sec, 1) AS refill_before,
+          -- A productive unknown-provider probe establishes useful capacity.
+          -- Preserve 90% of its target instead of discarding half after each
+          -- exploratory overshoot. Failed or sparse probes keep the stronger
+          -- decrease and no-progress policy. Prefer the attempt's active rules.
+          -- Early app feedback carries no rules: only an exclusively unknown
+          -- durable seed can select this policy, not an obsolete default row
+          -- beside declared operation rows.
+          h.unknown_provider AND h.refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+            AND h.rate_successes >= ${DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS}
+              AS productive_unknown_probe,
           GREATEST(h.retry_at_ms, COALESCE($5::bigint, nv.now_ms + $2::bigint)) AS deadline,
           (
             COALESCE($5::bigint, nv.now_ms + $2::bigint) > nv.now_ms
@@ -3122,17 +3355,25 @@ async function penalizeRateStateStatements(
                   - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
             END) AS cycle_ms,
           CASE
-            WHEN h.rate_successes <= 0 AND ($5::bigint IS NOT NULL OR h.seen_backpressure) THEN
+            -- A first unknown-provider rejection establishes its rate floor,
+            -- not a failed probe streak. Respect its explicit deadline even
+            -- when it expires in transit; the reduced dispatch budget still
+            -- paces the next request. Only a subsequent admitted probe with
+            -- no successes justifies the shared exponential pause.
+            WHEN h.rate_successes <= 0
+              AND ($5::bigint IS NOT NULL OR h.seen_backpressure)
+              AND (NOT h.unknown_provider OR h.seen_backpressure) THEN
               CASE WHEN h.seen_backpressure THEN
                 LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
               ELSE 1000 END
             ELSE NULL
           END AS no_progress_probe_interval_ms,
           nv.now_ms
-        FROM held h CROSS JOIN now_value nv
+        FROM seeded_hold h CROSS JOIN now_value nv
       ), throttle_ceiling AS (
         SELECT e.*,
-          GREATEST(${minimumRateRps},
+          GREATEST(CASE WHEN e.unknown_provider
+            THEN ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS} ELSE ${minimumRateRps} END,
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
               CASE WHEN e.no_progress_probe_interval_ms IS NOT NULL
                 -- A fresh provider deadline is not a throughput measurement.
@@ -3143,6 +3384,7 @@ async function penalizeRateStateStatements(
                 THEN GREATEST(${noProgressProbeRateRps},
                   1000.0 / e.no_progress_probe_interval_ms)
                 ELSE e.refill_before END
+              WHEN e.productive_unknown_probe THEN e.refill_before * ${PRODUCTIVE_UNKNOWN_PROVIDER_DECREASE_FACTOR}
               ELSE e.refill_before / 2 END
           ) AS next_ceiling_rps
         FROM evidence e
@@ -3151,11 +3393,12 @@ async function penalizeRateStateStatements(
           CASE WHEN e.new_episode THEN GREATEST(
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0
               THEN ${noProgressProbeRateRps}
-              ELSE ${minimumRateRps} END,
-            LEAST(
+              ELSE CASE WHEN e.unknown_provider
+                THEN ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS} ELSE ${minimumRateRps} END END,
+            CASE WHEN e.productive_unknown_probe THEN e.next_ceiling_rps ELSE LEAST(
             e.next_ceiling_rps,
             GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
-          )) ELSE e.refill_before END AS refill_after
+          ) END) ELSE e.refill_before END AS refill_after
         FROM throttle_ceiling e
       ), provider_hold AS (
         UPDATE ${scopeHoldTableName(input)} h SET
@@ -3165,21 +3408,33 @@ async function penalizeRateStateStatements(
           -- A repeated no-success throttle can impose a slower local probe
           -- interval than the normal adaptive floor. Keep that pause separate
           -- from retry_at_ms, which remains the provider's absolute deadline.
-          cool_until_ms = GREATEST(e.cool_until_ms, LEAST(
+          -- Only a newly admitted probe can start another backoff episode.
+          -- Gateway/runner copies and late in-flight siblings still extend
+          -- retry_at_ms, but must not slide the local no-progress pause.
+          cool_until_ms = CASE WHEN e.new_episode THEN GREATEST(e.cool_until_ms, LEAST(
             e.now_ms + $4::bigint,
             GREATEST(
               e.deadline,
-              e.now_ms + LEAST($2::bigint, $4::bigint),
+              -- cooldownMs was captured before transport/pool admission. When
+              -- an absolute deadline exists, adding that stale duration to
+              -- the database clock would restart part of the same hold.
+              -- Legacy duration-only callers still start their hold here.
+              CASE WHEN $5::bigint IS NULL
+                THEN e.now_ms + LEAST($2::bigint, $4::bigint)
+                ELSE e.now_ms END,
               e.now_ms + COALESCE(e.no_progress_probe_interval_ms, 0)
             )
-          )),
+          )) ELSE e.cool_until_ms END,
           tokens = CASE WHEN e.new_episode THEN 0 ELSE e.tokens END,
           rate_successes = e.rate_successes,
           rate_throttle_ceiling_rps = CASE WHEN e.new_episode
             THEN e.next_ceiling_rps ELSE e.rate_throttle_ceiling_rps END,
           rate_observation_started_at_ms = CASE WHEN e.new_episode THEN NULL ELSE e.rate_observation_started_at_ms END,
           rate_recovery_interval_ms = CASE WHEN e.new_episode
-            THEN COALESCE(e.no_progress_probe_interval_ms, e.cycle_ms)
+            THEN CASE WHEN e.unknown_provider AND e.rate_successes > 0
+              -- Retry-After is a deadline, not time spent proving capacity.
+              THEN GREATEST(1000, e.now_ms - COALESCE(e.rate_observation_started_at_ms, e.now_ms))
+              ELSE COALESCE(e.no_progress_probe_interval_ms, e.cycle_ms) END
             ELSE e.rate_recovery_interval_ms END,
           last_ramp_at_ms = CASE WHEN e.new_episode THEN e.deadline ELSE e.last_ramp_at_ms END,
           -- All pre-429 permits are revalidated against this durable hold
@@ -3193,6 +3448,7 @@ async function penalizeRateStateStatements(
         SELECT b.bucket_id, b.rule_id, b.refill_per_sec AS refill_before
         FROM ${bucketTableName(input)} b
         JOIN provider_hold h ON h.bucket_id = b.bucket_id
+        WHERE b.rule_id <> '${PHYSICAL_DISPATCH_RULE_ID}'
         ORDER BY b.rule_id FOR UPDATE OF b
       ), updated AS (
         -- Compatibility projections only: no per-rule learning history is
@@ -3235,6 +3491,7 @@ async function penalizeRateStateStatements(
       input.nowMs ?? null,
       PROVIDER_AIMD_COOLDOWN_CAP_MS,
       retryAtMs,
+      rules.map((rule) => rule.ruleId),
     ],
   );
   const rateRow = rateStateResult.rows[0];