deepline 0.3.332 → 0.3.334

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts

@@ -16,6 +16,7 @@ export interface PlayRunnerRuntimeDeadlineOptions {
 }
 
 export interface PlayRunnerRuntimeDeadlineHandle {
+  drainAtMs: number;
   /** Claim normal settlement. False means the deadline already won. */
   claimCompletion: () => boolean;
   /** Settles only if the deadline callback unexpectedly returns or rejects. */
@@ -60,6 +61,10 @@ export function startPlayRunnerRuntimeDeadline(
   options: PlayRunnerRuntimeDeadlineOptions,
 ): PlayRunnerRuntimeDeadlineHandle {
   const limitMs = Math.max(1, Math.floor(options.limitMs));
+  const drainDelayMs = options.onDrain
+    ? Math.max(1, limitMs - Math.min(options.drainLeadMs ?? 4_000, limitMs / 2))
+    : limitMs;
+  const drainAtMs = Date.now() + drainDelayMs;
   const setTimeoutImpl =
     options.setTimeoutImpl ??
     ((callback, delayMs) => setTimeout(callback, delayMs));
@@ -73,20 +78,14 @@ export function startPlayRunnerRuntimeDeadline(
     settleDeadline = resolve;
   });
   const drainHandle = options.onDrain
-    ? setTimeoutImpl(
-        () => {
-          if (fired || completionClaimed) return;
-          try {
-            options.onDrain?.();
-          } catch (error) {
-            options.onError?.(error);
-          }
-        },
-        Math.max(
-          1,
-          limitMs - Math.min(options.drainLeadMs ?? 4_000, limitMs / 2),
-        ),
-      )
+    ? setTimeoutImpl(() => {
+        if (fired || completionClaimed) return;
+        try {
+          options.onDrain?.();
+        } catch (error) {
+          options.onError?.(error);
+        }
+      }, drainDelayMs)
     : null;
   const handle = setTimeoutImpl(() => {
     if (fired || completionClaimed) return;
@@ -99,6 +98,7 @@ export function startPlayRunnerRuntimeDeadline(
   }, limitMs);
 
   return {
+    drainAtMs,
     claimCompletion: () => {
       if (fired || completionClaimed) return false;
       completionClaimed = true;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -736,7 +736,7 @@ export class AbsurdExecutionOperationTimeoutError extends Error {
  * park/resume seam without a real engine. */
 export type AbsurdSuspensionContext = Pick<
   TaskContext,
-  'sleepFor' | 'awaitEvent' | 'heartbeat'
+  'sleepFor' | 'sleepUntil' | 'awaitEvent' | 'heartbeat'
 >;
 
 /** Injected DB + app-runtime seams so the handler is testable against real SQL
@@ -1393,10 +1393,17 @@ export async function parkAndResolveSuspension(input: {
 
   if (suspension.kind === 'sleep') {
     // Throws SuspendTask on the first leg; returns once the deadline elapsed.
-    await ctx.sleepFor(
-      suspension.boundaryId,
-      Math.max(0, suspension.delayMs) / 1000,
-    );
+    if (suspension.wakeAtMs !== undefined) {
+      await ctx.sleepUntil(
+        suspension.boundaryId,
+        new Date(suspension.wakeAtMs),
+      );
+    } else {
+      await ctx.sleepFor(
+        suspension.boundaryId,
+        Math.max(0, suspension.delayMs) / 1000,
+      );
+    }
     return absurdCheckpointWithSleep(
       checkpoint,
       suspension.boundaryId,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -818,7 +818,12 @@ export function absurdCheckpointWithSleep(
     ...base,
     resolvedBoundaries: {
       ...resolvedBoundaries(base),
-      [boundaryId]: { kind: 'sleep', delayMs, completedAt: Date.now() },
+      [boundaryId]: {
+        ...resolvedBoundaries(base)[boundaryId],
+        kind: 'sleep',
+        delayMs,
+        completedAt: Date.now(),
+      },
     },
   };
 }

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -4,7 +4,10 @@
  * learn from copies of the same provider feedback. These CTEs are shared by
  * pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
  */
-import { DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS } from '../governor/rate-state-backend';
+import {
+  DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
+} from '../governor/rate-state-backend';
 
 const AIMD_RECOVERY_STEP_MS = 1_000;
 // Increase relative to the learned rate, not by a fixed RPS amount: a long
@@ -14,15 +17,17 @@ const AIMD_RECOVERY_STEP_MS = 1_000;
 const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
 // Keep upward probes small enough to retain headroom after a productive
 // decrease. Sparse successes use the earlier observation window; sustained
-// useful work can establish another small probe each second.
+// useful work at nearly the current target can establish another small probe
+// each second. Demand or historical successes cannot certify the next target.
 const UNKNOWN_PROVIDER_PROBE_STEP_RPS = 0.1;
 const UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS = 5_000;
-const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.8;
+const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.95;
 export function acquireRateScopeSql(
   table: string,
   successesParameter: string,
   minimumRateSql = 'NULL::float8',
   unknownProvider = false,
+  probeBaselineSupported = true,
 ): string {
   const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
     THEN h.refill_per_sec
@@ -35,11 +40,29 @@ export function acquireRateScopeSql(
   // Success counts are already applied once by the feedback writer. Measure
   // this post-hold epoch, excluding Retry-After and earlier idle history. A
   // productive second justifies a small new probe; a lone completion does not.
+  const probeStartedAtMs = probeBaselineSupported
+    ? 'GREATEST(h.rate_probe_started_at_ms, h.rate_observation_started_at_ms)'
+    : 'h.rate_observation_started_at_ms';
+  const probeSuccesses = probeBaselineSupported
+    ? 'GREATEST(0, h.rate_successes - h.rate_probe_successes_at_ramp)'
+    : 'h.rate_successes';
+  const productiveRateFraction = probeBaselineSupported
+    ? UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION
+    : 0.8;
+  const storedRecoveryInterval = `LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms))`;
+  const sparseRecoveryInterval = probeBaselineSupported
+    ? `CASE WHEN ${refillRate} > ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+      THEN ${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS} ELSE ${storedRecoveryInterval} END`
+    : storedRecoveryInterval;
+  const floorRecoveryInterval = probeBaselineSupported
+    ? `GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)`
+    : 'h.rate_recovery_interval_ms';
   const productiveRecovery = `h.rate_observation_started_at_ms IS NOT NULL
-    AND nv.now_ms - h.rate_observation_started_at_ms >= ${AIMD_RECOVERY_STEP_MS}
-    AND h.rate_successes * 1000.0
-      / GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - h.rate_observation_started_at_ms)
-      >= ${refillRate} * ${UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION}`;
+    ${probeBaselineSupported ? 'AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms' : ''}
+    AND nv.now_ms - ${probeStartedAtMs} >= ${AIMD_RECOVERY_STEP_MS}
+    AND ${probeSuccesses} * 1000.0
+      / GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - ${probeStartedAtMs})
+      >= ${refillRate} * ${productiveRateFraction}`;
   // Below useful seed capacity, a confirmed success can prove the next probe
   // after one dispatch interval at the current rate. Keeping the floor's 500ms
   // interval at 4/8 RPS waits through additional clean sends unnecessarily.
@@ -47,9 +70,29 @@ export function acquireRateScopeSql(
   const backpressuredRecoveryInterval = unknownProvider
     ? `CASE WHEN ${refillRate} >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
         THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
-          ELSE LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)) END
-        ELSE LEAST(h.rate_recovery_interval_ms, CEIL(1000.0 / ${refillRate})) END`
+          ELSE ${sparseRecoveryInterval} END
+        ELSE LEAST(${floorRecoveryInterval}, CEIL(1000.0 / ${refillRate})) END`
     : `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
+  // A local no-progress pause remains active after the provider's shorter
+  // Retry-After. Only time beyond every outstanding dispatch/hold is idle.
+  const observationWasIdle = `${successesParameter}::int = 0
+    AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
+      > GREATEST(1000, 2000.0 / ${refillRate})`;
+  // A denied refill may fold a completion before the next floor interval is
+  // due. Keep that fresh completion available for one later ramp rather than
+  // requiring another provider call at the old pace. Its baseline advances on
+  // every floor ramp, so time alone cannot reuse it for another increase.
+  const recoverySuccess =
+    unknownProvider && probeBaselineSupported
+      ? `(${successesParameter}::int > 0 OR (
+          h.seen_backpressure
+          AND ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+          AND h.rate_probe_started_at_ms >= h.last_ramp_at_ms
+          AND h.rate_probe_started_at_ms >= h.rate_observation_started_at_ms
+          AND h.rate_successes > h.rate_probe_successes_at_ramp
+          AND NOT (${observationWasIdle})
+        ))`
+      : `${successesParameter}::int > 0`;
   return `provider_hold_rows AS (
     SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
   ), provider_hold AS (
@@ -58,12 +101,8 @@ export function acquireRateScopeSql(
         GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
         h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
       ) AS tokens_after_refill,
-      ${successesParameter}::int = 0
-        -- A local no-progress pause remains an active observation episode,
-        -- even after the provider's shorter Retry-After has elapsed.
-        AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
-          > GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
-      CASE WHEN ${successesParameter}::int > 0
+      ${observationWasIdle} AS observation_was_idle,
+      CASE WHEN ${recoverySuccess}
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
             THEN ${backpressuredRecoveryInterval}
@@ -73,7 +112,7 @@ export function acquireRateScopeSql(
           THEN ${recoveryRate}
           ELSE ${refillRate} * 2 END)
         ELSE ${refillRate} END AS next_refill_per_sec,
-      CASE WHEN ${successesParameter}::int > 0
+      CASE WHEN ${recoverySuccess}
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
             THEN ${backpressuredRecoveryInterval}
@@ -87,13 +126,24 @@ export function acquireRateScopeSql(
 export function updateRateScopeSql(
   table: string,
   unknownProvider = false,
+  probeBaselineSupported = true,
 ): string {
-  // Floor recovery and productive probing measure different rates. Once the
-  // geometric ramp reaches useful capacity, start a fresh measurement instead
-  // of diluting its successful throughput with the earlier slow-start period.
+  // Keep the throttle epoch's productive evidence while measuring each upward
+  // probe from a new counter baseline. Losing the earlier useful work would
+  // turn an exploratory rejection into a zero-progress failure.
   const recoveredObservation = `${unknownProvider} AND ph.seen_backpressure
     AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
     AND ph.next_refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
+  const probedObservation = `${unknownProvider} AND ph.seen_backpressure
+    AND ph.refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+    AND ph.next_refill_per_sec > ph.refill_per_sec`;
+  const floorRampedObservation = `${unknownProvider} AND ph.seen_backpressure
+    AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+    AND ph.next_refill_per_sec > ph.refill_per_sec`;
+  const idleFloorObservation = `ph.observation_was_idle
+    AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
+  const rebaseProbe =
+    'ph.rate_probe_started_at_ms < ph.last_ramp_at_ms OR ph.rate_probe_started_at_ms < COALESCE(ph.rate_observation_started_at_ms, 0) OR ph.rate_probe_successes_at_ramp > ph.rate_successes';
   return `updated_scope AS (
     UPDATE ${table} h SET
       tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
@@ -112,13 +162,34 @@ export function updateRateScopeSql(
           AND ph.rate_observation_started_at_ms IS NULL THEN 0
         WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
         ELSE ph.rate_successes END,
-      -- Do not turn an earlier productive measurement window into a failed
-      -- probe streak when an unknown provider starts a new run after idle.
-      -- Active Retry-After/local cooldowns are excluded by observation_was_idle.
+      ${
+        unknownProvider && probeBaselineSupported
+          ? `rate_probe_successes_at_ramp = CASE
+        WHEN ${recoveredObservation} THEN 0
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN 0
+          WHEN ${idleFloorObservation} THEN ph.rate_successes
+          WHEN ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN ph.rate_successes
+          ELSE ph.rate_probe_successes_at_ramp END,
+      rate_probe_started_at_ms = CASE
+        WHEN ${idleFloorObservation} OR ${recoveredObservation} OR ${floorRampedObservation} OR ${probedObservation} OR (${rebaseProbe}) THEN nv.now_ms
+        WHEN (SELECT granted FROM grant_calc) > 0
+          AND (ph.rate_observation_started_at_ms IS NULL OR ph.observation_was_idle) THEN nv.now_ms
+        ELSE ph.rate_probe_started_at_ms END,`
+          : ''
+      }
+      -- At the floor, a new successful epoch must not inherit failed-probe
+      -- backoff. Above the floor, retain its useful measurement duration for
+      -- sparse upward probes; that rate already proves successful recovery.
       rate_recovery_interval_ms = CASE
         WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
-          AND ph.observation_was_idle AND ph.rate_successes > 0
-          THEN ${AIMD_RECOVERY_STEP_MS}
+          AND ph.rate_successes > 0
+          AND ph.next_refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS}
+          AND (ph.observation_was_idle
+            OR ph.rate_observation_started_at_ms IS NULL)
+          -- Zero is an unmeasured new episode, not a failed-probe interval.
+          -- A real failure establishes the interval before it can double.
+          THEN ${probeBaselineSupported ? 0 : AIMD_RECOVERY_STEP_MS}
         ELSE ph.rate_recovery_interval_ms END,
       rate_throttle_ceiling_rps = CASE
         WHEN (SELECT granted FROM grant_calc) > 0

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -288,6 +288,36 @@ function isMissingRateThrottleCeilingColumnError(error: unknown): boolean {
   );
 }
 
+/** Run the existing rate operation until the deploy-owned probe expansion lands. */
+function isMissingProbeBaselineColumnError(error: unknown): boolean {
+  const candidate = error as { code?: unknown; message?: unknown } | null;
+  return (
+    candidate?.code === '42703' &&
+    typeof candidate.message === 'string' &&
+    /\b(rate_probe_successes_at_ramp|rate_probe_started_at_ms)\b/.test(
+      candidate.message,
+    )
+  );
+}
+
+async function queryRateStateWithProbeBaseline<
+  Row extends Record<string, unknown>,
+>(
+  client: PostgresSchedulerQueryClient,
+  sql: (probeBaselineSupported: boolean) => string,
+  parameters: unknown[],
+) {
+  try {
+    return await client.query<Row>(sql(true), parameters);
+  } catch (error) {
+    if (!isMissingProbeBaselineColumnError(error)) throw error;
+    // PostgreSQL rejects the statement before any grant or reservation writes.
+    // Retry only that statement; previously accepted success feedback is not
+    // applied twice. No DDL, migration preflight, or global schema floor.
+    return await client.query<Row>(sql(false), parameters);
+  }
+}
+
 /**
  * v43 added `rate_requesters` and the lease `requester_id` column. Only an
  * acquire that carries a requester identity touches them, so during the
@@ -565,6 +595,8 @@ export async function ensurePostgresRateStateSchema(
       ADD COLUMN IF NOT EXISTS last_ramp_at_ms bigint NOT NULL DEFAULT 0,
       ADD COLUMN IF NOT EXISTS rate_observation_started_at_ms bigint,
       ADD COLUMN IF NOT EXISTS rate_successes bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS rate_probe_successes_at_ramp bigint NOT NULL DEFAULT 0,
+      ADD COLUMN IF NOT EXISTS rate_probe_started_at_ms bigint NOT NULL DEFAULT 0,
       ADD COLUMN IF NOT EXISTS rate_recovery_interval_ms bigint NOT NULL DEFAULT 1000,
       ADD COLUMN IF NOT EXISTS rate_throttle_ceiling_rps float8
   `);
@@ -916,6 +948,8 @@ type InternalAcquireOutcome = PlayRunnerRateStateAcquireResult & {
   observedRuleRows: number;
   /** True when a seed upsert would still change a stored row (config drift). */
   configDrift: boolean;
+  /** Local monotonic receipt time; never crosses the runtime boundary. */
+  serverClockReceivedAtMs?: number;
 };
 
 function acquireResultOf(
@@ -924,9 +958,17 @@ function acquireResultOf(
   const result: PlayRunnerRateStateAcquireResult & {
     observedRuleRows?: number;
     configDrift?: boolean;
+    serverClockReceivedAtMs?: number;
   } = { ...outcome };
   delete result.observedRuleRows;
   delete result.configDrift;
+  delete result.serverClockReceivedAtMs;
+  if (outcome.serverClockReceivedAtMs != null) {
+    result.serverClockElapsedMs = Math.max(
+      0,
+      Math.floor(performance.now() - outcome.serverClockReceivedAtMs),
+    );
+  }
   return result;
 }
 
@@ -1307,7 +1349,7 @@ async function acquirePureRate(
         concurrency: null,
       })
     : null;
-  const result = await client.query<{
+  const result = await queryRateStateWithProbeBaseline<{
     granted: string | number;
     wait_ms: string | number;
     observed_rule_rows: string | number;
@@ -1319,7 +1361,8 @@ async function acquirePureRate(
     reservation_ids: string[] | null;
     effective_rps: string | number;
   }>(
-    `
+    client,
+    (probeBaselineSupported) => `
       WITH now_value AS (
         SELECT COALESCE(
           $3::bigint,
@@ -1346,6 +1389,7 @@ async function acquirePureRate(
         '$5',
         minimumRateSqlForRules(rules),
         unknownProvider,
+        probeBaselineSupported,
       )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
@@ -1500,7 +1544,7 @@ async function acquirePureRate(
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
-      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
+      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider, probeBaselineSupported)},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -1669,6 +1713,8 @@ async function acquirePureRate(
       ...(requesterId ? [requesterId] : []),
     ],
   );
+  const serverClockReceivedAtMs =
+    input.nowMs == null ? performance.now() : undefined;
   const row = result.rows[0];
   if (!row) {
     // Seeding raced away or rows vanished; the caller re-seeds and retries.
@@ -1709,6 +1755,7 @@ async function acquirePureRate(
     granted,
     waitMs,
     serverNowMs,
+    serverClockReceivedAtMs,
     observedRuleRows,
     configDrift,
     leaseIds: [],
@@ -1741,29 +1788,39 @@ async function acquireWithConcurrency(
   seeds: RateBucketSeed[],
   requested: number,
 ): Promise<InternalAcquireOutcome> {
-  await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-  try {
-    await client.query(
-      `SELECT bucket_id FROM ${scopeHoldTableName(input)}
-       WHERE bucket_id = $1 FOR UPDATE`,
-      [input.bucketId],
-    );
-    const outcome = await grantWithConcurrency(
-      client,
-      input,
-      rules,
-      seeds,
-      requested,
-    );
-    await client.query('COMMIT');
-    return outcome;
-  } catch (error) {
+  let probeBaselineSupported = true;
+  while (true) {
+    await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
     try {
-      await client.query('ROLLBACK');
-    } catch {
-      // Preserve the original admission/storage error.
+      await client.query(
+        `SELECT bucket_id FROM ${scopeHoldTableName(input)}
+       WHERE bucket_id = $1 FOR UPDATE`,
+        [input.bucketId],
+      );
+      const outcome = await grantWithConcurrency(
+        client,
+        input,
+        rules,
+        seeds,
+        requested,
+        probeBaselineSupported,
+      );
+      await client.query('COMMIT');
+      return outcome;
+    } catch (error) {
+      try {
+        await client.query('ROLLBACK');
+      } catch {
+        // Preserve the original admission/storage error.
+      }
+      if (probeBaselineSupported && isMissingProbeBaselineColumnError(error)) {
+        // The failed grant aborts its transaction. Reacquire the same scope lock
+        // with a fresh snapshot; only the grant is repeated, never its feedback.
+        probeBaselineSupported = false;
+        continue;
+      }
+      throw error;
     }
-    throw error;
   }
 }
 
@@ -1773,6 +1830,7 @@ async function grantWithConcurrency(
   rules: AcquirePacingRule[],
   seeds: RateBucketSeed[],
   requested: number,
+  probeBaselineSupported: boolean,
 ): Promise<InternalAcquireOutcome> {
   const ruleIds = rules.map((rule) => rule.ruleId);
   const unknownProvider = rules.some(
@@ -1830,6 +1888,7 @@ async function grantWithConcurrency(
         '$10',
         minimumRateSqlForRules(rules),
         unknownProvider,
+        probeBaselineSupported,
       )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
@@ -2068,7 +2127,7 @@ async function grantWithConcurrency(
           GREATEST(1, FLOOR(MIN(next_refill_per_sec) / 100.0))::bigint AS burst_size
         FROM refreshed
       ),
-      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider)},
+      ${updateRateScopeSql(scopeHoldTableName(input), unknownProvider, probeBaselineSupported)},
       updated AS (
         UPDATE ${bucketTableName(input)} b
         SET
@@ -2283,6 +2342,8 @@ async function grantWithConcurrency(
       ...(requesterId ? [requesterId] : []),
     ],
   );
+  const serverClockReceivedAtMs =
+    input.nowMs == null ? performance.now() : undefined;
   const row = result.rows[0];
   if (!row) {
     return {
@@ -2327,6 +2388,7 @@ async function grantWithConcurrency(
     granted,
     waitMs,
     serverNowMs,
+    serverClockReceivedAtMs,
     observedRuleRows,
     configDrift,
     leaseIds: granted > 0 ? leaseIds : [],
@@ -2410,9 +2472,15 @@ export async function validatePostgresRateStatePermit(
  */
 export async function claimPostgresProviderDispatch(
   client: PostgresSchedulerQueryClient,
-  input: PlayRunnerRateStateValidateInput & { nowMs?: number },
+  input: PlayRunnerRateStateValidateInput & {
+    nowMs?: number;
+    retryAtMsAtGrant?: number;
+  },
 ): Promise<
-  PlayRunnerRateStateValidateResult & { physicalCadenceWait: boolean }
+  PlayRunnerRateStateValidateResult & {
+    physicalCadenceWait: boolean;
+    stalePermit?: boolean;
+  }
 > {
   return withRateStateAvailabilityGuard(input, async () => {
     const result = await client.query<{
@@ -2420,6 +2488,7 @@ export async function claimPostgresProviderDispatch(
       retry_at_ms: string | number;
       allowed: boolean;
       cadence_wait: boolean;
+      stale_permit: boolean;
     }>(
       `
         WITH now_value AS (
@@ -2438,6 +2507,7 @@ export async function claimPostgresProviderDispatch(
             GREATEST(COALESCE(s.retry_at_ms, 0), COALESCE(s.cool_until_ms, 0)) AS hold_until_ms,
             s.physical_rps,
             COALESCE(s.seen_backpressure, false) AS preserve_cadence_phase,
+            COALESCE(s.retry_at_ms > $3::bigint, false) AS stale_permit,
             GREATEST(1, FLOOR(s.physical_rps / 100))::int AS burst_size,
             GREATEST(1, CEIL(1000.0 / s.physical_rps))::bigint AS spacing_ms
           FROM now_value nv LEFT JOIN scope_rows s ON true
@@ -2453,6 +2523,7 @@ export async function claimPostgresProviderDispatch(
             c.now_ms + c.spacing_ms * c.burst_size, 0
           FROM cadence c
           WHERE c.hold_until_ms <= c.now_ms AND c.physical_rps > 0
+            AND NOT c.stale_permit
           ON CONFLICT (bucket_id, rule_id) DO UPDATE SET
             -- Always return the locked current cursor, including a denial.
             -- A snapshot read after a blocked upsert could miss a concurrent
@@ -2466,9 +2537,16 @@ export async function claimPostgresProviderDispatch(
             cool_until_ms = CASE
               WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
                 OR LEAST(existing.tokens, EXCLUDED.capacity) >= 1 THEN 0
-              ELSE LEAST(existing.next_dispatch_at_ms, EXCLUDED.next_dispatch_at_ms) END,
-            -- Persist debt decay at the new pace so repeated probes converge
-            -- to this deadline instead of moving it forward on every denial.
+              ELSE CASE WHEN EXCLUDED.refill_per_sec > existing.refill_per_sec
+                THEN LEAST(existing.next_dispatch_at_ms,
+                  GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
+                    + (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms))
+                ELSE existing.next_dispatch_at_ms END END,
+            -- Only a faster rate shortens existing debt. A contender can
+            -- sample the clock before waiting behind a newer cursor grant;
+            -- its denial must not rewind that grant at the same pace. Anchor
+            -- recovery at the latest cursor clock, then persist its deadline
+            -- so repeated probes converge without consuming another slot.
             next_dispatch_at_ms = CASE
               WHEN existing.next_dispatch_at_ms <= EXCLUDED.refilled_at_ms
                 -- Keep the cadence phase when a request reaches its slot
@@ -2487,23 +2565,30 @@ export async function claimPostgresProviderDispatch(
                   -- Cold discovery retains strict spacing until provider
                   -- backpressure establishes an adaptive cadence.
                   ELSE EXCLUDED.next_dispatch_at_ms END
-              ELSE LEAST(existing.next_dispatch_at_ms, EXCLUDED.next_dispatch_at_ms) END,
+              ELSE CASE WHEN EXCLUDED.refill_per_sec > existing.refill_per_sec
+                THEN LEAST(existing.next_dispatch_at_ms,
+                  GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
+                    + (EXCLUDED.next_dispatch_at_ms - EXCLUDED.refilled_at_ms))
+                ELSE existing.next_dispatch_at_ms END END,
             capacity = EXCLUDED.capacity,
             refill_per_sec = EXCLUDED.refill_per_sec,
             base_refill_per_sec = EXCLUDED.base_refill_per_sec,
             max_rps = EXCLUDED.max_rps,
-            refilled_at_ms = EXCLUDED.refilled_at_ms
+            refilled_at_ms = GREATEST(existing.refilled_at_ms, EXCLUDED.refilled_at_ms)
           RETURNING cool_until_ms
         )
         SELECT c.now_ms AS server_now_ms,
-          GREATEST(c.hold_until_ms, COALESCE(cursor.cool_until_ms, 0)) AS retry_at_ms,
-          c.hold_until_ms <= c.now_ms AND (
+          GREATEST(c.hold_until_ms, COALESCE(cursor.cool_until_ms, 0),
+            CASE WHEN c.stale_permit THEN c.now_ms ELSE 0 END) AS retry_at_ms,
+          NOT c.stale_permit AND c.hold_until_ms <= c.now_ms AND (
             c.physical_rps IS NULL OR cursor.cool_until_ms = 0
           ) AS allowed,
-          c.hold_until_ms <= c.now_ms AND COALESCE(cursor.cool_until_ms > c.now_ms, false) AS cadence_wait
+          NOT c.stale_permit AND c.hold_until_ms <= c.now_ms
+            AND COALESCE(cursor.cool_until_ms > c.now_ms, false) AS cadence_wait,
+          c.stale_permit
         FROM cadence c LEFT JOIN cursor ON true
       `,
-      [input.bucketId, input.nowMs ?? null],
+      [input.bucketId, input.nowMs ?? null, input.retryAtMsAtGrant ?? null],
     );
     const row = result.rows[0];
     if (!row || typeof row.allowed !== 'boolean') {
@@ -2516,6 +2601,9 @@ export async function claimPostgresProviderDispatch(
       physicalCadenceWait: row.cadence_wait,
       serverNowMs: Number(row.server_now_ms),
       retryAtMs: Number(row.retry_at_ms),
+      ...(input.retryAtMsAtGrant !== undefined
+        ? { stalePermit: row.stale_permit }
+        : {}),
     };
   });
 }
@@ -3362,7 +3450,13 @@ async function penalizeRateStateStatements(
             -- no successes justifies the shared exponential pause.
             WHEN h.rate_successes <= 0
               AND ($5::bigint IS NOT NULL OR h.seen_backpressure)
-              AND (NOT h.unknown_provider OR h.seen_backpressure) THEN
+              -- An unknown scope can only recover above its floor after
+              -- successful work. A cleared counter at that new epoch does
+              -- not turn its measurement duration into consecutive failures.
+              AND (NOT h.unknown_provider
+                OR h.refill_per_sec <= ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS})
+              AND (NOT h.unknown_provider OR (h.seen_backpressure
+                AND h.rate_recovery_interval_ms > 0)) THEN
               CASE WHEN h.seen_backpressure THEN
                 LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
               ELSE 1000 END
@@ -3392,6 +3486,7 @@ async function penalizeRateStateStatements(
         SELECT e.*,
           CASE WHEN e.new_episode THEN GREATEST(
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0
+              AND e.no_progress_probe_interval_ms IS NOT NULL
               THEN ${noProgressProbeRateRps}
               ELSE CASE WHEN e.unknown_provider
                 THEN ${DEFAULT_UNKNOWN_PROVIDER_MIN_RPS} ELSE ${minimumRateRps} END END,