deepline 0.3.323 → 0.3.324

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -4,21 +4,52 @@
  * learn from copies of the same provider feedback. These CTEs are shared by
  * pure-rate and concurrency-bearing admission, in the same atomic grant SQL.
  */
+import { DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS } from '../governor/rate-state-backend';
+
 const AIMD_RECOVERY_STEP_MS = 1_000;
 // Increase relative to the learned rate, not by a fixed RPS amount: a long
 // Retry-After can imply a very small quota, where even +1 RPS would be an
 // enormous and repeated over-probe. A 25% clean-success step still recovers a
 // healthy 1->5 RPS provider in eight observations instead of seventeen.
 const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
-
+// Keep upward probes small enough to retain headroom after a productive
+// decrease. Sparse successes use the earlier observation window; sustained
+// useful work can establish another small probe each second.
+const UNKNOWN_PROVIDER_PROBE_STEP_RPS = 0.1;
+const UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS = 5_000;
+const UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION = 0.8;
 export function acquireRateScopeSql(
   table: string,
   successesParameter: string,
   minimumRateSql = 'NULL::float8',
+  unknownProvider = false,
 ): string {
   const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
     THEN h.refill_per_sec
     ELSE GREATEST(h.refill_per_sec, ${minimumRateSql}) END`;
+  const recoveryRate = unknownProvider
+    ? `CASE WHEN ${refillRate} < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+        THEN LEAST(${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}, ${refillRate} * 2)
+        ELSE ((${refillRate})::numeric + ${UNKNOWN_PROVIDER_PROBE_STEP_RPS})::float8 END`
+    : `${refillRate} * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})`;
+  // Success counts are already applied once by the feedback writer. Measure
+  // this post-hold epoch, excluding Retry-After and earlier idle history. A
+  // productive second justifies a small new probe; a lone completion does not.
+  const productiveRecovery = `h.rate_observation_started_at_ms IS NOT NULL
+    AND nv.now_ms - h.rate_observation_started_at_ms >= ${AIMD_RECOVERY_STEP_MS}
+    AND h.rate_successes * 1000.0
+      / GREATEST(${AIMD_RECOVERY_STEP_MS}, nv.now_ms - h.rate_observation_started_at_ms)
+      >= ${refillRate} * ${UNKNOWN_PROVIDER_PRODUCTIVE_RATE_FRACTION}`;
+  // Below useful seed capacity, a confirmed success can prove the next probe
+  // after one dispatch interval at the current rate. Keeping the floor's 500ms
+  // interval at 4/8 RPS waits through additional clean sends unnecessarily.
+  // Above that seed, preserve the slower productive/sparse observation window.
+  const backpressuredRecoveryInterval = unknownProvider
+    ? `CASE WHEN ${refillRate} >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+        THEN CASE WHEN ${productiveRecovery} THEN ${AIMD_RECOVERY_STEP_MS}
+          ELSE LEAST(${UNKNOWN_PROVIDER_PROBE_INTERVAL_CAP_MS}, GREATEST(${AIMD_RECOVERY_STEP_MS}, h.rate_recovery_interval_ms)) END
+        ELSE LEAST(h.rate_recovery_interval_ms, CEIL(1000.0 / ${refillRate})) END`
+    : `LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})`;
   return `provider_hold_rows AS (
     SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
   ), provider_hold AS (
@@ -28,22 +59,24 @@ export function acquireRateScopeSql(
         h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
       ) AS tokens_after_refill,
       ${successesParameter}::int = 0
-        AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
+        -- A local no-progress pause remains an active observation episode,
+        -- even after the provider's shorter Retry-After has elapsed.
+        AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms, h.cool_until_ms)
           > GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
-            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            THEN ${backpressuredRecoveryInterval}
             ELSE h.rate_recovery_interval_ms END
           AND h.retry_at_ms <= nv.now_ms
         THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
-          THEN ${refillRate} * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
+          THEN ${recoveryRate}
           ELSE ${refillRate} * 2 END)
         ELSE ${refillRate} END AS next_refill_per_sec,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
-            THEN LEAST(h.rate_recovery_interval_ms, ${AIMD_RECOVERY_STEP_MS})
+            THEN ${backpressuredRecoveryInterval}
             ELSE h.rate_recovery_interval_ms END
           AND h.retry_at_ms <= nv.now_ms
         THEN nv.now_ms ELSE h.last_ramp_at_ms END AS next_ramp_at_ms
@@ -51,7 +84,16 @@ export function acquireRateScopeSql(
   )`;
 }
 
-export function updateRateScopeSql(table: string): string {
+export function updateRateScopeSql(
+  table: string,
+  unknownProvider = false,
+): string {
+  // Floor recovery and productive probing measure different rates. Once the
+  // geometric ramp reaches useful capacity, start a fresh measurement instead
+  // of diluting its successful throughput with the earlier slow-start period.
+  const recoveredObservation = `${unknownProvider} AND ph.seen_backpressure
+    AND ph.refill_per_sec < ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}
+    AND ph.next_refill_per_sec >= ${DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS}`;
   return `updated_scope AS (
     UPDATE ${table} h SET
       tokens = ph.tokens_after_refill - (SELECT granted FROM grant_calc),
@@ -59,14 +101,25 @@ export function updateRateScopeSql(table: string): string {
       refill_per_sec = ph.next_refill_per_sec,
       last_ramp_at_ms = ph.next_ramp_at_ms,
       rate_observation_started_at_ms = CASE
+        WHEN ${recoveredObservation} THEN CASE
+          WHEN (SELECT granted FROM grant_calc) > 0 THEN nv.now_ms ELSE NULL END
         WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN nv.now_ms
         WHEN (SELECT granted FROM grant_calc) > 0 THEN COALESCE(ph.rate_observation_started_at_ms, nv.now_ms)
         ELSE ph.rate_observation_started_at_ms END,
       rate_successes = CASE
+        WHEN ${recoveredObservation} THEN 0
         WHEN (SELECT granted FROM grant_calc) > 0
           AND ph.rate_observation_started_at_ms IS NULL THEN 0
         WHEN (SELECT granted FROM grant_calc) > 0 AND ph.observation_was_idle THEN 0
         ELSE ph.rate_successes END,
+      -- Do not turn an earlier productive measurement window into a failed
+      -- probe streak when an unknown provider starts a new run after idle.
+      -- Active Retry-After/local cooldowns are excluded by observation_was_idle.
+      rate_recovery_interval_ms = CASE
+        WHEN ${unknownProvider} AND (SELECT granted FROM grant_calc) > 0
+          AND ph.observation_was_idle AND ph.rate_successes > 0
+          THEN ${AIMD_RECOVERY_STEP_MS}
+        ELSE ph.rate_recovery_interval_ms END,
       rate_throttle_ceiling_rps = CASE
         WHEN (SELECT granted FROM grant_calc) > 0
           AND ph.rate_observation_started_at_ms IS NULL THEN NULL