deepline 0.3.201 → 0.3.203

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -307,7 +307,7 @@ type BreakerState = {
   consecutiveFailures: number;
   openedAt: number | null;
   lastFailureMessage: string;
-  halfOpenInFlight: boolean;
+  halfOpenProbe: symbol | null;
 };
 
 export type RuntimeSchedulerPoolPriority = 'ordinary' | 'critical';
@@ -645,6 +645,9 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
       maxAcquireQueueDepth,
       maxAcquireQueueWaitMs,
     );
+    // A pool created by the old breaker used a boolean claim. The new token
+    // does not inherit a stranded half-open claim across a hot reload.
+    existing.breaker.halfOpenProbe ??= null;
     return existing;
   }
 
@@ -671,7 +674,7 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
       consecutiveFailures: 0,
       openedAt: null,
       lastFailureMessage: '',
-      halfOpenInFlight: false,
+      halfOpenProbe: null,
     },
     stats: {
       acquires: 0,
@@ -698,31 +701,39 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
   return entry;
 }
 
-function breakerGate(entry: PoolEntry): void {
+function breakerGate(entry: PoolEntry): symbol | null {
   const { breaker } = entry;
-  if (breaker.openedAt === null) return;
+  if (breaker.openedAt === null) return null;
   const elapsed = Date.now() - breaker.openedAt;
   if (elapsed < RUNTIME_SCHEDULER_POOL_DEFAULTS.breaker.openMs) {
     entry.stats.breakerRejections += 1;
     throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
   }
   // Half-open: let exactly one caller through as the trial.
-  if (breaker.halfOpenInFlight) {
+  if (breaker.halfOpenProbe !== null) {
     entry.stats.breakerRejections += 1;
     throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
   }
-  breaker.halfOpenInFlight = true;
+  const probe = Symbol('runtime-scheduler-half-open-probe');
+  breaker.halfOpenProbe = probe;
+  return probe;
 }
 
 function recordBreakerSuccess(entry: PoolEntry): void {
   entry.breaker.consecutiveFailures = 0;
   entry.breaker.openedAt = null;
-  entry.breaker.halfOpenInFlight = false;
+  entry.breaker.halfOpenProbe = null;
 }
 
-function recordBreakerFailure(entry: PoolEntry, error: unknown): void {
+function recordBreakerFailure(
+  entry: PoolEntry,
+  error: unknown,
+  probe: symbol | null,
+): void {
   const { breaker } = entry;
-  breaker.halfOpenInFlight = false;
+  if (probe !== null && breaker.halfOpenProbe === probe) {
+    breaker.halfOpenProbe = null;
+  }
   breaker.consecutiveFailures += 1;
   breaker.lastFailureMessage =
     error instanceof Error ? error.message : String(error);
@@ -798,190 +809,46 @@ export async function withRuntimeSchedulerClient<T>(
     throwIfSchedulerAborted(signal);
     for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
       throwIfSchedulerAborted(signal);
-      breakerGate(entry);
-      if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
-        entry.stats.saturationRejections += 1;
-        throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
-      }
-      const acquireStart = performance.now();
-      entry.stats.peakTotalClients = Math.max(
-        entry.stats.peakTotalClients,
-        entry.pool.totalCount,
-      );
-      entry.stats.peakWaitingClients = Math.max(
-        entry.stats.peakWaitingClients,
-        entry.pool.waitingCount,
-      );
-      let poolClient;
+      const halfOpenProbe = breakerGate(entry);
+      let capacityObserved = false;
       try {
-        poolClient = await entry.pool.connect();
-      } catch (error) {
-        if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
+        if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
           entry.stats.saturationRejections += 1;
-          console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
-            maxClients: entry.maxClients,
-            totalClients: entry.pool.totalCount,
-            idleClients: entry.pool.idleCount,
-            waitingClients: entry.pool.waitingCount,
-          });
           throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
         }
-        if (isPgBouncerCapacityError(error)) {
-          recordCapacityRejection(entry, error);
-          throw new RuntimeSchedulerPgBouncerCapacityError(
-            error instanceof Error ? error.message : String(error),
-          );
-        }
-        recordBreakerFailure(entry, error);
-        entry.stats.connectionErrors += 1;
-        lastError = error;
-        if (attempt + 1 < maxAttempts) {
-          entry.stats.retries += 1;
-          await sleep(jitteredBackoffMs(attempt));
-          throwIfSchedulerAborted(signal);
-          continue;
-        }
-        throw error;
-      }
-      entry.stats.acquires += 1;
-      const acquireWaitMs = performance.now() - acquireStart;
-      entry.stats.peakTotalClients = Math.max(
-        entry.stats.peakTotalClients,
-        entry.pool.totalCount,
-      );
-      pushSample(entry.stats.acquireWait, acquireWaitMs);
-
-      // A checked-out pg client emits 'error' (not via the pool) when its TCP
-      // connection dies while idle — e.g. a worker holding a client across a
-      // long play execution. With no listener that is an uncaught exception
-      // that kills the process (observed: worker crash mid-150k-row run).
-      // Swallow it here; the next query on the dead client fails normally and
-      // flows through the connection-error retry/poison path below.
-      let checkedOutError: unknown = null;
-      const onCheckedOutError = (error: Error) => {
-        checkedOutError = error;
-        entry.stats.connectionErrors += 1;
-        console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
-          error: error.message,
-        });
-      };
-      poolClient.on('error', onCheckedOutError);
-      let clientReleased = false;
-      const releasePoolClient = (error?: Error) => {
-        if (clientReleased) return;
-        clientReleased = true;
-        poolClient.release(error);
-      };
-      const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
-      signal?.addEventListener('abort', onAbort, { once: true });
-
-      let queryCount = 0;
-      let queryMs = 0;
-      try {
-        throwIfSchedulerAborted(signal);
-        const result = await fn({
-          query: async (sql, params) => {
-            throwIfSchedulerAborted(signal);
-            const queryStartedAt = performance.now();
-            queryCount += 1;
-            try {
-              try {
-                const queryResult = await poolClient.query(sql, params);
-                throwIfSchedulerAborted(signal);
-                return { rows: queryResult.rows };
-              } catch (error) {
-                // node-postgres' query timeout and an AbortSignal reject the
-                // promise, but the server may still be processing the SQL.
-                // Never return that socket to the pool: doing so lets the
-                // next retry overlap the first statement on the same backend.
-                if (isRuntimeSchedulerQueryCancellationError(error)) {
-                  releasePoolClient(
-                    error instanceof Error ? error : new Error(String(error)),
-                  );
-                }
-                throw error;
-              }
-            } finally {
-              queryMs += performance.now() - queryStartedAt;
-            }
-          },
-        });
-        throwIfSchedulerAborted(signal);
-        recordBreakerSuccess(entry);
-        signal?.removeEventListener('abort', onAbort);
-        poolClient.removeListener('error', onCheckedOutError);
-        // The connection died mid-fn but fn happened to finish without another
-        // query — destroy the dead connection rather than pooling it.
-        releasePoolClient(
-          checkedOutError instanceof Error ? checkedOutError : undefined,
+        const acquireStart = performance.now();
+        entry.stats.peakTotalClients = Math.max(
+          entry.stats.peakTotalClients,
+          entry.pool.totalCount,
         );
-        if (trace) {
-          console.info(
-            '[runtime-scheduler-pool.operation] ' +
-              JSON.stringify({
-                runId: trace.runId,
-                phase: trace.phase,
-                outcome: 'success',
-                attempt: attempt + 1,
-                priority: trace.priority ?? 'ordinary',
-                admissionWaitMs,
-                acquireWaitMs,
-                queryCount,
-                queryMs,
-                totalMs: performance.now() - operationStartedAt,
-              }),
-          );
-        }
-        return result;
-      } catch (error) {
-        if (trace) {
-          console.info(
-            '[runtime-scheduler-pool.operation] ' +
-              JSON.stringify({
-                runId: trace.runId,
-                phase: trace.phase,
-                outcome: 'failed',
-                attempt: attempt + 1,
-                priority: trace.priority ?? 'ordinary',
-                admissionWaitMs,
-                acquireWaitMs,
-                queryCount,
-                queryMs,
-                totalMs: performance.now() - operationStartedAt,
-                errorName: error instanceof Error ? error.name : 'UnknownError',
-              }),
-          );
-        }
-        signal?.removeEventListener('abort', onAbort);
-        poolClient.removeListener('error', onCheckedOutError);
-        if (signal?.aborted) {
-          const cancellation = schedulerAbortReason(signal);
-          releasePoolClient(cancellation);
-          throw cancellation;
-        }
-        if (isPgBouncerCapacityError(error)) {
-          recordCapacityRejection(entry, error);
-          releasePoolClient();
-          throw new RuntimeSchedulerPgBouncerCapacityError(
-            error instanceof Error ? error.message : String(error),
-          );
-        }
-        if (isRuntimeSchedulerQueryCancellationError(error)) {
-          entry.stats.queryErrors += 1;
-          recordBreakerSuccess(entry);
-          releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
-          );
-          throw error;
-        }
-        if (isConnectionCategoryError(error)) {
-          recordBreakerFailure(entry, error);
+        entry.stats.peakWaitingClients = Math.max(
+          entry.stats.peakWaitingClients,
+          entry.pool.waitingCount,
+        );
+        let poolClient;
+        try {
+          poolClient = await entry.pool.connect();
+        } catch (error) {
+          if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
+            entry.stats.saturationRejections += 1;
+            console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
+              maxClients: entry.maxClients,
+              totalClients: entry.pool.totalCount,
+              idleClients: entry.pool.idleCount,
+              waitingClients: entry.pool.waitingCount,
+            });
+            throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
+          }
+          if (isPgBouncerCapacityError(error)) {
+            recordCapacityRejection(entry, error);
+            capacityObserved = true;
+            throw new RuntimeSchedulerPgBouncerCapacityError(
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+          recordBreakerFailure(entry, error, halfOpenProbe);
           entry.stats.connectionErrors += 1;
           lastError = error;
-          // Destroy the broken connection instead of returning it to the pool.
-          releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
-          );
           if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
@@ -990,30 +857,197 @@ export async function withRuntimeSchedulerClient<T>(
           }
           throw error;
         }
-        if (isRuntimeSchedulerRetryableTransactionError(error)) {
-          // PostgreSQL has aborted the transaction, so never return this
-          // client to the pool. Re-run the idempotent scheduler operation on a
-          // new session without opening the connection-failure breaker.
-          entry.stats.queryErrors += 1;
+        entry.stats.acquires += 1;
+        const acquireWaitMs = performance.now() - acquireStart;
+        entry.stats.peakTotalClients = Math.max(
+          entry.stats.peakTotalClients,
+          entry.pool.totalCount,
+        );
+        pushSample(entry.stats.acquireWait, acquireWaitMs);
+
+        // A checked-out pg client emits 'error' (not via the pool) when its TCP
+        // connection dies while idle — e.g. a worker holding a client across a
+        // long play execution. With no listener that is an uncaught exception
+        // that kills the process (observed: worker crash mid-150k-row run).
+        // Swallow it here; the next query on the dead client fails normally and
+        // flows through the connection-error retry/poison path below.
+        let checkedOutError: unknown = null;
+        const onCheckedOutError = (error: Error) => {
+          checkedOutError = error;
+          entry.stats.connectionErrors += 1;
+          console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
+            error: error.message,
+          });
+        };
+        poolClient.on('error', onCheckedOutError);
+        let clientReleased = false;
+        const releasePoolClient = (error?: Error) => {
+          if (clientReleased) return;
+          clientReleased = true;
+          poolClient.release(error);
+        };
+        const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
+        signal?.addEventListener('abort', onAbort, { once: true });
+
+        let queryCount = 0;
+        let queryMs = 0;
+        try {
+          throwIfSchedulerAborted(signal);
+          const result = await fn({
+            query: async (sql, params) => {
+              throwIfSchedulerAborted(signal);
+              const queryStartedAt = performance.now();
+              queryCount += 1;
+              try {
+                try {
+                  const queryResult = await poolClient.query(sql, params);
+                  throwIfSchedulerAborted(signal);
+                  return { rows: queryResult.rows };
+                } catch (error) {
+                  // node-postgres' query timeout and an AbortSignal reject the
+                  // promise, but the server may still be processing the SQL.
+                  // Never return that socket to the pool: doing so lets the
+                  // next retry overlap the first statement on the same backend.
+                  if (isRuntimeSchedulerQueryCancellationError(error)) {
+                    releasePoolClient(
+                      error instanceof Error ? error : new Error(String(error)),
+                    );
+                  }
+                  throw error;
+                }
+              } finally {
+                queryMs += performance.now() - queryStartedAt;
+              }
+            },
+          });
+          throwIfSchedulerAborted(signal);
           recordBreakerSuccess(entry);
-          lastError = error;
+          signal?.removeEventListener('abort', onAbort);
+          poolClient.removeListener('error', onCheckedOutError);
+          // The connection died mid-fn but fn happened to finish without another
+          // query — destroy the dead connection rather than pooling it.
           releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
+            checkedOutError instanceof Error ? checkedOutError : undefined,
           );
-          if (attempt + 1 < maxAttempts) {
-            entry.stats.retries += 1;
-            await sleep(jitteredBackoffMs(attempt));
-            throwIfSchedulerAborted(signal);
-            continue;
+          if (trace) {
+            console.info(
+              '[runtime-scheduler-pool.operation] ' +
+                JSON.stringify({
+                  runId: trace.runId,
+                  phase: trace.phase,
+                  outcome: 'success',
+                  attempt: attempt + 1,
+                  priority: trace.priority ?? 'ordinary',
+                  admissionWaitMs,
+                  acquireWaitMs,
+                  queryCount,
+                  queryMs,
+                  totalMs: performance.now() - operationStartedAt,
+                }),
+            );
           }
+          return result;
+        } catch (error) {
+          if (trace) {
+            console.info(
+              '[runtime-scheduler-pool.operation] ' +
+                JSON.stringify({
+                  runId: trace.runId,
+                  phase: trace.phase,
+                  outcome: 'failed',
+                  attempt: attempt + 1,
+                  priority: trace.priority ?? 'ordinary',
+                  admissionWaitMs,
+                  acquireWaitMs,
+                  queryCount,
+                  queryMs,
+                  totalMs: performance.now() - operationStartedAt,
+                  errorName:
+                    error instanceof Error ? error.name : 'UnknownError',
+                }),
+            );
+          }
+          signal?.removeEventListener('abort', onAbort);
+          poolClient.removeListener('error', onCheckedOutError);
+          if (signal?.aborted) {
+            const cancellation = schedulerAbortReason(signal);
+            releasePoolClient(cancellation);
+            throw cancellation;
+          }
+          if (isPgBouncerCapacityError(error)) {
+            recordCapacityRejection(entry, error);
+            capacityObserved = true;
+            releasePoolClient();
+            throw new RuntimeSchedulerPgBouncerCapacityError(
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+          if (isRuntimeSchedulerQueryCancellationError(error)) {
+            entry.stats.queryErrors += 1;
+            recordBreakerSuccess(entry);
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            throw error;
+          }
+          if (isConnectionCategoryError(error)) {
+            recordBreakerFailure(entry, error, halfOpenProbe);
+            entry.stats.connectionErrors += 1;
+            lastError = error;
+            // Destroy the broken connection instead of returning it to the pool.
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            if (attempt + 1 < maxAttempts) {
+              entry.stats.retries += 1;
+              await sleep(jitteredBackoffMs(attempt));
+              throwIfSchedulerAborted(signal);
+              continue;
+            }
+            throw error;
+          }
+          if (isRuntimeSchedulerRetryableTransactionError(error)) {
+            // PostgreSQL has aborted the transaction, so never return this
+            // client to the pool. Re-run the idempotent scheduler operation on a
+            // new session without opening the connection-failure breaker.
+            entry.stats.queryErrors += 1;
+            recordBreakerSuccess(entry);
+            lastError = error;
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            if (attempt + 1 < maxAttempts) {
+              entry.stats.retries += 1;
+              await sleep(jitteredBackoffMs(attempt));
+              throwIfSchedulerAborted(signal);
+              continue;
+            }
+            throw error;
+          }
+          // Query-level failure on a healthy connection: count it, return the
+          // connection to the pool, surface the error without retry.
+          entry.stats.queryErrors += 1;
+          recordBreakerSuccess(entry);
+          releasePoolClient();
           throw error;
         }
-        // Query-level failure on a healthy connection: count it, return the
-        // connection to the pool, surface the error without retry.
-        entry.stats.queryErrors += 1;
-        recordBreakerSuccess(entry);
-        releasePoolClient();
-        throw error;
+      } finally {
+        // A canceled, locally saturated, or capacity-rejected trial did not
+        // call either breaker recorder. Never let it own the probe forever.
+        // Only the trial that claimed this token may release it: another
+        // operation can close and reopen the breaker while this one awaits.
+        if (
+          halfOpenProbe !== null &&
+          entry.breaker.halfOpenProbe === halfOpenProbe
+        ) {
+          entry.breaker.halfOpenProbe = null;
+          if (capacityObserved && entry.breaker.openedAt !== null) {
+            // Capacity is not a connection failure. Keep the existing open
+            // state and try once more after the ordinary cooldown instead of
+            // probing the still-full database on every request.
+            entry.breaker.openedAt = Date.now();
+          }
+        }
       }
     }
     throw lastError instanceof Error

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -3080,6 +3080,13 @@ async function penalizeRateStateStatements(
                 GREATEST(h.retry_at_ms, COALESCE($5::bigint, nv.now_ms + $2::bigint))
                   - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
             END) AS cycle_ms,
+          CASE
+            WHEN $5::bigint IS NOT NULL AND h.rate_successes <= 0 THEN
+              CASE WHEN h.seen_backpressure THEN
+                LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
+              ELSE 1000 END
+            ELSE NULL
+          END AS no_progress_probe_interval_ms,
           nv.now_ms
         FROM held h CROSS JOIN now_value nv
       ), throttle_ceiling AS (
@@ -3087,12 +3094,12 @@ async function penalizeRateStateStatements(
           GREATEST(${MIN_ADAPTIVE_RPS_FLOOR},
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
               CASE WHEN $5::bigint IS NOT NULL
-                -- An explicit Retry-After is only the provider's next safe
-                -- dispatch deadline. It is not a rate measurement; with no
-                -- successful-call evidence, use one conservative probe per
-                -- second after that deadline instead of deriving a permanent
-                -- RPM limit from the length of the hold.
-                THEN GREATEST(e.refill_before, 1000.0 / e.cycle_ms)
+                -- A fresh provider deadline is not a throughput measurement.
+                -- Repeated probes with no successful work are evidence that
+                -- the scope is still unavailable, however. Space subsequent
+                -- probes exponentially (up to one per minute) across every
+                -- run sharing this provider credential.
+                THEN 1000.0 / e.no_progress_probe_interval_ms
                 ELSE e.refill_before END
               ELSE e.refill_before / 2 END
           ) AS next_ceiling_rps
@@ -3115,7 +3122,9 @@ async function penalizeRateStateStatements(
           rate_throttle_ceiling_rps = CASE WHEN e.new_episode
             THEN e.next_ceiling_rps ELSE e.rate_throttle_ceiling_rps END,
           rate_observation_started_at_ms = CASE WHEN e.new_episode THEN NULL ELSE e.rate_observation_started_at_ms END,
-          rate_recovery_interval_ms = CASE WHEN e.new_episode THEN e.cycle_ms ELSE e.rate_recovery_interval_ms END,
+          rate_recovery_interval_ms = CASE WHEN e.new_episode
+            THEN COALESCE(e.no_progress_probe_interval_ms, e.cycle_ms)
+            ELSE e.rate_recovery_interval_ms END,
           last_ramp_at_ms = CASE WHEN e.new_episode THEN e.deadline ELSE e.last_ramp_at_ms END,
           -- All pre-429 permits are revalidated against this durable hold
           -- before dispatch. Their obsolete schedule debt must not add a