deepline 0.3.200 → 0.3.202

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts

@@ -1,4 +1,10 @@
-import { cancelPostgresSchedulerRun } from './postgres';
+import {
+  abandonUnstartedModalCancellationReservations,
+  cancelPostgresSchedulerRun,
+  enqueuePostgresSchedulerSandboxCleanupForRun,
+  hasPendingPostgresSchedulerSandboxCleanupForRun,
+  readPostgresSchedulerResult,
+} from './postgres';
 import type {
   PostgresSchedulerOptions,
   PostgresSchedulerQueryClient,
@@ -14,31 +20,100 @@ import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
 
 /** Bound deadline reconciliation just like Absurd's own cancellation sweep. */
 export const ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT = 100;
+const QUEUED_CANCELLATION_RECONCILE_LIMIT = 20;
 
 function quoteAbsurdTable(table: string): string {
   return `"${table.replaceAll('"', '""')}"`;
 }
 
-/**
- * Project a play task cancelled by Absurd into the scheduler's authoritative
- * run state. This covers both indexed `max_delay` cancellation and the manual
- * cancellation lane. Manual cancellation stamps `cancel_at` in the adapter so
- * existing Absurd v4 queues are discoverable without a schema cutover.
- *
- * The engine can cancel an overdue task before or after a worker claims its
- * handler. A cancelled task has no handler path left to write the normal
- * terminal state. This reconciler
- * closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
- * terminal and its run-ledger outbox event, and is idempotent if a user cancel
- * or a racing worker already terminalized the run. The next claim/maintenance
- * pass invokes it after Absurd performs its bounded cancellation sweep.
- */
+async function finalizeCancelledRunAfterCleanup(input: {
+  client: PostgresSchedulerQueryClient;
+  runId: string;
+  reason: string;
+  schedulerSchema: string;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  controlKey: string;
+}): Promise<'pending' | 'cancelled' | 'already_terminal'> {
+  await input.client.query('BEGIN');
+  try {
+    await input.client.query(
+      `SELECT set_config('lock_timeout', $1, true),
+              set_config('statement_timeout', $2, true)`,
+      [
+        `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
+        `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
+      ],
+    );
+    // Sandbox admission locks the run before reserving an operation; resource
+    // writers lock attempts. Hold both through the final cleanup check.
+    await input.client.query(
+      `SELECT run_id FROM ${input.schedulerSchema}.runs
+       WHERE run_id = $1 FOR UPDATE`,
+      [input.runId],
+    );
+    await input.client.query(
+      `SELECT attempt FROM ${input.schedulerSchema}.attempts
+       WHERE run_id = $1 FOR UPDATE`,
+      [input.runId],
+    );
+    await abandonUnstartedModalCancellationReservations(
+      input.client,
+      { runId: input.runId },
+      input.schedulerOptions,
+    );
+    await enqueuePostgresSchedulerSandboxCleanupForRun(
+      input.client,
+      { runId: input.runId },
+      input.schedulerOptions,
+    );
+    if (
+      await hasPendingPostgresSchedulerSandboxCleanupForRun(
+        input.client,
+        { runId: input.runId },
+        input.schedulerOptions,
+      )
+    ) {
+      await input.client.query('COMMIT');
+      return 'pending';
+    }
+    const cancelled = await cancelPostgresSchedulerRun(
+      input.client,
+      { runId: input.runId, reason: input.reason },
+      input.schedulerOptions,
+    );
+    if (
+      !cancelled &&
+      !(await readPostgresSchedulerResult(
+        input.client,
+        input.runId,
+        input.schedulerOptions,
+      ))
+    ) {
+      throw new Error(
+        `Cancellation did not terminalize scheduler run "${input.runId}".`,
+      );
+    }
+    await input.client.query(
+      `DELETE FROM ${input.schedulerSchema}.control WHERE key = $1`,
+      [input.controlKey],
+    );
+    await input.client.query('COMMIT');
+    return cancelled ? 'cancelled' : 'already_terminal';
+  } catch (error) {
+    await input.client.query('ROLLBACK').catch(() => undefined);
+    throw error;
+  }
+}
+
+/** Put legacy manual and deadline cancellations into the same durable queue
+ * as caller requests. A queued task is excluded from later scans, so one
+ * pending sandbox cannot hide other cancelled tasks behind the batch limit. */
 export async function reconcileAbsurdRuntimeCancellations(input: {
   client: PostgresSchedulerQueryClient;
   queue: string;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   limit?: number;
-}): Promise<{ reconciled: number; skipped: number }> {
+}): Promise<{ queued: number }> {
   const requestedLimit = Math.floor(
     input.limit ?? ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
   );
@@ -54,16 +129,21 @@ export async function reconcileAbsurdRuntimeCancellations(input: {
       resolveRuntimeSchedulerSchemaEnv() ||
       'runtime_scheduler',
   );
-  const cancelled = await input.client.query<{
-    run_id: string | null;
-    cancellation_kind: 'manual' | 'deadline';
-  }>(
+  const queued = await input.client.query<{ key: string }>(
     `
-      SELECT run.run_id,
-             CASE
-               WHEN task.cancellation ? 'manual' THEN 'manual'
-               ELSE 'deadline'
-             END AS cancellation_kind
+      INSERT INTO ${schedulerSchema}.control (key, value)
+      SELECT 'runtime_cancel:' || run.run_id,
+             jsonb_build_object(
+               'reason', CASE
+                 WHEN task.cancellation ? 'manual'
+                   THEN COALESCE(
+                     NULLIF(task.cancellation->>'reason', ''),
+                     'Run cancellation was requested by the caller.'
+                   )
+                 ELSE 'Run did not start within its configured scheduler queue deadline.'
+               END,
+               'requestedAt', clock_timestamp()
+             )
       FROM ${schedulerSchema}.runs AS run
       JOIN absurd.${taskTable} AS task
         ON task.task_id::text = run.absurd_task_id
@@ -78,57 +158,79 @@ export async function reconcileAbsurdRuntimeCancellations(input: {
             AND task.cancelled_at >= task.cancel_at
           )
         )
+        AND NOT EXISTS (
+          SELECT 1 FROM ${schedulerSchema}.control AS pending
+          WHERE pending.key = 'runtime_cancel:' || run.run_id
+        )
       ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
       LIMIT $2
+      ON CONFLICT (key) DO NOTHING
+      RETURNING key
     `,
     [ABSURD_PLAY_RUN_TASK, limit],
   );
+  return { queued: queued.rows.length };
+}
 
+/** Drain caller-requested cancellations independently of their release lane. */
+export async function reconcileQueuedAbsurdRuntimeCancellations(input: {
+  client: PostgresSchedulerQueryClient;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  limit?: number;
+}): Promise<{ reconciled: number; skipped: number }> {
+  const schedulerSchema = quoteAbsurdTable(
+    input.schedulerOptions?.schema?.trim() ||
+      resolveRuntimeSchedulerSchemaEnv() ||
+      'runtime_scheduler',
+  );
+  const requestedLimit = Math.floor(
+    input.limit ?? QUEUED_CANCELLATION_RECONCILE_LIMIT,
+  );
+  const limit = Number.isFinite(requestedLimit)
+    ? Math.max(1, Math.min(QUEUED_CANCELLATION_RECONCILE_LIMIT, requestedLimit))
+    : QUEUED_CANCELLATION_RECONCILE_LIMIT;
+  const queued = await input.client.query<{
+    key: string;
+    reason: string | null;
+  }>(
+    `WITH selected AS (
+       SELECT key FROM ${schedulerSchema}.control
+       WHERE starts_with(key, 'runtime_cancel:')
+       ORDER BY updated_at ASC, key ASC
+       LIMIT $1
+       FOR UPDATE SKIP LOCKED
+     )
+     UPDATE ${schedulerSchema}.control AS request
+     SET updated_at = clock_timestamp()
+     FROM selected
+     WHERE request.key = selected.key
+     RETURNING request.key, request.value->>'reason' AS reason`,
+    [limit],
+  );
   let reconciled = 0;
   let skipped = 0;
-  for (const row of cancelled.rows) {
-    const runId = row.run_id?.trim();
+  for (const row of queued.rows) {
+    const runId = row.key.slice('runtime_cancel:'.length);
     if (!runId) {
-      skipped += 1;
-      continue;
+      throw new Error(`Malformed runtime cancellation request "${row.key}".`);
     }
-    await input.client.query('BEGIN');
     try {
-      await input.client.query(
-        `SELECT set_config('lock_timeout', $1, true),
-                set_config('statement_timeout', $2, true)`,
-        [
-          `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
-          `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
-        ],
-      );
-      const didCancel = await cancelPostgresSchedulerRun(
-        input.client,
-        {
-          runId,
-          reason:
-            row.cancellation_kind === 'manual'
-              ? 'Run cancellation was requested by the caller.'
-              : 'Run did not start within its configured scheduler queue deadline.',
-        },
-        input.schedulerOptions,
-      );
-      await input.client.query('COMMIT');
-      if (didCancel) reconciled += 1;
+      const outcome = await finalizeCancelledRunAfterCleanup({
+        client: input.client,
+        runId,
+        reason: row.reason || 'Cancelled play run.',
+        schedulerSchema,
+        schedulerOptions: input.schedulerOptions,
+        controlKey: row.key,
+      });
+      if (outcome === 'cancelled') reconciled += 1;
       else skipped += 1;
     } catch (error) {
-      await input.client.query('ROLLBACK').catch(() => undefined);
-      // A worker-held lock is expected to be retried by the next lane sweep;
-      // never let one blocked cancellation stall the claim loop or create a
-      // second unbounded cancellation pile.
       if (
         isRuntimeSchedulerQueryCancellationError(error) ||
         isRuntimeSchedulerRetryableTransactionError(error)
       ) {
         skipped += 1;
-        // The hardened pool destroys a client after a statement timeout or a
-        // retryable transaction error. Do not issue the next candidate on that
-        // same socket; the next worker sweep acquires a fresh client.
         break;
       }
       throw error;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -262,10 +262,13 @@ import { shouldTerminalizeSandboxCapacityRejection } from '../launch-source-poli
 import {
   ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
   reconcileAbsurdRuntimeCancellations,
+  reconcileQueuedAbsurdRuntimeCancellations,
 } from './absurd-deadline-reconciliation';
 import {
   abandonRuntimeSandboxOperationReservation,
   bindRuntimeSandboxCapacityLease,
+  readReservedDaytonaCancellationOperation,
+  recordRecoveredDaytonaCancellationSandbox,
   reserveRuntimeSandboxOperation,
   runtimeSandboxCapacityAdmissionMode,
   RuntimeSandboxCapacityLimitError,
@@ -373,8 +376,8 @@ export const ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS =
 const HEARTBEAT_THROTTLE_MS = (HEARTBEAT_CLAIM_TIMEOUT_SECONDS * 1000) / 3;
 const HEARTBEAT_TIMEOUT_RETRY_DELAY_MS = 1_000;
 /**
- * Sandbox deletion is best-effort cleanup after a durable terminal write. The
- * Daytona SDK may hang before its own delete timeout (for example while
+ * Sandbox deletion is a durable obligation and may precede a cancellation's
+ * terminal write. The Daytona SDK may hang before its own delete timeout (for example while
  * looking up an already-autostopped sandbox), so never let that external call
  * occupy an Absurd worker slot indefinitely.
  */
@@ -6644,6 +6647,75 @@ export async function reconcilePostgresSchedulerAmbiguousModalCreatesOnce(input:
   return { observed: intents.length, resourcesRecorded, resolvedAbsent };
 }
 
+async function reconcileReservedDaytonaCancellationOnce(input: {
+  runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
+  environment: 'preview' | 'production';
+  schedulerOptions: Pick<PostgresSchedulerOptions, 'schema'>;
+}): Promise<void> {
+  const operation = await input.runWithClient((client) =>
+    readReservedDaytonaCancellationOperation(
+      client,
+      input.environment,
+      input.schedulerOptions,
+    ),
+  );
+  if (!operation) return;
+  const [{ Daytona }, { loadDaytonaRequiredConfig }] = await Promise.all([
+    import('@daytonaio/sdk'),
+    import('../daytona-runtime-config'),
+  ]);
+  const daytona = new Daytona(
+    loadDaytonaRequiredConfig({ environment: input.environment }).clientOptions,
+  );
+  const sandboxName = `dl-${operation.providerOperationToken}`;
+  let sandbox: Awaited<ReturnType<typeof daytona.get>>;
+  try {
+    sandbox = await deleteRuntimeSandboxBounded({
+      deleteRuntimeSandbox: (name) => daytona.get(name),
+      sandboxId: sandboxName,
+      timeoutMs: ABSURD_SANDBOX_GC_TIMEOUT_MS,
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (!/\b(?:sandbox )?not found\b|\b404\b/i.test(message)) throw error;
+    if (
+      Date.now() - operation.createdAtMs <
+      RUNTIME_RELIABILITY_POLICY.recovery.horizonMs
+    ) {
+      return;
+    }
+    await input.runWithClient((client) =>
+      abandonRuntimeSandboxOperationReservation(
+        client,
+        { operationKey: operation.operationKey },
+        input.schedulerOptions,
+      ),
+    );
+    return;
+  }
+  if (sandbox.labels?.operationKey !== operation.providerOperationToken) {
+    throw new Error(
+      `Daytona cancellation lookup ${sandboxName} lacks its exact operation label.`,
+    );
+  }
+  if (!sandbox.organizationId) {
+    throw new Error(
+      `Daytona cancellation lookup ${sandboxName} lacks its routing organization.`,
+    );
+  }
+  await input.runWithClient((client) =>
+    recordRecoveredDaytonaCancellationSandbox(
+      client,
+      {
+        operationKey: operation.operationKey,
+        sandboxId: sandbox.id,
+        routingDomain: sandbox.organizationId,
+      },
+      input.schedulerOptions,
+    ),
+  );
+}
+
 export async function runPostgresSchedulerSandboxCleanupController(input: {
   runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
   deleteRuntimeSandbox: AbsurdRuntimeWorkerDeps['deleteRuntimeSandbox'];
@@ -6681,6 +6753,8 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
   const reportReconciliationPollFailure = createRateLimitedRuntimeLogger({
     log: (...args) => console.warn(...args),
   });
+  let reservedRecovery: Promise<void> | null = null;
+  let nextReservedRecoveryAt = 0;
 
   const startJob = (job: PostgresSchedulerSandboxCleanupJob): void => {
     const task = processPostgresSchedulerSandboxCleanupJob({
@@ -6726,6 +6800,37 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
   };
 
   while (!input.shouldStop()) {
+    if (
+      (!input.providers || input.providers.includes('daytona')) &&
+      !reservedRecovery &&
+      Date.now() >= nextReservedRecoveryAt
+    ) {
+      nextReservedRecoveryAt = Date.now() + 60_000;
+      const recovery = reconcileReservedDaytonaCancellationOnce(input)
+        .catch((error) => {
+          reportReconciliationPollFailure(
+            '[absurd-runtime.worker] daytona_cancellation_reservation_recovery_failed',
+            { error: error instanceof Error ? error.message : String(error) },
+          );
+        })
+        .finally(() => {
+          if (reservedRecovery === recovery) reservedRecovery = null;
+        });
+      reservedRecovery = recovery;
+    }
+    try {
+      await input.runWithClient((client) =>
+        reconcileQueuedAbsurdRuntimeCancellations({
+          client,
+          schedulerOptions: input.schedulerOptions,
+        }),
+      );
+    } catch (error) {
+      reportReconciliationPollFailure(
+        '[absurd-runtime.worker] cancellation_reconciliation_poll_failed',
+        { error: error instanceof Error ? error.message : String(error) },
+      );
+    }
     if (input.providers?.includes('modal')) {
       try {
         const reconciliation =
@@ -7082,9 +7187,9 @@ export async function startRuntimeWorkerHost(
         queue,
         schedulerOptions,
       });
-      if (cancellations.reconciled > 0 || cancellations.skipped > 0) {
+      if (cancellations.queued > 0) {
         reportCancellationProgress(
-          '[absurd-runtime.worker] cancellations_reconciled',
+          '[absurd-runtime.worker] cancellations_queued',
           {
             queue,
             releaseId,
@@ -7244,9 +7349,9 @@ export async function startRuntimeWorkerHost(
               queue: adoptedQueue,
               schedulerOptions,
             });
-            if (cancellations.reconciled > 0 || cancellations.skipped > 0) {
+            if (cancellations.queued > 0) {
               reportAdoptedCancellationProgress(
-                '[absurd-runtime.worker] cancellations_reconciled',
+                '[absurd-runtime.worker] cancellations_queued',
                 {
                   queue: adoptedQueue,
                   releaseId,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts

@@ -14,8 +14,8 @@
  *             Persist the absurd task id
  *             on the run row so a later cancel can target it.
  *   observe/result → reuse the postgres LISTEN/NOTIFY run handle.
- *   cancel  → one bounded transaction cancels the Absurd task and terminalizes
- *             the postgres scheduler run. Success means both states committed.
+ *   cancel  → commit the Absurd stop and a durable cancellation request;
+ *             the worker deletes sandboxes before the final run write.
  *   signal  → durable signal row (authoritative payload) + absurd emitEvent
  *             (the wakeup only; payload is re-read from the signal row).
  */
@@ -29,8 +29,6 @@ import {
   type PlaySchedulerSubmitInput,
 } from '../scheduler-backend';
 import {
-  cancelPostgresSchedulerRun,
-  postgresSchedulerCancellationOutboxLease,
   postgresSchedulerRunHandle,
   readPostgresSchedulerLaunch,
   readPostgresSchedulerAbsurdTaskId,
@@ -48,6 +46,8 @@ import {
   withRuntimeSchedulerClient,
   withRuntimeSchedulerTransaction,
 } from './postgres-pool';
+import { RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE } from '../scheduler-backend';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
 import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
 import {
   ABSURD_PLAY_RUN_MAX_ATTEMPTS,
@@ -200,7 +200,7 @@ function absurdRunHandle(input: {
   const runsQueue = absurdQueueNameForRelease(releaseId);
   const cancellationKey = `${poolOptions.url}::${effectiveOptions.schema ?? ''}::${input.runId}`;
 
-  const cancelAbsurdRunSynchronously = async (reason?: string) =>
+  const queueCancellation = async (reason?: string) =>
     withRuntimeSchedulerTransaction(
       poolOptions,
       {
@@ -208,9 +208,8 @@ function absurdRunHandle(input: {
         statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
       },
       async (client) => {
-        // Resolve the immutable release lane inside the same checkout and
-        // transaction that performs cancellation. The stop endpoint therefore
-        // has no preliminary lane lookup that can succeed or fail separately.
+        // Resolve the immutable release lane in the transaction that records
+        // the request. A failed final run write cannot roll this marker back.
         const launch = await readPostgresSchedulerLaunch(
           client,
           input.runId,
@@ -231,67 +230,73 @@ function absurdRunHandle(input: {
           input.runId,
           effectiveOptions,
         );
+        if (
+          await readPostgresSchedulerResult(
+            client,
+            input.runId,
+            effectiveOptions,
+          )
+        ) {
+          return false;
+        }
         if (taskId) {
-          const engineTaskCancelled = await cancelAbsurdTaskIfPresent(client, {
+          const engineTaskPresent = await cancelAbsurdTaskIfPresent(client, {
             queue: cancellationQueue,
             taskId,
             runId: input.runId,
           });
-          // Older v4 Absurd functions do not materialize a manual cancellation
-          // deadline or reason. Stamp both in the adapter so existing queues
-          // participate in the same durable reconciliation lane without a
-          // schema cutover. The explicit marker keeps a user cancel distinct
-          // from a task cancelled by its configured max_delay sweep.
-          if (engineTaskCancelled) {
+          if (engineTaskPresent) {
+            // Older workers only reconcile `manual` or `max_delay`. This
+            // marker is diagnostic; the control row is the cross-lane queue.
             await client.query(
               `UPDATE absurd.${quotePostgresIdentifier(`t_${cancellationQueue}`)}
                 SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
                     cancellation = COALESCE(cancellation, '{}'::jsonb)
-                      || jsonb_build_object('manual', true)
-              WHERE task_id = $1::uuid
-                AND state = 'cancelled'`,
+                      || jsonb_build_object('sandboxFirst', true)
+              WHERE task_id = $1::uuid`,
               [taskId],
             );
           }
         }
-        const cancelled = await cancelPostgresSchedulerRun(
-          client,
-          {
-            runId: input.runId,
-            reason,
-            outboxLease: postgresSchedulerCancellationOutboxLease(input.runId),
-          },
-          effectiveOptions,
+        const schedulerSchema = quotePostgresIdentifier(
+          effectiveOptions.schema?.trim() ||
+            resolveRuntimeSchedulerSchemaEnv() ||
+            'runtime_scheduler',
         );
-        // The update returning a row is proof that this transaction wrote the
-        // terminal state and outbox event. Only the idempotent retry/race path
-        // needs one read to prove another terminal writer already won.
-        if (!cancelled) {
-          const terminal = await readPostgresSchedulerResult(
-            client,
-            input.runId,
-            effectiveOptions,
-          );
-          if (!terminal) {
-            throw new Error(
-              `Cancellation did not terminalize scheduler run "${input.runId}".`,
-            );
-          }
-        }
+        await client.query(
+          `INSERT INTO ${schedulerSchema}.control (key, value)
+           VALUES ($1, jsonb_build_object(
+             'reason', left($2::text, 4000),
+             'requestedAt', clock_timestamp()
+           ))
+           ON CONFLICT (key) DO NOTHING`,
+          [
+            `runtime_cancel:${input.runId}`,
+            reason?.trim() || 'Cancelled play run.',
+          ],
+        );
+        return true;
       },
       {
         runId: input.runId,
-        phase: 'cancel_run_synchronously',
+        phase: 'queue_run_cancellation',
         priority: 'critical',
       },
+      { maxAttempts: 1 },
     );
 
   const requestCancellation = async (reason?: string) => {
     await runCancellationSingleFlight(cancellationKey, async () => {
-      // Engine cancellation and the scheduler terminal write commit together.
-      // A timeout rolls both back, so returning means the run is terminal and
-      // throwing means the caller can safely retry one unchanged operation.
-      await cancelAbsurdRunSynchronously(reason);
+      const queued = await queueCancellation(reason);
+      if (queued) {
+        const error = new Error(
+          `Cancellation queued for ${input.runId}; sandbox cleanup and final run update are pending.`,
+        );
+        Object.assign(error, {
+          code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
+        });
+        throw error;
+      }
     });
   };
 

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -3080,6 +3080,13 @@ async function penalizeRateStateStatements(
                 GREATEST(h.retry_at_ms, COALESCE($5::bigint, nv.now_ms + $2::bigint))
                   - COALESCE(h.rate_observation_started_at_ms, nv.now_ms)
             END) AS cycle_ms,
+          CASE
+            WHEN $5::bigint IS NOT NULL AND h.rate_successes <= 0 THEN
+              CASE WHEN h.seen_backpressure THEN
+                LEAST(60000, GREATEST(1000, h.rate_recovery_interval_ms * 2))
+              ELSE 1000 END
+            ELSE NULL
+          END AS no_progress_probe_interval_ms,
           nv.now_ms
         FROM held h CROSS JOIN now_value nv
       ), throttle_ceiling AS (
@@ -3087,12 +3094,12 @@ async function penalizeRateStateStatements(
           GREATEST(${MIN_ADAPTIVE_RPS_FLOOR},
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
               CASE WHEN $5::bigint IS NOT NULL
-                -- An explicit Retry-After is only the provider's next safe
-                -- dispatch deadline. It is not a rate measurement; with no
-                -- successful-call evidence, use one conservative probe per
-                -- second after that deadline instead of deriving a permanent
-                -- RPM limit from the length of the hold.
-                THEN GREATEST(e.refill_before, 1000.0 / e.cycle_ms)
+                -- A fresh provider deadline is not a throughput measurement.
+                -- Repeated probes with no successful work are evidence that
+                -- the scope is still unavailable, however. Space subsequent
+                -- probes exponentially (up to one per minute) across every
+                -- run sharing this provider credential.
+                THEN 1000.0 / e.no_progress_probe_interval_ms
                 ELSE e.refill_before END
               ELSE e.refill_before / 2 END
           ) AS next_ceiling_rps
@@ -3115,7 +3122,9 @@ async function penalizeRateStateStatements(
           rate_throttle_ceiling_rps = CASE WHEN e.new_episode
             THEN e.next_ceiling_rps ELSE e.rate_throttle_ceiling_rps END,
           rate_observation_started_at_ms = CASE WHEN e.new_episode THEN NULL ELSE e.rate_observation_started_at_ms END,
-          rate_recovery_interval_ms = CASE WHEN e.new_episode THEN e.cycle_ms ELSE e.rate_recovery_interval_ms END,
+          rate_recovery_interval_ms = CASE WHEN e.new_episode
+            THEN COALESCE(e.no_progress_probe_interval_ms, e.cycle_ms)
+            ELSE e.rate_recovery_interval_ms END,
           last_ramp_at_ms = CASE WHEN e.new_episode THEN e.deadline ELSE e.last_ramp_at_ms END,
           -- All pre-429 permits are revalidated against this durable hold
           -- before dispatch. Their obsolete schedule debt must not add a