deepline 0.3.208 → 0.3.210

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.208',
+  version: '0.3.210',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -122,6 +122,7 @@ import {
 import {
   AllAdmittedRowsFailedError,
   isProviderExhaustedFailure,
+  ProviderAdmissionWaitTimeoutError,
   ProviderExhaustedError,
 } from './run-failure';
 import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
@@ -18682,6 +18683,30 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           networkScope: null,
         },
       );
+    const providerAdmissionTimeoutError = (
+      error: ProviderAdmissionWaitTimeoutError,
+    ) =>
+      createToolHttpError(
+        toolErrorSchemaVersion,
+        error.message,
+        null,
+        503,
+        'repairable',
+        {
+          toolId,
+          provider: error.provider,
+          operation: toolId,
+          code: error.code,
+          origin: 'deepline',
+          category: 'rate_limit',
+          retryable: true,
+          statusCode: 503,
+          requestId: null,
+          retryAfterMs: null,
+          networkKind: null,
+          networkScope: null,
+        },
+      );
     const observeProviderWait = (retryAtMs: number) => {
       const startedAtMsByScope =
         (this.checkpoint.providerNoProgressStartedAtMsByScope ??= {});
@@ -19934,6 +19959,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
+              if (error instanceof ProviderAdmissionWaitTimeoutError) {
+                throw providerAdmissionTimeoutError(error);
+              }
               if (error instanceof ProviderExhaustedError) {
                 const retryAtMs = Date.parse(error.retryAt);
                 observeProviderWait(

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -20,6 +20,7 @@ import {
   type RateStateBackendKind,
 } from './rate-state-backend';
 import {
+  ProviderAdmissionWaitTimeoutError,
   ProviderExhaustedError,
   providerNameFromBucketId,
 } from '../run-failure';
@@ -36,12 +37,18 @@ import type { PlayRunnerRateStateValidateResult } from '../protocol';
  */
 export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 300_000;
 /**
- * Total time one acquirer may spend cycling through window-exhausted refills.
- * Each refill's sleep is bounded, but nothing bounded how many of them a row
- * could take; with several runs sharing one org's provider window a late
- * starter parked here for twenty minutes with its core idle.
+ * Max total duration of a cached rate-state hold before it is yielded or its
+ * stale local no-progress warning is cleared. This is distinct from the shorter
+ * local permit-admission budget below.
  */
 export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
+/**
+ * Total time one acquirer may spend waiting for a locally paced permit. This
+ * covers both empty-block refills and future-scheduled tickets. A spent budget
+ * is a repairable Deepline admission error, not another retry loop or a
+ * provider-owned Retry-After hold.
+ */
+export const PROVIDER_ADMISSION_WAIT_BUDGET_MS = 120_000;
 const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
 
 /**
@@ -91,6 +98,31 @@ export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
 const LEASE_BLOCK_MIN_TTL_MS = 250;
 const LEASE_BLOCK_MAX_TTL_MS = 5_000;
 const MAX_ACQUIRE_SLEEP_MS = 5_000;
+
+function abortReason(signal: AbortSignal): Error {
+  return signal.reason instanceof Error
+    ? signal.reason
+    : new Error('Rate-state acquire aborted.');
+}
+
+function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(abortReason(signal));
+  return new Promise((resolve, reject) => {
+    const cleanup = () => signal?.removeEventListener('abort', onAbort);
+    const timer = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      cleanup();
+      reject(abortReason(signal!));
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+    if (signal?.aborted) onAbort();
+  });
+}
+
 /**
  * Horizon used to size the next lease block from the rate actually in force.
  * Kept short so a block stays inside its own TTL and a stalled runner can never
@@ -156,7 +188,7 @@ interface PendingRelease {
 interface Options {
   schedulerSchema?: string | null;
   now?: () => number;
-  sleep?: (ms: number) => Promise<void>;
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
   onStoreFailure?: (info: { bucketId: string; error: string }) => void;
   /** Override the transport (tests inject a counting/faulting port). */
   port?: AppRuntimeRatePort;
@@ -177,7 +209,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly port: AppRuntimeRatePort;
   private readonly schedulerSchema: string | null;
   private readonly now: () => number;
-  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
   private readonly jitter: () => number;
   private readonly onStoreFailure: (info: {
     bucketId: string;
@@ -238,9 +270,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     this.port = options.port ?? contextRatePort(context);
     this.schedulerSchema = options.schedulerSchema?.trim() || null;
     this.now = options.now ?? (() => Date.now());
-    this.sleep =
-      options.sleep ??
-      ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.sleep = options.sleep ?? sleepWithAbort;
     this.jitter = options.jitter ?? Math.random;
     this.onStoreFailure =
       options.onStoreFailure ??
@@ -412,20 +442,47 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       const localPermit = this.drawFromBlock(bucketId, laneKey);
       if (localPermit) {
         const waitMs = localPermit.scheduledAtMs - this.now();
-        if (waitMs > 0) await this.sleep(waitMs);
+        const elapsedMs = Math.max(0, this.now() - acquireStartedAt);
+        const remainingWaitMs = Math.max(
+          0,
+          PROVIDER_ADMISSION_WAIT_BUDGET_MS - elapsedMs,
+        );
+        if (
+          elapsedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS ||
+          waitMs > remainingWaitMs
+        ) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw new ProviderAdmissionWaitTimeoutError({
+            provider: providerNameFromBucketId(bucketId),
+            waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+          });
+        }
+        try {
+          if (waitMs > 0) await this.sleep(waitMs, signal);
+        } catch (error) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw error;
+        }
         if (signal?.aborted) {
-          localPermit.permit.release();
-          throw signal.reason instanceof Error
-            ? signal.reason
-            : new Error('Rate-state acquire aborted.');
+          await this.cancelUndispatchedPermit(localPermit);
+          throw abortReason(signal);
+        }
+        if (
+          this.now() - acquireStartedAt >= PROVIDER_ADMISSION_WAIT_BUDGET_MS
+        ) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw new ProviderAdmissionWaitTimeoutError({
+            provider: providerNameFromBucketId(bucketId),
+            waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+          });
         }
         return localPermit.permit;
       }
       const waitedMs = this.now() - acquireStartedAt;
-      if (waitedMs >= RATE_ACQUIRE_MAX_WAIT_MS) {
-        throw new ProviderExhaustedError({
+      if (waitedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS) {
+        throw new ProviderAdmissionWaitTimeoutError({
           provider: providerNameFromBucketId(bucketId),
-          retryAtMs: this.now() + MAX_ACQUIRE_SLEEP_MS,
+          waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
         });
       }
       // Count this acquirer as waiting on the refill so the single-flight refill
@@ -1389,7 +1446,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private drawFromBlock(
     bucketId: string,
     laneKey: string,
-  ): { permit: PacingPermit; scheduledAtMs: number } | null {
+  ): {
+    permit: PacingPermit;
+    scheduledAtMs: number;
+    reservationId: string | null;
+  } | null {
     const block = this.liveBlock(bucketId, laneKey);
     if (!block) return null;
     // An emptied block stays until it expires: the next refill merges into it,
@@ -1399,9 +1460,24 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     return {
       permit: this.permitFor(bucketId, laneKey, block, scheduled),
       scheduledAtMs: scheduled[0],
+      reservationId: scheduled[2],
     };
   }
 
+  private async cancelUndispatchedPermit(input: {
+    permit: PacingPermit;
+    reservationId: string | null;
+  }): Promise<void> {
+    if (input.reservationId) {
+      await input.permit.cancel();
+      return;
+    }
+    // Older gateways may grant a permit without a refundable reservation id.
+    // Still release any concurrency lease; losing one rate token is safer than
+    // manufacturing an immediate permit during a rolling deploy.
+    input.permit.release();
+  }
+
   private drainBlocksForBucket(
     bucketId: string,
     beforeRetryAtMs?: number,
@@ -1546,7 +1622,7 @@ function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
     // This one shared default rule is synthesized for unknown providers whose
     // credential bucket spans operations. Keep the generic policy unchanged
     // for local/coordinator backends, but let Postgres start a new provider
-    // scope with one probe/sec before learning its sustainable rate.
+    // scope at 10 RPS before learning its sustainable rate.
     const seed =
       rule.ruleId === 'default:provider-scope'
         ? {

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -511,9 +511,9 @@ export function createPlayExecutionGovernor(
       if (!declaredPacing) {
         // bucketId is the authenticated quota scope. Unknown tools sharing a
         // credential must not each mint an independent copy of the default rate.
-        // Start its one shared lane at one probe/sec; the DB-owned AIMD quickly
-        // ramps on sustained success and reacts to Retry-After if the provider
-        // tells us the quota is lower.
+        // Start its one shared lane at 10 RPS, matching the generic fallback.
+        // The DB-owned AIMD ramps on sustained success and reacts to
+        // Retry-After if the provider tells us the quota is lower.
         pacing.rules = pacing.rules.map((rule) => ({
           ...rule,
           ruleId: 'default:provider-scope',

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -189,7 +189,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
   },
   pacing: {
     // Keep the generic fallback rate. The shared Postgres backend separately
-    // cold-starts unknown provider scopes at one probe/sec.
+    // cold-starts unknown provider scopes at 10 RPS.
     // Declared providers (rate-limit-definitions.ts) keep their own rates.
     defaultProviderRequestsPerSecond: 10,
     // This is the scheduler's resident-call envelope, not an RPS target.

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -15,7 +15,7 @@
 export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
 
 /** Cold-start rate for an unknown provider bucket shared across a fleet key. */
-export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 1;
+export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 10;
 
 /** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
 export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -240,6 +240,27 @@ export function providerExhaustedMessage(input: {
   );
 }
 
+/**
+ * The local rate governor could not offer a provider permit inside its bounded
+ * admission window. No provider request was dispatched, so this is a Deepline
+ * pacing failure rather than a provider Retry-After hold.
+ */
+export class ProviderAdmissionWaitTimeoutError extends Error {
+  readonly code = 'PROVIDER_ADMISSION_TIMEOUT';
+  readonly provider: string;
+  readonly waitBudgetMs: number;
+
+  constructor(input: { provider: string; waitBudgetMs: number }) {
+    const minutes = Math.max(1, Math.ceil(input.waitBudgetMs / 60_000));
+    super(
+      `PROVIDER_ADMISSION_TIMEOUT: Deepline could not grant a provider permit for ${input.provider} within its ${minutes}-minute wait budget. No provider request was sent.`,
+    );
+    this.name = 'ProviderAdmissionWaitTimeoutError';
+    this.provider = input.provider;
+    this.waitBudgetMs = input.waitBudgetMs;
+  }
+}
+
 /**
  * Control signal from the DB-authoritative pacer when a provider hold exceeds
  * the in-process wait threshold. The next provider call is not dispatched.

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -126,6 +126,9 @@ export type PlayRunLedgerSnapshot = {
   runId: string;
   playName?: string | null;
   status: PlayRunLedgerStatus;
+  /** Typed scheduler wait, when a durable waiting event supplied it. */
+  waitKind?: string | null;
+  waitUntil?: number | null;
   error?: string | null;
   createdAt?: number | null;
   startedAt?: number | null;
@@ -201,6 +204,8 @@ export type PlayRunLedgerEvent =
     })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.waiting';
+      waitKind?: string | null;
+      waitUntil?: number | null;
     })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.resumed';
@@ -541,6 +546,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
     runId: input.runId,
     playName: input.playName ?? null,
     status,
+    waitKind: null,
+    waitUntil: null,
     error: optionalNullableString(input.error) ?? null,
     createdAt: finiteNumber(input.createdAt) ?? null,
     startedAt,
@@ -677,6 +684,8 @@ export function normalizePlayRunLedgerSnapshot(
     playName:
       optionalNullableString(value.playName) ?? fallback.playName ?? null,
     status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
+    waitKind: optionalNullableString(value.waitKind),
+    waitUntil: finiteNumber(value.waitUntil),
     error,
     createdAt,
     startedAt,
@@ -1312,11 +1321,17 @@ export function reducePlayRunLedgerEvent(
       return withTiming({
         ...base,
         status: 'waiting',
+        waitKind: event.waitKind ?? null,
+        waitUntil: event.waitUntil ?? null,
+        activeActivityId: null,
       });
     case 'run.resumed':
       return withTiming({
         ...base,
         status: 'running',
+        waitKind: null,
+        waitUntil: null,
+        activeActivityId: null,
       });
     case 'run.completed':
       return (
@@ -1330,6 +1345,8 @@ export function reducePlayRunLedgerEvent(
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
           status: 'completed',
+          waitKind: null,
+          waitUntil: null,
           error: null,
           startedAt: base.startedAt ?? occurredAt,
           finishedAt: terminalFinishedAt(base, event.type, occurredAt),
@@ -1358,6 +1375,8 @@ export function reducePlayRunLedgerEvent(
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
           status: 'failed',
+          waitKind: null,
+          waitUntil: null,
           error:
             base.status === 'failed'
               ? (base.error ?? event.error ?? null)
@@ -1386,6 +1405,8 @@ export function reducePlayRunLedgerEvent(
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
           status: 'cancelled',
+          waitKind: null,
+          waitUntil: null,
           error:
             base.status === 'cancelled'
               ? (base.error ?? event.error ?? null)

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -14,18 +14,22 @@ const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
 export function acquireRateScopeSql(
   table: string,
   successesParameter: string,
+  minimumRateSql = 'NULL::float8',
 ): string {
+  const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
+    THEN h.refill_per_sec
+    ELSE GREATEST(h.refill_per_sec, ${minimumRateSql}) END`;
   return `provider_hold_rows AS (
     SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
   ), provider_hold AS (
     SELECT h.*,
       LEAST(
-        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, h.refill_per_sec),
-        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * h.refill_per_sec / 1000.0
+        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
+        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
       ) AS tokens_after_refill,
       ${successesParameter}::int = 0
         AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
-          > GREATEST(1000, 2000.0 / h.refill_per_sec) AS observation_was_idle,
+          > GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
@@ -33,9 +37,9 @@ export function acquireRateScopeSql(
             ELSE h.rate_recovery_interval_ms END
           AND h.retry_at_ms <= nv.now_ms
         THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
-          THEN h.refill_per_sec * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
-          ELSE h.refill_per_sec * 2 END)
-        ELSE h.refill_per_sec END AS next_refill_per_sec,
+          THEN ${refillRate} * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
+          ELSE ${refillRate} * 2 END)
+        ELSE ${refillRate} END AS next_refill_per_sec,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -390,9 +390,9 @@ function normalizeRules(
       return {
         ruleId: rule.ruleId,
         // Rates are fractional; only dispatched requests must be whole.
-        // Keep the shared, unhinted credential scope conservative even when
-        // an older runner submits the generic 10-RPS policy seed. The shared
-        // row's AIMD controller learns upward from successful observations.
+        // Normalize every runner to the canonical 10-RPS seed for an unknown
+        // shared credential scope. The shared row's AIMD state can still be
+        // lower after backpressure and learns upward from successful calls.
         requestsPerWindow: isUnknownProviderScope
           ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
           : rule.requestsPerWindow,
@@ -446,7 +446,13 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   // Preserve its fractional refill rate, with dispatch times resolved in ms.
   const capacity = Math.max(1, rule.requestsPerWindow);
   const base = (rule.requestsPerWindow / rule.windowMs) * 1000;
-  const minRps = Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
+  // Undeclared providers use the shared fallback rate as their operating
+  // floor. Explicit provider Retry-After deadlines still block dispatch; a
+  // prior learned AIMD rate must not leave later runs paced below the fallback.
+  const minRps =
+    rule.ruleId === 'default:provider-scope'
+      ? base
+      : Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
   const maxRps =
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
@@ -461,6 +467,12 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   };
 }
 
+function minimumRateSqlForRules(rules: readonly AcquirePacingRule[]): string {
+  return rules.some((rule) => rule.ruleId === 'default:provider-scope')
+    ? `${DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS}::float8`
+    : 'NULL::float8';
+}
+
 function positiveInt(value: number, label: string): number {
   if (!Number.isFinite(value) || value <= 0) {
     throw new Error(`${label} must be a positive finite number.`);
@@ -743,11 +755,10 @@ async function seedBucketRows(
         ON CONFLICT (bucket_id) DO UPDATE SET
           retry_at_ms = GREATEST(hold.retry_at_ms, EXCLUDED.retry_at_ms),
           refill_per_sec = CASE
-            -- Older runners seeded this shared credential row at the generic
-            -- default (10 RPS). If that old value never moved from its seed,
-            -- adopt the conservative shared-scope seed on the first acquire
-            -- after upgrade. Preserve any rate with actual ramp or throttle
-            -- evidence, and never erase a provider Retry-After hold.
+            -- If this row never moved from its seed, adopt the canonical
+            -- shared-scope seed on the first acquire after upgrade. Preserve
+            -- any rate with actual ramp or throttle evidence, and never erase
+            -- a provider Retry-After hold.
             WHEN EXISTS (
               SELECT 1 FROM seed_config
               WHERE rule_id = 'default:provider-scope'
@@ -1295,7 +1306,11 @@ async function acquirePureRate(
           adaptive_max_configured boolean
         )
       ),
-      ${acquireRateScopeSql(scopeHoldTableName(input), '$5')},
+      ${acquireRateScopeSql(
+        scopeHoldTableName(input),
+        '$5',
+        minimumRateSqlForRules(rules),
+      )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
@@ -1735,7 +1750,11 @@ async function acquireWithConcurrency(
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
       ),
-      ${acquireRateScopeSql(scopeHoldTableName(input), '$10')},
+      ${acquireRateScopeSql(
+        scopeHoldTableName(input),
+        '$10',
+        minimumRateSqlForRules(rules),
+      )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
@@ -3015,6 +3034,11 @@ async function penalizeRateStateStatements(
     return emptyRateStatePenaltyTelemetry();
   }
   const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
+  const minimumRateRps = rules.some(
+    (rule) => rule.ruleId === 'default:provider-scope',
+  )
+    ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
+    : MIN_ADAPTIVE_RPS_FLOOR;
   await applyConcurrencyObservations(
     client,
     input,
@@ -3091,7 +3115,7 @@ async function penalizeRateStateStatements(
         FROM held h CROSS JOIN now_value nv
       ), throttle_ceiling AS (
         SELECT e.*,
-          GREATEST(${MIN_ADAPTIVE_RPS_FLOOR},
+          GREATEST(${minimumRateRps},
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
               CASE WHEN $5::bigint IS NOT NULL
                 -- A fresh provider deadline is not a throughput measurement.
@@ -3106,7 +3130,7 @@ async function penalizeRateStateStatements(
         FROM evidence e
       ), learned AS (
         SELECT e.*,
-          CASE WHEN e.new_episode THEN GREATEST(${MIN_ADAPTIVE_RPS_FLOOR}, LEAST(
+          CASE WHEN e.new_episode THEN GREATEST(${minimumRateRps}, LEAST(
             e.next_ceiling_rps,
             GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
           )) ELSE e.refill_before END AS refill_after

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -78,6 +78,7 @@ import {
 } from '../work-receipts';
 import type { PlayStaticPipeline } from '../../plays/static-pipeline';
 import {
+  isPlayRunLedgerOpaqueEvent,
   LOG_TAIL_LIMIT,
   type AcceptedPlayRunLedgerEvent,
   type PlayRunLedgerEvent,
@@ -15920,6 +15921,64 @@ async function appendRunLedgerOutboxFact(
 
 export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-contract';
 
+/**
+ * A newer app can receive a projection from a worker that predates optional
+ * wait metadata. Treat only those absent fields as their canonical values;
+ * lease identity, event ordering, and every field the worker did send remain
+ * exact.
+ */
+function normalizeLegacyRunWaitingProjectionFields(
+  projectedEvents: readonly AcceptedPlayRunLedgerEvent[],
+  expectedEvents: readonly (
+    | AcceptedPlayRunLedgerEvent
+    | PlayRunLedgerIngressEvent
+  )[],
+): AcceptedPlayRunLedgerEvent[] {
+  return projectedEvents.map((event, index) => {
+    const expected = expectedEvents[index];
+    if (
+      isPlayRunLedgerOpaqueEvent(event) ||
+      event.type !== 'run.waiting' ||
+      expected?.type !== 'run.waiting' ||
+      ('opaque' in expected && expected.opaque === true)
+    ) {
+      return event;
+    }
+    const expectedEvent = expected as Record<string, unknown>;
+    const expectedWaitKind = expectedEvent.waitKind;
+    const expectedWaitUntil = expectedEvent.waitUntil;
+    return {
+      ...event,
+      ...(event.waitKind === undefined &&
+      (typeof expectedWaitKind === 'string' || expectedWaitKind === null)
+        ? { waitKind: expectedWaitKind }
+        : {}),
+      ...(event.waitUntil === undefined &&
+      (typeof expectedWaitUntil === 'number' || expectedWaitUntil === null)
+        ? { waitUntil: expectedWaitUntil }
+        : {}),
+    };
+  });
+}
+
+function runLedgerProjectionMatchesCanonicalPrefix(
+  projectedEvents: readonly AcceptedPlayRunLedgerEvent[],
+  expectedEvents: readonly (
+    | AcceptedPlayRunLedgerEvent
+    | PlayRunLedgerIngressEvent
+  )[],
+): boolean {
+  if (projectedEvents.length !== expectedEvents.length) return false;
+  return (
+    stableStringify(
+      normalizeLegacyRunWaitingProjectionFields(
+        projectedEvents,
+        expectedEvents,
+      ),
+    ) === stableStringify(expectedEvents)
+  );
+}
+
 /** Extend only our still-live lease before another bounded transport piece. */
 export async function renewPostgresRunLedgerDelivery(
   client: PostgresSchedulerQueryClient,
@@ -16074,12 +16133,17 @@ export async function assertPostgresRunLedgerDelivery(
           }),
         );
       }
-      const comparablePrefixEvents = input.normalizeProjectedEvents
+      const comparablePrefixEvents: readonly (
+        | AcceptedPlayRunLedgerEvent
+        | PlayRunLedgerIngressEvent
+      )[] = input.normalizeProjectedEvents
         ? input.normalizeProjectedEvents(prefixEvents)
         : prefixEvents;
       if (
-        stableStringify(input.projectedEvents) ===
-        stableStringify(comparablePrefixEvents)
+        runLedgerProjectionMatchesCanonicalPrefix(
+          input.projectedEvents,
+          comparablePrefixEvents,
+        )
       ) {
         return;
       }
@@ -16965,6 +17029,9 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
       runId: event.runId,
       occurredAt,
       source: 'system',
+      ...(event.eventType === 'run.waiting' && waitKind !== null
+        ? { waitKind }
+        : {}),
     } as PlayRunLedgerEvent;
     if (
       waitKind !== null &&

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -8,7 +8,12 @@ import {
   type ToolExecutionErrorSchemaVersion,
 } from '../plays/tool-execution-error';
 
-export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 2;
+// Give an explicitly replay-safe transient 5xx one additional bounded retry.
+// Preview gateway load runs can overlap brief Convex billing-plane 503s; this
+// adds a second recovery chance without replaying operations that did not opt
+// into the transient HTTP safety contract.
+export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
+const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS = 2;
 // A provider may need to finish the original request before it can replay the
 // result for this key. With no Retry-After signal, these two polls give the
 // original request ten seconds to finish without repeatedly hammering it.
@@ -344,7 +349,7 @@ function decideToolExecuteHttpRetry(input: {
   if (input.customerDbStorageUnavailable) {
     return {
       retryable: true,
-      attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+      attemptCap: TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS,
       reason: 'customer_db_storage_unavailable',
     };
   }

package/dist/cli/index.js

@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -5450,6 +5450,8 @@ function createEmptyPlayRunLedgerSnapshot(input2) {
     runId: input2.runId,
     playName: input2.playName ?? null,
     status,
+    waitKind: null,
+    waitUntil: null,
     error: optionalNullableString(input2.error) ?? null,
     createdAt: finiteNumber(input2.createdAt) ?? null,
     startedAt,
@@ -5546,6 +5548,8 @@ function normalizePlayRunLedgerSnapshot(value, fallback) {
     runId: optionalString(value.runId) ?? fallback.runId,
     playName: optionalNullableString(value.playName) ?? fallback.playName ?? null,
     status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
+    waitKind: optionalNullableString(value.waitKind),
+    waitUntil: finiteNumber(value.waitUntil),
     error,
     createdAt,
     startedAt,

package/dist/cli/index.mjs

@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -5445,6 +5445,8 @@ function createEmptyPlayRunLedgerSnapshot(input2) {
     runId: input2.runId,
     playName: input2.playName ?? null,
     status,
+    waitKind: null,
+    waitUntil: null,
     error: optionalNullableString(input2.error) ?? null,
     createdAt: finiteNumber(input2.createdAt) ?? null,
     startedAt,
@@ -5541,6 +5543,8 @@ function normalizePlayRunLedgerSnapshot(value, fallback) {
     runId: optionalString(value.runId) ?? fallback.runId,
     playName: optionalNullableString(value.playName) ?? fallback.playName ?? null,
     status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
+    waitKind: optionalNullableString(value.waitKind),
+    waitUntil: finiteNumber(value.waitUntil),
     error,
     createdAt,
     startedAt,

package/dist/index.js

@@ -864,7 +864,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -3211,6 +3211,8 @@ function createEmptyPlayRunLedgerSnapshot(input) {
     runId: input.runId,
     playName: input.playName ?? null,
     status,
+    waitKind: null,
+    waitUntil: null,
     error: optionalNullableString(input.error) ?? null,
     createdAt: finiteNumber(input.createdAt) ?? null,
     startedAt,
@@ -3307,6 +3309,8 @@ function normalizePlayRunLedgerSnapshot(value, fallback) {
     runId: optionalString(value.runId) ?? fallback.runId,
     playName: optionalNullableString(value.playName) ?? fallback.playName ?? null,
     status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
+    waitKind: optionalNullableString(value.waitKind),
+    waitUntil: finiteNumber(value.waitUntil),
     error,
     createdAt,
     startedAt,

package/dist/index.mjs

@@ -768,7 +768,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -3115,6 +3115,8 @@ function createEmptyPlayRunLedgerSnapshot(input) {
     runId: input.runId,
     playName: input.playName ?? null,
     status,
+    waitKind: null,
+    waitUntil: null,
     error: optionalNullableString(input.error) ?? null,
     createdAt: finiteNumber(input.createdAt) ?? null,
     startedAt,
@@ -3211,6 +3213,8 @@ function normalizePlayRunLedgerSnapshot(value, fallback) {
     runId: optionalString(value.runId) ?? fallback.runId,
     playName: optionalNullableString(value.playName) ?? fallback.playName ?? null,
     status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
+    waitKind: optionalNullableString(value.waitKind),
+    waitUntil: finiteNumber(value.waitUntil),
     error,
     createdAt,
     startedAt,

package/dist/release.d.mts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.208";
+    readonly version: "0.3.210";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.d.ts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.208";
+    readonly version: "0.3.210";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.js

@@ -74,7 +74,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.mjs

@@ -48,7 +48,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.208",
+  version: "0.3.210",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.208",
+  "version": "0.3.210",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",