deepline 0.1.290 → 0.1.292

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.290',
+  version: '0.1.292',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -358,11 +358,15 @@ type RuntimeApiRequest =
 export type WorkerRuntimeApiContext = {
   baseUrl: string;
   executorToken: string;
+  boundary?: 'app_runtime' | 'receipt_gateway';
   integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   vercelProtectionBypassToken?: string | null;
   runtimeTestFaultHeader?: string | null;
   fetch?: typeof fetch;
   requestTimeoutMs?: number | null;
+  /** A higher-level writer owns retries and preserves the exact request body. */
+  retryPolicy?: 'default' | 'none';
+  signal?: AbortSignal;
 };
 
 const APP_RUNTIME_API_RETRY_DELAYS_MS = [100, 250, 500, 1_000] as const;
@@ -874,6 +878,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   attemptStartedAt: number;
   httpStatus?: number;
   retryAfterMs?: number | null;
+  boundaryLabel: string;
 }): Promise<void> {
   if (!isRequestTimeoutAbort(input.error)) {
     recordAppRuntimeRetryFailure({
@@ -936,9 +941,18 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
     action: input.action,
     attempts: input.attempt,
     cause: input.error,
+    boundaryLabel: input.boundaryLabel,
   });
 }
 
+function runtimeApiBoundaryLabel(
+  context: Pick<WorkerRuntimeApiContext, 'boundary'>,
+): string {
+  return context.boundary === 'receipt_gateway'
+    ? 'Runtime receipt gateway'
+    : 'App runtime API';
+}
+
 export class AppRuntimeApiTransportError extends Error {
   readonly action: RuntimeApiRequest['action'];
   readonly attempts: number;
@@ -947,11 +961,12 @@ export class AppRuntimeApiTransportError extends Error {
     action: RuntimeApiRequest['action'];
     attempts: number;
     cause: unknown;
+    boundaryLabel?: string;
   }) {
     const causeMessage =
       input.cause instanceof Error ? input.cause.message : String(input.cause);
     super(
-      `App runtime API transport exhausted action=${input.action} attempts=${input.attempts}: ${causeMessage}`,
+      `${input.boundaryLabel ?? 'App runtime API'} transport exhausted action=${input.action} attempts=${input.attempts}: ${causeMessage}`,
       { cause: input.cause },
     );
     this.name = 'AppRuntimeApiTransportError';
@@ -979,9 +994,10 @@ export class AppRuntimeApiResponseError extends Error {
     requestId?: string | null;
     retryable: boolean;
     detail: string;
+    boundaryLabel?: string;
   }) {
     super(
-      `App runtime API ${input.action} failed with status ${input.status}` +
+      `${input.boundaryLabel ?? 'App runtime API'} ${input.action} failed with status ${input.status}` +
         `${input.code ? ` code=${input.code}` : ''}` +
         `${input.requestId ? ` request_id=${input.requestId}` : ''}: ` +
         input.detail,
@@ -994,6 +1010,60 @@ export class AppRuntimeApiResponseError extends Error {
   }
 }
 
+const APP_RUNTIME_CAPACITY_ERROR_CODES = new Set([
+  'receipt_db_admission_backpressure',
+  'scheduler_capacity',
+  'runtime_postgres_admission_backpressure',
+  'RUNTIME_SCHEDULER_SATURATED',
+]);
+
+/**
+ * The runtime persistence plane is temporarily full. This is an admission
+ * signal, not a transport failure: callers must park the same operation rather
+ * than spending the ordinary request retry ladder.
+ */
+export class AppRuntimeApiCapacityError extends AppRuntimeApiResponseError {
+  readonly retryAfterMs: number;
+
+  constructor(input: {
+    action: RuntimeApiRequest['action'];
+    status: number;
+    code: string;
+    requestId?: string | null;
+    detail: string;
+    retryAfterMs: number;
+  }) {
+    super({
+      action: input.action,
+      status: input.status,
+      code: input.code,
+      requestId: input.requestId,
+      retryable: true,
+      detail: input.detail,
+    });
+    this.name = 'AppRuntimeApiCapacityError';
+    this.retryAfterMs = Math.max(0, Math.floor(input.retryAfterMs));
+  }
+}
+
+export function isAppRuntimeApiCapacityError(
+  error: unknown,
+): error is AppRuntimeApiCapacityError {
+  if (error instanceof AppRuntimeApiCapacityError) return true;
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as {
+    name?: unknown;
+    code?: unknown;
+    retryAfterMs?: unknown;
+  };
+  return (
+    candidate.name === 'AppRuntimeApiCapacityError' &&
+    typeof candidate.code === 'string' &&
+    APP_RUNTIME_CAPACITY_ERROR_CODES.has(candidate.code) &&
+    typeof candidate.retryAfterMs === 'number'
+  );
+}
+
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
   return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
@@ -1012,13 +1082,15 @@ async function postAppRuntimeApi<TResponse>(
     throw new Error('Worker runtime API requires executorToken.');
   }
   const runtimeFetch = context.fetch ?? fetch;
+  const boundaryLabel = runtimeApiBoundaryLabel(context);
   const vercelHeaders = await vercelProtectionBypassHeaders({
     baseUrl,
     token: context.vercelProtectionBypassToken,
     fetchImpl: runtimeFetch,
   });
 
-  const maxAttempts = appRuntimeMaxAttempts(body.action);
+  const maxAttempts =
+    context.retryPolicy === 'none' ? 1 : appRuntimeMaxAttempts(body.action);
   const retryTelemetry: AppRuntimeRetryTelemetry = {
     enabled: isRuntimeStepReceiptAction(body.action),
     startedAt: Date.now(),
@@ -1052,7 +1124,12 @@ async function postAppRuntimeApi<TResponse>(
             : {}),
         },
         body: JSON.stringify(body),
-        signal: AbortSignal.timeout(requestTimeoutMs),
+        signal: context.signal
+          ? AbortSignal.any([
+              context.signal,
+              AbortSignal.timeout(requestTimeoutMs),
+            ])
+          : AbortSignal.timeout(requestTimeoutMs),
       });
     } catch (error) {
       if (
@@ -1091,6 +1168,7 @@ async function postAppRuntimeApi<TResponse>(
         action: body.action,
         attempts: attempt,
         cause: error,
+        boundaryLabel,
       });
     }
     if (response.ok) {
@@ -1114,6 +1192,7 @@ async function postAppRuntimeApi<TResponse>(
           telemetry: retryTelemetry,
           attemptStartedAt,
           httpStatus: response.status,
+          boundaryLabel,
         });
         continue;
       }
@@ -1143,7 +1222,7 @@ async function postAppRuntimeApi<TResponse>(
           finalAttemptStartedAt: attemptStartedAt,
         });
         throw new Error(
-          `App runtime API ${body.action} failed with status ${response.status}: response body timed out`,
+          `${boundaryLabel} ${body.action} failed with status ${response.status}: response body timed out`,
           { cause: error },
         );
       }
@@ -1156,9 +1235,25 @@ async function postAppRuntimeApi<TResponse>(
         attemptStartedAt,
         httpStatus: response.status,
         retryAfterMs: appRuntimeRetryAfterMs(response, ''),
+        boundaryLabel,
       });
       continue;
     }
+    const responseCode = appRuntimeErrorCode(response, responseText);
+    if (
+      response.status === 503 &&
+      responseCode &&
+      APP_RUNTIME_CAPACITY_ERROR_CODES.has(responseCode)
+    ) {
+      throw new AppRuntimeApiCapacityError({
+        action: body.action,
+        status: response.status,
+        code: responseCode,
+        requestId: response.headers.get('x-deepline-request-id')?.trim(),
+        detail: summarizeAppRuntimeErrorBody(responseText),
+        retryAfterMs: appRuntimeRetryAfterMs(response, responseText) ?? 1_000,
+      });
+    }
     if (
       attempt < maxAttempts &&
       isRetryableAppRuntimeResponse({
@@ -1204,7 +1299,7 @@ async function postAppRuntimeApi<TResponse>(
       maxAttempts,
       finalAttemptStartedAt: attemptStartedAt,
     });
-    const code = appRuntimeErrorCode(response, responseText);
+    const code = responseCode;
     const requestId = response.headers.get('x-deepline-request-id')?.trim();
     throw new AppRuntimeApiResponseError({
       action: body.action,
@@ -1221,10 +1316,11 @@ async function postAppRuntimeApi<TResponse>(
           body: responseText,
         }),
       detail: summarizeAppRuntimeErrorBody(responseText),
+      boundaryLabel,
     });
   }
 
-  throw new Error(`App runtime API ${body.action} failed after retries.`);
+  throw new Error(`${boundaryLabel} ${body.action} failed after retries.`);
 }
 
 type SignedR2ReadUrlResponse = {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -2052,30 +2052,6 @@ export class PlayContextImpl {
       byLeaseId.set(target.leaseId, keys);
     }
 
-    if (
-      this.#options.markRuntimeStepReceiptsRunning ||
-      this.#options.markRuntimeStepReceiptRunning
-    ) {
-      const receipts = await this.markRuntimeStepReceiptsRunning(
-        targets.map((target) => ({
-          key: target.receiptKey,
-          runId: this.currentReceiptOwnerRunId,
-          leaseId: target.leaseId,
-        })),
-      );
-      for (const target of targets) {
-        const receipt = receipts.get(target.receiptKey);
-        if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
-          throw new RuntimeReceiptLeaseLostError({
-            receiptKey: target.receiptKey,
-            runId: this.currentReceiptOwnerRunId,
-            leaseId: target.leaseId,
-          });
-        }
-      }
-      return;
-    }
-
     if (this.#options.heartbeatRuntimeStepReceipts) {
       for (const [leaseId, keys] of byLeaseId) {
         const receipts = await this.#options.heartbeatRuntimeStepReceipts({
@@ -5429,7 +5405,7 @@ export class PlayContextImpl {
       let activeFieldName: string | null = null;
 
       // Global row slot keeps concurrent maps in the same run under rowMax. The
-      // worker pool above enforces this map's requested/default concurrency.
+      // worker pool below enforces this map's requested/default concurrency.
       const globalRowSlot = await this.resourceGovernor.acquireRow({
         estimatedBytes: runtimeMapJsonByteLength(baseRow),
       });
@@ -6602,8 +6578,9 @@ export class PlayContextImpl {
             : physicalDirectKey,
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
           ...(directReceiptLeaseId &&
-          (this.#options.markRuntimeStepReceiptRunning ||
-            this.#options.markRuntimeStepReceiptsRunning)
+          (this.#options.heartbeatRuntimeStepReceipts ||
+            this.#options.getRuntimeStepReceipt ||
+            this.#options.getRuntimeStepReceipts)
             ? {
                 beforeProviderCall: async () => {
                   await this.assertRuntimeToolReceiptOwnership([
@@ -6618,19 +6595,6 @@ export class PlayContextImpl {
                     },
                   ]);
                 },
-                ...(this.#options.markRuntimeStepReceiptsQueued
-                  ? {
-                      parkProviderCall: async () => {
-                        await this.markRuntimeStepReceiptsQueued([
-                          {
-                            key: directCacheKey,
-                            runId: this.currentReceiptOwnerRunId,
-                            leaseId: directReceiptLeaseId,
-                          },
-                        ]);
-                      },
-                    }
-                  : {}),
               }
             : {}),
         });
@@ -8016,8 +7980,6 @@ export class PlayContextImpl {
                   {
                     beforeProviderCall: () =>
                       this.assertRuntimeToolReceiptOwnership([owner]),
-                    parkProviderCall: () =>
-                      this.parkRuntimeToolReceiptsQueued([owner]),
                     durableCallReceiptKey: receiptKey,
                     executionAuthScopeDigest: owner.executionAuthScopeDigest,
                     receiptLeaseExpiresAt: owner.receiptLeaseExpiresAt,
@@ -8298,8 +8260,6 @@ export class PlayContextImpl {
                       this.assertRuntimeToolReceiptOwnership(
                         batch.memberRequests,
                       ),
-                    parkProviderCall: () =>
-                      this.parkRuntimeToolReceiptsQueued(batch.memberRequests),
                   },
                 );
               },
@@ -8447,8 +8407,6 @@ export class PlayContextImpl {
                   {
                     beforeProviderCall: () =>
                       this.assertRuntimeToolReceiptOwnership([request]),
-                    parkProviderCall: () =>
-                      this.parkRuntimeToolReceiptsQueued([request]),
                     ...(request.receiptKey
                       ? {
                           durableCallReceiptKey: request.receiptKey,

package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts

@@ -0,0 +1,159 @@
+export type GatewayPostgresLane = 'ordinary' | 'commit';
+
+export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
+export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
+
+export type GatewayPostgresAdmissionSnapshot = {
+  active: Record<GatewayPostgresLane, number> & { total: number };
+  waiting: Record<GatewayPostgresLane, number>;
+  cancelled: number;
+  commitPriorityGrants: number;
+};
+
+export class GatewayPostgresAdmissionCancelledError extends Error {
+  constructor() {
+    super('Gateway Postgres admission was cancelled.');
+    this.name = 'GatewayPostgresAdmissionCancelledError';
+  }
+}
+
+type Waiter = {
+  lane: GatewayPostgresLane;
+  signal: AbortSignal | null;
+  onAbort: (() => void) | null;
+  resolve: (release: () => void) => void;
+  reject: (error: Error) => void;
+  settled: boolean;
+};
+
+export class GatewayPostgresAdmission {
+  readonly #waiting: Record<GatewayPostgresLane, Waiter[]> = {
+    ordinary: [],
+    commit: [],
+  };
+  #ordinaryActive = 0;
+  #commitActive = 0;
+  #cancelled = 0;
+  #commitPriorityGrants = 0;
+
+  async acquire(
+    lane: GatewayPostgresLane,
+    options: { signal?: AbortSignal } = {},
+  ): Promise<() => void> {
+    if (options.signal?.aborted) {
+      this.#cancelled += 1;
+      throw new GatewayPostgresAdmissionCancelledError();
+    }
+    if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
+      return this.#grant(lane);
+    }
+
+    return await new Promise<() => void>((resolve, reject) => {
+      const waiter: Waiter = {
+        lane,
+        signal: options.signal ?? null,
+        onAbort: null,
+        resolve,
+        reject,
+        settled: false,
+      };
+      waiter.onAbort = () => {
+        if (!this.#remove(waiter)) return;
+        this.#cancelled += 1;
+        reject(new GatewayPostgresAdmissionCancelledError());
+      };
+      waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
+      this.#waiting[lane].push(waiter);
+      if (waiter.signal?.aborted) {
+        waiter.onAbort();
+        return;
+      }
+      this.#drain();
+    });
+  }
+
+  snapshot(): GatewayPostgresAdmissionSnapshot {
+    return {
+      active: {
+        ordinary: this.#ordinaryActive,
+        commit: this.#commitActive,
+        total: this.#activeTotal(),
+      },
+      waiting: {
+        ordinary: this.#waiting.ordinary.length,
+        commit: this.#waiting.commit.length,
+      },
+      cancelled: this.#cancelled,
+      commitPriorityGrants: this.#commitPriorityGrants,
+    };
+  }
+
+  #activeTotal(): number {
+    return this.#ordinaryActive + this.#commitActive;
+  }
+
+  #canGrant(lane: GatewayPostgresLane): boolean {
+    if (this.#activeTotal() >= GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) return false;
+    return (
+      lane === 'commit' ||
+      this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE
+    );
+  }
+
+  #grant(lane: GatewayPostgresLane): () => void {
+    if (lane === 'ordinary') this.#ordinaryActive += 1;
+    else this.#commitActive += 1;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      if (lane === 'ordinary') this.#ordinaryActive -= 1;
+      else this.#commitActive -= 1;
+      this.#drain();
+    };
+  }
+
+  #drain(): void {
+    while (this.#activeTotal() < GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) {
+      const commit = this.#waiting.commit.shift();
+      if (commit) {
+        if (this.#waiting.ordinary.length > 0) {
+          this.#commitPriorityGrants += 1;
+        }
+        this.#resolve(commit);
+        continue;
+      }
+      if (
+        this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE &&
+        this.#waiting.ordinary.length > 0
+      ) {
+        this.#resolve(this.#waiting.ordinary.shift()!);
+        continue;
+      }
+      return;
+    }
+  }
+
+  #resolve(waiter: Waiter): void {
+    waiter.settled = true;
+    this.#cleanup(waiter);
+    waiter.resolve(this.#grant(waiter.lane));
+  }
+
+  #remove(waiter: Waiter): boolean {
+    if (waiter.settled) return false;
+    const queue = this.#waiting[waiter.lane];
+    const index = queue.indexOf(waiter);
+    if (index < 0) return false;
+    waiter.settled = true;
+    queue.splice(index, 1);
+    this.#cleanup(waiter);
+    return true;
+  }
+
+  #cleanup(waiter: Waiter): void {
+    if (waiter.signal && waiter.onAbort) {
+      waiter.signal.removeEventListener('abort', waiter.onAbort);
+    }
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -193,7 +193,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     // Duration alone decides. Neither hold is over threshold → let it sleep.
     if (
       coolWait <= PROVIDER_EXHAUSTED_MAX_WAIT_MS &&
-      !(claimActive && cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS)
+      !(
+        claimActive &&
+        cooldown.claimedRetryAtMs - now > PROVIDER_EXHAUSTED_MAX_WAIT_MS
+      )
     ) {
       return;
     }
@@ -273,20 +276,19 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   }
 
   /**
-   * How many permits to request on the next refill for a bucket. Buckets with any
-   * `maxConcurrency` rule stay at the fixed floor block: each permit pins a
-   * durable concurrency slot, so over-reserving would strand real slots. Pure-rate
-   * buckets demand-size: clamp the live waiter count to
-   * [block floor, window budget] where the window budget is the smallest
-   * `requestsPerWindow` across the pure-rate rules (never reserve more than one
-   * window can hold). A 200-row wave then lands in ~one round trip.
+   * Size the next lease from live demand and the provider's real capacity.
+   *
+   * Pure-rate buckets retain a small prefetch floor. Concurrency-bearing
+   * buckets do not: each unused permit pins a durable provider slot, so request
+   * only the live waiter count, capped by both the rate horizon and declared
+   * max concurrency. This lets a 30-concurrent provider fill in one round trip
+   * without reserving a single slot that cannot be used.
    */
   private requestedForBucket(
     bucketId: string,
     rules: readonly PacingRule[],
   ): number {
     const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
-    if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
     const configuredRps = Math.min(
       ...rules.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000),
     );
@@ -294,15 +296,28 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       1,
       Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
     );
-    const windowBudget = Math.min(
+    const rateBudget = Math.min(
       horizonBudget,
       ...rules
         .map((rule) => rule.requestsPerWindow)
         .filter((value) => Number.isFinite(value) && value > 0),
     );
-    if (!Number.isFinite(windowBudget) || windowBudget <= 0) return floor;
+    if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
     const waiters = this.waiters.get(bucketId) ?? 1;
-    return Math.max(floor, Math.min(waiters, Math.trunc(windowBudget)));
+    const concurrencyLimits = rules.flatMap((rule) =>
+      rule.maxConcurrency != null &&
+      Number.isFinite(rule.maxConcurrency) &&
+      rule.maxConcurrency > 0
+        ? [Math.trunc(rule.maxConcurrency)]
+        : [],
+    );
+    if (concurrencyLimits.length > 0) {
+      return Math.max(
+        1,
+        Math.min(waiters, Math.trunc(rateBudget), ...concurrencyLimits),
+      );
+    }
+    return Math.max(floor, Math.min(waiters, Math.trunc(rateBudget)));
   }
 
   /**
@@ -344,7 +359,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    */
   private providerExhaustedFromResponse(
     bucketId: string,
-    response: { waitMs: number; coolUntilMs?: number; claimedRetryAtMs?: number },
+    response: {
+      waitMs: number;
+      coolUntilMs?: number;
+      claimedRetryAtMs?: number;
+    },
   ): ProviderExhaustedError {
     const claimedRetryAtMs = Number(response.claimedRetryAtMs);
     const coolUntilMs = Number(response.coolUntilMs);
@@ -507,6 +526,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     response: {
       granted: number;
       leaseIds?: unknown;
+      serverNowMs?: unknown;
       scheduledAtMs?: unknown;
     },
   ): void {
@@ -531,9 +551,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     // has already debited the block under the legacy token-bucket contract, so
     // keep those permits immediately usable until every gateway serves the new
     // globally reserved timestamps.
-    const scheduledAtMs = Array.isArray(response.scheduledAtMs)
+    const gatewayScheduledAtMs = Array.isArray(response.scheduledAtMs)
       ? response.scheduledAtMs.map(Number)
       : Array.from({ length: response.granted }, () => this.now());
+    const serverNowMs = Number(response.serverNowMs);
+    const receivedAtMs = this.now();
+    const scheduledAtMs =
+      Number.isFinite(serverNowMs) && serverNowMs >= 0
+        ? gatewayScheduledAtMs.map(
+            (scheduledAtMs) =>
+              receivedAtMs + Math.max(0, scheduledAtMs - serverNowMs),
+          )
+        : gatewayScheduledAtMs;
     if (
       scheduledAtMs.length !== response.granted ||
       scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
@@ -545,12 +574,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     this.mergeBlock(
       bucketId,
       rateScopeToken,
-      scheduledAtMs.map(
-        (scheduledAtMs, index): [number, string | null] => [
-          scheduledAtMs,
-          hasConcurrency ? (leaseIds[index] ?? null) : null,
-        ],
-      ),
+      scheduledAtMs.map((scheduledAtMs, index): [number, string | null] => [
+        scheduledAtMs,
+        hasConcurrency ? (leaseIds[index] ?? null) : null,
+      ]),
       rulesKey,
       ordered,
     );
@@ -639,9 +666,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         bucketId,
         existing.rateScopeToken,
         existing.rules,
-        existing.permits.flatMap(([, leaseId]) =>
-          leaseId ? [leaseId] : [],
-        ),
+        existing.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
       );
     this.blocks.set(bucketId, {
       permits,
@@ -668,9 +693,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         bucketId,
         block.rateScopeToken,
         block.rules,
-        block.permits.flatMap(([, leaseId]) =>
-          leaseId ? [leaseId] : [],
-        ),
+        block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
       );
       return null;
     }
@@ -699,9 +722,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       bucketId,
       block.rateScopeToken,
       block.rules,
-      block.permits.flatMap(([, leaseId]) =>
-        leaseId ? [leaseId] : [],
-      ),
+      block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
     );
   }
 

package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts

@@ -1,3 +1,5 @@
+import { isAppRuntimeApiCapacityError } from './app-runtime-api';
+
 /**
  * Runtime persistence-failure circuit breaker.
  *
@@ -113,6 +115,7 @@ export function tripRuntimePersistenceLatch(
   latch: RuntimePersistenceLatch,
   error: unknown,
 ): void {
+  if (isAppRuntimeApiCapacityError(error)) return;
   if (latch.tripped) return;
   latch.tripped = true;
   latch.cause = formatLatchCause(error);

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -36,6 +36,12 @@ export type PlayRunnerRateStateAcquireInput = {
 export type PlayRunnerRateStateAcquireResult = {
   granted: number;
   waitMs: number;
+  /**
+   * Gateway clock used to produce `scheduledAtMs`. Runners translate the
+   * schedule into relative delays so cross-machine clock skew cannot compress
+   * provider spacing.
+   */
+  serverNowMs?: number;
   /**
    * One globally reserved dispatch timestamp per granted permit. The runtime
    * may cache the block, but must not dispatch a permit before its timestamp.