deepline 0.1.293 → 0.1.295

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.293',
+  version: '0.1.295',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -24,7 +24,6 @@ import type {
   PlayVisualNodeProgressMap,
 } from '@shared_libs/play-runtime/worker-api-types';
 import type { RuntimeStepReceipt } from '@shared_libs/play-runtime/ctx-types';
-import type { WorkReceiptFailureKind } from '@shared_libs/play-runtime/work-receipts';
 import type {
   CreateDbSessionRequest,
   CreateDbSessionResponse,
@@ -45,6 +44,7 @@ import {
 import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
 import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
 import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
+import type { RuntimeReceiptAction } from '@shared_libs/play-runtime/runtime-actions';
 
 export type StoredPlayArtifactPayload = {
   sourceCode: string;
@@ -204,156 +204,7 @@ type RuntimeApiRequest =
   | ({
       action: 'governor_budget_charge';
     } & PlayRunnerBudgetChargeInput)
-  | {
-      action: 'get_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-    }
-  | {
-      action: 'acquire_runtime_receipt_execution_lock';
-      playName: string;
-      runId: string;
-      key: string;
-      ownerExecutionId: string;
-      ttlMs: number;
-    }
-  | {
-      action: 'release_runtime_receipt_execution_lock';
-      playName: string;
-      runId: string;
-      key: string;
-      ownerExecutionId: string;
-    }
-  | {
-      action: 'claim_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      leaseId?: string;
-      runAttempt?: number | null;
-      leaseAware?: boolean;
-      reclaimRunning?: boolean;
-      forceRefresh?: boolean;
-      forceFailedRefresh?: boolean;
-    }
-  | {
-      action: 'complete_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      runAttempt?: number | null;
-      leaseId?: string | null;
-      output: unknown;
-    }
-  | {
-      action: 'release_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      runAttempt?: number | null;
-      leaseId?: string | null;
-    }
-  | {
-      action: 'fail_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      runAttempt?: number | null;
-      leaseId?: string | null;
-      error: string;
-      failureKind?: WorkReceiptFailureKind | null;
-    }
-  | {
-      action: 'skip_runtime_step_receipt';
-      playName: string;
-      runId: string;
-      key: string;
-      runAttempt?: number | null;
-      leaseId?: string | null;
-      output?: unknown;
-    }
-  | {
-      action: 'get_runtime_step_receipts';
-      playName: string;
-      runId: string;
-      keys: string[];
-    }
-  | {
-      action: 'claim_runtime_step_receipts';
-      playName: string;
-      runId: string;
-      keys: string[];
-      leaseIds?: string[];
-      runAttempt?: number | null;
-      leaseAware?: boolean;
-      reclaimRunning?: boolean;
-      forceRefresh?: boolean;
-      forceFailedRefresh?: boolean;
-    }
-  | {
-      action: 'mark_runtime_step_receipt_running';
-      playName: string;
-      runId: string;
-      key: string;
-      runAttempt?: number | null;
-      leaseId?: string | null;
-    }
-  | {
-      action: 'mark_runtime_step_receipts_running';
-      playName: string;
-      runId: string;
-      receipts: Array<{
-        key: string;
-        runId: string;
-        runAttempt?: number | null;
-        leaseId?: string | null;
-      }>;
-    }
-  | {
-      action: 'mark_runtime_step_receipts_queued';
-      playName: string;
-      runId: string;
-      receipts: Array<{
-        key: string;
-        runId: string;
-        runAttempt?: number | null;
-        leaseId?: string | null;
-      }>;
-    }
-  | {
-      action: 'complete_runtime_step_receipts';
-      playName: string;
-      runId: string;
-      receipts: Array<{
-        key: string;
-        runId: string;
-        runAttempt?: number | null;
-        leaseId?: string | null;
-        output: unknown;
-      }>;
-    }
-  | {
-      action: 'fail_runtime_step_receipts';
-      playName: string;
-      runId: string;
-      receipts: Array<{
-        key: string;
-        runId: string;
-        runAttempt?: number | null;
-        leaseId?: string | null;
-        error: string;
-        failureKind?: WorkReceiptFailureKind | null;
-      }>;
-    }
-  | {
-      action: 'heartbeat_runtime_step_receipts';
-      playName: string;
-      runId: string;
-      runAttempt?: number | null;
-      leaseId: string;
-      keys: string[];
-    };
+  | RuntimeReceiptAction;
 
 export type WorkerRuntimeApiContext = {
   baseUrl: string;

package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts

@@ -2,6 +2,7 @@ export type GatewayPostgresLane = 'ordinary' | 'commit';
 
 export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
 export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
+export const GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS = 8;
 
 export type GatewayPostgresAdmissionSnapshot = {
   active: Record<GatewayPostgresLane, number> & { total: number };
@@ -35,6 +36,7 @@ export class GatewayPostgresAdmission {
   #commitActive = 0;
   #cancelled = 0;
   #commitPriorityGrants = 0;
+  #consecutiveCommitPriorityGrants = 0;
 
   async acquire(
     lane: GatewayPostgresLane,
@@ -115,21 +117,38 @@ export class GatewayPostgresAdmission {
 
   #drain(): void {
     while (this.#activeTotal() < GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE) {
+      const ordinaryCanRun =
+        this.#waiting.ordinary.length > 0 &&
+        this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE;
+      if (
+        ordinaryCanRun &&
+        this.#waiting.commit.length > 0 &&
+        this.#consecutiveCommitPriorityGrants >=
+          GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS
+      ) {
+        this.#consecutiveCommitPriorityGrants = 0;
+        this.#resolve(this.#waiting.ordinary.shift()!);
+        continue;
+      }
       const commit = this.#waiting.commit.shift();
       if (commit) {
-        if (this.#waiting.ordinary.length > 0) {
+        if (ordinaryCanRun) {
           this.#commitPriorityGrants += 1;
+          this.#consecutiveCommitPriorityGrants += 1;
+        } else {
+          this.#consecutiveCommitPriorityGrants = 0;
         }
         this.#resolve(commit);
         continue;
       }
-      if (
-        this.#ordinaryActive < GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE &&
-        this.#waiting.ordinary.length > 0
-      ) {
+      if (ordinaryCanRun) {
+        this.#consecutiveCommitPriorityGrants = 0;
         this.#resolve(this.#waiting.ordinary.shift()!);
         continue;
       }
+      if (this.#waiting.ordinary.length === 0) {
+        this.#consecutiveCommitPriorityGrants = 0;
+      }
       return;
     }
   }

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -228,13 +228,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
     const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
     const rulesKey = rulesSignature(ordered);
+    const laneKey = rateLaneKey(bucketId, rulesKey);
 
     // Drawing a cached permit is a synchronous, single-threaded critical
     // section (`drawFromBlock` has no `await`), so concurrent acquires can never
     // hand out the same permit — no lock is needed for the fast path. The ONLY
     // thing that needs coordination is the block refill, which we single-flight
-    // per bucket: one network round-trip (and any window-exhausted sleep) serves
-    // the whole waiting wave, then everyone re-draws. This replaces the old
+    // per bucket + rule set: one network round-trip (and any window-exhausted
+    // sleep) serves the whole matching wave, then everyone re-draws. This replaces the old
     // per-bucket lock that held across the RPC + up-to-5s sleep and thereby
     // serialized every map row behind one sleeping acquirer.
     while (true) {
@@ -243,7 +244,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           ? signal.reason
           : new Error('Rate-state acquire aborted.');
       }
-      const localPermit = this.drawFromBlock(bucketId, rulesKey);
+      const localPermit = this.drawFromBlock(bucketId, laneKey);
       if (localPermit) {
         const waitMs = localPermit.scheduledAtMs - this.now();
         if (waitMs > 0) await this.sleep(waitMs);
@@ -258,19 +259,20 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       // Count this acquirer as waiting on the refill so the single-flight refill
       // can size its block to the live wave. Decremented once a block lands (or
       // the refill errors) — see the finally in `refillBlock`.
-      this.waiters.set(bucketId, (this.waiters.get(bucketId) ?? 0) + 1);
+      this.waiters.set(laneKey, (this.waiters.get(laneKey) ?? 0) + 1);
       try {
         await this.refillBlock(
           bucketId,
           input.rateScopeToken?.trim() || null,
           ordered,
           rulesKey,
+          laneKey,
           signal,
         );
       } finally {
-        const remaining = (this.waiters.get(bucketId) ?? 1) - 1;
-        if (remaining > 0) this.waiters.set(bucketId, remaining);
-        else this.waiters.delete(bucketId);
+        const remaining = (this.waiters.get(laneKey) ?? 1) - 1;
+        if (remaining > 0) this.waiters.set(laneKey, remaining);
+        else this.waiters.delete(laneKey);
       }
     }
   }
@@ -285,7 +287,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    * without reserving a single slot that cannot be used.
    */
   private requestedForBucket(
-    bucketId: string,
+    laneKey: string,
     rules: readonly PacingRule[],
   ): number {
     const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
@@ -294,7 +296,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     );
     const horizonBudget = Math.max(
       1,
-      Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
+      Math.ceil(this.effectiveRps.get(laneKey) ?? configuredRps),
     );
     const rateBudget = Math.min(
       horizonBudget,
@@ -303,7 +305,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         .filter((value) => Number.isFinite(value) && value > 0),
     );
     if (!Number.isFinite(rateBudget) || rateBudget <= 0) return floor;
-    const waiters = this.waiters.get(bucketId) ?? 1;
+    const waiters = this.waiters.get(laneKey) ?? 1;
     const concurrencyLimits = rules.flatMap((rule) =>
       rule.maxConcurrency != null &&
       Number.isFinite(rule.maxConcurrency) &&
@@ -391,22 +393,24 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
+    laneKey: string,
     signal?: AbortSignal,
   ): Promise<void> {
-    const existing = this.refills.get(bucketId);
+    const existing = this.refills.get(laneKey);
     if (existing) return existing;
     const promise = this.performRefill(
       bucketId,
       rateScopeToken,
       ordered,
       rulesKey,
+      laneKey,
       signal,
     ).finally(() => {
-      if (this.refills.get(bucketId) === promise) {
-        this.refills.delete(bucketId);
+      if (this.refills.get(laneKey) === promise) {
+        this.refills.delete(laneKey);
       }
     });
-    this.refills.set(bucketId, promise);
+    this.refills.set(laneKey, promise);
     return promise;
   }
 
@@ -415,6 +419,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
+    laneKey: string,
     signal?: AbortSignal,
   ): Promise<void> {
     let consecutiveFailures = 0;
@@ -435,24 +440,25 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         if (signal?.aborted) return;
       }
       try {
-        const observedSuccesses = this.pendingSuccesses.get(bucketId) ?? 0;
+        const successKey = pendingSuccessKey(bucketId, ordered);
+        const observedSuccesses = this.pendingSuccesses.get(successKey) ?? 0;
         const response = await this.port.acquire({
           bucketId,
           rateScopeToken,
           // Stamp adaptiveMaxRps on default-pacing rules so the DB row's ceiling
           // matches the old in-memory AIMD range.
           rules: withAdaptiveMaxRps(ordered),
-          requested: this.requestedForBucket(bucketId, ordered),
+          requested: this.requestedForBucket(laneKey, ordered),
           observedSuccesses,
           schedulerSchema: this.schedulerSchema,
         });
         if (observedSuccesses > 0) {
           const remaining = Math.max(
             0,
-            (this.pendingSuccesses.get(bucketId) ?? 0) - observedSuccesses,
+            (this.pendingSuccesses.get(successKey) ?? 0) - observedSuccesses,
           );
-          if (remaining > 0) this.pendingSuccesses.set(bucketId, remaining);
-          else this.pendingSuccesses.delete(bucketId);
+          if (remaining > 0) this.pendingSuccesses.set(successKey, remaining);
+          else this.pendingSuccesses.delete(successKey);
         }
         this.recordCooldown(bucketId, response);
         if (
@@ -460,7 +466,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           Number(response.effectiveRequestsPerSecond) > 0
         ) {
           this.effectiveRps.set(
-            bucketId,
+            laneKey,
             Number(response.effectiveRequestsPerSecond),
           );
         }
@@ -472,6 +478,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
             rateScopeToken,
             ordered,
             rulesKey,
+            laneKey,
             response,
           );
           return;
@@ -523,6 +530,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
+    laneKey: string,
     response: {
       granted: number;
       leaseIds?: unknown;
@@ -579,6 +587,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         hasConcurrency ? (leaseIds[index] ?? null) : null,
       ]),
       rulesKey,
+      laneKey,
       ordered,
     );
   }
@@ -589,7 +598,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     cooldownMs: number;
   }): Promise<void> {
     if (input.cooldownMs <= 0) return;
-    this.drainBlock(input.bucketId);
+    this.drainBlocksForBucket(input.bucketId);
     try {
       await this.port.penalize({
         bucketId: input.bucketId,
@@ -609,10 +618,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     }
   }
 
-  observeSuccess(input: { bucketId: string }): void {
+  observeSuccess(input: {
+    bucketId: string;
+    rules: readonly PacingRule[];
+  }): void {
+    const key = pendingSuccessKey(input.bucketId, input.rules);
     this.pendingSuccesses.set(
-      input.bucketId,
-      Math.min(1_000_000, (this.pendingSuccesses.get(input.bucketId) ?? 0) + 1),
+      key,
+      Math.min(1_000_000, (this.pendingSuccesses.get(key) ?? 0) + 1),
     );
   }
 
@@ -645,12 +658,13 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     rateScopeToken: string | null,
     permits: LeasedBlock['permits'],
     rulesKey: string,
+    laneKey: string,
     rules: PacingRule[],
   ): void {
     const lastScheduledAtMs = permits.at(-1)?.[0] ?? this.now();
     const freshExpiresAt =
       Math.max(this.now(), lastScheduledAtMs) + leasedBlockTtlMs(rules);
-    const existing = this.blocks.get(bucketId);
+    const existing = this.blocks.get(laneKey);
     if (
       existing &&
       existing.rulesKey === rulesKey &&
@@ -668,7 +682,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         existing.rules,
         existing.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
       );
-    this.blocks.set(bucketId, {
+    this.blocks.set(laneKey, {
       permits,
       rateScopeToken,
       expiresAt: freshExpiresAt,
@@ -679,12 +693,12 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
   private drawFromBlock(
     bucketId: string,
-    rulesKey: string,
+    laneKey: string,
   ): { permit: PacingPermit; scheduledAtMs: number } | null {
-    const block = this.blocks.get(bucketId);
+    const block = this.blocks.get(laneKey);
     if (!block) return null;
-    if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
-      this.blocks.delete(bucketId);
+    if (block.expiresAt <= this.now()) {
+      this.blocks.delete(laneKey);
       // Unused window permits are forfeited on TTL expiry (the server already
       // decremented the provider window for them, and the rule-scoped TTL is
       // capped so at most one block's worth is stranded per bucket/window).
@@ -699,10 +713,10 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     }
     const scheduled = block.permits.shift();
     if (!scheduled) {
-      this.blocks.delete(bucketId);
+      this.blocks.delete(laneKey);
       return null;
     }
-    if (block.permits.length === 0) this.blocks.delete(bucketId);
+    if (block.permits.length === 0) this.blocks.delete(laneKey);
     return {
       permit: this.permitFor(
         bucketId,
@@ -714,16 +728,18 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     };
   }
 
-  private drainBlock(bucketId: string): void {
-    const block = this.blocks.get(bucketId);
-    if (!block) return;
-    this.blocks.delete(bucketId);
-    this.releaseReserved(
-      bucketId,
-      block.rateScopeToken,
-      block.rules,
-      block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
-    );
+  private drainBlocksForBucket(bucketId: string): void {
+    const prefix = `${bucketId}\0`;
+    for (const [laneKey, block] of this.blocks) {
+      if (!laneKey.startsWith(prefix)) continue;
+      this.blocks.delete(laneKey);
+      this.releaseReserved(
+        bucketId,
+        block.rateScopeToken,
+        block.rules,
+        block.permits.flatMap(([, leaseId]) => (leaseId ? [leaseId] : [])),
+      );
+    }
   }
 
   private releaseReserved(
@@ -814,20 +830,22 @@ function isDefaultPacingRuleId(ruleId: string): boolean {
 }
 
 /**
- * Stamp `adaptiveMaxRps` onto default-pacing rules so the DB row's ceiling matches
- * the old in-memory AIMD range (base * 25). Declared rules pass through untouched
- * (the DB pins them at base). The field is extra on the wire; the server's
- * `normalizeRules` reads it, everyone else ignores it.
+ * Preserve explicit advisory ceilings and stamp the legacy base * 25 ceiling
+ * only onto undeclared default-pacing rules. Declared rules without a ceiling
+ * remain pinned at base.
  */
 function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
   return rules.map((rule) =>
-    isDefaultPacingRuleId(rule.ruleId)
-      ? ({
-          ...rule,
-          adaptiveMaxRps:
-            rule.requestsPerWindow * DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER,
-        } as PacingRule)
-      : rule,
+    rule.adaptiveMaxRps != null
+      ? rule
+      : isDefaultPacingRuleId(rule.ruleId)
+        ? ({
+            ...rule,
+            adaptiveMaxRps:
+              rule.requestsPerWindow *
+              DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER,
+          } as PacingRule)
+        : rule,
   );
 }
 
@@ -835,12 +853,23 @@ function rulesSignature(rules: readonly PacingRule[]): string {
   return [...rules]
     .map(
       (rule) =>
-        `${rule.ruleId}:${rule.requestsPerWindow}:${rule.windowMs}:${rule.maxConcurrency ?? ''}`,
+        `${rule.ruleId}:${rule.requestsPerWindow}:${rule.windowMs}:${rule.adaptiveMaxRps ?? ''}:${rule.maxConcurrency ?? ''}`,
     )
     .sort()
     .join('|');
 }
 
+function rateLaneKey(bucketId: string, rulesKey: string): string {
+  return `${bucketId}\0${rulesKey}`;
+}
+
+function pendingSuccessKey(
+  bucketId: string,
+  rules: readonly PacingRule[],
+): string {
+  return rateLaneKey(bucketId, rulesSignature(rules));
+}
+
 function leasedBlockTtlMs(rules: readonly PacingRule[]): number {
   const shortestWindowMs = Math.min(
     ...rules

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -267,6 +267,7 @@ export function createPlayExecutionGovernor(
     });
   const budgetState = input.budgetState ?? new InMemoryBudgetStateBackend();
   const providerByTool = new Map<string, string>();
+  const pacingRulesByTool = new Map<string, PacingRule[]>();
   const scopeByTool = new Map<string, [string, string]>();
   const providerConcurrencySlotsByTool = new Map<string, Semaphore>();
 
@@ -308,6 +309,7 @@ export function createPlayExecutionGovernor(
       // row is authoritative.
       const pacing = declaredPacing ?? defaultPacingForTool(toolId, policy);
       providerByTool.set(toolId, pacing.provider);
+      pacingRulesByTool.set(toolId, pacing.rules);
       return pacing;
     }
     const pacing = adaptiveAdmission.resolvePacing({
@@ -317,6 +319,7 @@ export function createPlayExecutionGovernor(
       fallbackPacing: defaultPacingForTool(toolId, policy),
     });
     providerByTool.set(toolId, pacing.provider);
+    pacingRulesByTool.set(toolId, pacing.rules);
     return pacing;
   }
 
@@ -486,9 +489,13 @@ export function createPlayExecutionGovernor(
 
     observeProviderSuccess(success) {
       if (dbAuthoritativePacing) {
+        if (!success.toolId) return;
         const rateScope = scopeFor(success.toolId, success.provider);
+        const rules = pacingRulesByTool.get(success.toolId);
+        if (!rules) return;
         input.rateState.observeSuccess?.({
           bucketId: rateScope[0],
+          rules,
         });
         return;
       }

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -11,6 +11,8 @@ export interface PacingRule {
   readonly ruleId: string;
   readonly requestsPerWindow: number;
   readonly windowMs: number;
+  /** Optional learned-rate ceiling in requests per second. */
+  readonly adaptiveMaxRps?: number | null;
   /**
    * Optional simultaneous-in-flight cap for this rule.
    *
@@ -35,6 +37,7 @@ export interface PlayQueueHint {
   ruleId: string;
   requestsPerWindow: number;
   windowMs: number;
+  adaptiveMaxRps?: number | null;
   maxConcurrency: number | null;
 }
 
@@ -88,6 +91,7 @@ export interface RateStateBackend {
   observeSuccess?(input: {
     bucketId: string;
     rateScopeToken?: string | null;
+    rules: readonly PacingRule[];
   }): void;
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts

@@ -41,6 +41,12 @@ export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
  * is required to resume execution.
  */
 export const RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
+/**
+ * Receipt commands are measured before the runtime client adds its request
+ * envelope. Keep a full 4 MiB below the gateway body ceiling so one legal
+ * 10 MiB receipt fits while compatible fat receipts split before transport.
+ */
+export const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
 /**
  * The terminal `result` line is the crash-recovery record in Daytona stdout.
  * Once that line is written, retries may still emit runner-owned diagnostics.

package/dist/bundling-sources/shared_libs/play-runtime/pacing.ts

@@ -8,7 +8,12 @@ export type ResolvedPacingPolicy = {
 
 type PacingPolicyQueueHint = Pick<
   PlayQueueHint,
-  'provider' | 'ruleId' | 'requestsPerWindow' | 'windowMs' | 'maxConcurrency'
+  | 'provider'
+  | 'ruleId'
+  | 'requestsPerWindow'
+  | 'windowMs'
+  | 'adaptiveMaxRps'
+  | 'maxConcurrency'
 >;
 
 function isFinitePositiveNumber(value: unknown): value is number {
@@ -35,6 +40,9 @@ export function pacingPolicyFromQueueHints(
         ruleId: hint.ruleId,
         requestsPerWindow: hint.requestsPerWindow,
         windowMs: hint.windowMs,
+        adaptiveMaxRps: isFinitePositiveNumber(hint.adaptiveMaxRps)
+          ? hint.adaptiveMaxRps
+          : null,
         maxConcurrency:
           typeof hint.maxConcurrency === 'number' &&
           Number.isFinite(hint.maxConcurrency) &&
@@ -65,6 +73,8 @@ export function pacingPolicyFromUnknownQueueHints(
             : Number.NaN,
         windowMs:
           typeof hint.windowMs === 'number' ? hint.windowMs : Number.NaN,
+        adaptiveMaxRps:
+          typeof hint.adaptiveMaxRps === 'number' ? hint.adaptiveMaxRps : null,
         maxConcurrency:
           typeof hint.maxConcurrency === 'number' ? hint.maxConcurrency : null,
       })),

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -111,9 +111,26 @@ export async function inspectDetachedDaytonaStartup(input: {
  * verification.
  */
 
-export type DetachedDaytonaCommandStart = {
-  sessionId: string;
-  cmdId: string;
+export type DetachedDaytonaRunnerState =
+  | { phase: 'preparing'; sessionId: string }
+  | {
+      phase: 'command_accepted';
+      sessionId: string;
+      cmdId: string;
+      baselineHeartbeatAt: string | null;
+    }
+  | {
+      phase: 'ready';
+      sessionId: string;
+      cmdId: string;
+      baselineHeartbeatAt: string | null;
+    };
+
+export type DetachedDaytonaRunnerSupervisor = {
+  start(input: {
+    command: string;
+    exitCodePath: string;
+  }): Promise<Extract<DetachedDaytonaRunnerState, { phase: 'ready' }>>;
 };
 
 // The durable heartbeat proves runner-process liveness plus gateway reachability.
@@ -235,44 +252,82 @@ export class DaytonaRunnerInitializationError extends Error {
 }
 
 /**
- * Start `command` detached in a fresh session. Returns the session + command id
- * immediately (no await on completion).
+ * Begin the independent control-plane work needed by a detached runner while
+ * its payload uploads. `start` crosses the customer-code side-effect boundary:
+ * it accepts one command, then returns only after the scheduler has observed
+ * durable runner liveness.
  */
-export async function startDetachedDaytonaCommand(input: {
+export function prepareDetachedDaytonaRunner(input: {
   sandbox: DaytonaSandbox;
   sessionId: string;
-  command: string;
-}): Promise<DetachedDaytonaCommandStart> {
-  await createDetachedDaytonaSession(input);
-  return executeDetachedDaytonaSessionCommand(input);
-}
-
-export async function createDetachedDaytonaSession(input: {
-  sandbox: DaytonaSandbox;
-  sessionId: string;
-}): Promise<void> {
-  await input.sandbox.process.createSession(input.sessionId);
-}
+  readiness: DetachedRunnerReadinessPort;
+  cancellation?: Promise<never>;
+}): DetachedDaytonaRunnerSupervisor {
+  const cancellation = input.cancellation ?? new Promise<never>(() => {});
+  const sessionCreation = input.sandbox.process
+    .createSession(input.sessionId)
+    .then(
+      () => ({ ok: true as const }),
+      (error: unknown) => ({ ok: false as const, error }),
+    );
+  const readinessBaseline = captureDetachedDaytonaRunnerReadinessBaseline({
+    readiness: input.readiness,
+  }).then(
+    (heartbeatAt) => ({ ok: true as const, heartbeatAt }),
+    (error: unknown) => ({ ok: false as const, error }),
+  );
+  let state: DetachedDaytonaRunnerState = {
+    phase: 'preparing',
+    sessionId: input.sessionId,
+  };
+  let started = false;
 
-export async function executeDetachedDaytonaSessionCommand(input: {
-  sandbox: DaytonaSandbox;
-  sessionId: string;
-  command: string;
-}): Promise<DetachedDaytonaCommandStart> {
-  const response = await input.sandbox.process.executeSessionCommand(
-    input.sessionId,
-    {
-      command: input.command,
-      runAsync: true,
+  return {
+    async start({ command, exitCodePath }) {
+      if (started) {
+        throw new Error(
+          `Detached Daytona runner session ${input.sessionId} was already started.`,
+        );
+      }
+      started = true;
+      const session = await Promise.race([sessionCreation, cancellation]);
+      if (session.ok === false) throw session.error;
+      const baseline = await Promise.race([readinessBaseline, cancellation]);
+      if (baseline.ok === false) throw baseline.error;
+      const response = await Promise.race([
+        input.sandbox.process.executeSessionCommand(input.sessionId, {
+          command,
+          runAsync: true,
+        }),
+        cancellation,
+      ]);
+      const cmdId = response?.cmdId;
+      if (!cmdId) {
+        throw new Error(
+          `Daytona executeSessionCommand did not return a cmdId for session ${input.sessionId}.`,
+        );
+      }
+      state = {
+        phase: 'command_accepted',
+        sessionId: input.sessionId,
+        cmdId,
+        baselineHeartbeatAt: baseline.heartbeatAt,
+      };
+      await Promise.race([
+        confirmDetachedDaytonaRunnerReady({
+          sandbox: input.sandbox,
+          sessionId: state.sessionId,
+          cmdId: state.cmdId,
+          exitCodePath,
+          readiness: input.readiness,
+          baselineHeartbeatAt: state.baselineHeartbeatAt,
+        }),
+        cancellation,
+      ]);
+      state = { ...state, phase: 'ready' };
+      return state;
     },
-  );
-  const cmdId = response?.cmdId;
-  if (!cmdId) {
-    throw new Error(
-      `Daytona executeSessionCommand did not return a cmdId for session ${input.sessionId}.`,
-    );
-  }
-  return { sessionId: input.sessionId, cmdId };
+  };
 }
 
 /**

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -34,11 +34,8 @@ import {
 } from './daytona-lifecycle';
 import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
 import {
-  captureDetachedDaytonaRunnerReadinessBaseline,
-  confirmDetachedDaytonaRunnerReady,
-  createDetachedDaytonaSession,
   DaytonaRunnerInitializationError,
-  executeDetachedDaytonaSessionCommand,
+  prepareDetachedDaytonaRunner,
 } from './daytona-session-execution';
 
 const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
@@ -894,20 +891,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           // Session allocation is independent of payload staging. Start it
           // now so Daytona control-plane latency overlaps compression/upload.
           const sessionId = `deepline-play-${randomUUID()}`;
-          const sessionCreation = createDetachedDaytonaSession({
+          const detachedRunner = prepareDetachedDaytonaRunner({
             sandbox,
             sessionId,
-          }).then(
-            () => ({ ok: true as const }),
-            (error: unknown) => ({ ok: false as const, error }),
-          );
-          const readinessBaseline =
-            captureDetachedDaytonaRunnerReadinessBaseline({
-              readiness: readRunnerReadiness,
-            }).then(
-              (heartbeatAt) => ({ ok: true as const, heartbeatAt }),
-              (error: unknown) => ({ ok: false as const, error }),
-            );
+            readiness: readRunnerReadiness,
+            cancellation: cancellationPromise,
+          });
 
           const uploadStartedAt = Date.now();
           let uploadDeadlineTimer: ReturnType<typeof setTimeout> | null = null;
@@ -996,28 +985,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           // gateway heartbeat. A Daytona cmdId alone does not prove liveness.
           // After that bounded startup check, the worker parks (releasing its
           // claim and slot) and wakes on terminal push or the ceiling timeout.
-          const session = await Promise.race([
-            sessionCreation,
-            cancellationPromise,
-          ]);
-          if (session.ok === false) throw session.error;
-          const baseline = await Promise.race([
-            readinessBaseline,
-            cancellationPromise,
-          ]);
-          if (baseline.ok === false) throw baseline.error;
-          let start: Awaited<
-            ReturnType<typeof executeDetachedDaytonaSessionCommand>
-          >;
+          let start: Awaited<ReturnType<typeof detachedRunner.start>>;
           try {
-            start = await Promise.race([
-              executeDetachedDaytonaSessionCommand({
-                sandbox,
-                sessionId,
-                command: stagedPayload.command,
-              }),
-              cancellationPromise,
-            ]);
+            start = await detachedRunner.start({
+              command: stagedPayload.command,
+              exitCodePath: stagedPayload.exitCodePath,
+            });
           } catch (error) {
             if (
               cancellationRequested ||
@@ -1030,17 +1003,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             // classification below decide fresh-sandbox retry vs loud failure.
             throw error;
           }
-          await Promise.race([
-            confirmDetachedDaytonaRunnerReady({
-              sandbox,
-              sessionId: start.sessionId,
-              cmdId: start.cmdId,
-              exitCodePath: stagedPayload.exitCodePath,
-              readiness: readRunnerReadiness,
-              baselineHeartbeatAt: baseline.heartbeatAt,
-            }),
-            cancellationPromise,
-          ]);
           const runnerAttempt = Math.max(
             0,
             Math.floor(config.context.runAttempt ?? 0),

package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts

@@ -3,6 +3,7 @@ import type { DbLogicalTable, DbSessionOperation } from './db-session';
 import type { PlayRowUpdate } from './ctx-types';
 import type { PlaySheetContract } from '../plays/static-pipeline';
 import type { PlayRunLedgerEvent } from './run-ledger';
+import type { WorkReceiptFailureKind } from './work-receipts';
 
 export type RuntimeAppendRunEventsAction = {
   action: 'append_run_events';
@@ -94,11 +95,47 @@ export type RuntimeReceiptAction =
       runId: string;
       key: string;
     }
+  | {
+      action: 'get_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      keys: string[];
+    }
+  | {
+      action: 'acquire_runtime_receipt_execution_lock';
+      playName: string;
+      runId: string;
+      key: string;
+      ownerExecutionId: string;
+      ttlMs: number;
+    }
+  | {
+      action: 'release_runtime_receipt_execution_lock';
+      playName: string;
+      runId: string;
+      key: string;
+      ownerExecutionId: string;
+    }
   | {
       action: 'claim_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      leaseId?: string;
+      runAttempt?: number | null;
+      leaseAware?: boolean;
+      reclaimRunning?: boolean;
+      forceRefresh?: boolean;
+      forceFailedRefresh?: boolean;
+    }
+  | {
+      action: 'claim_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      keys: string[];
+      leaseIds?: string[];
+      runAttempt?: number | null;
+      leaseAware?: boolean;
       reclaimRunning?: boolean;
       forceRefresh?: boolean;
       forceFailedRefresh?: boolean;
@@ -108,6 +145,8 @@ export type RuntimeReceiptAction =
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       output: unknown;
     }
   | {
@@ -115,7 +154,7 @@ export type RuntimeReceiptAction =
       playName: string;
       runId: string;
       key: string;
-      runAttempt?: number;
+      runAttempt?: number | null;
       leaseId?: string | null;
     }
   | {
@@ -123,16 +162,77 @@ export type RuntimeReceiptAction =
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       error: string;
+      failureKind?: WorkReceiptFailureKind | null;
     }
   | {
       action: 'skip_runtime_step_receipt';
       playName: string;
       runId: string;
       key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
       output?: unknown;
+    }
+  | {
+      action: 'mark_runtime_step_receipt_running';
+      playName: string;
+      runId: string;
+      key: string;
+      runAttempt?: number | null;
+      leaseId?: string | null;
+    }
+  | {
+      action: 'mark_runtime_step_receipts_running';
+      playName: string;
+      runId: string;
+      receipts: RuntimeReceiptLease[];
+    }
+  | {
+      action: 'mark_runtime_step_receipts_queued';
+      playName: string;
+      runId: string;
+      receipts: RuntimeReceiptLease[];
+    }
+  | {
+      action: 'complete_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      receipts: RuntimeReceiptCompletion[];
+    }
+  | {
+      action: 'fail_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      receipts: RuntimeReceiptFailure[];
+    }
+  | {
+      action: 'heartbeat_runtime_step_receipts';
+      playName: string;
+      runId: string;
+      runAttempt?: number | null;
+      leaseId: string;
+      keys: string[];
     };
 
+export type RuntimeReceiptLease = {
+  key: string;
+  runId: string;
+  runAttempt?: number | null;
+  leaseId?: string | null;
+};
+
+export type RuntimeReceiptCompletion = RuntimeReceiptLease & {
+  output: unknown;
+};
+
+export type RuntimeReceiptFailure = RuntimeReceiptLease & {
+  error: string;
+  failureKind?: WorkReceiptFailureKind | null;
+};
+
 export type RuntimePlayResolutionAction = {
   action: 'resolve_play';
   playRef: string;

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -9,6 +9,7 @@ import type {
   RuntimeStepReceipt,
   SkipRuntimeStepReceiptInput,
 } from './ctx-types';
+import { RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES } from './output-size-limits';
 import {
   acquireRuntimeReceiptExecutionLockViaAppRuntime,
   AppRuntimeApiResponseError,
@@ -120,6 +121,7 @@ export class RuntimeReceiptStoreAdapter {
     playName: string;
     runId: string;
     maxBatchSize?: number;
+    maxBatchBytes?: number;
     maxFlushMs?: number;
     maxBufferedBytes?: number;
   }) {
@@ -130,6 +132,8 @@ export class RuntimeReceiptStoreAdapter {
     this.#writer = new RuntimeReceiptWriter({
       batchKey: receiptCommandBatchKey,
       maxBatchSize: options.maxBatchSize,
+      maxBatchBytes:
+        options.maxBatchBytes ?? RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
       maxFlushMs: options.maxFlushMs,
       maxBufferedBytes: options.maxBufferedBytes,
       classifyRetryableError: classifyReceiptRetry,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts

@@ -2,6 +2,8 @@ type RetryableReceiptError = {
   retryAfterMs: number;
 };
 
+const MAX_RETRY_JITTER_RATIO = 0.25;
+
 type PendingWrite<Input, Output> = {
   input: Input;
   bytes: number;
@@ -320,7 +322,7 @@ export class RuntimeReceiptWriter<Input, Output> {
         const retry = this.#classifyRetryableError(error);
         if (!retry) throw error;
         await this.#waitForRetry(
-          Math.max(0, Math.floor(retry.retryAfterMs)),
+          jitteredRetryDelayMs(retry.retryAfterMs),
           batch,
         );
       }
@@ -489,6 +491,14 @@ function normalizePositiveInteger(
   return Math.max(1, Math.floor(value ?? fallback));
 }
 
+function jitteredRetryDelayMs(delayMs: number): number {
+  const normalized = Math.max(0, Math.floor(delayMs));
+  if (normalized === 0) return 0;
+  return (
+    normalized + Math.floor(normalized * MAX_RETRY_JITTER_RATIO * Math.random())
+  );
+}
+
 function abortReason(signal: AbortSignal | null): unknown {
   return signal?.reason ?? new Error('Runtime receipt write was aborted.');
 }

package/dist/cli/index.js

@@ -718,7 +718,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.293",
+  version: "0.1.295",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -1600,6 +1600,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
 var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
 var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
+var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
 var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
 
 // ../shared_libs/play-runtime/ledger-safe-payload.ts

package/dist/cli/index.mjs

@@ -703,7 +703,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.293",
+  version: "0.1.295",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -1585,6 +1585,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
 var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
 var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
+var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
 var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
 
 // ../shared_libs/play-runtime/ledger-safe-payload.ts

package/dist/index.js

@@ -438,7 +438,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.293",
+  version: "0.1.295",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -1320,6 +1320,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
 var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
 var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
+var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
 var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
 
 // ../shared_libs/play-runtime/ledger-safe-payload.ts

package/dist/index.mjs

@@ -367,7 +367,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.293",
+  version: "0.1.295",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -1249,6 +1249,7 @@ var CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
 var RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
 var RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
 var RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
+var RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
 var RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
 
 // ../shared_libs/play-runtime/ledger-safe-payload.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.293",
+  "version": "0.1.295",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {