deepline 0.2.9 → 0.2.11

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -1331,6 +1331,8 @@ export class PlayContextImpl {
   #options: ContextOptions;
   private readonly executionScope: RunExecutionScope;
   private logBuffer: string[] = [];
+  private fixtureProviderPacingBypassLogged = false;
+  private fixtureProviderPacingEnforcementLogged = false;
   private checkpoint: PlayCheckpoint;
   /**
    * Durable tool receipts are the replay/cache authority for the execution
@@ -4329,6 +4331,18 @@ export class PlayContextImpl {
       // streaming and sheet-backed.
       const materializedResultRowLimit = resolveMaterializeLimitCap();
       const immediateMaterializedRows: Record<string, unknown>[] = [];
+      let immediateMaterializedResidentBytes = 0;
+      let immediateMaterializedCacheEnabled = true;
+
+      const disableImmediateMaterializedCache = (reason: string) => {
+        if (!immediateMaterializedCacheEnabled) return;
+        immediateMaterializedRows.length = 0;
+        immediateMaterializedResidentBytes = 0;
+        immediateMaterializedCacheEnabled = false;
+        this.log(
+          `Dataset ${normalizedMapNamespace} result cache disabled (${reason}); using the durable runtime sheet.`,
+        );
+      };
 
       const persistMapRows = async (rows: PersistableMapRow[]) => {
         if (!this.#options.onMapRowsCompleted || rows.length === 0) {
@@ -4630,8 +4644,47 @@ export class PlayContextImpl {
         for (const row of mapResult.completedRows) {
           const materializedRow = this.toMaterializedOutputRow(row.data);
           if (previewRows.length < 5) previewRows.push(materializedRow);
-          if (immediateMaterializedRows.length < materializedResultRowLimit) {
-            immediateMaterializedRows.push(materializedRow);
+          if (
+            immediateMaterializedCacheEnabled &&
+            immediateMaterializedRows.length < materializedResultRowLimit
+          ) {
+            const estimatedRowResidentBytes =
+              runtimeMapJsonByteLength(materializedRow) *
+                NODE_RUNTIME_MAP_MATERIALIZED_MEMORY_MULTIPLIER +
+              NODE_RUNTIME_MAP_ROW_OVERHEAD_BYTES;
+            const observedRowCount = immediateMaterializedRows.length + 1;
+            const observedResidentBytes =
+              immediateMaterializedResidentBytes + estimatedRowResidentBytes;
+            // `net_new` deliberately reports only admitted rows, so its
+            // running total cannot predict the next page. Do not pre-scan an
+            // unknown-count dataset just to estimate this cache. Instead,
+            // bound the cache and active source page together while that page
+            // remains resident.
+            const projectedResidentBytes =
+              options?.mode === 'net_new'
+                ? observedResidentBytes +
+                  pageMemoryEstimate.estimatedResidentBytes
+                : Math.ceil(
+                    (observedResidentBytes / observedRowCount) *
+                      Math.max(observedRowCount, totalInputCount - failedCount),
+                  );
+            if (
+              projectedResidentBytes > mapMemoryLimits.materializedBudgetBytes
+            ) {
+              // All-or-nothing cache: retaining a prefix that cannot fit once
+              // the known input finishes gives authored code no complete fast
+              // path. More importantly, that prefix would overlap the next
+              // active page and can exhaust a fixed-size sandbox before the
+              // cache reaches its own independent budget. Project from the
+              // rows observed so far and move to the already-durable sheet
+              // early instead of waiting for resident memory to reach a cliff.
+              disableImmediateMaterializedCache(
+                `projected ${projectedResidentBytes} bytes exceeds ${mapMemoryLimits.materializedBudgetBytes}-byte budget`,
+              );
+            } else {
+              immediateMaterializedRows.push(materializedRow);
+              immediateMaterializedResidentBytes = observedResidentBytes;
+            }
           }
         }
         executedCount += rowsToExecute.length;
@@ -4738,6 +4791,7 @@ export class PlayContextImpl {
           materializedResultRowLimit,
         );
         if (
+          immediateMaterializedCacheEnabled &&
           immediateMaterializedRows.length === availableResultRows &&
           (limit !== undefined || successfulCount <= materializedResultRowLimit)
         ) {
@@ -4808,6 +4862,7 @@ export class PlayContextImpl {
         },
         previewRows,
         residentRows:
+          immediateMaterializedCacheEnabled &&
           immediateMaterializedRows.length === successfulCount
             ? immediateMaterializedRows
             : null,
@@ -4828,8 +4883,9 @@ export class PlayContextImpl {
           peek: async (limit) =>
             limit <= 0
               ? []
-              : immediateMaterializedRows.length >=
-                  Math.min(limit, successfulCount)
+              : immediateMaterializedCacheEnabled &&
+                  immediateMaterializedRows.length >=
+                    Math.min(limit, successfulCount)
                 ? immediateMaterializedRows.slice(0, limit)
                 : await readRuntimeBackedMapRows({
                     limit,
@@ -4840,7 +4896,10 @@ export class PlayContextImpl {
           iterate: () =>
             ({
               async *[Symbol.asyncIterator]() {
-                if (immediateMaterializedRows.length === successfulCount) {
+                if (
+                  immediateMaterializedCacheEnabled &&
+                  immediateMaterializedRows.length === successfulCount
+                ) {
                   for (const row of immediateMaterializedRows) yield row;
                   return;
                 }
@@ -7945,6 +8004,113 @@ export class PlayContextImpl {
 
   // ——— Batched tool call execution ———
 
+  private startPendingToolReceiptHeartbeat(requests: ToolCallRequest[]): {
+    stop: () => void;
+    leaseLost: () => unknown | null;
+  } | null {
+    const heartbeatReceipts = this.#options.heartbeatRuntimeStepReceipts;
+    const ownedRequests = requests.filter(
+      (request) => request.receiptKey && request.receiptLeaseId,
+    );
+    if (!heartbeatReceipts || ownedRequests.length === 0) return null;
+
+    const earliestLeaseExpiry = ownedRequests.reduce<string | null>(
+      (earliest, request) => {
+        const expiresAt = request.receiptLeaseExpiresAt ?? null;
+        if (!expiresAt) return earliest;
+        if (!earliest) return expiresAt;
+        return Date.parse(expiresAt) < Date.parse(earliest)
+          ? expiresAt
+          : earliest;
+      },
+      null,
+    );
+    let leaseLost: unknown | null = null;
+    const supervisor = createRuntimeReceiptHeartbeatSupervisor({
+      intervalMs: runtimeLeaseHeartbeatIntervalFromExpiry({
+        leaseExpiresAt: earliestLeaseExpiry,
+        fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+      }),
+      heartbeat: async () => {
+        // Receipt claims are acquired in bulk before the provider-shaped
+        // dispatcher admits every call. Renew the undispatched tail as well as
+        // in-flight calls; otherwise a slow provider can let later claims
+        // expire while they are still waiting behind the concurrency gate.
+        const unsettled = ownedRequests.filter((request) =>
+          this.toolCallResolvers.has(request.callId),
+        );
+        if (unsettled.length === 0) return 'terminal';
+        const byLeaseId = new Map<string, Map<string, ToolCallRequest[]>>();
+        for (const request of unsettled) {
+          const leaseId = request.receiptLeaseId!;
+          const byReceiptKey = byLeaseId.get(leaseId) ?? new Map();
+          const receiptKey = request.receiptKey!;
+          const group = byReceiptKey.get(receiptKey) ?? [];
+          group.push(request);
+          byReceiptKey.set(receiptKey, group);
+          byLeaseId.set(leaseId, byReceiptKey);
+        }
+        await Promise.all(
+          [...byLeaseId].map(async ([leaseId, byReceiptKey]) => {
+            // One content-addressed receipt can have many same-run followers.
+            // The store renews unique receipt rows, not request positions, so
+            // duplicate keys in one heartbeat would yield null duplicate
+            // positions and falsely stop the supervisor as if ownership were
+            // lost.
+            const entries = [...byReceiptKey];
+            const keys = entries.map(([receiptKey]) => receiptKey);
+            const receipts = await heartbeatReceipts({
+              runId: this.currentReceiptOwnerRunId,
+              runAttempt: this.currentRunAttempt,
+              leaseId,
+              keys,
+            });
+            for (let index = 0; index < entries.length; index += 1) {
+              const [receiptKey, requests] = entries[index]!;
+              // Completion can race this bulk heartbeat response. A settled
+              // receipt group no longer needs ownership and must not turn that
+              // race into a false lease-loss failure.
+              if (
+                !requests.some((request) =>
+                  this.toolCallResolvers.has(request.callId),
+                )
+              ) {
+                continue;
+              }
+              if (
+                !this.runtimeToolReceiptStillOwned(
+                  receipts[index] ?? null,
+                  leaseId,
+                )
+              ) {
+                throw new RuntimeReceiptLeaseLostError({
+                  receiptKey,
+                  runId: this.currentReceiptOwnerRunId,
+                  leaseId,
+                });
+              }
+            }
+          }),
+        );
+        return 'active';
+      },
+      isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
+      onLeaseLost: (error) => {
+        leaseLost ??= error;
+      },
+      onTransientFailure: (error) => {
+        this.log(
+          `Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      },
+    });
+    supervisor.start();
+    return {
+      stop: () => supervisor.stop(),
+      leaseLost: () => leaseLost,
+    };
+  }
+
   private async executeBatchedToolCalls(
     queuedToolCalls: ToolCallRequest[],
   ): Promise<void> {
@@ -8615,277 +8781,290 @@ export class PlayContextImpl {
           recordToolStep(durableRecoveredRequests);
         }
 
-        if (pendingRequests.length > 0) {
-          const strategy =
-            this.#options.getBatchOperationStrategy?.(toolId) ?? null;
-
-          if (strategy) {
-            const compiledBatches = compileRequestsWithStrategy({
-              requests: pendingRequests,
-              strategy,
-              getPayload: (request: ToolCallRequest) => request.input,
-            });
+        const pendingReceiptHeartbeat =
+          this.startPendingToolReceiptHeartbeat(pendingRequests);
+        try {
+          if (pendingRequests.length > 0) {
+            const strategy =
+              this.#options.getBatchOperationStrategy?.(toolId) ?? null;
+
+            if (strategy) {
+              const compiledBatches = compileRequestsWithStrategy({
+                requests: pendingRequests,
+                strategy,
+                getPayload: (request: ToolCallRequest) => request.input,
+              });
 
-            const batchSize =
-              compiledBatches.length > 0
-                ? await this.resourceGovernor.suggestedToolParallelism(
-                    compiledBatches[0]!.batchOperation,
-                    this.governor.policy.pacing
-                      .workerToolBatchDefaultParallelism,
-                  )
-                : this.governor.policy.pacing.workerToolBatchDefaultParallelism;
-            await executeChunkedRequests({
-              requests: compiledBatches,
-              batchSize,
-              execute: async (batch) => {
-                // Circuit breaker: skip dispatching this batch's provider call
-                // once a persistence failure has occurred in this run.
-                if (this.persistenceLatch.tripped) {
-                  this.persistenceLatch.preventedCallCount +=
-                    batch.memberRequests.length;
-                  throw new RuntimePersistenceCircuitOpenError(
-                    this.persistenceLatch,
+              const batchSize =
+                compiledBatches.length > 0
+                  ? await this.resourceGovernor.suggestedToolParallelism(
+                      compiledBatches[0]!.batchOperation,
+                      this.governor.policy.pacing
+                        .workerToolBatchDefaultParallelism,
+                    )
+                  : this.governor.policy.pacing
+                      .workerToolBatchDefaultParallelism;
+              await executeChunkedRequests({
+                requests: compiledBatches,
+                batchSize,
+                execute: async (batch) => {
+                  // Circuit breaker: skip dispatching this batch's provider call
+                  // once a persistence failure has occurred in this run.
+                  if (this.persistenceLatch.tripped) {
+                    this.persistenceLatch.preventedCallCount +=
+                      batch.memberRequests.length;
+                    throw new RuntimePersistenceCircuitOpenError(
+                      this.persistenceLatch,
+                    );
+                  }
+                  const receiptKeys = batch.memberRequests.map(
+                    (request) => request.cacheKey,
                   );
-                }
-                const receiptKeys = batch.memberRequests.map(
-                  (request) => request.cacheKey,
-                );
-                const aggregateReceiptKey = buildDurableToolAggregateReceiptKey(
-                  {
-                    receiptKeys,
-                    prefix: 'batch',
-                    aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
-                      orgId: this.#options.orgId,
-                      toolId: batch.batchOperation,
-                    }),
-                  },
-                );
-                return await this.callToolAPI(
-                  batch.batchOperation,
-                  batch.batchPayload,
-                  {
-                    durableCallReceiptKey: aggregateReceiptKey,
-                    executionAuthScopeDigest:
-                      batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
-                    providerIdempotencyKey:
-                      buildDurableToolAggregateProviderIdempotencyKey({
-                        aggregateReceiptKey,
-                        receiptKeys,
-                        providerIdempotencyKeys: batch.memberRequests.map(
-                          (request) =>
-                            this.providerIdempotencyKeyForToolCall({
-                              cacheKey: request.cacheKey,
-                              force: request.force === true,
-                              leaseId: request.receiptLeaseId,
-                            }),
-                        ),
+                  const aggregateReceiptKey =
+                    buildDurableToolAggregateReceiptKey({
+                      receiptKeys,
+                      prefix: 'batch',
+                      aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
+                        orgId: this.#options.orgId,
+                        toolId: batch.batchOperation,
                       }),
-                    receiptLeaseExpiresAt: batch.memberRequests.reduce<
-                      string | null
-                    >((earliest, request) => {
-                      const expiresAt = request.receiptLeaseExpiresAt ?? null;
-                      if (!expiresAt) return earliest;
-                      if (!earliest) return expiresAt;
-                      return Date.parse(expiresAt) < Date.parse(earliest)
-                        ? expiresAt
-                        : earliest;
-                    }, null),
-                    timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
-                      batch.memberRequests,
-                    ),
-                    beforeProviderCall: () =>
-                      this.assertRuntimeToolReceiptOwnership(
-                        batch.memberRequests,
-                      ),
-                    heartbeatReceipt: () =>
-                      this.renewRuntimeToolReceiptOwnership(
+                    });
+                  return await this.callToolAPI(
+                    batch.batchOperation,
+                    batch.batchPayload,
+                    {
+                      durableCallReceiptKey: aggregateReceiptKey,
+                      executionAuthScopeDigest:
+                        batch.memberRequests[0]?.executionAuthScopeDigest ??
+                        null,
+                      providerIdempotencyKey:
+                        buildDurableToolAggregateProviderIdempotencyKey({
+                          aggregateReceiptKey,
+                          receiptKeys,
+                          providerIdempotencyKeys: batch.memberRequests.map(
+                            (request) =>
+                              this.providerIdempotencyKeyForToolCall({
+                                cacheKey: request.cacheKey,
+                                force: request.force === true,
+                                leaseId: request.receiptLeaseId,
+                              }),
+                          ),
+                        }),
+                      receiptLeaseExpiresAt: batch.memberRequests.reduce<
+                        string | null
+                      >((earliest, request) => {
+                        const expiresAt = request.receiptLeaseExpiresAt ?? null;
+                        if (!expiresAt) return earliest;
+                        if (!earliest) return expiresAt;
+                        return Date.parse(expiresAt) < Date.parse(earliest)
+                          ? expiresAt
+                          : earliest;
+                      }, null),
+                      timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
                         batch.memberRequests,
                       ),
-                  },
-                );
-              },
-              onChunkComplete: async (chunkResults) => {
-                for (const entry of chunkResults) {
-                  if (entry.error !== undefined) {
-                    for (const request of entry.request.memberRequests) {
-                      await rejectWithLiveFollowers(request, entry.error);
+                      beforeProviderCall: () =>
+                        this.assertRuntimeToolReceiptOwnership(
+                          batch.memberRequests,
+                        ),
+                      heartbeatReceipt: () =>
+                        this.renewRuntimeToolReceiptOwnership(
+                          batch.memberRequests,
+                        ),
+                    },
+                  );
+                },
+                onChunkComplete: async (chunkResults) => {
+                  for (const entry of chunkResults) {
+                    if (entry.error !== undefined) {
+                      for (const request of entry.request.memberRequests) {
+                        await rejectWithLiveFollowers(request, entry.error);
+                      }
+                      continue;
+                    }
+                    const splitResults =
+                      entry.result != null
+                        ? entry.request.splitResults(entry.result)
+                        : entry.request.memberRequests.map(() => null);
+                    const resolvedResults =
+                      await this.resolveToolCallBatchResults(
+                        toolId,
+                        entry.request.memberRequests.map((request, index) => ({
+                          request,
+                          result: splitResults[index] ?? null,
+                        })),
+                      );
+
+                    for (
+                      let index = 0;
+                      index < entry.request.memberRequests.length;
+                      index += 1
+                    ) {
+                      const request = entry.request.memberRequests[index]!;
+                      if (resolvedResults[index] != null) {
+                        successfulLiveStepCallIds.add(request.callId);
+                      }
+                      resolveLiveFollowers(request, resolvedResults[index]);
                     }
-                    continue;
                   }
-                  const splitResults =
-                    entry.result != null
-                      ? entry.request.splitResults(entry.result)
-                      : entry.request.memberRequests.map(() => null);
+
+                  recordToolStep(
+                    chunkResults.flatMap(
+                      (entry) => entry.request.memberRequests,
+                    ),
+                  );
+                  this.#options.onBatchComplete?.(this.checkpoint);
+                },
+              });
+            } else {
+              const completionBuffer: Array<{
+                request: ToolCallRequest;
+                result: unknown | null;
+                metadata?: ToolResultMetadataInput | null;
+                jobId?: string;
+                meta?: Record<string, unknown>;
+                resolve: (value: unknown) => void;
+                reject: (error: unknown) => void;
+              }> = [];
+              const completionFailedCallIds = new Set<string>();
+              let completionFlushScheduled = false;
+              const flushCompletionBuffer = async (): Promise<void> => {
+                const entries = completionBuffer.splice(0);
+                if (entries.length === 0) return;
+                try {
                   const resolvedResults =
                     await this.resolveToolCallBatchResults(
                       toolId,
-                      entry.request.memberRequests.map((request, index) => ({
-                        request,
-                        result: splitResults[index] ?? null,
+                      entries.map((entry) => ({
+                        request: entry.request,
+                        result: entry.result,
+                        metadata: entry.metadata,
+                        jobId: entry.jobId,
+                        meta: entry.meta,
                       })),
                     );
-
-                  for (
-                    let index = 0;
-                    index < entry.request.memberRequests.length;
-                    index += 1
-                  ) {
-                    const request = entry.request.memberRequests[index]!;
-                    if (resolvedResults[index] != null) {
-                      successfulLiveStepCallIds.add(request.callId);
+                  for (let index = 0; index < entries.length; index += 1) {
+                    const entry = entries[index]!;
+                    const result = resolvedResults[index];
+                    if (result != null) {
+                      successfulLiveStepCallIds.add(entry.request.callId);
                     }
-                    resolveLiveFollowers(request, resolvedResults[index]);
+                    resolveLiveFollowers(entry.request, result);
+                    entry.resolve(result);
                   }
-                }
-
-                recordToolStep(
-                  chunkResults.flatMap((entry) => entry.request.memberRequests),
-                );
-                this.#options.onBatchComplete?.(this.checkpoint);
-              },
-            });
-          } else {
-            const completionBuffer: Array<{
-              request: ToolCallRequest;
-              result: unknown | null;
-              metadata?: ToolResultMetadataInput | null;
-              jobId?: string;
-              meta?: Record<string, unknown>;
-              resolve: (value: unknown) => void;
-              reject: (error: unknown) => void;
-            }> = [];
-            const completionFailedCallIds = new Set<string>();
-            let completionFlushScheduled = false;
-            const flushCompletionBuffer = async (): Promise<void> => {
-              const entries = completionBuffer.splice(0);
-              if (entries.length === 0) return;
-              try {
-                const resolvedResults = await this.resolveToolCallBatchResults(
-                  toolId,
-                  entries.map((entry) => ({
-                    request: entry.request,
-                    result: entry.result,
-                    metadata: entry.metadata,
-                    jobId: entry.jobId,
-                    meta: entry.meta,
-                  })),
-                );
-                for (let index = 0; index < entries.length; index += 1) {
-                  const entry = entries[index]!;
-                  const result = resolvedResults[index];
-                  if (result != null) {
-                    successfulLiveStepCallIds.add(entry.request.callId);
+                } catch (error) {
+                  for (const entry of entries) {
+                    completionFailedCallIds.add(entry.request.callId);
+                    await this.rejectToolCall(toolId, entry.request, error, {
+                      persistReceiptFailure: false,
+                    });
+                    entry.reject(error);
                   }
-                  resolveLiveFollowers(entry.request, result);
-                  entry.resolve(result);
                 }
-              } catch (error) {
-                for (const entry of entries) {
-                  completionFailedCallIds.add(entry.request.callId);
-                  await this.rejectToolCall(toolId, entry.request, error, {
-                    persistReceiptFailure: false,
+              };
+              const enqueueCompletion = (
+                request: ToolCallRequest,
+                execution: ParsedToolExecuteResponse,
+              ): Promise<unknown> =>
+                new Promise((resolve, reject) => {
+                  completionBuffer.push({
+                    request,
+                    result: execution.result ?? null,
+                    metadata: execution.metadata ?? null,
+                    jobId: execution.jobId,
+                    meta: execution.meta,
+                    resolve,
+                    reject,
                   });
-                  entry.reject(error);
-                }
-              }
-            };
-            const enqueueCompletion = (
-              request: ToolCallRequest,
-              execution: ParsedToolExecuteResponse,
-            ): Promise<unknown> =>
-              new Promise((resolve, reject) => {
-                completionBuffer.push({
-                  request,
-                  result: execution.result ?? null,
-                  metadata: execution.metadata ?? null,
-                  jobId: execution.jobId,
-                  meta: execution.meta,
-                  resolve,
-                  reject,
+                  if (completionFlushScheduled) return;
+                  completionFlushScheduled = true;
+                  setTimeout(() => {
+                    completionFlushScheduled = false;
+                    void flushCompletionBuffer();
+                  }, 0);
                 });
-                if (completionFlushScheduled) return;
-                completionFlushScheduled = true;
-                setTimeout(() => {
-                  completionFlushScheduled = false;
-                  void flushCompletionBuffer();
-                }, 0);
-              });
-            // Seed the dispatch width from the governor's provider-shaped
-            // parallelism instead of the flat policy.concurrency.toolCalls
-            // ceiling. The flat width launched every pending row at once into
-            // the pacer, so unhinted providers 429'd on the first burst before
-            // AIMD could halve the rate. The shaped estimate is derived from the
-            // provider's RPS/maxConcurrency pacing rules (see
-            // suggestedParallelism in governor.ts), floored at 1 and capped by
-            // the global tool-call concurrency ceiling. The pacer still gates
-            // each in-flight call, so this only trims the launch burst.
-            const toolCallConcurrencyCeiling =
-              this.governor.policy.concurrency.toolCalls;
-            const shapedToolParallelism =
-              await this.resourceGovernor.suggestedToolParallelism(
-                toolId,
+              // Seed the dispatch width from the governor's provider-shaped
+              // parallelism instead of the flat policy.concurrency.toolCalls
+              // ceiling. The flat width launched every pending row at once into
+              // the pacer, so unhinted providers 429'd on the first burst before
+              // AIMD could halve the rate. The shaped estimate is derived from the
+              // provider's RPS/maxConcurrency pacing rules (see
+              // suggestedParallelism in governor.ts), floored at 1 and capped by
+              // the global tool-call concurrency ceiling. The pacer still gates
+              // each in-flight call, so this only trims the launch burst.
+              const toolCallConcurrencyCeiling =
+                this.governor.policy.concurrency.toolCalls;
+              const shapedToolParallelism =
+                await this.resourceGovernor.suggestedToolParallelism(
+                  toolId,
+                  toolCallConcurrencyCeiling,
+                );
+              const dispatchWidth = Math.min(
                 toolCallConcurrencyCeiling,
+                Math.max(1, shapedToolParallelism),
               );
-            const dispatchWidth = Math.min(
-              toolCallConcurrencyCeiling,
-              Math.max(1, shapedToolParallelism),
-            );
-            const dispatchResults = await dispatchBoundedSettled(
-              pendingRequests,
-              dispatchWidth,
-              async (request) => {
-                // Circuit breaker: do not dispatch this provider call once a
-                // persistence failure has occurred in this run.
-                if (this.persistenceLatch.tripped) {
-                  this.persistenceLatch.preventedCallCount += 1;
-                  throw new RuntimePersistenceCircuitOpenError(
-                    this.persistenceLatch,
+              const dispatchResults = await dispatchBoundedSettled(
+                pendingRequests,
+                dispatchWidth,
+                async (request) => {
+                  // Circuit breaker: do not dispatch this provider call once a
+                  // persistence failure has occurred in this run.
+                  if (this.persistenceLatch.tripped) {
+                    this.persistenceLatch.preventedCallCount += 1;
+                    throw new RuntimePersistenceCircuitOpenError(
+                      this.persistenceLatch,
+                    );
+                  }
+                  const execution = await this.callToolExecutionAPI(
+                    toolId,
+                    request.input,
+                    {
+                      beforeProviderCall: () =>
+                        this.assertRuntimeToolReceiptOwnership([request]),
+                      ...(request.receiptKey
+                        ? {
+                            durableCallReceiptKey: request.receiptKey,
+                            executionAuthScopeDigest:
+                              request.executionAuthScopeDigest,
+                            providerIdempotencyKey:
+                              this.providerIdempotencyKeyForToolCall({
+                                cacheKey: request.receiptKey,
+                                force: request.force === true,
+                                leaseId: request.receiptLeaseId,
+                              }),
+                            receiptLeaseExpiresAt:
+                              request.receiptLeaseExpiresAt,
+                            heartbeatReceipt: () =>
+                              this.renewRuntimeToolReceiptOwnership([request]),
+                          }
+                        : {}),
+                      timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
+                        request,
+                      ]),
+                    },
                   );
-                }
-                const execution = await this.callToolExecutionAPI(
-                  toolId,
-                  request.input,
-                  {
-                    beforeProviderCall: () =>
-                      this.assertRuntimeToolReceiptOwnership([request]),
-                    ...(request.receiptKey
-                      ? {
-                          durableCallReceiptKey: request.receiptKey,
-                          executionAuthScopeDigest:
-                            request.executionAuthScopeDigest,
-                          providerIdempotencyKey:
-                            this.providerIdempotencyKeyForToolCall({
-                              cacheKey: request.receiptKey,
-                              force: request.force === true,
-                              leaseId: request.receiptLeaseId,
-                            }),
-                          receiptLeaseExpiresAt: request.receiptLeaseExpiresAt,
-                          heartbeatReceipt: () =>
-                            this.renewRuntimeToolReceiptOwnership([request]),
-                        }
-                      : {}),
-                    timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
-                      request,
-                    ]),
-                  },
-                );
-                return await enqueueCompletion(request, execution);
-              },
-            );
-            const failedEntries = dispatchResults.filter(
-              (entry) =>
-                entry.error !== undefined &&
-                !completionFailedCallIds.has(entry.job.callId),
-            );
-            await Promise.allSettled(
-              failedEntries.map((entry) =>
-                rejectWithLiveFollowers(entry.job, entry.error),
-              ),
-            );
-            recordToolStep(pendingRequests);
-            this.#options.onBatchComplete?.(this.checkpoint);
+                  return await enqueueCompletion(request, execution);
+                },
+              );
+              const failedEntries = dispatchResults.filter(
+                (entry) =>
+                  entry.error !== undefined &&
+                  !completionFailedCallIds.has(entry.job.callId),
+              );
+              await Promise.allSettled(
+                failedEntries.map((entry) =>
+                  rejectWithLiveFollowers(entry.job, entry.error),
+                ),
+              );
+              recordToolStep(pendingRequests);
+              this.#options.onBatchComplete?.(this.checkpoint);
+            }
           }
+        } finally {
+          pendingReceiptHeartbeat?.stop();
         }
+        const pendingReceiptLeaseLost = pendingReceiptHeartbeat?.leaseLost();
+        if (pendingReceiptLeaseLost) throw pendingReceiptLeaseLost;
         if (durableExistingRunningHandler) {
           await durableExistingRunningHandler;
         }
@@ -9321,11 +9500,44 @@ export class PlayContextImpl {
                 // is the fetch itself, so independently delayed runners cannot
                 // compress real provider arrivals after their tickets.
                 const protectionHeaders = await this.vercelProtectionHeaders();
-                const providerPermit =
-                  await this.resourceGovernor.acquireProviderPermit({
-                    toolId,
-                    signal: abortController?.signal,
-                  });
+                // Use the mode carried by this physical execution request.
+                // A restored execution scope may predate integration-mode
+                // authority metadata, while the signed launch and every tool
+                // request still carry options.integrationMode. Pacing the
+                // latter as live would make fixture runs both slow and
+                // unauditable even though the app never dispatches a provider
+                // request.
+                const fixtureExecution =
+                  this.#options.integrationMode === 'fixture';
+                const enforceFixtureProviderPacing =
+                  fixtureExecution &&
+                  this.#options.enforceFixtureProviderPacing === true;
+                const fixtureOnlyExecution =
+                  fixtureExecution && !enforceFixtureProviderPacing;
+                if (
+                  fixtureOnlyExecution &&
+                  !this.fixtureProviderPacingBypassLogged
+                ) {
+                  this.fixtureProviderPacingBypassLogged = true;
+                  this.log(
+                    'Fixture mode: provider pacing bypassed because no provider request is dispatched.',
+                  );
+                }
+                if (
+                  enforceFixtureProviderPacing &&
+                  !this.fixtureProviderPacingEnforcementLogged
+                ) {
+                  this.fixtureProviderPacingEnforcementLogged = true;
+                  this.log(
+                    'Fixture mode: provider pacing explicitly enforced for production-parity testing.',
+                  );
+                }
+                const providerPermit = fixtureOnlyExecution
+                  ? { release() {} }
+                  : await this.resourceGovernor.acquireProviderPermit({
+                      toolId,
+                      signal: abortController?.signal,
+                    });
                 const integrationFetchStartedAt = Date.now();
                 providerCallStartedAt = integrationFetchStartedAt;
                 try {