deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -23,12 +23,24 @@ import {
 } from './batch-runtime';
 import type { PlayQueueHint } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
+import type { WorkReceiptFailureKind } from './work-receipts';
 import {
   completedMapRowOutcome,
   failedMapRowOutcome,
   mapRowOutcomeRuntimeFields,
   resolveMapRowOutcomeKey,
 } from './map-row-outcome';
+import { RuntimeSheetRowsBlockedError } from './runtime-sheet-errors';
+import { buildChildRunId } from './child-run-id';
+import type { PlayCallGovernanceSnapshot } from './scheduler-backend';
+import {
+  RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER,
+  SYNTHETIC_RUN_HEADER,
+} from './coordinator-headers';
+export {
+  RuntimeSheetRowsBlockedError,
+  type RuntimeSheetBlockedRowDetail,
+} from './runtime-sheet-errors';
 import {
   createDefaultGovernanceSnapshot,
   createPlayExecutionGovernor,
@@ -64,30 +76,38 @@ import {
   TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
+  parseToolExecuteAuthScopeChangedError,
+  ToolExecuteAuthScopeChangedError,
 } from './tool-execute-retry-policy';
 import {
   buildDurableCtxCallCacheKey,
+  buildDurableRunPlaySemanticKey,
+  buildDurableToolAggregateProviderIdempotencyKey,
+  buildDurableToolAggregateReceiptKey,
   buildDurableToolCallAuthScopeDigest,
   buildDurableToolCallCacheKey,
+  buildDurableToolProviderIdempotencyKey,
+  buildDurableToolReceiptPrefix,
 } from './durable-call-cache';
 import {
-  QUERY_RESULT_DATASET_PAGE_SIZE,
-  isCustomerDbDatasetTool,
-  isQueryResultDatasetReadRequest,
-  isQueryResultDatasetTool,
-} from './query-result-dataset';
-import {
+  RuntimeReceiptLeaseLostError,
   RuntimeReceiptWaitTimeoutError,
   executeWithDurableRuntimeReceipt,
   resolveRuntimeToolReceiptWaitMaxAttempts,
+  runtimeReceiptFailureKindForError,
   runtimeReceiptOutput as durableRuntimeReceiptOutput,
   waitForCompletedRuntimeReceipt,
   type DurableReceiptExecutionStore,
 } from './durable-receipt-execution';
+import {
+  QUERY_RESULT_DATASET_PAGE_SIZE,
+  isCustomerDbDatasetTool,
+  isQueryResultDatasetReadRequest,
+  isQueryResultDatasetTool,
+} from './query-result-dataset';
 import { isRowIsolationExemptError } from './row-isolation';
 import {
   createRuntimePersistenceLatch,
-  PROVIDER_DISPATCH_WAVE_SIZE,
   RuntimePersistenceCircuitOpenError,
   tripRuntimePersistenceLatch,
   type RuntimePersistenceLatch,
@@ -119,7 +139,9 @@ import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from './runtime-constraints';
 import {
   PlayExecutionSuspendedError,
   PlayRowExecutionSuspendedError,
+  isPlayExecutionSuspendedError,
   isPlayRowExecutionSuspendedError,
+  type PlayExecutionSuspension,
 } from './suspension';
 import {
   createSecretRedactionContext,
@@ -171,6 +193,7 @@ import {
   type StepProgramDatasetColumnInput,
   type StepProgramDatasetOptions,
 } from './step-program-dataset-builder';
+import { readRuntimeSheetDatasetRows } from './runtime-api';
 
 /**
  * SECURITY: AsyncLocalStorage is per async execution, not a cross-run cache.
@@ -207,9 +230,118 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
 const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
+const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
+const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
 type SafeFetchModule = typeof import('@shared_libs/security/safe-fetch');
 let safeFetchModule: Promise<SafeFetchModule> | null = null;
 
+export async function waitForNodeRuntimeMapRowsVisible(input: {
+  mapName: string;
+  tableNamespace: string;
+  runId?: string | null;
+  expectedRows: number;
+  updatedRows: number;
+  readVisibleRowCount: () => Promise<number>;
+  sleep?: (ms: number) => Promise<void>;
+  log?: (line: string) => void;
+}): Promise<number> {
+  if (input.expectedRows <= 0) return 0;
+
+  const sleep =
+    input.sleep ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  let lastVisibleRows = -1;
+  for (
+    let attempt = 1;
+    attempt <= NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS;
+    attempt += 1
+  ) {
+    lastVisibleRows = await input.readVisibleRowCount();
+    if (lastVisibleRows >= input.expectedRows) {
+      if (attempt > 1) {
+        input.log?.(
+          `Runtime sheet visibility barrier satisfied ctx.dataset("${input.mapName}") ` +
+            `after ${attempt} read(s): visible ${lastVisibleRows}/${input.expectedRows}; ` +
+            `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
+        );
+      }
+      return lastVisibleRows;
+    }
+
+    if (attempt === 1) {
+      input.log?.(
+        `Runtime sheet visibility barrier waiting ctx.dataset("${input.mapName}"): ` +
+          `visible ${lastVisibleRows}/${input.expectedRows}; ` +
+          `wrote ${input.updatedRows}/${input.expectedRows}; run ${input.runId ?? 'unknown'}.`,
+      );
+    }
+    if (attempt < NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS) {
+      await sleep(NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS);
+    }
+  }
+
+  throw new Error(
+    `Runtime sheet visibility mismatch for ctx.dataset("${input.mapName}"): ` +
+      `expected ${input.expectedRows} terminal visible row(s), saw ${lastVisibleRows}; ` +
+      `write reported ${input.updatedRows}; run ${input.runId ?? 'unknown'}.`,
+  );
+}
+
+export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: {
+  mapName: string;
+  tableNamespace: string;
+  runId?: string | null;
+  expectedRows: number;
+  currentRows: Record<string, unknown>[];
+  failedRowCount: number;
+  readPersistedRows: (input: {
+    limit: number;
+    offset: number;
+    rowMode: 'all';
+  }) => Promise<Record<string, unknown>[]>;
+  log?: (line: string) => void;
+}): Promise<Record<string, unknown>[]> {
+  if (
+    input.failedRowCount > 0 ||
+    input.expectedRows <= 0 ||
+    input.currentRows.length >= input.expectedRows
+  ) {
+    return input.currentRows;
+  }
+
+  const persistedRows: Record<string, unknown>[] = [];
+  let offset = 0;
+  while (persistedRows.length < input.expectedRows) {
+    const page = await input.readPersistedRows({
+      limit: Math.min(10_000, input.expectedRows - persistedRows.length),
+      offset,
+      rowMode: 'all',
+    });
+    if (page.length === 0) break;
+    persistedRows.push(...page);
+    offset += page.length;
+  }
+
+  if (persistedRows.length < input.expectedRows) {
+    throw new Error(
+      `Runtime sheet finalization mismatch for ctx.dataset("${input.mapName}"): ` +
+        `expected ${input.expectedRows} terminal persisted row(s), saw ${persistedRows.length}; ` +
+        `in-memory Node map results reported ${input.currentRows.length}; run ${input.runId ?? 'unknown'}.`,
+    );
+  }
+
+  if (persistedRows.length > input.currentRows.length) {
+    input.log?.(
+      `Runtime sheet finalization reconciled ctx.dataset("${input.mapName}") ` +
+        `from ${input.currentRows.length}/${input.expectedRows} in-memory Node row(s) ` +
+        `to ${persistedRows.length} terminal persisted row(s).`,
+    );
+    return persistedRows;
+  }
+
+  return input.currentRows;
+}
+
 export function resolveToolRuntimeTimeoutMs(
   toolId: string,
   requestedTimeoutMs?: number,
@@ -304,6 +436,9 @@ function finitePositiveInteger(value: unknown): number | null {
 
 type ToolExecutionApiOptions = {
   timeoutMs?: number;
+  durableCallReceiptKey?: string | null;
+  executionAuthScopeDigest?: string | null;
+  providerIdempotencyKey?: string | null;
   customerDbDataset?: {
     limit: number;
     offset: number;
@@ -347,6 +482,16 @@ function persistableMapRowBytes(row: PersistableMapRow): number {
   return JSON.stringify(row).length;
 }
 
+function failedMapRowLogLabel(row: PersistableMapRow): string {
+  const rowId = row.data?.row_id;
+  if (typeof rowId === 'string' && rowId.trim()) return rowId.trim();
+  if (row.key?.trim()) return row.key.trim();
+  if (typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex)) {
+    return `input:${Math.floor(row.inputIndex)}`;
+  }
+  return 'unknown';
+}
+
 type FieldMapRunResult = {
   completedRows: PersistableMapRow[];
   failedRows: PersistableMapRow[];
@@ -508,9 +653,7 @@ function durableCtxKey(input: {
   staleAfterSeconds?: number | null;
 }): string {
   if (input.operation === 'tool') {
-    throw new Error(
-      'Durable ctx key is only for non-tool call boundaries; tools use buildDurableToolCallCacheKey.',
-    );
+    throw new Error('Tool calls use tool receipt keys.');
   }
   return buildDurableCtxCallCacheKey({
     orgId: input.orgId,
@@ -756,6 +899,13 @@ export class PlayContextImpl {
     eventKey: string;
     timeoutMs: number;
   }> = [];
+  private pendingChildPlayBoundaries: Array<{
+    boundaryId: string;
+    childRunId: string;
+    childPlayName: string;
+  }> = [];
+  private activeScheduledChildPlayCalls = 0;
+  private scheduledChildPlayQuiescenceWaiters: Array<() => void> = [];
   private processedRowCount = 0;
   private sleepBoundaryIndex = 0;
   private fetchCallIndex = 0;
@@ -891,13 +1041,18 @@ export class PlayContextImpl {
       options.workflowId ||
       'anonymous-play';
     const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
+    if (options.requireSharedRateState && !options.rateState) {
+      throw new Error(
+        'Shared rate-state backend is required for this Node runtime substrate.',
+      );
+    }
     this.governor = createPlayExecutionGovernor({
       adapter: 'cjs_node20',
       scope: {
         orgId: options.orgId ?? 'org',
         rootRunId: options.runId ?? options.workflowId ?? 'run',
       },
-      rateState: new InMemoryRateStateBackend(),
+      rateState: options.rateState ?? new InMemoryRateStateBackend(),
       resolvePacing: createPacingResolver(options.getToolQueueHints),
       // A root run keeps depth 0 (its first child play is depth 1) but still
       // seeds its own play id into the ancestry/currentPlayId so the cycle guard
@@ -923,6 +1078,263 @@ export class PlayContextImpl {
     return `${this.governance.currentRunId}:${localId}`;
   }
 
+  private shouldLaunchChildPlaysThroughScheduler(): boolean {
+    return (
+      this.#options.requireSharedRateState === true &&
+      this.#options.durableBoundaries === true
+    );
+  }
+
+  private childPlayBoundaryId(childRunId: string): string {
+    return this.durableBoundaryId(`runPlay:${childRunId}`);
+  }
+
+  private recordPendingChildPlayBoundary(input: {
+    boundaryId: string;
+    childRunId: string;
+    childPlayName: string;
+  }): void {
+    this.pendingChildPlayBoundaries.push(input);
+  }
+
+  private beginScheduledChildPlayCall(): { release: () => void } {
+    this.activeScheduledChildPlayCalls += 1;
+    let released = false;
+    return {
+      release: () => {
+        if (released) return;
+        released = true;
+        this.activeScheduledChildPlayCalls = Math.max(
+          0,
+          this.activeScheduledChildPlayCalls - 1,
+        );
+        if (this.activeScheduledChildPlayCalls !== 0) return;
+        for (const resolve of this.scheduledChildPlayQuiescenceWaiters.splice(
+          0,
+        )) {
+          resolve();
+        }
+      },
+    };
+  }
+
+  private async waitForScheduledChildPlayQuiescence(): Promise<void> {
+    if (this.activeScheduledChildPlayCalls === 0) return;
+    await new Promise<void>((resolve) => {
+      this.scheduledChildPlayQuiescenceWaiters.push(resolve);
+    });
+  }
+
+  private consumePendingChildPlaySuspension(): PlayExecutionSuspension | null {
+    if (this.pendingChildPlayBoundaries.length === 0) return null;
+    const boundaries = [
+      ...new Map(
+        this.pendingChildPlayBoundaries.map((boundary) => [
+          boundary.boundaryId,
+          boundary,
+        ]),
+      ).values(),
+    ];
+    this.pendingChildPlayBoundaries = [];
+    if (boundaries.length === 1) {
+      const boundary = boundaries[0]!;
+      return {
+        kind: 'child_play',
+        boundaryId: boundary.boundaryId,
+        childRunId: boundary.childRunId,
+        childPlayName: boundary.childPlayName,
+      };
+    }
+    return {
+      kind: 'child_play_batch',
+      boundaries,
+    };
+  }
+
+  private childPlayBoundaryOutput<TOutput>(input: {
+    boundaryId: string;
+    childRunId: string;
+    childPlayName: string;
+  }): { found: true; output: TOutput } | { found: false } {
+    const existing = this.checkpoint.resolvedBoundaries?.[input.boundaryId];
+    if (!existing || existing.kind !== 'child_play') return { found: false };
+    if (existing.childRunId !== input.childRunId) return { found: false };
+    if (existing.status === 'completed') {
+      return { found: true, output: existing.output as TOutput };
+    }
+    const message =
+      typeof existing.error === 'string' && existing.error.trim()
+        ? existing.error.trim()
+        : `Child play ${input.childPlayName} (${input.childRunId}) ${existing.status}.`;
+    throw new Error(message);
+  }
+
+  private recordPlayCallStep(input: {
+    playId: string;
+    description?: string | null;
+    nestedSteps?: PlayStep[];
+  }): void {
+    const step = {
+      type: 'play_call' as const,
+      playId: input.playId,
+      nestedSteps: input.nestedSteps ?? [],
+      description: normalizeStepDescription(input.description ?? undefined),
+    };
+    if (this.activeDatasetStep) {
+      this.activeDatasetStep.substeps.push(step);
+    } else {
+      this.steps.push(step);
+    }
+  }
+
+  private runtimeApiHeaders(): Record<string, string> {
+    if (!this.#options.executorToken?.trim()) {
+      throw new Error(
+        'ctx.runPlay scheduled child launch requires an executor token.',
+      );
+    }
+    return {
+      Authorization: `Bearer ${this.#options.executorToken.trim()}`,
+      'Content-Type': 'application/json',
+      ...(this.#options.runtimeSchedulerSchema?.trim()
+        ? {
+            [RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER]:
+              this.#options.runtimeSchedulerSchema.trim(),
+            [SYNTHETIC_RUN_HEADER]: '1',
+          }
+        : {}),
+      ...(this.#options.vercelProtectionBypassToken?.trim()
+        ? {
+            'x-vercel-protection-bypass':
+              this.#options.vercelProtectionBypassToken.trim(),
+          }
+        : {}),
+    };
+  }
+
+  private runtimeApiBaseUrl(): string {
+    const baseUrl = this.#options.baseUrl?.trim();
+    if (!baseUrl) {
+      throw new Error('ctx.runPlay scheduled child launch requires baseUrl.');
+    }
+    return baseUrl.replace(/\/$/, '');
+  }
+
+  private async launchScheduledChildPlay(input: {
+    key: string;
+    childRunId: string;
+    childPlayName: string;
+    childGovernance: GovernanceSnapshot;
+    childInput: Record<string, unknown>;
+    description?: string | null;
+  }): Promise<void> {
+    const parentPlayName =
+      this.governance.currentPlayId ||
+      this.#options.playName ||
+      this.#options.playId ||
+      'anonymous-play';
+    const governance: PlayCallGovernanceSnapshot = {
+      rootRunId: this.governance.rootRunId,
+      parentRunId: this.governance.currentRunId,
+      parentPlayName,
+      key: input.key,
+      ancestryPlayIds: [...this.governance.ancestryPlayIds],
+      callDepth: input.childGovernance.callDepth,
+      playCallCount: input.childGovernance.playCallCount,
+      toolCallCount: input.childGovernance.toolCallCount,
+      retryCount: input.childGovernance.retryCount,
+      descendantCount: input.childGovernance.descendantCount,
+      waterfallStepExecutions: input.childGovernance.waterfallStepExecutions,
+    };
+    const response = await fetch(
+      `${this.runtimeApiBaseUrl()}/api/v2/plays/run`,
+      {
+        method: 'POST',
+        headers: this.runtimeApiHeaders(),
+        body: JSON.stringify({
+          name: input.childPlayName,
+          workflowId: input.childRunId,
+          input: input.childInput,
+          profile: 'hatchet',
+          waitForCompletionMs: 0,
+          internalRunPlay: {
+            ...governance,
+            description: input.description ?? null,
+          },
+        }),
+      },
+    );
+    if (!response.ok) {
+      const body = (await response.json().catch(() => null)) as {
+        error?: unknown;
+      } | null;
+      const message =
+        typeof body?.error === 'string' && body.error.trim()
+          ? body.error.trim()
+          : response.statusText;
+      throw new Error(
+        `ctx.runPlay(${input.key}) failed to launch child play "${input.childPlayName}": ${message} (status ${response.status}).`,
+      );
+    }
+  }
+
+  private async readScheduledChildTerminal<TOutput>(input: {
+    childRunId: string;
+    childPlayName: string;
+  }): Promise<
+    | { status: 'completed'; output: TOutput }
+    | { status: 'failed' | 'cancelled'; error: string }
+    | null
+  > {
+    const response = await fetch(
+      `${this.runtimeApiBaseUrl()}/api/v2/internal/play-runtime`,
+      {
+        method: 'POST',
+        headers: this.runtimeApiHeaders(),
+        body: JSON.stringify({
+          action: 'read_child_run_terminal_snapshot',
+          parentRunId: this.governance.currentRunId,
+          childRunId: input.childRunId,
+          childPlayName: input.childPlayName,
+        }),
+      },
+    );
+    const body = (await response.json().catch(() => null)) as {
+      state?: {
+        data?: {
+          status?: unknown;
+          result?: unknown;
+          error?: unknown;
+        };
+      };
+      error?: unknown;
+    } | null;
+    if (!response.ok) {
+      const message =
+        typeof body?.error === 'string' && body.error.trim()
+          ? body.error.trim()
+          : response.statusText;
+      throw new Error(
+        `ctx.runPlay failed to read child terminal snapshot for "${input.childPlayName}": ${message} (status ${response.status}).`,
+      );
+    }
+    const data = body?.state?.data;
+    const status = String(data?.status ?? '').toLowerCase();
+    if (status === 'completed') {
+      return { status, output: data?.result as TOutput };
+    }
+    if (status === 'failed' || status === 'cancelled') {
+      return {
+        status,
+        error:
+          typeof data?.error === 'string' && data.error.trim()
+            ? data.error.trim()
+            : `Child play ${input.childPlayName} (${input.childRunId}) ${status}.`,
+      };
+    }
+    return null;
+  }
+
   private emitScopedRowUpdate(
     key: string | null,
     tableNamespace: string | null,
@@ -990,7 +1402,7 @@ export class PlayContextImpl {
       this.#options.runId
     ) {
       const response = await fetch(
-        `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`,
+        `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime`,
         {
           method: 'POST',
           headers: {
@@ -1102,6 +1514,7 @@ export class PlayContextImpl {
     runId: string,
     reclaimRunning = false,
     forceRefresh = false,
+    forceFailedRefresh = false,
   ): Promise<RuntimeStepReceipt | null> {
     if (!this.#options.claimRuntimeStepReceipt) {
       return null;
@@ -1109,8 +1522,11 @@ export class PlayContextImpl {
     const claimed = await this.#options.claimRuntimeStepReceipt({
       key,
       runId,
+      runAttempt: this.currentRunAttempt,
+      leaseAware: true,
       ...(reclaimRunning ? { reclaimRunning: true } : {}),
       ...(forceRefresh ? { forceRefresh: true } : {}),
+      ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
     });
     if (!claimed || typeof claimed.key !== 'string' || !claimed.key.trim()) {
       return null;
@@ -1126,6 +1542,7 @@ export class PlayContextImpl {
     key: string,
     runId: string,
     output: unknown | null,
+    leaseId?: string | null,
   ): Promise<RuntimeStepReceipt | null> {
     assertNoSecretTaint(output, 'ctx.step receipt output');
     if (!this.#options.completeRuntimeStepReceipt) {
@@ -1134,6 +1551,8 @@ export class PlayContextImpl {
     const completed = await this.#options.completeRuntimeStepReceipt({
       key,
       runId,
+      runAttempt: this.currentRunAttempt,
+      ...(leaseId ? { leaseId } : {}),
       output,
     });
     if (
@@ -1150,10 +1569,167 @@ export class PlayContextImpl {
     };
   }
 
+  private async releaseRuntimeStepReceipt(
+    key: string,
+    runId: string,
+    leaseId?: string | null,
+  ): Promise<RuntimeStepReceipt | null> {
+    if (!this.#options.releaseRuntimeStepReceipt) {
+      return null;
+    }
+    const released = await this.#options.releaseRuntimeStepReceipt({
+      key,
+      runId,
+      runAttempt: this.currentRunAttempt,
+      ...(leaseId ? { leaseId } : {}),
+    });
+    if (!released || typeof released.key !== 'string' || !released.key.trim()) {
+      return null;
+    }
+    return {
+      ...released,
+      key: released.key.trim(),
+      runId: released.runId ?? null,
+    };
+  }
+
+  private async heartbeatRuntimeStepReceipt(
+    key: string,
+    runId: string,
+    leaseId: string,
+  ): Promise<RuntimeStepReceipt | null> {
+    if (!this.#options.heartbeatRuntimeStepReceipts) {
+      return null;
+    }
+    const [receipt] = await this.#options.heartbeatRuntimeStepReceipts({
+      runId,
+      runAttempt: this.currentRunAttempt,
+      leaseId,
+      keys: [key],
+    });
+    if (!receipt || typeof receipt.key !== 'string' || !receipt.key.trim()) {
+      return null;
+    }
+    return {
+      ...receipt,
+      key: receipt.key.trim(),
+      runId: receipt.runId ?? null,
+    };
+  }
+
+  private async assertRuntimeToolReceiptOwnership(
+    requests: ToolCallRequest[],
+  ): Promise<void> {
+    const targets = requests.flatMap((request) => {
+      const receiptKey = request.receiptKey?.trim() || null;
+      if (!receiptKey) return [];
+      const leaseId = request.receiptLeaseId?.trim() || null;
+      if (!leaseId) return [];
+      return [{ receiptKey, leaseId }];
+    });
+    if (targets.length === 0) return;
+
+    const byLeaseId = new Map<string, string[]>();
+    for (const target of targets) {
+      const keys = byLeaseId.get(target.leaseId) ?? [];
+      keys.push(target.receiptKey);
+      byLeaseId.set(target.leaseId, keys);
+    }
+
+    if (this.#options.heartbeatRuntimeStepReceipts) {
+      for (const [leaseId, keys] of byLeaseId) {
+        const receipts = await this.#options.heartbeatRuntimeStepReceipts({
+          runId: this.currentRunId,
+          runAttempt: this.currentRunAttempt,
+          leaseId,
+          keys,
+        });
+        this.assertRuntimeToolReceiptHeartbeatResult(keys, leaseId, receipts);
+      }
+      return;
+    }
+
+    if (
+      !this.#options.getRuntimeStepReceipt &&
+      !this.#options.getRuntimeStepReceipts
+    ) {
+      throw new RuntimeReceiptLeaseLostError({
+        receiptKey: targets[0]?.receiptKey ?? 'unknown',
+        runId: this.currentRunId,
+        leaseId: targets[0]?.leaseId ?? 'unknown',
+      });
+    }
+
+    const latest = await this.getRuntimeStepReceipts(
+      targets.map((target) => target.receiptKey),
+    );
+    for (const target of targets) {
+      const receipt = latest.get(target.receiptKey);
+      if (!this.runtimeToolReceiptStillOwned(receipt, target.leaseId)) {
+        throw new RuntimeReceiptLeaseLostError({
+          receiptKey: target.receiptKey,
+          runId: this.currentRunId,
+          leaseId: target.leaseId,
+        });
+      }
+    }
+  }
+
+  private assertRuntimeToolReceiptHeartbeatResult(
+    keys: string[],
+    leaseId: string,
+    receipts: Array<RuntimeStepReceipt | null>,
+  ): void {
+    for (let index = 0; index < keys.length; index += 1) {
+      const receipt = this.normalizeRuntimeStepReceipt(
+        keys[index] ?? '',
+        receipts[index],
+      );
+      if (!this.runtimeToolReceiptStillOwned(receipt, leaseId)) {
+        throw new RuntimeReceiptLeaseLostError({
+          receiptKey: keys[index] ?? 'unknown',
+          runId: this.currentRunId,
+          leaseId,
+        });
+      }
+    }
+  }
+
+  private runtimeToolReceiptStillOwned(
+    receipt: RuntimeStepReceipt | null | undefined,
+    leaseId: string,
+  ): boolean {
+    if (!receipt || receipt.status !== 'running') return false;
+    if (receipt.leaseId !== leaseId) return false;
+    const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
+    return !ownerRunId || ownerRunId === this.currentRunId;
+  }
+
+  private runtimeToolReceiptStillOwnedByCurrentAttempt(
+    receipt: RuntimeStepReceipt | null | undefined,
+    leaseId: string | null | undefined,
+  ): boolean {
+    if (!receipt || receipt.status !== 'running') return false;
+    if (leaseId?.trim()) {
+      if (receipt.leaseId !== leaseId.trim()) return false;
+    } else if (receipt.leaseId != null) {
+      return false;
+    }
+    const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
+    if (ownerRunId !== this.currentRunId) return false;
+    const ownerAttempt =
+      typeof receipt.leaseOwnerAttempt === 'number'
+        ? receipt.leaseOwnerAttempt
+        : 0;
+    return ownerAttempt === this.currentRunAttempt;
+  }
+
   private async failRuntimeStepReceipt(
     key: string,
     runId: string,
     error: string,
+    leaseId?: string | null,
+    failureKind?: WorkReceiptFailureKind | null,
   ): Promise<RuntimeStepReceipt | null> {
     if (!this.#options.failRuntimeStepReceipt) {
       return null;
@@ -1161,6 +1737,9 @@ export class PlayContextImpl {
     const failed = await this.#options.failRuntimeStepReceipt({
       key,
       runId,
+      runAttempt: this.currentRunAttempt,
+      ...(leaseId ? { leaseId } : {}),
+      ...(failureKind ? { failureKind } : {}),
       error,
     });
     if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) {
@@ -1220,6 +1799,7 @@ export class PlayContextImpl {
     runId: string,
     reclaimRunning = false,
     forceRefresh = false,
+    forceFailedRefresh = false,
   ): Promise<Map<string, RuntimeStepReceipt>> {
     const uniqueKeys = [
       ...new Set(keys.map((key) => key.trim()).filter(Boolean)),
@@ -1229,8 +1809,11 @@ export class PlayContextImpl {
       ? await this.#options.claimRuntimeStepReceipts({
           keys: uniqueKeys,
           runId,
+          runAttempt: this.currentRunAttempt,
+          leaseAware: true,
           ...(reclaimRunning ? { reclaimRunning: true } : {}),
           ...(forceRefresh ? { forceRefresh: true } : {}),
+          ...(forceFailedRefresh ? { forceFailedRefresh: true } : {}),
         })
       : await Promise.all(
           uniqueKeys.map((key) =>
@@ -1239,6 +1822,7 @@ export class PlayContextImpl {
               runId,
               reclaimRunning,
               forceRefresh,
+              forceFailedRefresh,
             ),
           ),
         );
@@ -1254,26 +1838,44 @@ export class PlayContextImpl {
   }
 
   private async completeRuntimeStepReceipts(
-    receipts: Array<{ key: string; runId: string; output: unknown | null }>,
+    receipts: Array<{
+      key: string;
+      runId: string;
+      runAttempt?: number | null;
+      output: unknown | null;
+      leaseId?: string | null;
+    }>,
   ): Promise<Map<string, RuntimeStepReceipt>> {
     const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
     if (normalizedInputs.length === 0) return new Map();
     for (const receipt of normalizedInputs) {
       assertNoSecretTaint(receipt.output, 'ctx.tool receipt output');
     }
-    const completed = this.#options.completeRuntimeStepReceipts
-      ? await this.#options.completeRuntimeStepReceipts({
-          receipts: normalizedInputs,
-        })
-      : await Promise.all(
-          normalizedInputs.map((receipt) =>
-            this.completeRuntimeStepReceipt(
-              receipt.key,
-              receipt.runId,
-              receipt.output,
+    let completed: Array<RuntimeStepReceipt | null>;
+    try {
+      completed = this.#options.completeRuntimeStepReceipts
+        ? await this.#options.completeRuntimeStepReceipts({
+            receipts: normalizedInputs.map((receipt) => ({
+              ...receipt,
+              runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
+            })),
+          })
+        : await Promise.all(
+            normalizedInputs.map((receipt) =>
+              this.completeRuntimeStepReceipt(
+                receipt.key,
+                receipt.runId,
+                receipt.output,
+                receipt.leaseId,
+              ),
             ),
-          ),
-        );
+          );
+    } catch (error) {
+      await this.recoverRuntimeStepReceiptCompletionsAfterBulkError(
+        normalizedInputs,
+      );
+      throw error;
+    }
     const byKey = new Map<string, RuntimeStepReceipt>();
     for (let index = 0; index < completed.length; index += 1) {
       const normalized = this.normalizeRuntimeStepReceipt(
@@ -1282,17 +1884,123 @@ export class PlayContextImpl {
       );
       if (normalized) byKey.set(normalized.key, normalized);
     }
+    await this.reconcileMissingRuntimeStepReceiptCompletions(
+      normalizedInputs,
+      byKey,
+    );
     return byKey;
   }
 
+  private async reconcileMissingRuntimeStepReceiptCompletions(
+    receipts: Array<{
+      key: string;
+      runId: string;
+      runAttempt?: number | null;
+      output: unknown | null;
+      leaseId?: string | null;
+    }>,
+    completedByKey: Map<string, RuntimeStepReceipt>,
+  ): Promise<void> {
+    if (
+      receipts.length === completedByKey.size ||
+      (!this.#options.getRuntimeStepReceipt &&
+        !this.#options.getRuntimeStepReceipts)
+    ) {
+      return;
+    }
+    const missingKeys = receipts
+      .map((receipt) => receipt.key.trim())
+      .filter((key) => key && !completedByKey.has(key));
+    if (missingKeys.length === 0) return;
+    const latest = await this.getRuntimeStepReceipts(missingKeys);
+    for (const key of missingKeys) {
+      const recovered = latest.get(key);
+      if (
+        recovered?.status === 'completed' ||
+        recovered?.status === 'skipped'
+      ) {
+        completedByKey.set(key, recovered);
+      }
+    }
+    const recoveredCount = missingKeys.filter((key) =>
+      completedByKey.has(key),
+    ).length;
+    if (recoveredCount > 0) {
+      this.log(
+        `Runtime tool receipt completion response reconciled ${recoveredCount}/${missingKeys.length} missing receipt(s) from durable store read-back.`,
+      );
+    }
+  }
+
+  private async recoverRuntimeStepReceiptCompletionsAfterBulkError(
+    receipts: Array<{
+      key: string;
+      runId: string;
+      runAttempt?: number | null;
+      output: unknown | null;
+      leaseId?: string | null;
+    }>,
+  ): Promise<void> {
+    if (
+      receipts.length === 0 ||
+      (!this.#options.getRuntimeStepReceipt &&
+        !this.#options.getRuntimeStepReceipts)
+    ) {
+      return;
+    }
+    let latest: Map<string, RuntimeStepReceipt>;
+    try {
+      latest = await this.getRuntimeStepReceipts(
+        receipts.map((receipt) => receipt.key),
+      );
+    } catch {
+      return;
+    }
+    await Promise.allSettled(
+      receipts.map(async (receipt) => {
+        const recovered = latest.get(receipt.key);
+        if (
+          recovered?.status === 'completed' ||
+          recovered?.status === 'skipped'
+        ) {
+          return;
+        }
+        if (
+          !this.runtimeToolReceiptStillOwnedByCurrentAttempt(
+            recovered,
+            receipt.leaseId,
+          )
+        ) {
+          return;
+        }
+        await this.completeRuntimeStepReceipt(
+          receipt.key,
+          receipt.runId,
+          receipt.output,
+          receipt.leaseId,
+        );
+      }),
+    );
+  }
+
   private async failRuntimeStepReceipts(
-    receipts: Array<{ key: string; runId: string; error: string }>,
+    receipts: Array<{
+      key: string;
+      runId: string;
+      runAttempt?: number | null;
+      error: string;
+      leaseId?: string | null;
+      failureKind?: WorkReceiptFailureKind | null;
+    }>,
   ): Promise<Map<string, RuntimeStepReceipt>> {
     const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
     if (normalizedInputs.length === 0) return new Map();
     const failed = this.#options.failRuntimeStepReceipts
       ? await this.#options.failRuntimeStepReceipts({
-          receipts: normalizedInputs,
+          receipts: normalizedInputs.map((receipt) => ({
+            ...receipt,
+            runAttempt: receipt.runAttempt ?? this.currentRunAttempt,
+          })),
         })
       : await Promise.all(
           normalizedInputs.map((receipt) =>
@@ -1300,6 +2008,8 @@ export class PlayContextImpl {
               receipt.key,
               receipt.runId,
               receipt.error,
+              receipt.leaseId,
+              receipt.failureKind,
             ),
           ),
         );
@@ -1327,45 +2037,98 @@ export class PlayContextImpl {
     );
   }
 
-  private async durableToolCallCacheKey(input: {
+  private async durableToolCallCacheKeyForScope(input: {
     toolId: string;
     requestInput: Record<string, unknown>;
+    executionAuthScopeDigest?: string | null;
     staleAfterSeconds?: number | null;
+    playLocalScope?: string | null;
   }): Promise<string> {
     const providerActionVersion =
       (await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
       '';
+    const executionAuthScopeDigest =
+      input.executionAuthScopeDigest?.trim() ||
+      (await this.resolveToolAuthScopeDigest(input.toolId))?.trim() ||
+      null;
     return buildDurableToolCallCacheKey({
       orgId: this.#options.orgId,
-      playId: this.governance.currentPlayId,
+      playLocalScope: input.playLocalScope,
       toolId: input.toolId,
       requestInput: input.requestInput,
       authScopeDigest: buildDurableToolCallAuthScopeDigest({
         orgId: this.#options.orgId,
-        userEmail: this.#options.userEmail,
         toolId: input.toolId,
+        executionAuthScopeDigest,
       }),
       providerActionVersion,
       staleAfterSeconds: input.staleAfterSeconds,
     });
   }
 
+  private async durableToolCallCacheKey(input: {
+    toolId: string;
+    requestInput: Record<string, unknown>;
+    executionAuthScopeDigest?: string | null;
+    staleAfterSeconds?: number | null;
+  }): Promise<string> {
+    return await this.durableToolCallCacheKeyForScope({
+      ...input,
+      playLocalScope: this.governance.currentPlayId,
+    });
+  }
+
+  private async resolveToolAuthScopeDigest(
+    toolId: string,
+  ): Promise<string | null> {
+    const configured = await this.#options.getToolAuthScopeDigest?.(toolId);
+    if (typeof configured === 'string' && configured.trim()) {
+      return configured.trim();
+    }
+    return null;
+  }
+
+  private invalidateToolAuthScopeDigest(toolId: string): void {
+    this.#options.invalidateToolAuthScopeDigest?.(toolId);
+  }
+
   private durableReceiptExecutionStore(): DurableReceiptExecutionStore {
     return {
       enabled: Boolean(this.#options.claimRuntimeStepReceipt),
       get: (receiptKey) => this.getRuntimeStepReceipt(receiptKey),
       getMany: (receiptKeys) => this.getRuntimeStepReceipts(receiptKeys),
-      claim: (receiptKey, runId, reclaimRunning, forceRefresh) =>
+      claim: (
+        receiptKey,
+        runId,
+        reclaimRunning,
+        forceRefresh,
+        forceFailedRefresh,
+      ) =>
         this.claimRuntimeStepReceipt(
           receiptKey,
           runId,
           reclaimRunning,
           forceRefresh,
+          forceFailedRefresh,
+        ),
+      complete: (receiptKey, runId, output, leaseId) =>
+        this.completeRuntimeStepReceipt(receiptKey, runId, output, leaseId),
+      release: (receiptKey, runId, leaseId) =>
+        this.releaseRuntimeStepReceipt(receiptKey, runId, leaseId),
+      ...(this.#options.heartbeatRuntimeStepReceipts
+        ? {
+            heartbeat: (receiptKey, runId, leaseId) =>
+              this.heartbeatRuntimeStepReceipt(receiptKey, runId, leaseId),
+          }
+        : {}),
+      fail: (receiptKey, runId, error, leaseId, failureKind) =>
+        this.failRuntimeStepReceipt(
+          receiptKey,
+          runId,
+          error,
+          leaseId,
+          failureKind,
         ),
-      complete: (receiptKey, runId, output) =>
-        this.completeRuntimeStepReceipt(receiptKey, runId, output),
-      fail: (receiptKey, runId, error) =>
-        this.failRuntimeStepReceipt(receiptKey, runId, error),
       canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
       canPersistCompletion: Boolean(this.#options.completeRuntimeStepReceipt),
     };
@@ -1407,7 +2170,8 @@ export class PlayContextImpl {
         source: DurableReceiptRecoverySource,
       ) => T;
       onClaimedResult?: (output: T, receiptKey: string) => T;
-      execute: () => Promise<T>;
+      shouldPersistFailure?: (error: unknown) => boolean;
+      execute: (context: { leaseId: string | null }) => Promise<T>;
     },
   ): Promise<T> {
     const receiptKey =
@@ -1436,6 +2200,7 @@ export class PlayContextImpl {
       markSkipped: opts.markSkipped,
       onRecovered: opts.onRecovered,
       onClaimedResult: opts.onClaimedResult,
+      shouldPersistFailure: opts.shouldPersistFailure,
       formatError: (error) => this.formatRuntimeError(error),
       log: (message) => this.log(message),
       execute: opts.execute,
@@ -1446,6 +2211,13 @@ export class PlayContextImpl {
     return this.#options.runId ?? this.governance.currentRunId ?? 'unknown';
   }
 
+  private get currentRunAttempt(): number {
+    const attempt = this.#options.runAttempt;
+    return typeof attempt === 'number' && Number.isFinite(attempt)
+      ? Math.max(0, Math.floor(attempt))
+      : 0;
+  }
+
   private emitScopedFieldMetaUpdate(input: {
     rowId: number;
     key: string | null;
@@ -1659,12 +2431,17 @@ export class PlayContextImpl {
 
   private effectiveToolCallCachePolicy(options?: ToolCallOptions): {
     force: boolean;
+    forceFailedRefresh: boolean;
     staleAfterSeconds?: number | null;
   } {
     return {
       force:
         options?.force === true ||
         this.#options.cachePolicy?.forceToolRefresh === true,
+      forceFailedRefresh:
+        options?.force === true ||
+        this.#options.cachePolicy?.forceToolRefresh === true ||
+        this.#options.cachePolicy?.forceFailedToolRefresh === true,
       staleAfterSeconds: options?.staleAfterSeconds ?? null,
     };
   }
@@ -1685,6 +2462,19 @@ export class PlayContextImpl {
     return this.checkpoint.completedToolBatches[toolId]?.[rowCacheKey];
   }
 
+  private getCachedToolResultCandidate(
+    toolId: string,
+    rowCacheKeys: readonly string[],
+  ): { cacheKey: string; result: ToolBatchResult } | null {
+    for (const rowCacheKey of rowCacheKeys) {
+      const cached = this.getCachedToolResult(toolId, rowCacheKey);
+      if (cached?.done) {
+        return { cacheKey: rowCacheKey, result: cached };
+      }
+    }
+    return null;
+  }
+
   private cacheToolResult(
     toolId: string,
     rowCacheKey: string,
@@ -1914,6 +2704,7 @@ export class PlayContextImpl {
             {
               key: receiptKey,
               runId: this.currentRunId,
+              leaseId: request.receiptLeaseId ?? null,
               output: serializeToolExecuteResult(wrapped),
             },
           ])
@@ -1969,12 +2760,13 @@ export class PlayContextImpl {
             {
               key: receiptKey,
               runId: this.currentRunId,
+              leaseId: entry.request.receiptLeaseId ?? null,
               output: serializeToolExecuteResult(entry.wrapped),
             },
           ]
         : [];
     });
-    let completedByKey: Map<string, RuntimeStepReceipt>;
+    let completedByKey = new Map<string, RuntimeStepReceipt>();
     if (receiptInputs.length > 0) {
       try {
         completedByKey = await this.completeRuntimeStepReceipts(receiptInputs);
@@ -1982,10 +2774,15 @@ export class PlayContextImpl {
         // Completion failed AFTER successful (billed) tool calls: trip the
         // breaker so remaining rows stop dispatching new provider calls.
         tripRuntimePersistenceLatch(this.persistenceLatch, receiptError);
+        await Promise.all(
+          wrappedEntries.map((entry) =>
+            this.rejectToolCall(toolId, entry.request, receiptError, {
+              persistReceiptFailure: false,
+            }),
+          ),
+        );
         throw receiptError;
       }
-    } else {
-      completedByKey = new Map<string, RuntimeStepReceipt>();
     }
     const results: unknown[] = [];
     for (const entry of wrappedEntries) {
@@ -2009,9 +2806,23 @@ export class PlayContextImpl {
     completed: RuntimeStepReceipt | null | undefined,
   ): Promise<unknown> {
     const cacheKey = request.cacheKey;
+    const receiptKey = request.receiptKey?.trim() || null;
+    const canPersistReceiptCompletion = Boolean(
+      this.#options.completeRuntimeStepReceipt ||
+      this.#options.completeRuntimeStepReceipts,
+    );
+    if (
+      receiptKey &&
+      canPersistReceiptCompletion &&
+      completed?.status !== 'completed' &&
+      completed?.status !== 'skipped'
+    ) {
+      throw new Error(
+        `Durable tool call ${receiptKey} lost receipt ownership before completion.`,
+      );
+    }
     const finalWrapped =
-      (completed?.status === 'completed' || completed?.status === 'skipped') &&
-      completed.runId !== this.currentRunId
+      completed?.status === 'completed' || completed?.status === 'skipped'
         ? await this.wrapToolExecutionResult({
             toolId,
             status:
@@ -2030,8 +2841,8 @@ export class PlayContextImpl {
                 : {
                     kind: 'cache',
                     cacheKey,
-                    receiptKey: request.receiptKey ?? '',
-                    attachedToReceiptKey: request.receiptKey,
+                    receiptKey: receiptKey ?? '',
+                    attachedToReceiptKey: receiptKey,
                   },
             ),
           })
@@ -2076,7 +2887,9 @@ export class PlayContextImpl {
           {
             key: receiptKey,
             runId: this.currentRunId,
+            leaseId: request.receiptLeaseId ?? null,
             error: message,
+            failureKind: runtimeReceiptFailureKindForError(error),
           },
         ]);
       } catch (receiptError) {
@@ -2360,6 +3173,7 @@ export class PlayContextImpl {
       this.toOutputRow(item as Record<string, unknown>),
     );
     itemOriginalIndexes = materializedItems.map((_item, index) => index);
+    const mapStartSeedRowsByKey = new Map<string, Record<string, unknown>>();
     if (this.#options.onMapStart) {
       const shouldPassRowKey = explicitKeyResolver != null;
       // toSerializableCsvAliasedRow (not a plain spread): projected CSV
@@ -2381,6 +3195,7 @@ export class PlayContextImpl {
           playName: this.#options.playName,
           playId: this.#options.playId,
           runId: this.#options.runId,
+          executorToken: this.#options.executorToken,
           staticPipeline: this.#options.staticPipeline,
           forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
         },
@@ -2388,13 +3203,14 @@ export class PlayContextImpl {
       resolvedTableNamespace = normalizeTableNamespace(
         mapStartResult.tableNamespace,
       );
+      if ((mapStartResult.blockedRows?.length ?? 0) > 0) {
+        throw new RuntimeSheetRowsBlockedError({
+          tableNamespace: resolvedTableNamespace,
+          blockedRows: mapStartResult.blockedRows ?? [],
+        });
+      }
       const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
         resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
-      const pendingKeys = new Set(
-        mapStartResult.pendingRows.map((row, index) =>
-          persistedRowIdentity(row, index),
-        ),
-      );
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
       for (
         let index = 0;
@@ -2405,15 +3221,20 @@ export class PlayContextImpl {
         const rowKey = persistedRowIdentity(row, index);
         if (rowKey) pendingRowsByKey.set(rowKey, row);
       }
-      const pendingItems = materializedItems
+      for (const row of mapStartResult.completedRows ?? []) {
+        const rowKey = persistedRowIdentity(row);
+        if (!rowKey) continue;
+        mapStartSeedRowsByKey.set(rowKey, row);
+      }
+      const seededItems = materializedItems
         .map((item, index) => ({
           row: this.toOutputRow(item as Record<string, unknown>),
           index,
         }))
-        .filter(({ row, index }) => pendingKeys.has(rowIdentity(row, index)))
         .map(({ row, index }) => {
           const rowKey = rowIdentity(row, index);
-          const persisted = pendingRowsByKey.get(rowKey);
+          const persisted =
+            pendingRowsByKey.get(rowKey) ?? mapStartSeedRowsByKey.get(rowKey);
           // Persisted rows lose the non-enumerable CSV alias projections on
           // the sheet round-trip. Merge the persisted data/meta over the
           // in-memory row so key functions and column resolvers keep seeing
@@ -2423,8 +3244,8 @@ export class PlayContextImpl {
             index,
           };
         });
-      itemsToProcess = pendingItems.map((item) => item.row);
-      itemOriginalIndexes = pendingItems.map((item) => item.index);
+      itemsToProcess = seededItems.map((item) => item.row);
+      itemOriginalIndexes = seededItems.map((item) => item.index);
     }
 
     const mapScope = this.createMapExecutionScope({
@@ -2495,6 +3316,7 @@ export class PlayContextImpl {
             playName: this.#options.playName,
             playId: this.#options.playId,
             runId: this.#options.runId,
+            executorToken: this.#options.executorToken,
             tableNamespace: resolvedTableNamespace,
             rows: chunk,
             outputFields: datasetColumnNames.filter((field) =>
@@ -2580,7 +3402,7 @@ export class PlayContextImpl {
     const directCompletedResults = mapResult.completedRows.map((row) =>
       this.toPublicOutputRow(row.data),
     );
-    const results =
+    let results =
       mapResult.failedRows.length === 0 &&
       directCompletedResults.length === rawItems.length
         ? directCompletedResults
@@ -2633,6 +3455,49 @@ export class PlayContextImpl {
     }
     this.activeMapCellMeta = null;
 
+    if (this.#options.onMapRowsCompleted) {
+      results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
+        mapName: normalizedMapNamespace,
+        tableNamespace: resolvedTableNamespace,
+        runId: this.#options.runId,
+        expectedRows: totalInputCount,
+        currentRows: results,
+        failedRowCount: mapResult.failedRows.length,
+        log: (line) => this.log(line),
+        readPersistedRows: async (readInput) => {
+          if (
+            !this.#options.baseUrl ||
+            !this.#options.executorToken ||
+            !this.#options.playName ||
+            !this.#options.runId
+          ) {
+            throw new Error(
+              `Runtime sheet finalization mismatch for ctx.dataset("${normalizedMapNamespace}"): ` +
+                'cannot verify terminal persisted rows because the Node runtime context is missing ' +
+                'baseUrl, executorToken, playName, or runId.',
+            );
+          }
+          const result = await readRuntimeSheetDatasetRows(
+            {
+              baseUrl: this.#options.baseUrl,
+              executorToken: this.#options.executorToken,
+              playName: this.#options.playName,
+              userEmail: this.#options.userEmail,
+              runId: this.#options.runId,
+            },
+            {
+              tableNamespace: resolvedTableNamespace,
+              runId: this.#options.runId,
+              rowMode: readInput.rowMode,
+              limit: readInput.limit,
+              offset: readInput.offset,
+            },
+          );
+          return result.rows;
+        },
+      });
+    }
+
     return createDeferredPlayDataset({
       datasetKind: 'map',
       datasetId: createRuntimeDatasetId(
@@ -2909,7 +3774,7 @@ export class PlayContextImpl {
       this.lastDatasetStep = datasetStep;
       this.activeDatasetStep = null;
       this.log(
-        `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
+        `Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
       );
       return {
         completedRows: results.map((row, index) =>
@@ -3183,6 +4048,17 @@ export class PlayContextImpl {
       }
       return result.value;
     });
+    const accountedRowIndexes = new Set<number>();
+    for (const row of completedRowsToPersist) {
+      if (typeof row.inputIndex === 'number') {
+        accountedRowIndexes.add(row.inputIndex);
+      }
+    }
+    for (const row of failedRowsToPersist) {
+      if (typeof row.inputIndex === 'number') {
+        accountedRowIndexes.add(row.inputIndex);
+      }
+    }
     if (this.pendingRowEventBoundaries.length > 0) {
       const completedRowKeys = new Set<string>();
       for (let index = 0; index < settledResults.length; index += 1) {
@@ -3195,6 +4071,7 @@ export class PlayContextImpl {
           if (key) {
             completedRowKeys.add(key);
           }
+          accountedRowIndexes.add(executionRowIndex(index));
           continue;
         }
         const row = result as Record<string, unknown>;
@@ -3238,6 +4115,19 @@ export class PlayContextImpl {
         boundaries: uniqueBoundaries,
       });
     }
+    if (accountedRowIndexes.size !== items.length) {
+      const expectedRowIndexes = items.map((_item, index) =>
+        executionRowIndex(index),
+      );
+      const missingRowIndexes = expectedRowIndexes.filter(
+        (index) => !accountedRowIndexes.has(index),
+      );
+      throw new Error(
+        `Runtime map execution accounting mismatch for ctx.dataset("${normalizedTableNamespace}"): ` +
+          `expected ${items.length} row(s), accounted for ${accountedRowIndexes.size}; ` +
+          `missing input index(es): ${missingRowIndexes.slice(0, 10).join(', ') || 'unknown'}.`,
+      );
+    }
     const results = settledResults.filter(
       (result): result is Record<string, unknown> =>
         result !== WAITING_ROW && result !== FAILED_ROW,
@@ -3249,8 +4139,18 @@ export class PlayContextImpl {
         emitEventType: 'map.completed',
       });
     }
+    for (const failedRow of failedRowsToPersist.slice(0, 3)) {
+      this.log(
+        `row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`,
+      );
+    }
+    if (failedRowsToPersist.length > 3) {
+      this.log(
+        `${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`,
+      );
+    }
     this.log(
-      `Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} duplicate keys skipped)`,
+      `Map completed: ${results.length + completedRows} results (${results.length} succeeded, ${failedRowsToPersist.length} failed, ${completedRows} duplicate keys skipped)`,
     );
     return {
       completedRows: [...completedRowsToPersist].sort(
@@ -3560,6 +4460,13 @@ export class PlayContextImpl {
       if (!hasPendingWork) {
         const status = await raceSettled();
         if (status === 'done' && this.toolCallQueue.length === 0) {
+          if (!this.activeDatasetStep) {
+            const childPlaySuspension =
+              this.consumePendingChildPlaySuspension();
+            if (childPlaySuspension) {
+              throw new PlayExecutionSuspendedError(childPlaySuspension);
+            }
+          }
           break;
         }
 
@@ -3656,18 +4563,23 @@ export class PlayContextImpl {
       toolId,
       requestInput: input,
     });
+    let executionAuthScopeDigest =
+      (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
     const cacheableToolResult = !isQueryResultDatasetReadRequest(toolId, input);
-    const durableCacheKey = await this.durableToolCallCacheKey({
+    let durableCacheKey = await this.durableToolCallCacheKey({
       toolId,
       requestInput: input,
+      executionAuthScopeDigest,
       staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
     });
-    const durableReceiptKey = cacheableToolResult ? durableCacheKey : null;
+    const checkpointCacheKeys = [durableCacheKey];
     const eventWaitHandler =
       (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
     const store = rowContext.getStore();
 
-    const executeTool = async (): Promise<unknown> => {
+    const executeTool = async (context?: {
+      leaseId?: string | null;
+    }): Promise<unknown> => {
       if (eventWaitHandler) {
         return this.executeIntegrationEventWaitTool(
           toolId,
@@ -3694,17 +4606,17 @@ export class PlayContextImpl {
           ? null
           : toolCachePolicy.force
             ? null
-            : this.getCachedToolResult(toolId, directCacheKey);
-        if (cached?.done) {
-          this.log(`Tool cache hit: ${toolId} exact payload`);
+            : this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
+        if (cached) {
+          this.log(`Calling tool: ${toolId} recovered from checkpoint`);
           return await this.wrapToolExecutionResult({
             toolId,
-            status: cached.result == null ? 'no_result' : 'completed',
-            result: cached.result,
+            status: cached.result.result == null ? 'no_result' : 'completed',
+            result: cached.result.result,
             requestInput: input,
             execution: toolExecutionMetadataForOutcome({
               kind: 'checkpoint',
-              cacheKey: directCacheKey,
+              cacheKey: cached.cacheKey,
             }),
           });
         }
@@ -3714,6 +4626,17 @@ export class PlayContextImpl {
             : `Calling tool: ${toolId}`,
         );
         const execution = await this.callToolExecutionAPI(toolId, input, {
+          ...(cacheableToolResult
+            ? {
+                durableCallReceiptKey: directCacheKey,
+                executionAuthScopeDigest,
+                providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({
+                  cacheKey: directCacheKey,
+                  force: toolCachePolicy.force,
+                  leaseId: context?.leaseId ?? null,
+                }),
+              }
+            : {}),
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
         });
         const wrapped = await this.wrapToolExecutionResult({
@@ -3763,17 +4686,17 @@ export class PlayContextImpl {
         ? null
         : toolCachePolicy.force
           ? null
-          : this.getCachedToolResult(toolId, toolResultCacheKey);
-      if (cached?.done) {
-        this.log(`  Row ${rowId} ${toolId}: cache hit exact payload`);
+          : this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
+      if (cached) {
+        this.log(`  Row ${rowId} ${toolId}: recovered from checkpoint`);
         return await this.wrapToolExecutionResult({
           toolId,
-          status: cached.result == null ? 'no_result' : 'completed',
-          result: cached.result,
+          status: cached.result.result == null ? 'no_result' : 'completed',
+          result: cached.result.result,
           requestInput: input,
           execution: toolExecutionMetadataForOutcome({
             kind: 'checkpoint',
-            cacheKey: toolResultCacheKey,
+            cacheKey: cached.cacheKey,
           }),
         });
       }
@@ -3806,8 +4729,10 @@ export class PlayContextImpl {
           callId,
           cacheKey: toolResultCacheKey,
           cacheable: cacheableToolResult,
-          receiptKey: durableReceiptKey,
+          receiptKey: cacheableToolResult ? durableCacheKey : null,
+          executionAuthScopeDigest,
           force: toolCachePolicy.force,
+          forceFailedRefresh: toolCachePolicy.forceFailedRefresh,
           rowId,
           fieldName,
           toolId,
@@ -3827,41 +4752,81 @@ export class PlayContextImpl {
       return await executeTool();
     }
 
-    return this.executeWithRuntimeReceipt(
-      'tool',
-      normalizedKey,
-      this.currentRunId,
-      {
-        receiptKey: durableReceiptKey,
-        semanticKey: toolRequestIdentity,
-        force: toolCachePolicy.force,
-        staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
-        markSkipped: () => {
-          this.log(
-            `Tool cache hit: ${toolId} exact payload (${normalizedKey}:${toolRequestIdentity})`,
-          );
-        },
-        onClaimedResult: (output, receiptKey) =>
-          markToolExecuteResultExecutionOutcome(output, {
-            kind: 'live',
-            receiptKey,
-          }),
-        onRecovered: (output, _receipt, source) =>
-          markToolExecuteResultExecutionOutcome(
-            output,
-            toolExecutionOutcomeForDurableReceipt({
-              source,
-              receiptKey: durableCacheKey,
-            }),
-          ),
-        runningReceiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
-          typeof options?.receiptWaitMs === 'number'
-            ? { max_wait_ms: options.receiptWaitMs }
-            : input,
-        ),
-        execute: executeTool,
-      },
-    );
+    for (
+      let authScopeAttempt = 0;
+      authScopeAttempt < 2;
+      authScopeAttempt += 1
+    ) {
+      try {
+        return await this.executeWithRuntimeReceipt(
+          'tool',
+          normalizedKey,
+          this.currentRunId,
+          {
+            receiptKey: durableCacheKey,
+            semanticKey: toolRequestIdentity,
+            force: toolCachePolicy.force,
+            staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
+            markSkipped: () => {
+              this.log(
+                `ctx.tools.execute(${toolId}): no-op due completed receipt ${toolRequestIdentity} (label: ${normalizedKey})`,
+              );
+            },
+            onClaimedResult: (output, receiptKey) =>
+              markToolExecuteResultExecutionOutcome(output, {
+                kind: 'live',
+                receiptKey,
+              }),
+            onRecovered: (output, _receipt, source) =>
+              markToolExecuteResultExecutionOutcome(
+                output,
+                toolExecutionOutcomeForDurableReceipt({
+                  source,
+                  receiptKey: durableCacheKey,
+                }),
+              ),
+            shouldPersistFailure: (error) =>
+              !(error instanceof ToolExecuteAuthScopeChangedError),
+            execute: ({ leaseId }) => executeTool({ leaseId }),
+            runningReceiptWaitMaxAttempts:
+              resolveRuntimeToolReceiptWaitMaxAttempts(
+                typeof options?.receiptWaitMs === 'number'
+                  ? { max_wait_ms: options.receiptWaitMs }
+                  : input,
+              ),
+          },
+        );
+      } catch (error) {
+        if (
+          !(error instanceof ToolExecuteAuthScopeChangedError) ||
+          authScopeAttempt > 0
+        ) {
+          throw error;
+        }
+        // Make the bounded auth-scope re-claim observable in run logs instead
+        // of a silent continuation: the credential identity changed after the
+        // receipt key was prepared, so we evict the cached digest, re-resolve,
+        // and re-claim a fresh receipt under the new scope before retrying once.
+        this.log(
+          `ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` +
+            `(label: ${normalizedKey}); credential identity changed mid-run, ` +
+            `re-resolving auth scope and re-claiming a fresh receipt under the ` +
+            `new scope (attempt ${authScopeAttempt + 1}/2).`,
+        );
+        this.invalidateToolAuthScopeDigest(toolId);
+        executionAuthScopeDigest =
+          (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
+        durableCacheKey = await this.durableToolCallCacheKey({
+          toolId,
+          requestInput: input,
+          executionAuthScopeDigest,
+          staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
+        });
+        checkpointCacheKeys[0] = durableCacheKey;
+      }
+    }
+
+    throw new ToolExecuteAuthScopeChangedError();
   }
 
   private async executeIntegrationEventWaitTool(
@@ -3987,163 +4952,349 @@ export class PlayContextImpl {
     if (!resolvedName.trim()) {
       throw new Error('ctx.runPlay(...) requires a resolvable play name.');
     }
+    const scheduledChildPlayCall = this.shouldLaunchChildPlaysThroughScheduler()
+      ? this.beginScheduledChildPlayCall()
+      : null;
 
-    if (!this.#options.resolvePlay) {
-      throw new Error(
-        'ctx.runPlay(...) is unavailable because no play resolver was configured.',
-      );
-    }
-    const resolvePlay = this.#options.resolvePlay;
-    const resolvedPlay = await resolvePlay(resolvedName);
-    if (!resolvedPlay) {
-      throw new Error(
-        `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
-      );
-    }
-    const childRevisionFingerprint =
-      resolvedPlayRevisionFingerprint(resolvedPlay);
-
-    const store = rowContext.getStore();
-    const executePlayCall = async (): Promise<TOutput> => {
-      // Reserve depth + per-parent + descendant/play-call budget on the parent
-      // and fork the lineage-global snapshot for the child. forkChild owns the
-      // cycle guard, depth cap, per-parent cap, and play/descendant charges, so
-      // a recovered receipt (which skips this body) never consumes budget. The
-      // child run id stays deterministic — it matches the prior post-increment
-      // playCallCount — so receipt keys and replay are stable.
-      const childRunId = `${this.governance.rootRunId}:${resolvedName}:${this.governor.snapshot().playCallCount + 1}`;
-      const childGovernance = this.governor.forkChild({
+    try {
+      if (!this.#options.resolvePlay) {
+        throw new Error(
+          'ctx.runPlay(...) is unavailable because no play resolver was configured.',
+        );
+      }
+      const resolvePlay = this.#options.resolvePlay;
+      const resolvedPlay = await resolvePlay(resolvedName);
+      if (!resolvedPlay) {
+        throw new Error(
+          `Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
+        );
+      }
+      const childRevisionFingerprint =
+        resolvedPlayRevisionFingerprint(resolvedPlay);
+      const runPlayRowScope = rowContext.getStore();
+      const runPlaySemanticKey = buildDurableRunPlaySemanticKey({
         childPlayName: resolvedName,
-        childRunId,
+        childRevisionFingerprint,
+        input,
+        rowScope: runPlayRowScope
+          ? {
+              fieldName: runPlayRowScope.fieldName,
+              rowId: runPlayRowScope.rowId,
+              rowKey: runPlayRowScope.rowKey ?? null,
+              tableNamespace: runPlayRowScope.tableNamespace ?? null,
+            }
+          : null,
       });
-      const childPlaySlot = await this.governor.acquireChildPlaySlot();
-      const rowStore = rowContext.getStore();
-      const producer = {
-        kind: 'play' as const,
-        id: normalizedKey,
-        playId: resolvedName,
-        displayName: displayNameFromProducerId(resolvedName),
-        runId: childRunId,
-      };
 
-      try {
-        if (rowStore) {
-          this.emitScopedFieldMetaUpdate({
-            rowId: rowStore.rowId,
-            key: rowStore.rowKey ?? null,
-            tableNamespace: rowStore.tableNamespace ?? null,
-            fieldName: rowStore.fieldName,
-            status: 'running',
-            rowStatus: 'running',
-            stage: resolvedName,
-            provider: 'deepline_native',
-            error: null,
-            producer,
-            dataPatch: {},
-          });
-        }
-        const childContext = createPlayContext({
-          ...this.#options,
-          playId: resolvedName,
-          playName: resolvedName,
-          runId: childGovernance.currentRunId,
-          staticPipeline: resolvedPlay.staticPipeline ?? null,
-          checkpoint: this.checkpoint,
-          governance: childGovernance,
-          getRuntimeStepReceipt: undefined,
-          getRuntimeStepReceipts: undefined,
-          claimRuntimeStepReceipt: undefined,
-          claimRuntimeStepReceipts: undefined,
-          completeRuntimeStepReceipt: undefined,
-          completeRuntimeStepReceipts: undefined,
-          failRuntimeStepReceipt: undefined,
-          failRuntimeStepReceipts: undefined,
-          skipRuntimeStepReceipt: undefined,
-        });
-        const childExecution = this.executeResolvedPlay(
-          resolvedPlay,
-          childContext,
+      const executePlayCall = async (): Promise<TOutput> => {
+        // Reserve depth + per-parent + descendant/play-call budget on the parent
+        // and fork the lineage-global snapshot for the child. forkChild owns the
+        // cycle guard, depth cap, per-parent cap, and play/descendant charges, so
+        // a recovered receipt (which skips this body) never consumes budget.
+        //
+        // The child run id is derived through the shared canonical builder so the
+        // in-process runtime and the workers_edge coordinator produce the same
+        // FORMAT (`play/<slug>/run/child-<digest>`) via the same code. The digest
+        // is deterministic across replay because every input (parent lineage,
+        // call key, input) is stable. Per-substrate difference: in-process has no
+        // coordinator idempotency key and no compiled child graphHash, so it uses
+        // the semantic fallback with graphHash omitted; workers_edge feeds the
+        // durable child idempotency key. See child-run-id.ts.
+        const childRunId = buildChildRunId({
+          childPlayName: resolvedName,
+          parentRunId: this.currentRunId,
+          parentPlayName: this.#options.playName,
+          key: normalizedKey,
+          runPlaySemanticKey,
           input,
-        );
-        await childContext.drainQueuedWork([childExecution]);
-        const result = await childExecution;
-        if (rowStore) {
-          this.emitScopedFieldMetaUpdate({
-            rowId: rowStore.rowId,
-            key: rowStore.rowKey ?? null,
-            tableNamespace: rowStore.tableNamespace ?? null,
-            fieldName: rowStore.fieldName,
-            status: 'completed',
-            rowStatus: 'running',
-            stage: resolvedName,
-            provider: 'deepline_native',
-            error: null,
-            producer,
-            dataPatch: {},
-          });
-        }
-        const step = {
-          type: 'play_call' as const,
+          graphHash: null,
+        });
+        const childGovernance = this.governor.forkChild({
+          childPlayName: resolvedName,
+          childRunId,
+        });
+        const childPlaySlot = await this.governor.acquireChildSubmitSlot();
+        const rowStore = rowContext.getStore();
+        const producer = {
+          kind: 'play' as const,
+          id: normalizedKey,
           playId: resolvedName,
-          nestedSteps: childContext.getSteps(),
-          description: normalizeStepDescription(options?.description),
+          displayName: displayNameFromProducerId(resolvedName),
+          runId: childRunId,
         };
-        if (this.activeDatasetStep) {
-          this.activeDatasetStep.substeps.push(step);
-        } else {
-          this.steps.push(step);
-        }
-        return result as TOutput;
-      } catch (error) {
-        if (rowStore) {
-          this.emitScopedFieldMetaUpdate({
-            rowId: rowStore.rowId,
-            key: rowStore.rowKey ?? null,
-            tableNamespace: rowStore.tableNamespace ?? null,
-            fieldName: rowStore.fieldName,
-            status: 'failed',
-            rowStatus: 'running',
-            stage: resolvedName,
-            provider: 'deepline_native',
-            error: this.formatRuntimeError(error),
-            producer,
-            dataPatch: {},
+
+        try {
+          if (rowStore) {
+            this.emitScopedFieldMetaUpdate({
+              rowId: rowStore.rowId,
+              key: rowStore.rowKey ?? null,
+              tableNamespace: rowStore.tableNamespace ?? null,
+              fieldName: rowStore.fieldName,
+              status: 'running',
+              rowStatus: 'running',
+              stage: resolvedName,
+              provider: 'deepline_native',
+              error: null,
+              producer,
+              dataPatch: {},
+            });
+          }
+          if (this.shouldLaunchChildPlaysThroughScheduler()) {
+            const boundaryId = this.childPlayBoundaryId(
+              childGovernance.currentRunId,
+            );
+            const checkpointResult = this.childPlayBoundaryOutput<TOutput>({
+              boundaryId,
+              childRunId: childGovernance.currentRunId,
+              childPlayName: resolvedName,
+            });
+            if (checkpointResult.found) {
+              if (rowStore) {
+                this.emitScopedFieldMetaUpdate({
+                  rowId: rowStore.rowId,
+                  key: rowStore.rowKey ?? null,
+                  tableNamespace: rowStore.tableNamespace ?? null,
+                  fieldName: rowStore.fieldName,
+                  status: 'completed',
+                  rowStatus: 'running',
+                  stage: resolvedName,
+                  provider: 'deepline_native',
+                  error: null,
+                  producer,
+                  dataPatch: {},
+                });
+              }
+              this.recordPlayCallStep({
+                playId: resolvedName,
+                description: options?.description,
+              });
+              return checkpointResult.output;
+            }
+
+            await this.launchScheduledChildPlay({
+              key: normalizedKey,
+              childRunId: childGovernance.currentRunId,
+              childPlayName: resolvedName,
+              childGovernance,
+              childInput: input,
+              description: options?.description ?? null,
+            });
+            childPlaySlot.release();
+            const terminal = await this.readScheduledChildTerminal<TOutput>({
+              childRunId: childGovernance.currentRunId,
+              childPlayName: resolvedName,
+            });
+            if (terminal?.status === 'completed') {
+              this.checkpoint.resolvedBoundaries = {
+                ...(this.checkpoint.resolvedBoundaries ?? {}),
+                [boundaryId]: {
+                  kind: 'child_play',
+                  childRunId: childGovernance.currentRunId,
+                  childPlayName: resolvedName,
+                  status: 'completed',
+                  output: terminal.output,
+                  completedAt: Date.now(),
+                },
+              };
+              if (rowStore) {
+                this.emitScopedFieldMetaUpdate({
+                  rowId: rowStore.rowId,
+                  key: rowStore.rowKey ?? null,
+                  tableNamespace: rowStore.tableNamespace ?? null,
+                  fieldName: rowStore.fieldName,
+                  status: 'completed',
+                  rowStatus: 'running',
+                  stage: resolvedName,
+                  provider: 'deepline_native',
+                  error: null,
+                  producer,
+                  dataPatch: {},
+                });
+              }
+              this.recordPlayCallStep({
+                playId: resolvedName,
+                description: options?.description,
+              });
+              return terminal.output;
+            }
+            if (terminal) {
+              throw new Error(terminal.error);
+            }
+
+            const suspension = {
+              kind: 'child_play',
+              boundaryId,
+              childRunId: childGovernance.currentRunId,
+              childPlayName: resolvedName,
+            } as const;
+            this.recordPendingChildPlayBoundary(suspension);
+            scheduledChildPlayCall?.release();
+            await this.waitForScheduledChildPlayQuiescence();
+            throw new PlayExecutionSuspendedError(
+              this.consumePendingChildPlaySuspension() ?? suspension,
+            );
+          }
+          const childContext = createPlayContext({
+            ...this.#options,
+            playId: resolvedName,
+            playName: resolvedName,
+            runId: childGovernance.currentRunId,
+            executorToken: await this.mintChildExecutorToken({
+              parentRunId: this.currentRunId,
+              parentPlayName: this.#options.playName,
+              childRunId: childGovernance.currentRunId,
+              childPlayName: resolvedName,
+            }),
+            staticPipeline: resolvedPlay.staticPipeline ?? null,
+            checkpoint: this.checkpoint,
+            governance: childGovernance,
+            getRuntimeStepReceipt: undefined,
+            getRuntimeStepReceipts: undefined,
+            claimRuntimeStepReceipt: undefined,
+            claimRuntimeStepReceipts: undefined,
+            completeRuntimeStepReceipt: undefined,
+            completeRuntimeStepReceipts: undefined,
+            failRuntimeStepReceipt: undefined,
+            failRuntimeStepReceipts: undefined,
+            heartbeatRuntimeStepReceipts: undefined,
+            skipRuntimeStepReceipt: undefined,
           });
+          const childExecution = this.executeResolvedPlay(
+            resolvedPlay,
+            childContext,
+            input,
+          );
+          await childContext.drainQueuedWork([childExecution]);
+          const result = await childExecution;
+          if (rowStore) {
+            this.emitScopedFieldMetaUpdate({
+              rowId: rowStore.rowId,
+              key: rowStore.rowKey ?? null,
+              tableNamespace: rowStore.tableNamespace ?? null,
+              fieldName: rowStore.fieldName,
+              status: 'completed',
+              rowStatus: 'running',
+              stage: resolvedName,
+              provider: 'deepline_native',
+              error: null,
+              producer,
+              dataPatch: {},
+            });
+          }
+          const step = {
+            type: 'play_call' as const,
+            playId: resolvedName,
+            nestedSteps: childContext.getSteps(),
+            description: normalizeStepDescription(options?.description),
+          };
+          if (this.activeDatasetStep) {
+            this.activeDatasetStep.substeps.push(step);
+          } else {
+            this.steps.push(step);
+          }
+          return result as TOutput;
+        } catch (error) {
+          if (isPlayExecutionSuspendedError(error)) {
+            throw error;
+          }
+          if (rowStore) {
+            this.emitScopedFieldMetaUpdate({
+              rowId: rowStore.rowId,
+              key: rowStore.rowKey ?? null,
+              tableNamespace: rowStore.tableNamespace ?? null,
+              fieldName: rowStore.fieldName,
+              status: 'failed',
+              rowStatus: 'running',
+              stage: resolvedName,
+              provider: 'deepline_native',
+              error: this.formatRuntimeError(error),
+              producer,
+              dataPatch: {},
+            });
+          }
+          throw error;
+        } finally {
+          childPlaySlot.release();
         }
-        throw error;
-      } finally {
-        childPlaySlot.release();
-      }
-    };
+      };
 
-    if (store) {
-      return await executePlayCall();
+      return await this.executeWithRuntimeReceipt<TOutput>(
+        'runPlay',
+        normalizedKey,
+        this.currentRunId,
+        {
+          semanticKey: runPlaySemanticKey,
+          staleAfterSeconds: options?.staleAfterSeconds,
+          markSkipped: () => {
+            this.log(
+              `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
+            );
+          },
+          repairRunningReceiptForSameRunAfterWaitTimeout: true,
+          runningReceiptWaitMaxAttempts:
+            resolveRuntimeToolReceiptWaitMaxAttempts(input),
+          execute: executePlayCall,
+        },
+      );
+    } finally {
+      scheduledChildPlayCall?.release();
     }
+  }
 
-    return this.executeWithRuntimeReceipt<TOutput>(
-      'runPlay',
-      normalizedKey,
-      this.currentRunId,
+  private async mintChildExecutorToken(input: {
+    parentRunId: string;
+    parentPlayName?: string | null;
+    childRunId: string;
+    childPlayName: string;
+  }): Promise<string | undefined> {
+    if (!this.#options.executorToken || !this.#options.baseUrl) {
+      return this.#options.executorToken;
+    }
+    const parentPlayName = input.parentPlayName?.trim();
+    if (!parentPlayName) {
+      return this.#options.executorToken;
+    }
+    const response = await fetch(
+      `${this.#options.baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime/child-executor-token`,
       {
-        semanticKey: stableDigest(
-          stableStringify({
-            childPlayName: resolvedName,
-            childRevisionFingerprint,
-            input,
-          }),
-        ),
-        staleAfterSeconds: options?.staleAfterSeconds,
-        markSkipped: () => {
-          this.log(
-            `ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
-          );
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.#options.executorToken}`,
+          'Content-Type': 'application/json',
+          ...(this.#options.vercelProtectionBypassToken?.trim()
+            ? {
+                'x-vercel-protection-bypass':
+                  this.#options.vercelProtectionBypassToken.trim(),
+              }
+            : {}),
         },
-        repairRunningReceiptForSameRunAfterWaitTimeout: true,
-        runningReceiptWaitMaxAttempts:
-          resolveRuntimeToolReceiptWaitMaxAttempts(input),
-        execute: executePlayCall,
+        body: JSON.stringify({
+          parentRunId: input.parentRunId,
+          parentPlayName,
+          childRunId: input.childRunId,
+          childPlayName: input.childPlayName,
+        }),
       },
     );
+    const body = (await response.json().catch(() => null)) as {
+      executorToken?: unknown;
+      error?: unknown;
+    } | null;
+    if (!response.ok) {
+      const message =
+        typeof body?.error === 'string' && body.error.trim()
+          ? body.error.trim()
+          : response.statusText;
+      throw new Error(
+        `ctx.runPlay failed to mint child executor token for "${input.childPlayName}": ${message} (status ${response.status}).`,
+      );
+    }
+    if (typeof body?.executorToken !== 'string' || !body.executorToken.trim()) {
+      throw new Error(
+        `ctx.runPlay failed to mint child executor token for "${input.childPlayName}": response did not include executorToken.`,
+      );
+    }
+    return body.executorToken.trim();
   }
 
   /**
@@ -4630,7 +5781,11 @@ export class PlayContextImpl {
           const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
           liveFollowersByOwnerCallId.delete(owner.callId);
           for (const request of [owner, ...followers]) {
-            await this.rejectToolCall(toolId, request, error);
+            await this.rejectToolCall(toolId, request, error, {
+              persistReceiptFailure: !(
+                error instanceof ToolExecuteAuthScopeChangedError
+              ),
+            });
           }
         };
         const requestsWithLiveFollowers = (
@@ -4685,7 +5840,11 @@ export class PlayContextImpl {
         ) {
           const requestsByReceiptKey = new Map<
             string,
-            { requests: ToolCallRequest[]; forceRefresh: boolean }
+            {
+              requests: ToolCallRequest[];
+              forceRefresh: boolean;
+              forceFailedRefresh: boolean;
+            }
           >();
           const localOnlyRequests: ToolCallRequest[] = [];
           for (const request of pendingRequests) {
@@ -4697,17 +5856,37 @@ export class PlayContextImpl {
             const group = requestsByReceiptKey.get(receiptKey) ?? {
               requests: [],
               forceRefresh: false,
+              forceFailedRefresh: false,
             };
             group.requests.push(request);
             group.forceRefresh ||= request.force === true;
+            group.forceFailedRefresh ||= request.forceFailedRefresh === true;
             requestsByReceiptKey.set(receiptKey, group);
           }
           const normalReceiptKeys = [...requestsByReceiptKey]
-            .filter(([, group]) => !group.forceRefresh)
+            .filter(
+              ([, group]) => !group.forceRefresh && !group.forceFailedRefresh,
+            )
             .map(([receiptKey]) => receiptKey);
           const forcedReceiptKeys = [...requestsByReceiptKey]
             .filter(([, group]) => group.forceRefresh)
             .map(([receiptKey]) => receiptKey);
+          const failedRefreshReceiptKeys = [...requestsByReceiptKey]
+            .filter(
+              ([, group]) => !group.forceRefresh && group.forceFailedRefresh,
+            )
+            .map(([receiptKey]) => receiptKey);
+          const forcedExisting = forcedReceiptKeys.length
+            ? await this.getRuntimeStepReceipts(forcedReceiptKeys)
+            : new Map<string, RuntimeStepReceipt>();
+          const forcedRunningReceiptKeys = new Set(
+            [...forcedExisting]
+              .filter(([, receipt]) => receipt.status === 'running')
+              .map(([receiptKey]) => receiptKey),
+          );
+          const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
+            (receiptKey) => !forcedRunningReceiptKeys.has(receiptKey),
+          );
           const claims =
             normalReceiptKeys.length > 0
               ? await this.claimRuntimeStepReceipts(
@@ -4716,17 +5895,30 @@ export class PlayContextImpl {
                 )
               : new Map<string, RuntimeStepReceipt>();
           const forcedClaims =
-            forcedReceiptKeys.length > 0
+            claimableForcedReceiptKeys.length > 0
               ? await this.claimRuntimeStepReceipts(
-                  forcedReceiptKeys,
+                  claimableForcedReceiptKeys,
                   this.currentRunId,
+                  false,
                   true,
+                )
+              : new Map<string, RuntimeStepReceipt>();
+          const failedRefreshClaims =
+            failedRefreshReceiptKeys.length > 0
+              ? await this.claimRuntimeStepReceipts(
+                  failedRefreshReceiptKeys,
+                  this.currentRunId,
+                  false,
+                  false,
                   true,
                 )
               : new Map<string, RuntimeStepReceipt>();
           for (const [receiptKey, claim] of forcedClaims) {
             claims.set(receiptKey, claim);
           }
+          for (const [receiptKey, claim] of failedRefreshClaims) {
+            claims.set(receiptKey, claim);
+          }
           const claimedRequests: ToolCallRequest[] = [...localOnlyRequests];
           const durableRecoveredRequests: ToolCallRequest[] = [];
           const resolveRequestsFromReceipt = async (
@@ -4738,9 +5930,7 @@ export class PlayContextImpl {
             if (!request) return;
             const receiptKey = request.receiptKey?.trim() || null;
             if (!receiptKey) {
-              throw new Error(
-                `Durable tool receipt ${source} recovery requires a receipt key.`,
-              );
+              throw new Error(`${source} tool recovery needs receipt key.`);
             }
             const wrapped = await this.wrapToolExecutionResult({
               toolId,
@@ -4794,6 +5984,7 @@ export class PlayContextImpl {
           const reclaimExistingRunningReceiptAfterWait = async (
             receiptKey: string,
             requestsForKey: ToolCallRequest[],
+            forceAfterWait = false,
           ): Promise<void> => {
             const waitMaxAttempts =
               requestsForKey.length > 0
@@ -4808,15 +5999,18 @@ export class PlayContextImpl {
                   )
                 : undefined;
             try {
-              await resolveRequestsFromReceipt(
-                requestsForKey,
-                await this.waitForCompletedRuntimeToolReceipt(
-                  receiptKey,
-                  waitMaxAttempts,
-                ),
-                'in_flight',
+              const completed = await this.waitForCompletedRuntimeToolReceipt(
+                receiptKey,
+                waitMaxAttempts,
               );
-              return;
+              if (!forceAfterWait) {
+                await resolveRequestsFromReceipt(
+                  requestsForKey,
+                  completed,
+                  'in_flight',
+                );
+                return;
+              }
             } catch (error) {
               if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
                 for (const request of requestsForKey) {
@@ -4832,6 +6026,7 @@ export class PlayContextImpl {
                 [receiptKey],
                 this.currentRunId,
                 true,
+                forceAfterWait,
               )
             ).get(receiptKey);
             if (
@@ -4862,14 +6057,31 @@ export class PlayContextImpl {
             ) {
               const [owner, ...waiters] = requestsForKey;
               if (!owner) return;
+              if (!reclaimed.leaseId) {
+                throw new RuntimeReceiptLeaseLostError({
+                  receiptKey,
+                  runId: this.currentRunId,
+                  leaseId: 'missing',
+                });
+              }
+              owner.receiptLeaseId = reclaimed.leaseId ?? null;
               if (waiters.length > 0) {
                 liveFollowersByOwnerCallId.set(owner.callId, waiters);
               }
               try {
+                await this.assertRuntimeToolReceiptOwnership([owner]);
                 const execution = await this.callToolExecutionAPI(
                   toolId,
                   owner.input,
                   {
+                    durableCallReceiptKey: receiptKey,
+                    executionAuthScopeDigest: owner.executionAuthScopeDigest,
+                    providerIdempotencyKey:
+                      this.providerIdempotencyKeyForToolCall({
+                        cacheKey: owner.cacheKey,
+                        force: owner.force === true,
+                        leaseId: owner.receiptLeaseId,
+                      }),
                     timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([owner]),
                   },
                 );
@@ -4904,8 +6116,13 @@ export class PlayContextImpl {
             const requestsForKey = group.requests;
             const claim = claims.get(receiptKey);
             if (!claim) {
-              const [owner, ...waiters] = requestsForKey;
-              claimOwnerWithLiveFollowers(owner, waiters);
+              durableExistingRunningHandlers.push(
+                reclaimExistingRunningReceiptAfterWait(
+                  receiptKey,
+                  requestsForKey,
+                  group.forceRefresh,
+                ),
+              );
               continue;
             }
             if (claim?.status === 'completed' || claim?.status === 'skipped') {
@@ -4928,6 +6145,16 @@ export class PlayContextImpl {
               claim.claimState !== 'existing'
             ) {
               const [owner, ...waiters] = requestsForKey;
+              if (!claim.leaseId) {
+                throw new RuntimeReceiptLeaseLostError({
+                  receiptKey,
+                  runId: this.currentRunId,
+                  leaseId: 'missing',
+                });
+              }
+              if (owner) {
+                owner.receiptLeaseId = claim.leaseId ?? null;
+              }
               claimOwnerWithLiveFollowers(owner, waiters);
               continue;
             }
@@ -4939,6 +6166,7 @@ export class PlayContextImpl {
                 reclaimExistingRunningReceiptAfterWait(
                   receiptKey,
                   requestsForKey,
+                  group.forceRefresh,
                 ),
               );
               continue;
@@ -4947,6 +6175,7 @@ export class PlayContextImpl {
               reclaimExistingRunningReceiptAfterWait(
                 receiptKey,
                 requestsForKey,
+                group.forceRefresh,
               ),
             );
           }
@@ -4971,18 +6200,13 @@ export class PlayContextImpl {
               compiledBatches.length > 0
                 ? await this.governor.suggestedParallelism(
                     compiledBatches[0]!.batchOperation,
-                    4,
+                    this.governor.policy.pacing
+                      .workerToolBatchDefaultParallelism,
                   )
-                : 4;
+                : this.governor.policy.pacing.workerToolBatchDefaultParallelism;
             await executeChunkedRequests({
               requests: compiledBatches,
               batchSize,
-              // Bounded dispatch wave (postgres_fast parity with workers_edge):
-              // cap the provider calls (counted by batch member size) a single
-              // chunk dispatches before the latch is read again, so a
-              // persistence failure prevents the rest of a large batch group.
-              maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
-              weightOf: (batch) => batch.memberRequests.length,
               execute: async (batch) => {
                 // Circuit breaker: skip dispatching this batch's provider call
                 // once a persistence failure has occurred in this run.
@@ -4993,10 +6217,42 @@ export class PlayContextImpl {
                     this.persistenceLatch,
                   );
                 }
+                await this.assertRuntimeToolReceiptOwnership(
+                  batch.memberRequests,
+                );
+                const receiptKeys = batch.memberRequests.map(
+                  (request) => request.cacheKey,
+                );
+                const aggregateReceiptKey = buildDurableToolAggregateReceiptKey(
+                  {
+                    receiptKeys,
+                    prefix: 'batch',
+                    aggregateReceiptPrefix: buildDurableToolReceiptPrefix({
+                      orgId: this.#options.orgId,
+                      toolId: batch.batchOperation,
+                    }),
+                  },
+                );
                 return await this.callToolAPI(
                   batch.batchOperation,
                   batch.batchPayload,
                   {
+                    durableCallReceiptKey: aggregateReceiptKey,
+                    executionAuthScopeDigest:
+                      batch.memberRequests[0]?.executionAuthScopeDigest ?? null,
+                    providerIdempotencyKey:
+                      buildDurableToolAggregateProviderIdempotencyKey({
+                        aggregateReceiptKey,
+                        receiptKeys,
+                        providerIdempotencyKeys: batch.memberRequests.map(
+                          (request) =>
+                            this.providerIdempotencyKeyForToolCall({
+                              cacheKey: request.cacheKey,
+                              force: request.force === true,
+                              leaseId: request.receiptLeaseId,
+                            }),
+                        ),
+                      }),
                     timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners(
                       batch.memberRequests,
                     ),
@@ -5042,13 +6298,9 @@ export class PlayContextImpl {
               },
             });
           } else {
-            // Bounded dispatch wave (postgres_fast parity with workers_edge):
-            // an unbatched tool group can carry a whole chunk's rows, so cap the
-            // provider calls dispatched before the latch is read again to the
-            // wave size. Respect a lower provider-pacing suggestion when present.
-            const batchSize = Math.min(
-              await this.governor.suggestedParallelism(toolId, 50),
-              PROVIDER_DISPATCH_WAVE_SIZE,
+            const batchSize = await this.governor.suggestedParallelism(
+              toolId,
+              this.governor.policy.pacing.suggestedMaxParallelism,
             );
             for (
               let start = 0;
@@ -5071,10 +6323,24 @@ export class PlayContextImpl {
                     return;
                   }
                   try {
+                    await this.assertRuntimeToolReceiptOwnership([request]);
                     const execution = await this.callToolExecutionAPI(
                       toolId,
                       request.input,
                       {
+                        ...(request.receiptKey
+                          ? {
+                              durableCallReceiptKey: request.receiptKey,
+                              executionAuthScopeDigest:
+                                request.executionAuthScopeDigest,
+                              providerIdempotencyKey:
+                                this.providerIdempotencyKeyForToolCall({
+                                  cacheKey: request.receiptKey,
+                                  force: request.force === true,
+                                  leaseId: request.receiptLeaseId,
+                                }),
+                            }
+                          : {}),
                         timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
                           request,
                         ]),
@@ -5198,6 +6464,23 @@ export class PlayContextImpl {
       : execution.result;
   }
 
+  private providerIdempotencyKeyForToolCall(input: {
+    cacheKey: string;
+    force?: boolean;
+    leaseId?: string | null;
+  }): string {
+    const fallbackAttemptId =
+      input.force === true && !input.leaseId
+        ? `${this.currentRunId}:${crypto.randomUUID()}`
+        : null;
+    return buildDurableToolProviderIdempotencyKey({
+      receiptKey: input.cacheKey,
+      force: input.force,
+      receiptLeaseId: input.leaseId,
+      fallbackAttemptId,
+    });
+  }
+
   private async callToolExecutionAPI(
     toolId: string,
     input: Record<string, unknown>,
@@ -5240,6 +6523,20 @@ export class PlayContextImpl {
 
           while (true) {
             let response: Response;
+            const durableCallReceiptKey =
+              options?.durableCallReceiptKey?.trim() || null;
+            const executionAuthScopeDigest =
+              durableCallReceiptKey && options?.executionAuthScopeDigest
+                ? options.executionAuthScopeDigest.trim() || null
+                : durableCallReceiptKey
+                  ? ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ??
+                    null)
+                  : null;
+            const providerIdempotencyKey =
+              options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
+            const deeplineRequestId = providerIdempotencyKey
+              ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
+              : null;
             const abortController = timeoutMs ? new AbortController() : null;
             const timeoutHandle = timeoutMs
               ? setTimeout(() => {
@@ -5261,6 +6558,12 @@ export class PlayContextImpl {
                     V2_EXECUTE_RESPONSE_CONTRACT,
                   [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
                   [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+                  ...(deeplineRequestId
+                    ? {
+                        'x-deepline-request-id': deeplineRequestId,
+                        'x-deepline-idempotency-key': providerIdempotencyKey!,
+                      }
+                    : {}),
                   ...(this.#options.vercelProtectionBypassToken?.trim()
                     ? {
                         'x-vercel-protection-bypass':
@@ -5272,6 +6575,17 @@ export class PlayContextImpl {
                   payload: input,
                   metadata: {
                     parent_run_id: this.#options.runId,
+                    ...(durableCallReceiptKey
+                      ? {
+                          durable_call_receipt_key: durableCallReceiptKey,
+                          ...(executionAuthScopeDigest
+                            ? {
+                                execution_auth_scope_digest:
+                                  executionAuthScopeDigest,
+                              }
+                            : {}),
+                        }
+                      : {}),
                     ...(options?.customerDbDataset
                       ? {
                           query_result_dataset: {
@@ -5293,6 +6607,9 @@ export class PlayContextImpl {
                             : {}),
                         }
                       : {}),
+                    ...(providerIdempotencyKey
+                      ? { provider_idempotency_key: providerIdempotencyKey }
+                      : {}),
                   },
                   ...(this.#options.integrationMode
                     ? { integration_mode: this.#options.integrationMode }
@@ -5337,6 +6654,13 @@ export class PlayContextImpl {
 
             if (!response.ok) {
               const text = await response.text();
+              const authScopeChanged = parseToolExecuteAuthScopeChangedError({
+                status: response.status,
+                bodyText: text,
+              });
+              if (authScopeChanged) {
+                throw authScopeChanged;
+              }
               const httpFailureAttempt = httpFailureAttempts.next({
                 toolId,
                 status: response.status,