deepline 0.1.168 → 0.1.170

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -159,6 +159,8 @@ import {
   markWorkerToolReceiptResultCached,
   markWorkerToolReceiptResultExecution,
   planWorkerToolReceiptGroups,
+  resolveWorkerToolReceiptGroupWaitMaxAttempts,
+  resolveWorkerToolRuntimeTimeoutMs,
 } from './runtime/tool-receipts';
 // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
 // the long-lived Play Harness Worker via env.HARNESS. We import the
@@ -263,6 +265,7 @@ type RunRequest = {
   callbackUrl: string;
   executorToken: string;
   baseUrl: string;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   orgId: string;
   playName: string;
   graphHash?: string | null;
@@ -325,6 +328,14 @@ function getStringField(value: unknown, key: string): string | null {
   return typeof field === 'string' && field.trim() ? field : null;
 }
 
+function normalizeIntegrationMode(
+  value: unknown,
+): 'live' | 'eval_stub' | 'fixture' | null {
+  return value === 'live' || value === 'eval_stub' || value === 'fixture'
+    ? value
+    : null;
+}
+
 function getObjectField(
   value: unknown,
   key: string,
@@ -554,11 +565,13 @@ function resolveRuntimeDeadlineRemainingMs(runtimeDeadlineMs?: number): number {
 
 function resolveToolRuntimeApiTimeout(input: {
   requestInput: Record<string, unknown>;
+  timeoutMs?: number;
   runtimeDeadlineMs?: number;
 }): { timeoutMs: number; timeoutErrorMessage?: string } {
-  const toolTimeoutMs = resolveRuntimeToolReceiptWaitTimeoutMs(
-    input.requestInput,
-  );
+  const toolTimeoutMs =
+    typeof input.timeoutMs === 'number' && Number.isFinite(input.timeoutMs)
+      ? Math.max(1, Math.ceil(input.timeoutMs))
+      : resolveRuntimeToolReceiptWaitTimeoutMs(input.requestInput);
   const remainingMs = resolveRuntimeDeadlineRemainingMs(
     input.runtimeDeadlineMs,
   );
@@ -1256,6 +1269,7 @@ async function executeTool(
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
+  runtimeTimeoutMs?: number,
 ): Promise<ToolExecuteResult> {
   if (args.toolId === 'test_wait_for_event' && workflowStep) {
     const result = await waitForSyntheticIntegrationEvent(
@@ -1278,6 +1292,7 @@ async function executeTool(
     transientHttpRetrySafe,
     abortSignal,
     runtimeDeadlineMs,
+    runtimeTimeoutMs,
   );
 }
 
@@ -1291,6 +1306,7 @@ async function executeToolWithLifecycle(
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
+  runtimeTimeoutMs?: number,
 ): Promise<ToolExecuteResult> {
   callbacks?.onToolCalled?.(args.toolId, nowMs());
   try {
@@ -1303,6 +1319,7 @@ async function executeToolWithLifecycle(
       transientHttpRetrySafe,
       abortSignal,
       runtimeDeadlineMs,
+      runtimeTimeoutMs,
     );
   } catch (error) {
     callbacks?.onToolFailed?.(args.toolId, nowMs());
@@ -1320,6 +1337,8 @@ function normalizeToolExecuteArgs(request: unknown): {
   input: Record<string, unknown>;
   force?: boolean;
   staleAfterSeconds?: number;
+  timeoutMs?: number;
+  receiptWaitMs?: number;
 } {
   if (!isToolExecuteRecord(request)) {
     throw new Error(
@@ -1345,6 +1364,14 @@ function normalizeToolExecuteArgs(request: unknown): {
     ...(typeof request.staleAfterSeconds === 'number'
       ? { staleAfterSeconds: request.staleAfterSeconds }
       : {}),
+    ...(typeof request.timeoutMs === 'number' &&
+    Number.isFinite(request.timeoutMs)
+      ? { timeoutMs: request.timeoutMs }
+      : {}),
+    ...(typeof request.receiptWaitMs === 'number' &&
+    Number.isFinite(request.receiptWaitMs)
+      ? { receiptWaitMs: request.receiptWaitMs }
+      : {}),
   };
 }
 
@@ -1441,6 +1468,7 @@ async function callToolDirect(
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
+  runtimeTimeoutMs?: number,
 ): Promise<ToolExecuteResult> {
   const { id, toolId, input } = args;
   const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
@@ -1455,6 +1483,7 @@ async function callToolDirect(
     try {
       const runtimeApiTimeout = resolveToolRuntimeApiTimeout({
         requestInput: input,
+        timeoutMs: runtimeTimeoutMs,
         runtimeDeadlineMs,
       });
       res = await fetchRuntimeApi(
@@ -1472,6 +1501,9 @@ async function callToolDirect(
           body: JSON.stringify({
             payload: input,
             metadata: { parent_run_id: req.runId },
+            ...(req.integrationMode
+              ? { integration_mode: req.integrationMode }
+              : {}),
           }),
         },
         {
@@ -1628,6 +1660,7 @@ type WorkerToolBatchRequest = {
   receiptKey: string | null;
   force: boolean;
   receiptWaitMaxAttempts: number;
+  runtimeTimeoutMs?: number;
   toolId: string;
   input: Record<string, unknown>;
   workflowStep?: WorkflowStep;
@@ -1646,17 +1679,62 @@ type PreparedWorkerToolBatchRequests = {
   deferredClaimedRequests: Promise<ClaimedWorkerToolBatchRequest[]>[];
 };
 
+function resolveClaimedWorkerToolRuntimeTimeoutMs(
+  claimedRequests: ClaimedWorkerToolBatchRequest[],
+  input: { runtimeDeadlineMs?: number },
+): number | undefined {
+  return resolveWorkerToolRuntimeTimeoutMs(claimedRequests, {
+    resolveOwnerTimeoutMs: (request) =>
+      resolveToolRuntimeApiTimeout({
+        requestInput: request.input,
+        timeoutMs: request.runtimeTimeoutMs,
+        runtimeDeadlineMs: input.runtimeDeadlineMs,
+      }).timeoutMs,
+  });
+}
+
 const WORKER_TOOL_BATCH_GRACE_MS = 250;
 const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
 const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
 const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
-const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
+const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
+const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
 /**
  * Bounded number of per-row failure samples carried in chunk summaries and the
  * map's terminal partial-failure log. Every failed row is persisted with its
  * full error in the runtime sheet; the samples just keep run logs readable.
  */
 const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
+
+class RuntimeReceiptPersistenceError extends Error {
+  readonly errors: unknown[];
+
+  constructor(message: string, errors: unknown[] = []) {
+    super(message);
+    this.name = 'RuntimeReceiptPersistenceError';
+    this.errors = errors;
+  }
+}
+
+function isRuntimeReceiptPersistenceError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if (error instanceof RuntimeReceiptPersistenceError) return true;
+  if (error instanceof Error && error.name === 'RuntimeReceiptPersistenceError') {
+    return true;
+  }
+  const nestedErrors = (error as { errors?: unknown }).errors;
+  return (
+    Array.isArray(nestedErrors) &&
+    nestedErrors.some(isRuntimeReceiptPersistenceError)
+  );
+}
+
+function isRunFatalWorkerRowError(error: unknown): boolean {
+  return (
+    isRowIsolationExemptError(error) || isRuntimeReceiptPersistenceError(error)
+  );
+}
+
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1759,6 +1837,7 @@ class WorkerToolBatchScheduler {
     input: Record<string, unknown>,
     workflowStep?: WorkflowStep,
     options?: { force?: boolean; staleAfterSeconds?: number | null },
+    runtimeOptions?: { timeoutMs?: number; receiptWaitMs?: number },
   ): Promise<unknown> {
     const providerActionVersion =
       await this.resolveToolActionCacheVersion(toolId);
@@ -1776,7 +1855,14 @@ class WorkerToolBatchScheduler {
         cacheKey: receiptKey,
         receiptKey,
         force: options?.force === true,
-        receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(input),
+        receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
+          typeof runtimeOptions?.receiptWaitMs === 'number'
+            ? { max_wait_ms: runtimeOptions.receiptWaitMs }
+            : input,
+        ),
+        ...(typeof runtimeOptions?.timeoutMs === 'number'
+          ? { runtimeTimeoutMs: runtimeOptions.timeoutMs }
+          : {}),
         toolId,
         input,
         workflowStep,
@@ -1962,7 +2048,10 @@ class WorkerToolBatchScheduler {
     try {
       const receipt = await this.waitForDurableToolReceipt({
         receiptKey: input.receiptKey,
-        maxAttempts: request.receiptWaitMaxAttempts,
+        maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
+          input.group,
+          (groupRequest) => groupRequest.receiptWaitMaxAttempts,
+        ),
       });
       await this.resolveCompletedDurableToolReceiptGroup({
         group: input.group,
@@ -1982,6 +2071,7 @@ class WorkerToolBatchScheduler {
       !canReclaimTimedOutWorkerToolReceipt({
         ownerRunId: input.runningReceipt.runId,
         currentRunId: this.req.runId,
+        updatedAt: input.runningReceipt.updatedAt,
       })
     ) {
       this.rejectRawRequests(input.group, waitError);
@@ -2070,7 +2160,10 @@ class WorkerToolBatchScheduler {
       try {
         const receipt = await this.waitForDurableToolReceipt({
           receiptKey: input.receiptKey,
-          maxAttempts: request.receiptWaitMaxAttempts,
+          maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
+            input.group,
+            (groupRequest) => groupRequest.receiptWaitMaxAttempts,
+          ),
         });
         await this.resolveCompletedDurableToolReceiptGroup({
           group: input.group,
@@ -2100,9 +2193,9 @@ class WorkerToolBatchScheduler {
       await this.failDurableToolRequest(claimed, error);
       return error;
     } catch (receiptError) {
-      return new AggregateError(
-        [error, receiptError],
+      return new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipt could not be marked failed',
+        [error, receiptError],
       );
     }
   }
@@ -2459,25 +2552,41 @@ class WorkerToolBatchScheduler {
             signal: this.abortSignal,
           });
           try {
-            const result = await executeToolWithLifecycle(
-              this.req,
-              { id: request.id, toolId, input: request.input },
-              request.workflowStep,
-              this.callbacks,
-              (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
-              () => this.governor.chargeBudget('retry'),
-              toolContract?.retrySafeTransientHttp === true,
-              this.abortSignal,
-              this.runtimeDeadlineMs,
-            );
+            let result: unknown;
+            try {
+              result = await executeToolWithLifecycle(
+                this.req,
+                { id: request.id, toolId, input: request.input },
+                request.workflowStep,
+                this.callbacks,
+                (retryAfterMs) =>
+                  this.reportBackpressure(toolId, retryAfterMs),
+                () => this.governor.chargeBudget('retry'),
+                toolContract?.retrySafeTransientHttp === true,
+                this.abortSignal,
+                this.runtimeDeadlineMs,
+                resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
+                  runtimeDeadlineMs: this.runtimeDeadlineMs,
+                }),
+              );
+            } catch (error) {
+              this.rejectRequests(
+                claimed,
+                await this.failureForRejectedToolRequest(claimed, error),
+              );
+              return;
+            }
             this.settleRequests(
               claimed,
               await this.completeDurableToolRequest(claimed, result),
             );
-          } catch (error) {
+          } catch (receiptError) {
             this.rejectRequests(
               claimed,
-              await this.failureForRejectedToolRequest(claimed, error),
+              new RuntimeReceiptPersistenceError(
+                'Tool call succeeded but durable receipt could not be marked completed',
+                [receiptError],
+              ),
             );
           } finally {
             slot.release();
@@ -2629,6 +2738,9 @@ async function executeBatchedWorkerToolGroup(input: {
           toolContract?.retrySafeTransientHttp === true,
           input.abortSignal,
           input.runtimeDeadlineMs,
+          resolveClaimedWorkerToolRuntimeTimeoutMs(batch.memberRequests, {
+            runtimeDeadlineMs: input.runtimeDeadlineMs,
+          }),
         );
       } catch (error) {
         input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
@@ -2651,9 +2763,9 @@ async function executeBatchedWorkerToolGroup(input: {
           try {
             await input.failRequests(entry.request.memberRequests, entry.error);
           } catch (receiptError) {
-            rejection = new AggregateError(
-              [entry.error, receiptError],
+            rejection = new RuntimeReceiptPersistenceError(
               'Tool call failed and durable receipts could not be marked failed',
+              [entry.error, receiptError],
             );
           }
           for (const claimed of entry.request.memberRequests) {
@@ -2668,16 +2780,28 @@ async function executeBatchedWorkerToolGroup(input: {
           batchResult != null
             ? entry.request.splitResults(batchResult)
             : entry.request.memberRequests.map(() => null);
-        const completedResults = await input.completeRequests(
-          entry.request.memberRequests.map((claimed, index) => ({
-            claimed,
-            result: wrapWorkerToolResult(
-              claimed.request.toolId,
-              splitResults[index] ?? null,
-              toolMetadataFallback(claimed.request.toolId),
-            ),
-          })),
-        );
+        let completedResults: unknown[];
+        try {
+          completedResults = await input.completeRequests(
+            entry.request.memberRequests.map((claimed, index) => ({
+              claimed,
+              result: wrapWorkerToolResult(
+                claimed.request.toolId,
+                splitResults[index] ?? null,
+                toolMetadataFallback(claimed.request.toolId),
+              ),
+            })),
+          );
+        } catch (receiptError) {
+          const rejection = new RuntimeReceiptPersistenceError(
+            'Tool call succeeded but durable receipts could not be marked completed',
+            [receiptError],
+          );
+          for (const claimed of entry.request.memberRequests) {
+            input.rejectRequest(claimed, rejection);
+          }
+          continue;
+        }
         for (let index = 0; index < completedResults.length; index += 1) {
           const claimed = entry.request.memberRequests[index]!;
           const request = claimed.request;
@@ -2690,9 +2814,9 @@ async function executeBatchedWorkerToolGroup(input: {
     try {
       await input.failRequests(input.requests, error);
     } catch (receiptError) {
-      rejection = new AggregateError(
-        [error, receiptError],
+      rejection = new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipts could not be marked failed',
+        [error, receiptError],
       );
     }
     for (const claimed of input.requests) {
@@ -2728,6 +2852,7 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
   stepCellsSkipped: number;
   outputDatasetId: string;
   hash: string;
+  fatalError?: string;
   preview: T[];
   cachedRows?: T[];
 };
@@ -3816,6 +3941,15 @@ async function persistCompletedMapRows(input: {
     rows,
     outputFields,
   });
+  console.warn('[play-runner.persist_completed_map_rows.start]', {
+    runId: req.runId,
+    playName: req.playName,
+    tableNamespace,
+    rows: rows.length,
+    outputFields,
+    extraOutputFields: input.extraOutputFields ?? [],
+    contractColumnCount: sheetContract.columns.length,
+  });
   const persistRequest = {
     ...sessionScope,
     tableNamespace,
@@ -3830,6 +3964,8 @@ async function persistCompletedMapRows(input: {
         .filter((key): key is string => Boolean(key)),
     ),
   ];
+  const expectedVisibleRows =
+    expectedKeys.length > 0 ? expectedKeys.length : rows.length;
   const readVisibleRowCount = async () => {
     if (expectedKeys.length > 0) {
       const result = await harnessReadSheetDatasetRowKeys({
@@ -3852,12 +3988,24 @@ async function persistCompletedMapRows(input: {
     return result.rows.length;
   };
   const result = await harnessPersistCompletedSheetRows(persistRequest);
+  console.warn('[play-runner.persist_completed_map_rows.result]', {
+    runId: req.runId,
+    playName: req.playName,
+    tableNamespace,
+    rows: rows.length,
+    rowsWritten: result.rowsWritten,
+    expectedVisibleRows,
+    expectedKeyCount: expectedKeys.length,
+  });
   let visibleRows = -1;
   let retryWritten: number | null = null;
   let retryVisible: number | null = null;
-  if (rows.length <= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS) {
+  if (
+    rows.length <= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS ||
+    result.rowsWritten !== rows.length
+  ) {
     visibleRows = await readVisibleRowCount();
-    if (visibleRows < rows.length) {
+    if (visibleRows < expectedVisibleRows) {
       await harnessStartSheetDataset({
         ...sessionScope,
         tableNamespace,
@@ -3869,15 +4017,23 @@ async function persistCompletedMapRows(input: {
       const retry = await harnessPersistCompletedSheetRows(persistRequest);
       retryWritten = retry.rowsWritten;
       retryVisible = await readVisibleRowCount();
-      if (retryVisible < rows.length)
-        throw new Error(
-          `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}; run ${req.runId}.`,
-        );
+      if (retryVisible >= expectedVisibleRows) {
+        return {
+          rows: rows.length,
+          written: result.rowsWritten,
+          visible: visibleRows,
+          retryWritten,
+          retryVisible,
+        };
+      }
+      throw new Error(
+        `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
+      );
     }
   }
-  if (result.rowsWritten !== rows.length && visibleRows < rows.length) {
+  if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
     throw new Error(
-      `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; run ${req.runId}.`,
+      `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; run ${req.runId}.`,
     );
   }
   return {
@@ -4932,7 +5088,7 @@ function createMinimalWorkerCtx(
       let persistFailure: unknown = null;
       let scheduledLiveUpdateTimer: ReturnType<typeof setTimeout> | null = null;
       let liveUpdateFlushChain: Promise<void> = Promise.resolve();
-      let liveUpdateFailure: unknown = null;
+      let liveUpdateFailureCount = 0;
 
       const clearScheduledPersistTimer = () => {
         if (scheduledPersistTimer) {
@@ -5070,25 +5226,35 @@ function createMinimalWorkerCtx(
         const updates = pendingLiveRowUpdates.splice(0);
         const extraOutputFields = Array.from(generatedOutputFields);
         const task = liveUpdateFlushChain.then(async () => {
-          if (liveUpdateFailure) throw liveUpdateFailure;
-          await applyLiveMapRowUpdates({
-            req,
-            tableNamespace: name,
-            outputFields,
-            extraOutputFields,
-            updates,
-          });
-        });
-        liveUpdateFlushChain = task.catch((error) => {
-          liveUpdateFailure ??= error;
+          try {
+            await applyLiveMapRowUpdates({
+              req,
+              tableNamespace: name,
+              outputFields,
+              extraOutputFields,
+              updates,
+            });
+          } catch (error) {
+            liveUpdateFailureCount += 1;
+            if (liveUpdateFailureCount <= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
+              emitEvent({
+                type: 'log',
+                level: 'warn',
+                message:
+                  `Live row update flush failed for ctx.dataset("${name}") ` +
+                  `(non-fatal; terminal rows still persist): ${formatWorkerRowFailureMessage(error)}`,
+                ts: nowMs(),
+              });
+            }
+          }
         });
+        liveUpdateFlushChain = task.catch(() => undefined);
         return task;
       };
 
       const scheduleLiveRowUpdates = () => {
-        if (liveUpdateFailure) return;
         if (
-          pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
+          pendingLiveRowUpdates.length >= MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS
         ) {
           void flushLiveRowUpdates().catch(() => undefined);
           return;
@@ -5103,14 +5269,12 @@ function createMinimalWorkerCtx(
       const enqueueLiveRowUpdate = (
         update: Omit<PlayRowUpdate, 'rowId'> & { runId?: string },
       ): Promise<void> => {
-        if (liveUpdateFailure) {
-          return Promise.reject(liveUpdateFailure);
-        }
         pendingLiveRowUpdates.push(update);
         if (
-          pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
+          pendingLiveRowUpdates.length >= MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS
         ) {
-          return flushLiveRowUpdates();
+          void flushLiveRowUpdates().catch(() => undefined);
+          return Promise.resolve();
         }
         scheduleLiveRowUpdates();
         return Promise.resolve();
@@ -5293,7 +5457,7 @@ function createMinimalWorkerCtx(
                 } catch (rowError) {
                   // Abort/budget errors stay run-fatal and leave no partial
                   // state: rethrow immediately without recording the row.
-                  if (isRowIsolationExemptError(rowError)) {
+                  if (isRunFatalWorkerRowError(rowError)) {
                     throw rowError;
                   }
                   const message = formatWorkerRowFailureMessage(rowError);
@@ -5393,6 +5557,43 @@ function createMinimalWorkerCtx(
           concurrency,
         },
       });
+      const buildRowFailureSamples = () =>
+        failedRowEntries
+          .map((failure, executedIndex) =>
+            failure
+              ? {
+                  rowKey: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
+                  error: failure.error,
+                }
+              : null,
+          )
+          .filter(
+            (sample): sample is { rowKey: string; error: string } =>
+              sample !== null,
+          )
+          .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
+      const fatalMapChunkSummary = async (
+        error: unknown,
+      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => ({
+        chunkIndex,
+        rangeStart: baseOffset + chunkStart,
+        rangeEnd: baseOffset + chunkStart,
+        rowsRead: chunkRows.length,
+        rowsWritten: 0,
+        rowsExecuted: rowsToExecute.length,
+        rowsCached: 0,
+        rowsDuplicateReused: duplicateInputReuseCount,
+        rowsInserted,
+        rowsSkipped,
+        rowsFailed: failedExecutedRows,
+        rowFailureSamples: buildRowFailureSamples(),
+        stepCellsCompleted,
+        stepCellsSkipped,
+        outputDatasetId: `map:${name}`,
+        hash: await hashJson([]),
+        fatalError: formatWorkerRowFailureMessage(error),
+        preview: [],
+      });
       const persistRowsStartedAt = nowMs();
       recordRunnerPerfTrace({
         req,
@@ -5407,7 +5608,6 @@ function createMinimalWorkerCtx(
       try {
         await flushLiveRowUpdates();
         await liveUpdateFlushChain;
-        if (liveUpdateFailure) throw liveUpdateFailure;
         await enqueuePersistExecutedRows();
         await persistFlushChain;
         if (persistFailure) throw persistFailure;
@@ -5433,13 +5633,16 @@ function createMinimalWorkerCtx(
             error: error instanceof Error ? error.message : String(error),
           },
         });
-        throw error;
+        return await fatalMapChunkSummary(error);
       }
       const rejectedWorker = workerResults.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
       );
       if (rejectedWorker) {
+        if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
+          return await fatalMapChunkSummary(rejectedWorker.reason);
+        }
         throw rejectedWorker.reason;
       }
       const resultByKey = new Map<string, T & Record<string, unknown>>();
@@ -5494,20 +5697,7 @@ function createMinimalWorkerCtx(
         0,
         executedRows.length - failedExecutedRows,
       );
-      const rowFailureSamples = failedRowEntries
-        .map((failure, executedIndex) =>
-          failure
-            ? {
-                rowKey: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
-                error: failure.error,
-              }
-            : null,
-        )
-        .filter(
-          (sample): sample is { rowKey: string; error: string } =>
-            sample !== null,
-        )
-        .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
+      const rowFailureSamples = buildRowFailureSamples();
       const publicOut = out.map((row) => publicCsvOutputRow(row));
       const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
         ...row,
@@ -5756,6 +5946,15 @@ function createMinimalWorkerCtx(
         continue;
       }
       const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
+      if (chunkResult.fatalError) {
+        throw new Error(
+          `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
+            `outside the retryable chunk step. Provider calls already executed for ` +
+            `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
+            `Fix the runtime persistence cause and re-run to resume. ` +
+            `First error: ${chunkResult.fatalError}`,
+        );
+      }
       totalRowsWritten += chunkResult.rowsWritten;
       totalRowsExecuted += chunkResult.rowsExecuted;
       totalRowsCached += chunkResult.rowsCached;
@@ -6136,6 +6335,10 @@ function createMinimalWorkerCtx(
             force: request.force === true,
             staleAfterSeconds: request.staleAfterSeconds,
           },
+          {
+            timeoutMs: request.timeoutMs,
+            receiptWaitMs: request.receiptWaitMs,
+          },
         );
       },
     },
@@ -6274,6 +6477,7 @@ function createMinimalWorkerCtx(
                   orgId: req.orgId,
                   callbackBaseUrl: req.callbackUrl,
                   baseUrl: req.baseUrl,
+                  integrationMode: req.integrationMode ?? null,
                   parentExecutorToken: req.executorToken,
                   userEmail: req.userEmail ?? '',
                   profile: 'workers_edge',
@@ -7652,11 +7856,18 @@ function extractMaxCreditsPerRun(contractSnapshot: unknown): number | null {
     : null;
 }
 
+function shouldSkipWorkerComputeBilling(req: RunRequest): boolean {
+  return req.integrationMode === 'fixture' || req.integrationMode === 'eval_stub';
+}
+
 async function finalizeWorkerComputeBilling(input: {
   req: RunRequest;
   success: boolean;
   actionEstimate: number;
 }): Promise<void> {
+  if (shouldSkipWorkerComputeBilling(input.req)) {
+    return;
+  }
   const maxCreditsPerRun = extractMaxCreditsPerRun(input.req.contractSnapshot);
   await postRuntimeApi(input.req.baseUrl, input.req.executorToken, {
     action: 'compute_billing_finalize',
@@ -7712,6 +7923,7 @@ function runRequestFromWorkflowParams(
     callbackUrl: String(params.baseUrl ?? ''),
     executorToken: String(params.executorToken ?? ''),
     baseUrl: String(params.baseUrl ?? ''),
+    integrationMode: normalizeIntegrationMode(params.integrationMode),
     orgId: String(params.orgId ?? ''),
     playName: String(params.playName ?? ''),
     userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
@@ -8051,7 +8263,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
     // somewhere inside executeRunRequest. If it doesn't appear, the
     // throw is in the framework wrapper between the loader and run().
     console.log(
-      `${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl}`,
+      `${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl} integrationMode=${req.integrationMode ?? 'default'}`,
     );
     captureCoordinatorBinding(this.env);
     captureRuntimeApiBinding(this.env);
@@ -8065,6 +8277,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
       ms: 0,
       extra: {
         hasWorkflowStep: true,
+        integrationMode: req.integrationMode ?? null,
       },
     });
     // Fire the one-time wiring probe (deduplicated across runs in the