deepline 0.1.167 → 0.1.169

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -263,6 +263,7 @@ type RunRequest = {
   callbackUrl: string;
   executorToken: string;
   baseUrl: string;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   orgId: string;
   playName: string;
   graphHash?: string | null;
@@ -325,6 +326,14 @@ function getStringField(value: unknown, key: string): string | null {
   return typeof field === 'string' && field.trim() ? field : null;
 }
 
+function normalizeIntegrationMode(
+  value: unknown,
+): 'live' | 'eval_stub' | 'fixture' | null {
+  return value === 'live' || value === 'eval_stub' || value === 'fixture'
+    ? value
+    : null;
+}
+
 function getObjectField(
   value: unknown,
   key: string,
@@ -1472,6 +1481,9 @@ async function callToolDirect(
           body: JSON.stringify({
             payload: input,
             metadata: { parent_run_id: req.runId },
+            ...(req.integrationMode
+              ? { integration_mode: req.integrationMode }
+              : {}),
           }),
         },
         {
@@ -1650,13 +1662,44 @@ const WORKER_TOOL_BATCH_GRACE_MS = 250;
 const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
 const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
 const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
-const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
+const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 1_000;
+const MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS = 1_000;
 /**
  * Bounded number of per-row failure samples carried in chunk summaries and the
  * map's terminal partial-failure log. Every failed row is persisted with its
  * full error in the runtime sheet; the samples just keep run logs readable.
  */
 const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
+
+class RuntimeReceiptPersistenceError extends Error {
+  readonly errors: unknown[];
+
+  constructor(message: string, errors: unknown[] = []) {
+    super(message);
+    this.name = 'RuntimeReceiptPersistenceError';
+    this.errors = errors;
+  }
+}
+
+function isRuntimeReceiptPersistenceError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false;
+  if (error instanceof RuntimeReceiptPersistenceError) return true;
+  if (error instanceof Error && error.name === 'RuntimeReceiptPersistenceError') {
+    return true;
+  }
+  const nestedErrors = (error as { errors?: unknown }).errors;
+  return (
+    Array.isArray(nestedErrors) &&
+    nestedErrors.some(isRuntimeReceiptPersistenceError)
+  );
+}
+
+function isRunFatalWorkerRowError(error: unknown): boolean {
+  return (
+    isRowIsolationExemptError(error) || isRuntimeReceiptPersistenceError(error)
+  );
+}
+
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1982,6 +2025,7 @@ class WorkerToolBatchScheduler {
       !canReclaimTimedOutWorkerToolReceipt({
         ownerRunId: input.runningReceipt.runId,
         currentRunId: this.req.runId,
+        updatedAt: input.runningReceipt.updatedAt,
       })
     ) {
       this.rejectRawRequests(input.group, waitError);
@@ -2100,9 +2144,9 @@ class WorkerToolBatchScheduler {
       await this.failDurableToolRequest(claimed, error);
       return error;
     } catch (receiptError) {
-      return new AggregateError(
-        [error, receiptError],
+      return new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipt could not be marked failed',
+        [error, receiptError],
       );
     }
   }
@@ -2459,25 +2503,38 @@ class WorkerToolBatchScheduler {
             signal: this.abortSignal,
           });
           try {
-            const result = await executeToolWithLifecycle(
-              this.req,
-              { id: request.id, toolId, input: request.input },
-              request.workflowStep,
-              this.callbacks,
-              (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
-              () => this.governor.chargeBudget('retry'),
-              toolContract?.retrySafeTransientHttp === true,
-              this.abortSignal,
-              this.runtimeDeadlineMs,
-            );
+            let result: unknown;
+            try {
+              result = await executeToolWithLifecycle(
+                this.req,
+                { id: request.id, toolId, input: request.input },
+                request.workflowStep,
+                this.callbacks,
+                (retryAfterMs) =>
+                  this.reportBackpressure(toolId, retryAfterMs),
+                () => this.governor.chargeBudget('retry'),
+                toolContract?.retrySafeTransientHttp === true,
+                this.abortSignal,
+                this.runtimeDeadlineMs,
+              );
+            } catch (error) {
+              this.rejectRequests(
+                claimed,
+                await this.failureForRejectedToolRequest(claimed, error),
+              );
+              return;
+            }
             this.settleRequests(
               claimed,
               await this.completeDurableToolRequest(claimed, result),
             );
-          } catch (error) {
+          } catch (receiptError) {
             this.rejectRequests(
               claimed,
-              await this.failureForRejectedToolRequest(claimed, error),
+              new RuntimeReceiptPersistenceError(
+                'Tool call succeeded but durable receipt could not be marked completed',
+                [receiptError],
+              ),
             );
           } finally {
             slot.release();
@@ -2651,9 +2708,9 @@ async function executeBatchedWorkerToolGroup(input: {
           try {
             await input.failRequests(entry.request.memberRequests, entry.error);
           } catch (receiptError) {
-            rejection = new AggregateError(
-              [entry.error, receiptError],
+            rejection = new RuntimeReceiptPersistenceError(
               'Tool call failed and durable receipts could not be marked failed',
+              [entry.error, receiptError],
             );
           }
           for (const claimed of entry.request.memberRequests) {
@@ -2668,16 +2725,28 @@ async function executeBatchedWorkerToolGroup(input: {
           batchResult != null
             ? entry.request.splitResults(batchResult)
             : entry.request.memberRequests.map(() => null);
-        const completedResults = await input.completeRequests(
-          entry.request.memberRequests.map((claimed, index) => ({
-            claimed,
-            result: wrapWorkerToolResult(
-              claimed.request.toolId,
-              splitResults[index] ?? null,
-              toolMetadataFallback(claimed.request.toolId),
-            ),
-          })),
-        );
+        let completedResults: unknown[];
+        try {
+          completedResults = await input.completeRequests(
+            entry.request.memberRequests.map((claimed, index) => ({
+              claimed,
+              result: wrapWorkerToolResult(
+                claimed.request.toolId,
+                splitResults[index] ?? null,
+                toolMetadataFallback(claimed.request.toolId),
+              ),
+            })),
+          );
+        } catch (receiptError) {
+          const rejection = new RuntimeReceiptPersistenceError(
+            'Tool call succeeded but durable receipts could not be marked completed',
+            [receiptError],
+          );
+          for (const claimed of entry.request.memberRequests) {
+            input.rejectRequest(claimed, rejection);
+          }
+          continue;
+        }
         for (let index = 0; index < completedResults.length; index += 1) {
           const claimed = entry.request.memberRequests[index]!;
           const request = claimed.request;
@@ -2690,9 +2759,9 @@ async function executeBatchedWorkerToolGroup(input: {
     try {
       await input.failRequests(input.requests, error);
     } catch (receiptError) {
-      rejection = new AggregateError(
-        [error, receiptError],
+      rejection = new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipts could not be marked failed',
+        [error, receiptError],
       );
     }
     for (const claimed of input.requests) {
@@ -2728,6 +2797,7 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
   stepCellsSkipped: number;
   outputDatasetId: string;
   hash: string;
+  fatalError?: string;
   preview: T[];
   cachedRows?: T[];
 };
@@ -3816,6 +3886,15 @@ async function persistCompletedMapRows(input: {
     rows,
     outputFields,
   });
+  console.warn('[play-runner.persist_completed_map_rows.start]', {
+    runId: req.runId,
+    playName: req.playName,
+    tableNamespace,
+    rows: rows.length,
+    outputFields,
+    extraOutputFields: input.extraOutputFields ?? [],
+    contractColumnCount: sheetContract.columns.length,
+  });
   const persistRequest = {
     ...sessionScope,
     tableNamespace,
@@ -3830,6 +3909,8 @@ async function persistCompletedMapRows(input: {
         .filter((key): key is string => Boolean(key)),
     ),
   ];
+  const expectedVisibleRows =
+    expectedKeys.length > 0 ? expectedKeys.length : rows.length;
   const readVisibleRowCount = async () => {
     if (expectedKeys.length > 0) {
       const result = await harnessReadSheetDatasetRowKeys({
@@ -3852,12 +3933,24 @@ async function persistCompletedMapRows(input: {
     return result.rows.length;
   };
   const result = await harnessPersistCompletedSheetRows(persistRequest);
+  console.warn('[play-runner.persist_completed_map_rows.result]', {
+    runId: req.runId,
+    playName: req.playName,
+    tableNamespace,
+    rows: rows.length,
+    rowsWritten: result.rowsWritten,
+    expectedVisibleRows,
+    expectedKeyCount: expectedKeys.length,
+  });
   let visibleRows = -1;
   let retryWritten: number | null = null;
   let retryVisible: number | null = null;
-  if (rows.length <= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS) {
+  if (
+    rows.length <= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS ||
+    result.rowsWritten !== rows.length
+  ) {
     visibleRows = await readVisibleRowCount();
-    if (visibleRows < rows.length) {
+    if (visibleRows < expectedVisibleRows) {
       await harnessStartSheetDataset({
         ...sessionScope,
         tableNamespace,
@@ -3869,7 +3962,7 @@ async function persistCompletedMapRows(input: {
       const retry = await harnessPersistCompletedSheetRows(persistRequest);
       retryWritten = retry.rowsWritten;
       retryVisible = await readVisibleRowCount();
-      if (retryVisible >= rows.length) {
+      if (retryVisible >= expectedVisibleRows) {
         return {
           rows: rows.length,
           written: result.rowsWritten,
@@ -3879,13 +3972,13 @@ async function persistCompletedMapRows(input: {
         };
       }
       throw new Error(
-        `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}; run ${req.runId}.`,
+        `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}/${expectedVisibleRows}; run ${req.runId}.`,
       );
     }
   }
-  if (result.rowsWritten !== rows.length) {
+  if (result.rowsWritten !== rows.length && visibleRows < expectedVisibleRows) {
     throw new Error(
-      `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; run ${req.runId}.`,
+      `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}/${expectedVisibleRows}; run ${req.runId}.`,
     );
   }
   return {
@@ -4940,7 +5033,7 @@ function createMinimalWorkerCtx(
       let persistFailure: unknown = null;
       let scheduledLiveUpdateTimer: ReturnType<typeof setTimeout> | null = null;
       let liveUpdateFlushChain: Promise<void> = Promise.resolve();
-      let liveUpdateFailure: unknown = null;
+      let liveUpdateFailureCount = 0;
 
       const clearScheduledPersistTimer = () => {
         if (scheduledPersistTimer) {
@@ -5078,25 +5171,35 @@ function createMinimalWorkerCtx(
         const updates = pendingLiveRowUpdates.splice(0);
         const extraOutputFields = Array.from(generatedOutputFields);
         const task = liveUpdateFlushChain.then(async () => {
-          if (liveUpdateFailure) throw liveUpdateFailure;
-          await applyLiveMapRowUpdates({
-            req,
-            tableNamespace: name,
-            outputFields,
-            extraOutputFields,
-            updates,
-          });
-        });
-        liveUpdateFlushChain = task.catch((error) => {
-          liveUpdateFailure ??= error;
+          try {
+            await applyLiveMapRowUpdates({
+              req,
+              tableNamespace: name,
+              outputFields,
+              extraOutputFields,
+              updates,
+            });
+          } catch (error) {
+            liveUpdateFailureCount += 1;
+            if (liveUpdateFailureCount <= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
+              emitEvent({
+                type: 'log',
+                level: 'warn',
+                message:
+                  `Live row update flush failed for ctx.dataset("${name}") ` +
+                  `(non-fatal; terminal rows still persist): ${formatWorkerRowFailureMessage(error)}`,
+                ts: nowMs(),
+              });
+            }
+          }
         });
+        liveUpdateFlushChain = task.catch(() => undefined);
         return task;
       };
 
       const scheduleLiveRowUpdates = () => {
-        if (liveUpdateFailure) return;
         if (
-          pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
+          pendingLiveRowUpdates.length >= MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS
         ) {
           void flushLiveRowUpdates().catch(() => undefined);
           return;
@@ -5111,14 +5214,12 @@ function createMinimalWorkerCtx(
       const enqueueLiveRowUpdate = (
         update: Omit<PlayRowUpdate, 'rowId'> & { runId?: string },
       ): Promise<void> => {
-        if (liveUpdateFailure) {
-          return Promise.reject(liveUpdateFailure);
-        }
         pendingLiveRowUpdates.push(update);
         if (
-          pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
+          pendingLiveRowUpdates.length >= MAP_LIVE_UPDATE_FLUSH_CHUNK_ROWS
         ) {
-          return flushLiveRowUpdates();
+          void flushLiveRowUpdates().catch(() => undefined);
+          return Promise.resolve();
         }
         scheduleLiveRowUpdates();
         return Promise.resolve();
@@ -5301,7 +5402,7 @@ function createMinimalWorkerCtx(
                 } catch (rowError) {
                   // Abort/budget errors stay run-fatal and leave no partial
                   // state: rethrow immediately without recording the row.
-                  if (isRowIsolationExemptError(rowError)) {
+                  if (isRunFatalWorkerRowError(rowError)) {
                     throw rowError;
                   }
                   const message = formatWorkerRowFailureMessage(rowError);
@@ -5401,6 +5502,43 @@ function createMinimalWorkerCtx(
           concurrency,
         },
       });
+      const buildRowFailureSamples = () =>
+        failedRowEntries
+          .map((failure, executedIndex) =>
+            failure
+              ? {
+                  rowKey: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
+                  error: failure.error,
+                }
+              : null,
+          )
+          .filter(
+            (sample): sample is { rowKey: string; error: string } =>
+              sample !== null,
+          )
+          .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
+      const fatalMapChunkSummary = async (
+        error: unknown,
+      ): Promise<WorkerMapChunkSummary<T & Record<string, unknown>>> => ({
+        chunkIndex,
+        rangeStart: baseOffset + chunkStart,
+        rangeEnd: baseOffset + chunkStart,
+        rowsRead: chunkRows.length,
+        rowsWritten: 0,
+        rowsExecuted: rowsToExecute.length,
+        rowsCached: 0,
+        rowsDuplicateReused: duplicateInputReuseCount,
+        rowsInserted,
+        rowsSkipped,
+        rowsFailed: failedExecutedRows,
+        rowFailureSamples: buildRowFailureSamples(),
+        stepCellsCompleted,
+        stepCellsSkipped,
+        outputDatasetId: `map:${name}`,
+        hash: await hashJson([]),
+        fatalError: formatWorkerRowFailureMessage(error),
+        preview: [],
+      });
       const persistRowsStartedAt = nowMs();
       recordRunnerPerfTrace({
         req,
@@ -5415,7 +5553,6 @@ function createMinimalWorkerCtx(
       try {
         await flushLiveRowUpdates();
         await liveUpdateFlushChain;
-        if (liveUpdateFailure) throw liveUpdateFailure;
         await enqueuePersistExecutedRows();
         await persistFlushChain;
         if (persistFailure) throw persistFailure;
@@ -5441,13 +5578,16 @@ function createMinimalWorkerCtx(
             error: error instanceof Error ? error.message : String(error),
           },
         });
-        throw error;
+        return await fatalMapChunkSummary(error);
       }
       const rejectedWorker = workerResults.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
       );
       if (rejectedWorker) {
+        if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
+          return await fatalMapChunkSummary(rejectedWorker.reason);
+        }
         throw rejectedWorker.reason;
       }
       const resultByKey = new Map<string, T & Record<string, unknown>>();
@@ -5502,20 +5642,7 @@ function createMinimalWorkerCtx(
         0,
         executedRows.length - failedExecutedRows,
       );
-      const rowFailureSamples = failedRowEntries
-        .map((failure, executedIndex) =>
-          failure
-            ? {
-                rowKey: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
-                error: failure.error,
-              }
-            : null,
-        )
-        .filter(
-          (sample): sample is { rowKey: string; error: string } =>
-            sample !== null,
-        )
-        .slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
+      const rowFailureSamples = buildRowFailureSamples();
       const publicOut = out.map((row) => publicCsvOutputRow(row));
       const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
         ...row,
@@ -5764,6 +5891,15 @@ function createMinimalWorkerCtx(
         continue;
       }
       const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
+      if (chunkResult.fatalError) {
+        throw new Error(
+          `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
+            `outside the retryable chunk step. Provider calls already executed for ` +
+            `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
+            `Fix the runtime persistence cause and re-run to resume. ` +
+            `First error: ${chunkResult.fatalError}`,
+        );
+      }
       totalRowsWritten += chunkResult.rowsWritten;
       totalRowsExecuted += chunkResult.rowsExecuted;
       totalRowsCached += chunkResult.rowsCached;
@@ -6282,6 +6418,7 @@ function createMinimalWorkerCtx(
                   orgId: req.orgId,
                   callbackBaseUrl: req.callbackUrl,
                   baseUrl: req.baseUrl,
+                  integrationMode: req.integrationMode ?? null,
                   parentExecutorToken: req.executorToken,
                   userEmail: req.userEmail ?? '',
                   profile: 'workers_edge',
@@ -7660,11 +7797,18 @@ function extractMaxCreditsPerRun(contractSnapshot: unknown): number | null {
     : null;
 }
 
+function shouldSkipWorkerComputeBilling(req: RunRequest): boolean {
+  return req.integrationMode === 'fixture' || req.integrationMode === 'eval_stub';
+}
+
 async function finalizeWorkerComputeBilling(input: {
   req: RunRequest;
   success: boolean;
   actionEstimate: number;
 }): Promise<void> {
+  if (shouldSkipWorkerComputeBilling(input.req)) {
+    return;
+  }
   const maxCreditsPerRun = extractMaxCreditsPerRun(input.req.contractSnapshot);
   await postRuntimeApi(input.req.baseUrl, input.req.executorToken, {
     action: 'compute_billing_finalize',
@@ -7720,6 +7864,7 @@ function runRequestFromWorkflowParams(
     callbackUrl: String(params.baseUrl ?? ''),
     executorToken: String(params.executorToken ?? ''),
     baseUrl: String(params.baseUrl ?? ''),
+    integrationMode: normalizeIntegrationMode(params.integrationMode),
     orgId: String(params.orgId ?? ''),
     playName: String(params.playName ?? ''),
     userEmail: typeof params.userEmail === 'string' ? params.userEmail : null,
@@ -8059,7 +8204,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
     // somewhere inside executeRunRequest. If it doesn't appear, the
     // throw is in the framework wrapper between the loader and run().
     console.log(
-      `${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl}`,
+      `${runPrefix} TenantWorkflow.run entered baseUrl=${req.baseUrl} integrationMode=${req.integrationMode ?? 'default'}`,
     );
     captureCoordinatorBinding(this.env);
     captureRuntimeApiBinding(this.env);
@@ -8073,6 +8218,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
       ms: 0,
       extra: {
         hasWorkflowStep: true,
+        integrationMode: req.integrationMode ?? null,
       },
     });
     // Fire the one-time wiring probe (deduplicated across runs in the