deepline 0.1.130 → 0.1.132

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -168,6 +168,9 @@ const MAP_FRAME_FLUSH_ROW_INTERVAL = 100;
 /** Executed-row sheet persistence chunking: rows AND bytes, whichever first. */
 const MAP_PERSIST_CHUNK_ROWS = 2_000;
 const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024;
+const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
+const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
+const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
 const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
 const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
@@ -219,11 +222,143 @@ function comparePersistableMapRowsByInputIndex(
   return leftIndex - rightIndex;
 }
 
+function persistableMapRowIdentity(row: PersistableMapRow): string | null {
+  if (row.key) return `key:${row.key}`;
+  return typeof row.inputIndex === 'number' && Number.isFinite(row.inputIndex)
+    ? `index:${Math.floor(row.inputIndex)}`
+    : null;
+}
+
+function persistableMapRowBytes(row: PersistableMapRow): number {
+  return JSON.stringify(row).length;
+}
+
 type FieldMapRunResult = {
   completedRows: PersistableMapRow[];
   failedRows: PersistableMapRow[];
 };
 
+type IncrementalMapRowPersistence = {
+  persistRows: (rows: PersistableMapRow[]) => Promise<void>;
+  isPersisted: (row: PersistableMapRow) => boolean;
+  flush: () => Promise<void>;
+};
+
+function createIncrementalMapRowPersistence(
+  persistRows: (rows: PersistableMapRow[]) => Promise<void>,
+): IncrementalMapRowPersistence {
+  const persisted = new Set<string>();
+  const queued = new Set<string>();
+  let pendingRows: PersistableMapRow[] = [];
+  let pendingBytes = 0;
+  let pendingResolvers: Array<{
+    resolve: () => void;
+    reject: (error: unknown) => void;
+  }> = [];
+  let scheduledTimer: ReturnType<typeof setTimeout> | null = null;
+  let flushChain: Promise<void> = Promise.resolve();
+
+  const clearScheduledTimer = (): void => {
+    if (scheduledTimer) {
+      clearTimeout(scheduledTimer);
+      scheduledTimer = null;
+    }
+  };
+
+  const flushPendingRows = (): Promise<void> => {
+    clearScheduledTimer();
+    const rows = pendingRows;
+    const resolvers = pendingResolvers;
+    pendingRows = [];
+    pendingBytes = 0;
+    pendingResolvers = [];
+    if (rows.length === 0) {
+      resolvers.forEach(({ resolve }) => resolve());
+      return flushChain;
+    }
+    flushChain = flushChain
+      .then(async () => {
+        await persistRows(rows);
+        for (const row of rows) {
+          const identity = persistableMapRowIdentity(row);
+          if (identity) {
+            persisted.add(identity);
+            queued.delete(identity);
+          }
+        }
+      })
+      .then(
+        () => {
+          resolvers.forEach(({ resolve }) => resolve());
+        },
+        (error) => {
+          for (const row of rows) {
+            const identity = persistableMapRowIdentity(row);
+            if (identity) queued.delete(identity);
+          }
+          resolvers.forEach(({ reject }) => reject(error));
+          throw error;
+        },
+      );
+    return flushChain;
+  };
+
+  const scheduleFlush = (): void => {
+    if (scheduledTimer) return;
+    scheduledTimer = setTimeout(() => {
+      scheduledTimer = null;
+      void flushPendingRows().catch(() => {
+        // Each queued caller is rejected by flushPendingRows; this scheduled
+        // fire-and-forget path must not create a second unhandled rejection.
+      });
+    }, MAP_INCREMENTAL_PERSIST_INTERVAL_MS);
+  };
+
+  return {
+    persistRows: async (rows) => {
+      const freshRows: PersistableMapRow[] = [];
+      for (const row of rows) {
+        const identity = persistableMapRowIdentity(row);
+        if (identity && (persisted.has(identity) || queued.has(identity))) {
+          continue;
+        }
+        if (identity) queued.add(identity);
+        freshRows.push(row);
+      }
+      if (freshRows.length === 0) return;
+      await new Promise<void>((resolve, reject) => {
+        pendingRows.push(...freshRows);
+        pendingBytes += freshRows.reduce(
+          (total, row) => total + persistableMapRowBytes(row),
+          0,
+        );
+        pendingResolvers.push({ resolve, reject });
+        if (
+          pendingRows.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS ||
+          pendingBytes >= MAP_INCREMENTAL_PERSIST_CHUNK_BYTES
+        ) {
+          void flushPendingRows().catch(() => {
+            // Each queued caller is rejected by flushPendingRows.
+          });
+          return;
+        }
+        scheduleFlush();
+      });
+    },
+    isPersisted: (row) => {
+      const identity = persistableMapRowIdentity(row);
+      return identity ? persisted.has(identity) : false;
+    },
+    flush: async () => {
+      if (pendingRows.length > 0) {
+        clearScheduledTimer();
+        await flushPendingRows();
+      }
+      await flushChain;
+    },
+  };
+}
+
 class FailFastMapRowsError extends Error {
   readonly completedRows: PersistableMapRow[];
   readonly failedRows: PersistableMapRow[];
@@ -1819,6 +1954,7 @@ export class PlayContextImpl {
     let totalInputCount = 0;
     let rawItems: Record<string, unknown>[] = [];
     let itemsToProcess: Array<Record<string, unknown>> = [];
+    let itemOriginalIndexes: number[] = [];
     let completedItemsByKey: Map<string, Record<string, unknown>> | null = null;
     const datasetColumnNames = Object.keys(input);
     const stripFieldOutputs = (row: Record<string, unknown>) => {
@@ -1923,6 +2059,7 @@ export class PlayContextImpl {
     itemsToProcess = materializedItems.map((item) =>
       this.toOutputRow(item as Record<string, unknown>),
     );
+    itemOriginalIndexes = materializedItems.map((_item, index) => index);
     if (this.#options.onMapStart) {
       const shouldPassRowKey = explicitKeyResolver != null;
       // toSerializableCsvAliasedRow (not a plain spread): projected CSV
@@ -1968,7 +2105,7 @@ export class PlayContextImpl {
         const rowKey = persistedRowIdentity(row, index);
         if (rowKey) pendingRowsByKey.set(rowKey, row);
       }
-      itemsToProcess = materializedItems
+      const pendingItems = materializedItems
         .map((item, index) => ({
           row: this.toOutputRow(item as Record<string, unknown>),
           index,
@@ -1981,8 +2118,13 @@ export class PlayContextImpl {
           // the sheet round-trip. Merge the persisted data/meta over the
           // in-memory row so key functions and column resolvers keep seeing
           // projected aliases.
-          return persisted ? cloneCsvAliasedRow(row, persisted) : row;
+          return {
+            row: persisted ? cloneCsvAliasedRow(row, persisted) : row,
+            index,
+          };
         });
+      itemsToProcess = pendingItems.map((item) => item.row);
+      itemOriginalIndexes = pendingItems.map((item) => item.index);
       if (mapStartResult.completedRows.length > 0) {
         completedItemsByKey = new Map();
         for (
@@ -2008,7 +2150,7 @@ export class PlayContextImpl {
     const completedRowKeys =
       completedItemsByKey != null ? [...completedItemsByKey.keys()] : [];
     const pendingRowKeys = itemsToProcess.map((item, index) =>
-      rowIdentity(this.toOutputRow(item), index),
+      rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index),
     );
     this.setMapFrame({
       mapInvocationId: mapScope.mapInvocationId,
@@ -2040,9 +2182,10 @@ export class PlayContextImpl {
     >();
     for (let index = 0; index < itemsToProcess.length; index += 1) {
       const row = itemsToProcess[index]!;
-      const rowKey = rowIdentity(row, index);
+      const originalIndex = itemOriginalIndexes[index] ?? index;
+      const rowKey = rowIdentity(row, originalIndex);
       if (!rowsToExecuteByKey.has(rowKey)) {
-        rowsToExecuteByKey.set(rowKey, { row, originalIndex: index });
+        rowsToExecuteByKey.set(rowKey, { row, originalIndex });
       }
     }
     const rowsToExecuteEntries = [...rowsToExecuteByKey.entries()].map(
@@ -2090,6 +2233,9 @@ export class PlayContextImpl {
       }
       await flushChunk();
     };
+    const incrementalPersistence = this.#options.onMapRowsCompleted
+      ? createIncrementalMapRowPersistence(persistMapRows)
+      : null;
 
     let mapResult: FieldMapRunResult;
     try {
@@ -2108,6 +2254,7 @@ export class PlayContextImpl {
           executionRowIndexes: rowsToExecuteEntries.map(
             (entry) => entry.originalIndex,
           ),
+          incrementalPersistence,
         },
       );
     } catch (error) {
@@ -2116,11 +2263,17 @@ export class PlayContextImpl {
           error.completedRows.length > 0
             ? error.completedRows
             : error.failedRows;
+        await incrementalPersistence?.flush();
+        const unpersistedRows = incrementalPersistence
+          ? rowsToPersist.filter(
+              (row) => !incrementalPersistence.isPersisted(row),
+            )
+          : rowsToPersist;
         const persistStartedAt = Date.now();
-        await persistMapRows(rowsToPersist);
-        if (rowsToPersist.length > 0) {
+        await persistMapRows(unpersistedRows);
+        if (unpersistedRows.length > 0) {
           this.log(
-            `Persisted ${rowsToPersist.length} fail-fast rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
+            `Persisted ${unpersistedRows.length} fail-fast rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
           );
         }
         this.activeMapCellMeta = null;
@@ -2164,9 +2317,11 @@ export class PlayContextImpl {
     // source of truth, not this in-memory results array. Chunked by rows AND
     // bytes so large cells (scraped pages) never produce oversized writes.
     if (resultsByKey.size > 0 || mapResult.failedRows.length > 0) {
+      await incrementalPersistence?.flush();
       const mapCellMeta = this.activeMapCellMeta;
       const persistRows: PersistableMapRow[] = [];
       for (const row of mapResult.completedRows) {
+        if (incrementalPersistence?.isPersisted(row)) continue;
         const rowKey = row.key;
         const meta = mapCellMeta?.get(rowKey);
         persistRows.push({
@@ -2178,12 +2333,18 @@ export class PlayContextImpl {
           },
         });
       }
-      persistRows.push(...mapResult.failedRows);
+      persistRows.push(
+        ...mapResult.failedRows.filter(
+          (row) => !incrementalPersistence?.isPersisted(row),
+        ),
+      );
       const persistStartedAt = Date.now();
       await persistMapRows(persistRows);
-      this.log(
-        `Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
-      );
+      if (persistRows.length > 0) {
+        this.log(
+          `Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
+        );
+      }
     }
     this.activeMapCellMeta = null;
 
@@ -2255,6 +2416,7 @@ export class PlayContextImpl {
        * want to throttle a wide map.
        */
       concurrency?: number;
+      incrementalPersistence?: IncrementalMapRowPersistence | null;
     },
   ): Promise<FieldMapRunResult> {
     const fieldEntries = Object.entries(definition);
@@ -2299,6 +2461,14 @@ export class PlayContextImpl {
     const failFastOnRowError = runtimeOptions?.onRowError === 'fail';
     const completedRowsToPersist: PersistableMapRow[] = [];
     const failedRowsToPersist: PersistableMapRow[] = [];
+    const incrementalPersistence =
+      runtimeOptions?.incrementalPersistence ?? null;
+    const enqueueIncrementalPersist = (row: PersistableMapRow): void => {
+      void incrementalPersistence?.persistRows([row]).catch(() => {
+        // The final map-level flush awaits the same chain and surfaces the
+        // persistence failure loudly without holding a completed row slot open.
+      });
+    };
 
     if (completedRows > 0 || pendingRows !== totalRows) {
       this.log(
@@ -2605,7 +2775,7 @@ export class PlayContextImpl {
                 : {},
             });
             if (failFastOnRowError) {
-              failedRowsToPersist.push({
+              const failedRow: PersistableMapRow = {
                 key: rowKey,
                 inputIndex: rowIndex,
                 data: this.toPersistedOutputRow(
@@ -2616,21 +2786,23 @@ export class PlayContextImpl {
                   : {}),
                 status: 'failed',
                 error: formattedError,
-              });
+              };
+              failedRowsToPersist.push(failedRow);
               throw error;
             }
             const failedData = this.toPersistedOutputRow(
               cloneCsvAliasedRow(baseRow, computedFields),
             );
             const cellMetaPatch = this.activeMapCellMeta?.get(rowKey);
-            failedRowsToPersist.push({
+            const failedRow: PersistableMapRow = {
               key: rowKey,
               inputIndex: rowIndex,
               data: failedData,
               ...(cellMetaPatch ? { cellMetaPatch } : {}),
               status: 'failed',
               error: formattedError,
-            });
+            };
+            failedRowsToPersist.push(failedRow);
             updateMapFrameProgress({
               failedRowKey: rowKey,
             });
@@ -2642,6 +2814,7 @@ export class PlayContextImpl {
               error: formattedError,
               dataPatch: {},
             });
+            enqueueIncrementalPersist(failedRow);
             return FAILED_ROW;
           }
           const cellValue = this.serializeCellValue(value);
@@ -2682,14 +2855,16 @@ export class PlayContextImpl {
           dataPatch: {},
         });
         const publicRow = this.toPublicOutputRow(merged);
-        completedRowsToPersist.push({
+        const completedRow: PersistableMapRow = {
           key: rowKey,
           inputIndex: rowIndex,
           data: this.toPersistedOutputRow(merged),
           ...(this.activeMapCellMeta?.get(rowKey)
             ? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
             : {}),
-        });
+        };
+        completedRowsToPersist.push(completedRow);
+        enqueueIncrementalPersist(completedRow);
         return publicRow;
       } catch (error) {
         if (isPlayRowExecutionSuspendedError(error)) {
@@ -2807,6 +2982,7 @@ export class PlayContextImpl {
         ).values(),
       ];
       this.pendingRowEventBoundaries = [];
+      await incrementalPersistence?.flush();
       this.#options.onBatchComplete?.(this.checkpoint);
       throw new PlayExecutionSuspendedError({
         kind: 'integration_event_batch',
@@ -4960,7 +5136,42 @@ export class PlayContextImpl {
       [...byTool.entries()].map(async ([toolId, requests]) => {
         this.log(`Executing tool batch ${toolId}: ${requests.length} calls`);
 
+        const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
+          if (stepRequests.length === 0) return;
+          const stepResults: PlayStepRowResult[] = stepRequests.map((req) => {
+            const cachedResult = this.getCachedToolResult(
+              toolId,
+              this.buildToolResultCacheKey({
+                rowId: req.rowId,
+                tableNamespace: req.tableNamespace,
+                rowKey: req.rowKey ?? undefined,
+                callId: req.callId,
+              }),
+            );
+            return {
+              rowId: req.rowId,
+              status: cachedResult?.result != null ? 'completed' : 'failed',
+              success: cachedResult?.result != null,
+              value: cachedResult?.result,
+              error: cachedResult?.result != null ? null : 'Tool call failed',
+            };
+          });
+          const toolStep = {
+            type: 'tool' as const,
+            toolId,
+            // Keep the step trace preview-sized for large map pages.
+            results: compactRowResultsPreview(stepResults),
+            description: normalizeStepDescription(stepRequests[0]?.description),
+          };
+          if (this.activeDatasetStep) {
+            this.activeDatasetStep.substeps.push(toolStep);
+          } else {
+            this.steps.push(toolStep);
+          }
+        };
+
         const pendingRequests: ToolCallRequest[] = [];
+        const recoveredRequests: ToolCallRequest[] = [];
         for (const req of requests) {
           const cached = this.getCachedToolResult(
             toolId,
@@ -4978,10 +5189,12 @@ export class PlayContextImpl {
               resolver.resolve(cached.result);
               this.toolCallResolvers.delete(req.callId);
             }
+            recoveredRequests.push(req);
           } else {
             pendingRequests.push(req);
           }
         }
+        recordToolStep(recoveredRequests);
 
         if (pendingRequests.length > 0) {
           const strategy =
@@ -5001,49 +5214,47 @@ export class PlayContextImpl {
                     4,
                   )
                 : 4;
-            for (
-              let start = 0;
-              start < compiledBatches.length;
-              start += batchSize
-            ) {
-              const chunk = compiledBatches.slice(start, start + batchSize);
-              const settled = await Promise.allSettled(
-                chunk.map((batch) =>
-                  this.callToolAPI(batch.batchOperation, batch.batchPayload),
+            await executeChunkedRequests({
+              requests: compiledBatches,
+              batchSize,
+              execute: async (batch) =>
+                await this.callToolAPI(
+                  batch.batchOperation,
+                  batch.batchPayload,
                 ),
-              );
-
-              for (let index = 0; index < chunk.length; index += 1) {
-                const requestBatch = chunk[index]!;
-                const outcome = settled[index]!;
-                if (outcome.status === 'fulfilled') {
+              onChunkComplete: async (chunkResults) => {
+                for (const entry of chunkResults) {
+                  if (entry.error !== undefined) {
+                    for (const request of entry.request.memberRequests) {
+                      this.rejectToolCall(toolId, request, entry.error);
+                    }
+                    continue;
+                  }
                   const splitResults =
-                    outcome.value != null
-                      ? requestBatch.splitResults(outcome.value)
-                      : requestBatch.memberRequests.map(() => null);
+                    entry.result != null
+                      ? entry.request.splitResults(entry.result)
+                      : entry.request.memberRequests.map(() => null);
 
                   for (
                     let index = 0;
-                    index < requestBatch.memberRequests.length;
+                    index < entry.request.memberRequests.length;
                     index += 1
                   ) {
-                    const request = requestBatch.memberRequests[index]!;
+                    const request = entry.request.memberRequests[index]!;
                     await this.resolveToolCall(
                       toolId,
                       request,
                       splitResults[index] ?? null,
                     );
                   }
-                  continue;
                 }
 
-                for (const request of requestBatch.memberRequests) {
-                  this.rejectToolCall(toolId, request, outcome.reason);
-                }
-              }
-
-              this.#options.onBatchComplete?.(this.checkpoint);
-            }
+                recordToolStep(
+                  chunkResults.flatMap((entry) => entry.request.memberRequests),
+                );
+                this.#options.onBatchComplete?.(this.checkpoint);
+              },
+            });
           } else {
             const batchSize = await this.governor.suggestedParallelism(
               toolId,
@@ -5055,66 +5266,32 @@ export class PlayContextImpl {
               start += batchSize
             ) {
               const chunk = pendingRequests.slice(start, start + batchSize);
-              const settled = await Promise.allSettled(
-                chunk.map((request) =>
-                  this.callToolExecutionAPI(toolId, request.input),
-                ),
+              await Promise.allSettled(
+                chunk.map(async (request) => {
+                  try {
+                    const execution = await this.callToolExecutionAPI(
+                      toolId,
+                      request.input,
+                    );
+                    await this.resolveToolCall(
+                      toolId,
+                      request,
+                      execution?.result ?? null,
+                      execution?.metadata ?? null,
+                      execution?.jobId,
+                      execution?.meta,
+                    );
+                  } catch (error) {
+                    this.rejectToolCall(toolId, request, error);
+                  }
+                }),
               );
 
-              for (let index = 0; index < chunk.length; index += 1) {
-                const request = chunk[index]!;
-                const outcome = settled[index]!;
-                if (outcome.status === 'fulfilled') {
-                  await this.resolveToolCall(
-                    toolId,
-                    request,
-                    outcome.value?.result ?? null,
-                    outcome.value?.metadata ?? null,
-                    outcome.value?.jobId,
-                    outcome.value?.meta,
-                  );
-                  continue;
-                }
-
-                this.rejectToolCall(toolId, request, outcome.reason);
-              }
-
+              recordToolStep(chunk);
               this.#options.onBatchComplete?.(this.checkpoint);
             }
           }
         }
-
-        // Record step — nest under map if inside one
-        const stepResults: PlayStepRowResult[] = requests.map((req) => {
-          const cachedResult = this.getCachedToolResult(
-            toolId,
-            this.buildToolResultCacheKey({
-              rowId: req.rowId,
-              tableNamespace: req.tableNamespace,
-              rowKey: req.rowKey ?? undefined,
-              callId: req.callId,
-            }),
-          );
-          return {
-            rowId: req.rowId,
-            status: cachedResult?.result != null ? 'completed' : 'failed',
-            success: cachedResult?.result != null,
-            value: cachedResult?.result,
-            error: cachedResult?.result != null ? null : 'Tool call failed',
-          };
-        });
-        const toolStep = {
-          type: 'tool' as const,
-          toolId,
-          // Keep the step trace preview-sized for large map pages.
-          results: compactRowResultsPreview(stepResults),
-          description: normalizeStepDescription(requests[0]?.description),
-        };
-        if (this.activeDatasetStep) {
-          this.activeDatasetStep.substeps.push(toolStep);
-        } else {
-          this.steps.push(toolStep);
-        }
       }),
     );
   }