deepline 0.1.294 → 0.1.296

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.294',
+  version: '0.1.296',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -35,6 +35,7 @@ import type { PlayQueueHint } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
 import { stringifyPostgresJson } from './postgres-json';
 import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
+import { RuntimeSheetRowWriter } from './runtime-sheet-row-writer';
 import type { WorkReceiptFailureKind } from './work-receipts';
 import {
   completedMapRowOutcome,
@@ -300,9 +301,8 @@ const MAP_FRAME_FLUSH_ROW_INTERVAL = 100;
 /** Executed-row sheet persistence chunking: rows AND bytes, whichever first. */
 const MAP_PERSIST_CHUNK_ROWS = 2_000;
 const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024;
-const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
-const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
-const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
+const MAP_ROW_WRITER_BUFFER_BYTES = 16 * 1024 * 1024;
+const MAP_ROW_WRITER_MAX_FLUSH_MS = 100;
 const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
 // Tool scheduling lanes wait at most this long for same-lane row continuations.
 // The window is fixed from the first item and never resets on later arrivals.
@@ -778,123 +778,117 @@ type FieldMapRunResult = {
   failedRows: PersistableMapRow[];
 };
 
-type IncrementalMapRowPersistence = {
-  persistRows: (rows: PersistableMapRow[]) => Promise<void>;
+type RuntimeMapRowPersistence = {
+  persistRows: (rows: PersistableMapRow[]) => {
+    admitted: Promise<void>;
+    committed: Promise<void>;
+  };
   isPersisted: (row: PersistableMapRow) => boolean;
+  checkpoint: (updates: PlayRowUpdate[]) => Promise<void>;
   flush: () => Promise<void>;
 };
 
-function createIncrementalMapRowPersistence(
+function createRuntimeMapRowPersistence(
   persistRows: (rows: PersistableMapRow[]) => Promise<void>,
-): IncrementalMapRowPersistence {
+  options?: {
+    persistCheckpoint?: (updates: PlayRowUpdate[]) => Promise<void>;
+    onFailure?: (error: unknown) => void;
+  },
+): RuntimeMapRowPersistence {
   const persisted = new Set<string>();
   const queued = new Set<string>();
-  let pendingRows: PersistableMapRow[] = [];
-  let pendingBytes = 0;
-  let pendingResolvers: Array<{
-    resolve: () => void;
-    reject: (error: unknown) => void;
-  }> = [];
-  let scheduledTimer: ReturnType<typeof setTimeout> | null = null;
-  let flushChain: Promise<void> = Promise.resolve();
-
-  const clearScheduledTimer = (): void => {
-    if (scheduledTimer) {
-      clearTimeout(scheduledTimer);
-      scheduledTimer = null;
-    }
-  };
+  const writer = new RuntimeSheetRowWriter<PersistableMapRow, PlayRowUpdate>({
+    terminalKey: (row) => {
+      const identity = persistableMapRowIdentity(row);
+      if (!identity) {
+        throw new Error('Runtime Sheet terminal row is missing an identity.');
+      }
+      return identity;
+    },
+    checkpointKey: (update) => `key:${update.key}`,
+    mergeCheckpointUpdates: (current, incoming) => ({
+      ...current,
+      ...incoming,
+      dataPatch: {
+        ...(current.dataPatch ?? {}),
+        ...(incoming.dataPatch ?? {}),
+      },
+      cellMetaPatch: {
+        ...(current.cellMetaPatch ?? {}),
+        ...(incoming.cellMetaPatch ?? {}),
+      },
+    }),
+    estimateTerminalBytes: persistableMapRowBytes,
+    estimateCheckpointBytes: (update) => JSON.stringify(update).length,
+    maxBatchRows: MAP_PERSIST_CHUNK_ROWS,
+    maxBatchBytes: MAP_PERSIST_CHUNK_BYTES,
+    maxBufferedBytes: MAP_ROW_WRITER_BUFFER_BYTES,
+    maxFlushMs: MAP_ROW_WRITER_MAX_FLUSH_MS,
+    onFailure: options?.onFailure,
+    writeBatch: async (batch) => {
+      if (batch.kind === 'terminal') {
+        await persistRows([...batch.rows]);
+        return { committed: batch.rows.length };
+      }
+      if (batch.updates.length > 0) {
+        await options?.persistCheckpoint?.([...batch.updates]);
+      }
+      return { committed: batch.updates.length };
+    },
+  });
 
-  const flushPendingRows = (): Promise<void> => {
-    clearScheduledTimer();
-    const rows = pendingRows;
-    const resolvers = pendingResolvers;
-    pendingRows = [];
-    pendingBytes = 0;
-    pendingResolvers = [];
-    if (rows.length === 0) {
-      resolvers.forEach(({ resolve }) => resolve());
-      return flushChain;
-    }
-    flushChain = flushChain
-      .then(async () => {
-        await persistRows(rows);
-        for (const row of rows) {
-          const identity = persistableMapRowIdentity(row);
-          if (identity) {
-            persisted.add(identity);
-            queued.delete(identity);
-          }
+  return {
+    persistRows: (rows) => {
+      const admissions: Promise<void>[] = [];
+      const commits: Promise<void>[] = [];
+      const clearQueuedState = (identity: string | null) => {
+        if (identity) {
+          queued.delete(identity);
         }
-      })
-      .then(
-        () => {
-          resolvers.forEach(({ resolve }) => resolve());
-        },
-        (error) => {
-          for (const row of rows) {
-            const identity = persistableMapRowIdentity(row);
-            if (identity) queued.delete(identity);
-          }
-          resolvers.forEach(({ reject }) => reject(error));
-          throw error;
-        },
-      );
-    return flushChain;
-  };
-
-  const scheduleFlush = (): void => {
-    if (scheduledTimer) return;
-    scheduledTimer = setTimeout(() => {
-      scheduledTimer = null;
-      void flushPendingRows().catch(() => {
-        // Each queued caller is rejected by flushPendingRows; this scheduled
-        // fire-and-forget path must not create a second unhandled rejection.
-      });
-    }, MAP_INCREMENTAL_PERSIST_INTERVAL_MS);
-  };
+      };
 
-  return {
-    persistRows: async (rows) => {
-      const freshRows: PersistableMapRow[] = [];
       for (const row of rows) {
         const identity = persistableMapRowIdentity(row);
         if (identity && (persisted.has(identity) || queued.has(identity))) {
           continue;
         }
         if (identity) queued.add(identity);
-        freshRows.push(row);
-      }
-      if (freshRows.length === 0) return;
-      await new Promise<void>((resolve, reject) => {
-        pendingRows.push(...freshRows);
-        pendingBytes += freshRows.reduce(
-          (total, row) => total + persistableMapRowBytes(row),
-          0,
+        const settlement = writer.settle(row);
+        admissions.push(
+          settlement.admitted.catch((error) => {
+            clearQueuedState(identity);
+            throw error;
+          }),
         );
-        pendingResolvers.push({ resolve, reject });
-        if (
-          pendingRows.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS ||
-          pendingBytes >= MAP_INCREMENTAL_PERSIST_CHUNK_BYTES
-        ) {
-          void flushPendingRows().catch(() => {
-            // Each queued caller is rejected by flushPendingRows.
-          });
-          return;
-        }
-        scheduleFlush();
-      });
+        commits.push(
+          settlement.committed.then(
+            () => {
+              if (identity) {
+                persisted.add(identity);
+                clearQueuedState(identity);
+              }
+            },
+            (error) => {
+              clearQueuedState(identity);
+              throw error;
+            },
+          ),
+        );
+      }
+      return {
+        admitted: Promise.all(admissions).then(() => {}),
+        committed: Promise.all(commits).then(() => {}),
+      };
     },
     isPersisted: (row) => {
       const identity = persistableMapRowIdentity(row);
       return identity ? persisted.has(identity) : false;
     },
+    checkpoint: async (updates) => {
+      await writer.checkpoint(updates);
+    },
     flush: async () => {
-      if (pendingRows.length > 0) {
-        clearScheduledTimer();
-        await flushPendingRows();
-      }
-      await flushChain;
+      await writer.finish();
     },
   };
 }
@@ -1310,6 +1304,13 @@ export class PlayContextImpl {
    * completed/cached cell meta — cross-run reuse decisions read that meta.
    */
   private activeMapCellMeta: Map<string, Record<string, unknown>> | null = null;
+  /**
+   * Latest partial row patch per key for the active map. Terminal rows are
+   * removed after their Runtime Sheet settlement commits; any remainder is the
+   * exact partial-cell checkpoint that must cross the durability barrier before
+   * an integration-event suspension is published.
+   */
+  private activeMapCheckpointUpdates: Map<string, PlayRowUpdate> | null = null;
   private lastProgressHeartbeatAt = 0;
   private pendingRowEventBoundaries: Array<{
     boundaryId: string;
@@ -1642,6 +1643,33 @@ export class PlayContextImpl {
           : { ...update.cellMetaPatch },
       );
     }
+    if (key && this.activeMapCheckpointUpdates) {
+      const checkpointKey = `${tableNamespace ?? ''}\u0000${key}`;
+      const existing = this.activeMapCheckpointUpdates.get(checkpointKey);
+      const next: PlayRowUpdate = {
+        ...(existing ?? {
+          key,
+          rowId: update.rowId,
+          tableNamespace,
+        }),
+        key,
+        rowId: update.rowId,
+        tableNamespace,
+        ...(update.status !== undefined ? { status: update.status } : {}),
+        ...(update.stage !== undefined ? { stage: update.stage } : {}),
+        ...(update.provider !== undefined ? { provider: update.provider } : {}),
+        ...(update.error !== undefined ? { error: update.error } : {}),
+        dataPatch: {
+          ...(existing?.dataPatch ?? {}),
+          ...(update.dataPatch ?? {}),
+        },
+        cellMetaPatch: {
+          ...(existing?.cellMetaPatch ?? {}),
+          ...(update.cellMetaPatch ?? {}),
+        },
+      };
+      this.activeMapCheckpointUpdates.set(checkpointKey, next);
+    }
     if (!key || !this.#options.onRowUpdate) {
       return;
     }
@@ -1652,6 +1680,13 @@ export class PlayContextImpl {
     });
   }
 
+  private clearActiveMapCheckpointUpdate(
+    key: string,
+    tableNamespace: string,
+  ): void {
+    this.activeMapCheckpointUpdates?.delete(`${tableNamespace}\u0000${key}`);
+  }
+
   private emitExecutionEvent(event: PlayExecutionEvent): void {
     if (!this.#options.onExecutionEvent) {
       return;
@@ -3969,32 +4004,23 @@ export class PlayContextImpl {
         const flushChunk = async () => {
           if (chunk.length === 0) return;
           try {
-            const sheetFlushLease =
-              await this.resourceGovernor.acquireSheetFlush({
-                estimatedBytes: chunkBytes,
-                rowCount: chunk.length,
-              });
             const flushStartedAt = Date.now();
-            try {
-              await this.#options.onMapRowsCompleted!({
-                playName: this.currentPlayName,
-                playId: this.currentExecutionScope.logical.playId,
-                runId: this.currentRunId,
-                executorToken: this.#options.executorToken,
-                tableNamespace: resolvedTableNamespace,
-                rows: chunk,
-                outputFields: datasetColumnNames.filter((field) =>
-                  shouldPersistMapCellField(field),
-                ),
-                staticPipeline: this.currentStaticPipeline ?? null,
-              });
-              this.resourceGovernor.observe({
-                sheetFlushBytes: chunkBytes,
-                sheetFlushLatencyMs: Date.now() - flushStartedAt,
-              });
-            } finally {
-              sheetFlushLease.release();
-            }
+            await this.#options.onMapRowsCompleted!({
+              playName: this.currentPlayName,
+              playId: this.currentExecutionScope.logical.playId,
+              runId: this.currentRunId,
+              executorToken: this.#options.executorToken,
+              tableNamespace: resolvedTableNamespace,
+              rows: chunk,
+              outputFields: datasetColumnNames.filter((field) =>
+                shouldPersistMapCellField(field),
+              ),
+              staticPipeline: this.currentStaticPipeline ?? null,
+            });
+            this.resourceGovernor.observe({
+              sheetFlushBytes: chunkBytes,
+              sheetFlushLatencyMs: Date.now() - flushStartedAt,
+            });
           } catch (error) {
             tripRuntimePersistenceLatch(this.persistenceLatch, error);
             throw error;
@@ -4142,10 +4168,28 @@ export class PlayContextImpl {
         const rowsToExecute = rowsToExecuteEntries.map((entry) => entry.row);
         const rowsToExecuteMemoryEstimate =
           estimateRuntimeMapRowsMemory(rowsToExecute);
-        const incrementalPersistence =
-          createIncrementalMapRowPersistence(persistMapRows);
+        const incrementalPersistence = createRuntimeMapRowPersistence(
+          persistMapRows,
+          {
+            persistCheckpoint: this.#options.onMapRowsCheckpoint
+              ? (updates) =>
+                  this.#options.onMapRowsCheckpoint!({
+                    playName: this.currentPlayName,
+                    playId: this.currentExecutionScope.logical.playId,
+                    runId: this.currentRunId,
+                    executorToken: this.#options.executorToken,
+                    tableNamespace: resolvedTableNamespace,
+                    updates,
+                    staticPipeline: this.currentStaticPipeline ?? null,
+                  })
+              : undefined,
+            onFailure: (error) =>
+              tripRuntimePersistenceLatch(this.persistenceLatch, error),
+          },
+        );
 
         this.activeMapCellMeta = new Map();
+        this.activeMapCheckpointUpdates = new Map();
         this.setMapFrame({
           mapInvocationId: mapScope.mapInvocationId,
           mapNodeId: mapScope.mapNodeId ?? null,
@@ -4207,9 +4251,11 @@ export class PlayContextImpl {
             );
             await persistMapRows(unpersistedRows);
             this.activeMapCellMeta = null;
+            this.activeMapCheckpointUpdates = null;
             throw error.cause;
           }
           this.activeMapCellMeta = null;
+          this.activeMapCheckpointUpdates = null;
           throw error;
         }
 
@@ -4238,6 +4284,7 @@ export class PlayContextImpl {
         );
         await persistMapRows(persistRows);
         this.activeMapCellMeta = null;
+        this.activeMapCheckpointUpdates = null;
 
         for (const row of mapResult.completedRows) {
           const materializedRow = this.toMaterializedOutputRow(row.data);
@@ -4669,6 +4716,7 @@ export class PlayContextImpl {
     );
 
     this.activeMapCellMeta = new Map();
+    this.activeMapCheckpointUpdates = new Map();
     const staleCompletionKeys = new Set<string>();
     const persistMapRows = async (rows: PersistableMapRow[]) => {
       if (!this.#options.onMapRowsCompleted || rows.length === 0) {
@@ -4685,38 +4733,28 @@ export class PlayContextImpl {
       const flushChunk = async () => {
         if (chunk.length === 0) return;
         try {
-          const sheetFlushLease = await this.resourceGovernor.acquireSheetFlush(
-            {
-              estimatedBytes: chunkBytes,
-              rowCount: chunk.length,
-            },
-          );
           const flushStartedAt = Date.now();
-          try {
-            const writeResult = await this.#options.onMapRowsCompleted!({
-              playName: this.currentPlayName,
-              playId: this.currentExecutionScope.logical.playId,
-              runId: this.currentRunId,
-              executorToken: this.#options.executorToken,
-              tableNamespace: resolvedTableNamespace,
-              rows: chunk,
-              outputFields: datasetColumnNames.filter((field) =>
-                shouldPersistMapCellField(field),
-              ),
-              staticPipeline: this.currentStaticPipeline ?? null,
-            });
-            if (writeResult) {
-              for (const key of writeResult.staleDroppedKeys ?? []) {
-                staleCompletionKeys.add(key);
-              }
+          const writeResult = await this.#options.onMapRowsCompleted!({
+            playName: this.currentPlayName,
+            playId: this.currentExecutionScope.logical.playId,
+            runId: this.currentRunId,
+            executorToken: this.#options.executorToken,
+            tableNamespace: resolvedTableNamespace,
+            rows: chunk,
+            outputFields: datasetColumnNames.filter((field) =>
+              shouldPersistMapCellField(field),
+            ),
+            staticPipeline: this.currentStaticPipeline ?? null,
+          });
+          if (writeResult) {
+            for (const key of writeResult.staleDroppedKeys ?? []) {
+              staleCompletionKeys.add(key);
             }
-            this.resourceGovernor.observe({
-              sheetFlushBytes: chunkBytes,
-              sheetFlushLatencyMs: Date.now() - flushStartedAt,
-            });
-          } finally {
-            sheetFlushLease.release();
           }
+          this.resourceGovernor.observe({
+            sheetFlushBytes: chunkBytes,
+            sheetFlushLatencyMs: Date.now() - flushStartedAt,
+          });
         } catch (error) {
           // Output-sheet flush failed: trip the breaker so the dispatch loops
           // stop dispatching new provider calls for the rest of this map.
@@ -4740,7 +4778,22 @@ export class PlayContextImpl {
       await flushChunk();
     };
     const incrementalPersistence = this.#options.onMapRowsCompleted
-      ? createIncrementalMapRowPersistence(persistMapRows)
+      ? createRuntimeMapRowPersistence(persistMapRows, {
+          persistCheckpoint: this.#options.onMapRowsCheckpoint
+            ? (updates) =>
+                this.#options.onMapRowsCheckpoint!({
+                  playName: this.currentPlayName,
+                  playId: this.currentExecutionScope.logical.playId,
+                  runId: this.currentRunId,
+                  executorToken: this.#options.executorToken,
+                  tableNamespace: resolvedTableNamespace,
+                  updates,
+                  staticPipeline: this.currentStaticPipeline ?? null,
+                })
+            : undefined,
+          onFailure: (error) =>
+            tripRuntimePersistenceLatch(this.persistenceLatch, error),
+        })
       : null;
 
     let mapResult: FieldMapRunResult;
@@ -4785,9 +4838,11 @@ export class PlayContextImpl {
           );
         }
         this.activeMapCellMeta = null;
+        this.activeMapCheckpointUpdates = null;
         throw error.cause;
       }
       this.activeMapCellMeta = null;
+      this.activeMapCheckpointUpdates = null;
       throw error;
     }
     const resultsByKey = new Map<string, Record<string, unknown>>();
@@ -4852,6 +4907,7 @@ export class PlayContextImpl {
       }
     }
     this.activeMapCellMeta = null;
+    this.activeMapCheckpointUpdates = null;
 
     if (staleCompletionKeys.size > 0) {
       results = results.filter((row, index) => {
@@ -5083,7 +5139,7 @@ export class PlayContextImpl {
       progressCompletedOffset?: number;
       progressFailedOffset?: number;
       progressTotalRows?: number;
-      incrementalPersistence?: IncrementalMapRowPersistence | null;
+      incrementalPersistence?: RuntimeMapRowPersistence | null;
     },
   ): Promise<FieldMapRunResult> {
     const fieldEntries = Object.entries(definition);
@@ -5135,18 +5191,17 @@ export class PlayContextImpl {
     const enqueueIncrementalPersist = (
       row: PersistableMapRow,
       onCommitted: () => void,
-    ): void => {
+    ): Promise<void> => {
       if (!incrementalPersistence) {
         onCommitted();
-        return;
+        return Promise.resolve();
       }
-      void incrementalPersistence
-        .persistRows([row])
-        .then(onCommitted)
-        .catch(() => {
-          // The final map-level flush awaits the same chain and surfaces the
-          // persistence failure loudly without holding a completed row slot open.
-        });
+      const settlement = incrementalPersistence.persistRows([row]);
+      void settlement.committed.then(onCommitted).catch(() => {
+        // The terminal map barrier awaits the writer and surfaces the same
+        // latched failure. This continuation only publishes durable progress.
+      });
+      return settlement.admitted;
     };
 
     if (completedRows > 0 || pendingRows !== totalRows) {
@@ -5512,7 +5567,11 @@ export class PlayContextImpl {
               error: formattedError,
               dataPatch: {},
             });
-            enqueueIncrementalPersist(failedRow, () => {
+            await enqueueIncrementalPersist(failedRow, () => {
+              this.clearActiveMapCheckpointUpdate(
+                rowKey,
+                normalizedTableNamespace,
+              );
               updateMapFrameProgress({ failedRowKey: rowKey });
             });
             return FAILED_ROW;
@@ -5591,7 +5650,8 @@ export class PlayContextImpl {
         });
         retainedRowsMemoryTracker.track(completedRow);
         completedRowsToPersist.push(completedRow);
-        enqueueIncrementalPersist(completedRow, () => {
+        await enqueueIncrementalPersist(completedRow, () => {
+          this.clearActiveMapCheckpointUpdate(rowKey, normalizedTableNamespace);
           updateMapFrameProgress({ completedRowKey: rowKey });
         });
         return publicRow;
@@ -5739,8 +5799,18 @@ export class PlayContextImpl {
           ]),
         ).values(),
       ];
-      this.pendingRowEventBoundaries = [];
+      const checkpointUpdates = [
+        ...(this.activeMapCheckpointUpdates?.values() ?? []),
+      ].filter(
+        (update) =>
+          (update.tableNamespace ?? normalizedTableNamespace) ===
+          normalizedTableNamespace,
+      );
+      // A suspension is not observable until both terminal rows and partial
+      // completed-cell patches cross the same writer's durability barrier.
+      await incrementalPersistence?.checkpoint(checkpointUpdates);
       await incrementalPersistence?.flush();
+      this.pendingRowEventBoundaries = [];
       this.#options.onBatchComplete?.(this.checkpoint);
       throw new PlayExecutionSuspendedError({
         kind: 'integration_event_batch',

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -604,6 +604,20 @@ export interface ContextOptions {
     updated: number;
     staleDroppedKeys?: string[];
   }>;
+  /**
+   * Persists coalesced, non-terminal row patches before a map suspends.
+   * Terminal rows use onMapRowsCompleted; this narrow seam only preserves
+   * cells that completed before an integration-event boundary.
+   */
+  onMapRowsCheckpoint?: (input: {
+    playName?: string;
+    playId?: string;
+    runId: string;
+    executorToken?: string;
+    tableNamespace: string;
+    updates: PlayRowUpdate[];
+    staticPipeline?: PlayStaticPipeline | null;
+  }) => Promise<void>;
   playId?: string;
   runId?: string;
   /** Physical executor run that owns leases for an inline child context. */
@@ -1030,6 +1044,7 @@ export interface PlayRowUpdate {
   attemptOwnerRunId?: string | null;
   attemptSeq?: number | null;
   attemptExpiresAt?: string | null;
+  writeVersion?: number | null;
   status?: 'running' | 'completed' | 'failed';
   stage?: string | null;
   provider?: string | null;