deepline 0.1.85 → 0.1.88

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -600,7 +600,7 @@ type WorkerCtxCallbacks = {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
     forceFlush?: boolean;
-  }) => void;
+  }) => void | Promise<void>;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
   onToolCalled?: (toolId: string, at?: number) => void;
@@ -1592,6 +1592,7 @@ type WorkerToolBatchRequest = {
 };
 
 const WORKER_TOOL_BATCH_GRACE_MS = 250;
+const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1599,6 +1600,10 @@ const WORKER_TOOL_BATCH_GRACE_MS = 250;
 const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
 const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
 
+function sleepWorkerMs(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 function stepProgramColumnName(parentField: string, stepId: string): string {
   return sqlSafePlayColumnName(`${parentField}.${stepId}`);
 }
@@ -3542,8 +3547,11 @@ function createMinimalWorkerCtx(
       softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
     });
     const outputFields = fieldEntries.map(([field]) => field);
-    const updateMapProgress = (progress: LiveNodeProgressSnapshot) => {
-      callbacks?.onNodeProgress?.({
+    const updateMapProgress = (
+      progress: LiveNodeProgressSnapshot,
+      options?: { forceFlush?: boolean },
+    ): void | Promise<void> => {
+      return callbacks?.onNodeProgress?.({
         nodeId: mapNodeId,
         progress: {
           artifactTableNamespace: name,
@@ -3551,19 +3559,62 @@ function createMinimalWorkerCtx(
           ...progress,
           updatedAt: progress.updatedAt ?? nowMs(),
         },
-        forceFlush: true,
+        forceFlush: options?.forceFlush ?? true,
       });
     };
     const formatMapProgressMessage = (completed: number, total?: number) =>
       typeof total === 'number' && Number.isFinite(total) && total > 0
         ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
         : `${completed.toLocaleString()} rows processed`;
+    const formatMapPreparingMessage = (total?: number) =>
+      typeof total === 'number' && Number.isFinite(total) && total > 0
+        ? `Preparing ${total.toLocaleString()} rows`
+        : 'Preparing rows';
+    const formatMapQueuedMessage = (input: {
+      completed: number;
+      queued: number;
+      total?: number;
+    }) => {
+      const completed = Math.max(0, input.completed);
+      const queued = Math.max(0, input.queued);
+      if (completed > 0 && queued > 0) {
+        return `${completed.toLocaleString()} already satisfied, ${queued.toLocaleString()} queued`;
+      }
+      if (queued > 0) {
+        return `${queued.toLocaleString()} rows queued`;
+      }
+      return formatMapProgressMessage(completed, input.total);
+    };
+    const formatMapProcessingMessage = (rowsToExecute: number) =>
+      rowsToExecute > 0
+        ? `Processing ${rowsToExecute.toLocaleString()} rows`
+        : null;
+    const formatMapExecutionHeartbeatMessage = (input: {
+      rowsToExecute: number;
+      startedRows: number;
+      activeRows: number;
+      completedRows: number;
+    }) => {
+      const rowsToExecute = Math.max(0, input.rowsToExecute);
+      const startedRows = Math.max(0, input.startedRows);
+      const activeRows = Math.max(0, input.activeRows);
+      const completedRows = Math.max(0, input.completedRows);
+      const waitingRows = Math.max(0, rowsToExecute - startedRows);
+      const parts = [
+        activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
+        waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
+        completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
+      ].filter((part): part is string => Boolean(part));
+      const base =
+        formatMapProcessingMessage(rowsToExecute) ?? 'Processing rows';
+      return parts.length > 0 ? `${base} (${parts.join(', ')})` : base;
+    };
     callbacks?.onMapStarted?.(mapNodeId, mapStartedAt);
-    updateMapProgress({
+    await updateMapProgress({
       completed: 0,
       total: rowCountHint ?? undefined,
       startedAt: mapStartedAt,
-      message: formatMapProgressMessage(0, rowCountHint ?? undefined),
+      message: formatMapPreparingMessage(rowCountHint ?? undefined),
     });
     const explicitRowKeysSeen =
       opts?.key === undefined ? null : new Map<string, number>();
@@ -3637,6 +3688,8 @@ function createMinimalWorkerCtx(
       }
     };
 
+    let totalRowsWritten = 0;
+
     const processChunk = async (
       chunkRows: T[],
       chunkStart: number,
@@ -3681,17 +3734,20 @@ function createMinimalWorkerCtx(
           completedRows: prepared.completedRows.length,
         },
       });
-      updateMapProgress({
-        completed: prepared.completedRows.length,
-        total: chunkRows.length,
+      const progressTotalRows = rowCountHint ?? chunkRows.length;
+      const preparedCompletedRows = Math.min(
+        progressTotalRows,
+        totalRowsWritten + prepared.completedRows.length,
+      );
+      await updateMapProgress({
+        completed: preparedCompletedRows,
+        total: progressTotalRows,
         startedAt: mapStartedAt,
-        message:
-          prepared.pendingRows.length > 0
-            ? `${prepared.pendingRows.length.toLocaleString()} rows queued`
-            : formatMapProgressMessage(
-                prepared.completedRows.length,
-                chunkRows.length,
-              ),
+        message: formatMapQueuedMessage({
+          completed: preparedCompletedRows,
+          queued: prepared.pendingRows.length,
+          total: progressTotalRows,
+        }),
       });
       const pendingKeys = new Set<string>();
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -3735,38 +3791,83 @@ function createMinimalWorkerCtx(
           new Set(chunkEntries.map((entry) => entry.rowKey)).size,
       );
       const rowsToExecute = uniqueRowsToExecuteEntries.map(({ row }) => row);
+      const processingMessage = formatMapProcessingMessage(
+        rowsToExecute.length,
+      );
+      if (processingMessage) {
+        await updateMapProgress({
+          completed: preparedCompletedRows,
+          total: progressTotalRows,
+          startedAt: mapStartedAt,
+          message: processingMessage,
+        });
+      }
       const rowsInserted = prepared.inserted + missingPreparedRows.length;
       const rowsSkipped = Math.max(
         0,
         prepared.skipped - missingPreparedRows.length,
       );
-      let settledToolRequests = 0;
-      let lastToolProgressAt = 0;
-      const reportSettledToolRequests = (count: number) => {
-        if (count <= 0) return;
-        settledToolRequests += count;
+      let completedExecutedRows = 0;
+      let startedExecutedRows = 0;
+      let activeExecutedRows = 0;
+      let lastChunkProgressAt = 0;
+      let lastExecutionHeartbeatAt = nowMs();
+      const completedRowsForProgress = () =>
+        Math.min(
+          progressTotalRows,
+          totalRowsWritten +
+            prepared.completedRows.length +
+            completedExecutedRows,
+        );
+      const reportExecutionHeartbeat = (force = false) => {
         const now = nowMs();
-        const estimatedCompleted = Math.min(
-          chunkRows.length,
-          prepared.completedRows.length + settledToolRequests,
+        if (
+          !force &&
+          now - lastExecutionHeartbeatAt < MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+        ) {
+          return;
+        }
+        lastExecutionHeartbeatAt = now;
+        void updateMapProgress(
+          {
+            completed: completedRowsForProgress(),
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapExecutionHeartbeatMessage({
+              rowsToExecute: rowsToExecute.length,
+              startedRows: startedExecutedRows,
+              activeRows: activeExecutedRows,
+              completedRows: completedExecutedRows,
+            }),
+          },
+          { forceFlush: force },
         );
-        const isTerminalEstimate = estimatedCompleted >= chunkRows.length;
+      };
+      const reportChunkProgress = (force = false) => {
+        const now = nowMs();
+        const completed = completedRowsForProgress();
+        const isTerminalEstimate = completed >= progressTotalRows;
         if (
+          !force &&
           !isTerminalEstimate &&
-          now - lastToolProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
+          now - lastChunkProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
         ) {
           return;
         }
-        lastToolProgressAt = now;
-        updateMapProgress({
-          completed: estimatedCompleted,
-          total: chunkRows.length,
-          startedAt: mapStartedAt,
-          message: formatMapProgressMessage(
-            estimatedCompleted,
-            chunkRows.length,
-          ),
-        });
+        lastChunkProgressAt = now;
+        void updateMapProgress(
+          {
+            completed,
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapProgressMessage(completed, progressTotalRows),
+          },
+          { forceFlush: force },
+        );
+      };
+      const reportSettledToolRequests = (count: number) => {
+        if (count <= 0) return;
+        reportChunkProgress(false);
       };
       // Row concurrency comes from the Governor: an explicit map concurrency is
       // clamped to the policy row-max, otherwise the policy default. Each row
@@ -3812,7 +3913,12 @@ function createMinimalWorkerCtx(
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              let rowMarkedActive = false;
               try {
+                startedExecutedRows += 1;
+                activeExecutedRows += 1;
+                rowMarkedActive = true;
+                reportExecutionHeartbeat(false);
                 const entry = uniqueRowsToExecuteEntries[myIndex]!;
                 const pendingRow = pendingRowsByKey.get(entry.rowKey);
                 const row = pendingRow
@@ -3985,7 +4091,13 @@ function createMinimalWorkerCtx(
                     ? cellMetaPatch
                     : undefined;
                 executedRows[myIndex] = enriched as T & Record<string, unknown>;
+                completedExecutedRows += 1;
+                reportChunkProgress(false);
               } finally {
+                if (rowMarkedActive) {
+                  activeExecutedRows = Math.max(0, activeExecutedRows - 1);
+                  reportExecutionHeartbeat(false);
+                }
                 rowSlot.release();
               }
             }
@@ -4031,7 +4143,27 @@ function createMinimalWorkerCtx(
         });
       };
       const workersStartedAt = nowMs();
-      const workerResults = await Promise.allSettled(workers);
+      // Track completion with a boolean flag rather than narrowing a
+      // closure-assigned `| null` variable: TypeScript's control-flow analysis
+      // does not see the assignment inside `.then(...)`, so a
+      // `while (results === null)` loop would narrow it to `never` afterwards.
+      let workerSettled = false;
+      const workerResultsPromise = Promise.allSettled(workers).then(
+        (results) => {
+          workerSettled = true;
+          return results;
+        },
+      );
+      while (!workerSettled) {
+        await Promise.race([
+          workerResultsPromise,
+          sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
+        ]);
+        if (!workerSettled) {
+          reportExecutionHeartbeat(false);
+        }
+      }
+      const workerResults = await workerResultsPromise;
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.execute_workers',
@@ -4198,7 +4330,7 @@ function createMinimalWorkerCtx(
         `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
       const completedAt = nowMs();
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
-      updateMapProgress({
+      void updateMapProgress({
         completed: totalRowsWritten,
         total: totalRowsWritten,
         completedAt,
@@ -4249,7 +4381,6 @@ function createMinimalWorkerCtx(
       });
     };
 
-    let totalRowsWritten = 0;
     let chunkIndex = 0;
     let chunkStart = 0;
     for await (const chunkRows of iterDatasetChunks(inputRows, rowsPerChunk)) {
@@ -4263,7 +4394,7 @@ function createMinimalWorkerCtx(
       totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
       totalRowsInserted += chunkResult.rowsInserted;
       totalRowsSkipped += chunkResult.rowsSkipped;
-      updateMapProgress({
+      await updateMapProgress({
         completed: totalRowsWritten,
         total: rowCountHint ?? undefined,
         message: formatMapProgressMessage(
@@ -5374,6 +5505,10 @@ async function executeRunRequest(
   ];
   let lastLedgerFlushAt = startedAt;
   let ledgerFlushInFlight: Promise<void> = Promise.resolve();
+  let ledgerFlushQueueDepth = 0;
+  let lastCoordinatorProgressPublishAt = 0;
+  let coordinatorProgressPublishInFlight: Promise<void> = Promise.resolve();
+  let coordinatorProgressPublishQueueDepth = 0;
 
   const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
@@ -5496,6 +5631,36 @@ async function executeRunRequest(
     });
   };
 
+  const flushCoordinatorProgressEvent = (force: boolean): Promise<void> => {
+    const now = nowMs();
+    if (
+      !force &&
+      now - lastCoordinatorProgressPublishAt <
+        MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+    ) {
+      return Promise.resolve();
+    }
+    if (!force && coordinatorProgressPublishQueueDepth > 0) {
+      return Promise.resolve();
+    }
+    lastCoordinatorProgressPublishAt = now;
+    coordinatorProgressPublishQueueDepth += 1;
+    coordinatorProgressPublishInFlight = coordinatorProgressPublishInFlight
+      .catch(() => undefined)
+      .then(async () => {
+        try {
+          await publishCoordinatorProgressEvent(now);
+        } finally {
+          coordinatorProgressPublishQueueDepth = Math.max(
+            0,
+            coordinatorProgressPublishQueueDepth - 1,
+          );
+        }
+      })
+      .catch(() => undefined);
+    return force ? coordinatorProgressPublishInFlight : Promise.resolve();
+  };
+
   const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
     updateStepProgress({
       nodeId: event.nodeId,
@@ -5595,15 +5760,19 @@ async function executeRunRequest(
     return events;
   };
 
-  const flushLedgerEvents = (force: boolean): void => {
-    if (!options?.persistResultDatasets) return;
+  const flushLedgerEvents = (force: boolean): Promise<void> => {
+    if (!options?.persistResultDatasets) return Promise.resolve();
     const now = nowMs();
     if (!force && now - lastLedgerFlushAt < RUN_LEDGER_FLUSH_INTERVAL_MS) {
-      return;
+      return Promise.resolve();
+    }
+    if (!force && ledgerFlushQueueDepth > 0) {
+      return Promise.resolve();
     }
     const events = drainPendingLedgerEvents(now);
-    if (events.length === 0) return;
+    if (events.length === 0) return Promise.resolve();
     lastLedgerFlushAt = now;
+    ledgerFlushQueueDepth += 1;
     ledgerFlushInFlight = ledgerFlushInFlight
       .catch(() => undefined)
       .then(async () => {
@@ -5616,10 +5785,12 @@ async function executeRunRequest(
         } catch {
           pendingLedgerEvents = [...events, ...pendingLedgerEvents];
           throw new Error('runtime run-ledger append failed');
+        } finally {
+          ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
         }
-        await publishCoordinatorProgressEvent(now).catch(() => undefined);
       })
       .catch(() => undefined);
+    return force ? ledgerFlushInFlight : Promise.resolve();
   };
 
   const flushTerminalLedgerEvents = async (
@@ -5661,7 +5832,12 @@ async function executeRunRequest(
   const workerCallbacks: WorkerCtxCallbacks = {
     onNodeProgress: (input) => {
       updateStepProgress(input);
-      flushLedgerEvents(Boolean(input.forceFlush));
+      const force = Boolean(input.forceFlush);
+      const ledgerFlush = flushLedgerEvents(force);
+      const progressFlush = flushCoordinatorProgressEvent(force);
+      return force
+        ? Promise.all([ledgerFlush, progressFlush]).then(() => undefined)
+        : Promise.resolve();
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),