deepline 0.1.83 → 0.1.88

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -163,11 +163,19 @@ import {
 } from './runtime/tool-http-errors';
 import {
   StepProgramDatasetBuilder,
+  type StepProgramDatasetColumnInput,
   type StepProgramDatasetOptions,
 } from '../../../shared_libs/play-runtime/step-program-dataset-builder';
 import {
   DEEPLINE_CELL_META_FIELD,
+  normalizeCellStalenessPolicy,
+  previousCellFromValue,
+  resolveCompletedCellStalenessMeta,
   shouldRecomputeCell,
+  type AuthoredCellStalenessPolicyByField,
+  type AuthoredStaleAfterSeconds,
+  type CellStalenessPolicyByField,
+  type PreviousCell,
 } from '../../../shared_libs/play-runtime/cell-staleness';
 
 // The play's default export. The bundler injects this — see bundle-play-file.ts.
@@ -592,7 +600,7 @@ type WorkerCtxCallbacks = {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
     forceFlush?: boolean;
-  }) => void;
+  }) => void | Promise<void>;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
   onToolCalled?: (toolId: string, at?: number) => void;
@@ -717,10 +725,7 @@ function publicCsvStorageRow<T extends Record<string, unknown>>(row: T): T {
     storageRow[fieldName] =
       'value' in descriptor ? descriptor.value : publicRow[fieldName];
   }
-  for (const runtimeField of [
-    '__deeplineRowKey',
-    '__deeplineCellMetaPatch',
-  ]) {
+  for (const runtimeField of ['__deeplineRowKey', '__deeplineCellMetaPatch']) {
     if (Object.prototype.hasOwnProperty.call(row, runtimeField)) {
       storageRow[runtimeField] = row[runtimeField];
     }
@@ -1587,6 +1592,7 @@ type WorkerToolBatchRequest = {
 };
 
 const WORKER_TOOL_BATCH_GRACE_MS = 250;
+const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1594,6 +1600,10 @@ const WORKER_TOOL_BATCH_GRACE_MS = 250;
 const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
 const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
 
+function sleepWorkerMs(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 function stepProgramColumnName(parentField: string, stepId: string): string {
   return sqlSafePlayColumnName(`${parentField}.${stepId}`);
 }
@@ -1973,6 +1983,7 @@ type WorkerStepResolver = (
   row: Record<string, unknown>,
   ctx: unknown,
   index: number,
+  previousCell?: PreviousCell,
 ) => Promise<unknown> | unknown;
 
 type WorkerConditionalStepResolver = {
@@ -1987,7 +1998,7 @@ type WorkerConditionalStepResolver = {
 
 type WorkerStepProgramStep = {
   name: string;
-  staleAfterSeconds?: number;
+  staleAfterSeconds?: AuthoredStaleAfterSeconds;
   resolver:
     | WorkerStepResolver
     | WorkerConditionalStepResolver
@@ -2042,6 +2053,7 @@ async function executeWorkerStepResolver(
   row: Record<string, unknown>,
   ctx: unknown,
   index: number,
+  previousCell?: PreviousCell,
   recorder?: {
     parentField: string;
     path: string[];
@@ -2051,7 +2063,14 @@ async function executeWorkerStepResolver(
   if (isWorkerConditionalStepResolver(resolver)) {
     const shouldRun = await resolver.when(row, index);
     return shouldRun
-      ? await executeWorkerStepResolver(resolver.run, row, ctx, index, recorder)
+      ? await executeWorkerStepResolver(
+          resolver.run,
+          row,
+          ctx,
+          index,
+          previousCell,
+          recorder,
+        )
       : {
           value: resolver.elseValue ?? null,
           status: 'skipped',
@@ -2069,7 +2088,14 @@ async function executeWorkerStepResolver(
     };
   }
   if (typeof resolver === 'function') {
-    return { value: await (resolver as WorkerStepResolver)(row, ctx, index) };
+    return {
+      value: await (resolver as WorkerStepResolver)(
+        row,
+        ctx,
+        index,
+        previousCell,
+      ),
+    };
   }
   return { value: resolver };
 }
@@ -2095,6 +2121,7 @@ async function executeWorkerStepProgram(
         currentRow,
         ctx,
         index,
+        undefined,
         recorder
           ? {
               ...recorder,
@@ -3005,7 +3032,7 @@ async function prepareMapRows(input: {
   tableNamespace: string;
   rows: Record<string, unknown>[];
   outputFields: string[];
-  cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
+  cellPolicies?: CellStalenessPolicyByField;
 }): Promise<{
   inserted: number;
   skipped: number;
@@ -3161,18 +3188,21 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
         if (!coordinatorUrl) {
           throw new Error('Coordinator rate acquire is unavailable.');
         }
-        const res = await fetch(`${coordinatorUrl.replace(/\/$/, '')}/rate-acquire`, {
-          method: 'POST',
-          headers: {
-            'x-deepline-request-id': makeRequestId(),
-            ...coordinatorRequestHeaders({
-              runId: req.runId,
-              contentType: 'application/json',
-              internalToken: req.coordinatorInternalToken,
-            }),
+        const res = await fetch(
+          `${coordinatorUrl.replace(/\/$/, '')}/rate-acquire`,
+          {
+            method: 'POST',
+            headers: {
+              'x-deepline-request-id': makeRequestId(),
+              ...coordinatorRequestHeaders({
+                runId: req.runId,
+                contentType: 'application/json',
+                internalToken: req.coordinatorInternalToken,
+              }),
+            },
+            body: JSON.stringify(input),
           },
-          body: JSON.stringify(input),
-        });
+        );
         if (!res.ok) {
           const text = await res.text().catch(() => '');
           throw new Error(
@@ -3494,7 +3524,8 @@ function createMinimalWorkerCtx(
           index: number,
         ) => Promise<unknown> | unknown)
     >,
-    cellPolicies?: Record<string, { staleAfterSeconds?: number }>,
+    cellPolicies?: CellStalenessPolicyByField,
+    authoredCellPolicies?: AuthoredCellStalenessPolicyByField,
     opts?: WorkerMapOptions,
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
@@ -3516,8 +3547,11 @@ function createMinimalWorkerCtx(
       softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
     });
     const outputFields = fieldEntries.map(([field]) => field);
-    const updateMapProgress = (progress: LiveNodeProgressSnapshot) => {
-      callbacks?.onNodeProgress?.({
+    const updateMapProgress = (
+      progress: LiveNodeProgressSnapshot,
+      options?: { forceFlush?: boolean },
+    ): void | Promise<void> => {
+      return callbacks?.onNodeProgress?.({
         nodeId: mapNodeId,
         progress: {
           artifactTableNamespace: name,
@@ -3525,19 +3559,62 @@ function createMinimalWorkerCtx(
           ...progress,
           updatedAt: progress.updatedAt ?? nowMs(),
         },
-        forceFlush: true,
+        forceFlush: options?.forceFlush ?? true,
       });
     };
     const formatMapProgressMessage = (completed: number, total?: number) =>
       typeof total === 'number' && Number.isFinite(total) && total > 0
         ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
         : `${completed.toLocaleString()} rows processed`;
+    const formatMapPreparingMessage = (total?: number) =>
+      typeof total === 'number' && Number.isFinite(total) && total > 0
+        ? `Preparing ${total.toLocaleString()} rows`
+        : 'Preparing rows';
+    const formatMapQueuedMessage = (input: {
+      completed: number;
+      queued: number;
+      total?: number;
+    }) => {
+      const completed = Math.max(0, input.completed);
+      const queued = Math.max(0, input.queued);
+      if (completed > 0 && queued > 0) {
+        return `${completed.toLocaleString()} already satisfied, ${queued.toLocaleString()} queued`;
+      }
+      if (queued > 0) {
+        return `${queued.toLocaleString()} rows queued`;
+      }
+      return formatMapProgressMessage(completed, input.total);
+    };
+    const formatMapProcessingMessage = (rowsToExecute: number) =>
+      rowsToExecute > 0
+        ? `Processing ${rowsToExecute.toLocaleString()} rows`
+        : null;
+    const formatMapExecutionHeartbeatMessage = (input: {
+      rowsToExecute: number;
+      startedRows: number;
+      activeRows: number;
+      completedRows: number;
+    }) => {
+      const rowsToExecute = Math.max(0, input.rowsToExecute);
+      const startedRows = Math.max(0, input.startedRows);
+      const activeRows = Math.max(0, input.activeRows);
+      const completedRows = Math.max(0, input.completedRows);
+      const waitingRows = Math.max(0, rowsToExecute - startedRows);
+      const parts = [
+        activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
+        waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
+        completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
+      ].filter((part): part is string => Boolean(part));
+      const base =
+        formatMapProcessingMessage(rowsToExecute) ?? 'Processing rows';
+      return parts.length > 0 ? `${base} (${parts.join(', ')})` : base;
+    };
     callbacks?.onMapStarted?.(mapNodeId, mapStartedAt);
-    updateMapProgress({
+    await updateMapProgress({
       completed: 0,
       total: rowCountHint ?? undefined,
       startedAt: mapStartedAt,
-      message: formatMapProgressMessage(0, rowCountHint ?? undefined),
+      message: formatMapPreparingMessage(rowCountHint ?? undefined),
     });
     const explicitRowKeysSeen =
       opts?.key === undefined ? null : new Map<string, number>();
@@ -3611,6 +3688,8 @@ function createMinimalWorkerCtx(
       }
     };
 
+    let totalRowsWritten = 0;
+
     const processChunk = async (
       chunkRows: T[],
       chunkStart: number,
@@ -3655,17 +3734,20 @@ function createMinimalWorkerCtx(
           completedRows: prepared.completedRows.length,
         },
       });
-      updateMapProgress({
-        completed: prepared.completedRows.length,
-        total: chunkRows.length,
+      const progressTotalRows = rowCountHint ?? chunkRows.length;
+      const preparedCompletedRows = Math.min(
+        progressTotalRows,
+        totalRowsWritten + prepared.completedRows.length,
+      );
+      await updateMapProgress({
+        completed: preparedCompletedRows,
+        total: progressTotalRows,
         startedAt: mapStartedAt,
-        message:
-          prepared.pendingRows.length > 0
-            ? `${prepared.pendingRows.length.toLocaleString()} rows queued`
-            : formatMapProgressMessage(
-                prepared.completedRows.length,
-                chunkRows.length,
-              ),
+        message: formatMapQueuedMessage({
+          completed: preparedCompletedRows,
+          queued: prepared.pendingRows.length,
+          total: progressTotalRows,
+        }),
       });
       const pendingKeys = new Set<string>();
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -3709,38 +3791,83 @@ function createMinimalWorkerCtx(
           new Set(chunkEntries.map((entry) => entry.rowKey)).size,
       );
       const rowsToExecute = uniqueRowsToExecuteEntries.map(({ row }) => row);
+      const processingMessage = formatMapProcessingMessage(
+        rowsToExecute.length,
+      );
+      if (processingMessage) {
+        await updateMapProgress({
+          completed: preparedCompletedRows,
+          total: progressTotalRows,
+          startedAt: mapStartedAt,
+          message: processingMessage,
+        });
+      }
       const rowsInserted = prepared.inserted + missingPreparedRows.length;
       const rowsSkipped = Math.max(
         0,
         prepared.skipped - missingPreparedRows.length,
       );
-      let settledToolRequests = 0;
-      let lastToolProgressAt = 0;
-      const reportSettledToolRequests = (count: number) => {
-        if (count <= 0) return;
-        settledToolRequests += count;
+      let completedExecutedRows = 0;
+      let startedExecutedRows = 0;
+      let activeExecutedRows = 0;
+      let lastChunkProgressAt = 0;
+      let lastExecutionHeartbeatAt = nowMs();
+      const completedRowsForProgress = () =>
+        Math.min(
+          progressTotalRows,
+          totalRowsWritten +
+            prepared.completedRows.length +
+            completedExecutedRows,
+        );
+      const reportExecutionHeartbeat = (force = false) => {
         const now = nowMs();
-        const estimatedCompleted = Math.min(
-          chunkRows.length,
-          prepared.completedRows.length + settledToolRequests,
+        if (
+          !force &&
+          now - lastExecutionHeartbeatAt < MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+        ) {
+          return;
+        }
+        lastExecutionHeartbeatAt = now;
+        void updateMapProgress(
+          {
+            completed: completedRowsForProgress(),
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapExecutionHeartbeatMessage({
+              rowsToExecute: rowsToExecute.length,
+              startedRows: startedExecutedRows,
+              activeRows: activeExecutedRows,
+              completedRows: completedExecutedRows,
+            }),
+          },
+          { forceFlush: force },
         );
-        const isTerminalEstimate = estimatedCompleted >= chunkRows.length;
+      };
+      const reportChunkProgress = (force = false) => {
+        const now = nowMs();
+        const completed = completedRowsForProgress();
+        const isTerminalEstimate = completed >= progressTotalRows;
         if (
+          !force &&
           !isTerminalEstimate &&
-          now - lastToolProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
+          now - lastChunkProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
         ) {
           return;
         }
-        lastToolProgressAt = now;
-        updateMapProgress({
-          completed: estimatedCompleted,
-          total: chunkRows.length,
-          startedAt: mapStartedAt,
-          message: formatMapProgressMessage(
-            estimatedCompleted,
-            chunkRows.length,
-          ),
-        });
+        lastChunkProgressAt = now;
+        void updateMapProgress(
+          {
+            completed,
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapProgressMessage(completed, progressTotalRows),
+          },
+          { forceFlush: force },
+        );
+      };
+      const reportSettledToolRequests = (count: number) => {
+        if (count <= 0) return;
+        reportChunkProgress(false);
       };
       // Row concurrency comes from the Governor: an explicit map concurrency is
       // clamped to the policy row-max, otherwise the policy default. Each row
@@ -3760,6 +3887,8 @@ function createMinimalWorkerCtx(
               reused?: boolean;
               runId?: string;
               completedAt?: number;
+              staleAt?: number | null;
+              staleAfterSeconds?: number | null;
             }
           >
         | undefined
@@ -3784,12 +3913,25 @@ function createMinimalWorkerCtx(
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              let rowMarkedActive = false;
               try {
+                startedExecutedRows += 1;
+                activeExecutedRows += 1;
+                rowMarkedActive = true;
+                reportExecutionHeartbeat(false);
                 const entry = uniqueRowsToExecuteEntries[myIndex]!;
-                const row = pendingRowsByKey.has(entry.rowKey)
+                const pendingRow = pendingRowsByKey.get(entry.rowKey);
+                const row = pendingRow
                   ? ({
                       ...entry.row,
-                      ...publicCsvInputRow(pendingRowsByKey.get(entry.rowKey)!),
+                      ...publicCsvInputRow(pendingRow),
+                      ...(pendingRow[DEEPLINE_CELL_META_FIELD] &&
+                      typeof pendingRow[DEEPLINE_CELL_META_FIELD] === 'object'
+                        ? {
+                            [DEEPLINE_CELL_META_FIELD]:
+                              pendingRow[DEEPLINE_CELL_META_FIELD],
+                          }
+                        : {}),
                     } as T & Record<string, unknown>)
                   : entry.row;
                 const absoluteIndex = entry.absoluteIndex;
@@ -3804,6 +3946,8 @@ function createMinimalWorkerCtx(
                     reused?: boolean;
                     runId?: string;
                     completedAt?: number;
+                    staleAt?: number | null;
+                    staleAfterSeconds?: number | null;
                   }
                 > = {};
                 const waterfallOutputs: RecordedWaterfallOutput[] = [];
@@ -3857,10 +4001,25 @@ function createMinimalWorkerCtx(
                         ? (rawCellMeta as {
                             status?: string;
                             completedAt?: number;
+                            staleAt?: number | null;
+                            staleAfterSeconds?: number | null;
                           })
                         : null,
                     policy: cellPolicies?.[key],
                   });
+                  const previousCell = previousCellFromValue({
+                    hasValue: isCompletedWorkerFieldValue(enriched[key]),
+                    value: enriched[key],
+                    meta:
+                      rawCellMeta && typeof rawCellMeta === 'object'
+                        ? (rawCellMeta as {
+                            status?: string;
+                            completedAt?: number;
+                            staleAt?: number | null;
+                            staleAfterSeconds?: number | null;
+                          })
+                        : null,
+                  });
                   if (reuseDecision.action === 'reuse') {
                     cellMetaPatch[key] = {
                       status: 'cached',
@@ -3875,6 +4034,7 @@ function createMinimalWorkerCtx(
                     enriched,
                     rowCtx,
                     absoluteIndex,
+                    previousCell,
                     isWorkerStepProgram(value)
                       ? {
                           parentField: key,
@@ -3892,11 +4052,18 @@ function createMinimalWorkerCtx(
                       runId: req.runId,
                     };
                   } else {
+                    const completedAt = nowMs();
+                    const stalenessMeta = resolveCompletedCellStalenessMeta({
+                      policy: authoredCellPolicies?.[key],
+                      value: resolved.value,
+                      completedAt,
+                    });
                     cellMetaPatch[key] = {
                       status: 'completed',
                       stage: key,
                       runId: req.runId,
-                      completedAt: nowMs(),
+                      completedAt,
+                      ...stalenessMeta,
                     };
                   }
                 }
@@ -3924,7 +4091,13 @@ function createMinimalWorkerCtx(
                     ? cellMetaPatch
                     : undefined;
                 executedRows[myIndex] = enriched as T & Record<string, unknown>;
+                completedExecutedRows += 1;
+                reportChunkProgress(false);
               } finally {
+                if (rowMarkedActive) {
+                  activeExecutedRows = Math.max(0, activeExecutedRows - 1);
+                  reportExecutionHeartbeat(false);
+                }
                 rowSlot.release();
               }
             }
@@ -3970,7 +4143,27 @@ function createMinimalWorkerCtx(
         });
       };
       const workersStartedAt = nowMs();
-      const workerResults = await Promise.allSettled(workers);
+      // Track completion with a boolean flag rather than narrowing a
+      // closure-assigned `| null` variable: TypeScript's control-flow analysis
+      // does not see the assignment inside `.then(...)`, so a
+      // `while (results === null)` loop would narrow it to `never` afterwards.
+      let workerSettled = false;
+      const workerResultsPromise = Promise.allSettled(workers).then(
+        (results) => {
+          workerSettled = true;
+          return results;
+        },
+      );
+      while (!workerSettled) {
+        await Promise.race([
+          workerResultsPromise,
+          sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
+        ]);
+        if (!workerSettled) {
+          reportExecutionHeartbeat(false);
+        }
+      }
+      const workerResults = await workerResultsPromise;
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.execute_workers',
@@ -4137,7 +4330,7 @@ function createMinimalWorkerCtx(
         `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
       const completedAt = nowMs();
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
-      updateMapProgress({
+      void updateMapProgress({
         completed: totalRowsWritten,
         total: totalRowsWritten,
         completedAt,
@@ -4188,7 +4381,6 @@ function createMinimalWorkerCtx(
       });
     };
 
-    let totalRowsWritten = 0;
     let chunkIndex = 0;
     let chunkStart = 0;
     for await (const chunkRows of iterDatasetChunks(inputRows, rowsPerChunk)) {
@@ -4202,7 +4394,7 @@ function createMinimalWorkerCtx(
       totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
       totalRowsInserted += chunkResult.rowsInserted;
       totalRowsSkipped += chunkResult.rowsSkipped;
-      updateMapProgress({
+      await updateMapProgress({
         completed: totalRowsWritten,
         total: rowCountHint ?? undefined,
         message: formatMapProgressMessage(
@@ -4266,14 +4458,31 @@ function createMinimalWorkerCtx(
             program.steps.map((step) => [step.name, step.resolver]),
           );
           const cellPolicies = Object.fromEntries(
+            program.steps.map((step) => [
+              step.name,
+              step.staleAfterSeconds === undefined
+                ? {}
+                : normalizeCellStalenessPolicy({
+                    staleAfterSeconds: step.staleAfterSeconds,
+                  }),
+            ]),
+          ) as CellStalenessPolicyByField;
+          const authoredCellPolicies = Object.fromEntries(
             program.steps.map((step) => [
               step.name,
               step.staleAfterSeconds === undefined
                 ? {}
                 : { staleAfterSeconds: step.staleAfterSeconds },
             ]),
+          ) as AuthoredCellStalenessPolicyByField;
+          return runMap(
+            this.name,
+            this.rows,
+            fields,
+            cellPolicies,
+            authoredCellPolicies,
+            opts,
           );
-          return runMap(this.name, this.rows, fields, cellPolicies, opts);
         },
         {
           emptyColumnName:
@@ -4288,7 +4497,9 @@ function createMinimalWorkerCtx(
 
     withColumn(
       name: string,
-      resolver: WorkerStepProgramStep['resolver'],
+      resolver: StepProgramDatasetColumnInput<
+        WorkerStepProgramStep['resolver']
+      >,
       options?: StepProgramDatasetOptions,
     ): this {
       this.builder.withColumn(name, resolver, options);
@@ -4492,14 +4703,31 @@ function createMinimalWorkerCtx(
           fieldsDef.steps.map((step) => [step.name, step.resolver]),
         );
         const cellPolicies = Object.fromEntries(
+          fieldsDef.steps.map((step) => [
+            step.name,
+            step.staleAfterSeconds === undefined
+              ? {}
+              : normalizeCellStalenessPolicy({
+                  staleAfterSeconds: step.staleAfterSeconds,
+                }),
+          ]),
+        ) as CellStalenessPolicyByField;
+        const authoredCellPolicies = Object.fromEntries(
           fieldsDef.steps.map((step) => [
             step.name,
             step.staleAfterSeconds === undefined
               ? {}
               : { staleAfterSeconds: step.staleAfterSeconds },
           ]),
+        ) as AuthoredCellStalenessPolicyByField;
+        return runMap(
+          name,
+          rows,
+          fields,
+          cellPolicies,
+          authoredCellPolicies,
+          opts,
         );
-        return runMap(name, rows, fields, cellPolicies, opts);
       }
       throw new Error(
         'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
@@ -5277,6 +5505,10 @@ async function executeRunRequest(
   ];
   let lastLedgerFlushAt = startedAt;
   let ledgerFlushInFlight: Promise<void> = Promise.resolve();
+  let ledgerFlushQueueDepth = 0;
+  let lastCoordinatorProgressPublishAt = 0;
+  let coordinatorProgressPublishInFlight: Promise<void> = Promise.resolve();
+  let coordinatorProgressPublishQueueDepth = 0;
 
   const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
@@ -5399,6 +5631,36 @@ async function executeRunRequest(
     });
   };
 
+  const flushCoordinatorProgressEvent = (force: boolean): Promise<void> => {
+    const now = nowMs();
+    if (
+      !force &&
+      now - lastCoordinatorProgressPublishAt <
+        MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+    ) {
+      return Promise.resolve();
+    }
+    if (!force && coordinatorProgressPublishQueueDepth > 0) {
+      return Promise.resolve();
+    }
+    lastCoordinatorProgressPublishAt = now;
+    coordinatorProgressPublishQueueDepth += 1;
+    coordinatorProgressPublishInFlight = coordinatorProgressPublishInFlight
+      .catch(() => undefined)
+      .then(async () => {
+        try {
+          await publishCoordinatorProgressEvent(now);
+        } finally {
+          coordinatorProgressPublishQueueDepth = Math.max(
+            0,
+            coordinatorProgressPublishQueueDepth - 1,
+          );
+        }
+      })
+      .catch(() => undefined);
+    return force ? coordinatorProgressPublishInFlight : Promise.resolve();
+  };
+
   const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
     updateStepProgress({
       nodeId: event.nodeId,
@@ -5498,15 +5760,19 @@ async function executeRunRequest(
     return events;
   };
 
-  const flushLedgerEvents = (force: boolean): void => {
-    if (!options?.persistResultDatasets) return;
+  const flushLedgerEvents = (force: boolean): Promise<void> => {
+    if (!options?.persistResultDatasets) return Promise.resolve();
     const now = nowMs();
     if (!force && now - lastLedgerFlushAt < RUN_LEDGER_FLUSH_INTERVAL_MS) {
-      return;
+      return Promise.resolve();
+    }
+    if (!force && ledgerFlushQueueDepth > 0) {
+      return Promise.resolve();
     }
     const events = drainPendingLedgerEvents(now);
-    if (events.length === 0) return;
+    if (events.length === 0) return Promise.resolve();
     lastLedgerFlushAt = now;
+    ledgerFlushQueueDepth += 1;
     ledgerFlushInFlight = ledgerFlushInFlight
       .catch(() => undefined)
       .then(async () => {
@@ -5519,10 +5785,12 @@ async function executeRunRequest(
         } catch {
           pendingLedgerEvents = [...events, ...pendingLedgerEvents];
           throw new Error('runtime run-ledger append failed');
+        } finally {
+          ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
         }
-        await publishCoordinatorProgressEvent(now).catch(() => undefined);
       })
       .catch(() => undefined);
+    return force ? ledgerFlushInFlight : Promise.resolve();
   };
 
   const flushTerminalLedgerEvents = async (
@@ -5564,7 +5832,12 @@ async function executeRunRequest(
   const workerCallbacks: WorkerCtxCallbacks = {
     onNodeProgress: (input) => {
       updateStepProgress(input);
-      flushLedgerEvents(Boolean(input.forceFlush));
+      const force = Boolean(input.forceFlush);
+      const ledgerFlush = flushLedgerEvents(force);
+      const progressFlush = flushCoordinatorProgressEvent(force);
+      return force
+        ? Promise.all([ledgerFlush, progressFlush]).then(() => undefined)
+        : Promise.resolve();
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),