deepline 0.1.85 → 0.1.89

package/dist/repo/apps/play-runner-workers/src/entry.ts

@@ -62,7 +62,7 @@ import {
 } from './child-play-await';
 import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
 import {
-  adaptV2ExecuteResponseToToolResult,
+  parseToolExecuteResponse,
   createToolExecuteResult,
   deserializeToolExecuteResult,
   isSerializedToolExecuteResult,
@@ -600,7 +600,7 @@ type WorkerCtxCallbacks = {
     nodeId: string;
     progress: LiveNodeProgressSnapshot;
     forceFlush?: boolean;
-  }) => void;
+  }) => void | Promise<void>;
   onMapStarted?: (nodeId: string, at?: number) => void;
   onMapCompleted?: (nodeId: string, at?: number) => void;
   onToolCalled?: (toolId: string, at?: number) => void;
@@ -1289,18 +1289,12 @@ async function callToolDirect(
     });
     if (res.ok) {
       const body = (await res.json()) as Record<string, unknown>;
-      const { result } = adaptV2ExecuteResponseToToolResult(body);
-      const status =
-        typeof body.status === 'string'
-          ? body.status
-          : result == null
-            ? 'no_result'
-            : 'completed';
+      const parsed = parseToolExecuteResponse(toolId, body);
       return wrapWorkerToolResult(
         toolId,
-        result,
-        parseExecuteToolMetadata(toolId, body),
-        status,
+        parsed.result,
+        parsed.metadata,
+        parsed.status,
       );
     }
 
@@ -1338,121 +1332,6 @@ async function callToolDirect(
   throw lastError ?? new Error(`tool ${toolId} failed before execution.`);
 }
 
-function parseExecuteToolMetadata(
-  toolId: string,
-  data: Record<string, unknown>,
-): ToolResultMetadataInput | null {
-  const metadata = data._metadata;
-  if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) {
-    return null;
-  }
-  const tool = (metadata as Record<string, unknown>).tool;
-  if (!tool || typeof tool !== 'object' || Array.isArray(tool)) return null;
-  const record = tool as Record<string, unknown>;
-  const metadataToolId =
-    typeof record.toolId === 'string' && record.toolId.trim()
-      ? record.toolId.trim()
-      : toolId;
-  return {
-    toolId: metadataToolId,
-    extractors: parseExtractorMetadata(record.extractors),
-    targetGetters: parseGetterMetadata(record.targetGetters),
-    listExtractorPaths: parseStringArray(record.listExtractorPaths),
-    listIdentityGetters: parseGetterMetadata(record.listIdentityGetters),
-  };
-}
-
-function parseExtractorMetadata(
-  value: unknown,
-): ToolResultMetadataInput['extractors'] {
-  if (!value || typeof value !== 'object' || Array.isArray(value)) {
-    return undefined;
-  }
-  const entries = Object.entries(value as Record<string, unknown>).flatMap(
-    ([key, descriptor]) => {
-      if (
-        !descriptor ||
-        typeof descriptor !== 'object' ||
-        Array.isArray(descriptor)
-      ) {
-        return [];
-      }
-      const record = descriptor as Record<string, unknown>;
-      const paths = parseStringArray(record.paths);
-      if (paths.length === 0) return [];
-      const transforms = parseStringArray(record.transforms);
-      const enumValues = parseStringArray(record.enum);
-      const overrides = parseExtractorOverrides(record.overrides);
-      return [
-        [
-          key,
-          {
-            paths,
-            ...(transforms.length > 0 ? { transforms } : {}),
-            ...(enumValues.length > 0 ? { enum: enumValues } : {}),
-            ...(overrides.length > 0 ? { overrides } : {}),
-          },
-        ],
-      ] as const;
-    },
-  );
-  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
-}
-
-function parseExtractorOverrides(
-  value: unknown,
-): NonNullable<
-  NonNullable<ToolResultMetadataInput['extractors']>[string]['overrides']
-> {
-  if (!Array.isArray(value)) return [];
-  return value.flatMap((entry) => {
-    if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
-      return [];
-    }
-    const record = entry as Record<string, unknown>;
-    const paths = parseStringArray(record.paths);
-    if (paths.length === 0) return [];
-    const equals =
-      record.equals === null ||
-      typeof record.equals === 'string' ||
-      typeof record.equals === 'number' ||
-      typeof record.equals === 'boolean'
-        ? record.equals
-        : true;
-    const overrideValue = record.value;
-    if (
-      overrideValue !== null &&
-      typeof overrideValue !== 'string' &&
-      typeof overrideValue !== 'number' &&
-      typeof overrideValue !== 'boolean'
-    ) {
-      return [];
-    }
-    return [{ paths, equals, value: overrideValue }];
-  });
-}
-
-function parseGetterMetadata(
-  value: unknown,
-): Record<string, readonly string[]> | undefined {
-  if (!value || typeof value !== 'object' || Array.isArray(value)) {
-    return undefined;
-  }
-  const entries = Object.entries(value as Record<string, unknown>)
-    .map(([key, paths]) => [key, parseStringArray(paths)] as const)
-    .filter(
-      (entry): entry is readonly [string, string[]] => entry[1].length > 0,
-    );
-  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
-}
-
-function parseStringArray(value: unknown): string[] {
-  if (!Array.isArray(value)) return [];
-  return value
-    .map((entry) => (typeof entry === 'string' ? entry.trim() : ''))
-    .filter(Boolean);
-}
-
 function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
   if (toolId === 'test_rate_limit') {
     return {
@@ -1592,6 +1471,7 @@ type WorkerToolBatchRequest = {
 };
 
 const WORKER_TOOL_BATCH_GRACE_MS = 250;
+const MAP_EXECUTION_HEARTBEAT_INTERVAL_MS = 5_000;
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1599,6 +1479,10 @@ const WORKER_TOOL_BATCH_GRACE_MS = 250;
 const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
 const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
 
+function sleepWorkerMs(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 function stepProgramColumnName(parentField: string, stepId: string): string {
   return sqlSafePlayColumnName(`${parentField}.${stepId}`);
 }
@@ -2951,42 +2835,71 @@ function augmentSheetContractWithDatasetFields(input: {
   outputFields?: readonly string[];
 }): PlaySheetContract {
   const outputFields = new Set(input.outputFields ?? []);
-  const existingFields = new Set(
-    input.contract.columns.flatMap((column) =>
-      typeof column.field === 'string' ? [column.field] : [],
-    ),
-  );
-  const existingSqlNames = new Set(
-    input.contract.columns.map((column) => column.sqlName),
-  );
-  const columns = [...input.contract.columns];
   const candidateFields = new Set<string>();
   for (const row of input.rows) {
     for (const field of Object.keys(row)) {
-      candidateFields.add(field);
+      if (isDatasetPayloadField(field)) {
+        candidateFields.add(field);
+      }
     }
   }
   for (const field of outputFields) {
-    candidateFields.add(field);
-  }
-  for (const field of candidateFields) {
-    if (!isDatasetPayloadField(field) || existingFields.has(field)) {
-      continue;
+    if (isDatasetPayloadField(field)) {
+      candidateFields.add(field);
     }
-    const sqlName = sqlSafePlayColumnName(field);
-    if (existingSqlNames.has(sqlName)) {
-      continue;
+  }
+
+  const existingFields = new Set<string>();
+  const existingSqlNames = new Set<string>();
+  const inputColumns: PlaySheetContract['columns'] = [];
+  const outputColumns: PlaySheetContract['columns'] = [];
+  const appendColumn = (
+    target: PlaySheetContract['columns'],
+    column: PlaySheetContract['columns'][number],
+  ) => {
+    const field = typeof column.field === 'string' ? column.field : column.id;
+    const sqlName = column.sqlName.trim();
+    if (
+      !field ||
+      !sqlName ||
+      existingFields.has(field) ||
+      existingSqlNames.has(sqlName)
+    ) {
+      return;
     }
     existingFields.add(field);
     existingSqlNames.add(sqlName);
-    columns.push({
+    target.push(column);
+  };
+
+  for (const column of input.contract.columns) {
+    const field = typeof column.field === 'string' ? column.field : column.id;
+    if (
+      column.source === 'input' &&
+      field === input.contract.tableNamespace &&
+      !candidateFields.has(field)
+    ) {
+      continue;
+    }
+    appendColumn(
+      column.source === 'input' ? inputColumns : outputColumns,
+      column,
+    );
+  }
+
+  for (const field of candidateFields) {
+    if (existingFields.has(field)) continue;
+    const sqlName = sqlSafePlayColumnName(field);
+    if (existingSqlNames.has(sqlName)) continue;
+    appendColumn(outputFields.has(field) ? outputColumns : inputColumns, {
       id: `runtime:${input.contract.tableNamespace}:${field}`,
       sqlName,
       source: outputFields.has(field) ? 'datasetColumn' : 'input',
       field,
     });
   }
-  return { ...input.contract, columns };
+
+  return { ...input.contract, columns: [...inputColumns, ...outputColumns] };
 }
 
 async function persistCompletedMapRows(input: {
@@ -3542,8 +3455,11 @@ function createMinimalWorkerCtx(
       softWorkflowStepBudget: plan?.chunkPlan.softWorkflowStepBudget,
     });
     const outputFields = fieldEntries.map(([field]) => field);
-    const updateMapProgress = (progress: LiveNodeProgressSnapshot) => {
-      callbacks?.onNodeProgress?.({
+    const updateMapProgress = (
+      progress: LiveNodeProgressSnapshot,
+      options?: { forceFlush?: boolean },
+    ): void | Promise<void> => {
+      return callbacks?.onNodeProgress?.({
         nodeId: mapNodeId,
         progress: {
           artifactTableNamespace: name,
@@ -3551,19 +3467,62 @@ function createMinimalWorkerCtx(
           ...progress,
           updatedAt: progress.updatedAt ?? nowMs(),
         },
-        forceFlush: true,
+        forceFlush: options?.forceFlush ?? true,
       });
     };
     const formatMapProgressMessage = (completed: number, total?: number) =>
       typeof total === 'number' && Number.isFinite(total) && total > 0
         ? `${completed.toLocaleString()} / ${total.toLocaleString()} rows processed`
         : `${completed.toLocaleString()} rows processed`;
+    const formatMapPreparingMessage = (total?: number) =>
+      typeof total === 'number' && Number.isFinite(total) && total > 0
+        ? `Preparing ${total.toLocaleString()} rows`
+        : 'Preparing rows';
+    const formatMapQueuedMessage = (input: {
+      completed: number;
+      queued: number;
+      total?: number;
+    }) => {
+      const completed = Math.max(0, input.completed);
+      const queued = Math.max(0, input.queued);
+      if (completed > 0 && queued > 0) {
+        return `${completed.toLocaleString()} already satisfied, ${queued.toLocaleString()} queued`;
+      }
+      if (queued > 0) {
+        return `${queued.toLocaleString()} rows queued`;
+      }
+      return formatMapProgressMessage(completed, input.total);
+    };
+    const formatMapProcessingMessage = (rowsToExecute: number) =>
+      rowsToExecute > 0
+        ? `Processing ${rowsToExecute.toLocaleString()} rows`
+        : null;
+    const formatMapExecutionHeartbeatMessage = (input: {
+      rowsToExecute: number;
+      startedRows: number;
+      activeRows: number;
+      completedRows: number;
+    }) => {
+      const rowsToExecute = Math.max(0, input.rowsToExecute);
+      const startedRows = Math.max(0, input.startedRows);
+      const activeRows = Math.max(0, input.activeRows);
+      const completedRows = Math.max(0, input.completedRows);
+      const waitingRows = Math.max(0, rowsToExecute - startedRows);
+      const parts = [
+        activeRows > 0 ? `${activeRows.toLocaleString()} active` : null,
+        waitingRows > 0 ? `${waitingRows.toLocaleString()} waiting` : null,
+        completedRows > 0 ? `${completedRows.toLocaleString()} done` : null,
+      ].filter((part): part is string => Boolean(part));
+      const base =
+        formatMapProcessingMessage(rowsToExecute) ?? 'Processing rows';
+      return parts.length > 0 ? `${base} (${parts.join(', ')})` : base;
+    };
     callbacks?.onMapStarted?.(mapNodeId, mapStartedAt);
-    updateMapProgress({
+    await updateMapProgress({
       completed: 0,
       total: rowCountHint ?? undefined,
       startedAt: mapStartedAt,
-      message: formatMapProgressMessage(0, rowCountHint ?? undefined),
+      message: formatMapPreparingMessage(rowCountHint ?? undefined),
     });
     const explicitRowKeysSeen =
       opts?.key === undefined ? null : new Map<string, number>();
@@ -3637,6 +3596,8 @@ function createMinimalWorkerCtx(
       }
     };
 
+    let totalRowsWritten = 0;
+
     const processChunk = async (
       chunkRows: T[],
       chunkStart: number,
@@ -3681,17 +3642,20 @@ function createMinimalWorkerCtx(
           completedRows: prepared.completedRows.length,
         },
       });
-      updateMapProgress({
-        completed: prepared.completedRows.length,
-        total: chunkRows.length,
+      const progressTotalRows = rowCountHint ?? chunkRows.length;
+      const preparedCompletedRows = Math.min(
+        progressTotalRows,
+        totalRowsWritten + prepared.completedRows.length,
+      );
+      await updateMapProgress({
+        completed: preparedCompletedRows,
+        total: progressTotalRows,
         startedAt: mapStartedAt,
-        message:
-          prepared.pendingRows.length > 0
-            ? `${prepared.pendingRows.length.toLocaleString()} rows queued`
-            : formatMapProgressMessage(
-                prepared.completedRows.length,
-                chunkRows.length,
-              ),
+        message: formatMapQueuedMessage({
+          completed: preparedCompletedRows,
+          queued: prepared.pendingRows.length,
+          total: progressTotalRows,
+        }),
       });
       const pendingKeys = new Set<string>();
       const pendingRowsByKey = new Map<string, Record<string, unknown>>();
@@ -3735,38 +3699,83 @@ function createMinimalWorkerCtx(
           new Set(chunkEntries.map((entry) => entry.rowKey)).size,
       );
       const rowsToExecute = uniqueRowsToExecuteEntries.map(({ row }) => row);
+      const processingMessage = formatMapProcessingMessage(
+        rowsToExecute.length,
+      );
+      if (processingMessage) {
+        await updateMapProgress({
+          completed: preparedCompletedRows,
+          total: progressTotalRows,
+          startedAt: mapStartedAt,
+          message: processingMessage,
+        });
+      }
       const rowsInserted = prepared.inserted + missingPreparedRows.length;
       const rowsSkipped = Math.max(
         0,
         prepared.skipped - missingPreparedRows.length,
       );
-      let settledToolRequests = 0;
-      let lastToolProgressAt = 0;
-      const reportSettledToolRequests = (count: number) => {
-        if (count <= 0) return;
-        settledToolRequests += count;
+      let completedExecutedRows = 0;
+      let startedExecutedRows = 0;
+      let activeExecutedRows = 0;
+      let lastChunkProgressAt = 0;
+      let lastExecutionHeartbeatAt = nowMs();
+      const completedRowsForProgress = () =>
+        Math.min(
+          progressTotalRows,
+          totalRowsWritten +
+            prepared.completedRows.length +
+            completedExecutedRows,
+        );
+      const reportExecutionHeartbeat = (force = false) => {
         const now = nowMs();
-        const estimatedCompleted = Math.min(
-          chunkRows.length,
-          prepared.completedRows.length + settledToolRequests,
+        if (
+          !force &&
+          now - lastExecutionHeartbeatAt < MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+        ) {
+          return;
+        }
+        lastExecutionHeartbeatAt = now;
+        void updateMapProgress(
+          {
+            completed: completedRowsForProgress(),
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapExecutionHeartbeatMessage({
+              rowsToExecute: rowsToExecute.length,
+              startedRows: startedExecutedRows,
+              activeRows: activeExecutedRows,
+              completedRows: completedExecutedRows,
+            }),
+          },
+          { forceFlush: force },
         );
-        const isTerminalEstimate = estimatedCompleted >= chunkRows.length;
+      };
+      const reportChunkProgress = (force = false) => {
+        const now = nowMs();
+        const completed = completedRowsForProgress();
+        const isTerminalEstimate = completed >= progressTotalRows;
         if (
+          !force &&
           !isTerminalEstimate &&
-          now - lastToolProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
+          now - lastChunkProgressAt < RUN_LEDGER_FLUSH_INTERVAL_MS
         ) {
           return;
         }
-        lastToolProgressAt = now;
-        updateMapProgress({
-          completed: estimatedCompleted,
-          total: chunkRows.length,
-          startedAt: mapStartedAt,
-          message: formatMapProgressMessage(
-            estimatedCompleted,
-            chunkRows.length,
-          ),
-        });
+        lastChunkProgressAt = now;
+        void updateMapProgress(
+          {
+            completed,
+            total: progressTotalRows,
+            startedAt: mapStartedAt,
+            message: formatMapProgressMessage(completed, progressTotalRows),
+          },
+          { forceFlush: force },
+        );
+      };
+      const reportSettledToolRequests = (count: number) => {
+        if (count <= 0) return;
+        reportChunkProgress(false);
       };
       // Row concurrency comes from the Governor: an explicit map concurrency is
       // clamped to the policy row-max, otherwise the policy default. Each row
@@ -3812,7 +3821,12 @@ function createMinimalWorkerCtx(
               const rowSlot = await governor.acquireRowSlot({
                 signal: abortSignal,
               });
+              let rowMarkedActive = false;
               try {
+                startedExecutedRows += 1;
+                activeExecutedRows += 1;
+                rowMarkedActive = true;
+                reportExecutionHeartbeat(false);
                 const entry = uniqueRowsToExecuteEntries[myIndex]!;
                 const pendingRow = pendingRowsByKey.get(entry.rowKey);
                 const row = pendingRow
@@ -3985,7 +3999,13 @@ function createMinimalWorkerCtx(
                     ? cellMetaPatch
                     : undefined;
                 executedRows[myIndex] = enriched as T & Record<string, unknown>;
+                completedExecutedRows += 1;
+                reportChunkProgress(false);
               } finally {
+                if (rowMarkedActive) {
+                  activeExecutedRows = Math.max(0, activeExecutedRows - 1);
+                  reportExecutionHeartbeat(false);
+                }
                 rowSlot.release();
               }
             }
@@ -4031,7 +4051,27 @@ function createMinimalWorkerCtx(
         });
       };
       const workersStartedAt = nowMs();
-      const workerResults = await Promise.allSettled(workers);
+      // Track completion with a boolean flag rather than narrowing a
+      // closure-assigned `| null` variable: TypeScript's control-flow analysis
+      // does not see the assignment inside `.then(...)`, so a
+      // `while (results === null)` loop would narrow it to `never` afterwards.
+      let workerSettled = false;
+      const workerResultsPromise = Promise.allSettled(workers).then(
+        (results) => {
+          workerSettled = true;
+          return results;
+        },
+      );
+      while (!workerSettled) {
+        await Promise.race([
+          workerResultsPromise,
+          sleepWorkerMs(MAP_EXECUTION_HEARTBEAT_INTERVAL_MS),
+        ]);
+        if (!workerSettled) {
+          reportExecutionHeartbeat(false);
+        }
+      }
+      const workerResults = await workerResultsPromise;
       recordRunnerPerfTrace({
         req,
         phase: 'runner.map_chunk.execute_workers',
@@ -4198,7 +4238,7 @@ function createMinimalWorkerCtx(
         `inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
       const completedAt = nowMs();
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
-      updateMapProgress({
+      void updateMapProgress({
         completed: totalRowsWritten,
         total: totalRowsWritten,
         completedAt,
@@ -4249,7 +4289,6 @@ function createMinimalWorkerCtx(
       });
     };
 
-    let totalRowsWritten = 0;
     let chunkIndex = 0;
     let chunkStart = 0;
     for await (const chunkRows of iterDatasetChunks(inputRows, rowsPerChunk)) {
@@ -4263,7 +4302,7 @@ function createMinimalWorkerCtx(
       totalRowsDuplicateReused += chunkResult.rowsDuplicateReused;
       totalRowsInserted += chunkResult.rowsInserted;
       totalRowsSkipped += chunkResult.rowsSkipped;
-      updateMapProgress({
+      await updateMapProgress({
         completed: totalRowsWritten,
         total: rowCountHint ?? undefined,
         message: formatMapProgressMessage(
@@ -5374,6 +5413,10 @@ async function executeRunRequest(
   ];
   let lastLedgerFlushAt = startedAt;
   let ledgerFlushInFlight: Promise<void> = Promise.resolve();
+  let ledgerFlushQueueDepth = 0;
+  let lastCoordinatorProgressPublishAt = 0;
+  let coordinatorProgressPublishInFlight: Promise<void> = Promise.resolve();
+  let coordinatorProgressPublishQueueDepth = 0;
 
   const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
@@ -5496,6 +5539,36 @@ async function executeRunRequest(
     });
   };
 
+  const flushCoordinatorProgressEvent = (force: boolean): Promise<void> => {
+    const now = nowMs();
+    if (
+      !force &&
+      now - lastCoordinatorProgressPublishAt <
+        MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
+    ) {
+      return Promise.resolve();
+    }
+    if (!force && coordinatorProgressPublishQueueDepth > 0) {
+      return Promise.resolve();
+    }
+    lastCoordinatorProgressPublishAt = now;
+    coordinatorProgressPublishQueueDepth += 1;
+    coordinatorProgressPublishInFlight = coordinatorProgressPublishInFlight
+      .catch(() => undefined)
+      .then(async () => {
+        try {
+          await publishCoordinatorProgressEvent(now);
+        } finally {
+          coordinatorProgressPublishQueueDepth = Math.max(
+            0,
+            coordinatorProgressPublishQueueDepth - 1,
+          );
+        }
+      })
+      .catch(() => undefined);
+    return force ? coordinatorProgressPublishInFlight : Promise.resolve();
+  };
+
   const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
     updateStepProgress({
       nodeId: event.nodeId,
@@ -5595,15 +5668,19 @@ async function executeRunRequest(
     return events;
   };
 
-  const flushLedgerEvents = (force: boolean): void => {
-    if (!options?.persistResultDatasets) return;
+  const flushLedgerEvents = (force: boolean): Promise<void> => {
+    if (!options?.persistResultDatasets) return Promise.resolve();
     const now = nowMs();
     if (!force && now - lastLedgerFlushAt < RUN_LEDGER_FLUSH_INTERVAL_MS) {
-      return;
+      return Promise.resolve();
+    }
+    if (!force && ledgerFlushQueueDepth > 0) {
+      return Promise.resolve();
     }
     const events = drainPendingLedgerEvents(now);
-    if (events.length === 0) return;
+    if (events.length === 0) return Promise.resolve();
     lastLedgerFlushAt = now;
+    ledgerFlushQueueDepth += 1;
     ledgerFlushInFlight = ledgerFlushInFlight
       .catch(() => undefined)
       .then(async () => {
@@ -5616,10 +5693,12 @@ async function executeRunRequest(
         } catch {
           pendingLedgerEvents = [...events, ...pendingLedgerEvents];
           throw new Error('runtime run-ledger append failed');
+        } finally {
+          ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
         }
-        await publishCoordinatorProgressEvent(now).catch(() => undefined);
       })
       .catch(() => undefined);
+    return force ? ledgerFlushInFlight : Promise.resolve();
   };
 
   const flushTerminalLedgerEvents = async (
@@ -5661,7 +5740,12 @@ async function executeRunRequest(
   const workerCallbacks: WorkerCtxCallbacks = {
     onNodeProgress: (input) => {
       updateStepProgress(input);
-      flushLedgerEvents(Boolean(input.forceFlush));
+      const force = Boolean(input.forceFlush);
+      const ledgerFlush = flushLedgerEvents(force);
+      const progressFlush = flushCoordinatorProgressEvent(force);
+      return force
+        ? Promise.all([ledgerFlush, progressFlush]).then(() => undefined)
+        : Promise.resolve();
     },
     onMapStarted: (nodeId, at) => stepLifecycle?.onMapStarted(nodeId, at),
     onMapCompleted: (nodeId, at) => stepLifecycle?.onMapCompleted(nodeId, at),