deepline 0.3.283 → 0.3.285

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -35,6 +35,7 @@ import { runtimeWorkCountersFromStats } from '../runtime-work-counters';
 import {
   buildPlayRunLedgerEventsFromStatusPatch,
   createEmptyPlayRunLedgerSnapshot,
+  type PlayMapRowActivitySource,
   type PlayRunLedgerEvent,
 } from '../run-ledger';
 import {
@@ -186,6 +187,21 @@ function callsInFlightMessage(input: {
     : `${active} calls in flight`;
 }
 
+function mapRowProgressMessage(input: {
+  completed: number;
+  total: number | undefined;
+  supersededRows?: number;
+  activeRows?: number;
+}): string | undefined {
+  const settled = rowsProcessedMessage(
+    input.completed,
+    input.total,
+    input.supersededRows,
+  );
+  if (input.activeRows === undefined) return settled;
+  return `${settled ?? `${input.completed.toLocaleString()} rows processed`} · ${input.activeRows.toLocaleString()} rows processing`;
+}
+
 export type PostgresSchedulerProgressReporter = {
   /**
    * Consume one raw runner event. Synchronous and cheap on purpose: this is
@@ -426,6 +442,10 @@ export function createPostgresSchedulerProgressReporter(input: {
   let deferredFlushTimer: ReturnType<typeof setTimeout> | null = null;
   let totalRows = 0;
   let activeMapNodeId: string | null = null;
+  // Private per-node provenance, reset for each map invocation. Tool-batch
+  // cardinality is call evidence; an explicit resolver sample owns row counts.
+  const mapsWithResolverActivity = new Set<string>();
+  const mapInvocationIds = new Map<string, string>();
   let pendingDatasetLifecycleEvents: Array<
     Extract<
       PlayExecutionEvent,
@@ -688,6 +708,18 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(current.waitingRows !== undefined
           ? { waitingRows: current.waitingRows }
           : {}),
+        ...(current.rowActivitySource !== undefined
+          ? { rowActivitySource: current.rowActivitySource }
+          : {}),
+        ...(current.mapInvocationId !== undefined
+          ? { mapInvocationId: current.mapInvocationId }
+          : {}),
+        ...(current.rowActivityStartedAt !== undefined
+          ? { rowActivityStartedAt: current.rowActivityStartedAt }
+          : {}),
+        ...(current.rowActivityObservedAt !== undefined
+          ? { rowActivityObservedAt: current.rowActivityObservedAt }
+          : {}),
         ...(current.completedRows !== undefined
           ? { completedRows: current.completedRows }
           : {}),
@@ -719,6 +751,21 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(patch.waitingRows !== undefined
           ? { waitingRows: patch.waitingRows }
           : {}),
+        ...(patch.rowActivitySource !== undefined
+          ? { rowActivitySource: patch.rowActivitySource }
+          : {}),
+        ...(mapInvocationIds.has(nodeId)
+          ? { mapInvocationId: mapInvocationIds.get(nodeId) }
+          : {}),
+        ...(patch.mapInvocationId !== undefined
+          ? { mapInvocationId: patch.mapInvocationId }
+          : {}),
+        ...(patch.rowActivityStartedAt !== undefined
+          ? { rowActivityStartedAt: patch.rowActivityStartedAt }
+          : {}),
+        ...(patch.rowActivityObservedAt !== undefined
+          ? { rowActivityObservedAt: patch.rowActivityObservedAt }
+          : {}),
         ...(patch.completedRows !== undefined
           ? { completedRows: patch.completedRows }
           : {}),
@@ -758,6 +805,15 @@ export function createPostgresSchedulerProgressReporter(input: {
   );
   stepLifecycle.markPreDatasetStepsStarted();
 
+  const sourceForMapLog = (nodeId: string): PlayMapRowActivitySource => {
+    if (mapsWithResolverActivity.has(nodeId)) return 'resolver';
+    // Coalescing must retain a witnessed map start, settlement or resume in
+    // this batch. Otherwise its subsequent compatibility log hides the fact
+    // that resets the preceding batch's resolver authority.
+    const current = liveNodeProgress[nodeId]?.rowActivitySource;
+    return current === 'map_event' || current === 'map_start' ? current : 'log';
+  };
+
   const markMapStartedFromLog = (line: string) => {
     const match = line.match(MAP_START_LOG_PATTERN);
     if (!match) {
@@ -795,18 +851,26 @@ export function createPostgresSchedulerProgressReporter(input: {
     );
     const failed = Math.max(0, Number(current.failed ?? 0));
     const settled = completed + failed;
+    const startedRows = Math.max(Number(current.startedRows ?? 0), settled);
     setNodeProgress(nodeRef.nodeId, {
       completed,
       failed,
       total: effectiveTotal > 0 ? effectiveTotal : undefined,
-      startedRows: Math.max(Number(current.startedRows ?? 0), settled),
-      activeRows: 0,
+      rowActivitySource: sourceForMapLog(nodeRef.nodeId),
+      startedRows,
+      activeRows: current.activeRows ?? 0,
       waitingRows:
-        effectiveTotal > 0 ? Math.max(0, effectiveTotal - settled) : undefined,
+        effectiveTotal > 0
+          ? Math.max(0, effectiveTotal - startedRows)
+          : undefined,
       completedRows: completed,
       artifactTableNamespace: nodeRef.artifactTableNamespace,
       startedAt: current.startedAt ?? Date.now(),
-      message: rowsProcessedMessage(completed, effectiveTotal),
+      message: mapRowProgressMessage({
+        completed,
+        total: effectiveTotal,
+        activeRows: current.activeRows,
+      }),
     });
   };
 
@@ -837,23 +901,31 @@ export function createPostgresSchedulerProgressReporter(input: {
     const completed = Math.max(0, Number(current.completed ?? 0));
     const failed = Math.max(0, Number(current.failed ?? 0));
     const settled = completed + failed;
+    const startedRows = Math.max(Number(current.startedRows ?? 0), settled);
     setNodeProgress(nodeRef.nodeId, {
       completed,
       failed,
       total: effectiveTotal > 0 ? effectiveTotal : undefined,
-      startedRows: Math.max(Number(current.startedRows ?? 0), settled),
-      activeRows: 0,
+      rowActivitySource: sourceForMapLog(nodeRef.nodeId),
+      startedRows,
+      activeRows: current.activeRows ?? 0,
       waitingRows:
-        effectiveTotal > 0 ? Math.max(0, effectiveTotal - settled) : undefined,
+        effectiveTotal > 0
+          ? Math.max(0, effectiveTotal - startedRows)
+          : undefined,
       completedRows: completed,
       artifactTableNamespace: nodeRef.artifactTableNamespace,
       startedAt: current.startedAt ?? Date.now(),
-      message: rowsProcessedMessage(completed, effectiveTotal),
+      message: mapRowProgressMessage({
+        completed,
+        total: effectiveTotal,
+        activeRows: current.activeRows,
+      }),
     });
   };
 
   const markToolBatchInFlightFromLog = (line: string) => {
-    if (!activeMapNodeId) {
+    if (!activeMapNodeId || mapsWithResolverActivity.has(activeMapNodeId)) {
       return;
     }
     const match = line.match(PLAY_TOOL_BATCH_LOG_PATTERN);
@@ -876,6 +948,7 @@ export function createPostgresSchedulerProgressReporter(input: {
       settled + activeRows,
     );
     setNodeProgress(activeMapNodeId, {
+      rowActivitySource: sourceForMapLog(activeMapNodeId),
       startedRows: total > 0 ? Math.min(total, startedRows) : startedRows,
       activeRows,
       waitingRows:
@@ -1007,6 +1080,7 @@ export function createPostgresSchedulerProgressReporter(input: {
     if (!nodeId) {
       return;
     }
+    mapInvocationIds.set(nodeId, event.mapInvocationId);
     activeMapNodeId = nodeId;
     const eventTotalRows = 'totalRows' in event ? event.totalRows : undefined;
     if (
@@ -1021,11 +1095,15 @@ export function createPostgresSchedulerProgressReporter(input: {
       event.type === 'map.started' ||
       event.type === 'map.resumed'
     ) {
+      mapsWithResolverActivity.delete(nodeId);
       stepLifecycle.onMapStarted(nodeId);
       setNodeProgress(nodeId, {
         completed: event.completedRows,
         failed: 0,
         total: eventTotalRows,
+        rowActivitySource: 'map_start',
+        rowActivityStartedAt: event.at,
+        rowActivityObservedAt: event.at,
         startedRows: event.completedRows + (event.supersededRows ?? 0),
         activeRows: 0,
         waitingRows:
@@ -1055,6 +1133,13 @@ export function createPostgresSchedulerProgressReporter(input: {
       return;
     }
     if (event.type === 'map.progress') {
+      if (
+        event.startedRows !== undefined ||
+        event.activeRows !== undefined ||
+        event.waitingRows !== undefined
+      ) {
+        mapsWithResolverActivity.add(nodeId);
+      }
       const current = liveNodeProgress[nodeId] ?? {};
       const settled = Math.max(
         0,
@@ -1066,32 +1151,42 @@ export function createPostgresSchedulerProgressReporter(input: {
           : 0,
         Number(current.total ?? 0),
       );
-      const startedRows = Math.max(settled, Number(current.startedRows ?? 0));
-      const activeRows = Math.max(0, startedRows - settled);
+      const startedRows = Math.max(
+        settled,
+        Number(event.startedRows ?? current.startedRows ?? 0),
+      );
+      const activeRows = Math.max(0, event.activeRows ?? startedRows - settled);
       setNodeProgress(nodeId, {
         completed: event.completedRows,
         failed: event.failedRows,
         total: progressTotalRows > 0 ? progressTotalRows : undefined,
+        rowActivitySource: mapsWithResolverActivity.has(nodeId)
+          ? 'resolver'
+          : 'map_event',
+        rowActivityObservedAt: event.at,
         startedRows,
         activeRows,
         waitingRows:
-          progressTotalRows > 0
+          event.waitingRows ??
+          (progressTotalRows > 0
             ? Math.max(0, progressTotalRows - startedRows)
-            : undefined,
+            : undefined),
         completedRows: event.completedRows,
         ...(event.supersededRows !== undefined
           ? { supersededRows: event.supersededRows }
           : {}),
         artifactTableNamespace: event.artifactTableNamespace,
-        message: rowsProcessedMessage(
-          event.completedRows,
-          progressTotalRows,
-          event.supersededRows,
-        ),
+        message: mapRowProgressMessage({
+          completed: event.completedRows,
+          total: progressTotalRows,
+          supersededRows: event.supersededRows,
+          activeRows: event.activeRows,
+        }),
       });
       return;
     }
     if (event.type === 'map.completed' || event.type === 'map.failed') {
+      mapsWithResolverActivity.delete(nodeId);
       if (event.type === 'map.failed') {
         stepLifecycle.onMapFailed(nodeId);
       } else {
@@ -1109,6 +1204,8 @@ export function createPostgresSchedulerProgressReporter(input: {
         completed: event.completedRows,
         failed: event.failedRows,
         total: event.totalRows,
+        rowActivitySource: 'map_event',
+        rowActivityObservedAt: event.at,
         startedRows:
           terminalTotal > 0
             ? Math.min(terminalTotal, processedRows)
@@ -1131,10 +1228,14 @@ export function createPostgresSchedulerProgressReporter(input: {
       return;
     }
     if (event.type === 'map.suspended') {
+      mapsWithResolverActivity.add(nodeId);
       setNodeProgress(nodeId, {
         completed: event.completedRows,
         failed: event.failedRows,
         total: event.totalRows,
+        activeRows: 0,
+        rowActivitySource: 'resolver',
+        rowActivityObservedAt: event.at,
         completedRows: event.completedRows,
         artifactTableNamespace: event.artifactTableNamespace,
         message: rowsProcessedMessage(event.completedRows, event.totalRows),
@@ -1339,6 +1440,8 @@ export function createPostgresSchedulerProgressReporter(input: {
         completed,
         failed,
         total: total > 0 ? total : undefined,
+        rowActivitySource: 'map_event',
+        mapInvocationId: frame.mapInvocationId,
         startedRows: total > 0 ? total : completed + failed,
         activeRows: 0,
         waitingRows: 0,
@@ -1413,6 +1516,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         completed,
         failed,
         total: total > 0 ? total : undefined,
+        rowActivitySource: 'map_event',
         startedRows: total > 0 ? total : completed + failed + supersededRows,
         activeRows: 0,
         waitingRows: 0,