deepline 0.3.247 → 0.3.249

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -20,6 +20,10 @@ import {
 import type { PlayVisualNodeProgressMap } from '../worker-api-types';
 import type { PlayExecutionEvent } from '../ctx-types';
 import type { PlayActivityObservation } from '../activity-observation';
+import {
+  normalizePlayLogContext,
+  type PlayLogContext,
+} from '../log-context';
 import type {
   PlayRunnerLogEvent,
   PlayRunnerResult,
@@ -232,7 +236,8 @@ export type PostgresSchedulerProgressReporter = {
 };
 
 export type PostgresSchedulerDurableLogMode =
-  'progress_reporter' | 'direct_worker';
+  | 'progress_reporter'
+  | 'direct_worker';
 
 function shouldPublishDurableLogEvent(
   mode: PostgresSchedulerDurableLogMode,
@@ -274,33 +279,53 @@ type TerminalLogReplayPayload = NonNullable<
   RuntimeStatusUpdate['terminalLogReplay']
 >;
 
-function terminalReplayLinesForMode(input: {
+function terminalReplayEntriesForMode(input: {
   resultLogs: readonly string[];
+  resultLogContexts?: readonly (PlayLogContext | null)[];
   durableLogMode: PostgresSchedulerDurableLogMode;
-}): string[] {
-  return input.resultLogs
-    .map((line) => line.trim())
-    .filter(
-      (line) =>
-        line.length > 0 &&
-        shouldPublishTerminalResultLog(input.durableLogMode, line),
-    );
+}): Array<{ line: string; context: PlayLogContext | null }> {
+  if (
+    input.resultLogContexts !== undefined &&
+    input.resultLogContexts.length !== input.resultLogs.length
+  ) {
+    throw new TypeError('Terminal log contexts must align with result logs.');
+  }
+  return input.resultLogs.flatMap((rawLine, index) => {
+    const line = rawLine.trim();
+    if (
+      line.length === 0 ||
+      !shouldPublishTerminalResultLog(input.durableLogMode, line)
+    ) {
+      return [];
+    }
+    return [
+      {
+        line,
+        context: normalizePlayLogContext(input.resultLogContexts?.[index]),
+      },
+    ];
+  });
 }
 
 function terminalReplayPayloadForMode(input: {
   resultLogs: readonly string[];
+  resultLogContexts?: readonly (PlayLogContext | null)[];
   durableLogMode: PostgresSchedulerDurableLogMode;
   liveLogs: readonly string[];
+  liveLogContexts: readonly (PlayLogContext | null)[];
   liveLogsTotalCount: number;
   replayAllTerminalLogs: boolean;
   producerAttempt?: number;
 }): TerminalLogReplayPayload | null {
-  let lines = terminalReplayLinesForMode(input);
+  if (input.liveLogs.length !== input.liveLogContexts.length) {
+    throw new TypeError('Live log contexts must align with live log lines.');
+  }
+  let entries = terminalReplayEntriesForMode(input);
   // Detached recovery is a complete attempt-local buffer. Its total count
   // gives the replay an offset of zero on that attempt's positional channel;
   // producerAttempt fences stale deliveries while the ledger absorbs only an
   // exact retransmission from that same attempt.
-  let totalCount = lines.length;
+  let totalCount = entries.length;
   if (
     input.durableLogMode === 'direct_worker' &&
     !input.replayAllTerminalLogs
@@ -311,24 +336,27 @@ function terminalReplayPayloadForMode(input: {
       if (!key) continue;
       observedCounts.set(key, (observedCounts.get(key) ?? 0) + 1);
     }
-    const boundedLines: string[] = [];
-    for (const line of lines) {
-      const key = line.trim().replace(CTX_LOG_TIMESTAMP_PREFIX_RE, '');
+    const boundedEntries: typeof entries = [];
+    for (const entry of entries) {
+      const key = entry.line.trim().replace(CTX_LOG_TIMESTAMP_PREFIX_RE, '');
       const remaining = observedCounts.get(key) ?? 0;
       if (remaining > 0) {
         observedCounts.set(key, remaining - 1);
         continue;
       }
-      boundedLines.push(line);
+      boundedEntries.push(entry);
     }
-    lines = boundedLines;
-    totalCount = input.liveLogsTotalCount + lines.length;
+    entries = boundedEntries;
+    totalCount = input.liveLogsTotalCount + entries.length;
   }
-  if (lines.length === 0) {
+  if (entries.length === 0) {
     return null;
   }
+  const lines = entries.map((entry) => entry.line);
+  const lineContexts = entries.map((entry) => entry.context);
   return {
     lines,
+    ...(lineContexts.some(Boolean) ? { lineContexts } : {}),
     ...(input.replayAllTerminalLogs ? { dedupeMode: 'exact' as const } : {}),
     ...(typeof input.producerAttempt === 'number'
       ? { producerAttempt: input.producerAttempt }
@@ -385,12 +413,14 @@ export function createPostgresSchedulerProgressReporter(input: {
   let liveNodeProgress: PlayVisualNodeProgressMap = {};
   let liveNodeProgressDirty = false;
   let liveLogs: string[] = [];
+  let liveLogContexts: Array<PlayLogContext | null> = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
   let liveLogsDirty = false;
   let workCounters: RuntimeStatusUpdate['workCounters'];
   let workCountersDirty = false;
   let lastFlushAt = 0;
-  let delayedFlushTimer: ReturnType<typeof setTimeout> | null = null;
+  let lastProgressFlushAt = 0;
+  let deferredFlushTimer: ReturnType<typeof setTimeout> | null = null;
   let totalRows = 0;
   let activeMapNodeId: string | null = null;
   let pendingDatasetLifecycleEvents: Array<
@@ -403,6 +433,7 @@ export function createPostgresSchedulerProgressReporter(input: {
 
   type ProgressFlushRequest = {
     status: 'running' | 'completed' | 'failed';
+    force: boolean;
     required: boolean;
     terminalLogReplay: TerminalLogReplayPayload | null;
     waiters: Array<{
@@ -423,6 +454,7 @@ export function createPostgresSchedulerProgressReporter(input: {
       next.status !== 'running'
         ? next.status
         : (current?.status ?? next.status),
+    force: next.force || current?.force === true,
     required: next.required || current?.required === true,
     terminalLogReplay:
       next.terminalLogReplay ?? current?.terminalLogReplay ?? null,
@@ -459,17 +491,32 @@ export function createPostgresSchedulerProgressReporter(input: {
           const request = pendingProgressUpload;
           pendingProgressUpload = null;
 
-          const nextProgress = { ...liveNodeProgress };
+          // Customer log latency is independent of row-count progress sampling.
+          const includeProgress =
+            request.force ||
+            request.required ||
+            request.status !== 'running' ||
+            Date.now() - lastProgressFlushAt >=
+              getAdaptiveRunEventIntervalMs(totalRows);
+          const nextProgress = includeProgress
+            ? { ...liveNodeProgress }
+            : undefined;
           const nextLogs = liveLogs.slice();
+          const nextLogContexts = liveLogContexts.slice();
           const nextLogTotalCount = liveLogsTotalCount;
-          const nextDatasetLifecycleEvents = pendingDatasetLifecycleEvents;
-          pendingDatasetLifecycleEvents = [];
-          const nextActivityObservations = [
-            ...pendingActivityObservations.values(),
-          ];
+          const nextDatasetLifecycleEvents = includeProgress
+            ? pendingDatasetLifecycleEvents
+            : [];
+          const nextActivityObservations = includeProgress
+            ? [...pendingActivityObservations.values()]
+            : [];
           const nextWorkCounters = workCountersDirty ? workCounters : undefined;
-          pendingActivityObservations.clear();
-          liveNodeProgressDirty = false;
+          if (includeProgress) {
+            pendingDatasetLifecycleEvents = [];
+            pendingActivityObservations.clear();
+            liveNodeProgressDirty = false;
+            lastProgressFlushAt = Date.now();
+          }
           liveLogsDirty = false;
           workCountersDirty = false;
 
@@ -480,6 +527,7 @@ export function createPostgresSchedulerProgressReporter(input: {
                 status: request.status,
                 liveNodeProgress: nextProgress,
                 liveLogs: nextLogs,
+                liveLogContexts: nextLogContexts,
                 liveLogTotalCount: nextLogTotalCount,
                 liveLogProducerAttempt: producerAttempt,
                 datasetLifecycleEvents: nextDatasetLifecycleEvents,
@@ -531,30 +579,31 @@ export function createPostgresSchedulerProgressReporter(input: {
       pendingDatasetLifecycleEvents.length === 0 &&
       pendingActivityObservations.size === 0
     ) {
-      if (delayedFlushTimer) {
-        clearTimeout(delayedFlushTimer);
-        delayedFlushTimer = null;
+      if (deferredFlushTimer) {
+        clearTimeout(deferredFlushTimer);
+        deferredFlushTimer = null;
       }
       return activeProgressUpload ?? Promise.resolve();
     }
     const now = Date.now();
-    const intervalMs = getAdaptiveRunEventIntervalMs(totalRows);
-    if (!force && now - lastFlushAt < intervalMs) {
-      if (!delayedFlushTimer) {
-        delayedFlushTimer = setTimeout(
-          () => {
-            delayedFlushTimer = null;
-            void flush().catch(() => undefined);
-          },
-          Math.max(1, intervalMs - (now - lastFlushAt)),
-        );
-        delayedFlushTimer.unref?.();
-      }
+    const interval = liveLogsDirty
+      ? 1_500
+      : getAdaptiveRunEventIntervalMs(totalRows);
+    const remaining = interval - (now - lastFlushAt);
+    if (!force && remaining > 0) {
+      // Quiet runs must flush too: waiting for another event can retain one
+      // customer log until a long-running operation finally completes.
+      if (deferredFlushTimer !== null) clearTimeout(deferredFlushTimer);
+      deferredFlushTimer = setTimeout(() => {
+        deferredFlushTimer = null;
+        void flush().catch(() => undefined);
+      }, remaining);
+      deferredFlushTimer.unref?.();
       return activeProgressUpload ?? Promise.resolve();
     }
-    if (delayedFlushTimer) {
-      clearTimeout(delayedFlushTimer);
-      delayedFlushTimer = null;
+    if (deferredFlushTimer !== null) {
+      clearTimeout(deferredFlushTimer);
+      deferredFlushTimer = null;
     }
     lastFlushAt = now;
     const requested = new Promise<void>((resolve, reject) => {
@@ -562,6 +611,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         pendingProgressUpload,
         {
           status,
+          force,
           required,
           terminalLogReplay: input?.terminalLogReplay ?? null,
         },
@@ -579,15 +629,19 @@ export function createPostgresSchedulerProgressReporter(input: {
     void flush({ force }).catch(() => undefined);
   };
 
-  const appendLiveLog = (line: string) => {
+  const appendLiveLog = (line: string, context?: unknown) => {
     const trimmed = line.trim();
     if (!trimmed) {
       return;
     }
     liveLogsTotalCount += 1;
     liveLogs = [...liveLogs, trimmed];
+    liveLogContexts = [...liveLogContexts, normalizePlayLogContext(context)];
     if (liveLogs.length > LIVE_LOG_TAIL_LIMIT) {
       liveLogs = liveLogs.slice(liveLogs.length - LIVE_LOG_TAIL_LIMIT);
+      liveLogContexts = liveLogContexts.slice(
+        liveLogContexts.length - LIVE_LOG_TAIL_LIMIT,
+      );
     }
     liveLogsDirty = true;
   };
@@ -1138,13 +1192,17 @@ export function createPostgresSchedulerProgressReporter(input: {
       return;
     }
     if (type === 'log') {
-      const logEvent = event as { line?: unknown; source?: unknown };
+      const logEvent = event as {
+        line?: unknown;
+        source?: unknown;
+        context?: unknown;
+      };
       const rawLine = logEvent.line;
       if (typeof rawLine !== 'string' || !rawLine) {
         return;
       }
       if (shouldPublishDurableLogEvent(durableLogMode, logEvent)) {
-        appendLiveLog(rawLine);
+        appendLiveLog(rawLine, logEvent.context);
       }
       markMapStartedFromLog(rawLine);
       markMapStartedFromPerfLog(rawLine);
@@ -1373,7 +1431,8 @@ export function createPostgresSchedulerProgressReporter(input: {
   ): Promise<void> => {
     const terminalCounters = runtimeWorkCountersFromStats(result?.stats);
     const terminalSegment = result?.stats?.runtimeWorkCounterSegment as
-      { invocationSeq?: unknown; segmentId?: unknown } | undefined;
+      | { invocationSeq?: unknown; segmentId?: unknown }
+      | undefined;
     if (
       terminalCounters &&
       terminalCounters.provider429s !== undefined &&
@@ -1423,8 +1482,10 @@ export function createPostgresSchedulerProgressReporter(input: {
           ? null
           : terminalReplayPayloadForMode({
               resultLogs: result.logs,
+              resultLogContexts: result.logContexts,
               durableLogMode,
               liveLogs,
+              liveLogContexts,
               liveLogsTotalCount,
               replayAllTerminalLogs: options.replayAllTerminalLogs === true,
               producerAttempt:
@@ -1502,6 +1563,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       lastCheckpointAt: finalUpdate.lastCheckpointAt,
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
+      liveLogContexts: finalUpdate.liveLogContexts,
       liveLogsChannelOffset: Math.max(0, liveLogTotalCount - liveLogs.length),
       liveLogsProducerAttempt:
         finalUpdate.liveLogProducerAttempt ?? input.attempt,