deepline 0.3.290 → 0.3.291

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts

@@ -3,7 +3,10 @@ import {
   type PlayRunnerEvent,
   type PlayRunnerResult,
 } from '../protocol';
-import type { PlayLogContext } from '../log-context';
+import type {
+  PlayLogContext,
+  PlayLogMetadata,
+} from '../log-context';
 import { parseCtxFetchHttpFailureDiagnostic } from '../log-provenance';
 import {
   RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
@@ -201,9 +204,14 @@ function retainArrayWithinJsonByteLimit<T>(
  * Retain the normal newest log tail, but reserve space for the canonical
  * ctx.fetch failure diagnostics first. `runtimeDiagnostics` powers the final
  * warning, while the matching raw log line is the durable evidence customers
- * see through `runs logs --debug` after progress events could not be pushed.
+ * see through `runs logs` after progress events could not be pushed.
  */
-function retainTerminalLogTailWithRuntimeDiagnostics(logs: readonly string[]): {
+function retainTerminalLogTailWithRuntimeDiagnostics(
+  logs: readonly string[],
+  logContexts?: readonly (PlayLogContext | null)[],
+  logMetadata?: readonly (PlayLogMetadata | null)[],
+  logIds?: readonly string[],
+): {
   values: string[];
   indexes: number[];
   truncated: boolean;
@@ -215,8 +223,16 @@ function retainTerminalLogTailWithRuntimeDiagnostics(logs: readonly string[]): {
     // array. Re-serializing the retained 128 KiB tail for every historical
     // line made large completed runs exhaust their terminal-flush deadline.
     const separatorBytes = retainedIndexes.size ? 1 : 0;
-    const measurement = jsonByteLengthUpTo(
+    // The aligned arrays cross this same transport. Counting their bytes
+    // preserves the existing diagnostic allowance and customer-output headroom.
+    const entry = [
       logs[index]!,
+      ...(logContexts ? [logContexts[index] ?? null] : []),
+      ...(logMetadata ? [logMetadata[index] ?? null] : []),
+      ...(logIds ? [logIds[index]] : []),
+    ];
+    const measurement = jsonByteLengthUpTo(
+      entry.length === 1 ? entry[0] : entry,
       TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES - retainedBytes - separatorBytes,
     );
     if (measurement.exceeded) return;
@@ -270,20 +286,29 @@ function appendTerminalDiagnosticsMarker(
   truncatedFields: string[],
   logContexts?: Array<PlayLogContext | null>,
   logIds?: string[],
+  logMetadata?: Array<PlayLogMetadata | null>,
 ): {
   logs: string[];
   logContexts?: Array<PlayLogContext | null>;
   logIds?: string[];
+  logMetadata?: Array<PlayLogMetadata | null>;
 } {
   const marker =
     `[deepline] terminal diagnostic metadata was bounded (${truncatedFields.join(', ')}); ` +
     'the authored play output is complete.';
   const marked = [marker, ...logs];
   const markedContexts = logContexts ? [null, ...logContexts] : undefined;
+  const markedMetadata = logMetadata ? [null, ...logMetadata] : undefined;
   const markedIds = logIds ? [crypto.randomUUID(), ...logIds] : undefined;
+  const logPayload = () => ({
+    logs: marked,
+    ...(markedContexts ? { logContexts: markedContexts } : {}),
+    ...(markedMetadata ? { logMetadata: markedMetadata } : {}),
+    ...(markedIds ? { logIds: markedIds } : {}),
+  });
   while (
     marked.length > 1 &&
-    !fitsJsonByteLimit(marked, TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
+    !fitsJsonByteLimit(logPayload(), TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
   ) {
     // Preserve canonical ctx.fetch evidence when a marker itself needs room.
     // Other tail lines are diagnostic context, but this line is the durable
@@ -295,16 +320,20 @@ function appendTerminalDiagnosticsMarker(
     const removeAt = removableIndex === -1 ? 1 : removableIndex;
     marked.splice(removeAt, 1);
     markedContexts?.splice(removeAt, 1);
+    markedMetadata?.splice(removeAt, 1);
     markedIds?.splice(removeAt, 1);
   }
   const logsFit = fitsJsonByteLimit(
-    marked,
+    logPayload(),
     TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES,
   );
   const retainedLogs = logsFit ? marked : [marker];
   const retainedContexts = logsFit ? markedContexts : [null];
   return {
     logs: retainedLogs,
+    ...(markedMetadata?.some(Boolean)
+      ? { logMetadata: logsFit ? markedMetadata : [null] }
+      : {}),
     ...(markedIds
       ? { logIds: logsFit ? markedIds : markedIds.slice(0, 1) }
       : {}),
@@ -323,18 +352,29 @@ function appendTerminalDiagnosticsMarker(
 function boundSettledTerminalTransportDiagnostics(
   result: SettledRunnerTerminalTransportResult,
 ): SettledRunnerTerminalTransportResult {
-  const logProjection = retainTerminalLogTailWithRuntimeDiagnostics(
-    result.logs,
-  );
   if (
     result.logContexts !== undefined &&
     result.logContexts.length !== result.logs.length
   ) {
     throw new TypeError('Terminal log contexts must align with result logs.');
   }
+  if (
+    result.logMetadata !== undefined &&
+    result.logMetadata.length !== result.logs.length
+  )
+    throw new TypeError('Terminal log metadata must align with logs.');
+  const logProjection = retainTerminalLogTailWithRuntimeDiagnostics(
+    result.logs,
+    result.logContexts,
+    result.logMetadata,
+    result.logIds,
+  );
   const logContexts = result.logContexts?.length
     ? logProjection.indexes.map((index) => result.logContexts![index] ?? null)
     : undefined;
+  const logMetadata = result.logMetadata?.length
+    ? logProjection.indexes.map((index) => result.logMetadata![index] ?? null)
+    : undefined;
   if (result.logIds && result.logIds.length !== result.logs.length)
     throw new TypeError('Terminal log identities must align with result logs.');
   const logIds = result.logIds
@@ -384,16 +424,19 @@ function boundSettledTerminalTransportDiagnostics(
             truncatedFields,
             logContexts,
             logIds,
+            logMetadata,
           )
         : {
             logs: logProjection.values,
             ...(logIds ? { logIds } : {}),
             ...(logContexts?.some(Boolean) ? { logContexts } : {}),
+            ...(logMetadata?.some(Boolean) ? { logMetadata } : {}),
           };
     return {
       ...result,
       logs: boundedLogs.logs,
       logContexts: boundedLogs.logContexts,
+      logMetadata: boundedLogs.logMetadata,
       logIds: boundedLogs.logIds,
       stats: statsProjection.value,
       steps: stepsProjection.values,
@@ -427,17 +470,20 @@ function boundSettledTerminalTransportDiagnostics(
           truncatedFields,
           logContexts,
           logIds,
+          logMetadata,
         )
       : {
           logs: logProjection.values,
           ...(logIds ? { logIds } : {}),
           ...(logContexts?.some(Boolean) ? { logContexts } : {}),
+          ...(logMetadata?.some(Boolean) ? { logMetadata } : {}),
         };
   return {
     ...result,
     error: error.value,
     logs: boundedLogs.logs,
     logContexts: boundedLogs.logContexts,
+    logMetadata: boundedLogs.logMetadata,
     logIds: boundedLogs.logIds,
     stats: statsProjection.value,
     steps: stepsProjection.values,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -3040,8 +3040,14 @@ async function executeAssignedAbsurdAttempt(input: {
       });
     }
 
-    // A failed run's settled counts must reach the ledger BEFORE its terminal
-    // makes the steps immutable, so they ride the terminal outbox row.
+    // Terminal step errors and failed-map settled counts ride the existing
+    // terminal outbox row. They no longer need a progress write immediately
+    // ahead of the terminal in the same ordered run stream.
+    const observedTerminalProgressEvents =
+      progressReporter.collectTerminalFailureProgressEvents?.() ?? [];
+    if (observedTerminalProgressEvents.length > 0) {
+      progressReporter.markTerminalFailureProgressPersisted?.();
+    }
     const failedTerminalProgressEvents =
       runnerResult.status === 'failed'
         ? await collectFailedTerminalSettledProgressEvents({
@@ -3059,8 +3065,14 @@ async function executeAssignedAbsurdAttempt(input: {
           ...options,
           terminalLogOutbox: input.fromDetachedRunner ? 'ctx_log_tail' : 'omit',
           sandboxCrashDiagnostic,
-          ...(failedTerminalProgressEvents.length > 0
-            ? { failedTerminalProgressEvents }
+          ...(observedTerminalProgressEvents.length > 0 ||
+          failedTerminalProgressEvents.length > 0
+            ? {
+                terminalProgressEvents: [
+                  ...observedTerminalProgressEvents,
+                  ...failedTerminalProgressEvents,
+                ],
+              }
             : {}),
         },
       ),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -24,6 +24,8 @@ import type { PlayExecutionEvent } from '../ctx-types';
 import type { PlayActivityObservation } from '../activity-observation';
 import {
   normalizePlayLogContext,
+  normalizePlayLogMetadata,
+  type PlayLogMetadata,
   type PlayLogContext,
 } from '../log-context';
 import type {
@@ -62,6 +64,7 @@ const PLAY_TOOL_BATCH_LOG_PATTERN =
   /(?:^|\]\s*)Executing tool batch\s+([A-Za-z0-9_:-]+):\s+(\d+)\s+calls\b/i;
 const CTX_LOG_TIMESTAMP_PREFIX_RE = /^\[\d{4}-\d{2}-\d{2}T[\d:.]+Z\]\s+/;
 const LIVE_LOG_TAIL_LIMIT = 1_000;
+const STEP_FAILURE_PROGRESS_DEBOUNCE_MS = 250;
 
 function docflowInvocationSequence(invocationId?: string): number {
   if (!invocationId) return -1;
@@ -257,6 +260,11 @@ export type PostgresSchedulerProgressReporter = {
       skipTerminalLogReplay?: boolean;
     },
   ): Promise<void>;
+  /** Fold observed step failures into the terminal outbox instead of a
+   * separate, immediately-preceding progress write. */
+  collectTerminalFailureProgressEvents?(): PlayRunLedgerEvent[];
+  /** Cancel deferred failure progress once the terminal transaction owns it. */
+  markTerminalFailureProgressPersisted?(): void;
 };
 
 export type PostgresSchedulerDurableLogMode =
@@ -295,11 +303,13 @@ type TerminalLogReplayPayload = NonNullable<
 function terminalReplayEntriesForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogMetadata?: readonly (PlayLogMetadata | null)[];
   resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
 }): Array<{
   line: string;
   context: PlayLogContext | null;
+  metadata: PlayLogMetadata | null;
   entryId: string | null;
 }> {
   if (
@@ -308,6 +318,12 @@ function terminalReplayEntriesForMode(input: {
   ) {
     throw new TypeError('Terminal log contexts must align with result logs.');
   }
+  if (
+    input.resultLogMetadata !== undefined &&
+    input.resultLogMetadata.length !== input.resultLogs.length
+  ) {
+    throw new TypeError('Terminal log metadata must align with result logs.');
+  }
   if (
     input.resultLogIds &&
     input.resultLogIds.length !== input.resultLogs.length
@@ -326,6 +342,7 @@ function terminalReplayEntriesForMode(input: {
       {
         line,
         context: normalizePlayLogContext(input.resultLogContexts?.[index]),
+        metadata: normalizePlayLogMetadata(input.resultLogMetadata?.[index]),
         entryId: input.resultLogIds?.[index] ?? null,
       },
     ];
@@ -335,6 +352,7 @@ function terminalReplayEntriesForMode(input: {
 function terminalReplayPayloadForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogMetadata?: readonly (PlayLogMetadata | null)[];
   resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
   liveLogs: readonly string[];
@@ -381,9 +399,11 @@ function terminalReplayPayloadForMode(input: {
   }
   const lines = entries.map((entry) => entry.line);
   const lineContexts = entries.map((entry) => entry.context);
+  const lineMetadata = entries.map((entry) => entry.metadata);
   return {
     lines,
     ...(lineContexts.some(Boolean) ? { lineContexts } : {}),
+    ...(lineMetadata.some(Boolean) ? { lineMetadata } : {}),
     ...(input.resultLogIds
       ? { lineIds: entries.map((entry) => entry.entryId) }
       : {}),
@@ -446,6 +466,7 @@ export function createPostgresSchedulerProgressReporter(input: {
   let nodeProgressSealed = false;
   let liveLogs: string[] = [];
   let liveLogContexts: Array<PlayLogContext | null> = [];
+  let liveLogMetadata: Array<PlayLogMetadata | null> = [];
   let liveLogIds: Array<string | null> = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
   let liveLogsDirty = false;
@@ -487,6 +508,9 @@ export function createPostgresSchedulerProgressReporter(input: {
 
   let activeProgressUpload: Promise<void> | null = null;
   let pendingProgressUpload: ProgressFlushRequest | null = null;
+  let deferredStepFailureFlushTimer: ReturnType<typeof setTimeout> | null =
+    null;
+  let terminalFailureProgressPersisted = false;
 
   const mergeProgressFlushRequest = (
     current: ProgressFlushRequest | null,
@@ -549,6 +573,7 @@ export function createPostgresSchedulerProgressReporter(input: {
             : undefined;
           const nextLogs = liveLogs.slice();
           const nextLogContexts = liveLogContexts.slice();
+          const nextLogMetadata = liveLogMetadata.slice();
           const nextLogIds = liveLogIds.slice();
           const nextLogTotalCount = liveLogsTotalCount;
           const nextDatasetLifecycleEvents = includeProgress
@@ -572,9 +597,42 @@ export function createPostgresSchedulerProgressReporter(input: {
               await publishStatusUpdate({
                 playId: launch.runId,
                 status: request.status,
-                liveNodeProgress: nodeProgressSealed ? {} : nextProgress,
+                // On failure only observed causal errors may precede the
+                // terminal. Partial map counters cannot prove completion.
+                liveNodeProgress: nodeProgressSealed
+                  ? terminalFailureProgressPersisted
+                    ? {}
+                    : Object.fromEntries(
+                        Object.entries(nextProgress ?? {})
+                          .filter(
+                            ([, progress]) =>
+                              progress.error || progress.nodeIo?.error,
+                          )
+                          .map(([id, progress]) => [
+                            id,
+                            {
+                              startedAt: progress.startedAt,
+                              completedAt: progress.completedAt,
+                              updatedAt: progress.updatedAt,
+                              error:
+                                progress.error ??
+                                progress.nodeIo?.error ??
+                                undefined,
+                              nodeIo: progress.nodeIo,
+                            },
+                          ]),
+                      )
+                  : terminalFailureProgressPersisted
+                    ? Object.fromEntries(
+                        Object.entries(nextProgress ?? {}).filter(
+                          ([, progress]) =>
+                            !progress.error && !progress.nodeIo?.error,
+                        ),
+                      )
+                    : nextProgress,
                 liveLogs: nextLogs,
                 liveLogContexts: nextLogContexts,
+                liveLogMetadata: nextLogMetadata,
                 liveLogIds: nextLogIds,
                 liveLogTotalCount: nextLogTotalCount,
                 ...(input.initialLiveLogTotalCount !== undefined
@@ -684,6 +742,7 @@ export function createPostgresSchedulerProgressReporter(input: {
     line: string,
     context?: unknown,
     entryId?: unknown,
+    metadata?: unknown,
   ) => {
     const trimmed = line.trim();
     if (!trimmed) {
@@ -692,9 +751,11 @@ export function createPostgresSchedulerProgressReporter(input: {
     liveLogsTotalCount += 1;
     liveLogs = [...liveLogs, trimmed];
     liveLogContexts = [...liveLogContexts, normalizePlayLogContext(context)];
+    liveLogMetadata = [...liveLogMetadata, normalizePlayLogMetadata(metadata)];
     liveLogIds = [...liveLogIds, typeof entryId === 'string' ? entryId : null];
     if (liveLogs.length > LIVE_LOG_TAIL_LIMIT) {
       liveLogs = liveLogs.slice(liveLogs.length - LIVE_LOG_TAIL_LIMIT);
+      liveLogMetadata = liveLogMetadata.slice(-LIVE_LOG_TAIL_LIMIT);
       liveLogIds = liveLogIds.slice(liveLogIds.length - LIVE_LOG_TAIL_LIMIT);
       liveLogContexts = liveLogContexts.slice(
         liveLogContexts.length - LIVE_LOG_TAIL_LIMIT,
@@ -761,6 +822,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(current.completedAt !== undefined && !clearCompletedAt
           ? { completedAt: current.completedAt }
           : {}),
+        ...(current.error !== undefined ? { error: current.error } : {}),
         ...(current.nodeIo !== undefined ? { nodeIo: current.nodeIo } : {}),
         ...(current.conditionCounts !== undefined
           ? { conditionCounts: current.conditionCounts }
@@ -810,6 +872,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(patch.completedAt !== undefined
           ? { completedAt: patch.completedAt }
           : {}),
+        ...(patch.error !== undefined ? { error: patch.error } : {}),
         ...(patch.nodeIo !== undefined ? { nodeIo: patch.nodeIo } : {}),
         ...(patch.conditionCounts !== undefined
           ? { conditionCounts: patch.conditionCounts }
@@ -994,6 +1057,34 @@ export function createPostgresSchedulerProgressReporter(input: {
   };
 
   const applyExecutionEvent = (event: PlayExecutionEvent) => {
+    if (
+      event.type === 'step.started' ||
+      event.type === 'step.completed' ||
+      event.type === 'step.failed'
+    ) {
+      setNodeProgress(event.stepId, {
+        ...(event.type === 'step.started'
+          ? { startedAt: event.at }
+          : { completedAt: event.at }),
+        ...(event.type === 'step.failed' ? { error: event.error } : {}),
+      });
+      if (event.type === 'step.failed') {
+        // A failing step is often followed immediately by run.failed. Give
+        // that terminal transaction a chance to carry the error in its own
+        // ordered outbox row, avoiding a redundant Convex append ahead of it.
+        if (deferredStepFailureFlushTimer !== null) {
+          clearTimeout(deferredStepFailureFlushTimer);
+        }
+        deferredStepFailureFlushTimer = setTimeout(() => {
+          deferredStepFailureFlushTimer = null;
+          if (!terminalFailureProgressPersisted) scheduleFlush(true);
+        }, STEP_FAILURE_PROGRESS_DEBOUNCE_MS);
+        deferredStepFailureFlushTimer.unref?.();
+      } else {
+        scheduleFlush(true);
+      }
+      return;
+    }
     if (event.type === 'docflow.node.hit') {
       setNodeProgress(event.nodeId, {
         startedAt: event.at,
@@ -1340,6 +1431,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         line?: unknown;
         source?: unknown;
         context?: unknown;
+        metadata?: unknown;
         entryId?: unknown;
       };
       const rawLine = logEvent.line;
@@ -1347,7 +1439,12 @@ export function createPostgresSchedulerProgressReporter(input: {
         return;
       }
       if (shouldPublishDurableLogEvent(durableLogMode, logEvent)) {
-        appendLiveLog(rawLine, logEvent.context, logEvent.entryId);
+        appendLiveLog(
+          rawLine,
+          logEvent.context,
+          logEvent.entryId,
+          logEvent.metadata,
+        );
       }
       markMapStartedFromLog(rawLine);
       markMapStartedFromPerfLog(rawLine);
@@ -1613,13 +1710,14 @@ export function createPostgresSchedulerProgressReporter(input: {
     // A failed result's terminal is already decided: every caller that
     // finalizes one does so after (or while) the scheduler commits
     // `run.failed`, and the ledger freezes a terminal run's steps. Its step
-    // facts from here are projections of this reporter's own partial view — a
+    // counts from here are projections of this reporter's own partial view — a
     // parked worker's fold saw page-sized map totals and never saw the
     // runner's failures — so they used to land after `run.failed` claiming
     // docflow nodes were "running" and a 50,000-row map had "completed" 2,000.
     // The settled counts that ARE true travel in the terminal outbox row
     // (`buildFailedTerminalSettledProgressEvents`). Logs, work counters,
-    // dataset and activity facts still flush.
+    // dataset and activity still flush; causal step failures already ride the
+    // terminal outbox row when one was committed.
     if (result?.status === 'failed') {
       nodeProgressSealed = true;
     } else if (result?.status === 'completed') {
@@ -1649,6 +1747,7 @@ export function createPostgresSchedulerProgressReporter(input: {
           : terminalReplayPayloadForMode({
               resultLogs: result.logs,
               resultLogContexts: result.logContexts,
+              resultLogMetadata: result.logMetadata,
               resultLogIds: result.logIds,
               durableLogMode,
               liveLogs,
@@ -1661,11 +1760,47 @@ export function createPostgresSchedulerProgressReporter(input: {
     });
   };
 
+  const collectTerminalFailureProgressEvents = (): PlayRunLedgerEvent[] => {
+    const failedProgress = Object.fromEntries(
+      Object.entries(liveNodeProgress).filter(
+        ([, progress]) => progress.error || progress.nodeIo?.error,
+      ),
+    );
+    if (Object.keys(failedProgress).length === 0) return [];
+    return buildPlayRunLedgerEventsFromStatusPatch({
+      patch: {
+        runId: launch.runId,
+        status: 'running',
+        lastCheckpointAt: Date.now(),
+        liveNodeProgress: failedProgress,
+      },
+      previousSnapshot: createEmptyPlayRunLedgerSnapshot({
+        runId: launch.runId,
+        status: 'queued',
+      }),
+      source: 'worker',
+    }).filter(
+      (event) =>
+        event.type === RUN_EVENT_KIND.stepProgress ||
+        event.type === RUN_EVENT_KIND.stepFailed,
+    );
+  };
+
+  const markTerminalFailureProgressPersisted = (): void => {
+    terminalFailureProgressPersisted = true;
+    if (deferredStepFailureFlushTimer !== null) {
+      clearTimeout(deferredStepFailureFlushTimer);
+      deferredStepFailureFlushTimer = null;
+    }
+  };
+
   return {
     onRunnerEvent,
     applyDurableObservations,
     applyTerminalSheetOutcomes,
     finalize,
+    collectTerminalFailureProgressEvents,
+    markTerminalFailureProgressPersisted,
   };
 }
 
@@ -1853,6 +1988,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
       liveLogContexts: finalUpdate.liveLogContexts,
+      liveLogMetadata: finalUpdate.liveLogMetadata,
       liveLogIds: finalUpdate.liveLogIds,
       liveLogsChannelOffset: Math.max(0, liveLogTotalCount - liveLogs.length),
       liveLogsProducerAttempt: