deepline 0.3.289 → 0.3.291

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -11,7 +11,6 @@ import { RUN_EVENT_KIND } from '../run-event-catalog';
  * absurd-specific wiring (queue, event names, checkpoint boundary shaping).
  */
 import { createHash } from 'node:crypto';
-import { AsyncLocalStorage } from 'node:async_hooks';
 import type {
   PlayCheckpoint,
   PlayExecutionEvent,
@@ -22,11 +21,6 @@ import type { PlayRunLedgerEvent } from '../run-ledger';
 import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
 import type { WorkerRuntimeApiContext } from '../app-runtime-api';
 import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
-import {
-  PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
-  PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES,
-  PLAY_ADMISSION_MAX_PARALLELISM,
-} from '../admission-transport-policy';
 import { assertKnownResolvedBoundary } from '../checkpoint-boundaries';
 import type { Absurd, Log, RetryStrategy } from '@deepline/absurd/sdk';
 import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
@@ -37,8 +31,6 @@ import {
 } from './postgres';
 import { createAbsurdForRuntimeScheduler } from './absurd-pool';
 import {
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS,
-  resolveWorkflowRuntimePostgresUrl,
   withRuntimeSchedulerClient,
   type RuntimeSchedulerPoolOptions,
 } from './postgres-pool';
@@ -548,137 +540,14 @@ export type AbsurdRuntimeResult = {
   error?: string;
 };
 
-// All queues in one worker share this single process pool. Scheduler queries
-// are short and batched; execution concurrency belongs to Daytona/Fly slots,
-// not to a growing set of database clients. Keeping this at two makes a
-// horizontally scaled fleet predictable against the shared PgBouncer budget.
-export const ABSURD_RUNTIME_POOL_MAX_CLIENTS =
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
-
-/**
- * Vercel Fluid invocations share a process and its scheduler pool. A single
- * client serializes unrelated launches, reads, and critical lease lookups;
- * one held connection can make every other request hit its acquire timeout.
- * Use the shared two-client control-plane budget so another request can make
- * progress while keeping each instance's PgBouncer footprint bounded.
- */
-export const ABSURD_SERVERLESS_POOL_MAX_CLIENTS =
-  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
-
-/**
- * A Vercel isolate may survive long after a webhook has returned. Keep its
- * scheduler clients only long enough to reuse them for an immediate follow-up,
- * then return idle PgBouncer clients before the source's delayed Workpool
- * recovery is due. The worker fleet keeps the ordinary 30-second idle lifetime.
- */
-export const ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS = 1_000;
-
-// Ordinary Vercel requests share the bounded control-plane pool above. The
-// private batch adapter is different: Workpool elects one global up-to-1,024-
-// source collection before entering this context, so one batch request may use
-// its larger bounded scheduler capacity without multiplying it per source.
-// Other Workpool adapters can still overlap during rollout.
-type ServerlessAdmissionPoolContext = {
-  maxClients: number;
-  maxAcquireQueueDepth: number;
-  maxAcquireQueueWaitMs: number;
-};
-
-const serverlessAdmissionPoolContext =
-  new AsyncLocalStorage<ServerlessAdmissionPoolContext>();
-
-export async function withServerlessAdmissionPoolMaxClients<T>(input: {
-  maxClients: number;
-  maxAcquireQueueDepth?: number;
-  maxAcquireQueueWaitMs?: number;
-  fn: () => Promise<T>;
-}): Promise<T> {
-  if (
-    !Number.isInteger(input.maxClients) ||
-    input.maxClients < 1 ||
-    input.maxClients > PLAY_ADMISSION_MAX_PARALLELISM
-  ) {
-    throw new Error(
-      'Serverless admission pool capacity is outside the supported range.',
-    );
-  }
-  const maxAcquireQueueDepth =
-    input.maxAcquireQueueDepth ?? PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES;
-  const maxAcquireQueueWaitMs =
-    input.maxAcquireQueueWaitMs ?? PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS;
-  if (
-    !Number.isSafeInteger(maxAcquireQueueDepth) ||
-    maxAcquireQueueDepth < 1 ||
-    maxAcquireQueueDepth > PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES
-  ) {
-    throw new Error(
-      'Serverless admission acquire queue capacity is outside the supported range.',
-    );
-  }
-  if (
-    !Number.isSafeInteger(maxAcquireQueueWaitMs) ||
-    maxAcquireQueueWaitMs < 5_000 ||
-    maxAcquireQueueWaitMs > PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS
-  ) {
-    throw new Error(
-      'Serverless admission acquire queue wait is outside the supported range.',
-    );
-  }
-  return await serverlessAdmissionPoolContext.run(
-    {
-      maxClients: input.maxClients,
-      maxAcquireQueueDepth,
-      maxAcquireQueueWaitMs,
-    },
-    input.fn,
-  );
-}
-
-/**
- * The private batch adapter owns this request-local scheduler pool. Reserve
- * one client for executor-token and other critical control-plane traffic; the
- * rest may submit the independently idempotent batch sources in parallel.
- */
-export function serverlessAdmissionSubmitMaxActive(): number | null {
-  const context = serverlessAdmissionPoolContext.getStore();
-  return context === undefined ? null : Math.max(1, context.maxClients - 1);
-}
-
-/**
- * Resolve the runtime scheduler Postgres pool options for the absurd control
- * plane. Loud failure over silent fallback: the absurd worker and scheduler
- * only run under Node where `WORKFLOW_RUNTIME_POSTGRES` is configured.
- */
-export function resolveAbsurdPoolOptions(
-  explicitUrl?: string | null,
-): RuntimeSchedulerPoolOptions {
-  const url = explicitUrl?.trim() || resolveWorkflowRuntimePostgresUrl();
-  if (!url) {
-    throw new Error(
-      'absurd scheduler backend requires WORKFLOW_RUNTIME_POSTGRES (runtime ' +
-        'scheduler Postgres URL). Refusing to fall back to local execution.',
-    );
-  }
-  const serverless = process.env.VERCEL?.trim();
-  const context = serverlessAdmissionPoolContext.getStore();
-  return {
-    url,
-    applicationName: 'deepline-absurd-runtime',
-    maxClients:
-      serverless || context !== undefined
-        ? (context?.maxClients ?? ABSURD_SERVERLESS_POOL_MAX_CLIENTS)
-        : ABSURD_RUNTIME_POOL_MAX_CLIENTS,
-    ...(serverless || context !== undefined
-      ? { idleTimeoutMs: ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS }
-      : {}),
-    ...(context
-      ? {
-          maxAcquireQueueDepth: context.maxAcquireQueueDepth,
-          maxAcquireQueueWaitMs: context.maxAcquireQueueWaitMs,
-        }
-      : {}),
-  };
-}
+export {
+  ABSURD_RUNTIME_POOL_MAX_CLIENTS,
+  ABSURD_SERVERLESS_POOL_MAX_CLIENTS,
+  ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS,
+  withServerlessAdmissionPoolMaxClients,
+  serverlessAdmissionSubmitMaxActive,
+  resolveAbsurdPoolOptions,
+} from './absurd-pool-options';
 
 export function normalizeAbsurdSchedulerOptions(
   options: { url?: string | null; schema?: string | null } = {},

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -22,6 +22,7 @@
  * connection. Notifications are strictly a wakeup hint — the tables remain
  * the source of truth and every waiter still reads the DB.
  */
+import { attachDatabasePool } from '@vercel/functions/db-connections';
 import { Client, Pool } from 'pg';
 import {
   RuntimeSchedulerPgBouncerCapacityError,
@@ -175,6 +176,8 @@ export const RUNTIME_SCHEDULER_POOL_DEFAULTS = {
   keepAliveIntervalMs: 60_000,
 } as const;
 
+export const RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS = 1_000;
+
 /**
  * A control-plane process only needs a few short scheduler transactions at
  * once. This shared cap keeps horizontally scaled workers and the receipt
@@ -197,8 +200,8 @@ export type RuntimeSchedulerPoolOptions = {
   maxClients?: number;
   /**
    * Serverless launchers must not pin a scarce PgBouncer client between
-   * requests. Long-lived workers retain the default; Vercel callers opt into
-   * a short idle lifetime through the shared Absurd pool resolver.
+   * requests. Vercel pools share one short idle lifetime; this override only
+   * applies to long-lived processes.
    */
   idleTimeoutMs?: number;
   /**
@@ -447,6 +450,7 @@ export class RuntimeSchedulerPoolAdmission {
 
 type PoolEntry = {
   pool: Pool;
+  databasePoolAttached?: boolean;
   maxClients: number;
   maxAcquireQueueDepth: number;
   admission: RuntimeSchedulerPoolAdmission;
@@ -723,7 +727,7 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
   assertRuntimeSchedulerHotPathUrl(options.url);
   const maxClients =
     options.maxClients ?? RUNTIME_SCHEDULER_POOL_DEFAULTS.maxClients;
-  const idleTimeoutMs =
+  const requestedIdleTimeoutMs =
     options.idleTimeoutMs ?? RUNTIME_SCHEDULER_POOL_DEFAULTS.idleTimeoutMs;
   const maxAcquireQueueDepth =
     options.maxAcquireQueueDepth ??
@@ -738,11 +742,17 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
         'instead of widening a single process pool.',
     );
   }
-  if (!Number.isSafeInteger(idleTimeoutMs) || idleTimeoutMs < 0) {
+  if (
+    !Number.isSafeInteger(requestedIdleTimeoutMs) ||
+    requestedIdleTimeoutMs < 0
+  ) {
     throw new Error(
-      `Runtime scheduler pool idleTimeoutMs=${idleTimeoutMs} must be a non-negative integer.`,
+      `Runtime scheduler pool idleTimeoutMs=${requestedIdleTimeoutMs} must be a non-negative integer.`,
     );
   }
+  const idleTimeoutMs = process.env.VERCEL?.trim()
+    ? RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS
+    : requestedIdleTimeoutMs;
   if (
     !Number.isSafeInteger(maxAcquireQueueDepth) ||
     maxAcquireQueueDepth < 1 ||
@@ -1049,6 +1059,10 @@ export async function withRuntimeSchedulerClient<T>(
         let queryCount = 0;
         let queryMs = 0;
         try {
+          if (!entry.databasePoolAttached) {
+            attachDatabasePool(entry.pool);
+            entry.databasePoolAttached = true;
+          }
           throwIfSchedulerAborted(signal);
           const result = await fn({
             query: async (sql, params) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -14,6 +14,7 @@ import { RUN_EVENT_KIND } from '../run-event-catalog';
  * Runtime Sheet row durability belongs to the context-level row writer.
  */
 import { PlayStepLifecycleTracker } from '../step-lifecycle-tracker';
+import { isDurableDirectRunnerLog } from '../log-provenance';
 import {
   updateRunStatusViaAppRuntime,
   type RuntimeStatusUpdate,
@@ -23,6 +24,8 @@ import type { PlayExecutionEvent } from '../ctx-types';
 import type { PlayActivityObservation } from '../activity-observation';
 import {
   normalizePlayLogContext,
+  normalizePlayLogMetadata,
+  type PlayLogMetadata,
   type PlayLogContext,
 } from '../log-context';
 import type {
@@ -61,6 +64,7 @@ const PLAY_TOOL_BATCH_LOG_PATTERN =
   /(?:^|\]\s*)Executing tool batch\s+([A-Za-z0-9_:-]+):\s+(\d+)\s+calls\b/i;
 const CTX_LOG_TIMESTAMP_PREFIX_RE = /^\[\d{4}-\d{2}-\d{2}T[\d:.]+Z\]\s+/;
 const LIVE_LOG_TAIL_LIMIT = 1_000;
+const STEP_FAILURE_PROGRESS_DEBOUNCE_MS = 250;
 
 function docflowInvocationSequence(invocationId?: string): number {
   if (!invocationId) return -1;
@@ -256,6 +260,11 @@ export type PostgresSchedulerProgressReporter = {
       skipTerminalLogReplay?: boolean;
     },
   ): Promise<void>;
+  /** Fold observed step failures into the terminal outbox instead of a
+   * separate, immediately-preceding progress write. */
+  collectTerminalFailureProgressEvents?(): PlayRunLedgerEvent[];
+  /** Cancel deferred failure progress once the terminal transaction owns it. */
+  markTerminalFailureProgressPersisted?(): void;
 };
 
 export type PostgresSchedulerDurableLogMode =
@@ -268,18 +277,7 @@ function shouldPublishDurableLogEvent(
     | Pick<PlayRunnerLogEvent, 'source' | 'line'>
     | { source?: unknown; line?: unknown },
 ): boolean {
-  // Daytona host lifecycle logs use source "worker" (sandbox
-  // create/upload/execute). Keep those in worker logs while still storing
-  // play-authored ctx.log events in the durable Run Log Stream. The one
-  // exception is the structured performance marker: it is bounded (one
-  // terminal phase observation per runner phase), run-scoped, and is the
-  // canonical black-box regression signal for dispatcher throughput. Do not
-  // broaden this to arbitrary worker output.
-  return (
-    mode !== 'direct_worker' ||
-    event.source !== 'worker' ||
-    (typeof event.line === 'string' && event.line.startsWith('[perf-trace] '))
-  );
+  return mode !== 'direct_worker' || isDurableDirectRunnerLog(event);
 }
 
 function shouldPublishTerminalResultLog(
@@ -305,14 +303,33 @@ type TerminalLogReplayPayload = NonNullable<
 function terminalReplayEntriesForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogMetadata?: readonly (PlayLogMetadata | null)[];
+  resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
-}): Array<{ line: string; context: PlayLogContext | null }> {
+}): Array<{
+  line: string;
+  context: PlayLogContext | null;
+  metadata: PlayLogMetadata | null;
+  entryId: string | null;
+}> {
   if (
     input.resultLogContexts !== undefined &&
     input.resultLogContexts.length !== input.resultLogs.length
   ) {
     throw new TypeError('Terminal log contexts must align with result logs.');
   }
+  if (
+    input.resultLogMetadata !== undefined &&
+    input.resultLogMetadata.length !== input.resultLogs.length
+  ) {
+    throw new TypeError('Terminal log metadata must align with result logs.');
+  }
+  if (
+    input.resultLogIds &&
+    input.resultLogIds.length !== input.resultLogs.length
+  ) {
+    throw new TypeError('Terminal log identities must align with result logs.');
+  }
   return input.resultLogs.flatMap((rawLine, index) => {
     const line = rawLine.trim();
     if (
@@ -325,6 +342,8 @@ function terminalReplayEntriesForMode(input: {
       {
         line,
         context: normalizePlayLogContext(input.resultLogContexts?.[index]),
+        metadata: normalizePlayLogMetadata(input.resultLogMetadata?.[index]),
+        entryId: input.resultLogIds?.[index] ?? null,
       },
     ];
   });
@@ -333,6 +352,8 @@ function terminalReplayEntriesForMode(input: {
 function terminalReplayPayloadForMode(input: {
   resultLogs: readonly string[];
   resultLogContexts?: readonly (PlayLogContext | null)[];
+  resultLogMetadata?: readonly (PlayLogMetadata | null)[];
+  resultLogIds?: readonly string[];
   durableLogMode: PostgresSchedulerDurableLogMode;
   liveLogs: readonly string[];
   liveLogContexts: readonly (PlayLogContext | null)[];
@@ -351,6 +372,7 @@ function terminalReplayPayloadForMode(input: {
   let totalCount = entries.length;
   if (
     input.durableLogMode === 'direct_worker' &&
+    !input.resultLogIds &&
     !input.replayAllTerminalLogs
   ) {
     const observedCounts = new Map<string, number>();
@@ -377,9 +399,14 @@ function terminalReplayPayloadForMode(input: {
   }
   const lines = entries.map((entry) => entry.line);
   const lineContexts = entries.map((entry) => entry.context);
+  const lineMetadata = entries.map((entry) => entry.metadata);
   return {
     lines,
     ...(lineContexts.some(Boolean) ? { lineContexts } : {}),
+    ...(lineMetadata.some(Boolean) ? { lineMetadata } : {}),
+    ...(input.resultLogIds
+      ? { lineIds: entries.map((entry) => entry.entryId) }
+      : {}),
     ...(input.replayAllTerminalLogs ? { dedupeMode: 'exact' as const } : {}),
     ...(typeof input.producerAttempt === 'number'
       ? { producerAttempt: input.producerAttempt }
@@ -439,6 +466,8 @@ export function createPostgresSchedulerProgressReporter(input: {
   let nodeProgressSealed = false;
   let liveLogs: string[] = [];
   let liveLogContexts: Array<PlayLogContext | null> = [];
+  let liveLogMetadata: Array<PlayLogMetadata | null> = [];
+  let liveLogIds: Array<string | null> = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
   let liveLogsDirty = false;
   let workCounters: RuntimeStatusUpdate['workCounters'];
@@ -479,6 +508,9 @@ export function createPostgresSchedulerProgressReporter(input: {
 
   let activeProgressUpload: Promise<void> | null = null;
   let pendingProgressUpload: ProgressFlushRequest | null = null;
+  let deferredStepFailureFlushTimer: ReturnType<typeof setTimeout> | null =
+    null;
+  let terminalFailureProgressPersisted = false;
 
   const mergeProgressFlushRequest = (
     current: ProgressFlushRequest | null,
@@ -541,6 +573,8 @@ export function createPostgresSchedulerProgressReporter(input: {
             : undefined;
           const nextLogs = liveLogs.slice();
           const nextLogContexts = liveLogContexts.slice();
+          const nextLogMetadata = liveLogMetadata.slice();
+          const nextLogIds = liveLogIds.slice();
           const nextLogTotalCount = liveLogsTotalCount;
           const nextDatasetLifecycleEvents = includeProgress
             ? pendingDatasetLifecycleEvents
@@ -563,10 +597,47 @@ export function createPostgresSchedulerProgressReporter(input: {
               await publishStatusUpdate({
                 playId: launch.runId,
                 status: request.status,
-                liveNodeProgress: nodeProgressSealed ? {} : nextProgress,
+                // On failure only observed causal errors may precede the
+                // terminal. Partial map counters cannot prove completion.
+                liveNodeProgress: nodeProgressSealed
+                  ? terminalFailureProgressPersisted
+                    ? {}
+                    : Object.fromEntries(
+                        Object.entries(nextProgress ?? {})
+                          .filter(
+                            ([, progress]) =>
+                              progress.error || progress.nodeIo?.error,
+                          )
+                          .map(([id, progress]) => [
+                            id,
+                            {
+                              startedAt: progress.startedAt,
+                              completedAt: progress.completedAt,
+                              updatedAt: progress.updatedAt,
+                              error:
+                                progress.error ??
+                                progress.nodeIo?.error ??
+                                undefined,
+                              nodeIo: progress.nodeIo,
+                            },
+                          ]),
+                      )
+                  : terminalFailureProgressPersisted
+                    ? Object.fromEntries(
+                        Object.entries(nextProgress ?? {}).filter(
+                          ([, progress]) =>
+                            !progress.error && !progress.nodeIo?.error,
+                        ),
+                      )
+                    : nextProgress,
                 liveLogs: nextLogs,
                 liveLogContexts: nextLogContexts,
+                liveLogMetadata: nextLogMetadata,
+                liveLogIds: nextLogIds,
                 liveLogTotalCount: nextLogTotalCount,
+                ...(input.initialLiveLogTotalCount !== undefined
+                  ? { liveLogBatchOffset: nextLogTotalCount - nextLogs.length }
+                  : {}),
                 liveLogProducerAttempt: producerAttempt,
                 datasetLifecycleEvents: nextDatasetLifecycleEvents,
                 activityObservations: nextActivityObservations,
@@ -667,7 +738,12 @@ export function createPostgresSchedulerProgressReporter(input: {
     void flush({ force }).catch(() => undefined);
   };
 
-  const appendLiveLog = (line: string, context?: unknown) => {
+  const appendLiveLog = (
+    line: string,
+    context?: unknown,
+    entryId?: unknown,
+    metadata?: unknown,
+  ) => {
     const trimmed = line.trim();
     if (!trimmed) {
       return;
@@ -675,8 +751,12 @@ export function createPostgresSchedulerProgressReporter(input: {
     liveLogsTotalCount += 1;
     liveLogs = [...liveLogs, trimmed];
     liveLogContexts = [...liveLogContexts, normalizePlayLogContext(context)];
+    liveLogMetadata = [...liveLogMetadata, normalizePlayLogMetadata(metadata)];
+    liveLogIds = [...liveLogIds, typeof entryId === 'string' ? entryId : null];
     if (liveLogs.length > LIVE_LOG_TAIL_LIMIT) {
       liveLogs = liveLogs.slice(liveLogs.length - LIVE_LOG_TAIL_LIMIT);
+      liveLogMetadata = liveLogMetadata.slice(-LIVE_LOG_TAIL_LIMIT);
+      liveLogIds = liveLogIds.slice(liveLogIds.length - LIVE_LOG_TAIL_LIMIT);
       liveLogContexts = liveLogContexts.slice(
         liveLogContexts.length - LIVE_LOG_TAIL_LIMIT,
       );
@@ -742,6 +822,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(current.completedAt !== undefined && !clearCompletedAt
           ? { completedAt: current.completedAt }
           : {}),
+        ...(current.error !== undefined ? { error: current.error } : {}),
         ...(current.nodeIo !== undefined ? { nodeIo: current.nodeIo } : {}),
         ...(current.conditionCounts !== undefined
           ? { conditionCounts: current.conditionCounts }
@@ -791,6 +872,7 @@ export function createPostgresSchedulerProgressReporter(input: {
         ...(patch.completedAt !== undefined
           ? { completedAt: patch.completedAt }
           : {}),
+        ...(patch.error !== undefined ? { error: patch.error } : {}),
         ...(patch.nodeIo !== undefined ? { nodeIo: patch.nodeIo } : {}),
         ...(patch.conditionCounts !== undefined
           ? { conditionCounts: patch.conditionCounts }
@@ -975,6 +1057,34 @@ export function createPostgresSchedulerProgressReporter(input: {
   };
 
   const applyExecutionEvent = (event: PlayExecutionEvent) => {
+    if (
+      event.type === 'step.started' ||
+      event.type === 'step.completed' ||
+      event.type === 'step.failed'
+    ) {
+      setNodeProgress(event.stepId, {
+        ...(event.type === 'step.started'
+          ? { startedAt: event.at }
+          : { completedAt: event.at }),
+        ...(event.type === 'step.failed' ? { error: event.error } : {}),
+      });
+      if (event.type === 'step.failed') {
+        // A failing step is often followed immediately by run.failed. Give
+        // that terminal transaction a chance to carry the error in its own
+        // ordered outbox row, avoiding a redundant Convex append ahead of it.
+        if (deferredStepFailureFlushTimer !== null) {
+          clearTimeout(deferredStepFailureFlushTimer);
+        }
+        deferredStepFailureFlushTimer = setTimeout(() => {
+          deferredStepFailureFlushTimer = null;
+          if (!terminalFailureProgressPersisted) scheduleFlush(true);
+        }, STEP_FAILURE_PROGRESS_DEBOUNCE_MS);
+        deferredStepFailureFlushTimer.unref?.();
+      } else {
+        scheduleFlush(true);
+      }
+      return;
+    }
     if (event.type === 'docflow.node.hit') {
       setNodeProgress(event.nodeId, {
         startedAt: event.at,
@@ -1321,13 +1431,20 @@ export function createPostgresSchedulerProgressReporter(input: {
         line?: unknown;
         source?: unknown;
         context?: unknown;
+        metadata?: unknown;
+        entryId?: unknown;
       };
       const rawLine = logEvent.line;
       if (typeof rawLine !== 'string' || !rawLine) {
         return;
       }
       if (shouldPublishDurableLogEvent(durableLogMode, logEvent)) {
-        appendLiveLog(rawLine, logEvent.context);
+        appendLiveLog(
+          rawLine,
+          logEvent.context,
+          logEvent.entryId,
+          logEvent.metadata,
+        );
       }
       markMapStartedFromLog(rawLine);
       markMapStartedFromPerfLog(rawLine);
@@ -1593,13 +1710,14 @@ export function createPostgresSchedulerProgressReporter(input: {
     // A failed result's terminal is already decided: every caller that
     // finalizes one does so after (or while) the scheduler commits
     // `run.failed`, and the ledger freezes a terminal run's steps. Its step
-    // facts from here are projections of this reporter's own partial view — a
+    // counts from here are projections of this reporter's own partial view — a
     // parked worker's fold saw page-sized map totals and never saw the
     // runner's failures — so they used to land after `run.failed` claiming
     // docflow nodes were "running" and a 50,000-row map had "completed" 2,000.
     // The settled counts that ARE true travel in the terminal outbox row
     // (`buildFailedTerminalSettledProgressEvents`). Logs, work counters,
-    // dataset and activity facts still flush.
+    // dataset and activity still flush; causal step failures already ride the
+    // terminal outbox row when one was committed.
     if (result?.status === 'failed') {
       nodeProgressSealed = true;
     } else if (result?.status === 'completed') {
@@ -1629,6 +1747,8 @@ export function createPostgresSchedulerProgressReporter(input: {
           : terminalReplayPayloadForMode({
               resultLogs: result.logs,
               resultLogContexts: result.logContexts,
+              resultLogMetadata: result.logMetadata,
+              resultLogIds: result.logIds,
               durableLogMode,
               liveLogs,
               liveLogContexts,
@@ -1640,11 +1760,47 @@ export function createPostgresSchedulerProgressReporter(input: {
     });
   };
 
+  const collectTerminalFailureProgressEvents = (): PlayRunLedgerEvent[] => {
+    const failedProgress = Object.fromEntries(
+      Object.entries(liveNodeProgress).filter(
+        ([, progress]) => progress.error || progress.nodeIo?.error,
+      ),
+    );
+    if (Object.keys(failedProgress).length === 0) return [];
+    return buildPlayRunLedgerEventsFromStatusPatch({
+      patch: {
+        runId: launch.runId,
+        status: 'running',
+        lastCheckpointAt: Date.now(),
+        liveNodeProgress: failedProgress,
+      },
+      previousSnapshot: createEmptyPlayRunLedgerSnapshot({
+        runId: launch.runId,
+        status: 'queued',
+      }),
+      source: 'worker',
+    }).filter(
+      (event) =>
+        event.type === RUN_EVENT_KIND.stepProgress ||
+        event.type === RUN_EVENT_KIND.stepFailed,
+    );
+  };
+
+  const markTerminalFailureProgressPersisted = (): void => {
+    terminalFailureProgressPersisted = true;
+    if (deferredStepFailureFlushTimer !== null) {
+      clearTimeout(deferredStepFailureFlushTimer);
+      deferredStepFailureFlushTimer = null;
+    }
+  };
+
   return {
     onRunnerEvent,
     applyDurableObservations,
     applyTerminalSheetOutcomes,
     finalize,
+    collectTerminalFailureProgressEvents,
+    markTerminalFailureProgressPersisted,
   };
 }
 
@@ -1832,6 +1988,8 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
       liveLogContexts: finalUpdate.liveLogContexts,
+      liveLogMetadata: finalUpdate.liveLogMetadata,
+      liveLogIds: finalUpdate.liveLogIds,
       liveLogsChannelOffset: Math.max(0, liveLogTotalCount - liveLogs.length),
       liveLogsProducerAttempt:
         finalUpdate.liveLogProducerAttempt ?? input.attempt,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -10,6 +10,7 @@ import type {
 import {
   DEFAULT_ADVISORY_PROVIDER_MAX_RPS,
   DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+  DEFAULT_UNKNOWN_PROVIDER_MIN_RPS,
   PROVIDER_AIMD_COOLDOWN_CAP_MS,
   type PacingRule,
 } from '../governor/rate-state-backend';
@@ -431,7 +432,8 @@ function normalizeRules(
  * per second; `capacity` is one window of burst. The adaptive `[min,max]` range
  * is derived value-first so the same wire shape drives both declared and
  * default-pacing buckets:
- *   - min_rps = min(5/60, max)
+ *   - min_rps = 2 for the unknown shared-provider scope
+ *   - min_rps = min(5/60, max) for declared rules
  *     (a lower explicit adaptive maximum remains authoritative)
  *   - max_rps = max(base, 100)     (catalogued-rate recommendation)
  *   - max_rps = explicit adaptiveMaxRps when supplied
@@ -454,11 +456,11 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
       : Math.max(base, DEFAULT_ADVISORY_PROVIDER_MAX_RPS);
-  // Undeclared providers keep their existing 10-RPS fallback. Other provider
-  // scopes use the five-RPM floor, capped by an explicit adaptive maximum.
+  // The unknown-provider seed is a guess, not a quota floor. Declared provider
+  // scopes keep the five-RPM floor, capped by an explicit adaptive maximum.
   const minRps =
     rule.ruleId === 'default:provider-scope'
-      ? base
+      ? DEFAULT_UNKNOWN_PROVIDER_MIN_RPS
       : Math.min(MIN_ADAPTIVE_RPS_FLOOR, maxRps);
   return {
     rule_id: rule.ruleId,
@@ -472,7 +474,7 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
 
 function minimumRateRpsForRules(rules: readonly AcquirePacingRule[]): number {
   if (rules.some((rule) => rule.ruleId === 'default:provider-scope')) {
-    return DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS;
+    return DEFAULT_UNKNOWN_PROVIDER_MIN_RPS;
   }
   const explicitAdaptiveMaxima = rules.flatMap((rule) => {
     if (rule.adaptiveMaxRps == null || rule.adaptiveMaxRps <= 0) return [];