deepline 0.3.270 → 0.3.272

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -23,6 +23,7 @@ import {
 } from './cell-provenance';
 import { isLedgerTerminalResultRef } from './output-size-limits';
 import type { PlayLogContext } from './log-context';
+import { RUN_EVENT_KIND, type RunEventSource } from './run-event-catalog';
 
 export type PlayRunLedgerStatus =
   | 'queued'
@@ -41,11 +42,7 @@ export type PlayRunLedgerStepStatus =
   | 'failed'
   | 'skipped';
 
-export type PlayRunLedgerEventSource =
-  | 'worker'
-  | 'convex'
-  | 'coordinator'
-  | 'system';
+export type PlayRunLedgerEventSource = RunEventSource;
 
 /**
  * Live progress for a single step/node.
@@ -228,51 +225,51 @@ type PlayRunLedgerBaseEvent = {
 
 export type PlayRunLedgerEvent =
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.created';
+      type: typeof RUN_EVENT_KIND.runCreated;
       playName?: string | null;
       status?: PlayRunLedgerStatus;
       runtimeBackend?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.started';
+      type: typeof RUN_EVENT_KIND.runStarted;
       playName?: string | null;
       runtimeBackend?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.waiting';
+      type: typeof RUN_EVENT_KIND.runWaiting;
       waitKind?: string | null;
       waitUntil?: number | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.resumed';
+      type: typeof RUN_EVENT_KIND.runResumed;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.completed';
+      type: typeof RUN_EVENT_KIND.runCompleted;
       result?: unknown;
       resultSummary?: unknown;
       terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.failed';
+      type: typeof RUN_EVENT_KIND.runFailed;
       error?: string | null;
       result?: unknown;
       resultSummary?: unknown;
       terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'run.cancelled';
+      type: typeof RUN_EVENT_KIND.runCancelled;
       error?: string | null;
       result?: unknown;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'step.started';
+      type: typeof RUN_EVENT_KIND.stepStarted;
       stepId: string;
       label?: string;
       kind?: string;
       artifactTableNamespace?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'step.progress';
+      type: typeof RUN_EVENT_KIND.stepProgress;
       stepId: string;
       label?: string;
       kind?: string;
@@ -280,14 +277,14 @@ export type PlayRunLedgerEvent =
       progress: PlayRunLedgerStepProgress;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'step.completed';
+      type: typeof RUN_EVENT_KIND.stepCompleted;
       stepId: string;
       label?: string;
       kind?: string;
       artifactTableNamespace?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'step.failed';
+      type: typeof RUN_EVENT_KIND.stepFailed;
       stepId: string;
       label?: string;
       kind?: string;
@@ -295,14 +292,14 @@ export type PlayRunLedgerEvent =
       artifactTableNamespace?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'step.skipped';
+      type: typeof RUN_EVENT_KIND.stepSkipped;
       stepId: string;
       label?: string;
       kind?: string;
       artifactTableNamespace?: string | null;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'log.appended';
+      type: typeof RUN_EVENT_KIND.logAppended;
       lines: string[];
       /** Search-only context aligned to `lines`; stripped before Convex. */
       lineContexts?: Array<PlayLogContext | null>;
@@ -330,7 +327,7 @@ export type PlayRunLedgerEvent =
       logsTruncated?: boolean;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'dataset.lifecycle';
+      type: typeof RUN_EVENT_KIND.datasetLifecycle;
       datasetId: string;
       path: string;
       tableNamespace: string;
@@ -343,11 +340,11 @@ export type PlayRunLedgerEvent =
       bornFrom?: PlayDatasetBornFrom;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'activity.observed';
+      type: typeof RUN_EVENT_KIND.activityObserved;
       observation: PlayActivityObservation;
     })
   | (PlayRunLedgerBaseEvent & {
-      type: 'work.counters';
+      type: typeof RUN_EVENT_KIND.workCounters;
       attempt: number;
       invocationSeq: number;
       segmentId: string;
@@ -548,7 +545,10 @@ function normalizeWorkCounters(
 
 function advanceNextWorkCounterSegment(
   current: PlayRunLedgerWorkCounters,
-  event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
+  event: Extract<
+    PlayRunLedgerEvent,
+    { type: typeof RUN_EVENT_KIND.workCounters }
+  >,
 ): PlayRunLedgerWorkCounters {
   const completedLogicalCalls =
     current.completedLogicalCalls + current.logicalCalls;
@@ -594,7 +594,10 @@ function advanceNextWorkCounterSegment(
 
 function advanceWorkCounters(
   current: PlayRunLedgerWorkCounters | undefined,
-  event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
+  event: Extract<
+    PlayRunLedgerEvent,
+    { type: typeof RUN_EVENT_KIND.workCounters }
+  >,
 ): PlayRunLedgerWorkCounters | undefined {
   if (
     !nonnegativeSafeInteger(event.attempt) ||
@@ -1307,9 +1310,9 @@ function withTiming(snapshot: PlayRunLedgerSnapshot): PlayRunLedgerSnapshot {
 }
 
 const TERMINAL_STATUS_BY_EVENT_TYPE = {
-  'run.completed': 'completed',
-  'run.failed': 'failed',
-  'run.cancelled': 'cancelled',
+  [RUN_EVENT_KIND.runCompleted]: 'completed',
+  [RUN_EVENT_KIND.runFailed]: 'failed',
+  [RUN_EVENT_KIND.runCancelled]: 'cancelled',
 } as const satisfies Partial<Record<string, PlayRunLedgerStatus>>;
 
 function isRawPlatformDeployResetText(
@@ -1387,7 +1390,7 @@ function conflictingTerminalSnapshot(
   const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
   if (
     base.status === 'completed' &&
-    eventType === 'run.failed' &&
+    eventType === RUN_EVENT_KIND.runFailed &&
     isPlatformDeployFailureText(eventError)
   ) {
     return withTiming(
@@ -1398,7 +1401,7 @@ function conflictingTerminalSnapshot(
   }
   if (
     base.status === 'failed' &&
-    eventType === 'run.failed' &&
+    eventType === RUN_EVENT_KIND.runFailed &&
     isPlatformDeployFailureText(eventError)
   ) {
     return withTiming(base);
@@ -1538,11 +1541,11 @@ function terminalFinishedAt(
 
 function isStepLifecycleEvent(event: PlayRunLedgerEvent): boolean {
   return (
-    event.type === 'step.started' ||
-    event.type === 'step.progress' ||
-    event.type === 'step.completed' ||
-    event.type === 'step.failed' ||
-    event.type === 'step.skipped'
+    event.type === RUN_EVENT_KIND.stepStarted ||
+    event.type === RUN_EVENT_KIND.stepProgress ||
+    event.type === RUN_EVENT_KIND.stepCompleted ||
+    event.type === RUN_EVENT_KIND.stepFailed ||
+    event.type === RUN_EVENT_KIND.stepSkipped
   );
 }
 
@@ -1557,7 +1560,10 @@ function isStepLifecycleEvent(event: PlayRunLedgerEvent): boolean {
  */
 function refineTerminalSyntheticStepStart(
   snapshot: PlayRunLedgerSnapshot,
-  event: Extract<PlayRunLedgerEvent, { type: 'step.progress' }>,
+  event: Extract<
+    PlayRunLedgerEvent,
+    { type: typeof RUN_EVENT_KIND.stepProgress }
+  >,
 ): PlayRunLedgerSnapshot | null {
   const current = snapshot.stepsById[event.stepId];
   const startedAt = event.progress.startedAt;
@@ -1607,7 +1613,7 @@ export function reducePlayRunLedgerEvent(
   // `step.*` facts must be a true no-op here rather than relying on a gateway
   // process to remember terminal state.
   if (isTerminalPlayRunLedgerStatus(snapshot.status)) {
-    if (event.type === 'step.progress') {
+    if (event.type === RUN_EVENT_KIND.stepProgress) {
       return refineTerminalSyntheticStepStart(snapshot, event) ?? snapshot;
     }
     if (isStepLifecycleEvent(event)) return snapshot;
@@ -1617,7 +1623,8 @@ export function reducePlayRunLedgerEvent(
   // must be byte-for-byte inert: even advancing `updatedAt` would re-publish
   // a stale terminal snapshot to every observer.
   if (
-    (event.type === 'run.waiting' || event.type === 'run.resumed') &&
+    (event.type === RUN_EVENT_KIND.runWaiting ||
+      event.type === RUN_EVENT_KIND.runResumed) &&
     shouldIgnoreNonTerminalAfterStickyTerminal(snapshot)
   ) {
     return snapshot;
@@ -1628,16 +1635,17 @@ export function reducePlayRunLedgerEvent(
     updatedAt: Math.max(snapshot.updatedAt ?? 0, occurredAt),
   };
   const ignoreNonTerminalAfterStickyTerminal =
-    (event.type === 'run.created' || event.type === 'run.started') &&
+    (event.type === RUN_EVENT_KIND.runCreated ||
+      event.type === RUN_EVENT_KIND.runStarted) &&
     shouldIgnoreNonTerminalAfterStickyTerminal(base);
 
   switch (event.type) {
-    case 'work.counters': {
+    case RUN_EVENT_KIND.workCounters: {
       const workCounters = advanceWorkCounters(base.workCounters, event);
       if (workCounters === base.workCounters) return snapshot;
       return withTiming({ ...base, workCounters });
     }
-    case 'run.created': {
+    case RUN_EVENT_KIND.runCreated: {
       if (ignoreNonTerminalAfterStickyTerminal) {
         return withTiming(base);
       }
@@ -1657,7 +1665,7 @@ export function reducePlayRunLedgerEvent(
           (createdStatus === 'running' ? occurredAt : base.startedAt),
       });
     }
-    case 'run.started':
+    case RUN_EVENT_KIND.runStarted:
       if (ignoreNonTerminalAfterStickyTerminal) {
         return withTiming(base);
       }
@@ -1671,7 +1679,7 @@ export function reducePlayRunLedgerEvent(
             : 'running',
         startedAt: base.startedAt ?? occurredAt,
       });
-    case 'run.waiting':
+    case RUN_EVENT_KIND.runWaiting:
       return withTiming({
         ...base,
         status: 'waiting',
@@ -1679,7 +1687,7 @@ export function reducePlayRunLedgerEvent(
         waitUntil: event.waitUntil ?? null,
         activeActivityId: null,
       });
-    case 'run.resumed':
+    case RUN_EVENT_KIND.runResumed:
       return withTiming({
         ...base,
         status: 'running',
@@ -1687,7 +1695,7 @@ export function reducePlayRunLedgerEvent(
         waitUntil: null,
         activeActivityId: null,
       });
-    case 'run.completed': {
+    case RUN_EVENT_KIND.runCompleted: {
       const terminalWorkCounterSegment =
         (event.source === 'worker' || event.source === 'system') &&
         (!isTerminalPlayRunLedgerStatus(base.status) ||
@@ -1732,7 +1740,7 @@ export function reducePlayRunLedgerEvent(
         })
       );
     }
-    case 'run.failed': {
+    case RUN_EVENT_KIND.runFailed: {
       const terminalWorkCounterSegment =
         (event.source === 'worker' || event.source === 'system') &&
         (!isTerminalPlayRunLedgerStatus(base.status) ||
@@ -1782,7 +1790,7 @@ export function reducePlayRunLedgerEvent(
         })
       );
     }
-    case 'run.cancelled':
+    case RUN_EVENT_KIND.runCancelled:
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
           allowStaleSameStatusPayloadMerge:
@@ -1815,7 +1823,7 @@ export function reducePlayRunLedgerEvent(
           terminalSource: nextTerminalSource(base, event.source),
         })
       );
-    case 'log.appended':
+    case RUN_EVENT_KIND.logAppended:
       return withTiming(
         appendLogLines(base, event.lines, {
           ...(typeof event.firstSeq === 'number'
@@ -1824,7 +1832,7 @@ export function reducePlayRunLedgerEvent(
           ...(event.logsTruncated === true ? { logsTruncated: true } : {}),
         }),
       );
-    case 'step.started': {
+    case RUN_EVENT_KIND.stepStarted: {
       const current = base.stepsById[event.stepId];
       const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const shouldRefineSyntheticStart =
@@ -1872,7 +1880,7 @@ export function reducePlayRunLedgerEvent(
             : (base.activeArtifactTableNamespace ?? null),
       });
     }
-    case 'step.progress': {
+    case RUN_EVENT_KIND.stepProgress: {
       const current = base.stepsById[event.stepId];
       const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const mergedProgress = {
@@ -1969,18 +1977,19 @@ export function reducePlayRunLedgerEvent(
               : (base.activeArtifactTableNamespace ?? null),
       });
     }
-    case 'step.completed':
-    case 'step.failed':
-    case 'step.skipped': {
+    case RUN_EVENT_KIND.stepCompleted:
+    case RUN_EVENT_KIND.stepFailed:
+    case RUN_EVENT_KIND.stepSkipped: {
       const current = base.stepsById[event.stepId];
       const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
       const preserveFailedStatus =
-        current?.status === 'failed' && event.type !== 'step.failed';
+        current?.status === 'failed' &&
+        event.type !== RUN_EVENT_KIND.stepFailed;
       const status = preserveFailedStatus
         ? 'failed'
-        : event.type === 'step.completed'
+        : event.type === RUN_EVENT_KIND.stepCompleted
           ? 'completed'
-          : event.type === 'step.failed'
+          : event.type === RUN_EVENT_KIND.stepFailed
             ? 'failed'
             : 'skipped';
       const completedAt =
@@ -1994,7 +2003,7 @@ export function reducePlayRunLedgerEvent(
         kind: event.kind ?? current?.kind,
         status,
         error:
-          event.type === 'step.failed'
+          event.type === RUN_EVENT_KIND.stepFailed
             ? (event.error ?? current?.error ?? null)
             : preserveFailedStatus
               ? (current?.error ?? null)
@@ -2032,7 +2041,7 @@ export function reducePlayRunLedgerEvent(
             : null,
       });
     }
-    case 'dataset.lifecycle': {
+    case RUN_EVENT_KIND.datasetLifecycle: {
       const current = base.datasetsById[event.datasetId];
       const persistedRows = optionalFiniteNumber(event.persistedRows);
       const succeededRows = optionalFiniteNumber(event.succeededRows);
@@ -2082,7 +2091,7 @@ export function reducePlayRunLedgerEvent(
         resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
       });
     }
-    case 'activity.observed': {
+    case RUN_EVENT_KIND.activityObserved: {
       const current = base.activitiesById[event.observation.activityId];
       if (current && current.observedAt >= event.observation.observedAt) {
         return withTiming(base);
@@ -2142,16 +2151,20 @@ function progressSignature(
 
 function terminalEventTypeForStatus(
   status: PlayRunLedgerStatus,
-): 'run.completed' | 'run.failed' | 'run.cancelled' | null {
-  if (status === 'completed') return 'run.completed';
+):
+  | typeof RUN_EVENT_KIND.runCompleted
+  | typeof RUN_EVENT_KIND.runFailed
+  | typeof RUN_EVENT_KIND.runCancelled
+  | null {
+  if (status === 'completed') return RUN_EVENT_KIND.runCompleted;
   if (
     status === 'failed' ||
     status === 'terminated' ||
     status === 'timed_out'
   ) {
-    return 'run.failed';
+    return RUN_EVENT_KIND.runFailed;
   }
-  if (status === 'cancelled') return 'run.cancelled';
+  if (status === 'cancelled') return RUN_EVENT_KIND.runCancelled;
   return null;
 }
 
@@ -2266,7 +2279,7 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
     index += MAX_LEDGER_LOG_LINES_PER_EVENT
   ) {
     events.push({
-      type: 'log.appended',
+      type: RUN_EVENT_KIND.logAppended,
       runId: input.runId,
       source,
       occurredAt,
@@ -2386,7 +2399,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
       isTerminalPlayRunLedgerStatus(status))
   ) {
     events.push({
-      type: 'run.started',
+      type: RUN_EVENT_KIND.runStarted,
       runId: patch.runId,
       playName: patch.playName ?? input.previousSnapshot.playName ?? null,
       source,
@@ -2401,7 +2414,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
     const previousStep = input.previousSnapshot.stepsById[stepId];
     if (progress.startedAt && !previousStep?.startedAt) {
       events.push({
-        type: 'step.started',
+        type: RUN_EVENT_KIND.stepStarted,
         runId: patch.runId,
         source,
         occurredAt: progress.startedAt,
@@ -2449,7 +2462,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
       progressSignature(previousStep?.progress)
     ) {
       events.push({
-        type: 'step.progress',
+        type: RUN_EVENT_KIND.stepProgress,
         runId: patch.runId,
         source,
         occurredAt: progress.updatedAt ?? checkpointAt,
@@ -2468,7 +2481,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
 
     if (progress.completedAt && !previousStep?.completedAt) {
       events.push({
-        type: 'step.completed',
+        type: RUN_EVENT_KIND.stepCompleted,
         runId: patch.runId,
         source,
         occurredAt: progress.completedAt,
@@ -2484,7 +2497,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
     !isTerminalPlayRunLedgerStatus(input.previousSnapshot.status)
   ) {
     const occurredAt = latestCompletedAt ?? checkpointAt;
-    if (terminalType === 'run.completed') {
+    if (terminalType === RUN_EVENT_KIND.runCompleted) {
       events.push({
         type: terminalType,
         runId: patch.runId,

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -1,3 +1,4 @@
+import { RUN_EVENT_KIND } from '../run-event-catalog';
 import { hydratePlayRunnerConfig } from '../protocol';
 import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
@@ -2404,7 +2405,7 @@ async function run(
               {
                 type: 'execution_event',
                 event: {
-                  type: 'dataset.lifecycle',
+                  type: RUN_EVENT_KIND.datasetLifecycle,
                   datasetId,
                   path: settlement.path,
                   tableNamespace: settlement.tableNamespace,
@@ -2620,7 +2621,7 @@ async function run(
         {
           type: 'execution_event',
           event: {
-            type: 'dataset.lifecycle',
+            type: RUN_EVENT_KIND.datasetLifecycle,
             datasetId,
             path: datasetPath,
             tableNamespace,
@@ -3193,7 +3194,7 @@ async function run(
         {
           type: 'execution_event',
           event: {
-            type: 'dataset.lifecycle',
+            type: RUN_EVENT_KIND.datasetLifecycle,
             datasetId: settlement.datasetId,
             path: settlement.path,
             tableNamespace: settlement.tableNamespace,
@@ -3281,7 +3282,7 @@ async function run(
         {
           type: 'execution_event',
           event: {
-            type: 'dataset.lifecycle',
+            type: RUN_EVENT_KIND.datasetLifecycle,
             datasetId: createRuntimeDatasetId(
               config.context.playName!,
               csvDataset.tableNamespace,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -151,6 +151,7 @@ import {
   markPostgresSchedulerAttemptWaiting,
   materializePostgresSchedulerFiles,
   listPostgresSchedulerCompletedRunnerTerminals,
+  type PostgresSchedulerCompletedRunnerTerminal,
   listPostgresSchedulerAmbiguousModalCreateIntents,
   recordPostgresSchedulerAttemptRuntimeResource,
   recordPostgresSchedulerAmbiguousModalCreateIntent,
@@ -1939,7 +1940,12 @@ async function readTerminalSheetRowOutcomes(input: {
   runId: string;
   attempt: number;
 }): Promise<
-  Array<{ tableNamespace: string; completed: number; failed: number; total: number }>
+  Array<{
+    tableNamespace: string;
+    completed: number;
+    failed: number;
+    total: number;
+  }>
 > {
   const sheetNamespaces = [
     ...new Set(
@@ -2014,11 +2020,14 @@ async function collectFailedTerminalSettledProgressEvents(input: {
       occurredAt: Date.now(),
     });
   } catch (error) {
-    console.warn('[absurd-runtime.worker] failed_terminal_settled_counts_failed', {
-      runId: input.runId,
-      attempt: input.attempt,
-      error: error instanceof Error ? error.message : String(error),
-    });
+    console.warn(
+      '[absurd-runtime.worker] failed_terminal_settled_counts_failed',
+      {
+        runId: input.runId,
+        attempt: input.attempt,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    );
     return [];
   }
 }
@@ -2073,7 +2082,11 @@ function requestTerminalOutboxProjection(input: {
         attempted: projection?.attempted ?? null,
         projected: projection?.projected ?? null,
       });
-      if (projection && projection.attempted === 0 && !projection.targetDelivered) {
+      if (
+        projection &&
+        projection.attempted === 0 &&
+        !projection.targetDelivered
+      ) {
         // Nothing was claimable for this run: another dispatcher holds its
         // outbox lease or a predecessor is not yet due. Delivery now depends
         // on that holder or the projector's next wave.
@@ -5748,17 +5761,21 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   absurdReleaseId?: string | null;
   limit?: number;
+  /** Preselected by the fixed controller's one-query cross-lane scan. */
+  terminals?: readonly PostgresSchedulerCompletedRunnerTerminal[];
 }): Promise<{ recovered: number; skipped: number }> {
   const options = input.schedulerOptions ?? {};
-  const terminals = await listPostgresSchedulerCompletedRunnerTerminals(
-    input.client,
-    {
-      limit: input.limit,
-      absurdReleaseId: input.absurdReleaseId,
-      includeFailed: true,
-    },
-    options,
-  );
+  const terminals =
+    input.terminals ??
+    (await listPostgresSchedulerCompletedRunnerTerminals(
+      input.client,
+      {
+        limit: input.limit,
+        absurdReleaseId: input.absurdReleaseId,
+        includeFailed: true,
+      },
+      options,
+    ));
   let recovered = 0;
   let skipped = 0;
 
@@ -5955,18 +5972,40 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminalsAcrossLanes(in
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   limitPerLane?: number;
 }): Promise<{ recovered: number; skipped: number }> {
+  const releaseIds = [
+    ...new Set(input.releaseIds.map((id) => id.trim()).filter(Boolean)),
+  ];
+  if (releaseIds.length === 0) return { recovered: 0, skipped: 0 };
+  const terminals = await listPostgresSchedulerCompletedRunnerTerminals(
+    input.client,
+    {
+      absurdReleaseIds: releaseIds,
+      includeFailed: true,
+      limit: input.limitPerLane,
+    },
+    input.schedulerOptions,
+  );
+  const terminalsByReleaseId = new Map<
+    string,
+    PostgresSchedulerCompletedRunnerTerminal[]
+  >();
+  for (const terminal of terminals) {
+    if (!terminal.releaseId) continue;
+    const laneTerminals = terminalsByReleaseId.get(terminal.releaseId) ?? [];
+    laneTerminals.push(terminal);
+    terminalsByReleaseId.set(terminal.releaseId, laneTerminals);
+  }
+
   let recovered = 0;
   let skipped = 0;
-  for (const releaseId of [
-    ...new Set(input.releaseIds.map((id) => id.trim())),
-  ]) {
-    if (!releaseId) continue;
+  for (const releaseId of releaseIds) {
     const outcome = await recoverAbsurdRuntimeCompletedRunnerTerminals({
       client: input.client,
       queue: absurdQueueNameForRelease(releaseId),
       absurdReleaseId: releaseId,
       schedulerOptions: input.schedulerOptions,
       limit: input.limitPerLane,
+      terminals: terminalsByReleaseId.get(releaseId) ?? [],
     });
     recovered += outcome.recovered;
     skipped += outcome.skipped;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -1,3 +1,4 @@
+import { RUN_EVENT_KIND } from '../run-event-catalog';
 /**
  * Shared constants and pure helpers for the `absurd` play-runtime scheduler
  * profile — imported by both the scheduler backend (`absurd.ts`, submit side)
@@ -823,7 +824,7 @@ export function absurdTerminalLedgerEvent(
   const occurredAt = Date.now();
   if (result.status === 'completed') {
     return {
-      type: 'run.completed',
+      type: RUN_EVENT_KIND.runCompleted,
       runId: result.runId,
       occurredAt,
       source: 'worker',
@@ -832,7 +833,7 @@ export function absurdTerminalLedgerEvent(
   }
   if (result.status === 'failed') {
     return {
-      type: 'run.failed',
+      type: RUN_EVENT_KIND.runFailed,
       runId: result.runId,
       occurredAt,
       source: 'worker',