deepline 0.3.301 → 0.3.303

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.301',
+  version: '0.3.303',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/sdk/src/runs/watch-polling.ts

@@ -95,7 +95,7 @@ async function backfillRotatedLogTail(input: {
 }): Promise<PlayRunLiveSnapshot> {
   // A late first attach intentionally shows the retained tail and its gap
   // marker. Backfill only lines that rotated out after this watcher attached.
-  if (!input.previous.runSignature) {
+  if (!input.previous.runSignature || input.snapshot.logsUnavailable) {
     return input.snapshot;
   }
   const gap = resolvePlayRunLogGap(input.snapshot, input.previous.lastLogSeq);

package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts

@@ -84,6 +84,8 @@ export type PlayRunLiveSnapshot = {
   status: PlayRunLiveStatus;
   updatedAt: number | null;
   logs: string[];
+  /** Present when an optional canonical log read failed for this snapshot. */
+  logsUnavailable?: true;
   activeArtifactTableNamespace: string | null;
   resultTableNamespace: string | null;
   nodeStates: PlayVisualNodeStateSnapshot[];

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -5,7 +5,7 @@ import {
   dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope,
   type PostgresSchedulerOptions,
   type PostgresSchedulerOutboxDrainResult,
-  type PostgresSchedulerQueryClient,
+  type RunLedgerOutboxWithClient,
 } from '../../scheduler-backends/postgres';
 
 import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports';
@@ -17,9 +17,7 @@ export type GatewayRunLedgerProjectionAdapterInput = {
    * against the active outbox lease while retaining supported transport hooks.
    */
   runtimeApi: WorkerRuntimeApiContext;
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>;
+  withClient: RunLedgerOutboxWithClient;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
 };
 

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -99,6 +99,12 @@ export type PlayRunLiveSnapshot = {
    * crossed the retention cap (ADR-0009). Additive/optional on the wire.
    */
   logsTruncated?: boolean;
+  /**
+   * The canonical log store could not be read for this snapshot. The run
+   * status remains authoritative; callers should retry logs on the next poll
+   * or use the paginated logs endpoint. The log cursor does not advance.
+   */
+  logsUnavailable?: true;
   activeArtifactTableNamespace: string | null;
   resultTableNamespace: string | null;
   datasets?: PlayRunLedgerDatasetSnapshot[];
@@ -573,6 +579,13 @@ function diffLogLines(input: {
   lastLogSeq: number;
 }): { lines: string[]; lastLogSeq: number; firstSeq: number | null } {
   const { logs, totalLogCount } = input.snapshot;
+  if (input.snapshot.logsUnavailable) {
+    return {
+      lines: [],
+      lastLogSeq: input.lastLogSeq,
+      firstSeq: null,
+    };
+  }
   if (totalLogCount <= input.lastLogSeq) {
     return { lines: [], lastLogSeq: input.lastLogSeq, firstSeq: null };
   }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -694,12 +694,22 @@ async function acquireOneShotDaytonaSandbox(input: {
     },
   );
   input.emitStage('create:start');
+  const admissionRequestedAt = Date.now();
+  let admissionWaitMs = 0;
+  let admissionQueued = false;
+  const create = () => {
+    admissionWaitMs = Date.now() - admissionRequestedAt;
+    if (admissionQueued) {
+      input.emitStage('create:admission_ready', { admissionWaitMs });
+    }
+    return createRetriedOneShotDaytonaSandbox(input);
+  };
   const result = input.startAdmission
-    ? await input.startAdmission.run(
-        () => createRetriedOneShotDaytonaSandbox(input),
-        input.cancellationSignal,
-      )
-    : await createRetriedOneShotDaytonaSandbox(input);
+    ? await input.startAdmission.run(create, input.cancellationSignal, () => {
+        admissionQueued = true;
+        input.emitStage('create:admission_wait');
+      })
+    : await create();
   // The runner's egress probe measures the sandbox-to-estate round trip; the
   // sandbox's target names the region that round trip crosses.
   console.info('[play-runner.daytona.sandbox_acquired]', {
@@ -779,6 +789,7 @@ async function acquireOneShotDaytonaSandbox(input: {
     sandboxId: sandbox.id,
     attempt: result.attempt,
     elapsedMs: billingStartedAt - input.startedAt,
+    admissionWaitMs,
     attemptElapsedMs: result.attemptElapsedMs,
     // Daytona's default snapshot is intentionally provider-managed. Record the
     // resolved immutable snapshot identifier so a slow acquisition can be

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts

@@ -1,7 +1,11 @@
 import { RUNTIME_CAPACITY_POLICY } from '../../runtime-capacity-policy';
 
 export type DaytonaSandboxStartAdmission = {
-  run<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
+  run<T>(
+    operation: () => Promise<T>,
+    signal?: AbortSignal,
+    onQueued?: () => void,
+  ): Promise<T>;
 };
 
 /**
@@ -45,6 +49,7 @@ export function createDaytonaSandboxStartAdmission(
     async run<T>(
       operation: () => Promise<T>,
       signal?: AbortSignal,
+      onQueued?: () => void,
     ): Promise<T> {
       signal?.throwIfAborted();
       // Preserve eager prepare semantics when capacity is free: the provider
@@ -57,6 +62,7 @@ export function createDaytonaSandboxStartAdmission(
               return createRelease();
             })()
           : await new Promise<() => void>((resolve, reject) => {
+              onQueued?.();
               const accept = (release: () => void) => {
                 signal?.removeEventListener('abort', abort);
                 resolve(release);

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -66,11 +66,13 @@ export const RUNTIME_CAPACITY_POLICY = {
   sandboxAcquisition: {
     /**
      * Starting a Daytona sandbox is a provider control-plane transition, not
-     * an Absurd execution slot. Serialize starts within each worker process;
-     * once started, all eight execution slots may run concurrently.
+     * an Absurd execution slot. Allow up to five concurrent starts within one
+     * worker process; the fleet-wide limiter paces lifecycle admissions, and
+     * this semaphore bounds concurrent create/retry ladders. A worker has
+     * eight execution slots.
      */
-    daytonaConcurrentStartsPerProcess: 1,
-    /** Fleet-wide AIMD start pacing, separate from execution-slot scaling. */
+    daytonaConcurrentStartsPerProcess: 5,
+    /** Fleet-wide AIMD lifecycle-admission pacing. */
     daytonaInitialStartsPerMinute: 120,
     daytonaMaximumStartsPerMinute: 600,
   },

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -870,8 +870,8 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
             ...absurdRuntimeApiFromLaunch(launch),
             ...(signal ? { signal } : {}),
           },
-          withClient: (operation) =>
-            withRuntimeSchedulerClient(getPool(), operation),
+          withClient: (operation, trace) =>
+            withRuntimeSchedulerClient(getPool(), operation, trace),
           schedulerOptions: options,
         }),
       }).deliver(targetEventId ? { targetEventId } : {}),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -642,6 +642,8 @@ export type PostgresSchedulerRunStatusSnapshot = {
   error: string | null;
   suspension: unknown | null;
   staticPipeline: PlayStaticPipeline | null;
+  /** Included only when requested by the admission-projection read path. */
+  createdProjectionPending?: boolean;
 };
 
 type RunStatusRow = RunRow & {
@@ -651,6 +653,7 @@ type RunStatusRow = RunRow & {
   runtime_profile: string | null;
   launch_json: unknown;
   definition_scope?: string | null;
+  created_projection_pending?: boolean;
   suspension_json: unknown | null;
   updated_at: Date | null;
 };
@@ -17380,11 +17383,23 @@ export async function readPostgresSchedulerRunStatus(
   options?: Pick<PostgresSchedulerOptions, 'schema'> & {
     includeDetails?: boolean;
     includeAdmissionScope?: boolean;
+    includeCreatedProjectionPending?: boolean;
   },
 ): Promise<PostgresSchedulerRunStatusSnapshot | null> {
   const admissionScopeSelection = options?.includeAdmissionScope
-    ? ", launch_json ->> 'definitionScope' AS definition_scope"
+    ? ", scheduler_run.launch_json ->> 'definitionScope' AS definition_scope"
     : '';
+  const createdProjectionJoin = options?.includeCreatedProjectionPending
+    ? `LEFT JOIN ${tableName('outbox', options)} AS created_event
+         ON created_event.run_id = scheduler_run.run_id
+        AND created_event.event_id = $2`
+    : '';
+  const createdProjectionSelection = options?.includeCreatedProjectionPending
+    ? ', (created_event.event_id IS NOT NULL AND created_event.delivered_at IS NULL) AS created_projection_pending'
+    : '';
+  const values = options?.includeCreatedProjectionPending
+    ? [runId, `run-created:${runId}`]
+    : [runId];
   if (options?.includeDetails === false) {
     // An admission projection gap needs lifecycle truth only. Launch, result,
     // suspension and error JSON may contain row/call-sized detail: never even
@@ -17403,15 +17418,20 @@ export async function readPostgresSchedulerRunStatus(
         | 'started_at'
         | 'finished_at'
         | 'definition_scope'
+        | 'created_projection_pending'
       >
     >(
       `
-      SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
-        created_at, updated_at, started_at, finished_at${admissionScopeSelection}
-      FROM ${tableName('runs', options)}
-      WHERE run_id = $1
+      SELECT scheduler_run.run_id, scheduler_run.org_id, scheduler_run.status,
+        scheduler_run.play_name, scheduler_run.graph_hash,
+        scheduler_run.runtime_profile, scheduler_run.created_at,
+        scheduler_run.updated_at, scheduler_run.started_at,
+        scheduler_run.finished_at${admissionScopeSelection}${createdProjectionSelection}
+      FROM ${tableName('runs', options)} AS scheduler_run
+      ${createdProjectionJoin}
+      WHERE scheduler_run.run_id = $1
     `,
-      [runId],
+      values,
     );
     const row = result.rows[0];
     if (!row) return null;
@@ -17441,17 +17461,25 @@ export async function readPostgresSchedulerRunStatus(
       error: null,
       suspension: null,
       staticPipeline: null,
+      ...(options.includeCreatedProjectionPending
+        ? { createdProjectionPending: row.created_projection_pending === true }
+        : {}),
     };
   }
   const result = await client.query<RunStatusRow>(
     `
-      SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
-        launch_json, suspension_json, terminal_result_json, terminal_error_json,
-        runtime_timing_json, created_at, updated_at, started_at, finished_at${admissionScopeSelection}
-      FROM ${tableName('runs', options)}
-      WHERE run_id = $1
+      SELECT scheduler_run.run_id, scheduler_run.org_id, scheduler_run.status,
+        scheduler_run.play_name, scheduler_run.graph_hash,
+        scheduler_run.runtime_profile, scheduler_run.launch_json,
+        scheduler_run.suspension_json, scheduler_run.terminal_result_json,
+        scheduler_run.terminal_error_json, scheduler_run.runtime_timing_json,
+        scheduler_run.created_at, scheduler_run.updated_at,
+        scheduler_run.started_at, scheduler_run.finished_at${admissionScopeSelection}${createdProjectionSelection}
+      FROM ${tableName('runs', options)} AS scheduler_run
+      ${createdProjectionJoin}
+      WHERE scheduler_run.run_id = $1
     `,
-    [runId],
+    values,
   );
   const row = result.rows[0] ?? null;
   if (!row) return null;
@@ -17487,6 +17515,9 @@ export async function readPostgresSchedulerRunStatus(
         : null,
     suspension: row.suspension_json ?? null,
     staticPipeline: staticPipelineFromContractSnapshot(launch.contractSnapshot),
+    ...(options?.includeCreatedProjectionPending
+      ? { createdProjectionPending: row.created_projection_pending === true }
+      : {}),
   };
 }
 
@@ -19526,6 +19557,40 @@ export type RunLedgerOutboxDeliveredRow = {
   lagMs: number;
 };
 
+export type RunLedgerOutboxClientTrace = {
+  runId: string;
+  phase: string;
+  priority?: 'ordinary' | 'critical';
+};
+
+export type RunLedgerOutboxWithClient = <T>(
+  operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  trace?: RunLedgerOutboxClientTrace,
+) => Promise<T>;
+
+function logRunLedgerOutboxSettlementFailure(input: {
+  runId: string;
+  eventIds: readonly string[];
+  stage: 'continuation_release' | 'aborted_release' | 'failure_settlement';
+  settlementError: unknown;
+  deliveryError?: unknown;
+}): void {
+  const errorMessage = (error: unknown) =>
+    (error instanceof Error ? error.message : String(error)).slice(0, 500);
+  console.error('[run-ledger-outbox.settlement_failed]', {
+    runId: input.runId,
+    stage: input.stage,
+    eventCount: input.eventIds.length,
+    eventIds: input.eventIds.slice(0, 8),
+    settlementError: errorMessage(input.settlementError),
+    deliveryError:
+      input.deliveryError === undefined
+        ? null
+        : errorMessage(input.deliveryError),
+    recovery: 'lease_expiry',
+  });
+}
+
 export async function dispatchRunLedgerOutboxToLedgerOnce(
   client: PostgresSchedulerQueryClient,
   input: RunLedgerOutboxDispatchInput,
@@ -19570,9 +19635,7 @@ export async function dispatchRunLedgerOutboxToLedgerThroughEvent(
  * network calls. Each callback owns only the claim or settlement query phase.
  */
 export async function dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope(
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>,
+  withClient: RunLedgerOutboxWithClient,
   input: RunLedgerOutboxDispatchInput & {
     targetEventId: string;
     maxEvents?: number;
@@ -19606,9 +19669,7 @@ type RunLedgerOutboxAuthorityScope = {
 };
 
 async function dispatchRunLedgerOutboxThroughEventWithScope(
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>,
+  withClient: RunLedgerOutboxWithClient,
   input: RunLedgerOutboxDispatchInput & {
     targetEventId: string;
     maxEvents?: number;
@@ -19745,9 +19806,7 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
  * background projector use this exact implementation.
  */
 export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>,
+  withClient: RunLedgerOutboxWithClient,
   input: RunLedgerOutboxDispatchInput & { waitForRunLock?: boolean },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<PostgresSchedulerOutboxProjectionResult> {
@@ -19761,9 +19820,7 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
 }
 
 async function dispatchRunLedgerOutboxBatchWithScope(
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>,
+  withClient: RunLedgerOutboxWithClient,
   input: RunLedgerOutboxDispatchInput & { waitForRunLock?: boolean },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
   scope?: RunLedgerOutboxDrainScope,
@@ -19788,14 +19845,31 @@ async function dispatchRunLedgerOutboxBatchWithScope(
   const releasePendingContinuation = async () => {
     const nextClaimed = continuation.current?.nextClaimed ?? [];
     if (nextClaimed.length === 0) return;
-    await withClient((client) =>
-      releasePostgresSchedulerOutboxLease(
-        client,
-        nextClaimed.map((event) => event.eventId),
-        { ...options, leaseOwner },
-      ),
-    );
-    nextClaimed.length = 0;
+    const eventIds = nextClaimed.map((event) => event.eventId);
+    try {
+      await withClient(
+        (client) =>
+          releasePostgresSchedulerOutboxLease(client, eventIds, {
+            ...options,
+            leaseOwner,
+          }),
+        {
+          runId: continuation.current?.runId ?? nextClaimed[0]!.runId,
+          phase: 'run_ledger_outbox_lease_release',
+          priority: 'critical',
+        },
+      );
+      nextClaimed.length = 0;
+    } catch (settlementError) {
+      // Keep the in-memory claim visible to this drain and let its durable
+      // lease expire if even critical pool admission cannot settle it.
+      logRunLedgerOutboxSettlementFailure({
+        runId: continuation.current?.runId ?? nextClaimed[0]!.runId,
+        eventIds,
+        stage: 'continuation_release',
+        settlementError,
+      });
+    }
   };
   const claimEvents = async (client: PostgresSchedulerQueryClient) => {
     // The checkout is exclusive. Hold the existing scheduler run's row lock
@@ -20497,13 +20571,31 @@ async function dispatchRunLedgerOutboxBatchWithScope(
       // projection attempt or scheduling backoff, and wake the durable
       // projector. If the append already reached the ledger, the projector's
       // replay carries the same idempotency key and is a no-op there.
-      const released = await withClient((client) =>
-        releasePostgresSchedulerOutboxLease(
-          client,
-          events.map((event) => event.eventId),
-          { ...options, leaseOwner },
-        ),
-      );
+      const eventIds = events.map((event) => event.eventId);
+      let released: string[];
+      try {
+        released = await withClient(
+          (client) =>
+            releasePostgresSchedulerOutboxLease(client, eventIds, {
+              ...options,
+              leaseOwner,
+            }),
+          {
+            runId: events[0]!.runId,
+            phase: 'run_ledger_outbox_lease_release',
+            priority: 'critical',
+          },
+        );
+      } catch (settlementError) {
+        logRunLedgerOutboxSettlementFailure({
+          runId: events[0]!.runId,
+          eventIds,
+          stage: 'aborted_release',
+          settlementError,
+          deliveryError: error,
+        });
+        released = eventIds;
+      }
       return {
         attempted: events.length,
         projected: 0,
@@ -20515,22 +20607,34 @@ async function dispatchRunLedgerOutboxBatchWithScope(
       };
     }
     if (error instanceof DeferredRuntimeProjectionInputError) {
-      const outcome = await withClient((client) =>
-        settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
-          ...options,
-          leaseOwner,
-        }),
+      const outcome = await withClient(
+        (client) =>
+          settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
+            ...options,
+            leaseOwner,
+          }),
+        {
+          runId: events[0]!.runId,
+          phase: 'run_ledger_outbox_failure_settlement',
+          priority: 'critical',
+        },
       );
       // Earlier facts may already be indexed. Only the invalid immutable row
       // is quarantined; valid predecessors remain deliverable in causal order.
-      const released = await withClient((client) =>
-        releasePostgresSchedulerOutboxLease(
-          client,
-          events
-            .filter((event) => event.eventId !== error.eventId)
-            .map((event) => event.eventId),
-          { ...options, leaseOwner },
-        ),
+      const released = await withClient(
+        (client) =>
+          releasePostgresSchedulerOutboxLease(
+            client,
+            events
+              .filter((event) => event.eventId !== error.eventId)
+              .map((event) => event.eventId),
+            { ...options, leaseOwner },
+          ),
+        {
+          runId: events[0]!.runId,
+          phase: 'run_ledger_outbox_lease_release',
+          priority: 'critical',
+        },
       );
       const failed = outcome === 'blocked' ? [error.eventId] : [];
       const retrying =
@@ -20550,52 +20654,79 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         retryingEventIds: retrying,
       };
     }
-    const settlement = await withClient(async (client) => {
-      await client.query('BEGIN');
-      try {
-        const blocked: string[] = [];
-        const retrying: string[] = [];
-        const ownedRetrying: string[] = [];
-        for (const event of events) {
-          const outcome = await settlePostgresSchedulerOutboxFailure(
-            client,
-            event.eventId,
-            error,
-            { ...options, leaseOwner },
-          );
-          if (outcome === 'blocked') blocked.push(event.eventId);
-          else retrying.push(event.eventId);
-          if (outcome === 'retrying') ownedRetrying.push(event.eventId);
-        }
-        // All members of this failed append must become due together. Keep
-        // the existing per-row retry policy, taking its latest deadline only
-        // for rows this owner actually settled, before releasing their locks.
-        if (ownedRetrying.length > 1) {
-          await client.query(
-            `
-            WITH retry AS (
-              SELECT max(next_attempt_at) AS retry_at
-              FROM ${tableName('outbox', options)}
-              WHERE event_id = ANY($1::text[])
-                AND delivered_at IS NULL AND failed_at IS NULL
-            )
-            UPDATE ${tableName('outbox', options)} AS outbox
-            SET next_attempt_at = retry.retry_at
-            FROM retry
-            WHERE outbox.event_id = ANY($1::text[])
-              AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
-              AND retry.retry_at IS NOT NULL
-          `,
-            [ownedRetrying],
-          );
-        }
-        await client.query('COMMIT');
-        return { blocked, retrying };
-      } catch (settlementError) {
-        await client.query('ROLLBACK');
-        throw settlementError;
-      }
-    });
+    let settlement: { blocked: string[]; retrying: string[] };
+    try {
+      settlement = await withClient(
+        async (client) => {
+          await client.query('BEGIN');
+          try {
+            const blocked: string[] = [];
+            const retrying: string[] = [];
+            const ownedRetrying: string[] = [];
+            for (const event of events) {
+              const outcome = await settlePostgresSchedulerOutboxFailure(
+                client,
+                event.eventId,
+                error,
+                { ...options, leaseOwner },
+              );
+              if (outcome === 'blocked') blocked.push(event.eventId);
+              else retrying.push(event.eventId);
+              if (outcome === 'retrying') ownedRetrying.push(event.eventId);
+            }
+            // Keep every member of a failed append on the same retry deadline.
+            if (ownedRetrying.length > 1) {
+              await client.query(
+                `
+                WITH retry AS (
+                  SELECT max(next_attempt_at) AS retry_at
+                  FROM ${tableName('outbox', options)}
+                  WHERE event_id = ANY($1::text[])
+                    AND delivered_at IS NULL AND failed_at IS NULL
+                )
+                UPDATE ${tableName('outbox', options)} AS outbox
+                SET next_attempt_at = retry.retry_at
+                FROM retry
+                WHERE outbox.event_id = ANY($1::text[])
+                  AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
+                  AND retry.retry_at IS NOT NULL
+              `,
+                [ownedRetrying],
+              );
+            }
+            await client.query('COMMIT');
+            return { blocked, retrying };
+          } catch (settlementError) {
+            await client.query('ROLLBACK');
+            throw settlementError;
+          }
+        },
+        {
+          runId: events[0]!.runId,
+          phase: 'run_ledger_outbox_failure_settlement',
+          priority: 'critical',
+        },
+      );
+    } catch (settlementError) {
+      // The delivery error is already known. If the critical settlement
+      // retry also fails, durable lease expiry remains the recovery path.
+      logRunLedgerOutboxSettlementFailure({
+        runId: events[0]!.runId,
+        eventIds: events.map((event) => event.eventId),
+        stage: 'failure_settlement',
+        settlementError,
+        deliveryError: error,
+      });
+      return {
+        attempted: events.length,
+        projected: 0,
+        eventIds: [],
+        failed: 0,
+        failedEventIds: [],
+        retrying: events.length,
+        retryingEventIds: events.map((event) => event.eventId),
+      };
+    }
     const failure = runLedgerDeliveryFailure({ error });
     console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
       runId: input.runId ?? null,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -15,10 +15,10 @@ import {
   readPostgresSchedulerOutboxNextDueAt,
   type PostgresSchedulerOptions,
   type PostgresSchedulerOutboxProjectionResult,
-  type PostgresSchedulerQueryClient,
   type RunLedgerOutboxDeliveredRow,
   type RunLedgerOutboxHeadPriority,
   type RunLedgerOutboxTokenAuthority,
+  type RunLedgerOutboxWithClient,
 } from './postgres';
 import {
   acquirePgNotifyHub,
@@ -561,9 +561,7 @@ export type RunCreatedInlineDeliveryOutcome =
  */
 export async function deliverRunCreatedOutboxEventInline(input: {
   runId: string;
-  withClient: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>;
+  withClient: RunLedgerOutboxWithClient;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   tokenAuthority: RunLedgerOutboxTokenAuthority;
   deadlineMs?: number;