deepline 0.3.258 → 0.3.259

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -100,6 +100,7 @@ import {
   runtimeWorkCountersFromResources,
   runtimeWorkCountersFromStats,
 } from '../runtime-work-counters';
+import { readPlayLaunchSurface } from '../launch-source-policy';
 import {
   isCtxFetchHttpFailureDiagnostic,
   parseCtxFetchHttpFailureDiagnostic,
@@ -311,7 +312,9 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
 // v50 adds a partial run-scoped outbox index for projection lease verification.
 // v51 also creates that index in isolated (transactional) bootstraps and makes
 // deploy readiness require it; v50 only built it on the concurrent path.
-export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 51;
+// v52 indexes only delivered outbox rows that still contain raw log bodies.
+// The concurrent build lets retention skip already-scrubbed history.
+export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 52;
 const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
   'scheduler_schema_migrations';
 export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -446,6 +449,15 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
   /** Internal provider-read coordinates which must survive terminal-worker
    * loss. They are persisted only on the matching durable cleanup obligation. */
   sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
+  /** Settled progress facts for a failed runner result, folded before the
+   * terminal write. They travel in the `run.failed` outbox row ahead of the
+   * terminal fact, the same carrier a direct completion uses, so the ledger
+   * settles them before the run's terminal makes its steps immutable. */
+  failedTerminalProgressEvents?: readonly unknown[];
+  /** Forwarded to `failPostgresSchedulerAttempt`: a failure terminal written
+   * for a detached-runner attempt must not overwrite an authoritative pushed
+   * runner terminal that won after the attempt was claimed. */
+  preserveRunnerTerminal?: boolean;
 };
 
 export type PostgresSchedulerWorkerOnceResult =
@@ -900,6 +912,20 @@ export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_MS =
 export const POSTGRES_SCHEDULER_OUTBOX_LOG_BODY_RETENTION_MS =
   7 * 24 * 60 * 60 * 1_000;
 export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_BATCH_LIMIT = 1_000;
+// Keep this expression identical in the partial index and retention query so
+// PostgreSQL can prove that the ordered candidate scan is index-qualified.
+// Strict JSONPath plus the array guards matches the old jsonb_array_elements
+// eligibility test, including present-but-null log fields.
+const DELIVERED_OUTBOX_RAW_LOG_BODY_PREDICATE = `
+  (
+    (event_type = 'run.events'
+      AND jsonb_typeof(payload_json->'events') = 'array'
+      AND payload_json @? 'strict $.events[*] ? (@.type == "log.appended" && (exists(@.lines) || exists(@.lineContexts)))'::jsonpath)
+    OR (event_type IN ('run.completed', 'run.failed')
+      AND (payload_json ? 'terminalLogLines'
+        OR (jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
+          AND payload_json @? 'strict $.terminalProgressEvents[*] ? (@.type == "log.appended" && (exists(@.lines) || exists(@.lineContexts)))'::jsonpath)))
+  )`;
 const RUN_LEDGER_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
 const COMPUTE_BILLING_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
 
@@ -1933,6 +1959,15 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             AND index_row.indisready AND index_row.indisvalid
           )
         )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_outbox_delivered_log_body_idx'
+            AND index_row.indrelid = to_regclass($3)
+            AND index_row.indisready AND index_row.indisvalid
+        )
         AND (
           $6::boolean = false
           OR EXISTS (
@@ -3852,6 +3887,12 @@ export async function ensurePostgresSchedulerSchema(
           ON ${tableName('outbox', options)} (delivered_at, event_id)
           WHERE delivered_at IS NOT NULL AND failed_at IS NULL
       `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_delivered_log_body_idx
+          ON ${tableName('outbox', options)} (delivered_at, event_id)
+          WHERE delivered_at IS NOT NULL AND failed_at IS NULL
+            AND ${DELIVERED_OUTBOX_RAW_LOG_BODY_PREDICATE}
+      `);
       await client.query(`
         CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_blocked_oldest_idx
           ON ${tableName('outbox', options)} (created_at, event_id)
@@ -4004,6 +4045,13 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
     definition: `(delivered_at, event_id)
       WHERE delivered_at IS NOT NULL AND failed_at IS NULL`,
   },
+  {
+    name: 'runtime_scheduler_outbox_delivered_log_body_idx',
+    table: 'outbox',
+    definition: `(delivered_at, event_id)
+      WHERE delivered_at IS NOT NULL AND failed_at IS NULL
+        AND ${DELIVERED_OUTBOX_RAW_LOG_BODY_PREDICATE}`,
+  },
   {
     name: 'runtime_scheduler_outbox_blocked_oldest_idx',
     table: 'outbox',
@@ -5053,6 +5101,7 @@ async function submitPostgresSchedulerRunTransaction(
             'staticPipeline', launch_payload.launch_json -> 'staticPipeline',
             'source', launch_payload.launch_json ->> 'source',
             'triggerSource', launch_payload.launch_json ->> 'triggerSource',
+            'launchSurface', launch_payload.launch_json ->> 'launchSurface',
             'inputFileId', launch_payload.launch_json ->> 'inputFileId',
             'inputBytes', launch_payload.launch_json -> 'inputBytes',
             'inputSha256', launch_payload.launch_json ->> 'inputSha256',
@@ -7184,6 +7233,10 @@ export async function markPostgresSchedulerAttemptWaiting(
   const eventKey = suspensionEventKey(input.suspension);
   const detachedRunnerCeilingDeadlineMs =
     detachedRunnerCeilingDeadlineMsFromSuspension(input.suspension);
+  const sleepWakeAtMs = sleepWakeAtMsFromSuspension(
+    input.suspension,
+    Date.now(),
+  );
   const waiting = await client.query<{ run_id: string; event_id: string }>(
     `
       WITH waiting_attempt AS (
@@ -7252,7 +7305,8 @@ export async function markPostgresSchedulerAttemptWaiting(
           'run.waiting',
           jsonb_strip_nulls(jsonb_build_object(
             'kind', NULLIF(btrim($4::jsonb ->> 'kind'), ''),
-            'eventKey', NULLIF($7::text, '')
+            'eventKey', NULLIF($7::text, ''),
+            'waitUntil', $10::double precision
           )),
           ${causalOutboxCreatedAt('waiting_run.run_id', options)}
         FROM waiting_run
@@ -7270,6 +7324,7 @@ export async function markPostgresSchedulerAttemptWaiting(
       eventKey,
       input.preserveHeartbeatAt === true,
       detachedRunnerCeilingDeadlineMs,
+      sleepWakeAtMs,
     ],
   );
   const row = waiting.rows[0];
@@ -7283,7 +7338,29 @@ export async function markPostgresSchedulerAttemptWaiting(
  * lane. Other wait kinds intentionally return null: event/HITL waits are not
  * execution timeouts.
  */
-function detachedRunnerCeilingDeadlineMsFromSuspension(
+/**
+ * When a `ctx.sleep` park will wake, measured from the moment the scheduler
+ * records the wait. Observers render it ("sleeping until 14:05:09") so a sleep
+ * is never described as an approval or event wait.
+ */
+export function sleepWakeAtMsFromSuspension(
+  suspension: unknown,
+  nowMs: number,
+): number | null {
+  if (
+    !suspension ||
+    typeof suspension !== 'object' ||
+    Array.isArray(suspension)
+  ) {
+    return null;
+  }
+  const candidate = suspension as { kind?: unknown; delayMs?: unknown };
+  if (candidate.kind !== 'sleep') return null;
+  const delayMs = Number(candidate.delayMs);
+  return Number.isFinite(delayMs) && delayMs >= 0 ? nowMs + delayMs : null;
+}
+
+export function detachedRunnerCeilingDeadlineMsFromSuspension(
   suspension: unknown,
 ): number | null {
   if (
@@ -7964,6 +8041,204 @@ export async function deferPostgresSchedulerAttemptForCapacity(
   return deferred.rows.length > 0;
 }
 
+/** `error_json.kind` stamped on an attempt retired by an executor-token
+ * outage. Consecutive attempts carrying it form one bounded outage. */
+export const EXECUTOR_TOKEN_UNAVAILABLE_ATTEMPT_ERROR_KIND =
+  'executor_token_unavailable';
+
+/**
+ * Return a fresh, pre-code attempt to durable queue admission after the
+ * executor-token route was transiently unreachable. Like a capacity deferral,
+ * no Play code ran and no engine attempt is spent. Unlike capacity, an
+ * unreachable app may never recover, so the deferral is bounded by wall-clock:
+ * the outage starts at the oldest attempt in the unbroken tail of attempts
+ * retired by this same outage, and once it is older than `maxOutageMs` the
+ * attempt is left untouched (`deferred: false`) for the caller to terminalize.
+ */
+export async function deferPostgresSchedulerAttemptForExecutorTokenOutage(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    runId: string;
+    attempt: number;
+    leaseToken: string;
+    reason: string;
+    maxOutageMs: number;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<{ deferred: boolean; outageStartedAtMs: number | null }> {
+  const result = await client.query<{
+    outage_started_at_ms: number | string | null;
+    deferred: boolean;
+  }>(
+    `
+      WITH current_attempt AS (
+        SELECT created_at
+        FROM ${tableName('attempts', options)}
+        WHERE run_id = $1
+          AND attempt = $2
+          AND lease_token = $3
+          AND status IN ('claimed', 'running')
+      ),
+      outage AS (
+        SELECT LEAST(
+          (SELECT created_at FROM current_attempt),
+          (
+            SELECT min(prior.created_at)
+            FROM ${tableName('attempts', options)} AS prior
+            WHERE prior.run_id = $1
+              AND prior.attempt < $2
+              AND prior.error_json ->> 'kind' = $6
+              AND NOT EXISTS (
+                SELECT 1
+                FROM ${tableName('attempts', options)} AS gap
+                WHERE gap.run_id = $1
+                  AND gap.attempt > prior.attempt
+                  AND gap.attempt < $2
+                  AND gap.error_json ->> 'kind' IS DISTINCT FROM $6
+              )
+          )
+        ) AS started_at
+      ),
+      deferred_attempt AS (
+        UPDATE ${tableName('attempts', options)}
+        SET status = 'stale',
+            lease_expires_at = NULL,
+            heartbeat_at = now(),
+            error_json = $4::jsonb,
+            updated_at = now()
+        WHERE run_id = $1
+          AND attempt = $2
+          AND status IN ('claimed', 'running')
+          AND lease_token = $3
+          AND (SELECT started_at FROM outage) >
+            clock_timestamp() - ($5::double precision * interval '1 millisecond')
+        RETURNING run_id
+      ),
+      requeued_run AS (
+        UPDATE ${tableName('runs', options)}
+        SET status = 'queued',
+            suspension_json = NULL,
+            wait_deadline = NULL,
+            updated_at = now()
+        WHERE run_id IN (SELECT run_id FROM deferred_attempt)
+          AND status IN ('queued', 'running', 'waiting')
+        RETURNING run_id
+      )
+      SELECT
+        (extract(epoch FROM (SELECT started_at FROM outage)) * 1000)::float8
+          AS outage_started_at_ms,
+        EXISTS (SELECT 1 FROM requeued_run) AS deferred
+    `,
+    [
+      input.runId,
+      input.attempt,
+      input.leaseToken,
+      stringifyPostgresJson({
+        kind: EXECUTOR_TOKEN_UNAVAILABLE_ATTEMPT_ERROR_KIND,
+        message: input.reason,
+      }),
+      input.maxOutageMs,
+      EXECUTOR_TOKEN_UNAVAILABLE_ATTEMPT_ERROR_KIND,
+    ],
+  );
+  const row = result.rows[0];
+  const startedAtMs =
+    row?.outage_started_at_ms == null ? NaN : Number(row.outage_started_at_ms);
+  return {
+    deferred: row?.deferred === true,
+    outageStartedAtMs: Number.isFinite(startedAtMs) ? startedAtMs : null,
+  };
+}
+
+/**
+ * Take the fence of one parked detached-runner attempt so its owner can write
+ * a failure terminal. This is the single-run form of the overdue-ceiling claim
+ * (same guards, same `waiting -> running` transition and `run.resumed` fact):
+ * it requires the exact attempt and lease token, refuses when an authoritative
+ * runner terminal was already pushed, and refuses when a newer attempt exists.
+ * Returns false (no write) when any guard fails.
+ */
+export async function claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string; attempt: number; leaseToken: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const claimed = await client.query<{ run_id: string }>(
+    `
+      WITH candidate AS MATERIALIZED (
+        SELECT runs.run_id, attempts.attempt
+        FROM ${tableName('runs', options)} AS runs
+        JOIN ${tableName('attempts', options)} AS attempts
+          ON attempts.run_id = runs.run_id
+         AND attempts.attempt = (runs.suspension_json ->> 'runnerAttempt')::integer
+        WHERE runs.run_id = $1
+          AND runs.status = 'waiting'
+          AND runs.suspension_json ->> 'kind' = 'detached_runner'
+          AND attempts.attempt = $2
+          AND attempts.status = 'waiting'
+          AND attempts.lease_token = $3
+          AND (
+            attempts.runner_result_json IS NULL
+            OR attempts.runner_result_json ->> 'status' = 'suspended'
+          )
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${tableName('attempts', options)} AS newer
+            WHERE newer.run_id = attempts.run_id
+              AND newer.attempt > attempts.attempt
+              AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
+          )
+        FOR UPDATE OF runs, attempts
+      ),
+      running_attempt AS (
+        UPDATE ${tableName('attempts', options)} AS target
+        SET status = 'running',
+            heartbeat_at = now(),
+            updated_at = now()
+        FROM candidate
+        WHERE target.run_id = candidate.run_id
+          AND target.attempt = candidate.attempt
+          AND target.status = 'waiting'
+          AND target.lease_token = $3
+        RETURNING target.run_id
+      ),
+      running_run AS (
+        UPDATE ${tableName('runs', options)} AS target
+        SET status = 'running',
+            suspension_json = NULL,
+            wait_deadline = NULL,
+            updated_at = now()
+        WHERE target.run_id IN (SELECT run_id FROM running_attempt)
+          AND target.status = 'waiting'
+        RETURNING target.run_id
+      ),
+      resumed_outbox AS (
+        INSERT INTO ${tableName('outbox', options)} (
+          event_id, run_id, event_type, payload_json
+        )
+        SELECT
+          $4,
+          running_run.run_id,
+          'run.resumed',
+          jsonb_build_object(
+            'kind', 'detached_runner',
+            'eventKey', 'detached_runner_terminal_salvage'
+          )
+        FROM running_run
+        ON CONFLICT (event_id) DO NOTHING
+      )
+      SELECT run_id FROM running_run
+    `,
+    [
+      input.runId,
+      input.attempt,
+      input.leaseToken,
+      `run-resumed:detached-runner-salvage:${input.runId}:${input.attempt}`,
+    ],
+  );
+  return claimed.rows.length > 0;
+}
+
 export type PostgresSchedulerRecoveryCircuit = {
   dependency: string;
   state: 'closed' | 'open' | 'half_open';
@@ -12733,6 +13008,9 @@ export async function failPostgresSchedulerAttempt(
      * hard ceiling in the same fenced statement. */
     preserveRunnerTerminal?: boolean;
     sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
+    /** Progress facts the ledger must apply before `run.failed` (see
+     * `failedTerminalProgressEvents`). Omitted from the payload when empty. */
+    terminalProgressEvents?: readonly unknown[];
   },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<boolean> {
@@ -13021,7 +13299,11 @@ export async function failPostgresSchedulerAttempt(
             'terminalLogContexts', $14::jsonb,
             'resultSummary', $13::jsonb,
             'terminalWorkCounterSegment', $15::jsonb
-          ),
+          ) || CASE
+            WHEN jsonb_array_length($16::jsonb) > 0
+              THEN jsonb_build_object('terminalProgressEvents', $16::jsonb)
+            ELSE '{}'::jsonb
+          END,
           ${causalOutboxCreatedAt('run_terminal.run_id', options)}
         FROM run_terminal
         ON CONFLICT (event_id) DO NOTHING
@@ -13047,6 +13329,7 @@ export async function failPostgresSchedulerAttempt(
       stringifyPostgresJson(resultSummary),
       JSON.stringify(terminalLogContexts),
       stringifyPostgresJson(terminalWorkCounterSegment ?? null),
+      stringifyPostgresJson(input.terminalProgressEvents ?? []),
     ],
   );
   return failed.rows.length > 0;
@@ -14130,6 +14413,12 @@ export async function executePostgresSchedulerClaim(
         terminalLogLines: terminalLogProjection.lines,
         terminalLogContexts: terminalLogProjection.lineContexts,
         sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
+        ...(options.preserveRunnerTerminal
+          ? { preserveRunnerTerminal: true }
+          : {}),
+        ...(runnerFailureResult && options.failedTerminalProgressEvents?.length
+          ? { terminalProgressEvents: options.failedTerminalProgressEvents }
+          : {}),
       },
       options,
     );
@@ -17021,8 +17310,18 @@ export async function readPendingPostgresSchedulerOutboxEvents(
         -- terminal facts. Project it first so a lifecycle backlog cannot make
         -- an accepted run invisible; per-run predecessor checks above retain
         -- strict ordering within each run.
+        -- A terminal head comes next: it is a run's last fact (one per run, so
+        -- it cannot starve anything), its predecessors are already delivered
+        -- (checked above), and it is what a caller is waiting on. Under a
+        -- lifecycle backlog, FIFO left a committed terminal queued behind other
+        -- runs' progress for seconds (live-run-observation-e2e: 4.9 s from
+        -- commit to first claim while the projector drained continuously).
         ORDER BY
-          CASE WHEN candidate.event_type = 'run.created' THEN 0 ELSE 1 END,
+          CASE
+            WHEN candidate.event_type = 'run.created' THEN 0
+            WHEN candidate.event_id LIKE 'run-terminal:%' THEN 1
+            ELSE 2
+          END,
           coalesce(candidate.next_attempt_at, candidate.created_at),
           candidate.created_at,
           candidate.event_id
@@ -17416,37 +17715,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
     await client.query(
       `WITH candidates AS (
          SELECT event_id
-           FROM ${tableName('outbox', options)} AS outbox
+           FROM ${tableName('outbox', options)}
           WHERE delivered_at IS NOT NULL
             AND failed_at IS NULL
             AND delivered_at < $1::timestamptz
-            AND (
-              (event_type = 'run.events'
-               AND jsonb_typeof(payload_json->'events') = 'array'
-               AND EXISTS (
-                 SELECT 1
-                   FROM jsonb_array_elements(
-                     CASE WHEN jsonb_typeof(payload_json->'events') = 'array'
-                       THEN payload_json->'events' ELSE '[]'::jsonb END
-                 ) AS item
-                  WHERE item->>'type' = 'log.appended'
-                    AND (item ? 'lines' OR item ? 'lineContexts')
-               ))
-              OR (event_type IN ('run.completed', 'run.failed')
-                  AND (
-                    payload_json ? 'terminalLogLines'
-                    OR (jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
-                        AND EXISTS (
-                          SELECT 1
-                            FROM jsonb_array_elements(
-                              CASE WHEN jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
-                                THEN payload_json->'terminalProgressEvents' ELSE '[]'::jsonb END
-                            ) AS item
-                           WHERE item->>'type' = 'log.appended'
-                             AND (item ? 'lines' OR item ? 'lineContexts')
-                        ))
-                  ))
-            )
+            AND ${DELIVERED_OUTBOX_RAW_LOG_BODY_PREDICATE}
           ORDER BY delivered_at ASC, event_id ASC
           LIMIT $2::integer
           FOR UPDATE SKIP LOCKED
@@ -17720,8 +17993,29 @@ export async function markPostgresSchedulerOutboxDelivered(
     leaseOwner?: string | null;
   },
 ): Promise<number> {
-  if (eventIds.length === 0) return 0;
-  const result = await client.query<{ event_id: string }>(
+  return (
+    await markPostgresSchedulerOutboxDeliveredRows(client, eventIds, options)
+  ).length;
+}
+
+/**
+ * Lease-fenced acknowledgement that also reports each row's event type and
+ * commit-to-acknowledgement lag on the database clock, so lag metrics never
+ * depend on skew between the projector host and Postgres.
+ */
+export async function markPostgresSchedulerOutboxDeliveredRows(
+  client: PostgresSchedulerQueryClient,
+  eventIds: string[],
+  options?: Pick<PostgresSchedulerOptions, 'schema'> & {
+    leaseOwner?: string | null;
+  },
+): Promise<RunLedgerOutboxDeliveredRow[]> {
+  if (eventIds.length === 0) return [];
+  const result = await client.query<{
+    event_id: string;
+    event_type: string;
+    lag_ms: number | string | null;
+  }>(
     `
       WITH delivered AS (
         UPDATE ${tableName('outbox', options)}
@@ -17733,9 +18027,14 @@ export async function markPostgresSchedulerOutboxDelivered(
         WHERE event_id = ANY($1::text[])
           AND delivered_at IS NULL
           AND ($2::text IS NULL OR lease_owner = $2)
-        RETURNING event_id
+        RETURNING event_id,
+                  event_type,
+                  (extract(epoch FROM (delivered_at - created_at)) * 1000)::float8
+                    AS lag_ms
       )
       SELECT event_id,
+             event_type,
+             lag_ms,
              pg_notify(
                $3,
                json_build_object('eventId', event_id, 'kind', 'outbox_delivered')::text
@@ -17748,7 +18047,51 @@ export async function markPostgresSchedulerOutboxDelivered(
       postgresSchedulerNotifyChannel('terminal', options),
     ],
   );
-  return result.rows.length;
+  return result.rows.map((row) => {
+    const lagMs = Number(row.lag_ms);
+    return {
+      eventType: row.event_type,
+      lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
+    };
+  });
+}
+
+/**
+ * Give an unfinished delivery lease back without settling a failure. Used
+ * only when the caller abandoned its own latency budget (the inline admission
+ * attempt): the row keeps its attempt count and due time, and the outbox
+ * channel wakes the durable projector to take it immediately.
+ */
+export async function releasePostgresSchedulerOutboxLease(
+  client: PostgresSchedulerQueryClient,
+  eventIds: string[],
+  options: Pick<PostgresSchedulerOptions, 'schema'> & {
+    leaseOwner: string;
+  },
+): Promise<string[]> {
+  const leaseOwner = options.leaseOwner.trim();
+  if (eventIds.length === 0 || !leaseOwner) return [];
+  const result = await client.query<{ event_id: string }>(
+    `
+      WITH released AS (
+        UPDATE ${tableName('outbox', options)}
+        SET lease_owner = NULL,
+            lease_expires_at = NULL
+        WHERE event_id = ANY($1::text[])
+          AND delivered_at IS NULL
+          AND lease_owner = $2
+        RETURNING event_id
+      )
+      SELECT event_id,
+             pg_notify(
+               $3,
+               json_build_object('eventId', event_id, 'kind', 'outbox_released')::text
+             )
+      FROM released
+    `,
+    [eventIds, leaseOwner, postgresSchedulerNotifyChannel('outbox', options)],
+  );
+  return result.rows.map((row) => row.event_id);
 }
 
 function normalizeOutboxIngressPreview(
@@ -17915,6 +18258,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
     // fabricates an integration-event log from its missing event key.
     if (waitKind === 'detached_runner') return [];
     const eventKey = stringPayloadField(event.payload, 'eventKey');
+    const waitUntil = numberPayloadField(event.payload, 'waitUntil');
     const lifecycle = {
       type: event.eventType,
       runId: event.runId,
@@ -17923,6 +18267,9 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
       ...(event.eventType === 'run.waiting' && waitKind !== null
         ? { waitKind }
         : {}),
+      ...(event.eventType === 'run.waiting' && waitUntil !== null
+        ? { waitUntil }
+        : {}),
     } as PlayRunLedgerEvent;
     if (
       waitKind !== null &&
@@ -18181,6 +18528,39 @@ export type RunLedgerOutboxDispatchInput = {
    * app-side authoritative projector. The Fly worker MUST pass `gateway`.
    */
   tokenAuthority?: RunLedgerOutboxTokenAuthority;
+  /**
+   * Run-scoped claims only: claim nothing unless this exact event is the
+   * run's pending causal head. The inline admission attempt uses it so it can
+   * only ever deliver the one `run.created` row it just committed.
+   */
+  headEventId?: string | null;
+  /**
+   * Caller-owned deadline for the app-runtime append. When it fires, the
+   * attempt is abandoned rather than failed: its lease is released without a
+   * projection-attempt increment or retry backoff, so the durable projector
+   * can take the same row immediately under the same idempotency key.
+   */
+  signal?: AbortSignal;
+  /**
+   * Observation hook for acknowledged rows (event type and commit-to-ack lag
+   * measured on the database clock). Never affects delivery.
+   */
+  onDelivered?: (deliveries: readonly RunLedgerOutboxDeliveredRow[]) => void;
+  /**
+   * Background lease holders (the outbox projector and the worker's
+   * untargeted fast path) set this: after acknowledging a batch, also carry
+   * the run's terminal fact through when it is now pending (see
+   * `continueRunLedgerOutboxThroughTerminal`). Callers that drive their own
+   * bounded batches over a pre-leased prefix (cancellation, inline
+   * `run.created`, the targeted terminal drain) leave it unset.
+   */
+  continueThroughTerminal?: boolean;
+};
+
+/** One acknowledged outbox row, for bounded-cardinality lag metrics. */
+export type RunLedgerOutboxDeliveredRow = {
+  eventType: string;
+  lagMs: number;
 };
 
 export async function dispatchRunLedgerOutboxToLedgerOnce(
@@ -18267,6 +18647,7 @@ export async function dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope
         {
           ...input,
           waitForRunLock: true,
+          continueThroughTerminal: false,
           limit:
             remaining === null ? batchSize : Math.min(batchSize, remaining),
         },
@@ -18400,6 +18781,7 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
             JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
             WHERE head.failed_at IS NULL
               AND coalesce(head.next_attempt_at, head.created_at) <= now()
+              AND ($3::text IS NULL OR head.event_id = $3::text)
               AND NOT EXISTS (
                 SELECT 1 FROM ${tableName('outbox', options)} AS active
                 WHERE active.run_id = runs.run_id
@@ -18410,7 +18792,7 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
               )
             FOR UPDATE OF runs SKIP LOCKED
           `,
-            [targetRunId, leaseOwner],
+            [targetRunId, leaseOwner, input.headEventId?.trim() || null],
           )
         : await client.query<{ run_id: string }>(`
             WITH heads AS MATERIALIZED (
@@ -18437,10 +18819,18 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
                   AND active.lease_owner IS NOT NULL
                   AND active.lease_expires_at > now()
               )
+            -- admission_first: admission, then a run whose head is its
+            -- terminal (its last fact; see
+            -- readPendingPostgresSchedulerOutboxEvents), then the FIFO
+            -- backlog. oldest_ready: pure age order, the anti-starvation turn.
             ORDER BY ${
               input.headPriority === 'oldest_ready'
                 ? ''
-                : "CASE WHEN head.event_type = 'run.created' THEN 0 ELSE 1 END,"
+                : `CASE
+                       WHEN head.event_type = 'run.created' THEN 0
+                       WHEN head.event_id LIKE 'run-terminal:%' THEN 1
+                       ELSE 2
+                     END,`
             }
                      head.created_at, runs.run_id
             FOR UPDATE OF runs SKIP LOCKED LIMIT 1
@@ -18531,6 +18921,8 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
           }
         : {}),
       retryPolicy: 'none' as const,
+      // The caller-owned admission deadline (see `signal` above) wins.
+      ...(input.signal ? { signal: input.signal } : {}),
     };
     const renewDelivery = () =>
       withClient((client) =>
@@ -18598,14 +18990,28 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
         });
       }
     }
-    const markedDelivered = await withClient((client) =>
-      markPostgresSchedulerOutboxDelivered(
+    const deliveredRows = await withClient((client) =>
+      markPostgresSchedulerOutboxDeliveredRows(
         client,
         events.map((event) => event.eventId),
         { ...options, leaseOwner },
       ),
     );
-    return {
+    const markedDelivered = deliveredRows.length;
+    if (input.onDelivered && deliveredRows.length > 0) {
+      try {
+        input.onDelivered(deliveredRows);
+      } catch (observerError) {
+        // Metrics are observation only; they never change a settled delivery.
+        console.warn('[run-ledger-outbox.delivery_observer_failed]', {
+          error:
+            observerError instanceof Error
+              ? observerError.message
+              : String(observerError),
+        });
+      }
+    }
+    const delivered: PostgresSchedulerOutboxProjectionResult = {
       attempted: events.length,
       projected: markedDelivered,
       eventIds:
@@ -18617,7 +19023,42 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
       retrying: 0,
       retryingEventIds: [],
     };
+    if (
+      input.continueThroughTerminal !== true ||
+      input.headEventId ||
+      markedDelivered !== events.length
+    ) {
+      return delivered;
+    }
+    return await continueRunLedgerOutboxThroughTerminal(
+      withClient,
+      { input, leaseOwner, runId: first.runId, delivered, events },
+      options,
+    );
   } catch (error) {
+    if (input.signal?.aborted) {
+      // The caller's latency budget expired; this is not a delivery failure.
+      // Release the lease (fenced to this owner) without charging a
+      // projection attempt or scheduling backoff, and wake the durable
+      // projector. If the append already reached the ledger, the projector's
+      // replay carries the same idempotency key and is a no-op there.
+      const released = await withClient((client) =>
+        releasePostgresSchedulerOutboxLease(
+          client,
+          events.map((event) => event.eventId),
+          { ...options, leaseOwner },
+        ),
+      );
+      return {
+        attempted: events.length,
+        projected: 0,
+        eventIds: [],
+        failed: 0,
+        failedEventIds: [],
+        retrying: released.length,
+        retryingEventIds: released,
+      };
+    }
     const settlement = await withClient(async (client) => {
       const blocked: string[] = [];
       const retrying: string[] = [];
@@ -18650,6 +19091,68 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
   }
 }
 
+/**
+ * Whoever holds a run's outbox lease delivers its terminal fact.
+ *
+ * A worker commits a run's terminal and immediately asks to deliver it
+ * (`run-terminal:<runId>`). When another dispatcher (normally the projector)
+ * is mid-append on an earlier event of the same run, that run-scoped claim
+ * finds the run leased and correctly claims nothing, and the terminal then
+ * waited for the projector's next wave: seconds of a finished run still
+ * reading "running" (live-run-observation-e2e, 4.2 s). So the lease holder,
+ * having acknowledged its batch, carries the run's terminal fact through in
+ * the same turn when it is now pending. The terminal is the run's last fact,
+ * so this is one finite causal drain (the same leased, idempotent
+ * `dispatchRunLedgerOutboxToLedgerThroughEvent` the worker uses), not a loop;
+ * a run without a pending terminal costs one indexed read.
+ */
+async function continueRunLedgerOutboxThroughTerminal(
+  withClient: <T>(
+    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>,
+  input: {
+    input: RunLedgerOutboxDispatchInput & { waitForRunLock?: boolean };
+    leaseOwner: string;
+    runId: string;
+    delivered: PostgresSchedulerOutboxProjectionResult;
+    events: readonly PostgresSchedulerOutboxEvent[];
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<PostgresSchedulerOutboxProjectionResult> {
+  const terminalEventId = `run-terminal:${input.runId}`;
+  if (input.events.some((event) => event.eventId === terminalEventId)) {
+    return input.delivered;
+  }
+  const terminal = await withClient((client) =>
+    readPostgresSchedulerOutboxDeliveryStatus(
+      client,
+      { runId: input.runId, eventId: terminalEventId },
+      { ...options, includeBlocked: false },
+    ),
+  );
+  if (!terminal.exists || terminal.delivered) return input.delivered;
+  const drained =
+    await dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope(
+      withClient,
+      {
+        ...input.input,
+        runId: input.runId,
+        leaseOwner: input.leaseOwner,
+        targetEventId: terminalEventId,
+      },
+      options,
+    );
+  return {
+    attempted: input.delivered.attempted + drained.attempted,
+    projected: input.delivered.projected + drained.projected,
+    eventIds: [...input.delivered.eventIds, ...drained.eventIds],
+    failed: drained.failed,
+    failedEventIds: drained.failedEventIds,
+    retrying: drained.retrying,
+    retryingEventIds: drained.retryingEventIds,
+  };
+}
+
 async function projectPostgresSchedulerCreatedEvent(
   runtimeApi: WorkerRuntimeApiContext,
   event: PostgresSchedulerOutboxEvent,
@@ -18660,6 +19163,7 @@ async function projectPostgresSchedulerCreatedEvent(
   const raw = (key: string) => payloadField(event.payload, key);
   const definitionScope = value('definitionScope');
   const triggerSource = value('triggerSource');
+  const launchSurface = readPlayLaunchSurface(value('launchSurface'));
   const secretRefs = raw('secretRefs');
   const inputSummary = raw('inputSummary');
   const inputBytes = number('inputBytes');
@@ -18678,6 +19182,9 @@ async function projectPostgresSchedulerCreatedEvent(
     workflowId: event.runId,
     runId: event.runId,
     workflowFamilyKey: value('workflowFamilyKey') ?? event.runId,
+    // Written in the admission transaction, so this is when the run was
+    // accepted, not when the projection happened to deliver it.
+    acceptedAt: event.createdAt.getTime(),
     ...(value('parentRunId') ? { parentRunId: value('parentRunId') } : {}),
     ...(value('rootRunId') ? { rootRunId: value('rootRunId') } : {}),
     ...(value('asyncLaunchKey')
@@ -18729,6 +19236,7 @@ async function projectPostgresSchedulerCreatedEvent(
     triggerSource === 'sql_listener'
       ? { triggerSource }
       : {}),
+    ...(launchSurface ? { launchSurface } : {}),
     ...(inputFileId ? { inputFileId } : {}),
     ...(inputBytes !== null ? { inputBytes } : {}),
     ...(inputSha256 ? { inputSha256 } : {}),