deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -0,0 +1,355 @@
+/**
+ * Recovery loop for durable Run Ledger outbox events.
+ *
+ * Runtime workers may accelerate their terminal event, but this controller is
+ * the independent delivery owner for durable lifecycle facts. In particular,
+ * it projects `run.created` before an execution slot is available. It has one
+ * durable source of truth (the outbox), one direct Postgres LISTEN connection
+ * as a wake hint, and one exact timer for the next due row. There is
+ * deliberately no worker fleet polling cadence.
+ */
+import {
+  dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope,
+  postgresSchedulerNotifyChannel,
+  readPostgresSchedulerOutboxNextDueAt,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerOutboxProjectionResult,
+} from './postgres';
+import {
+  acquirePgNotifyHub,
+  withRuntimeSchedulerClient,
+  type PgNotifyHubLease,
+  type PgNotifyHubState,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+
+const RECOVERY_FAILURE_RETRY_BASE_MS = 5_000;
+const RECOVERY_FAILURE_RETRY_MAX_MS = 60_000;
+// LISTEN/NOTIFY is the normal, immediate wake path. This fallback exists only
+// while the optional direct connection is unavailable. It is intentionally
+// short: a durable `run.created` fact is on the execution causal path, so a
+// direct-session outage must add one small indexed poll rather than tens of
+// seconds of worker deferrals. Only the dedicated projector performs this
+// fallback; the worker fleet still never polls the outbox.
+export const NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS = 1_000;
+const MAX_TIMEOUT_MS = 2_147_483_647;
+/**
+ * Two short claim/deliver waves match each projector's bounded scheduler pool.
+ * The claim query uses SKIP LOCKED, so the wave can advance separate runs
+ * without splitting the ordered lifecycle of any one run.
+ */
+export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 2;
+
+type RecoveryTimer = ReturnType<typeof setTimeout>;
+
+export type RunLedgerOutboxRecoveryLogger = Pick<Console, 'info' | 'warn'>;
+
+export type RunLedgerOutboxRecoveryDeps = {
+  /** One bounded durable claim/delivery attempt. */
+  dispatchOnce: () => Promise<PostgresSchedulerOutboxProjectionResult>;
+  /** Earliest pending retry time or active lease expiry in the durable outbox. */
+  readNextDueAt: () => Promise<Date | null>;
+  /** A direct LISTEN connection. It is a hint only; a null lease is safe. */
+  acquireNotifyHub: () => PgNotifyHubLease | null;
+  /** Bounded number of independent runs this replica may deliver at once. */
+  parallelism?: number;
+  now?: () => number;
+  setTimer?: (callback: () => void, delayMs: number) => RecoveryTimer;
+  clearTimer?: (timer: RecoveryTimer) => void;
+  logger?: RunLedgerOutboxRecoveryLogger;
+};
+
+export type RunLedgerOutboxRecoveryController = {
+  /**
+   * Ask the singleton projector to re-read durable outbox work. This is
+   * intentionally a coalesced wake, never an unbounded concurrent drain.
+   */
+  wake: (reason?: string) => void;
+  /** Whether the optional direct LISTEN hint is currently usable. */
+  notifyState: () => PgNotifyHubState | 'unavailable';
+  close: () => Promise<void>;
+};
+
+function boundedFailureDelayMs(failures: number): number {
+  const exponent = Math.min(Math.max(0, failures - 1), 4);
+  return Math.min(
+    RECOVERY_FAILURE_RETRY_MAX_MS,
+    RECOVERY_FAILURE_RETRY_BASE_MS * 2 ** exponent,
+  );
+}
+
+/**
+ * Start a bounded, work-conserving recovery controller from injected seams.
+ * Subscription happens before the first database scan, so a terminal commit
+ * cannot slip between startup scan and listener registration.
+ */
+export function startRunLedgerOutboxRecovery(
+  deps: RunLedgerOutboxRecoveryDeps,
+): RunLedgerOutboxRecoveryController {
+  const now = deps.now ?? Date.now;
+  const setTimer = deps.setTimer ?? setTimeout;
+  const clearTimer = deps.clearTimer ?? clearTimeout;
+  const logger = deps.logger ?? console;
+  const requestedParallelism = Math.floor(deps.parallelism ?? 1);
+  const parallelism = Number.isFinite(requestedParallelism)
+    ? Math.max(
+        1,
+        Math.min(
+          RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
+          requestedParallelism,
+        ),
+      )
+    : 1;
+  let closed = false;
+  let requested = false;
+  let failureCount = 0;
+  let timer: RecoveryTimer | null = null;
+  let drainPromise: Promise<void> | null = null;
+  let cooldownUntilMs: number | null = null;
+  let hubLease: PgNotifyHubLease | null = null;
+  let unsubscribeNotify: (() => void) | undefined;
+  let unsubscribeState: (() => void) | undefined;
+  let requestDrain: (reason: string) => void = () => undefined;
+
+  const clearWakeTimer = () => {
+    if (!timer) return;
+    clearTimer(timer);
+    timer = null;
+  };
+
+  const listenerIsAvailable = () => hubLease?.hub.state() === 'listening';
+
+  const attachNotifyHub = () => {
+    if (closed || hubLease) return;
+    const lease = deps.acquireNotifyHub();
+    if (!lease) return;
+    hubLease = lease;
+    unsubscribeNotify = lease.hub.subscribe(() =>
+      requestDrain('postgres_notify'),
+    );
+    unsubscribeState = lease.hub.subscribeState((state) => {
+      // A reconnect is a durable catch-up boundary. The controller re-reads
+      // Postgres rather than trusting payloads from the notification channel.
+      if (state === 'listening') requestDrain('listener_connected');
+      // `down` and `standby` are safe: scheduleNextDue installs the indexed
+      // safety wake after this scan reaches idle.
+      else requestDrain('listener_unavailable');
+    });
+  };
+
+  requestDrain = (reason: string) => {
+    if (closed) return;
+    requested = true;
+    if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
+    clearWakeTimer();
+    if (drainPromise) return;
+    drainPromise = drain(reason)
+      .catch((error) => {
+        // drain() handles expected database/projection faults itself. This is a
+        // final guard so the singleton process never dies silently.
+        logger.warn('[run-ledger-outbox.recovery_unhandled_failure]', {
+          reason,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      })
+      .finally(() => {
+        drainPromise = null;
+        if (requested && !closed) requestDrain('coalesced_wake');
+      });
+  };
+
+  const recordProjectionRetry = (retrying: number) => {
+    failureCount += 1;
+    logger.warn('[run-ledger-outbox.recovery_retry]', {
+      stage: 'projection',
+      retrying,
+    });
+  };
+
+  const scheduleFailureRetry = (
+    stage: 'dispatch' | 'next_due',
+    error: unknown,
+  ) => {
+    failureCount += 1;
+    // A rejected pooled query is normally worth exponential backoff. The
+    // exception is an unavailable LISTEN session: this dedicated projector is
+    // the one durable poller, and its one-second safety wake is what keeps
+    // `run.created` delivery live while a direct PostgreSQL connection is
+    // refused (for example, during a connection-capacity spike). Do not let a
+    // process-local exponential cooldown silently disable that fallback.
+    const delayMs = listenerIsAvailable()
+      ? boundedFailureDelayMs(failureCount)
+      : NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS;
+    logger.warn('[run-ledger-outbox.recovery_retry]', {
+      stage,
+      delayMs,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    clearWakeTimer();
+    cooldownUntilMs = now() + delayMs;
+    timer = setTimer(() => {
+      timer = null;
+      cooldownUntilMs = null;
+      requestDrain('recovery_error_retry');
+    }, delayMs);
+    timer.unref?.();
+  };
+
+  const scheduleNextDue = async (): Promise<void> => {
+    let dueAt: Date | null;
+    try {
+      dueAt = await deps.readNextDueAt();
+      failureCount = 0;
+    } catch (error) {
+      scheduleFailureRetry('next_due', error);
+      return;
+    }
+    if (closed || requested) return;
+    // When LISTEN is healthy the exact durable due time is sufficient. If the
+    // optional listener is absent, down, or lost an election, an indexed
+    // short safety wake discovers a fresh row that otherwise had no
+    // notification or existing due time to wake this replica.
+    const exactDueMs = dueAt ? Math.max(0, dueAt.getTime() - now()) : null;
+    const delayMs = listenerIsAvailable()
+      ? exactDueMs
+      : Math.min(exactDueMs ?? Infinity, NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS);
+    if (delayMs === null || !Number.isFinite(delayMs)) return;
+    clearWakeTimer();
+    timer = setTimer(
+      () => {
+        timer = null;
+        if (!listenerIsAvailable()) attachNotifyHub();
+        requestDrain(dueAt ? 'next_due' : 'listener_safety_wake');
+      },
+      Math.min(MAX_TIMEOUT_MS, delayMs),
+    );
+    timer.unref?.();
+  };
+
+  const drain = async (_reason: string): Promise<void> => {
+    while (requested && !closed) {
+      requested = false;
+      const settled = await Promise.allSettled(
+        Array.from({ length: parallelism }, () => deps.dispatchOnce()),
+      );
+      const rejected = settled.find(
+        (result): result is PromiseRejectedResult =>
+          result.status === 'rejected',
+      );
+      if (rejected) {
+        scheduleFailureRetry('dispatch', rejected.reason);
+        return;
+      }
+
+      const results = settled.map(
+        (result) =>
+          (
+            result as PromiseFulfilledResult<PostgresSchedulerOutboxProjectionResult>
+          ).value,
+      );
+      const retrying = results.reduce(
+        (total, result) => total + result.retrying,
+        0,
+      );
+
+      if (retrying > 0) {
+        // The failed row owns its own durable retry clock. Do NOT stop this
+        // drain: the claim query preserves causal order within a run, while
+        // independent runs are safe to project now. Returning here turns one
+        // transient projection failure into global head-of-line blocking when
+        // this is the only healthy projector replica.
+        recordProjectionRetry(retrying);
+      } else {
+        failureCount = 0;
+      }
+
+      // Continue until no event was claimable. A transient failure schedules
+      // its own durable retry, so the next pass naturally skips that run while
+      // moving unrelated runs forward. A permanent failure is settled once and
+      // likewise cannot block another run's causal prefix.
+      if (results.some((result) => result.attempted > 0)) {
+        requested = true;
+        continue;
+      }
+      await scheduleNextDue();
+    }
+  };
+
+  // Subscribe first, then scan. Notifications are lossy by design, so every
+  // wake re-reads durable state and reconnect transitions also trigger a scan.
+  attachNotifyHub();
+  requestDrain('startup');
+
+  return {
+    wake: (reason = 'external_safety_wake') => requestDrain(reason),
+    notifyState: () => hubLease?.hub.state() ?? 'unavailable',
+    close: async () => {
+      if (closed) return;
+      closed = true;
+      clearWakeTimer();
+      cooldownUntilMs = null;
+      unsubscribeNotify?.();
+      unsubscribeState?.();
+      await drainPromise;
+      hubLease?.release();
+    },
+  };
+}
+
+/**
+ * Production wiring shared by each stateless Fly projector replica. Every
+ * replica uses gateway authority because Fly holds no signing key; leased
+ * outbox rows and idempotent appends provide the coordination boundary.
+ */
+export function startRunLedgerOutboxRecoveryController(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  /** Direct LISTEN is a latency hint and may be disabled for isolated schemas. */
+  useDirectNotify?: boolean;
+}): RunLedgerOutboxRecoveryController {
+  const schedulerOptions = input.schedulerOptions ?? {};
+  return startRunLedgerOutboxRecovery({
+    acquireNotifyHub: () =>
+      input.useDirectNotify === false
+        ? null
+        : acquirePgNotifyHub(
+            runLedgerOutboxRecoveryNotifyHubInput({
+              poolOptions: input.poolOptions,
+              schedulerOptions,
+            }),
+          ),
+    dispatchOnce: () =>
+      dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+        (operation) => withRuntimeSchedulerClient(input.poolOptions, operation),
+        { tokenAuthority: 'gateway', limit: 8 },
+        schedulerOptions,
+      ),
+    parallelism: Math.min(
+      RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
+      Math.max(1, input.poolOptions.maxClients ?? 1),
+    ),
+    readNextDueAt: () =>
+      withRuntimeSchedulerClient(input.poolOptions, (client) =>
+        readPostgresSchedulerOutboxNextDueAt(client, schedulerOptions),
+      ),
+  });
+}
+
+/** Production projector replicas LISTEN; durable row leases arbitrate work. */
+export function runLedgerOutboxRecoveryNotifyHubInput(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+}): {
+  url: string;
+  channel: string;
+  applicationName: string;
+} {
+  return {
+    url: input.poolOptions.url,
+    channel: postgresSchedulerNotifyChannel(
+      'outbox',
+      input.schedulerOptions ?? {},
+    ),
+    applicationName: 'deepline-run-ledger-outbox-recovery',
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts

@@ -0,0 +1,46 @@
+import {
+  createPlayRunTerminalJob,
+  createPostgresPlayRunTerminalPorts,
+  resolvePlayRunTerminalAuthority,
+  type PlayRunTerminalAuthority,
+  type PlayRunTerminalOutcome,
+} from '../play-runs/terminal';
+import type { PostgresSchedulerQueryClient } from './postgres';
+
+/**
+ * Compatibility facade for older receipt-gateway imports. New code belongs in
+ * the Play Run Terminal Module, whose job owns the transition law and whose
+ * Postgres Adapter owns scheduler SQL.
+ */
+export type RunnerTerminalAttemptAuthority = PlayRunTerminalAuthority;
+export const resolveRunnerTerminalAttemptAuthority =
+  resolvePlayRunTerminalAuthority;
+export type RunnerTerminalIngressOutcome = PlayRunTerminalOutcome;
+
+/**
+ * Accept a detached runner's terminal result.
+ *
+ * Eligible completions commit their terminal state, cleanup obligations, and
+ * Absurd retirement event atomically. Compatibility outcomes persist the
+ * terminal receipt before queuing the retirement event. Callers do not need to
+ * know the event name or notification channel.
+ */
+export async function acceptRunnerTerminal(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    queueName: string;
+    runId: string;
+    attempt: number;
+    authority?: RunnerTerminalAttemptAuthority | null;
+    result: unknown;
+  },
+  options?: { schema?: string },
+): Promise<RunnerTerminalIngressOutcome> {
+  const job = createPlayRunTerminalJob(
+    createPostgresPlayRunTerminalPorts({
+      client,
+      schema: options?.schema,
+    }),
+  );
+  return await job.accept(input);
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runner-push.ts

@@ -0,0 +1,60 @@
+import type { PostgresSchedulerQueryClient } from './postgres';
+import {
+  heartbeatPostgresSchedulerDetachedRunner,
+  readPostgresSchedulerRunnerLiveness,
+} from './postgres';
+
+/**
+ * Detached-runner heartbeat control for the receipt gateway.
+ *
+ * The worker parks after the detached runner start — it holds no claim while
+ * the sandbox executes — so a parked run has no Absurd claim to extend.
+ * Heartbeats read the Play Run status, update attempt liveness, and carry the
+ * cancellation signal back to the runner.
+ *
+ * Terminal ingress lives in the Run Terminal Transition Module.
+ */
+
+export type RunnerHeartbeatOutcome =
+  | { ok: true; schedulerStatus: 'queued' | 'running' | 'waiting' }
+  | { ok: false; terminal: 'cancelled' | 'failed' | 'completed' };
+
+export async function runnerHeartbeat(
+  client: PostgresSchedulerQueryClient,
+  input: { runId: string; attempt: number },
+  options?: { schema?: string },
+): Promise<RunnerHeartbeatOutcome | null> {
+  const status = await heartbeatPostgresSchedulerDetachedRunner(
+    client,
+    input,
+    options,
+  );
+  if (status === null) return null;
+  if (status === 'cancelled') return { ok: false, terminal: 'cancelled' };
+  if (status === 'failed') return { ok: false, terminal: 'failed' };
+  if (status === 'completed') return { ok: false, terminal: 'completed' };
+  return { ok: true, schedulerStatus: status };
+}
+
+export async function runnerReadiness(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    runId: string;
+    attempt: number;
+    baselineHeartbeatAt?: string | null;
+  },
+  options?: { schema?: string },
+): Promise<{ ready: boolean; heartbeatAt: string | null } | null> {
+  const liveness = await readPostgresSchedulerRunnerLiveness(
+    client,
+    { runId: input.runId, attempt: input.attempt },
+    options,
+  );
+  if (!liveness) return null;
+  return {
+    ready:
+      input.baselineHeartbeatAt !== undefined &&
+      liveness.heartbeatAt !== input.baselineHeartbeatAt,
+    heartbeatAt: liveness.heartbeatAt,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-api-origin.ts

@@ -0,0 +1,33 @@
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+
+/**
+ * Resolve the app origin used for authenticated runtime API calls.
+ *
+ * New Play Launch Envelopes carry an executor-reachable runtime API origin.
+ * Older launch rows predate that field and retain the callback URL fallback.
+ */
+export function runtimeApiBaseUrlFromLaunch(
+  launch: Pick<PlaySchedulerSubmitInput, 'baseUrl' | 'runtimeApiBaseUrl'>,
+): string {
+  return launch.runtimeApiBaseUrl?.trim() || launch.baseUrl;
+}
+
+/**
+ * Resolve the origin whose JWKS verifies a run's executor token.
+ *
+ * The API endpoint remains launch-pinned so a staged Vercel deployment is
+ * genuinely exercised before promotion. The signing origin instead remains
+ * the stable public production authority, which both gateway fleets already
+ * trust. Legacy launch rows deliberately preserve the old single-origin
+ * behavior.
+ */
+export function runtimeTokenOriginFromLaunch(
+  launch: Pick<
+    PlaySchedulerSubmitInput,
+    'baseUrl' | 'runtimeApiBaseUrl' | 'runtimeTokenOrigin'
+  >,
+): string {
+  return (
+    launch.runtimeTokenOrigin?.trim() || runtimeApiBaseUrlFromLaunch(launch)
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-gateway-health.ts

@@ -0,0 +1,64 @@
+import {
+  LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER,
+  LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE,
+} from '../durable-invocation-fence';
+
+const RECEIPT_GATEWAY_HEALTH_PATH = '/healthz';
+const RECEIPT_GATEWAY_HEALTH_TIMEOUT_MS = 5_000;
+
+function gatewayOrigin(gatewayBaseUrl: string): string {
+  try {
+    const url = new URL(gatewayBaseUrl);
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+      throw new Error('unsupported protocol');
+    }
+    return url.origin;
+  } catch {
+    throw new Error(
+      'Runtime gateway URL must be a valid absolute URL using HTTP(S).',
+    );
+  }
+}
+
+export async function verifyRuntimeReceiptGatewayHealth(input: {
+  gatewayBaseUrl: string;
+  fetchImpl?: typeof fetch;
+  timeoutMs?: number;
+}): Promise<void> {
+  let origin: string;
+  try {
+    origin = gatewayOrigin(input.gatewayBaseUrl);
+  } catch {
+    throw new Error(
+      'DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL must be a valid absolute URL using HTTP(S).',
+    );
+  }
+
+  const response = await (input.fetchImpl ?? fetch)(
+    `${origin}${RECEIPT_GATEWAY_HEALTH_PATH}`,
+    {
+      method: 'GET',
+      redirect: 'manual',
+      signal: AbortSignal.timeout(
+        input.timeoutMs ?? RECEIPT_GATEWAY_HEALTH_TIMEOUT_MS,
+      ),
+    },
+  );
+  if (!response.ok) {
+    throw new Error(
+      `Receipt gateway ${origin} health check returned HTTP ${response.status}.`,
+    );
+  }
+  // This proves only receipt-gateway identity and liveness. v2 execution
+  // capability is checked by the caller of execute-fenced-v2 against the
+  // execution receiver; making it a worker boot condition would block valid
+  // ordinary and receipt traffic during a split-gateway rollout.
+  if (
+    response.headers.get(LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER) !==
+    LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE
+  ) {
+    throw new Error(
+      `Receipt gateway ${origin} did not attest itself with ${LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_HEADER}: ${LEGACY_DURABLE_INVOCATION_FENCE_ATTESTATION_VALUE}.`,
+    );
+  }
+}