deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-metrics.ts

@@ -0,0 +1,120 @@
+import type { WorkerQueueMetrics } from './worker-queue-metrics';
+import {
+  renderRuntimeProcessResourceMetrics,
+  type WorkerResourceMetrics,
+} from './worker-resource-metrics';
+
+/** A `/metrics` response is rendered solely from a process-local snapshot. */
+export type AbsurdWorkerMetricsSnapshot = Readonly<{
+  queue: WorkerQueueMetrics;
+  schema: string;
+  releaseId: string;
+  queueName: string;
+  workerName: string | null;
+  pool: Readonly<{
+    totalCount: number;
+    idleCount: number;
+    waitingCount: number;
+    capacityRejections: number;
+    saturationRejections: number;
+    breakerRejections: number;
+  }>;
+  resources: WorkerResourceMetrics;
+  launchReads: readonly Readonly<{
+    caller: string;
+    releaseId: string;
+    found: number;
+    missing: number;
+    errors: number;
+    durationMsTotal: number;
+  }>[];
+}>;
+
+function prometheusLabel(value: string): string {
+  return value
+    .replaceAll('\\', '\\\\')
+    .replaceAll('"', '\\"')
+    .replaceAll('\n', '\\n');
+}
+
+/**
+ * This deliberately has no database Adapter. Callers must collect all values
+ * while doing normal work, then pass the in-memory snapshot to this renderer.
+ */
+export function renderAbsurdWorkerMetrics(
+  input: AbsurdWorkerMetricsSnapshot,
+): string {
+  const labels = `{schema="${prometheusLabel(input.schema)}",worker_release_id="${prometheusLabel(input.releaseId)}",worker_queue="${prometheusLabel(input.queueName)}",worker_name="${prometheusLabel(input.workerName ?? 'unnamed')}"}`;
+  return [
+    '# HELP deepline_absurd_runtime_worker_info Current worker identity. A series exists only while this Fly Machine is scrapeable.',
+    '# TYPE deepline_absurd_runtime_worker_info gauge',
+    `deepline_absurd_runtime_worker_info${labels} 1`,
+    '# HELP deepline_absurd_runtime_queue_metrics_available Whether this worker can provide current global queue state without a database scan.',
+    '# TYPE deepline_absurd_runtime_queue_metrics_available gauge',
+    `deepline_absurd_runtime_queue_metrics_available${labels} 0`,
+    '# HELP deepline_absurd_runtime_queue_depth Queued attempts.',
+    '# TYPE deepline_absurd_runtime_queue_depth gauge',
+    `deepline_absurd_runtime_queue_depth${labels} ${input.queue.queued}`,
+    '# HELP deepline_absurd_runtime_running Running attempts.',
+    '# TYPE deepline_absurd_runtime_running gauge',
+    `deepline_absurd_runtime_running${labels} ${input.queue.running}`,
+    '# HELP deepline_absurd_runtime_oldest_queued_seconds Age of the oldest queued attempt.',
+    '# TYPE deepline_absurd_runtime_oldest_queued_seconds gauge',
+    `deepline_absurd_runtime_oldest_queued_seconds${labels} ${input.queue.oldest_queued_s}`,
+    '# HELP deepline_absurd_runtime_oldest_running_seconds Age of the oldest running attempt.',
+    '# TYPE deepline_absurd_runtime_oldest_running_seconds gauge',
+    `deepline_absurd_runtime_oldest_running_seconds${labels} ${input.queue.oldest_running_s}`,
+    '# HELP deepline_absurd_runtime_stale_leases Running attempts whose lease expired.',
+    '# TYPE deepline_absurd_runtime_stale_leases gauge',
+    `deepline_absurd_runtime_stale_leases${labels} ${input.queue.stale_leases}`,
+    '# HELP deepline_absurd_runtime_completed_runner_handoffs Completed detached-runner terminals awaiting scheduler finalization.',
+    '# TYPE deepline_absurd_runtime_completed_runner_handoffs gauge',
+    `deepline_absurd_runtime_completed_runner_handoffs${labels} ${input.queue.completed_runner_handoffs}`,
+    '# HELP deepline_absurd_runtime_running_without_lease Running attempts with no lease expiry.',
+    '# TYPE deepline_absurd_runtime_running_without_lease gauge',
+    `deepline_absurd_runtime_running_without_lease${labels} ${input.queue.running_without_lease}`,
+    '# HELP deepline_absurd_runtime_stale_heartbeats Running attempts without a recent heartbeat.',
+    '# TYPE deepline_absurd_runtime_stale_heartbeats gauge',
+    `deepline_absurd_runtime_stale_heartbeats${labels} ${input.queue.stale_heartbeats}`,
+    '# HELP deepline_absurd_runtime_play_run_dead_letters Failed Play Runs awaiting operator recovery.',
+    '# TYPE deepline_absurd_runtime_play_run_dead_letters gauge',
+    `deepline_absurd_runtime_play_run_dead_letters${labels} ${input.queue.play_run_dead_letters}`,
+    '# HELP deepline_absurd_runtime_scheduler_pool_clients Checked-out and idle scheduler pool clients.',
+    '# TYPE deepline_absurd_runtime_scheduler_pool_clients gauge',
+    `deepline_absurd_runtime_scheduler_pool_clients${labels} ${input.pool.totalCount}`,
+    '# HELP deepline_absurd_runtime_scheduler_pool_idle_clients Idle scheduler pool clients.',
+    '# TYPE deepline_absurd_runtime_scheduler_pool_idle_clients gauge',
+    `deepline_absurd_runtime_scheduler_pool_idle_clients${labels} ${input.pool.idleCount}`,
+    '# HELP deepline_absurd_runtime_scheduler_pool_waiters Pending scheduler pool acquires.',
+    '# TYPE deepline_absurd_runtime_scheduler_pool_waiters gauge',
+    `deepline_absurd_runtime_scheduler_pool_waiters${labels} ${input.pool.waitingCount}`,
+    '# HELP deepline_absurd_runtime_scheduler_capacity_rejections_total Deferred work due to PgBouncer or local pool capacity.',
+    '# TYPE deepline_absurd_runtime_scheduler_capacity_rejections_total counter',
+    `deepline_absurd_runtime_scheduler_capacity_rejections_total${labels} ${input.pool.capacityRejections}`,
+    '# HELP deepline_absurd_runtime_scheduler_saturation_rejections_total Local scheduler pool saturation rejections.',
+    '# TYPE deepline_absurd_runtime_scheduler_saturation_rejections_total counter',
+    `deepline_absurd_runtime_scheduler_saturation_rejections_total${labels} ${input.pool.saturationRejections}`,
+    '# HELP deepline_absurd_runtime_scheduler_breaker_rejections_total Database failure circuit rejections.',
+    '# TYPE deepline_absurd_runtime_scheduler_breaker_rejections_total counter',
+    `deepline_absurd_runtime_scheduler_breaker_rejections_total${labels} ${input.pool.breakerRejections}`,
+    renderRuntimeProcessResourceMetrics({
+      service: 'scheduler',
+      machine: input.workerName ?? 'local',
+      metrics: input.resources,
+    }),
+    '# HELP deepline_absurd_runtime_launch_reads_total Immutable launch lookups by callsite and resolved release.',
+    '# TYPE deepline_absurd_runtime_launch_reads_total counter',
+    '# HELP deepline_absurd_runtime_launch_read_duration_milliseconds_total Total immutable launch-read database time by callsite and resolved release.',
+    '# TYPE deepline_absurd_runtime_launch_read_duration_milliseconds_total counter',
+    ...input.launchReads.flatMap((metric) => {
+      const metricLabels = `${labels.slice(0, -1)},caller="${prometheusLabel(metric.caller)}",launch_release_id="${prometheusLabel(metric.releaseId)}"`;
+      return [
+        `deepline_absurd_runtime_launch_reads_total${metricLabels},outcome="found"} ${metric.found}`,
+        `deepline_absurd_runtime_launch_reads_total${metricLabels},outcome="missing"} ${metric.missing}`,
+        `deepline_absurd_runtime_launch_reads_total${metricLabels},outcome="error"} ${metric.errors}`,
+        `deepline_absurd_runtime_launch_read_duration_milliseconds_total${metricLabels}} ${metric.durationMsTotal}`,
+      ];
+    }),
+    '',
+  ].join('\n');
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts

@@ -0,0 +1,76 @@
+import {
+  normalizePlayRuntimeBackend,
+  type PlayRuntimeBackendId,
+} from '../backend';
+import { runtimeSandboxPlacementPoliciesForBackend } from '../runtime-sandbox-placement-policy';
+
+/**
+ * Boot-time required-env contract for the Absurd runtime worker entrypoint.
+ *
+ * Extracted from `apps/absurd-runtime-worker/index.ts` so the invariant is
+ * importable (and testable) without launching the worker process.
+ *
+ * SECURITY INVARIANT: the Fly worker NEVER holds
+ * an auth signing key. It proves its active scheduler attempt to the launch-
+ * pinned Vercel deployment and receives a short-lived executor JWT; the gateway
+ * verifies that JWT against the deployment JWKS. So `BETTER_AUTH_SECRET` must
+ * never appear in this required set — a worker that demanded it would either
+ * refuse to boot without a key it must not have, or (worse) be handed one and
+ * start minting tokens locally, defeating the lane-isolation model.
+ * `tests/lib/plays/absurd-worker-required-env.test.ts` guards this.
+ *
+ * - `WORKFLOW_RUNTIME_POSTGRES`: the runtime scheduler Postgres URL.
+ * - provider credentials are selected by `DEEPLINE_PLAY_RUNNER_BACKEND`.
+ * - `DEEPLINE_INTERNAL_TOKEN`: authenticates the worker -> app runtime API
+ *   (ledger, billing, sessions, and the executor-JWT gateways).
+ * - `DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL`: the only receipt and Runtime Sheet
+ *   persistence path available to Daytona runners.
+ */
+export const ABSURD_WORKER_REQUIRED_ENV = [
+  'WORKFLOW_RUNTIME_POSTGRES',
+  'DEEPLINE_INTERNAL_TOKEN',
+  'DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL',
+] as const;
+
+export function runtimeSandboxProviderCapabilities(
+  backend: PlayRuntimeBackendId,
+): Array<'daytona' | 'modal'> {
+  return [
+    ...new Set(
+      runtimeSandboxPlacementPoliciesForBackend(backend).flatMap(
+        (policy) => policy.cleanupProviders,
+      ),
+    ),
+  ];
+}
+
+/**
+ * A signing-key env that the Fly worker must never require at boot. The worker
+ * is signing-key-free by design; requiring any of these would violate the lane-
+ * isolation invariant above.
+ */
+export const ABSURD_WORKER_FORBIDDEN_REQUIRED_ENV = [
+  'BETTER_AUTH_SECRET',
+] as const;
+
+export function assertAbsurdWorkerRequiredEnv(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+): void {
+  const backend = normalizePlayRuntimeBackend(
+    env.DEEPLINE_PLAY_RUNNER_BACKEND?.trim() || 'daytona',
+  );
+  const providerRequired = [
+    ...new Set(
+      runtimeSandboxPlacementPoliciesForBackend(backend).flatMap(
+        (policy) => policy.requiredCredentialEnv,
+      ),
+    ),
+  ];
+  const missing = [...providerRequired, ...ABSURD_WORKER_REQUIRED_ENV].filter(
+    (name) => !env[name]?.trim(),
+  );
+  if (missing.length === 0) return;
+  throw new Error(
+    `Missing required absurd runtime worker configuration: ${missing.join(', ')}.`,
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts

@@ -0,0 +1,476 @@
+/**
+ * `absurd` play-runtime scheduler backend.
+ *
+ * Postgres-native durable execution: the vendored Absurd engine
+ * (`packages/absurd`) provides the queue/claim/park/resume control plane,
+ * while the battle-tested Postgres run-state store (`postgres.ts`) remains the
+ * source of truth for launch rows, attempts, receipts, signals, and terminal
+ * state. This module is greenfield and does not import legacy scheduler
+ * modules.
+ *
+ * Lifecycle:
+ *   submit  → ensure schemas + queues → insert launch row (postgres) → spawn an
+ *             absurd task (idempotencyKey=runId, orgKey=orgId) on `runs`.
+ *             Persist the absurd task id
+ *             on the run row so a later cancel can target it.
+ *   observe/result → reuse the postgres LISTEN/NOTIFY run handle.
+ *   cancel  → postgres terminal cancel intent + absurd cancelTask.
+ *   signal  → durable signal row (authoritative payload) + absurd emitEvent
+ *             (the wakeup only; payload is re-read from the signal row).
+ */
+import { recordPerfTrace } from '../perf-trace';
+import { CTX_RUN_PLAY_INLINE_ONLY } from '../child-execution-strategy';
+import {
+  defaultRuntimeSandboxReconciliation,
+  runtimeSandboxDeleteSucceeded,
+  runtimeSandboxRefIdentityKey,
+  runtimeSandboxRefFromResource,
+  type RuntimeSandboxRef,
+} from '../runner-backends/runtime-sandbox-reconciliation';
+import {
+  PLAY_SCHEDULER_BACKENDS,
+  type PlaySchedulerBackend,
+  type PlaySchedulerRunHandle,
+  type PlaySchedulerSubmitInput,
+} from '../scheduler-backend';
+import {
+  postgresSchedulerRunHandle,
+  readPostgresSchedulerLaunch,
+  readPostgresSchedulerAbsurdTaskId,
+  readPostgresSchedulerRuntimeResourcesForRun,
+  recordPostgresSchedulerSignal,
+} from './postgres';
+import {
+  createPlayRunAdmissionJob,
+  createPostgresPlayRunAdmissionPorts,
+} from '../play-runs/admission';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
+import {
+  ABSURD_PLAY_RUN_MAX_ATTEMPTS,
+  ABSURD_PLAY_RUN_RETRY_STRATEGY,
+  ABSURD_DAYTONA_SANDBOX_CLEANUP_TASK,
+  absurdIntegrationEventName,
+  absurdQueueNameForRelease,
+  absurdReleaseIdForLaunch,
+  absurdReleaseIdForSuffix,
+  createAbsurdRuntimeClient,
+  ensureAbsurdRuntimeReady,
+  normalizeAbsurdSchedulerOptions,
+  resolveAbsurdPoolOptions,
+  type AbsurdDaytonaSandboxCleanupTaskParams,
+  type AbsurdSchedulerOptions,
+} from './absurd-shared';
+
+export type AbsurdSchedulerBackendOptions = AbsurdSchedulerOptions & {
+  /**
+   * Explicit per-request queue suffix, resolved in `lazy.ts` from the
+   * per-request scheduler schema. When absent, queue names fall back to the
+   * process env suffix (Fly worker / dev:v2) or base names (unit tests / prod).
+   * This is what makes the shared hosted preview app enqueue onto the SAME
+   * suffixed queues the isolated Fly worker drains.
+   */
+  queueSuffix?: string | null;
+};
+
+function recordAbsurdSubmitPhase(
+  input: Pick<PlaySchedulerSubmitInput, 'runId' | 'playName' | 'graphHash'>,
+  phase: string,
+  startedAt: number,
+  extra: Record<string, unknown> = {},
+): void {
+  recordPerfTrace({
+    source: 'server',
+    runId: input.runId,
+    playName: input.playName,
+    graphHash: input.graphHash,
+    phase: `server.absurd_${phase}`,
+    ms: Date.now() - startedAt,
+    extra,
+  });
+}
+
+function absurdRunHandle(input: {
+  runId: string;
+  schedulerOptions: AbsurdSchedulerOptions;
+  queueSuffix?: string | null;
+  releaseId?: string | null;
+  initialState?: Record<string, unknown> | null;
+}): PlaySchedulerRunHandle & { initialState?: Record<string, unknown> } {
+  const schedulerOptions = input.schedulerOptions;
+  const delegate = postgresSchedulerRunHandle(input.runId, {
+    ...schedulerOptions,
+    // Release lanes can contain workers that predate the terminal-notify
+    // contract. Until the release registry versions this capability, polling
+    // every second is the only safe completion bound for attached runs.
+    // TODO(runtime-terminal-notify): use a low-risk two-phase rollout. First,
+    // enable notifications only as an additive wake hint while retaining the
+    // one-second safety poll for mixed-version workers. After every pre-notify
+    // lane and suspended run has drained, long-lived non-Vercel listeners may
+    // move to the sparse safety cadence. Vercel must keep polling unless a
+    // fixed-cardinality notification relay owns the direct LISTEN sessions;
+    // PostgreSQL notifications are never the durable source of truth.
+    terminalNotifications: false,
+  });
+  const poolOptions = resolveAbsurdPoolOptions(schedulerOptions.url);
+
+  const initialStateSchema =
+    typeof input.initialState?.schedulerSchema === 'string' &&
+    input.initialState.schedulerSchema.trim()
+      ? input.initialState.schedulerSchema.trim()
+      : null;
+  const effectiveOptions: AbsurdSchedulerOptions = initialStateSchema
+    ? { schema: initialStateSchema }
+    : schedulerOptions;
+
+  // The run rides ONE release lane queue for its whole life. Prefer the explicit
+  // release id (from the launch envelope). Fall back to the suffix-derived
+  // release so direct backend construction in unit tests (no releaseId, no
+  // suffix, env cleared) stays on the base `runs` queue and worker/dev:v2 keep
+  // their suffixed queues.
+  const releaseId =
+    input.releaseId?.trim() || absurdReleaseIdForSuffix(input.queueSuffix);
+  const runsQueue = absurdQueueNameForRelease(releaseId);
+
+  const cancelAbsurdTask = async () => {
+    const taskId = await withRuntimeSchedulerClient(poolOptions, (client) =>
+      readPostgresSchedulerAbsurdTaskId(client, input.runId, effectiveOptions),
+    );
+    if (!taskId) return;
+    const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
+    await absurd.cancelTask(taskId, runsQueue);
+  };
+
+  const cleanupPersistedRuntimeSandboxes = async () => {
+    const resources = await withRuntimeSchedulerClient(poolOptions, (client) =>
+      readPostgresSchedulerRuntimeResourcesForRun(
+        client,
+        { runId: input.runId },
+        effectiveOptions,
+      ),
+    );
+    const productionRefs = new Map<string, RuntimeSandboxRef>();
+    const workerOwnedRefs = new Map<string, RuntimeSandboxRef>();
+    for (const resource of resources) {
+      if (
+        !resource ||
+        typeof resource !== 'object' ||
+        Array.isArray(resource)
+      ) {
+        continue;
+      }
+      const record = resource as Record<string, unknown>;
+      const ref = runtimeSandboxRefFromResource(record);
+      if (!ref) continue;
+      const refKey = runtimeSandboxRefIdentityKey(ref);
+      // Modal credentials and all preview provider credentials are worker-only.
+      // This explicit task keeps deletion on the owning release lane. Legacy
+      // Daytona records predate runtimeEnvironment, so retain their marker.
+      const runtimeEnvironment =
+        record.runtimeEnvironment === 'preview' ||
+        record.daytonaEnvironment === 'preview'
+          ? 'preview'
+          : 'production';
+      if (runtimeEnvironment === 'preview' || ref.provider === 'modal') {
+        workerOwnedRefs.set(refKey, ref);
+        continue;
+      }
+      productionRefs.set(refKey, ref);
+    }
+    if (workerOwnedRefs.size > 0) {
+      const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
+      await Promise.all(
+        [...workerOwnedRefs.values()].map((runtimeSandboxRef) => {
+          // sandboxId remains in the payload for workers deployed before the
+          // provider-scoped cleanup cutover. New workers use the versioned ref.
+          const params: AbsurdDaytonaSandboxCleanupTaskParams & {
+            runtimeSandboxRef: RuntimeSandboxRef;
+          } = {
+            runId: input.runId,
+            sandboxId: runtimeSandboxRef.resourceId,
+            runtimeSandboxRef,
+          };
+          return absurd.spawn(ABSURD_DAYTONA_SANDBOX_CLEANUP_TASK, params, {
+            queue: runsQueue,
+            idempotencyKey: `sandbox-cleanup:${input.runId}:${runtimeSandboxRefIdentityKey(runtimeSandboxRef)}`,
+            maxAttempts: ABSURD_PLAY_RUN_MAX_ATTEMPTS,
+            retryStrategy: ABSURD_PLAY_RUN_RETRY_STRATEGY,
+          });
+        }),
+      );
+    }
+    if (productionRefs.size === 0) return;
+    const cleanupResults = await Promise.allSettled(
+      [...productionRefs.values()].map(async (ref) => {
+        const outcome = await defaultRuntimeSandboxReconciliation.delete(ref);
+        if (!runtimeSandboxDeleteSucceeded(outcome)) {
+          throw new Error(
+            `${ref.provider} sandbox cleanup did not delete ${ref.resourceId}.`,
+          );
+        }
+      }),
+    );
+    for (const result of cleanupResults) {
+      if (result.status === 'rejected') {
+        console.warn('[absurd.runtime_sandbox_cleanup_failed]', {
+          runId: input.runId,
+          error:
+            result.reason instanceof Error
+              ? result.reason.message
+              : String(result.reason),
+        });
+      }
+    }
+  };
+
+  return {
+    ...delegate,
+    runId: input.runId,
+    initialState: input.initialState ?? undefined,
+    cancel: async () => {
+      const cleanup = cleanupPersistedRuntimeSandboxes();
+      await Promise.all([delegate.cancel(), cancelAbsurdTask(), cleanup]);
+    },
+    signal: async (payload) => {
+      if (payload.kind === 'cancel') {
+        const cleanup = cleanupPersistedRuntimeSandboxes();
+        await Promise.all([
+          delegate.signal(payload),
+          cancelAbsurdTask(),
+          cleanup,
+        ]);
+        return;
+      }
+      if (payload.kind !== 'integration_event' || !payload.eventKey) {
+        throw new Error(
+          `absurd scheduler signal requires kind=integration_event with an eventKey (got ${payload.kind}).`,
+        );
+      }
+      const eventKey = payload.eventKey;
+      const data =
+        payload.data &&
+        typeof payload.data === 'object' &&
+        !Array.isArray(payload.data)
+          ? { ...(payload.data as Record<string, unknown>) }
+          : {};
+      const boundaryId =
+        typeof data.boundaryId === 'string' && data.boundaryId.trim()
+          ? data.boundaryId.trim()
+          : null;
+      if (boundaryId) {
+        delete data.boundaryId;
+      }
+      const eventPayload = { eventKey, boundaryId, data };
+
+      // Durable signal row is authoritative: the worker claims it for the real
+      // payload on wake. It is written BEFORE the absurd event so a fast wake
+      // never races ahead of the payload.
+      await withRuntimeSchedulerClient(poolOptions, (client) =>
+        recordPostgresSchedulerSignal(
+          client,
+          { runId: input.runId, eventKey, boundaryId, payload: eventPayload },
+          effectiveOptions,
+        ),
+      );
+
+      // The absurd event is only the wakeup. Scoped by runId so it wakes just
+      // this run on the shared queue.
+      const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
+      const eventName = absurdIntegrationEventName(input.runId, eventKey);
+      await absurd.emitEvent(eventName, null, runsQueue);
+      console.info('[absurd-runtime.signal] event_emitted', {
+        runId: input.runId,
+        eventKey,
+        boundaryId,
+        releaseId,
+        queue: runsQueue,
+        schema: effectiveOptions.schema ?? null,
+      });
+    },
+  };
+}
+
+export function createAbsurdSchedulerBackend(
+  schedulerOptions: AbsurdSchedulerBackendOptions = {},
+): PlaySchedulerBackend {
+  const normalizedSchedulerOptions =
+    normalizeAbsurdSchedulerOptions(schedulerOptions);
+  const queueSuffix = schedulerOptions.queueSuffix?.trim() || null;
+  return {
+    id: PLAY_SCHEDULER_BACKENDS.absurd,
+    async submit(input) {
+      const legacyParentRunId = (
+        input as PlaySchedulerSubmitInput & {
+          playCallGovernance?: { parentRunId?: unknown } | null;
+        }
+      ).playCallGovernance?.parentRunId;
+      if (typeof legacyParentRunId === 'string' && legacyParentRunId.trim()) {
+        throw new Error(
+          `${CTX_RUN_PLAY_INLINE_ONLY}: scheduled ctx.runPlay children are unsupported. ` +
+            'ctx.runPlay composes a resolved scalar child inline; submit independent work as a top-level play.',
+        );
+      }
+      const submitStartedAt = Date.now();
+      const poolOptions = resolveAbsurdPoolOptions(
+        normalizedSchedulerOptions.url,
+      );
+      // The run pins its release lane at launch: prefer the release id stamped
+      // in the envelope; fall back to the suffix-derived release for pre-release
+      // launch rows and unit tests (enforce-when-exists / allow-when-absent).
+      const releaseId = absurdReleaseIdForLaunch({
+        absurdReleaseId: input.absurdReleaseId ?? null,
+      });
+      const queue = absurdQueueNameForRelease(releaseId);
+      // The launch row and the actual play-run task are one Postgres commit.
+      // Returning from submit therefore means the durable work is runnable;
+      // there is no outbox, dispatcher, or second admission boundary.
+      let submissionQueue = queue;
+      const admissionParams = {
+        schedulerUrl: poolOptions.url,
+        schedulerSchema: normalizedSchedulerOptions.schema,
+      };
+      let spawn = await withRuntimeSchedulerSubmitAdmission(
+        admissionParams,
+        async (permit) => {
+          recordAbsurdSubmitPhase(input, 'submit_admitted', submitStartedAt, {
+            queueDepth: permit.queueDepth,
+            waitMs: permit.waitMs,
+          });
+          await ensureAbsurdRuntimeReady(
+            poolOptions,
+            normalizedSchedulerOptions,
+            queueSuffix,
+            releaseId,
+          );
+          recordAbsurdSubmitPhase(input, 'schema_ready', submitStartedAt);
+
+          // Loud, structured submit log: the #1 way an absurd run silently hangs
+          // is an app→worker queue-name mismatch (the shared preview app
+          // enqueuing on a queue no Fly worker drains).
+          console.info('[absurd-runtime.submit] enqueue', {
+            runId: input.runId,
+            orgId: input.orgId,
+            queue,
+            releaseId,
+            schedulerSchema: normalizedSchedulerOptions.schema ?? null,
+            queueSuffix: queueSuffix ?? null,
+          });
+          return withRuntimeSchedulerClient(
+            poolOptions,
+            async (client) =>
+              (
+                await createPlayRunAdmissionJob(
+                  createPostgresPlayRunAdmissionPorts({
+                    client,
+                    schema: normalizedSchedulerOptions.schema,
+                  }),
+                ).admit({
+                  launch: input,
+                  queueName: queue,
+                })
+              ).receipt,
+            {
+              runId: input.runId,
+              phase: 'launch_persist_and_enqueue',
+            },
+          );
+        },
+      );
+      if (spawn.taskRecoveryRequired) {
+        const existingLaunch = await withRuntimeSchedulerClient(
+          poolOptions,
+          (client) =>
+            readPostgresSchedulerLaunch(
+              client,
+              input.runId,
+              normalizedSchedulerOptions,
+              { caller: 'scheduler_submit_recovery' },
+            ),
+        );
+        if (!existingLaunch) {
+          throw new Error(
+            `Play run "${input.runId}" requires task recovery but its launch row is missing.`,
+          );
+        }
+        const recoveryReleaseId = absurdReleaseIdForLaunch(existingLaunch);
+        submissionQueue = absurdQueueNameForRelease(recoveryReleaseId);
+        spawn = await withRuntimeSchedulerSubmitAdmission(
+          admissionParams,
+          async () => {
+            await ensureAbsurdRuntimeReady(
+              poolOptions,
+              normalizedSchedulerOptions,
+              queueSuffix,
+              recoveryReleaseId,
+            );
+            return withRuntimeSchedulerClient(
+              poolOptions,
+              async (client) =>
+                (
+                  await createPlayRunAdmissionJob(
+                    createPostgresPlayRunAdmissionPorts({
+                      client,
+                      schema: normalizedSchedulerOptions.schema,
+                    }),
+                  ).admit({
+                    launch: input,
+                    queueName: submissionQueue,
+                    recoverExistingTask: true,
+                  })
+                ).receipt,
+              {
+                runId: input.runId,
+                phase: 'launch_recovery_persist_and_enqueue',
+              },
+            );
+          },
+        );
+        if (!spawn.taskId) {
+          throw new Error(
+            `Play run "${input.runId}" task recovery did not create an Absurd task.`,
+          );
+        }
+      }
+      recordAbsurdSubmitPhase(input, 'spawned', submitStartedAt, {
+        created: spawn.runCreated,
+        taskCreated: spawn.taskCreated,
+        queue: submissionQueue,
+      });
+
+      return absurdRunHandle({
+        runId: input.runId,
+        schedulerOptions: normalizedSchedulerOptions,
+        queueSuffix,
+        releaseId,
+        initialState: {
+          scheduler: PLAY_SCHEDULER_BACKENDS.absurd,
+          schedulerSchema: normalizedSchedulerOptions.schema ?? null,
+          absurdReleaseId: releaseId,
+          absurdTaskId: spawn.taskId,
+          runCreated: spawn.runCreated,
+          taskCreated: spawn.taskCreated,
+        },
+      });
+    },
+    async attach(runId, options) {
+      // The initialState may carry the run's stamped release id (from a prior
+      // submit's handle or a caller that read the launch row); prefer it so the
+      // cancel/signal wakeup lands on the run's lane, not this process's default.
+      const initialStateReleaseId =
+        typeof options?.initialState?.absurdReleaseId === 'string' &&
+        options.initialState.absurdReleaseId.trim()
+          ? options.initialState.absurdReleaseId.trim()
+          : null;
+      return absurdRunHandle({
+        runId,
+        schedulerOptions: normalizedSchedulerOptions,
+        queueSuffix,
+        releaseId: initialStateReleaseId,
+        initialState: options?.initialState ?? null,
+      });
+    },
+  };
+}
+
+export const absurdSchedulerBackend: PlaySchedulerBackend =
+  createAbsurdSchedulerBackend();