deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -0,0 +1,897 @@
+/**
+ * Shared constants and pure helpers for the `absurd` play-runtime scheduler
+ * profile — imported by both the scheduler backend (`absurd.ts`, submit side)
+ * and the durable task handler (`absurd-runtime-worker.ts`, worker side).
+ *
+ * This module is greenfield and does not import legacy scheduler modules. The
+ * battle-tested Postgres run-state store
+ * (`postgres.ts`) remains the source of truth for launch rows, attempts,
+ * receipts, signals, and terminal state; `absurd-shared` only adds the
+ * absurd-specific wiring (queue, event names, checkpoint boundary shaping).
+ */
+import { createHash } from 'node:crypto';
+import type {
+  PlayCheckpoint,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+} from '../ctx-types';
+import type { PlayExecutionSuspension } from '../suspension';
+import type { PlayRunLedgerEvent } from '../run-ledger';
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import type { WorkerRuntimeApiContext } from '../app-runtime-api';
+import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
+import { assertKnownResolvedBoundary } from '../checkpoint-boundaries';
+import type { Absurd, Log, RetryStrategy } from '@deepline/absurd/sdk';
+import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
+import {
+  ensureAbsurdSchema,
+  failPostgresSchedulerScheduledChildWaits,
+  type PostgresSchedulerWorkerOnceResult,
+} from './postgres';
+import { createAbsurdForRuntimeScheduler } from './absurd-pool';
+import {
+  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS,
+  resolveWorkflowRuntimePostgresUrl,
+  withRuntimeSchedulerClient,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
+import {
+  ABSURD_DEV_RELEASE_PREFIX,
+  ABSURD_PROD_DEFAULT_SCHEMA,
+  ABSURD_QUEUE_SUFFIX_ENV,
+  ABSURD_SCHEDULER_SCHEMA_ENV,
+  absurdDevReleaseIdForQueueSuffix,
+  resolveAbsurdQueueSuffix,
+  resolveAbsurdQueueSuffixForSchema,
+} from './absurd-queue-suffix';
+
+export {
+  ABSURD_PROD_DEFAULT_SCHEMA,
+  ABSURD_QUEUE_SUFFIX_ENV,
+  ABSURD_SCHEDULER_SCHEMA_ENV,
+  resolveAbsurdQueueSuffix,
+  resolveAbsurdQueueSuffixForSchema,
+} from './absurd-queue-suffix';
+
+type EnvMap = Record<string, string | undefined>;
+
+/**
+ * Per-worktree ENGINE-table isolation — option (a): queue-NAME scoping.
+ *
+ * The vendored Absurd SQL hardcodes the schema literal `absurd`, and dev:v2
+ * points every worktree at the SAME docker Postgres (`deepline_dev`), so all
+ * worktrees write into one shared `absurd` schema. Absurd partitions its state
+ * BY QUEUE: each queue owns its own `t_/r_/c_/e_/w_` tables
+ * (`absurd.t_<queue>`, `absurd.r_<queue>`, ...). Two worktrees with DISTINCT
+ * queue names therefore touch DISJOINT tables inside the shared schema. We scope
+ * the queue NAMES by a per-worktree suffix instead of rewriting the schema
+ * literal (option (b)), which would mean patching the whole SQL blob and every
+ * cross-queue `absurd.` function reference — far riskier for zero extra benefit.
+ *
+ * The suffix comes from a DEDICATED env var `DEEPLINE_ABSURD_QUEUE_SUFFIX` (set
+ * per worktree by dev-v2-runtime.sh), NOT from DEEPLINE_RUNTIME_SCHEDULER_SCHEMA.
+ * Keeping it separate means unit tests — which never set it, and which mutate
+ * the scheduler-schema env — keep the canonical base name `runs`
+ * and stay hermetic. Unset (tests, prod default) => base names, so nothing
+ * changes anywhere the suffix is absent.
+ *
+ * Agreement across processes: the backend (submit/cancel/signal) and the worker
+ * (task registration + poll loop) both import THESE constants, so within a
+ * process they are identical, and across the app + worker processes they agree
+ * because dev:v2 exports the same suffix into both environments. Queue metrics
+ * read the run-state store's `attempts` table, isolated by
+ * DEEPLINE_RUNTIME_SCHEDULER_SCHEMA, so they never touch these queue-name tables.
+ */
+/**
+ * Worker release identity env var. Baked into the Fly image at build time as an
+ * ARG/ENV (see the Fly Dockerfile), so a given machine's process knows exactly
+ * which release lane it belongs to. Absent in dev/unit tests => the dev-collapse
+ * rule applies and lane checks pass trivially.
+ */
+export const ABSURD_RELEASE_ID_ENV = 'DEEPLINE_RUNTIME_DEPLOY_VERSION';
+
+/**
+ * Sentinel release id used everywhere no real deploy version exists (unit tests,
+ * plain local `dev:v2` with no worktree suffix, prod default). Maps to the plain
+ * base `runs` queue — byte-identical to today.
+ */
+export const ABSURD_DEV_RELEASE_ID = 'dev';
+
+/**
+ * Prefix stamped on the dev-collapse release id when a worktree/preview queue
+ * suffix is active. `dev-<suffix>` deterministically maps back to today's
+ * `runs_<suffix>` queue via `absurdQueueNameForRelease`, so local dev is
+ * unchanged while the release id still travels on the run for the lane guard.
+ */
+const ABSURD_RUNS_QUEUE_BASE = 'runs';
+
+/** True for any release lane derived from the durable Absurd runs queue. */
+export function isAbsurdRunsQueue(queueName: string): boolean {
+  return (
+    queueName === ABSURD_RUNS_QUEUE_BASE ||
+    queueName.startsWith(`${ABSURD_RUNS_QUEUE_BASE}_`)
+  );
+}
+
+/**
+ * Longest queue-name suffix we allow before hashing. Absurd derives engine
+ * tables `t_<queue>` / `r_<queue>` (2-char prefixes) from the queue name, and
+ * Postgres caps identifiers at 63 bytes, so `runs_` (5) + prefix (2) + suffix
+ * must stay under 63. We cap the release-derived suffix well below that and hash
+ * any overflow deterministically so two long release ids never collide onto one
+ * lane.
+ */
+const ABSURD_RELEASE_QUEUE_SUFFIX_MAX = 40;
+
+/**
+ * A worker whose scheduler schema is anything OTHER than the prod default is
+ * running an isolated lane topology — a per-PR preview fleet or a per-worktree
+ * dev fleet — where the release lane (and therefore the queue it must poll) is
+ * NOT the base `runs` queue but `runs_<suffix>`. In that topology the suffix is
+ * the SINGLE provisioned lane identity: the submitting app derives it from the
+ * per-request schema and the polling worker reads it from its own env, and both
+ * MUST agree or every run enqueues onto a queue no worker drains and hangs.
+ *
+ * This returns true exactly when the worker's schema demands a provisioned
+ * suffix. The boot guard uses it to refuse to start (loudly, naming the missing
+ * variable) rather than silently fall back to the base `runs` queue — the
+ * dev-collapse fallback is safe ONLY when the topology can guarantee all parties
+ * the same inputs, which an isolated-schema worker with no suffix cannot.
+ * Delegates to the shared topology predicate so the Daytona egress guard and
+ * this lane guard can never disagree about what "isolated" means.
+ */
+export function absurdWorkerSchemaRequiresQueueSuffix(
+  schedulerSchema: string | null | undefined,
+): boolean {
+  return isIsolatedRuntimeSchedulerSchema(schedulerSchema);
+}
+
+/**
+ * Sanitize an arbitrary release id into a queue-name segment: lowercase, collapse
+ * non-`[a-z0-9_]` to `_`, trim leading/trailing `_`. Overflow past
+ * `ABSURD_RELEASE_QUEUE_SUFFIX_MAX` is truncated AND fingerprinted with a short
+ * sha256 tail so two long-but-distinct release ids can never alias onto the same
+ * lane. Returns `''` for an empty/garbage input so the caller can fail loudly.
+ */
+function sanitizeReleaseQueueSegment(releaseId: string): string {
+  const normalized = releaseId
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '');
+  if (!normalized) return '';
+  if (normalized.length <= ABSURD_RELEASE_QUEUE_SUFFIX_MAX) return normalized;
+  // Deterministic overflow: keep a readable head, append a hash tail so the full
+  // (untruncated) release id disambiguates collisions.
+  const hashTail = createHash('sha256')
+    .update(normalized)
+    .digest('hex')
+    .slice(0, 8);
+  const headBudget = ABSURD_RELEASE_QUEUE_SUFFIX_MAX - hashTail.length - 1;
+  return `${normalized.slice(0, headBudget).replace(/_+$/g, '')}_${hashTail}`;
+}
+
+/**
+ * Resolve THIS worker process's release lane id (the "current" lane). Precedence,
+ * matching the dev-collapse rule:
+ *   1. queue suffix set (worktree/local dev/preview CI) => `dev-<suffix>`, which
+ *      maps back to today's `runs_<suffix>` queue — local dev is unchanged.
+ *   2. deploy version set (Fly release machine) => sanitized deploy version.
+ *   3. neither (unit tests / prod default) => `dev` => plain `runs` queue.
+ */
+export function currentAbsurdReleaseId(env: EnvMap = process.env): string {
+  const suffix = resolveAbsurdQueueSuffixFromEnv(env);
+  if (suffix) return `${ABSURD_DEV_RELEASE_PREFIX}${suffix}`;
+  const deployVersion = env[ABSURD_RELEASE_ID_ENV]?.trim();
+  if (deployVersion) {
+    const sanitized = sanitizeReleaseQueueSegment(deployVersion);
+    if (!sanitized) {
+      throw new Error(
+        `absurd release id: ${ABSURD_RELEASE_ID_ENV}="${deployVersion}" ` +
+          `sanitizes to empty. A deploy version must contain at least one ` +
+          `[a-z0-9] character.`,
+      );
+    }
+    return sanitized;
+  }
+  return ABSURD_DEV_RELEASE_ID;
+}
+
+/**
+ * Release id as persisted in `runs.launch_json`. Production launches retain the
+ * raw registry id, while the worker lane id above is sanitized/hashed to fit an
+ * Absurd queue name. Preview suffixes still use the dev-collapse id because the
+ * suffix outranks a baked deploy version in both the app and worker topology.
+ */
+export function currentAbsurdLaunchReleaseId(
+  env: EnvMap = process.env,
+): string {
+  const suffix = resolveAbsurdQueueSuffixFromEnv(env);
+  if (suffix) return `${ABSURD_DEV_RELEASE_PREFIX}${suffix}`;
+  return env[ABSURD_RELEASE_ID_ENV]?.trim() || ABSURD_DEV_RELEASE_ID;
+}
+
+/**
+ * The release lane queue for a given release id. This is the ONE queue a run and
+ * all of its descendants/wakeups ride for its whole life.
+ *
+ * Dev-collapse mapping (byte-identical to today's suffix behavior):
+ *   - `dev`          => `runs`            (plain base queue)
+ *   - `dev-<suffix>` => `runs_<suffix>`   (existing worktree/preview queue)
+ *   - `<release>`    => `runs_<sanitized-release>`  (a real Fly release lane)
+ *
+ * Loud failure: an empty/whitespace release id is a programming error (the
+ * resolver never returns one), so we throw rather than silently landing work on
+ * the base queue where no lane worker may be draining.
+ */
+export function absurdQueueNameForRelease(releaseId: string): string {
+  const trimmed = releaseId?.trim();
+  if (!trimmed) {
+    throw new Error(
+      'absurdQueueNameForRelease requires a non-empty release id. ' +
+        'Resolve it via currentAbsurdReleaseId() or the launch row.',
+    );
+  }
+  if (trimmed === ABSURD_DEV_RELEASE_ID) {
+    return absurdQueueName(ABSURD_RUNS_QUEUE_BASE, '');
+  }
+  if (trimmed.startsWith(ABSURD_DEV_RELEASE_PREFIX)) {
+    const suffix = normalizeQueueSuffixValue(
+      trimmed.slice(ABSURD_DEV_RELEASE_PREFIX.length),
+    );
+    return absurdQueueName(ABSURD_RUNS_QUEUE_BASE, suffix);
+  }
+  const segment = sanitizeReleaseQueueSegment(trimmed);
+  if (!segment) {
+    throw new Error(
+      `absurdQueueNameForRelease: release id "${trimmed}" sanitizes to empty.`,
+    );
+  }
+  return absurdQueueName(ABSURD_RUNS_QUEUE_BASE, segment);
+}
+
+/**
+ * Loud lane guard used belt-and-suspenders when a worker executes a claimed run.
+ * The monogamous poll loop makes a mismatch structurally unreachable; this exists
+ * to catch operator error (e.g. a lane machine pointed at the wrong queue). The
+ * message is operator-actionable: it names both ids and the remediation.
+ */
+export function assertAbsurdReleaseLaneMatches(input: {
+  runId: string;
+  runReleaseId: string;
+  workerReleaseId: string;
+}): void {
+  if (input.runReleaseId === input.workerReleaseId) return;
+  // A release id reaches this guard in one of two forms: the raw deploy slug
+  // stamped onto a launch from the release registry (`main-<sha>`), or the
+  // sanitized lane id a worker reports via `currentAbsurdReleaseId()`
+  // (`main_<sha>_<hash>`). Both map to the SAME lane queue, which is the real
+  // claim boundary — a run lands on, and is claimed off, its queue regardless
+  // of which spelling named it. Compare derived queues so the two spellings of
+  // one lane match. Without this, a production-length id (whose sanitized form
+  // diverges from the raw slug once it overflows the queue-segment budget)
+  // fails this guard on EVERY attempt even though the run correctly landed on
+  // this worker's queue; short dev/preview ids never overflow, so raw ==
+  // sanitized and they slipped past this guard in CI.
+  if (
+    absurdQueueNameForRelease(input.runReleaseId) ===
+    absurdQueueNameForRelease(input.workerReleaseId)
+  ) {
+    return;
+  }
+  throw new Error(
+    `absurd release lane mismatch for run ${input.runId}: the run was launched ` +
+      `on release "${input.runReleaseId}" but this worker serves release ` +
+      `"${input.workerReleaseId}". A run must finish on the release that ` +
+      `launched it. Remediation: start a lane machine for "${input.runReleaseId}" ` +
+      `(queue ${absurdQueueNameForRelease(input.runReleaseId)}).`,
+  );
+}
+
+/**
+ * Seconds a foreign, non-adoptable task is parked for before the owning lane
+ * gets another chance at it. Matches the historical release-guard deferral so
+ * the observable retry cadence is unchanged.
+ */
+export const ABSURD_LANE_ADOPTION_DEFER_SECONDS = 30;
+
+/** Absurd worker-id suffix that opts a poll loop into cross-lane claims. */
+export const ABSURD_CONTINUATION_WORKER_SUFFIX = ':continuations';
+
+/**
+ * Worker id for an adoption poll loop. The `:continuations` suffix is load
+ * bearing: vendored `claim_task` patch (g) admits such a worker only for runs
+ * still `sleeping` or carrying a non-null event-wake payload, so a fresh or
+ * ordinary retry-pending task on the adopted queue is never handed to it.
+ */
+export function absurdContinuationWorkerId(input: {
+  workerBase: string;
+  queue: string;
+}): string {
+  return `${input.workerBase}:${input.queue}${ABSURD_CONTINUATION_WORKER_SUFFIX}`;
+}
+
+export type AbsurdReleaseLaneDisposition =
+  /** The run belongs to this worker's own lane. */
+  | { kind: 'own' }
+  /** A retired lane's parked durable wait resumes on this newer release. */
+  | { kind: 'adopt'; adoptedReleaseId: string }
+  /** Foreign, not adoptable: park it without spending a task attempt. */
+  | { kind: 'defer'; reason: string; delaySeconds: number }
+  /** Foreign lane that still claims — operator error, fail loudly. */
+  | { kind: 'refuse' };
+
+/**
+ * Decide what this worker may do with a claimed run, given the durable lane
+ * facts read from the scheduler Postgres.
+ *
+ * This is the adapter half of the adoption rule (lane handoff). The engine's
+ * `:continuations` fence already stops most cross-lane claims; this decides the
+ * cases the engine cannot see, because `sleeping` in the engine covers durable
+ * suspensions, retry backoff, and capacity deferrals alike.
+ *
+ * `laneClaiming` and `parkedInAdoptableWait` come from
+ * `readPostgresSchedulerRunLaneState`, never from a timer or a machine count.
+ */
+export function resolveAbsurdReleaseLaneDisposition(input: {
+  runId: string;
+  runReleaseId: string;
+  workerReleaseId: string;
+  /** Does the run's OWN lane still claim work off its queue? */
+  laneClaiming: boolean;
+  /** Is the run parked in a sleep/integration-event wait right now? */
+  parkedInAdoptableWait: boolean;
+}): AbsurdReleaseLaneDisposition {
+  if (
+    input.runReleaseId === input.workerReleaseId ||
+    absurdQueueNameForRelease(input.runReleaseId) ===
+      absurdQueueNameForRelease(input.workerReleaseId)
+  ) {
+    return { kind: 'own' };
+  }
+  // A lane that still claims owns every one of its rows. Reaching this branch
+  // means a machine is polling a queue it does not serve; stay loud.
+  if (input.laneClaiming) return { kind: 'refuse' };
+  if (!input.parkedInAdoptableWait) {
+    return {
+      kind: 'defer',
+      reason:
+        `Run ${input.runId} rides retired release lane "${input.runReleaseId}" ` +
+        `but is not parked in an adoptable durable wait. Ordinary queued and ` +
+        `running work is never adopted; leaving it for its owning lane.`,
+      delaySeconds: ABSURD_LANE_ADOPTION_DEFER_SECONDS,
+    };
+  }
+  return { kind: 'adopt', adoptedReleaseId: input.runReleaseId };
+}
+
+/**
+ * Adoption target queues for a worker, given the release ids the scheduler
+ * reported as adoptable. Two release-id spellings of one lane collapse to one
+ * queue, and this worker's own queue is never attached twice — which is why a
+ * dev-collapse topology (every run stamped `dev`) attaches nothing at all.
+ */
+export function absurdAdoptionQueuesForReleases(input: {
+  workerReleaseId: string;
+  adoptableReleaseIds: readonly string[];
+}): string[] {
+  const ownQueue = absurdQueueNameForRelease(input.workerReleaseId);
+  const queues = new Set<string>();
+  for (const releaseId of input.adoptableReleaseIds) {
+    const trimmed = releaseId?.trim();
+    if (!trimmed) continue;
+    const queue = absurdQueueNameForRelease(trimmed);
+    if (queue === ownQueue) continue;
+    queues.add(queue);
+  }
+  return [...queues].sort();
+}
+
+/**
+ * Env-driven queue suffix read (worker/dev:v2). Extracted so
+ * `currentAbsurdReleaseId` can take an explicit env for hermetic tests while
+ * `resolveAbsurdQueueSuffix` keeps its process.env signature.
+ */
+function resolveAbsurdQueueSuffixFromEnv(env: EnvMap): string {
+  return resolveAbsurdQueueSuffix(env);
+}
+
+function normalizeQueueSuffixValue(raw: string): string {
+  return raw
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+    .slice(0, 32);
+}
+
+function absurdQueueName(base: string, suffix: string): string {
+  return suffix ? `${base}_${suffix}` : base;
+}
+
+/**
+ * Resolve the per-worktree/per-PR queue suffix for a submit/signal/cancel that
+ * carries an EXPLICIT per-request scheduler schema (threaded from the CLI via
+ * the runtime-scheduler-schema header → `createAbsurdSchedulerBackend({schema})`).
+ *
+ * Why this exists: the module-level `ABSURD_RUNS_QUEUE` constants read the
+ * `DEEPLINE_ABSURD_QUEUE_SUFFIX` env var ONCE at import. That works for the Fly
+ * worker (env set as a secret) and dev:v2 (env exported into every process),
+ * but NOT for the shared hosted preview app, which learns the per-PR schema
+ * only per-request and has no such env — so its enqueue would land on the base
+ * `runs` queue while the worker drains `runs_<suffix>`, and every run hangs.
+ *
+ * Precedence: the env suffix applies when the explicit schema is native to the
+ * process. A different explicit schema identifies a remote runtime, so derive
+ * the SAME suffix its worker uses — sha256(schema)[:12]. Prod-default or an
+ * unset schema retains the process suffix when present.
+ */
+/**
+ * Normalize an explicit queue suffix (from the request layer via
+ * `resolveAbsurdQueueSuffixForSchema`). An empty/undefined value falls back to
+ * the process env suffix — so the Fly worker and dev:v2 (env set) keep their
+ * suffixed queues, direct backend construction in unit tests (no suffix, env
+ * cleared) stays on base names, and the shared hosted app (env absent) uses the
+ * explicit per-request suffix threaded down from `lazy.ts`.
+ */
+function effectiveAbsurdQueueSuffix(explicit?: string | null): string {
+  const normalized = explicit?.trim();
+  if (normalized) return normalizeQueueSuffixValue(normalized);
+  return resolveAbsurdQueueSuffix();
+}
+
+/**
+ * Map an effective queue suffix to a release id under the dev-collapse rule.
+ * Empty suffix => the process default release (`dev`, or the baked deploy
+ * version on a Fly lane machine); a non-empty suffix => `dev-<suffix>`, which
+ * `absurdQueueNameForRelease` maps back to today's `runs_<suffix>` queue. This
+ * is how the shared hosted preview app (which threads an explicit per-PR suffix
+ * derived from the schema) stamps the SAME release id the Fly preview worker
+ * serves, without touching local-dev behavior.
+ */
+export function absurdReleaseIdForSuffix(
+  explicitSuffix?: string | null,
+  env: EnvMap = process.env,
+): string {
+  const suffix = effectiveAbsurdQueueSuffix(explicitSuffix);
+  if (suffix) return absurdDevReleaseIdForQueueSuffix(suffix);
+  return currentAbsurdReleaseId(env);
+}
+
+/**
+ * Resolve the release lane a launch row belongs to. Enforce-when-exists /
+ * allow-when-absent: a run stamped with `absurdReleaseId` rides its lane; a
+ * pre-release launch row (no stamp — the only compat allowance in this PR) falls
+ * back to THIS process's dev-collapse id so preview/burn-in continuity holds.
+ */
+export function absurdReleaseIdForLaunch(
+  launch: { absurdReleaseId?: string | null },
+  env: EnvMap = process.env,
+): string {
+  const stamped = launch.absurdReleaseId?.trim();
+  if (stamped) return stamped;
+  return currentAbsurdReleaseId(env);
+}
+
+/** The release lane queue a launch row rides for its whole lifecycle. */
+export function absurdQueueNameForLaunch(
+  launch: { absurdReleaseId?: string | null },
+  env: EnvMap = process.env,
+): string {
+  return absurdQueueNameForRelease(absurdReleaseIdForLaunch(launch, env));
+}
+
+/** Shared run queue for top-level plays and their parked resumptions. */
+export function absurdRunsQueueForSuffix(suffix?: string | null): string {
+  return absurdQueueName(
+    ABSURD_RUNS_QUEUE_BASE,
+    effectiveAbsurdQueueSuffix(suffix),
+  );
+}
+
+/** Compatibility shape for admin callers that enumerate scheduler queues. */
+export function absurdAllQueuesForSuffix(suffix?: string | null): string[] {
+  return [absurdRunsQueueForSuffix(suffix)];
+}
+
+export const ABSURD_RUNS_QUEUE = absurdRunsQueueForSuffix();
+
+/** Durable task for one top-level play run. */
+export {
+  ABSURD_PLAY_RUN_MAX_ATTEMPTS,
+  ABSURD_PLAY_RUN_RETRY_STRATEGY,
+  ABSURD_PLAY_RUN_TASK,
+  type AbsurdPlayRunTaskParams,
+} from './absurd-play-run-task';
+/** Worker-owned cleanup for a Daytona sandbox in this release lane. */
+export const ABSURD_DAYTONA_SANDBOX_CLEANUP_TASK = 'daytona-sandbox-cleanup';
+
+/** Terminal engine rows are diagnostic scratch state, not the durable run
+ * ledger. Keep a six-hour incident window, then let the scaler's bounded
+ * maintenance pass remove them so claims do not rescan an ever-growing table. */
+export const ABSURD_RUNS_CLEANUP_TTL = '6 hours';
+export const ABSURD_RUNS_CLEANUP_LIMIT = 1_000;
+
+export type AbsurdDaytonaSandboxCleanupTaskParams = {
+  runId: string;
+  sandboxId: string;
+};
+
+/** Schema is `string | undefined` (never null) so it is directly assignable to
+ * the Postgres store's `Pick<PostgresSchedulerOptions, 'schema'>`. */
+export type AbsurdSchedulerOptions = {
+  url?: string | null;
+  schema?: string;
+};
+
+export type AbsurdRuntimeResult = {
+  ok: true;
+  runId: string;
+  schedulerStatus: 'completed' | 'failed' | 'idle';
+  output?: unknown;
+  error?: string;
+};
+
+// All queues in one worker share this single process pool. Scheduler queries
+// are short and batched; execution concurrency belongs to Daytona/Fly slots,
+// not to a growing set of database clients. Keeping this at two makes a
+// horizontally scaled fleet predictable against the shared PgBouncer budget.
+export const ABSURD_RUNTIME_POOL_MAX_CLIENTS =
+  RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
+
+/**
+ * A Vercel invocation has its own module cache and pool. Letting every
+ * short-lived launch hold the worker-sized cap multiplies the shared
+ * PgBouncer budget under a burst of webhook deliveries. A launch submits one
+ * durable transaction at a time, so one checkout preserves correctness while
+ * the long-lived Fly worker keeps its two-client control-plane budget.
+ */
+export const ABSURD_SERVERLESS_POOL_MAX_CLIENTS = 1;
+
+/**
+ * Resolve the runtime scheduler Postgres pool options for the absurd control
+ * plane. Loud failure over silent fallback: the absurd worker and scheduler
+ * only run under Node where `WORKFLOW_RUNTIME_POSTGRES` is configured.
+ */
+export function resolveAbsurdPoolOptions(
+  explicitUrl?: string | null,
+): RuntimeSchedulerPoolOptions {
+  const url = explicitUrl?.trim() || resolveWorkflowRuntimePostgresUrl();
+  if (!url) {
+    throw new Error(
+      'absurd scheduler backend requires WORKFLOW_RUNTIME_POSTGRES (runtime ' +
+        'scheduler Postgres URL). Refusing to fall back to local execution.',
+    );
+  }
+  return {
+    url,
+    applicationName: 'deepline-absurd-runtime',
+    maxClients: process.env.VERCEL?.trim()
+      ? ABSURD_SERVERLESS_POOL_MAX_CLIENTS
+      : ABSURD_RUNTIME_POOL_MAX_CLIENTS,
+  };
+}
+
+export function normalizeAbsurdSchedulerOptions(
+  options: { url?: string | null; schema?: string | null } = {},
+): AbsurdSchedulerOptions {
+  const url = options.url?.trim() || null;
+  const schema = options.schema?.trim();
+  return {
+    ...(url ? { url } : {}),
+    ...(schema ? { schema } : {}),
+  };
+}
+
+const absurdLog: Log = {
+  log: (...args) => console.log('[absurd-runtime]', ...args),
+  info: (...args) => console.info('[absurd-runtime]', ...args),
+  warn: (...args) => console.warn('[absurd-runtime]', ...args),
+  error: (...args) => console.error('[absurd-runtime]', ...args),
+};
+
+/**
+ * Construct an Absurd client on the hardened runtime scheduler pool. The
+ * client's `queueName` is the queue its `startWorker` poll loop drains and the
+ * default target for `emitEvent`/`spawn` without an explicit queue; defaults to
+ * the root `runs` queue. Task registrations are attached by the worker.
+ */
+export function createAbsurdRuntimeClient(
+  poolOptions: RuntimeSchedulerPoolOptions,
+  queueName: string = ABSURD_RUNS_QUEUE,
+): Absurd {
+  return createAbsurdForRuntimeScheduler(poolOptions, {
+    queueName,
+    log: absurdLog,
+  });
+}
+
+const absurdReadyPromises = new Map<string, Promise<void>>();
+
+/**
+ * Ensure, once per process (keyed by pool url + schema): the vendored absurd
+ * schema and the shared queue with its
+ * policies. Idempotent and safe under concurrent submits — the memoized promise
+ * is shared; a failure evicts the cache so the next caller retries loudly.
+ */
+export function ensureAbsurdRuntimeReady(
+  poolOptions: RuntimeSchedulerPoolOptions,
+  schedulerOptions: AbsurdSchedulerOptions = {},
+  queueSuffix?: string | null,
+  releaseId?: string | null,
+): Promise<void> {
+  const schema = normalizeAbsurdSchedulerOptions(schedulerOptions).schema ?? '';
+  // Queue name derivation: an explicit release id (from the launch envelope or
+  // the worker's own `currentAbsurdReleaseId()`) wins — that is the ONE lane the
+  // run rides. Otherwise fall back to the explicit per-request suffix (threaded
+  // from `lazy.ts`) so the shared hosted app still creates the SAME suffixed
+  // queue the Fly worker drains, and worker/dev:v2 with an env suffix are
+  // unchanged. Under dev-collapse the release id `dev-<suffix>` maps back to
+  // exactly `runs_<suffix>`, so this is byte-identical to today for local dev.
+  const runsQueue = releaseId?.trim()
+    ? absurdQueueNameForRelease(releaseId)
+    : absurdRunsQueueForSuffix(queueSuffix);
+  const key = `${poolOptions.url}::${schema}::${runsQueue}`;
+  const existing = absurdReadyPromises.get(key);
+  if (existing) return existing;
+  const promise = (async () => {
+    await withRuntimeSchedulerClient(poolOptions, (client) =>
+      ensureAbsurdSchema(client),
+    );
+    // Hard-cutover cleanup: an old worker may have persisted a run waiting
+    // on a scheduled ctx.runPlay child. This runtime intentionally has no such
+    // scheduler anymore, so make the affected run visibly failed and emit
+    // its terminal outbox fact rather than leaving it parked forever.
+    await withRuntimeSchedulerClient(poolOptions, (client) =>
+      failPostgresSchedulerScheduledChildWaits(client, schedulerOptions),
+    );
+    const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
+    const existingQueues = new Set(await absurd.listQueues());
+    if (!existingQueues.has(runsQueue)) {
+      await absurd.createQueue(runsQueue);
+    }
+    // Idempotent upsert: parked runs hold no slot; fair claim ordering chooses
+    // the next runnable work without imposing an account-level claim cap.
+    await absurd.setQueuePolicy(runsQueue, {
+      cleanupTtl: ABSURD_RUNS_CLEANUP_TTL,
+      cleanupLimit: ABSURD_RUNS_CLEANUP_LIMIT,
+    });
+  })().catch((error) => {
+    absurdReadyPromises.delete(key);
+    throw error;
+  });
+  absurdReadyPromises.set(key, promise);
+  return promise;
+}
+
+/**
+ * Absurd wakeup event name for an integration-event suspension. Scoped by
+ * runId so a broadcast emit only wakes the intended run (absurd events are
+ * queue-scoped and immutable/first-emit-wins, so an UNSCOPED name would both
+ * cross-wake unrelated runs and permanently pin one payload). The authoritative
+ * payload is NEVER read from this event — the worker claims the durable signal
+ * row (`claimPostgresSchedulerSignal`) for the real payload.
+ */
+export function absurdIntegrationEventName(
+  runId: string,
+  eventKey: string,
+): string {
+  return `evt:${runId}:${eventKey}`;
+}
+
+/**
+ * Wake an attempt parked on a detached push-execution runner (B2-final). The
+ * name is ATTEMPT-scoped on purpose: absurd events are queue-scoped and
+ * first-emit-wins, so a stale attempt's late terminal emit names a key nobody
+ * is parked on and can never wake (or clobber) a newer generation. The terminal
+ * payload is read from scheduler Postgres (`runner_result_json` on the attempt
+ * row); the Absurd event is only a durable wakeup.
+ */
+export function absurdRunnerTerminalEventName(
+  runId: string,
+  runnerAttempt: number,
+): string {
+  return `runner-terminal:${runId}:${runnerAttempt}`;
+}
+
+/** Build the worker→app runtime API context (ledger + billing callbacks). */
+export function absurdRuntimeApiFromLaunch(
+  launch: PlaySchedulerSubmitInput,
+): WorkerRuntimeApiContext {
+  return {
+    baseUrl: runtimeApiBaseUrlFromLaunch(launch),
+    executorToken: launch.executorToken,
+    integrationMode: launch.integrationMode ?? null,
+    vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
+    runtimeTestFaultHeader: launch.runtimeTestFaultHeader ?? null,
+  };
+}
+
+/** Terminal run-ledger event for a settled attempt (source=worker). */
+export function absurdTerminalLedgerEvent(
+  result: PostgresSchedulerWorkerOnceResult,
+): PlayRunLedgerEvent | null {
+  const occurredAt = Date.now();
+  if (result.status === 'completed') {
+    return {
+      type: 'run.completed',
+      runId: result.runId,
+      occurredAt,
+      source: 'worker',
+      result: result.output,
+    };
+  }
+  if (result.status === 'failed') {
+    return {
+      type: 'run.failed',
+      runId: result.runId,
+      occurredAt,
+      source: 'worker',
+      error: result.error,
+    };
+  }
+  return null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+
+/**
+ * Normalize an arbitrary runner checkpoint into the PlayCheckpoint envelope.
+ *
+ * Compatibility contract (`docs/runtime-compatibility-contract.md`): unknown
+ * TOP-LEVEL fields are PASSED THROUGH. This normalizer's result is written back
+ * to the durable checkpoint on every park/resume, so rebuilding the envelope
+ * from only the keys this build knows would ERASE state a newer build wrote —
+ * silently, and permanently, on the first wait a run takes. The known keys keep
+ * exactly the validation they had: they always override the passthrough.
+ */
+export function absurdCheckpointBase(checkpoint: unknown): PlayCheckpoint {
+  const base = isRecord(checkpoint) ? checkpoint : {};
+  const durableCallCacheEpochMs = base.durableCallCacheEpochMs;
+  const normalized = {
+    ...base,
+    completedBatches:
+      (base.completedBatches as PlayCheckpoint['completedBatches']) ?? {},
+    completedToolBatches:
+      (base.completedToolBatches as PlayCheckpoint['completedToolBatches']) ??
+      {},
+    resolvedWaterfalls:
+      (base.resolvedWaterfalls as PlayCheckpoint['resolvedWaterfalls']) ?? {},
+    resolvedBoundaries:
+      (base.resolvedBoundaries as PlayCheckpoint['resolvedBoundaries']) ?? {},
+  } as PlayCheckpoint;
+  // Same strictness as before the passthrough: a non-numeric/negative epoch or a
+  // non-record mapFrames is DROPPED rather than carried forward by the spread.
+  if (
+    typeof durableCallCacheEpochMs === 'number' &&
+    Number.isFinite(durableCallCacheEpochMs) &&
+    durableCallCacheEpochMs >= 0
+  ) {
+    normalized.durableCallCacheEpochMs = durableCallCacheEpochMs;
+  } else {
+    delete normalized.durableCallCacheEpochMs;
+  }
+  if (isRecord(base.mapFrames)) {
+    normalized.mapFrames = base.mapFrames as PlayCheckpoint['mapFrames'];
+  } else {
+    delete normalized.mapFrames;
+  }
+  return normalized;
+}
+
+function resolvedBoundaries(
+  checkpoint: PlayCheckpoint,
+): NonNullable<PlayCheckpoint['resolvedBoundaries']> {
+  return checkpoint.resolvedBoundaries ?? {};
+}
+
+/** Boundary output object the play runtime reads when an integration event
+ * resolves (or times out). Mirrors the belt-and-suspenders envelope carried by
+ * the durable signal store today. */
+export function absurdIntegrationEventBoundaryOutput(input: {
+  eventKey: string;
+  signalData: Record<string, unknown> | null;
+  messageRef?: { channel: string; ts: string } | null;
+  timedOut: boolean;
+}): Record<string, unknown> {
+  if (input.timedOut) {
+    return {
+      resumed: false,
+      timed_out: true,
+      signal: input.eventKey,
+      event_key: input.eventKey,
+      message_ref: input.messageRef ?? null,
+      event_payload: null,
+      response: null,
+    };
+  }
+  const data = input.signalData ?? {};
+  return {
+    ...data,
+    resumed: true,
+    timed_out: false,
+    signal: input.eventKey,
+    event_key: input.eventKey,
+    message_ref: input.messageRef ?? null,
+    event_payload: input.signalData ?? null,
+    response: { eventKey: input.eventKey, data: input.signalData ?? null },
+  };
+}
+
+/** Return a checkpoint with the sleep boundary marked complete. */
+export function absurdCheckpointWithSleep(
+  checkpoint: unknown,
+  boundaryId: string,
+  delayMs: number,
+): PlayCheckpoint {
+  const base = absurdCheckpointBase(checkpoint);
+  return {
+    ...base,
+    resolvedBoundaries: {
+      ...resolvedBoundaries(base),
+      [boundaryId]: { kind: 'sleep', delayMs, completedAt: Date.now() },
+    },
+  };
+}
+
+/** Return a checkpoint with the integration-event boundary resolved. */
+export function absurdCheckpointWithIntegrationEvent(input: {
+  checkpoint: unknown;
+  boundaryId: string;
+  eventKey: string;
+  timeoutMs: number;
+  messageRef?: { channel: string; ts: string };
+  output: Record<string, unknown>;
+}): PlayCheckpoint {
+  const base = absurdCheckpointBase(input.checkpoint);
+  const existingBoundary = resolvedBoundaries(base)[input.boundaryId];
+  // Resolving a wait replaces its output, not the semantics captured when it
+  // was armed. In particular, an optional timeout policy must survive through
+  // the timeout wake so a later deployment cannot reinterpret the result.
+  const existingIntegrationBoundary =
+    existingBoundary?.kind === 'integration_event' ? existingBoundary : {};
+  return {
+    ...base,
+    resolvedBoundaries: {
+      ...resolvedBoundaries(base),
+      [input.boundaryId]: {
+        ...existingIntegrationBoundary,
+        kind: 'integration_event',
+        eventKey: input.eventKey,
+        timeoutMs: input.timeoutMs,
+        ...(input.messageRef ? { messageRef: input.messageRef } : {}),
+        output: input.output,
+        completedAt: Date.now(),
+      },
+    },
+  };
+}
+
+/** True when a boundary id is already resolved in the checkpoint (idempotent
+ * replay guard).
+ *
+ * An entry whose `kind` this build does not know THROWS rather than reporting
+ * "resolved" for a boundary it cannot actually replay — see
+ * `assertKnownResolvedBoundary`. */
+export function absurdBoundaryResolved(
+  checkpoint: unknown,
+  boundaryId: string,
+): boolean {
+  const base = absurdCheckpointBase(checkpoint);
+  return Boolean(
+    assertKnownResolvedBoundary(
+      boundaryId,
+      resolvedBoundaries(base)[boundaryId],
+    ),
+  );
+}
+
+/** Runner progress callbacks are a subset of the runner-backend contract; kept
+ * here so both the worker and its tests share one shape. */
+export type AbsurdRunnerProgressCallbacks = {
+  onLog?: (event: unknown) => void;
+  onRowUpdate?: (update: PlayRowUpdate) => void;
+  onExecutionEvent?: (event: PlayExecutionEvent) => void;
+};