deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts

@@ -0,0 +1,124 @@
+import type { RetryStrategy } from '@deepline/absurd/sdk';
+import {
+  createAbsurdRuntimeClient,
+  resolveAbsurdPoolOptions,
+  resolveAbsurdQueueSuffix,
+} from './absurd-shared';
+import { ensureAbsurdSchema } from './postgres';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+
+/**
+ * GitHub deploy-on-push build tasks ride their own fixed Absurd queue,
+ * deliberately OUTSIDE the release-lane model: builds are not versioned by
+ * play-runtime release, so pinning them to a `runs_<release>` lane would
+ * strand queued deploys on lane retirement. A plain rolling deploy of the
+ * deploy worker is correct.
+ */
+export const PLAY_DEPLOY_QUEUE = 'deploys';
+export const PLAY_DEPLOY_TASK_NAME = 'play-deploy';
+
+/**
+ * The queue this process submits to / drains. Production and plain local
+ * dev use the fixed base name; a worktree/preview with
+ * DEEPLINE_ABSURD_QUEUE_SUFFIX gets `deploys_<suffix>` so multiple local
+ * stacks sharing one docker Postgres never consume each other's builds
+ * (Absurd partitions its engine tables BY QUEUE, same as the runs lanes).
+ */
+export function playDeployQueueName(): string {
+  const suffix = resolveAbsurdQueueSuffix();
+  return suffix ? `${PLAY_DEPLOY_QUEUE}_${suffix}` : PLAY_DEPLOY_QUEUE;
+}
+export const PLAY_DEPLOY_MAX_ATTEMPTS = 5;
+export const PLAY_DEPLOY_RETRY_STRATEGY: RetryStrategy = {
+  kind: 'exponential',
+  baseSeconds: 5,
+  maxSeconds: 60,
+};
+/**
+ * Retained deploy rows are bounded like the runs queue: a stuck or finished
+ * deploy task must not accumulate forever.
+ *
+ * NOTE: this queue previously set a per-org claim cap. Upstream Absurd
+ * removed account-level caps from queue policy (see the comment on the runs
+ * queue in absurd-shared.ts: parked work holds no slot and fair claim
+ * ordering picks the next runnable task). Build parallelism is now bounded
+ * by worker slots, not per-org policy. If per-org fairness is wanted again
+ * it has to be expressed at claim time, not here.
+ */
+export const PLAY_DEPLOY_CLEANUP_TTL = '6 hours';
+export const PLAY_DEPLOY_CLEANUP_LIMIT = 1_000;
+
+/**
+ * Task payload stays tiny (32 KiB control-plane cap): the deploy token plus
+ * addressing. The worker fetches everything else — repo, commit, project
+ * settings, a GitHub installation token — from the app API with the token,
+ * so state is always read fresh (supersession, disabled projects).
+ */
+export type PlayDeployTaskParams = {
+  deploymentId: string;
+  orgId: string;
+  playProjectId: string;
+  deployToken: string;
+  appBaseUrl: string;
+};
+
+const deployQueueReadyPromises = new Map<string, Promise<void>>();
+
+/**
+ * Ensure the deploys queue exists with its policy. Idempotent per process;
+ * a failure evicts the memo so the next caller retries loudly. Deliberately
+ * NOT ensureAbsurdRuntimeReady — that helper hardcodes the runs-lane
+ * derivation and its policies.
+ */
+export function ensurePlayDeployQueueReady(input?: {
+  url?: string | null;
+  schema?: string | null;
+}): Promise<void> {
+  const poolOptions = resolveAbsurdPoolOptions(input?.url);
+  const schema = input?.schema?.trim() ?? '';
+  const queueName = playDeployQueueName();
+  const key = `${poolOptions.url}::${schema}::${queueName}`;
+  const existing = deployQueueReadyPromises.get(key);
+  if (existing) return existing;
+  const promise = (async () => {
+    await withRuntimeSchedulerClient(poolOptions, (client) =>
+      ensureAbsurdSchema(client),
+    );
+    const absurd = createAbsurdRuntimeClient(poolOptions, queueName);
+    const existingQueues = new Set(await absurd.listQueues());
+    if (!existingQueues.has(queueName)) {
+      await absurd.createQueue(queueName);
+    }
+    await absurd.setQueuePolicy(queueName, {
+      cleanupTtl: PLAY_DEPLOY_CLEANUP_TTL,
+      cleanupLimit: PLAY_DEPLOY_CLEANUP_LIMIT,
+    });
+  })().catch((error) => {
+    deployQueueReadyPromises.delete(key);
+    throw error;
+  });
+  deployQueueReadyPromises.set(key, promise);
+  return promise;
+}
+
+/**
+ * Enqueue one deploy build task, idempotent on the deployment id. Called by
+ * the webhook receiver AFTER the Convex playDeployments row exists and
+ * BEFORE the delivery is marked processed — a crash in between lets
+ * GitHub's redelivery re-run both steps safely.
+ */
+export async function enqueuePlayDeployTask(
+  params: PlayDeployTaskParams,
+): Promise<void> {
+  const poolOptions = resolveAbsurdPoolOptions();
+  await ensurePlayDeployQueueReady();
+  const queueName = playDeployQueueName();
+  const absurd = createAbsurdRuntimeClient(poolOptions, queueName);
+  await absurd.spawn(PLAY_DEPLOY_TASK_NAME, params, {
+    queue: queueName,
+    idempotencyKey: `play-deploy:${params.deploymentId}`,
+    orgKey: params.orgId,
+    maxAttempts: PLAY_DEPLOY_MAX_ATTEMPTS,
+    retryStrategy: PLAY_DEPLOY_RETRY_STRATEGY,
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts

@@ -0,0 +1,68 @@
+import type { RetryStrategy } from '@deepline/absurd/sdk';
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+
+export const ABSURD_PLAY_RUN_TASK = 'play-run';
+
+export const ABSURD_PLAY_RUN_RETRY_STRATEGY: RetryStrategy = {
+  kind: 'exponential',
+  baseSeconds: RUNTIME_RELIABILITY_POLICY.engine.retryBaseSeconds,
+  factor: 2,
+  maxSeconds: RUNTIME_RELIABILITY_POLICY.engine.retryMaxSeconds,
+};
+
+export const ABSURD_PLAY_RUN_MAX_ATTEMPTS =
+  RUNTIME_RELIABILITY_POLICY.engine.maxAttempts;
+
+export type AbsurdPlayRunTaskParams = {
+  runId: string;
+  orgId: string;
+  /**
+   * Retained release lanes may still validate this field before loading the
+   * launch row. Every newly submitted task is a root.
+   */
+  nestingDepth: number;
+  schedulerSchema?: string | null;
+};
+
+/**
+ * The one task ABI for an ordinary Play Run.
+ *
+ * Callers choose only the release queue. Task identity, fairness, retries, and
+ * queue-delay policy stay inside the scheduler adapter.
+ */
+export function buildAbsurdPlayRunTask(input: {
+  launch: Pick<
+    PlaySchedulerSubmitInput,
+    'runId' | 'orgId' | 'queueMaxDelaySeconds' | 'queuePriority'
+  >;
+  schedulerSchema?: string | null;
+}) {
+  const params: AbsurdPlayRunTaskParams = {
+    runId: input.launch.runId,
+    orgId: input.launch.orgId,
+    nestingDepth: 0,
+    schedulerSchema: input.schedulerSchema ?? null,
+  };
+  const maxDelaySeconds = input.launch.queueMaxDelaySeconds;
+  const priority = input.launch.queuePriority;
+  return {
+    taskName: ABSURD_PLAY_RUN_TASK,
+    params,
+    options: {
+      idempotency_key: input.launch.runId,
+      org_key: input.launch.orgId,
+      max_attempts: ABSURD_PLAY_RUN_MAX_ATTEMPTS,
+      retry_strategy: {
+        kind: ABSURD_PLAY_RUN_RETRY_STRATEGY.kind,
+        base_seconds: ABSURD_PLAY_RUN_RETRY_STRATEGY.baseSeconds,
+        factor: ABSURD_PLAY_RUN_RETRY_STRATEGY.factor,
+        max_seconds: ABSURD_PLAY_RUN_RETRY_STRATEGY.maxSeconds,
+      },
+      ...(typeof priority === 'number' ? { priority } : {}),
+      ...(typeof maxDelaySeconds === 'number'
+        ? { cancellation: { max_delay: maxDelaySeconds } }
+        : {}),
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts

@@ -0,0 +1,193 @@
+/**
+ * Src-coupled bridge between the vendored Absurd SDK (`packages/absurd`) and
+ * the hardened runtime scheduler Postgres pool (`./postgres-pool.ts`).
+ *
+ * workspace packages stay free of `src/**` imports, so the Absurd client takes an
+ * injected `Queryable`. This module supplies that handle by delegating every
+ * query through `withRuntimeSchedulerClient`, which is the process-singleton
+ * `pg.Pool` with saturation backpressure, a circuit breaker, and idempotent
+ * connection-error retries. Each Absurd SQL call is a self-contained plpgsql
+ * transaction, so a per-call client checkout is correct — Absurd never relies on
+ * cross-call session state.
+ */
+import { createRequire } from 'node:module';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import {
+  Absurd,
+  type AbsurdOptions,
+  type Queryable,
+  type QueryResult,
+} from '@deepline/absurd/sdk';
+import {
+  ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  ABSURD_SCHEMA_VERSION,
+} from '@deepline/absurd/pool';
+import {
+  withRuntimeSchedulerClient,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+
+export { ABSURD_SCHEMA_VERSION };
+
+/**
+ * Builds an SDK `Queryable` backed by the hardened runtime scheduler pool.
+ * Every query runs on a freshly checked-out pooled client and releases it,
+ * inheriting the pool's backpressure/breaker/retry guarantees.
+ */
+export function absurdQueryableForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+): Queryable {
+  return {
+    query: async <Row = Record<string, unknown>>(
+      sql: string,
+      params?: unknown[],
+    ): Promise<QueryResult<Row>> => {
+      return withRuntimeSchedulerClient(poolOptions, async (client) => {
+        const result = await client.query<Row & Record<string, unknown>>(
+          sql,
+          params,
+        );
+        return { rows: result.rows as Row[] };
+      });
+    },
+  };
+}
+
+/**
+ * Constructs an Absurd client wired to the runtime scheduler pool.
+ */
+export function createAbsurdForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+  options: Omit<AbsurdOptions, 'db'> = {},
+): Absurd {
+  return new Absurd({
+    ...options,
+    db: absurdQueryableForRuntimeScheduler(poolOptions),
+  });
+}
+
+let cachedSchemaSql: string | null = null;
+let cachedNoSchedulerOrgCapCompatibilitySql: string | null = null;
+
+/**
+ * The vendored schema version cannot represent Deepline-owned compatibility
+ * patches. Keep an explicit durable marker in the shared `absurd` schema so
+ * cold Vercel instances do not execute `CREATE OR REPLACE FUNCTION` on every
+ * first submission. Replacing a function while claims are running is DDL
+ * contention, not useful admission work.
+ */
+export const ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION =
+  'deepline-no-scheduler-org-cap-v1';
+
+function schemaSection(sql: string, start: string, end: string): string {
+  const startIndex = sql.indexOf(start);
+  const endIndex = sql.indexOf(end);
+  if (startIndex < 0 || endIndex < 0 || endIndex <= startIndex) {
+    throw new Error('Vendored absurd v5 schema upgrade markers are malformed.');
+  }
+  return sql.slice(startIndex + start.length, endIndex).trim();
+}
+
+/**
+ * Reads the vendored `absurd.sql` schema text. Cached after first read. Node
+ * only — the scheduler control plane and worker run under Node, never workerd.
+ */
+export function readAbsurdSchemaSql(): string {
+  if (cachedSchemaSql !== null) {
+    return cachedSchemaSql;
+  }
+  const packagePoolPath = createRequire(import.meta.url).resolve(
+    '@deepline/absurd/pool',
+  );
+  const schemaPath = path.resolve(
+    path.dirname(packagePoolPath),
+    ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  );
+  const sql = readFileSync(schemaPath, 'utf8');
+  if (!sql.includes('create schema if not exists absurd')) {
+    throw new Error(
+      `Vendored absurd schema at ${schemaPath} is missing or malformed.`,
+    );
+  }
+  cachedSchemaSql = sql;
+  return sql;
+}
+
+/**
+ * Applies no-cap behavior without changing the v4 schema contract.
+ *
+ * During a rolling release older workers still cold-start against v4 and may
+ * submit `per_org_concurrency`. Keeping the marker, column, function result
+ * shape, and accepted input key lets both versions run. The replacement
+ * claim function deliberately ignores the retired setting, so it cannot limit
+ * scheduler claims even while its compatibility-only column exists.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
+  if (cachedNoSchedulerOrgCapCompatibilitySql !== null) {
+    return cachedNoSchedulerOrgCapCompatibilitySql;
+  }
+  const sql = readAbsurdSchemaSql();
+  const claim = schemaSection(
+    sql,
+    '-- Workers call this to reserve a task from a given queue',
+    '-- Markes a run as completed',
+  );
+  const noCapClaim = claim
+    .replace(
+      `  -- DEEPLINE PATCH (a): per-org admission cap from queue policy (null=unlimited).\n  v_per_org_cap integer;\n`,
+      '',
+    )
+    .replace(
+      `\n  select per_org_concurrency into v_per_org_cap\n  from absurd.queues\n  where queue_name = p_queue_name;\n`,
+      '',
+    )
+    .replace(
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority + admission caps.\n`,
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority.\n`,
+    )
+    .replace(
+      `  --   * admission — when the queue sets per_org_concurrency, an org may only\n  --                 admit (cap - inflight) head-of-line tasks; an org already at\n  --                 the cap is skipped entirely.\n  --\n`,
+      '',
+    )
+    .replace(
+      `     admissible as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n         where $5::integer is null\n            or org_rank <= ($5::integer - inflight_cnt)\n     ),\n     ranked as (\n        select run_id\n          from admissible\n`,
+      `     ranked as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n`,
+    )
+    .replace(
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until, v_per_org_cap;',
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until;',
+    );
+
+  if (
+    noCapClaim.includes('v_per_org_cap') ||
+    noCapClaim.includes('$5::integer')
+  ) {
+    throw new Error(
+      'Absurd no-cap compatibility patch did not remove the retired claim gate.',
+    );
+  }
+  // The installed v4 claim function has the identical signature and result
+  // type. Replacing only its body is safe for old and new processes alike.
+  cachedNoSchedulerOrgCapCompatibilitySql = noCapClaim.replace(
+    'create function absurd.claim_task',
+    'create or replace function absurd.claim_task',
+  );
+  return cachedNoSchedulerOrgCapCompatibilitySql;
+}
+
+/**
+ * Marks the no-cap compatibility patch after its replacement claim function
+ * has committed. A function keeps the marker in Absurd's global schema and
+ * makes it valid for every runtime scheduler namespace that shares it.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(): string {
+  return `
+    CREATE OR REPLACE FUNCTION absurd.deepline_no_scheduler_org_cap_compatibility_version()
+    RETURNS text
+    LANGUAGE sql
+    AS $$
+      SELECT '${ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION}'::text
+    $$
+  `;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts

@@ -0,0 +1,269 @@
+/**
+ * Bounded lifecycle cleanup for historical Absurd release queues.
+ *
+ * `runtimeReleases` is authoritative for live/draining lanes, but early
+ * release-lane deployments created physical `absurd.queues` entries before
+ * the lifecycle registry had a tombstone for every lane.  Those queues are
+ * still safe to reclaim once they are old, unowned, and have no nonterminal
+ * task.  This module deliberately owns only those orphan queues; registered
+ * retired releases continue through the stronger release-id/run-row gate in
+ * `runtime-lanes.ts`.
+ */
+import type { AbsurdLaneTaskCounts } from './absurd-runtime-worker';
+import type {
+  RuntimeSchedulerPoolOptions,
+  RuntimeSchedulerQueryClient,
+} from './postgres-pool';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+import { absurdQueueNameForRelease } from './absurd-shared';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+
+export const ABSURD_ORPHAN_QUEUE_GRACE_MS = 7 * 24 * 60 * 60 * 1_000;
+export const ABSURD_ORPHAN_QUEUE_BATCH_LIMIT = 5;
+/** Query a bounded page then rotate it. A blocked historical queue must never
+ * prevent later empty queues from being reclaimed. */
+export const ABSURD_ORPHAN_QUEUE_SCAN_LIMIT = 100;
+
+export type AbsurdQueueLifecycleCandidate = {
+  queueName: string;
+  createdAt: string;
+};
+
+export type AbsurdQueueLifecycleOutcome = {
+  scanned: number;
+  skippedProtected: string[];
+  skippedNonTerminal: Array<{
+    queueName: string;
+    taskCounts: AbsurdLaneTaskCounts;
+  }>;
+  dropped: string[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+};
+
+function quoteIdent(value: string): string {
+  return `"${value.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Atomically prove one orphan queue empty and remove it.
+ *
+ * `absurd.drop_queue` intentionally does not decide whether a queue is safe
+ * to delete. The controller locks scheduler runs, the queue registry, and the
+ * task table in submit order. Scheduler-run ownership, task count, and drop
+ * therefore share one serialization boundary rather than a racy sequence.
+ */
+async function dropAbsurdQueueIfStillEmpty(
+  client: RuntimeSchedulerQueryClient,
+  queueName: string,
+  schedulerSchema: string,
+): Promise<{ dropped: boolean; taskCounts: AbsurdLaneTaskCounts }> {
+  let transactionOpen = false;
+  try {
+    await client.query('BEGIN');
+    transactionOpen = true;
+    await client.query(
+      `SELECT set_config('lock_timeout', '2000ms', true),
+              set_config('statement_timeout', '15000ms', true)`,
+    );
+    // Scheduler submit writes its run row before spawning the Absurd task.
+    // Match that lock order: freeze nonterminal run inserts/updates before the
+    // queue registry and task table, avoiding both a deadlock and the
+    // run-without-task gap.
+    await client.query(
+      `LOCK TABLE ${quoteIdent(schedulerSchema)}.runs IN SHARE MODE`,
+    );
+    // `ensure_queue_tables` and `spawn_task` consult this registry before
+    // touching queue tables. Holding this lock makes the following task-table
+    // gate atomic with respect to a late enqueue.
+    await client.query('LOCK TABLE absurd.queues IN ACCESS EXCLUSIVE MODE');
+    const taskTable = `t_${queueName}`;
+    const present = await client.query<{ present: boolean }>(
+      `SELECT to_regclass(format('absurd.%I', $1::text)) IS NOT NULL AS present`,
+      [taskTable],
+    );
+    let taskCounts: AbsurdLaneTaskCounts = {
+      nonTerminal: 0,
+      claimable: 0,
+      running: 0,
+    };
+    if (present.rows[0]?.present) {
+      await client.query(
+        `LOCK TABLE absurd.${quoteIdent(taskTable)} IN ACCESS EXCLUSIVE MODE`,
+      );
+      const taskResult = await client.query<{ non_terminal: string }>(
+        `SELECT count(*) FILTER (WHERE state IN ('pending', 'running', 'sleeping'))::text AS non_terminal
+           FROM absurd.${quoteIdent(taskTable)}`,
+      );
+      taskCounts = {
+        ...taskCounts,
+        nonTerminal:
+          Number.parseInt(taskResult.rows[0]?.non_terminal ?? '0', 10) || 0,
+      };
+    }
+    const schedulerRuns = await client.query<{
+      release_id: string | null;
+      non_terminal: string;
+    }>(
+      `SELECT launch_json ->> 'absurdReleaseId' AS release_id,
+              count(*)::text AS non_terminal
+         FROM ${quoteIdent(schedulerSchema)}.runs
+        WHERE status IN ('queued', 'running', 'waiting')
+        GROUP BY launch_json ->> 'absurdReleaseId'`,
+    );
+    for (const row of schedulerRuns.rows) {
+      const releaseId = row.release_id?.trim();
+      if (!releaseId) continue;
+      let runQueue: string;
+      try {
+        runQueue = absurdQueueNameForRelease(releaseId);
+      } catch {
+        continue;
+      }
+      if (runQueue !== queueName) continue;
+      taskCounts.nonTerminal +=
+        Number.parseInt(row.non_terminal ?? '0', 10) || 0;
+    }
+    if (taskCounts.nonTerminal > 0) {
+      await client.query('ROLLBACK');
+      transactionOpen = false;
+      return { dropped: false, taskCounts };
+    }
+    await client.query('SELECT absurd.drop_queue($1)', [queueName]);
+    await client.query('COMMIT');
+    transactionOpen = false;
+    return { dropped: true, taskCounts };
+  } catch (error) {
+    if (transactionOpen) await client.query('ROLLBACK').catch(() => undefined);
+    throw error;
+  }
+}
+
+/** Pure rotating selection shared by the controller and its focused tests. */
+export function selectRotatingAbsurdOrphanQueues(input: {
+  candidates: readonly AbsurdQueueLifecycleCandidate[];
+  limit?: number;
+}): {
+  candidates: AbsurdQueueLifecycleCandidate[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+} {
+  const candidates = [...input.candidates].sort((left, right) => {
+    const created = left.createdAt.localeCompare(right.createdAt);
+    return created === 0
+      ? left.queueName.localeCompare(right.queueName)
+      : created;
+  });
+  if (candidates.length === 0) {
+    return { candidates: [], nextCursor: null };
+  }
+  const count = Math.min(
+    candidates.length,
+    Math.max(1, Math.floor(input.limit ?? ABSURD_ORPHAN_QUEUE_BATCH_LIMIT)),
+  );
+  const selected = candidates.slice(0, count);
+  return {
+    candidates: selected,
+    nextCursor: selected.at(-1) ?? null,
+  };
+}
+
+/**
+ * Reclaim only old `runs_*` release queues which no current controller/Fly
+ * owner can reference. The base `runs` queue is intentionally never eligible:
+ * it is the durable dev/default queue and cannot be reverse-mapped to one
+ * release safely.
+ *
+ * The caller supplies a fresh snapshot of every active, draining, retiring,
+ * rollback-held, and tombstoned queue. We still re-read each task table just
+ * before `drop_queue`; an empty queue is the destructive gate. Work is capped
+ * so a historical backlog cannot steal capacity reconciliation time.
+ */
+export async function cleanupAbsurdOrphanReleaseQueues(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  schedulerSchema?: string | null;
+  protectedQueueNames: ReadonlySet<string>;
+  nowMs?: number;
+  cursor?: AbsurdQueueLifecycleCandidate | null;
+  limit?: number;
+  scanLimit?: number;
+  withClient?: <T>(
+    callback: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+}): Promise<AbsurdQueueLifecycleOutcome> {
+  const nowMs = input.nowMs ?? Date.now();
+  const graceCutoff = new Date(nowMs - ABSURD_ORPHAN_QUEUE_GRACE_MS);
+  const schedulerSchema =
+    input.schedulerSchema?.trim() ||
+    resolveRuntimeSchedulerSchemaEnv() ||
+    'runtime_scheduler';
+  const withClient =
+    input.withClient ??
+    ((callback) => withRuntimeSchedulerClient(input.poolOptions, callback));
+
+  return withClient(async (client) => {
+    const readCandidates = (cursor?: AbsurdQueueLifecycleCandidate | null) =>
+      client.query<AbsurdQueueLifecycleCandidate>(
+        `SELECT queue_name AS "queueName", created_at::text AS "createdAt"
+         FROM absurd.queues
+        WHERE queue_name LIKE 'runs\\_%' ESCAPE '\\'
+          AND queue_name <> 'runs'
+          AND created_at <= $1
+          ${
+            cursor
+              ? 'AND (created_at, queue_name) > ($2::timestamptz, $3::text)'
+              : ''
+          }
+        ORDER BY created_at ASC, queue_name ASC
+        LIMIT $${cursor ? 4 : 2}`,
+        cursor
+          ? [
+              graceCutoff.toISOString(),
+              cursor.createdAt,
+              cursor.queueName,
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ]
+          : [
+              graceCutoff.toISOString(),
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ],
+      );
+    let candidatesResult = await readCandidates(input.cursor);
+    if (candidatesResult.rows.length === 0 && input.cursor) {
+      candidatesResult = await readCandidates();
+    }
+    const selection = selectRotatingAbsurdOrphanQueues({
+      candidates: candidatesResult.rows,
+      limit: input.limit,
+    });
+    const skippedProtected: string[] = [];
+    const skippedNonTerminal: AbsurdQueueLifecycleOutcome['skippedNonTerminal'] =
+      [];
+    const dropped: string[] = [];
+
+    for (const candidate of selection.candidates) {
+      if (input.protectedQueueNames.has(candidate.queueName)) {
+        skippedProtected.push(candidate.queueName);
+        continue;
+      }
+      const removal = await dropAbsurdQueueIfStillEmpty(
+        client,
+        candidate.queueName,
+        schedulerSchema,
+      );
+      if (!removal.dropped) {
+        skippedNonTerminal.push({
+          queueName: candidate.queueName,
+          taskCounts: removal.taskCounts,
+        });
+        continue;
+      }
+      dropped.push(candidate.queueName);
+    }
+    return {
+      scanned: selection.candidates.length,
+      skippedProtected,
+      skippedNonTerminal,
+      dropped,
+      nextCursor: selection.nextCursor,
+    };
+  });
+}