deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts

@@ -0,0 +1,269 @@
+/**
+ * Bounded lifecycle cleanup for historical Absurd release queues.
+ *
+ * `runtimeReleases` is authoritative for live/draining lanes, but early
+ * release-lane deployments created physical `absurd.queues` entries before
+ * the lifecycle registry had a tombstone for every lane.  Those queues are
+ * still safe to reclaim once they are old, unowned, and have no nonterminal
+ * task.  This module deliberately owns only those orphan queues; registered
+ * retired releases continue through the stronger release-id/run-row gate in
+ * `runtime-lanes.ts`.
+ */
+import type { AbsurdLaneTaskCounts } from './absurd-runtime-worker';
+import type {
+  RuntimeSchedulerPoolOptions,
+  RuntimeSchedulerQueryClient,
+} from './postgres-pool';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+import { absurdQueueNameForRelease } from './absurd-shared';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+
+export const ABSURD_ORPHAN_QUEUE_GRACE_MS = 7 * 24 * 60 * 60 * 1_000;
+export const ABSURD_ORPHAN_QUEUE_BATCH_LIMIT = 5;
+/** Query a bounded page then rotate it. A blocked historical queue must never
+ * prevent later empty queues from being reclaimed. */
+export const ABSURD_ORPHAN_QUEUE_SCAN_LIMIT = 100;
+
+export type AbsurdQueueLifecycleCandidate = {
+  queueName: string;
+  createdAt: string;
+};
+
+export type AbsurdQueueLifecycleOutcome = {
+  scanned: number;
+  skippedProtected: string[];
+  skippedNonTerminal: Array<{
+    queueName: string;
+    taskCounts: AbsurdLaneTaskCounts;
+  }>;
+  dropped: string[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+};
+
+function quoteIdent(value: string): string {
+  return `"${value.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Atomically prove one orphan queue empty and remove it.
+ *
+ * `absurd.drop_queue` intentionally does not decide whether a queue is safe
+ * to delete. The controller locks scheduler runs, the queue registry, and the
+ * task table in submit order. Scheduler-run ownership, task count, and drop
+ * therefore share one serialization boundary rather than a racy sequence.
+ */
+async function dropAbsurdQueueIfStillEmpty(
+  client: RuntimeSchedulerQueryClient,
+  queueName: string,
+  schedulerSchema: string,
+): Promise<{ dropped: boolean; taskCounts: AbsurdLaneTaskCounts }> {
+  let transactionOpen = false;
+  try {
+    await client.query('BEGIN');
+    transactionOpen = true;
+    await client.query(
+      `SELECT set_config('lock_timeout', '2000ms', true),
+              set_config('statement_timeout', '15000ms', true)`,
+    );
+    // Scheduler submit writes its run row before spawning the Absurd task.
+    // Match that lock order: freeze nonterminal run inserts/updates before the
+    // queue registry and task table, avoiding both a deadlock and the
+    // run-without-task gap.
+    await client.query(
+      `LOCK TABLE ${quoteIdent(schedulerSchema)}.runs IN SHARE MODE`,
+    );
+    // `ensure_queue_tables` and `spawn_task` consult this registry before
+    // touching queue tables. Holding this lock makes the following task-table
+    // gate atomic with respect to a late enqueue.
+    await client.query('LOCK TABLE absurd.queues IN ACCESS EXCLUSIVE MODE');
+    const taskTable = `t_${queueName}`;
+    const present = await client.query<{ present: boolean }>(
+      `SELECT to_regclass(format('absurd.%I', $1::text)) IS NOT NULL AS present`,
+      [taskTable],
+    );
+    let taskCounts: AbsurdLaneTaskCounts = {
+      nonTerminal: 0,
+      claimable: 0,
+      running: 0,
+    };
+    if (present.rows[0]?.present) {
+      await client.query(
+        `LOCK TABLE absurd.${quoteIdent(taskTable)} IN ACCESS EXCLUSIVE MODE`,
+      );
+      const taskResult = await client.query<{ non_terminal: string }>(
+        `SELECT count(*) FILTER (WHERE state IN ('pending', 'running', 'sleeping'))::text AS non_terminal
+           FROM absurd.${quoteIdent(taskTable)}`,
+      );
+      taskCounts = {
+        ...taskCounts,
+        nonTerminal:
+          Number.parseInt(taskResult.rows[0]?.non_terminal ?? '0', 10) || 0,
+      };
+    }
+    const schedulerRuns = await client.query<{
+      release_id: string | null;
+      non_terminal: string;
+    }>(
+      `SELECT launch_json ->> 'absurdReleaseId' AS release_id,
+              count(*)::text AS non_terminal
+         FROM ${quoteIdent(schedulerSchema)}.runs
+        WHERE status IN ('queued', 'running', 'waiting')
+        GROUP BY launch_json ->> 'absurdReleaseId'`,
+    );
+    for (const row of schedulerRuns.rows) {
+      const releaseId = row.release_id?.trim();
+      if (!releaseId) continue;
+      let runQueue: string;
+      try {
+        runQueue = absurdQueueNameForRelease(releaseId);
+      } catch {
+        continue;
+      }
+      if (runQueue !== queueName) continue;
+      taskCounts.nonTerminal +=
+        Number.parseInt(row.non_terminal ?? '0', 10) || 0;
+    }
+    if (taskCounts.nonTerminal > 0) {
+      await client.query('ROLLBACK');
+      transactionOpen = false;
+      return { dropped: false, taskCounts };
+    }
+    await client.query('SELECT absurd.drop_queue($1)', [queueName]);
+    await client.query('COMMIT');
+    transactionOpen = false;
+    return { dropped: true, taskCounts };
+  } catch (error) {
+    if (transactionOpen) await client.query('ROLLBACK').catch(() => undefined);
+    throw error;
+  }
+}
+
+/** Pure rotating selection shared by the controller and its focused tests. */
+export function selectRotatingAbsurdOrphanQueues(input: {
+  candidates: readonly AbsurdQueueLifecycleCandidate[];
+  limit?: number;
+}): {
+  candidates: AbsurdQueueLifecycleCandidate[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+} {
+  const candidates = [...input.candidates].sort((left, right) => {
+    const created = left.createdAt.localeCompare(right.createdAt);
+    return created === 0
+      ? left.queueName.localeCompare(right.queueName)
+      : created;
+  });
+  if (candidates.length === 0) {
+    return { candidates: [], nextCursor: null };
+  }
+  const count = Math.min(
+    candidates.length,
+    Math.max(1, Math.floor(input.limit ?? ABSURD_ORPHAN_QUEUE_BATCH_LIMIT)),
+  );
+  const selected = candidates.slice(0, count);
+  return {
+    candidates: selected,
+    nextCursor: selected.at(-1) ?? null,
+  };
+}
+
+/**
+ * Reclaim only old `runs_*` release queues which no current controller/Fly
+ * owner can reference. The base `runs` queue is intentionally never eligible:
+ * it is the durable dev/default queue and cannot be reverse-mapped to one
+ * release safely.
+ *
+ * The caller supplies a fresh snapshot of every active, draining, retiring,
+ * rollback-held, and tombstoned queue. We still re-read each task table just
+ * before `drop_queue`; an empty queue is the destructive gate. Work is capped
+ * so a historical backlog cannot steal capacity reconciliation time.
+ */
+export async function cleanupAbsurdOrphanReleaseQueues(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  schedulerSchema?: string | null;
+  protectedQueueNames: ReadonlySet<string>;
+  nowMs?: number;
+  cursor?: AbsurdQueueLifecycleCandidate | null;
+  limit?: number;
+  scanLimit?: number;
+  withClient?: <T>(
+    callback: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+}): Promise<AbsurdQueueLifecycleOutcome> {
+  const nowMs = input.nowMs ?? Date.now();
+  const graceCutoff = new Date(nowMs - ABSURD_ORPHAN_QUEUE_GRACE_MS);
+  const schedulerSchema =
+    input.schedulerSchema?.trim() ||
+    resolveRuntimeSchedulerSchemaEnv() ||
+    'runtime_scheduler';
+  const withClient =
+    input.withClient ??
+    ((callback) => withRuntimeSchedulerClient(input.poolOptions, callback));
+
+  return withClient(async (client) => {
+    const readCandidates = (cursor?: AbsurdQueueLifecycleCandidate | null) =>
+      client.query<AbsurdQueueLifecycleCandidate>(
+        `SELECT queue_name AS "queueName", created_at::text AS "createdAt"
+         FROM absurd.queues
+        WHERE queue_name LIKE 'runs\\_%' ESCAPE '\\'
+          AND queue_name <> 'runs'
+          AND created_at <= $1
+          ${
+            cursor
+              ? 'AND (created_at, queue_name) > ($2::timestamptz, $3::text)'
+              : ''
+          }
+        ORDER BY created_at ASC, queue_name ASC
+        LIMIT $${cursor ? 4 : 2}`,
+        cursor
+          ? [
+              graceCutoff.toISOString(),
+              cursor.createdAt,
+              cursor.queueName,
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ]
+          : [
+              graceCutoff.toISOString(),
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ],
+      );
+    let candidatesResult = await readCandidates(input.cursor);
+    if (candidatesResult.rows.length === 0 && input.cursor) {
+      candidatesResult = await readCandidates();
+    }
+    const selection = selectRotatingAbsurdOrphanQueues({
+      candidates: candidatesResult.rows,
+      limit: input.limit,
+    });
+    const skippedProtected: string[] = [];
+    const skippedNonTerminal: AbsurdQueueLifecycleOutcome['skippedNonTerminal'] =
+      [];
+    const dropped: string[] = [];
+
+    for (const candidate of selection.candidates) {
+      if (input.protectedQueueNames.has(candidate.queueName)) {
+        skippedProtected.push(candidate.queueName);
+        continue;
+      }
+      const removal = await dropAbsurdQueueIfStillEmpty(
+        client,
+        candidate.queueName,
+        schedulerSchema,
+      );
+      if (!removal.dropped) {
+        skippedNonTerminal.push({
+          queueName: candidate.queueName,
+          taskCounts: removal.taskCounts,
+        });
+        continue;
+      }
+      dropped.push(candidate.queueName);
+    }
+    return {
+      scanned: selection.candidates.length,
+      skippedProtected,
+      skippedNonTerminal,
+      dropped,
+      nextCursor: selection.nextCursor,
+    };
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto';
+import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
+
+type EnvMap = Record<string, string | undefined>;
+
+/** Per-process suffix used to isolate Absurd queue tables. */
+export const ABSURD_QUEUE_SUFFIX_ENV = 'DEEPLINE_ABSURD_QUEUE_SUFFIX';
+
+/** Scheduler schema native to this process. */
+export const ABSURD_SCHEDULER_SCHEMA_ENV = 'DEEPLINE_RUNTIME_SCHEDULER_SCHEMA';
+
+/** The production scheduler schema uses the canonical, unsuffixed queues. */
+export const ABSURD_PROD_DEFAULT_SCHEMA = PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
+
+export const ABSURD_DEV_RELEASE_PREFIX = 'dev-';
+
+function normalizeQueueSuffixValue(raw: string): string {
+  return raw
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+    .slice(0, 32);
+}
+
+/** Read and normalize the queue suffix owned by this process. */
+export function resolveAbsurdQueueSuffix(env: EnvMap = process.env): string {
+  const raw = env[ABSURD_QUEUE_SUFFIX_ENV]?.trim();
+  return raw ? normalizeQueueSuffixValue(raw) : '';
+}
+
+/**
+ * Resolve the queue suffix for an explicitly selected scheduler schema.
+ * The process-owned suffix applies only to its native schema. A different
+ * explicitly selected schema is a remote runtime and therefore derives the
+ * stable schema fingerprint provisioned by that runtime's worker workflow.
+ */
+export function resolveAbsurdQueueSuffixForSchema(
+  schema?: string | null,
+  env: EnvMap = process.env,
+): string {
+  const normalizedSchema = schema?.trim();
+  const envSuffix = resolveAbsurdQueueSuffix(env);
+  if (!normalizedSchema || normalizedSchema === ABSURD_PROD_DEFAULT_SCHEMA) {
+    return envSuffix;
+  }
+  const nativeSchema = env[ABSURD_SCHEDULER_SCHEMA_ENV]?.trim();
+  if (envSuffix && nativeSchema === normalizedSchema) return envSuffix;
+  return resolveAbsurdQueueSuffixForIsolatedSchema(normalizedSchema);
+}
+
+/** Pure queue identity for an explicitly selected isolated scheduler schema. */
+export function resolveAbsurdQueueSuffixForIsolatedSchema(
+  schema: string,
+): string {
+  const normalizedSchema = schema.trim();
+  if (!normalizedSchema || normalizedSchema === ABSURD_PROD_DEFAULT_SCHEMA) {
+    throw new Error('An isolated scheduler schema is required.');
+  }
+  return normalizeQueueSuffixValue(
+    createHash('sha256').update(normalizedSchema).digest('hex').slice(0, 12),
+  );
+}
+
+/** Map a non-empty isolated queue suffix to its dev-collapse release id. */
+export function absurdDevReleaseIdForQueueSuffix(suffix: string): string {
+  const normalized = normalizeQueueSuffixValue(suffix.trim());
+  if (!normalized) {
+    throw new Error(
+      'An isolated Absurd runtime environment requires a non-empty queue suffix.',
+    );
+  }
+  return `${ABSURD_DEV_RELEASE_PREFIX}${normalized}`;
+}