deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts

@@ -0,0 +1,193 @@
+/**
+ * Src-coupled bridge between the vendored Absurd SDK (`packages/absurd`) and
+ * the hardened runtime scheduler Postgres pool (`./postgres-pool.ts`).
+ *
+ * workspace packages stay free of `src/**` imports, so the Absurd client takes an
+ * injected `Queryable`. This module supplies that handle by delegating every
+ * query through `withRuntimeSchedulerClient`, which is the process-singleton
+ * `pg.Pool` with saturation backpressure, a circuit breaker, and idempotent
+ * connection-error retries. Each Absurd SQL call is a self-contained plpgsql
+ * transaction, so a per-call client checkout is correct — Absurd never relies on
+ * cross-call session state.
+ */
+import { createRequire } from 'node:module';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import {
+  Absurd,
+  type AbsurdOptions,
+  type Queryable,
+  type QueryResult,
+} from '@deepline/absurd/sdk';
+import {
+  ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  ABSURD_SCHEMA_VERSION,
+} from '@deepline/absurd/pool';
+import {
+  withRuntimeSchedulerClient,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+
+export { ABSURD_SCHEMA_VERSION };
+
+/**
+ * Builds an SDK `Queryable` backed by the hardened runtime scheduler pool.
+ * Every query runs on a freshly checked-out pooled client and releases it,
+ * inheriting the pool's backpressure/breaker/retry guarantees.
+ */
+export function absurdQueryableForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+): Queryable {
+  return {
+    query: async <Row = Record<string, unknown>>(
+      sql: string,
+      params?: unknown[],
+    ): Promise<QueryResult<Row>> => {
+      return withRuntimeSchedulerClient(poolOptions, async (client) => {
+        const result = await client.query<Row & Record<string, unknown>>(
+          sql,
+          params,
+        );
+        return { rows: result.rows as Row[] };
+      });
+    },
+  };
+}
+
+/**
+ * Constructs an Absurd client wired to the runtime scheduler pool.
+ */
+export function createAbsurdForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+  options: Omit<AbsurdOptions, 'db'> = {},
+): Absurd {
+  return new Absurd({
+    ...options,
+    db: absurdQueryableForRuntimeScheduler(poolOptions),
+  });
+}
+
+let cachedSchemaSql: string | null = null;
+let cachedNoSchedulerOrgCapCompatibilitySql: string | null = null;
+
+/**
+ * The vendored schema version cannot represent Deepline-owned compatibility
+ * patches. Keep an explicit durable marker in the shared `absurd` schema so
+ * cold Vercel instances do not execute `CREATE OR REPLACE FUNCTION` on every
+ * first submission. Replacing a function while claims are running is DDL
+ * contention, not useful admission work.
+ */
+export const ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION =
+  'deepline-no-scheduler-org-cap-v1';
+
+function schemaSection(sql: string, start: string, end: string): string {
+  const startIndex = sql.indexOf(start);
+  const endIndex = sql.indexOf(end);
+  if (startIndex < 0 || endIndex < 0 || endIndex <= startIndex) {
+    throw new Error('Vendored absurd v5 schema upgrade markers are malformed.');
+  }
+  return sql.slice(startIndex + start.length, endIndex).trim();
+}
+
+/**
+ * Reads the vendored `absurd.sql` schema text. Cached after first read. Node
+ * only — the scheduler control plane and worker run under Node, never workerd.
+ */
+export function readAbsurdSchemaSql(): string {
+  if (cachedSchemaSql !== null) {
+    return cachedSchemaSql;
+  }
+  const packagePoolPath = createRequire(import.meta.url).resolve(
+    '@deepline/absurd/pool',
+  );
+  const schemaPath = path.resolve(
+    path.dirname(packagePoolPath),
+    ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  );
+  const sql = readFileSync(schemaPath, 'utf8');
+  if (!sql.includes('create schema if not exists absurd')) {
+    throw new Error(
+      `Vendored absurd schema at ${schemaPath} is missing or malformed.`,
+    );
+  }
+  cachedSchemaSql = sql;
+  return sql;
+}
+
+/**
+ * Applies no-cap behavior without changing the v4 schema contract.
+ *
+ * During a rolling release older workers still cold-start against v4 and may
+ * submit `per_org_concurrency`. Keeping the marker, column, function result
+ * shape, and accepted input key lets both versions run. The replacement
+ * claim function deliberately ignores the retired setting, so it cannot limit
+ * scheduler claims even while its compatibility-only column exists.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
+  if (cachedNoSchedulerOrgCapCompatibilitySql !== null) {
+    return cachedNoSchedulerOrgCapCompatibilitySql;
+  }
+  const sql = readAbsurdSchemaSql();
+  const claim = schemaSection(
+    sql,
+    '-- Workers call this to reserve a task from a given queue',
+    '-- Markes a run as completed',
+  );
+  const noCapClaim = claim
+    .replace(
+      `  -- DEEPLINE PATCH (a): per-org admission cap from queue policy (null=unlimited).\n  v_per_org_cap integer;\n`,
+      '',
+    )
+    .replace(
+      `\n  select per_org_concurrency into v_per_org_cap\n  from absurd.queues\n  where queue_name = p_queue_name;\n`,
+      '',
+    )
+    .replace(
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority + admission caps.\n`,
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority.\n`,
+    )
+    .replace(
+      `  --   * admission — when the queue sets per_org_concurrency, an org may only\n  --                 admit (cap - inflight) head-of-line tasks; an org already at\n  --                 the cap is skipped entirely.\n  --\n`,
+      '',
+    )
+    .replace(
+      `     admissible as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n         where $5::integer is null\n            or org_rank <= ($5::integer - inflight_cnt)\n     ),\n     ranked as (\n        select run_id\n          from admissible\n`,
+      `     ranked as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n`,
+    )
+    .replace(
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until, v_per_org_cap;',
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until;',
+    );
+
+  if (
+    noCapClaim.includes('v_per_org_cap') ||
+    noCapClaim.includes('$5::integer')
+  ) {
+    throw new Error(
+      'Absurd no-cap compatibility patch did not remove the retired claim gate.',
+    );
+  }
+  // The installed v4 claim function has the identical signature and result
+  // type. Replacing only its body is safe for old and new processes alike.
+  cachedNoSchedulerOrgCapCompatibilitySql = noCapClaim.replace(
+    'create function absurd.claim_task',
+    'create or replace function absurd.claim_task',
+  );
+  return cachedNoSchedulerOrgCapCompatibilitySql;
+}
+
+/**
+ * Marks the no-cap compatibility patch after its replacement claim function
+ * has committed. A function keeps the marker in Absurd's global schema and
+ * makes it valid for every runtime scheduler namespace that shares it.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(): string {
+  return `
+    CREATE OR REPLACE FUNCTION absurd.deepline_no_scheduler_org_cap_compatibility_version()
+    RETURNS text
+    LANGUAGE sql
+    AS $$
+      SELECT '${ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION}'::text
+    $$
+  `;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-lifecycle.ts

@@ -0,0 +1,269 @@
+/**
+ * Bounded lifecycle cleanup for historical Absurd release queues.
+ *
+ * `runtimeReleases` is authoritative for live/draining lanes, but early
+ * release-lane deployments created physical `absurd.queues` entries before
+ * the lifecycle registry had a tombstone for every lane.  Those queues are
+ * still safe to reclaim once they are old, unowned, and have no nonterminal
+ * task.  This module deliberately owns only those orphan queues; registered
+ * retired releases continue through the stronger release-id/run-row gate in
+ * `runtime-lanes.ts`.
+ */
+import type { AbsurdLaneTaskCounts } from './absurd-runtime-worker';
+import type {
+  RuntimeSchedulerPoolOptions,
+  RuntimeSchedulerQueryClient,
+} from './postgres-pool';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+import { absurdQueueNameForRelease } from './absurd-shared';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+
+export const ABSURD_ORPHAN_QUEUE_GRACE_MS = 7 * 24 * 60 * 60 * 1_000;
+export const ABSURD_ORPHAN_QUEUE_BATCH_LIMIT = 5;
+/** Query a bounded page then rotate it. A blocked historical queue must never
+ * prevent later empty queues from being reclaimed. */
+export const ABSURD_ORPHAN_QUEUE_SCAN_LIMIT = 100;
+
+export type AbsurdQueueLifecycleCandidate = {
+  queueName: string;
+  createdAt: string;
+};
+
+export type AbsurdQueueLifecycleOutcome = {
+  scanned: number;
+  skippedProtected: string[];
+  skippedNonTerminal: Array<{
+    queueName: string;
+    taskCounts: AbsurdLaneTaskCounts;
+  }>;
+  dropped: string[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+};
+
+function quoteIdent(value: string): string {
+  return `"${value.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Atomically prove one orphan queue empty and remove it.
+ *
+ * `absurd.drop_queue` intentionally does not decide whether a queue is safe
+ * to delete. The controller locks scheduler runs, the queue registry, and the
+ * task table in submit order. Scheduler-run ownership, task count, and drop
+ * therefore share one serialization boundary rather than a racy sequence.
+ */
+async function dropAbsurdQueueIfStillEmpty(
+  client: RuntimeSchedulerQueryClient,
+  queueName: string,
+  schedulerSchema: string,
+): Promise<{ dropped: boolean; taskCounts: AbsurdLaneTaskCounts }> {
+  let transactionOpen = false;
+  try {
+    await client.query('BEGIN');
+    transactionOpen = true;
+    await client.query(
+      `SELECT set_config('lock_timeout', '2000ms', true),
+              set_config('statement_timeout', '15000ms', true)`,
+    );
+    // Scheduler submit writes its run row before spawning the Absurd task.
+    // Match that lock order: freeze nonterminal run inserts/updates before the
+    // queue registry and task table, avoiding both a deadlock and the
+    // run-without-task gap.
+    await client.query(
+      `LOCK TABLE ${quoteIdent(schedulerSchema)}.runs IN SHARE MODE`,
+    );
+    // `ensure_queue_tables` and `spawn_task` consult this registry before
+    // touching queue tables. Holding this lock makes the following task-table
+    // gate atomic with respect to a late enqueue.
+    await client.query('LOCK TABLE absurd.queues IN ACCESS EXCLUSIVE MODE');
+    const taskTable = `t_${queueName}`;
+    const present = await client.query<{ present: boolean }>(
+      `SELECT to_regclass(format('absurd.%I', $1::text)) IS NOT NULL AS present`,
+      [taskTable],
+    );
+    let taskCounts: AbsurdLaneTaskCounts = {
+      nonTerminal: 0,
+      claimable: 0,
+      running: 0,
+    };
+    if (present.rows[0]?.present) {
+      await client.query(
+        `LOCK TABLE absurd.${quoteIdent(taskTable)} IN ACCESS EXCLUSIVE MODE`,
+      );
+      const taskResult = await client.query<{ non_terminal: string }>(
+        `SELECT count(*) FILTER (WHERE state IN ('pending', 'running', 'sleeping'))::text AS non_terminal
+           FROM absurd.${quoteIdent(taskTable)}`,
+      );
+      taskCounts = {
+        ...taskCounts,
+        nonTerminal:
+          Number.parseInt(taskResult.rows[0]?.non_terminal ?? '0', 10) || 0,
+      };
+    }
+    const schedulerRuns = await client.query<{
+      release_id: string | null;
+      non_terminal: string;
+    }>(
+      `SELECT launch_json ->> 'absurdReleaseId' AS release_id,
+              count(*)::text AS non_terminal
+         FROM ${quoteIdent(schedulerSchema)}.runs
+        WHERE status IN ('queued', 'running', 'waiting')
+        GROUP BY launch_json ->> 'absurdReleaseId'`,
+    );
+    for (const row of schedulerRuns.rows) {
+      const releaseId = row.release_id?.trim();
+      if (!releaseId) continue;
+      let runQueue: string;
+      try {
+        runQueue = absurdQueueNameForRelease(releaseId);
+      } catch {
+        continue;
+      }
+      if (runQueue !== queueName) continue;
+      taskCounts.nonTerminal +=
+        Number.parseInt(row.non_terminal ?? '0', 10) || 0;
+    }
+    if (taskCounts.nonTerminal > 0) {
+      await client.query('ROLLBACK');
+      transactionOpen = false;
+      return { dropped: false, taskCounts };
+    }
+    await client.query('SELECT absurd.drop_queue($1)', [queueName]);
+    await client.query('COMMIT');
+    transactionOpen = false;
+    return { dropped: true, taskCounts };
+  } catch (error) {
+    if (transactionOpen) await client.query('ROLLBACK').catch(() => undefined);
+    throw error;
+  }
+}
+
+/** Pure rotating selection shared by the controller and its focused tests. */
+export function selectRotatingAbsurdOrphanQueues(input: {
+  candidates: readonly AbsurdQueueLifecycleCandidate[];
+  limit?: number;
+}): {
+  candidates: AbsurdQueueLifecycleCandidate[];
+  nextCursor: AbsurdQueueLifecycleCandidate | null;
+} {
+  const candidates = [...input.candidates].sort((left, right) => {
+    const created = left.createdAt.localeCompare(right.createdAt);
+    return created === 0
+      ? left.queueName.localeCompare(right.queueName)
+      : created;
+  });
+  if (candidates.length === 0) {
+    return { candidates: [], nextCursor: null };
+  }
+  const count = Math.min(
+    candidates.length,
+    Math.max(1, Math.floor(input.limit ?? ABSURD_ORPHAN_QUEUE_BATCH_LIMIT)),
+  );
+  const selected = candidates.slice(0, count);
+  return {
+    candidates: selected,
+    nextCursor: selected.at(-1) ?? null,
+  };
+}
+
+/**
+ * Reclaim only old `runs_*` release queues which no current controller/Fly
+ * owner can reference. The base `runs` queue is intentionally never eligible:
+ * it is the durable dev/default queue and cannot be reverse-mapped to one
+ * release safely.
+ *
+ * The caller supplies a fresh snapshot of every active, draining, retiring,
+ * rollback-held, and tombstoned queue. We still re-read each task table just
+ * before `drop_queue`; an empty queue is the destructive gate. Work is capped
+ * so a historical backlog cannot steal capacity reconciliation time.
+ */
+export async function cleanupAbsurdOrphanReleaseQueues(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  schedulerSchema?: string | null;
+  protectedQueueNames: ReadonlySet<string>;
+  nowMs?: number;
+  cursor?: AbsurdQueueLifecycleCandidate | null;
+  limit?: number;
+  scanLimit?: number;
+  withClient?: <T>(
+    callback: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+}): Promise<AbsurdQueueLifecycleOutcome> {
+  const nowMs = input.nowMs ?? Date.now();
+  const graceCutoff = new Date(nowMs - ABSURD_ORPHAN_QUEUE_GRACE_MS);
+  const schedulerSchema =
+    input.schedulerSchema?.trim() ||
+    resolveRuntimeSchedulerSchemaEnv() ||
+    'runtime_scheduler';
+  const withClient =
+    input.withClient ??
+    ((callback) => withRuntimeSchedulerClient(input.poolOptions, callback));
+
+  return withClient(async (client) => {
+    const readCandidates = (cursor?: AbsurdQueueLifecycleCandidate | null) =>
+      client.query<AbsurdQueueLifecycleCandidate>(
+        `SELECT queue_name AS "queueName", created_at::text AS "createdAt"
+         FROM absurd.queues
+        WHERE queue_name LIKE 'runs\\_%' ESCAPE '\\'
+          AND queue_name <> 'runs'
+          AND created_at <= $1
+          ${
+            cursor
+              ? 'AND (created_at, queue_name) > ($2::timestamptz, $3::text)'
+              : ''
+          }
+        ORDER BY created_at ASC, queue_name ASC
+        LIMIT $${cursor ? 4 : 2}`,
+        cursor
+          ? [
+              graceCutoff.toISOString(),
+              cursor.createdAt,
+              cursor.queueName,
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ]
+          : [
+              graceCutoff.toISOString(),
+              input.scanLimit ?? ABSURD_ORPHAN_QUEUE_SCAN_LIMIT,
+            ],
+      );
+    let candidatesResult = await readCandidates(input.cursor);
+    if (candidatesResult.rows.length === 0 && input.cursor) {
+      candidatesResult = await readCandidates();
+    }
+    const selection = selectRotatingAbsurdOrphanQueues({
+      candidates: candidatesResult.rows,
+      limit: input.limit,
+    });
+    const skippedProtected: string[] = [];
+    const skippedNonTerminal: AbsurdQueueLifecycleOutcome['skippedNonTerminal'] =
+      [];
+    const dropped: string[] = [];
+
+    for (const candidate of selection.candidates) {
+      if (input.protectedQueueNames.has(candidate.queueName)) {
+        skippedProtected.push(candidate.queueName);
+        continue;
+      }
+      const removal = await dropAbsurdQueueIfStillEmpty(
+        client,
+        candidate.queueName,
+        schedulerSchema,
+      );
+      if (!removal.dropped) {
+        skippedNonTerminal.push({
+          queueName: candidate.queueName,
+          taskCounts: removal.taskCounts,
+        });
+        continue;
+      }
+      dropped.push(candidate.queueName);
+    }
+    return {
+      scanned: selection.candidates.length,
+      skippedProtected,
+      skippedNonTerminal,
+      dropped,
+      nextCursor: selection.nextCursor,
+    };
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-queue-suffix.ts

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto';
+import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
+
+type EnvMap = Record<string, string | undefined>;
+
+/** Per-process suffix used to isolate Absurd queue tables. */
+export const ABSURD_QUEUE_SUFFIX_ENV = 'DEEPLINE_ABSURD_QUEUE_SUFFIX';
+
+/** Scheduler schema native to this process. */
+export const ABSURD_SCHEDULER_SCHEMA_ENV = 'DEEPLINE_RUNTIME_SCHEDULER_SCHEMA';
+
+/** The production scheduler schema uses the canonical, unsuffixed queues. */
+export const ABSURD_PROD_DEFAULT_SCHEMA = PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA;
+
+export const ABSURD_DEV_RELEASE_PREFIX = 'dev-';
+
+function normalizeQueueSuffixValue(raw: string): string {
+  return raw
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+    .slice(0, 32);
+}
+
+/** Read and normalize the queue suffix owned by this process. */
+export function resolveAbsurdQueueSuffix(env: EnvMap = process.env): string {
+  const raw = env[ABSURD_QUEUE_SUFFIX_ENV]?.trim();
+  return raw ? normalizeQueueSuffixValue(raw) : '';
+}
+
+/**
+ * Resolve the queue suffix for an explicitly selected scheduler schema.
+ * The process-owned suffix applies only to its native schema. A different
+ * explicitly selected schema is a remote runtime and therefore derives the
+ * stable schema fingerprint provisioned by that runtime's worker workflow.
+ */
+export function resolveAbsurdQueueSuffixForSchema(
+  schema?: string | null,
+  env: EnvMap = process.env,
+): string {
+  const normalizedSchema = schema?.trim();
+  const envSuffix = resolveAbsurdQueueSuffix(env);
+  if (!normalizedSchema || normalizedSchema === ABSURD_PROD_DEFAULT_SCHEMA) {
+    return envSuffix;
+  }
+  const nativeSchema = env[ABSURD_SCHEDULER_SCHEMA_ENV]?.trim();
+  if (envSuffix && nativeSchema === normalizedSchema) return envSuffix;
+  return resolveAbsurdQueueSuffixForIsolatedSchema(normalizedSchema);
+}
+
+/** Pure queue identity for an explicitly selected isolated scheduler schema. */
+export function resolveAbsurdQueueSuffixForIsolatedSchema(
+  schema: string,
+): string {
+  const normalizedSchema = schema.trim();
+  if (!normalizedSchema || normalizedSchema === ABSURD_PROD_DEFAULT_SCHEMA) {
+    throw new Error('An isolated scheduler schema is required.');
+  }
+  return normalizeQueueSuffixValue(
+    createHash('sha256').update(normalizedSchema).digest('hex').slice(0, 12),
+  );
+}
+
+/** Map a non-empty isolated queue suffix to its dev-collapse release id. */
+export function absurdDevReleaseIdForQueueSuffix(suffix: string): string {
+  const normalized = normalizeQueueSuffixValue(suffix.trim());
+  if (!normalized) {
+    throw new Error(
+      'An isolated Absurd runtime environment requires a non-empty queue suffix.',
+    );
+  }
+  return `${ABSURD_DEV_RELEASE_PREFIX}${normalized}`;
+}