deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-deadline-reconciliation.ts

@@ -0,0 +1,88 @@
+import { cancelPostgresSchedulerRun } from './postgres';
+import type {
+  PostgresSchedulerOptions,
+  PostgresSchedulerQueryClient,
+} from './postgres';
+import { ABSURD_PLAY_RUN_TASK } from './absurd-shared';
+import { resolveRuntimeSchedulerSchemaEnv } from './runtime-scheduler-schema';
+
+/** Bound deadline reconciliation just like Absurd's own cancellation sweep. */
+export const ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT = 100;
+
+function quoteAbsurdTable(table: string): string {
+  return `"${table.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Project a play task cancelled by Absurd's indexed `max_delay` sweep into the
+ * scheduler's authoritative run state.
+ *
+ * The engine can cancel an overdue task before or after a worker claims its
+ * handler. A cancelled task has no handler path left to write the normal
+ * terminal state. This reconciler
+ * closes that gap: `cancelPostgresSchedulerRun` atomically writes the runtime
+ * terminal and its run-ledger outbox event, and is idempotent if a user cancel
+ * or a racing worker already terminalized the run. The next empty claim invokes
+ * it immediately after Absurd performs its bounded cancellation sweep.
+ */
+export async function reconcileAbsurdRuntimeMaxDelayCancellations(input: {
+  client: PostgresSchedulerQueryClient;
+  queue: string;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  limit?: number;
+}): Promise<{ reconciled: number; skipped: number }> {
+  const requestedLimit = Math.floor(
+    input.limit ?? ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT,
+  );
+  const limit = Number.isFinite(requestedLimit)
+    ? Math.max(
+        1,
+        Math.min(ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT, requestedLimit),
+      )
+    : ABSURD_CANCELLED_DEADLINE_RECONCILE_LIMIT;
+  const taskTable = quoteAbsurdTable(`t_${input.queue}`);
+  const schedulerSchema = quoteAbsurdTable(
+    input.schedulerOptions?.schema?.trim() ||
+      resolveRuntimeSchedulerSchemaEnv() ||
+      'runtime_scheduler',
+  );
+  const cancelled = await input.client.query<{ run_id: string | null }>(
+    `
+      SELECT task.params ->> 'runId' AS run_id
+      FROM absurd.${taskTable} AS task
+      JOIN ${schedulerSchema}.runs AS run
+        ON run.run_id = task.params ->> 'runId'
+       AND run.status IN ('queued', 'running', 'waiting')
+      WHERE task.state = 'cancelled'
+        AND task.task_name = $1
+        AND task.cancellation ? 'max_delay'
+        AND task.cancel_at IS NOT NULL
+        AND task.cancelled_at >= task.cancel_at
+      ORDER BY task.cancelled_at ASC NULLS LAST, task.task_id ASC
+      LIMIT $2
+    `,
+    [ABSURD_PLAY_RUN_TASK, limit],
+  );
+
+  let reconciled = 0;
+  let skipped = 0;
+  for (const row of cancelled.rows) {
+    const runId = row.run_id?.trim();
+    if (!runId) {
+      skipped += 1;
+      continue;
+    }
+    const didCancel = await cancelPostgresSchedulerRun(
+      input.client,
+      {
+        runId,
+        reason:
+          'Run did not start within its configured scheduler queue deadline.',
+      },
+      input.schedulerOptions,
+    );
+    if (didCancel) reconciled += 1;
+    else skipped += 1;
+  }
+  return { reconciled, skipped };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts

@@ -0,0 +1,124 @@
+import type { RetryStrategy } from '@deepline/absurd/sdk';
+import {
+  createAbsurdRuntimeClient,
+  resolveAbsurdPoolOptions,
+  resolveAbsurdQueueSuffix,
+} from './absurd-shared';
+import { ensureAbsurdSchema } from './postgres';
+import { withRuntimeSchedulerClient } from './postgres-pool';
+
+/**
+ * GitHub deploy-on-push build tasks ride their own fixed Absurd queue,
+ * deliberately OUTSIDE the release-lane model: builds are not versioned by
+ * play-runtime release, so pinning them to a `runs_<release>` lane would
+ * strand queued deploys on lane retirement. A plain rolling deploy of the
+ * deploy worker is correct.
+ */
+export const PLAY_DEPLOY_QUEUE = 'deploys';
+export const PLAY_DEPLOY_TASK_NAME = 'play-deploy';
+
+/**
+ * The queue this process submits to / drains. Production and plain local
+ * dev use the fixed base name; a worktree/preview with
+ * DEEPLINE_ABSURD_QUEUE_SUFFIX gets `deploys_<suffix>` so multiple local
+ * stacks sharing one docker Postgres never consume each other's builds
+ * (Absurd partitions its engine tables BY QUEUE, same as the runs lanes).
+ */
+export function playDeployQueueName(): string {
+  const suffix = resolveAbsurdQueueSuffix();
+  return suffix ? `${PLAY_DEPLOY_QUEUE}_${suffix}` : PLAY_DEPLOY_QUEUE;
+}
+export const PLAY_DEPLOY_MAX_ATTEMPTS = 5;
+export const PLAY_DEPLOY_RETRY_STRATEGY: RetryStrategy = {
+  kind: 'exponential',
+  baseSeconds: 5,
+  maxSeconds: 60,
+};
+/**
+ * Retained deploy rows are bounded like the runs queue: a stuck or finished
+ * deploy task must not accumulate forever.
+ *
+ * NOTE: this queue previously set a per-org claim cap. Upstream Absurd
+ * removed account-level caps from queue policy (see the comment on the runs
+ * queue in absurd-shared.ts: parked work holds no slot and fair claim
+ * ordering picks the next runnable task). Build parallelism is now bounded
+ * by worker slots, not per-org policy. If per-org fairness is wanted again
+ * it has to be expressed at claim time, not here.
+ */
+export const PLAY_DEPLOY_CLEANUP_TTL = '6 hours';
+export const PLAY_DEPLOY_CLEANUP_LIMIT = 1_000;
+
+/**
+ * Task payload stays tiny (32 KiB control-plane cap): the deploy token plus
+ * addressing. The worker fetches everything else — repo, commit, project
+ * settings, a GitHub installation token — from the app API with the token,
+ * so state is always read fresh (supersession, disabled projects).
+ */
+export type PlayDeployTaskParams = {
+  deploymentId: string;
+  orgId: string;
+  playProjectId: string;
+  deployToken: string;
+  appBaseUrl: string;
+};
+
+const deployQueueReadyPromises = new Map<string, Promise<void>>();
+
+/**
+ * Ensure the deploys queue exists with its policy. Idempotent per process;
+ * a failure evicts the memo so the next caller retries loudly. Deliberately
+ * NOT ensureAbsurdRuntimeReady — that helper hardcodes the runs-lane
+ * derivation and its policies.
+ */
+export function ensurePlayDeployQueueReady(input?: {
+  url?: string | null;
+  schema?: string | null;
+}): Promise<void> {
+  const poolOptions = resolveAbsurdPoolOptions(input?.url);
+  const schema = input?.schema?.trim() ?? '';
+  const queueName = playDeployQueueName();
+  const key = `${poolOptions.url}::${schema}::${queueName}`;
+  const existing = deployQueueReadyPromises.get(key);
+  if (existing) return existing;
+  const promise = (async () => {
+    await withRuntimeSchedulerClient(poolOptions, (client) =>
+      ensureAbsurdSchema(client),
+    );
+    const absurd = createAbsurdRuntimeClient(poolOptions, queueName);
+    const existingQueues = new Set(await absurd.listQueues());
+    if (!existingQueues.has(queueName)) {
+      await absurd.createQueue(queueName);
+    }
+    await absurd.setQueuePolicy(queueName, {
+      cleanupTtl: PLAY_DEPLOY_CLEANUP_TTL,
+      cleanupLimit: PLAY_DEPLOY_CLEANUP_LIMIT,
+    });
+  })().catch((error) => {
+    deployQueueReadyPromises.delete(key);
+    throw error;
+  });
+  deployQueueReadyPromises.set(key, promise);
+  return promise;
+}
+
+/**
+ * Enqueue one deploy build task, idempotent on the deployment id. Called by
+ * the webhook receiver AFTER the Convex playDeployments row exists and
+ * BEFORE the delivery is marked processed — a crash in between lets
+ * GitHub's redelivery re-run both steps safely.
+ */
+export async function enqueuePlayDeployTask(
+  params: PlayDeployTaskParams,
+): Promise<void> {
+  const poolOptions = resolveAbsurdPoolOptions();
+  await ensurePlayDeployQueueReady();
+  const queueName = playDeployQueueName();
+  const absurd = createAbsurdRuntimeClient(poolOptions, queueName);
+  await absurd.spawn(PLAY_DEPLOY_TASK_NAME, params, {
+    queue: queueName,
+    idempotencyKey: `play-deploy:${params.deploymentId}`,
+    orgKey: params.orgId,
+    maxAttempts: PLAY_DEPLOY_MAX_ATTEMPTS,
+    retryStrategy: PLAY_DEPLOY_RETRY_STRATEGY,
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-run-task.ts

@@ -0,0 +1,68 @@
+import type { RetryStrategy } from '@deepline/absurd/sdk';
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+
+export const ABSURD_PLAY_RUN_TASK = 'play-run';
+
+export const ABSURD_PLAY_RUN_RETRY_STRATEGY: RetryStrategy = {
+  kind: 'exponential',
+  baseSeconds: RUNTIME_RELIABILITY_POLICY.engine.retryBaseSeconds,
+  factor: 2,
+  maxSeconds: RUNTIME_RELIABILITY_POLICY.engine.retryMaxSeconds,
+};
+
+export const ABSURD_PLAY_RUN_MAX_ATTEMPTS =
+  RUNTIME_RELIABILITY_POLICY.engine.maxAttempts;
+
+export type AbsurdPlayRunTaskParams = {
+  runId: string;
+  orgId: string;
+  /**
+   * Retained release lanes may still validate this field before loading the
+   * launch row. Every newly submitted task is a root.
+   */
+  nestingDepth: number;
+  schedulerSchema?: string | null;
+};
+
+/**
+ * The one task ABI for an ordinary Play Run.
+ *
+ * Callers choose only the release queue. Task identity, fairness, retries, and
+ * queue-delay policy stay inside the scheduler adapter.
+ */
+export function buildAbsurdPlayRunTask(input: {
+  launch: Pick<
+    PlaySchedulerSubmitInput,
+    'runId' | 'orgId' | 'queueMaxDelaySeconds' | 'queuePriority'
+  >;
+  schedulerSchema?: string | null;
+}) {
+  const params: AbsurdPlayRunTaskParams = {
+    runId: input.launch.runId,
+    orgId: input.launch.orgId,
+    nestingDepth: 0,
+    schedulerSchema: input.schedulerSchema ?? null,
+  };
+  const maxDelaySeconds = input.launch.queueMaxDelaySeconds;
+  const priority = input.launch.queuePriority;
+  return {
+    taskName: ABSURD_PLAY_RUN_TASK,
+    params,
+    options: {
+      idempotency_key: input.launch.runId,
+      org_key: input.launch.orgId,
+      max_attempts: ABSURD_PLAY_RUN_MAX_ATTEMPTS,
+      retry_strategy: {
+        kind: ABSURD_PLAY_RUN_RETRY_STRATEGY.kind,
+        base_seconds: ABSURD_PLAY_RUN_RETRY_STRATEGY.baseSeconds,
+        factor: ABSURD_PLAY_RUN_RETRY_STRATEGY.factor,
+        max_seconds: ABSURD_PLAY_RUN_RETRY_STRATEGY.maxSeconds,
+      },
+      ...(typeof priority === 'number' ? { priority } : {}),
+      ...(typeof maxDelaySeconds === 'number'
+        ? { cancellation: { max_delay: maxDelaySeconds } }
+        : {}),
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool.ts

@@ -0,0 +1,193 @@
+/**
+ * Src-coupled bridge between the vendored Absurd SDK (`packages/absurd`) and
+ * the hardened runtime scheduler Postgres pool (`./postgres-pool.ts`).
+ *
+ * workspace packages stay free of `src/**` imports, so the Absurd client takes an
+ * injected `Queryable`. This module supplies that handle by delegating every
+ * query through `withRuntimeSchedulerClient`, which is the process-singleton
+ * `pg.Pool` with saturation backpressure, a circuit breaker, and idempotent
+ * connection-error retries. Each Absurd SQL call is a self-contained plpgsql
+ * transaction, so a per-call client checkout is correct — Absurd never relies on
+ * cross-call session state.
+ */
+import { createRequire } from 'node:module';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import {
+  Absurd,
+  type AbsurdOptions,
+  type Queryable,
+  type QueryResult,
+} from '@deepline/absurd/sdk';
+import {
+  ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  ABSURD_SCHEMA_VERSION,
+} from '@deepline/absurd/pool';
+import {
+  withRuntimeSchedulerClient,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+
+export { ABSURD_SCHEMA_VERSION };
+
+/**
+ * Builds an SDK `Queryable` backed by the hardened runtime scheduler pool.
+ * Every query runs on a freshly checked-out pooled client and releases it,
+ * inheriting the pool's backpressure/breaker/retry guarantees.
+ */
+export function absurdQueryableForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+): Queryable {
+  return {
+    query: async <Row = Record<string, unknown>>(
+      sql: string,
+      params?: unknown[],
+    ): Promise<QueryResult<Row>> => {
+      return withRuntimeSchedulerClient(poolOptions, async (client) => {
+        const result = await client.query<Row & Record<string, unknown>>(
+          sql,
+          params,
+        );
+        return { rows: result.rows as Row[] };
+      });
+    },
+  };
+}
+
+/**
+ * Constructs an Absurd client wired to the runtime scheduler pool.
+ */
+export function createAbsurdForRuntimeScheduler(
+  poolOptions: RuntimeSchedulerPoolOptions,
+  options: Omit<AbsurdOptions, 'db'> = {},
+): Absurd {
+  return new Absurd({
+    ...options,
+    db: absurdQueryableForRuntimeScheduler(poolOptions),
+  });
+}
+
+let cachedSchemaSql: string | null = null;
+let cachedNoSchedulerOrgCapCompatibilitySql: string | null = null;
+
+/**
+ * The vendored schema version cannot represent Deepline-owned compatibility
+ * patches. Keep an explicit durable marker in the shared `absurd` schema so
+ * cold Vercel instances do not execute `CREATE OR REPLACE FUNCTION` on every
+ * first submission. Replacing a function while claims are running is DDL
+ * contention, not useful admission work.
+ */
+export const ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION =
+  'deepline-no-scheduler-org-cap-v1';
+
+function schemaSection(sql: string, start: string, end: string): string {
+  const startIndex = sql.indexOf(start);
+  const endIndex = sql.indexOf(end);
+  if (startIndex < 0 || endIndex < 0 || endIndex <= startIndex) {
+    throw new Error('Vendored absurd v5 schema upgrade markers are malformed.');
+  }
+  return sql.slice(startIndex + start.length, endIndex).trim();
+}
+
+/**
+ * Reads the vendored `absurd.sql` schema text. Cached after first read. Node
+ * only — the scheduler control plane and worker run under Node, never workerd.
+ */
+export function readAbsurdSchemaSql(): string {
+  if (cachedSchemaSql !== null) {
+    return cachedSchemaSql;
+  }
+  const packagePoolPath = createRequire(import.meta.url).resolve(
+    '@deepline/absurd/pool',
+  );
+  const schemaPath = path.resolve(
+    path.dirname(packagePoolPath),
+    ABSURD_SCHEMA_SQL_RELATIVE_PATH,
+  );
+  const sql = readFileSync(schemaPath, 'utf8');
+  if (!sql.includes('create schema if not exists absurd')) {
+    throw new Error(
+      `Vendored absurd schema at ${schemaPath} is missing or malformed.`,
+    );
+  }
+  cachedSchemaSql = sql;
+  return sql;
+}
+
+/**
+ * Applies no-cap behavior without changing the v4 schema contract.
+ *
+ * During a rolling release older workers still cold-start against v4 and may
+ * submit `per_org_concurrency`. Keeping the marker, column, function result
+ * shape, and accepted input key lets both versions run. The replacement
+ * claim function deliberately ignores the retired setting, so it cannot limit
+ * scheduler claims even while its compatibility-only column exists.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilitySql(): string {
+  if (cachedNoSchedulerOrgCapCompatibilitySql !== null) {
+    return cachedNoSchedulerOrgCapCompatibilitySql;
+  }
+  const sql = readAbsurdSchemaSql();
+  const claim = schemaSection(
+    sql,
+    '-- Workers call this to reserve a task from a given queue',
+    '-- Markes a run as completed',
+  );
+  const noCapClaim = claim
+    .replace(
+      `  -- DEEPLINE PATCH (a): per-org admission cap from queue policy (null=unlimited).\n  v_per_org_cap integer;\n`,
+      '',
+    )
+    .replace(
+      `\n  select per_org_concurrency into v_per_org_cap\n  from absurd.queues\n  where queue_name = p_queue_name;\n`,
+      '',
+    )
+    .replace(
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority + admission caps.\n`,
+      `  -- DEEPLINE PATCH (a): fair per-org claiming + priority.\n`,
+    )
+    .replace(
+      `  --   * admission — when the queue sets per_org_concurrency, an org may only\n  --                 admit (cap - inflight) head-of-line tasks; an org already at\n  --                 the cap is skipped entirely.\n  --\n`,
+      '',
+    )
+    .replace(
+      `     admissible as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n         where $5::integer is null\n            or org_rank <= ($5::integer - inflight_cnt)\n     ),\n     ranked as (\n        select run_id\n          from admissible\n`,
+      `     ranked as (\n        select run_id,\n               priority,\n               available_at,\n               created_at,\n               (org_rank + inflight_cnt) as fair_rank\n          from scored\n`,
+    )
+    .replace(
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until, v_per_org_cap;',
+      'return query execute v_sql using v_now, v_qty, v_worker_id, v_claim_until;',
+    );
+
+  if (
+    noCapClaim.includes('v_per_org_cap') ||
+    noCapClaim.includes('$5::integer')
+  ) {
+    throw new Error(
+      'Absurd no-cap compatibility patch did not remove the retired claim gate.',
+    );
+  }
+  // The installed v4 claim function has the identical signature and result
+  // type. Replacing only its body is safe for old and new processes alike.
+  cachedNoSchedulerOrgCapCompatibilitySql = noCapClaim.replace(
+    'create function absurd.claim_task',
+    'create or replace function absurd.claim_task',
+  );
+  return cachedNoSchedulerOrgCapCompatibilitySql;
+}
+
+/**
+ * Marks the no-cap compatibility patch after its replacement claim function
+ * has committed. A function keeps the marker in Absurd's global schema and
+ * makes it valid for every runtime scheduler namespace that shares it.
+ */
+export function readAbsurdSchemaNoSchedulerOrgCapCompatibilityMarkerSql(): string {
+  return `
+    CREATE OR REPLACE FUNCTION absurd.deepline_no_scheduler_org_cap_compatibility_version()
+    RETURNS text
+    LANGUAGE sql
+    AS $$
+      SELECT '${ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION}'::text
+    $$
+  `;
+}