deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts

@@ -0,0 +1,47 @@
+/**
+ * Keeps receipt-completion bursts from occupying every local database client.
+ * Heartbeats and initial runner readiness deliberately do not use this gate.
+ */
+export class GatewayHeavyRequestAdmission {
+  readonly #waiting: Array<() => void> = [];
+  #active = 0;
+
+  constructor(readonly maxActive: number) {
+    if (!Number.isInteger(maxActive) || maxActive < 1) {
+      throw new Error('Gateway heavy-request admission requires maxActive >= 1.');
+    }
+  }
+
+  async acquire(signal?: AbortSignal): Promise<() => void> {
+    if (signal?.aborted) throw new Error('Gateway request admission was cancelled.');
+    if (this.#active < this.maxActive) return this.#grant();
+    return await new Promise<() => void>((resolve, reject) => {
+      const grant = () => {
+        signal?.removeEventListener('abort', abort);
+        resolve(this.#grant());
+      };
+      const abort = () => {
+        const index = this.#waiting.indexOf(grant);
+        if (index >= 0) this.#waiting.splice(index, 1);
+        reject(new Error('Gateway request admission was cancelled.'));
+      };
+      signal?.addEventListener('abort', abort, { once: true });
+      this.#waiting.push(grant);
+    });
+  }
+
+  snapshot(): { active: number; waiting: number } {
+    return { active: this.#active, waiting: this.#waiting.length };
+  }
+
+  #grant(): () => void {
+    this.#active += 1;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      this.#active -= 1;
+      this.#waiting.shift()?.();
+    };
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts

@@ -0,0 +1,147 @@
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import type { PlayRunnerResult } from '../protocol';
+import {
+  createDirectWorkerProgressReporter,
+  type PostgresSchedulerProgressReporter,
+} from './postgres-progress';
+
+/**
+ * Stateless gateway fold for runner progress.
+ *
+ * HORIZONTAL-SCALING INVARIANT: every request creates a fresh reporter and is
+ * acknowledged only after its events reach the durable app-runtime projection.
+ * The runner supplies the absolute durable-log offset, so Fly may spread every
+ * consecutive batch and the terminal across different Machines without sticky
+ * routing, shared memory, or a replica-affine cursor. Attempt fencing belongs to
+ * the durable Play Run Ledger, never to a gateway process.
+ */
+
+export interface GatewayProgressRegistry {
+  ingest: (input: {
+    launch: PlaySchedulerSubmitInput;
+    runId: string;
+    attempt: number;
+    events: unknown[];
+    liveLogOffset: number;
+    isCurrentAttempt: () => Promise<boolean>;
+  }) => Promise<void>;
+  /** Flush + drop a run's reporter (e.g. on the run's terminal). */
+  finalize: (runId: string, attempt: number) => Promise<void>;
+  /**
+   * Finalize a pushed detached runner when the gateway, rather than a wake
+   * leg, writes the terminal. Creates a reporter when no progress batch was
+   * received so terminal step state is still projected.
+   */
+  finalizeTerminal: (input: {
+    launch: PlaySchedulerSubmitInput;
+    runId: string;
+    attempt: number;
+    result: PlayRunnerResult;
+    liveLogOffset: number;
+    isCurrentAttempt: () => Promise<boolean>;
+    replayTerminalLogs?: boolean;
+    events?: unknown[];
+  }) => Promise<void>;
+  size: () => number;
+}
+
+/**
+ * Project a runner's final observation tail before its terminal transition.
+ * Suspended and failed attempts need the same ordering as completed attempts:
+ * either terminal can wake/permit a successor whose attempt fence would make
+ * a late predecessor progress upload inert.
+ */
+export async function projectGatewayRunnerTerminalProgress(input: {
+  registry: GatewayProgressRegistry;
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  attempt: number;
+  result: PlayRunnerResult;
+  events: unknown[];
+  liveLogOffset: number;
+  isCurrentAttempt: () => Promise<boolean>;
+  replayTerminalLogs?: boolean;
+}): Promise<void> {
+  if (input.result.status === 'completed') {
+    // Fold the observation tail and terminal step state in one reporter. Two
+    // separate reporters require two fenced app uploads on the critical path.
+    // The reporter owns cursor advancement and acknowledges the whole fold
+    // before the caller can commit the terminal fact.
+    await input.registry.finalizeTerminal({
+      launch: input.launch,
+      runId: input.runId,
+      attempt: input.attempt,
+      result: input.result,
+      events: input.events,
+      liveLogOffset: input.liveLogOffset,
+      isCurrentAttempt: input.isCurrentAttempt,
+      replayTerminalLogs: input.replayTerminalLogs,
+    });
+  } else if (input.events.length > 0) {
+    await input.registry.ingest({
+      launch: input.launch,
+      runId: input.runId,
+      attempt: input.attempt,
+      events: input.events,
+      liveLogOffset: input.liveLogOffset,
+      isCurrentAttempt: input.isCurrentAttempt,
+    });
+  }
+}
+
+export function createGatewayProgressRegistry(options?: {
+  createReporter?: (
+    launch: PlaySchedulerSubmitInput,
+    attempt: number,
+    isCurrentAttempt: () => boolean | Promise<boolean>,
+    initialLiveLogTotalCount: number,
+  ) => PostgresSchedulerProgressReporter;
+}): GatewayProgressRegistry {
+  const createReporter =
+    options?.createReporter ??
+    ((launch, attempt, isCurrentAttempt, initialLiveLogTotalCount) =>
+      createDirectWorkerProgressReporter({
+        launch,
+        producerAttempt: attempt,
+        isCurrentAttempt,
+        initialLiveLogTotalCount,
+      }));
+
+  return {
+    async ingest({ launch, attempt, events, liveLogOffset, isCurrentAttempt }) {
+      const reporter = createReporter(
+        launch,
+        attempt,
+        isCurrentAttempt,
+        liveLogOffset,
+      );
+      for (const event of events) {
+        reporter.onRunnerEvent(event);
+      }
+      await reporter.finalize(null, {
+        status: 'running',
+        required: true,
+      });
+    },
+    async finalize() {},
+    async finalizeTerminal(input) {
+      const reporter = createReporter(
+        input.launch,
+        input.attempt,
+        input.isCurrentAttempt,
+        input.liveLogOffset,
+      );
+      for (const event of input.events ?? []) reporter.onRunnerEvent(event);
+      await reporter.finalize(input.result, {
+        status: 'running',
+        required: true,
+        skipTerminalLogReplay: input.replayTerminalLogs !== true,
+        replayAllTerminalLogs: input.replayTerminalLogs === true,
+        terminalLogProducerAttempt: input.attempt,
+      });
+    },
+    size() {
+      return 0;
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/index.ts

@@ -0,0 +1,27 @@
+/**
+ * Resolver for Play Scheduler backends — the Adapters that own play workflow
+ * lifecycle (submit, attach, observe, signal, cancel, result).
+ *
+ * Absurd is the only durable scheduler.
+ */
+
+import {
+  normalizePlaySchedulerBackend,
+  PLAY_SCHEDULER_BACKENDS,
+  type PlaySchedulerBackend,
+  type PlaySchedulerBackendId,
+} from '../scheduler-backend';
+import { absurdSchedulerBackend } from './absurd';
+
+export function resolveSchedulerBackend(
+  backendId?: PlaySchedulerBackendId | string | null,
+): PlaySchedulerBackend {
+  const backend = normalizePlaySchedulerBackend(backendId);
+  if (backend === PLAY_SCHEDULER_BACKENDS.absurd) {
+    return absurdSchedulerBackend;
+  }
+  throw new Error(`Unsupported scheduler backend: ${backend}`);
+}
+
+export { absurdSchedulerBackend };
+export type { PlaySchedulerBackend };

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/lazy.ts

@@ -0,0 +1,50 @@
+import {
+  normalizePlaySchedulerBackend,
+  PLAY_SCHEDULER_BACKENDS,
+  type PlaySchedulerBackend,
+  type PlaySchedulerBackendId,
+} from '../scheduler-backend';
+import { resolveAbsurdQueueSuffixForSchema } from '../scheduler-backends/absurd-shared';
+
+export type SchedulerBackendResolveOptions = {
+  postgresUrl?: string | null;
+  postgresSchema?: string | null;
+  resolveQueueSuffixForSchema?: (schema: string) => string;
+};
+
+export async function resolveSchedulerBackendLazy(
+  backendId?: PlaySchedulerBackendId | string | null,
+  options: SchedulerBackendResolveOptions = {},
+): Promise<PlaySchedulerBackend> {
+  const backend = normalizePlaySchedulerBackend(backendId);
+  if (backend === PLAY_SCHEDULER_BACKENDS.absurd) {
+    const { createAbsurdSchedulerBackend, absurdSchedulerBackend } =
+      await import('./absurd');
+    const schema = options.postgresSchema?.trim();
+    if (schema) {
+      // Resolve the queue suffix HERE (the request path) rather than deriving it
+      // inside the backend: the shared hosted app has no
+      // DEEPLINE_ABSURD_QUEUE_SUFFIX env, so it must derive the SAME suffix the
+      // Fly worker was provisioned with from the per-request schema, to enqueue
+      // onto the queues that worker actually drains. For a preview namespace
+      // that identity includes the sandbox placement policy, so this must go
+      // through resolveAbsurdQueueSuffixForRuntimeSchema and not hash the bare
+      // schema. Unit tests construct the backend directly (bypassing this path)
+      // and stay on base queues.
+      const resolveQueueSuffixForSchema =
+        options.resolveQueueSuffixForSchema ?? resolveAbsurdQueueSuffixForSchema;
+      return createAbsurdSchedulerBackend({
+        url: options.postgresUrl?.trim() || null,
+        schema,
+        queueSuffix: resolveQueueSuffixForSchema(schema),
+      });
+    }
+    if (options.postgresUrl?.trim()) {
+      return createAbsurdSchedulerBackend({
+        url: options.postgresUrl.trim(),
+      });
+    }
+    return absurdSchedulerBackend;
+  }
+  throw new Error(`Unsupported scheduler backend: ${backend}`);
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/play-run-recovery.ts

@@ -0,0 +1,98 @@
+import {
+  isPlayRunRecoveryError,
+  PLAY_RUN_RECOVERY_DEFER_POLICY,
+  recoveryDeferDelaySeconds,
+} from '../play-run-recovery-policy';
+import type { PlayCheckpoint } from '../ctx-types';
+import type { PlayRunnerResult } from '../protocol';
+import {
+  deferPostgresSchedulerAttemptForRecovery,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerQueryClient,
+} from './postgres';
+
+/**
+ * Scheduler-owned transition for failures proved to have happened before
+ * customer code. Provider Adapters supply typed evidence; this Module owns the
+ * durable defer budget, dependency circuit decision, and exhausted outcome.
+ */
+export type PlayRunRecoveryOutcome =
+  | { kind: 'not_recoverable' }
+  | {
+      kind: 'deferred';
+      dependency: 'modal' | 'runtime';
+      reason: string;
+      deferCount: number;
+      deadlineAtMs: number | null;
+      delaySeconds: number;
+    }
+  | { kind: 'exhausted'; terminalResult: PlayRunnerResult };
+
+export async function transitionPreCodePlayRunRecovery(input: {
+  error: unknown;
+  runId: string;
+  attempt: number;
+  leaseToken: string;
+  checkpoint: PlayCheckpoint | null;
+  client: PostgresSchedulerQueryClient;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  nowMs?: number;
+}): Promise<PlayRunRecoveryOutcome> {
+  if (
+    !isPlayRunRecoveryError(input.error) ||
+    input.error.decision.stage !== 'pre_code' ||
+    input.error.decision.action !== 'defer'
+  ) {
+    return { kind: 'not_recoverable' };
+  }
+  const decision = input.error.decision;
+  const recovery = await deferPostgresSchedulerAttemptForRecovery(
+    input.client,
+    {
+      runId: input.runId,
+      attempt: input.attempt,
+      leaseToken: input.leaseToken,
+      reason: decision.reason,
+      dependency: decision.dependency,
+      // Runtime-wide operator holds/throttles are already durable policy.
+      // Only Modal-observed outages own the provider circuit.
+      openCircuit:
+        decision.dependency === 'modal' &&
+        decision.reason !== 'modal_circuit_open_before_execution',
+      nowMs: input.nowMs ?? Date.now(),
+      maxDefers: PLAY_RUN_RECOVERY_DEFER_POLICY.maxDefers,
+      horizonMs: PLAY_RUN_RECOVERY_DEFER_POLICY.horizonMs,
+      deferBaseMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferBaseSeconds * 1_000,
+      deferMaxMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferMaxSeconds * 1_000,
+    },
+    input.schedulerOptions,
+  );
+  if (recovery.deferred && recovery.deferCount !== null) {
+    return {
+      kind: 'deferred',
+      dependency: decision.dependency,
+      reason: decision.reason,
+      deferCount: recovery.deferCount,
+      deadlineAtMs: recovery.deadlineAtMs,
+      delaySeconds: recoveryDeferDelaySeconds(recovery.deferCount),
+    };
+  }
+  if (recovery.exhausted) {
+    return {
+      kind: 'exhausted',
+      terminalResult: {
+        status: 'failed',
+        error:
+          `PLATFORM_RECOVERY_EXHAUSTED: ${decision.dependency} ` +
+          'remained unavailable before customer execution began through the bounded recovery horizon. ' +
+          'No customer code was started; inspect and redrive this run after the dependency is healthy.',
+        logs: [],
+        stats: {},
+        steps: [],
+        checkpoint: input.checkpoint,
+        tableNamespace: null,
+      },
+    };
+  }
+  return { kind: 'not_recoverable' };
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts

@@ -0,0 +1,260 @@
+import { BudgetStateLimitError } from '../governor/budget-state-backend';
+import type {
+  PlayRunnerBudgetChargeInput,
+  PlayRunnerBudgetChargeResult,
+} from '../protocol';
+import type { PostgresSchedulerQueryClient } from './postgres';
+import { stringifyPostgresJson } from '../postgres-json';
+
+const MAX_BUDGET_CHARGES_PER_REQUEST = 16;
+
+function quoteIdent(value: string): string {
+  return `"${value.replace(/"/g, '""')}"`;
+}
+
+function tableName(input: { schedulerSchema?: string | null }): string {
+  const schema = input.schedulerSchema?.trim() || 'runtime_scheduler';
+  return `${quoteIdent(schema)}.${quoteIdent('governor_budget_state')}`;
+}
+
+function reservationTableName(input: {
+  schedulerSchema?: string | null;
+}): string {
+  const schema = input.schedulerSchema?.trim() || 'runtime_scheduler';
+  return `${quoteIdent(schema)}.${quoteIdent('governor_budget_reservations')}`;
+}
+
+export async function ensurePostgresBudgetStateSchema(
+  client: PostgresSchedulerQueryClient,
+  input: { schedulerSchema?: string | null } = {},
+): Promise<void> {
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${tableName(input)} (
+      org_id text NOT NULL,
+      root_run_id text NOT NULL,
+      counters_json jsonb NOT NULL DEFAULT '{}'::jsonb,
+      updated_at timestamptz NOT NULL DEFAULT now(),
+      PRIMARY KEY (org_id, root_run_id)
+    )
+  `);
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${reservationTableName(input)} (
+      org_id text NOT NULL,
+      root_run_id text NOT NULL,
+      reservation_id text NOT NULL,
+      charges_json jsonb NOT NULL,
+      counters_json jsonb,
+      created_at timestamptz NOT NULL DEFAULT now(),
+      PRIMARY KEY (org_id, root_run_id, reservation_id)
+    )
+  `);
+}
+
+function normalizedCharges(input: PlayRunnerBudgetChargeInput) {
+  if (input.charges.length > MAX_BUDGET_CHARGES_PER_REQUEST) {
+    throw new Error(
+      `Governor budget accepts at most ${MAX_BUDGET_CHARGES_PER_REQUEST} charges per request.`,
+    );
+  }
+  const merged = new Map<string, { amount: number; limit: number }>();
+  for (const charge of input.charges) {
+    const key = charge.key.trim();
+    if (!key || key.length > 256) {
+      throw new Error('Governor budget key must be 1-256 characters.');
+    }
+    if (!Number.isSafeInteger(charge.amount) || charge.amount <= 0) {
+      throw new Error(
+        `Governor budget ${key} amount must be a positive integer.`,
+      );
+    }
+    if (!Number.isSafeInteger(charge.limit) || charge.limit <= 0) {
+      throw new Error(
+        `Governor budget ${key} limit must be a positive integer.`,
+      );
+    }
+    const current = merged.get(key);
+    merged.set(key, {
+      amount: (current?.amount ?? 0) + charge.amount,
+      limit: Math.min(current?.limit ?? charge.limit, charge.limit),
+    });
+  }
+  return [...merged.entries()].map(([key, value]) => ({ key, ...value }));
+}
+
+function sameCharges(
+  left: Array<{ key: string; amount: number; limit: number }>,
+  right: Array<{ key: string; amount: number; limit: number }>,
+): boolean {
+  return (
+    left.length === right.length &&
+    left.every((charge, index) => {
+      const other = right[index];
+      return (
+        other?.key === charge.key &&
+        Number(other.amount) === charge.amount &&
+        Number(other.limit) === charge.limit
+      );
+    })
+  );
+}
+
+/** Atomically reserve one or more root-run counters. One row serializes sibling
+ * sandboxes, and a multi-counter child reservation either commits all or none. */
+export async function chargePostgresBudgetState(
+  client: PostgresSchedulerQueryClient,
+  input: PlayRunnerBudgetChargeInput & { orgId: string },
+): Promise<PlayRunnerBudgetChargeResult> {
+  const charges = normalizedCharges(input);
+  if (charges.length === 0) return { counters: {} };
+  const reservationId = input.reservationId.trim();
+  if (!reservationId || reservationId.length > 256) {
+    throw new Error('Governor budget reservationId must be 1-256 characters.');
+  }
+  for (const charge of charges) {
+    if (charge.amount > charge.limit) {
+      throw new BudgetStateLimitError(charge.key, charge.amount, charge.limit);
+    }
+  }
+  await client.query('BEGIN');
+  try {
+    const insertedReservation = await client.query<{
+      reservation_id: string;
+    }>(
+      `
+        INSERT INTO ${reservationTableName(input)} (
+          org_id,
+          root_run_id,
+          reservation_id,
+          charges_json
+        ) VALUES ($1, $2, $3, $4::jsonb)
+        ON CONFLICT (org_id, root_run_id, reservation_id) DO NOTHING
+        RETURNING reservation_id
+      `,
+      [
+        input.orgId,
+        input.rootRunId,
+        reservationId,
+        stringifyPostgresJson(charges),
+      ],
+    );
+    if (!insertedReservation.rows[0]) {
+      const existing = await client.query<{
+        charges_json: typeof charges;
+        counters_json: Record<string, number> | null;
+      }>(
+        `
+          SELECT charges_json, counters_json
+          FROM ${reservationTableName(input)}
+          WHERE org_id = $1 AND root_run_id = $2 AND reservation_id = $3
+          FOR UPDATE
+        `,
+        [input.orgId, input.rootRunId, reservationId],
+      );
+      const reservation = existing.rows[0];
+      if (!reservation) {
+        throw new Error('Governor budget reservation disappeared.');
+      }
+      if (!sameCharges(reservation.charges_json, charges)) {
+        throw new Error(
+          'Governor budget reservationId was reused with different charges.',
+        );
+      }
+      if (!reservation.counters_json) {
+        throw new Error('Governor budget reservation is incomplete.');
+      }
+      await client.query('COMMIT');
+      return { counters: reservation.counters_json };
+    }
+
+    const charged = await client.query<{
+      counters_json: Record<string, number>;
+    }>(
+      `
+      WITH charges AS (
+        SELECT *
+        FROM jsonb_to_recordset($3::jsonb)
+          AS charge(key text, amount bigint, "limit" bigint)
+      )
+      INSERT INTO ${tableName(input)} AS budget (
+        org_id,
+        root_run_id,
+        counters_json,
+        updated_at
+      )
+      SELECT $1,
+             $2,
+             jsonb_object_agg(charges.key, to_jsonb(charges.amount)),
+             now()
+      FROM charges
+      ON CONFLICT (org_id, root_run_id) DO UPDATE
+      SET counters_json = budget.counters_json || (
+            SELECT jsonb_object_agg(
+              charges.key,
+              to_jsonb(
+                COALESCE((budget.counters_json ->> charges.key)::bigint, 0) +
+                charges.amount
+              )
+            )
+            FROM charges
+          ),
+          updated_at = now()
+      WHERE NOT EXISTS (
+        SELECT 1
+        FROM charges
+        WHERE COALESCE((budget.counters_json ->> charges.key)::bigint, 0) +
+              charges.amount > charges."limit"
+      )
+      RETURNING counters_json
+    `,
+      [input.orgId, input.rootRunId, stringifyPostgresJson(charges)],
+    );
+    const counters = charged.rows[0]?.counters_json;
+    if (counters) {
+      await client.query(
+        `
+          UPDATE ${reservationTableName(input)}
+          SET counters_json = $4::jsonb
+          WHERE org_id = $1 AND root_run_id = $2 AND reservation_id = $3
+        `,
+        [
+          input.orgId,
+          input.rootRunId,
+          reservationId,
+          stringifyPostgresJson(counters),
+        ],
+      );
+      await client.query('COMMIT');
+      return { counters };
+    }
+
+    const current = await client.query<{
+      counters_json: Record<string, number>;
+    }>(
+      `
+      SELECT counters_json
+      FROM ${tableName(input)}
+      WHERE org_id = $1 AND root_run_id = $2
+    `,
+      [input.orgId, input.rootRunId],
+    );
+    const observedCounters = current.rows[0]?.counters_json ?? {};
+    const violation = charges.find(
+      (charge) =>
+        Number(observedCounters[charge.key] ?? 0) + charge.amount >
+        charge.limit,
+    );
+    if (!violation) {
+      throw new Error(
+        'Governor budget reservation failed without a limit violation.',
+      );
+    }
+    throw new BudgetStateLimitError(
+      violation.key,
+      Number(observedCounters[violation.key] ?? 0) + violation.amount,
+      violation.limit,
+    );
+  } catch (error) {
+    await client.query('ROLLBACK').catch(() => {});
+    throw error;
+  }
+}