deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-operation.ts

@@ -0,0 +1,207 @@
+/**
+ * The one durable lifecycle for a physical managed sandbox.
+ *
+ * Keep these values aligned with `runtime_sandbox_operation_state` in the
+ * scheduler schema.  This is deliberately an enum, rather than an open
+ * string union: adding a lifecycle state must update every transition below.
+ */
+export enum RuntimeSandboxOperationState {
+  Reserved = 'reserved',
+  Live = 'live',
+  Deleting = 'deleting',
+  Deleted = 'deleted',
+  Abandoned = 'abandoned',
+}
+
+export enum RuntimeSandboxProvider {
+  Daytona = 'daytona',
+  Modal = 'modal',
+}
+
+export type RuntimeSandboxOperationReserved = {
+  state: RuntimeSandboxOperationState.Reserved;
+  operationKey: string;
+  provider: RuntimeSandboxProvider;
+  runId: string;
+  attempt: number;
+  orgId: string;
+  sandboxId: null;
+  routingDomain: null;
+  deletedAtMs: null;
+};
+
+export type RuntimeSandboxOperationLive = {
+  state: RuntimeSandboxOperationState.Live;
+  operationKey: string;
+  provider: RuntimeSandboxProvider;
+  runId: string;
+  attempt: number;
+  orgId: string;
+  sandboxId: string;
+  routingDomain: string | null;
+  deletedAtMs: null;
+};
+
+export type RuntimeSandboxOperationDeleting = {
+  state: RuntimeSandboxOperationState.Deleting;
+  operationKey: string;
+  provider: RuntimeSandboxProvider;
+  runId: string;
+  attempt: number;
+  orgId: string;
+  sandboxId: string;
+  routingDomain: string | null;
+  deletedAtMs: null;
+};
+
+export type RuntimeSandboxOperationDeleted = {
+  state: RuntimeSandboxOperationState.Deleted;
+  operationKey: string;
+  provider: RuntimeSandboxProvider;
+  runId: string;
+  attempt: number;
+  orgId: string;
+  sandboxId: string;
+  routingDomain: string | null;
+  deletedAtMs: number;
+};
+
+/** Confirmed no-resource outcome. It cannot be promoted to a live sandbox. */
+export type RuntimeSandboxOperationAbandoned = {
+  state: RuntimeSandboxOperationState.Abandoned;
+  operationKey: string;
+  provider: RuntimeSandboxProvider;
+  runId: string;
+  attempt: number;
+  orgId: string;
+  sandboxId: null;
+  routingDomain: null;
+  deletedAtMs: null;
+};
+
+export type RuntimeSandboxOperation =
+  | RuntimeSandboxOperationReserved
+  | RuntimeSandboxOperationLive
+  | RuntimeSandboxOperationDeleting
+  | RuntimeSandboxOperationDeleted
+  | RuntimeSandboxOperationAbandoned;
+
+/** Every legal transition is explicit and closed over the enum. */
+export const RUNTIME_SANDBOX_OPERATION_TRANSITIONS: Readonly<
+  Record<RuntimeSandboxOperationState, readonly RuntimeSandboxOperationState[]>
+> = {
+  [RuntimeSandboxOperationState.Reserved]: [
+    RuntimeSandboxOperationState.Live,
+    RuntimeSandboxOperationState.Deleting,
+    RuntimeSandboxOperationState.Abandoned,
+  ],
+  [RuntimeSandboxOperationState.Live]: [RuntimeSandboxOperationState.Deleting],
+  [RuntimeSandboxOperationState.Deleting]: [
+    RuntimeSandboxOperationState.Deleted,
+  ],
+  [RuntimeSandboxOperationState.Deleted]: [],
+  [RuntimeSandboxOperationState.Abandoned]: [],
+};
+
+export function canTransitionRuntimeSandboxOperation(
+  from: RuntimeSandboxOperationState,
+  to: RuntimeSandboxOperationState,
+): boolean {
+  return RUNTIME_SANDBOX_OPERATION_TRANSITIONS[from].includes(to);
+}
+
+type RuntimeSandboxOperationDatabaseRow = {
+  operation_key: unknown;
+  provider: unknown;
+  run_id: unknown;
+  attempt: unknown;
+  org_id: unknown;
+  sandbox_id: unknown;
+  routing_domain: unknown;
+  state: unknown;
+  delete_confirmed_at_ms: unknown;
+};
+
+function requiredText(value: unknown, name: string): string {
+  if (typeof value !== 'string' || !value.trim()) {
+    throw new Error(`Invalid sandbox operation ${name}.`);
+  }
+  return value;
+}
+
+/** The sole boundary that converts database text into the closed TS FSM. */
+export function decodeRuntimeSandboxOperation(
+  row: RuntimeSandboxOperationDatabaseRow,
+): RuntimeSandboxOperation {
+  const provider = requiredText(row.provider, 'provider');
+  if (
+    provider !== RuntimeSandboxProvider.Daytona &&
+    provider !== RuntimeSandboxProvider.Modal
+  ) {
+    throw new Error(`Unknown sandbox operation provider: ${provider}.`);
+  }
+  const state = requiredText(row.state, 'state');
+  const base = {
+    operationKey: requiredText(row.operation_key, 'operationKey'),
+    provider,
+    runId: requiredText(row.run_id, 'runId'),
+    attempt: Number(row.attempt),
+    orgId: requiredText(row.org_id, 'orgId'),
+  } as const;
+  if (!Number.isSafeInteger(base.attempt) || base.attempt < 1) {
+    throw new Error('Invalid sandbox operation attempt.');
+  }
+  const sandboxId =
+    row.sandbox_id === null ? null : requiredText(row.sandbox_id, 'sandboxId');
+  const routingDomain =
+    row.routing_domain === null
+      ? null
+      : requiredText(row.routing_domain, 'routingDomain');
+  const deletedAtMs =
+    row.delete_confirmed_at_ms === null
+      ? null
+      : Number(row.delete_confirmed_at_ms);
+  switch (state) {
+    case RuntimeSandboxOperationState.Reserved:
+    case RuntimeSandboxOperationState.Abandoned:
+      if (
+        sandboxId !== null ||
+        routingDomain !== null ||
+        deletedAtMs !== null
+      ) {
+        throw new Error(`Invalid ${state} sandbox operation shape.`);
+      }
+      return {
+        ...base,
+        state,
+        sandboxId: null,
+        routingDomain: null,
+        deletedAtMs: null,
+      };
+    case RuntimeSandboxOperationState.Live:
+    case RuntimeSandboxOperationState.Deleting:
+      if (sandboxId === null || deletedAtMs !== null) {
+        throw new Error(`Invalid ${state} sandbox operation shape.`);
+      }
+      return { ...base, state, sandboxId, routingDomain, deletedAtMs: null };
+    case RuntimeSandboxOperationState.Deleted:
+      if (
+        sandboxId === null ||
+        deletedAtMs === null ||
+        !Number.isFinite(deletedAtMs)
+      ) {
+        throw new Error('Invalid deleted sandbox operation shape.');
+      }
+      return { ...base, state, sandboxId, routingDomain, deletedAtMs };
+    default:
+      throw new Error(`Unknown sandbox operation state: ${state}.`);
+  }
+}
+
+export function runtimeSandboxProviderForResourceKind(
+  kind: 'daytona_sandbox' | 'modal_sandbox',
+): RuntimeSandboxProvider {
+  return kind === 'daytona_sandbox'
+    ? RuntimeSandboxProvider.Daytona
+    : RuntimeSandboxProvider.Modal;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-scheduler-schema.ts

@@ -0,0 +1,10 @@
+export const LOCAL_RUNTIME_SCHEDULER_SCHEMA_ENV =
+  'DEEPLINE_LOCAL_RUNTIME_SCHEDULER_SCHEMA';
+export const RUNTIME_SCHEDULER_SCHEMA_ENV = 'DEEPLINE_RUNTIME_SCHEDULER_SCHEMA';
+
+export function resolveRuntimeSchedulerSchemaEnv(): string | null {
+  const localSchema = process.env[LOCAL_RUNTIME_SCHEDULER_SCHEMA_ENV]?.trim();
+  if (localSchema) return localSchema;
+  const schema = process.env[RUNTIME_SCHEDULER_SCHEMA_ENV]?.trim();
+  return schema || null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts

@@ -0,0 +1,123 @@
+export class RuntimeSchedulerSaturatedError extends Error {
+  readonly queueDepth: number;
+
+  constructor(queueDepth: number) {
+    super(
+      `Runtime scheduler DB pool is saturated (${queueDepth} pending acquires). ` +
+        'Backpressure: shed or retry with backoff instead of queueing more work.',
+    );
+    this.name = 'RuntimeSchedulerSaturatedError';
+    this.queueDepth = queueDepth;
+  }
+}
+
+/** Scheduler admission is temporarily full, not broken. */
+export class RuntimeSchedulerCapacityError extends Error {
+  readonly retryAfterMs: number;
+
+  constructor(message: string, retryAfterMs: number) {
+    super(message);
+    this.name = 'RuntimeSchedulerCapacityError';
+    this.retryAfterMs = retryAfterMs;
+  }
+}
+
+export class RuntimeSchedulerPgBouncerCapacityError extends RuntimeSchedulerCapacityError {
+  constructor(cause: string) {
+    super(
+      `Runtime scheduler PgBouncer is at connection capacity: ${cause}`,
+      5_000,
+    );
+    this.name = 'RuntimeSchedulerPgBouncerCapacityError';
+  }
+}
+
+/**
+ * The app deliberately keeps bulk launch transactions off the last control
+ * plane connection. This is a bounded, process-local admission limit, not a
+ * durable scheduler queue: a caller has not been admitted until its existing
+ * atomic launch transaction commits.
+ */
+export class RuntimeSchedulerSubmissionAdmissionError extends RuntimeSchedulerCapacityError {
+  readonly reason: 'full' | 'timed_out';
+
+  constructor(reason: 'full' | 'timed_out', queueDepth: number) {
+    super(
+      `Runtime scheduler submission admission ${reason} (${queueDepth} queued launches).`,
+      1_000,
+    );
+    this.name = 'RuntimeSchedulerSubmissionAdmissionError';
+    this.reason = reason;
+  }
+}
+
+export function isRuntimeSchedulerCapacityError(
+  error: unknown,
+): error is RuntimeSchedulerCapacityError | RuntimeSchedulerSaturatedError {
+  if (
+    error instanceof RuntimeSchedulerCapacityError ||
+    error instanceof RuntimeSchedulerSaturatedError
+  ) {
+    return true;
+  }
+  if (!error || typeof error !== 'object') return false;
+  const candidate = error as { name?: unknown; message?: unknown };
+  if (typeof candidate.name !== 'string') return false;
+  if (typeof candidate.message !== 'string') return false;
+  return (
+    (candidate.name === 'RuntimeSchedulerPgBouncerCapacityError' &&
+      candidate.message.includes(
+        'Runtime scheduler PgBouncer is at connection capacity:',
+      )) ||
+    (candidate.name === 'RuntimeSchedulerCapacityError' &&
+      candidate.message.startsWith('Runtime scheduler ')) ||
+    (candidate.name === 'RuntimeSchedulerSubmissionAdmissionError' &&
+      candidate.message.startsWith(
+        'Runtime scheduler submission admission ',
+      )) ||
+    (candidate.name === 'RuntimeSchedulerSaturatedError' &&
+      candidate.message.startsWith('Runtime scheduler DB pool is saturated ('))
+  );
+}
+
+/** Restore the typed capacity signal after a text-only protocol boundary. */
+export function parseRuntimeSchedulerCapacityError(
+  message: string,
+): RuntimeSchedulerCapacityError | RuntimeSchedulerSaturatedError | null {
+  const pgBouncerPrefix =
+    'Runtime scheduler PgBouncer is at connection capacity: ';
+  const pgBouncerIndex = message.indexOf(pgBouncerPrefix);
+  if (pgBouncerIndex >= 0) {
+    return new RuntimeSchedulerPgBouncerCapacityError(
+      message
+        .slice(pgBouncerIndex + pgBouncerPrefix.length)
+        .split(/\r?\n/, 1)[0]!,
+    );
+  }
+
+  const saturated = message.match(
+    /Runtime scheduler DB pool is saturated \((\d+) pending acquires\)\./,
+  );
+  if (saturated)
+    return new RuntimeSchedulerSaturatedError(Number(saturated[1]));
+
+  const admission = message.match(
+    /Runtime scheduler submission admission (full|timed_out) \((\d+) queued launches\)\./,
+  );
+  return admission
+    ? new RuntimeSchedulerSubmissionAdmissionError(
+        admission[1] as 'full' | 'timed_out',
+        Number(admission[2]),
+      )
+    : null;
+}
+
+export function runtimeSchedulerCapacityRetryAfterMs(
+  error: RuntimeSchedulerCapacityError | RuntimeSchedulerSaturatedError,
+): number {
+  const retryAfterMs = (error as { retryAfterMs?: unknown }).retryAfterMs;
+  if (typeof retryAfterMs === 'number' && Number.isFinite(retryAfterMs)) {
+    return Math.max(0, retryAfterMs);
+  }
+  return error.name === 'RuntimeSchedulerSaturatedError' ? 1_000 : 5_000;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-launch-read-observability.ts

@@ -0,0 +1,78 @@
+/**
+ * In-process telemetry for immutable launch lookups. Keep labels bounded: the
+ * callers are an explicit union and releases are immutable deploy lanes. Never
+ * put run ids, play names, users, or organizations in these metrics.
+ */
+export type SchedulerLaunchReadCaller =
+  | 'play_run_control'
+  | 'scheduler_submit_recovery'
+  | 'runtime_worker_execute'
+  | 'runtime_worker_terminal_recovery'
+  | 'direct_terminal_billing'
+  | 'runtime_worker_outbox_projection'
+  | 'receipt_gateway_auth'
+  | 'receipt_gateway_lazy'
+  | 'test';
+
+export type SchedulerLaunchReadOutcome = 'found' | 'missing' | 'error';
+
+type Counter = {
+  caller: SchedulerLaunchReadCaller;
+  releaseId: string;
+  found: number;
+  missing: number;
+  errors: number;
+  total: number;
+  durationMsTotal: number;
+};
+
+const counters = new Map<string, Counter>();
+
+function counterFor(input: {
+  caller: SchedulerLaunchReadCaller;
+  releaseId: string | null;
+}): Counter {
+  const releaseId = input.releaseId?.trim() || 'unassigned';
+  const key = `${input.caller}:${releaseId}`;
+  const existing = counters.get(key);
+  if (existing) return existing;
+  const counter: Counter = {
+    caller: input.caller,
+    releaseId,
+    found: 0,
+    missing: 0,
+    errors: 0,
+    total: 0,
+    durationMsTotal: 0,
+  };
+  counters.set(key, counter);
+  return counter;
+}
+
+export function recordSchedulerLaunchRead(input: {
+  caller: SchedulerLaunchReadCaller;
+  releaseId: string | null;
+  durationMs: number;
+  outcome: SchedulerLaunchReadOutcome;
+}): void {
+  const counter = counterFor(input);
+  counter.total += 1;
+  counter.durationMsTotal += Math.max(0, input.durationMs);
+  if (input.outcome === 'found') counter.found += 1;
+  if (input.outcome === 'missing') counter.missing += 1;
+  if (input.outcome === 'error') counter.errors += 1;
+}
+
+export function schedulerLaunchReadMetricsSnapshot(): Counter[] {
+  return [...counters.values()]
+    .map((counter) => ({ ...counter }))
+    .sort(
+      (left, right) =>
+        left.caller.localeCompare(right.caller) ||
+        left.releaseId.localeCompare(right.releaseId),
+    );
+}
+
+export function resetSchedulerLaunchReadMetricsForTest(): void {
+  counters.clear();
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/submit-admission.ts

@@ -0,0 +1,169 @@
+/**
+ * Process-local admission before the atomic Postgres launch transaction.
+ *
+ * A Vercel app process uses a deliberately tiny two-client scheduler pool.
+ * Bulk submissions used to consume both clients while executor-token lease
+ * reads needed one, so a burst turned a retryable token lookup into a failed
+ * engine leg. One bulk submit at a time keeps a control-plane client free.
+ *
+ * This is intentionally not durable state and does not acknowledge a launch.
+ * The existing submit transaction remains the only admission boundary. If the
+ * process dies, its ordinary launch-record recovery path remains authoritative.
+ *
+ * Do not wire an HTTP abort signal into this queue. At this point the caller
+ * has already persisted its launch record; silently dropping it on disconnect
+ * would make a durable start look abandoned. Explicit run cancellation and
+ * durable recovery own that decision instead.
+ */
+import { RuntimeSchedulerSubmissionAdmissionError } from './scheduler-capacity';
+
+export const RUNTIME_SCHEDULER_SUBMIT_ADMISSION = {
+  maxActive: 1,
+  maxQueued: 128,
+  maxWaitMs: 60_000,
+} as const;
+
+type Waiter = {
+  enqueuedAt: number;
+  resolve: (permit: RuntimeSchedulerSubmitAdmissionPermit) => void;
+  reject: (error: Error) => void;
+  timeout: ReturnType<typeof setTimeout>;
+};
+
+type AdmissionEntry = {
+  active: number;
+  waiters: Waiter[];
+};
+
+type AdmissionRegistry = Map<string, AdmissionEntry>;
+
+const REGISTRY_SYMBOL = Symbol.for(
+  'deepline.runtimeSchedulerSubmitAdmission.registry',
+);
+
+function registry(): AdmissionRegistry {
+  const target = globalThis as typeof globalThis & Record<symbol, unknown>;
+  const existing = target[REGISTRY_SYMBOL];
+  if (existing instanceof Map) return existing as AdmissionRegistry;
+  const created: AdmissionRegistry = new Map<string, AdmissionEntry>();
+  target[REGISTRY_SYMBOL] = created;
+  return created;
+}
+
+export type RuntimeSchedulerSubmitAdmissionParams = {
+  /** Kept in-memory only. Do not log this value because it contains DB auth. */
+  schedulerUrl: string;
+  schedulerSchema: string | null | undefined;
+};
+
+export type RuntimeSchedulerSubmitAdmissionPermit = {
+  queueDepth: number;
+  waitMs: number;
+  release: () => void;
+};
+
+function keyFor(params: RuntimeSchedulerSubmitAdmissionParams): string {
+  return `${params.schedulerUrl}\u0000${params.schedulerSchema ?? ''}`;
+}
+
+function removeWaiter(entry: AdmissionEntry, waiter: Waiter): boolean {
+  const index = entry.waiters.indexOf(waiter);
+  if (index < 0) return false;
+  entry.waiters.splice(index, 1);
+  return true;
+}
+
+function pruneEntry(key: string, entry: AdmissionEntry): void {
+  if (entry.active === 0 && entry.waiters.length === 0) {
+    registry().delete(key);
+  }
+}
+
+function permitFor(
+  key: string,
+  entry: AdmissionEntry,
+  queueDepth: number,
+  enqueuedAt: number,
+): RuntimeSchedulerSubmitAdmissionPermit {
+  let released = false;
+  return {
+    queueDepth,
+    waitMs: Date.now() - enqueuedAt,
+    release: () => {
+      if (released) return;
+      released = true;
+      const next = entry.waiters.shift();
+      if (next) {
+        clearTimeout(next.timeout);
+        next.resolve(
+          permitFor(key, entry, entry.waiters.length, next.enqueuedAt),
+        );
+        return;
+      }
+      entry.active -= 1;
+      pruneEntry(key, entry);
+    },
+  };
+}
+
+/**
+ * Wait for the next local bulk-submit slot. This never makes a launch durable
+ * or returns success to an API caller; it merely controls when the caller may
+ * start the existing atomic scheduler transaction.
+ */
+export async function acquireRuntimeSchedulerSubmitAdmission(
+  params: RuntimeSchedulerSubmitAdmissionParams,
+): Promise<RuntimeSchedulerSubmitAdmissionPermit> {
+  const key = keyFor(params);
+  const entries = registry();
+  const entry = entries.get(key) ?? { active: 0, waiters: [] };
+  entries.set(key, entry);
+  const enqueuedAt = Date.now();
+
+  if (entry.active < RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxActive) {
+    entry.active += 1;
+    return permitFor(key, entry, 0, enqueuedAt);
+  }
+
+  if (entry.waiters.length >= RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxQueued) {
+    pruneEntry(key, entry);
+    throw new RuntimeSchedulerSubmissionAdmissionError(
+      'full',
+      entry.waiters.length,
+    );
+  }
+
+  return new Promise<RuntimeSchedulerSubmitAdmissionPermit>(
+    (resolve, reject) => {
+      const waiter: Waiter = {
+        enqueuedAt,
+        resolve,
+        reject,
+        timeout: setTimeout(() => {
+          if (!removeWaiter(entry, waiter)) return;
+          reject(
+            new RuntimeSchedulerSubmissionAdmissionError(
+              'timed_out',
+              entry.waiters.length,
+            ),
+          );
+          pruneEntry(key, entry);
+        }, RUNTIME_SCHEDULER_SUBMIT_ADMISSION.maxWaitMs),
+      };
+      entry.waiters.push(waiter);
+    },
+  );
+}
+
+/** Run submit-time control-plane work while holding the bounded permit. */
+export async function withRuntimeSchedulerSubmitAdmission<T>(
+  params: RuntimeSchedulerSubmitAdmissionParams,
+  operation: (permit: RuntimeSchedulerSubmitAdmissionPermit) => Promise<T>,
+): Promise<T> {
+  const permit = await acquireRuntimeSchedulerSubmitAdmission(params);
+  try {
+    return await operation(permit);
+  } finally {
+    permit.release();
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/terminal-contract.ts

@@ -0,0 +1,20 @@
+/**
+ * The terminal transition contract a worker release declares. Declaring it
+ * proves the owning runtime release has the durable compute-billing controller
+ * required before a capped completion may bypass the parked Absurd invocation.
+ *
+ * There is exactly one. A worker that declares nothing — or anything this build
+ * does not recognize — takes the compatibility wake path, which is what every
+ * pre-declaration lane has always done.
+ */
+export const DIRECT_COMPLETED_TERMINAL_CONTRACT =
+  'direct-completed-v2' as const;
+
+export type DirectCompletedTerminalContract =
+  typeof DIRECT_COMPLETED_TERMINAL_CONTRACT;
+
+export function directCompletedTerminalContract(
+  value: unknown,
+): DirectCompletedTerminalContract | null {
+  return value === DIRECT_COMPLETED_TERMINAL_CONTRACT ? value : null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-queue-metrics.ts

@@ -0,0 +1,34 @@
+/**
+ * Global scheduler queue state cannot be collected independently by every
+ * worker. The production metrics collector scrapes every worker, so a
+ * per-scrape aggregate would multiply PlanetScale load by the fleet size.
+ */
+export type WorkerQueueMetrics = {
+  queued: string;
+  running: string;
+  oldest_queued_s: string;
+  oldest_running_s: string;
+  stale_leases: string;
+  completed_runner_handoffs: string;
+  running_without_lease: string;
+  stale_heartbeats: string;
+  play_run_dead_letters: string;
+};
+
+/**
+ * Preserve the established metric series while explicitly reporting that this
+ * individual worker does not own a fresh global queue-state projection.
+ */
+export function unavailableWorkerQueueMetrics(): WorkerQueueMetrics {
+  return {
+    queued: 'NaN',
+    running: 'NaN',
+    oldest_queued_s: 'NaN',
+    oldest_running_s: 'NaN',
+    stale_leases: 'NaN',
+    completed_runner_handoffs: 'NaN',
+    running_without_lease: 'NaN',
+    stale_heartbeats: 'NaN',
+    play_run_dead_letters: 'NaN',
+  };
+}