deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -0,0 +1,835 @@
+import { createHash, randomUUID } from 'node:crypto';
+import {
+  withRuntimeSchedulerClient,
+  type RuntimeSchedulerPoolOptions,
+} from './postgres-pool';
+import type {
+  PostgresSchedulerOptions,
+  PostgresSchedulerQueryClient,
+} from './postgres';
+import { RuntimeSchedulerCapacityError } from './scheduler-capacity';
+import { RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME } from '../runner-backends/types';
+import {
+  RuntimeSandboxOperationState,
+  RuntimeSandboxProvider,
+} from './runtime-sandbox-operation';
+
+/** The safe baseline when an organization has no current override. */
+export const DEFAULT_PLAY_SANDBOX_CONCURRENCY = 4;
+export const MAX_PLAY_SANDBOX_CONCURRENCY = 250;
+/** A crashed worker cannot retain capacity forever. */
+export const RUNTIME_SANDBOX_CAPACITY_LEASE_MS = 50 * 60 * 1_000;
+
+/**
+ * Ship the mirror and resource accounting before allowing it to defer work.
+ * This gives operators a safe seed/drain window for pre-existing sandboxes.
+ */
+export type RuntimeSandboxCapacityAdmissionMode = 'observe' | 'enforce';
+
+export function runtimeSandboxCapacityAdmissionMode(
+  env: Record<string, string | undefined> = process.env,
+): RuntimeSandboxCapacityAdmissionMode {
+  const value = env.DEEPLINE_RUNTIME_SANDBOX_CAPACITY_ADMISSION?.trim();
+  if (!value || value === 'observe') return 'observe';
+  if (value === 'enforce') return 'enforce';
+  throw new Error(
+    'DEEPLINE_RUNTIME_SANDBOX_CAPACITY_ADMISSION must be observe or enforce.',
+  );
+}
+
+/** Compatibility boundary for existing callback implementations. New lifecycle
+ * code uses `RuntimeSandboxProvider`; this accepts the equivalent serialized
+ * provider id without widening to arbitrary text. */
+export type RuntimeSandboxProviderId = `${RuntimeSandboxProvider}`;
+
+/** Stable <=63-character Daytona label for a durable operation key. */
+export function runtimeSandboxProviderOperationToken(
+  operationKey: string,
+): string {
+  const key = assertId(operationKey, 'operationKey');
+  return `dlop-${createHash('sha256').update(key).digest('base64url')}`;
+}
+
+function quoteIdent(value: string): string {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) {
+    throw new Error(`Invalid runtime scheduler schema: ${value}`);
+  }
+  return `"${value}"`;
+}
+
+function schemaTables(options?: Pick<PostgresSchedulerOptions, 'schema'>) {
+  const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
+  return {
+    cache: `${schema}."runtime_org_sandbox_limit_cache"`,
+    leases: `${schema}."runtime_sandbox_capacity_leases"`,
+    operations: `${schema}."runtime_sandbox_operations"`,
+    cleanup: `${schema}."sandbox_cleanup_jobs_v2"`,
+  };
+}
+
+function assertId(value: string, name: string): string {
+  const normalized = value.trim();
+  if (!normalized || normalized.length > 256) {
+    throw new Error(`${name} is required and must be at most 256 characters.`);
+  }
+  return normalized;
+}
+
+function assertLimit(value: number): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    value < 1 ||
+    value > MAX_PLAY_SANDBOX_CONCURRENCY
+  ) {
+    throw new Error(
+      `Sandbox concurrency must be an integer from 1 through ${MAX_PLAY_SANDBOX_CONCURRENCY}.`,
+    );
+  }
+  return value;
+}
+
+export class RuntimeSandboxCapacityLimitError extends RuntimeSchedulerCapacityError {
+  readonly orgId: string;
+  readonly limit: number;
+  readonly activeLeases: number;
+
+  constructor(input: { orgId: string; limit: number; activeLeases: number }) {
+    super(
+      `Runtime scheduler sandbox capacity is full for organization ${input.orgId} (${input.activeLeases}/${input.limit} active leases).`,
+      15_000,
+    );
+    this.name = RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME;
+    this.orgId = input.orgId;
+    this.limit = input.limit;
+    this.activeLeases = input.activeLeases;
+  }
+}
+
+export class RuntimeSandboxOperationNotRetryableError extends Error {
+  constructor(readonly state: RuntimeSandboxOperationState) {
+    super(`Sandbox operation is ${state}; it cannot resume provider create.`);
+    this.name = 'RuntimeSandboxOperationNotRetryableError';
+  }
+}
+
+export async function syncRuntimeOrgSandboxLimitCache(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    orgId: string;
+    configuredLimit: number | null;
+    expiresAt: number | null;
+    sourceRevision: number;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<{ syncedAtMs: number; effectiveLimit: number }> {
+  const orgId = assertId(input.orgId, 'orgId');
+  const limit =
+    input.configuredLimit === null
+      ? DEFAULT_PLAY_SANDBOX_CONCURRENCY
+      : assertLimit(input.configuredLimit);
+  if (!Number.isSafeInteger(input.sourceRevision) || input.sourceRevision < 0) {
+    throw new Error('sourceRevision must be a non-negative safe integer.');
+  }
+  if (
+    input.expiresAt !== null &&
+    (!Number.isFinite(input.expiresAt) || input.expiresAt <= 0)
+  ) {
+    throw new Error(
+      'expiresAt must be a positive epoch milliseconds value or null.',
+    );
+  }
+  const { cache } = schemaTables(options);
+  const result = await client.query<{ synced_at_ms: number }>(
+    `
+      INSERT INTO ${cache} (
+        org_id, "limit", expires_at, source_revision, synced_at
+      )
+      VALUES (
+        $1,
+        $2,
+        CASE WHEN $3::double precision IS NULL THEN NULL
+          ELSE to_timestamp($3::double precision / 1000.0) END,
+        $4,
+        clock_timestamp()
+      )
+      ON CONFLICT (org_id) DO UPDATE
+      SET "limit" = EXCLUDED."limit",
+          expires_at = EXCLUDED.expires_at,
+          source_revision = EXCLUDED.source_revision,
+          synced_at = clock_timestamp()
+      WHERE ${cache}.source_revision <= EXCLUDED.source_revision
+      RETURNING floor(extract(epoch FROM synced_at) * 1000)::bigint AS synced_at_ms
+    `,
+    [orgId, limit, input.expiresAt, input.sourceRevision],
+  );
+  return {
+    // A stale reconciler write is intentionally a no-op; the newer cache row
+    // remains the source used by workers.
+    syncedAtMs: Number(result.rows[0]?.synced_at_ms ?? Date.now()),
+    effectiveLimit: limit,
+  };
+}
+
+export async function syncRuntimeOrgSandboxLimitCacheViaRuntimeScheduler(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  orgId: string;
+  configuredLimit: number | null;
+  expiresAt: number | null;
+  sourceRevision: number;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+}): Promise<{ syncedAtMs: number; effectiveLimit: number }> {
+  return await withRuntimeSchedulerClient(input.poolOptions, (client) =>
+    syncRuntimeOrgSandboxLimitCache(client, input, input.schedulerOptions),
+  );
+}
+
+export type RuntimeOrgSandboxLimitCacheRow = {
+  limit: number;
+  expiresAt: number | null;
+  sourceRevision: number;
+  syncedAtMs: number;
+};
+
+export async function readRuntimeOrgSandboxLimitCache(
+  client: PostgresSchedulerQueryClient,
+  input: { orgId: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeOrgSandboxLimitCacheRow | null> {
+  const { cache } = schemaTables(options);
+  const result = await client.query<{
+    limit: number;
+    expires_at_ms: number | null;
+    source_revision: number;
+    synced_at_ms: number;
+  }>(
+    `
+      SELECT
+        "limit",
+        CASE WHEN expires_at IS NULL THEN NULL
+          ELSE floor(extract(epoch FROM expires_at) * 1000)::bigint END
+          AS expires_at_ms,
+        source_revision,
+        floor(extract(epoch FROM synced_at) * 1000)::bigint AS synced_at_ms
+      FROM ${cache}
+      WHERE org_id = $1
+    `,
+    [assertId(input.orgId, 'orgId')],
+  );
+  const row = result.rows[0];
+  return row
+    ? {
+        limit: Number(row.limit),
+        expiresAt:
+          row.expires_at_ms === null ? null : Number(row.expires_at_ms),
+        sourceRevision: Number(row.source_revision),
+        syncedAtMs: Number(row.synced_at_ms),
+      }
+    : null;
+}
+
+export async function readRuntimeOrgSandboxLimitCacheViaRuntimeScheduler(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  orgId: string;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+}): Promise<RuntimeOrgSandboxLimitCacheRow | null> {
+  return await withRuntimeSchedulerClient(input.poolOptions, (client) =>
+    readRuntimeOrgSandboxLimitCache(client, input, input.schedulerOptions),
+  );
+}
+
+export async function reserveRuntimeSandboxCapacityLease(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    orgId: string;
+    runId: string;
+    provider: RuntimeSandboxProviderId;
+    leaseId?: string;
+    /** Required by current workers: the operation intent precedes provider I/O. */
+    attempt?: number;
+    environment?: 'preview' | 'production';
+    mode?: RuntimeSandboxCapacityAdmissionMode;
+    /** Resume the one unresolved Daytona create for this run, if present. */
+    resumePendingDaytonaOperation?: boolean;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<
+  | {
+      granted: true;
+      leaseId: string;
+      providerOperationToken: string | null;
+      limit: number;
+      activeLeases: number;
+      wouldDefer: boolean;
+      resumeExistingProviderOperation: boolean;
+    }
+  | { granted: false; limit: number; activeLeases: number }
+> {
+  const orgId = assertId(input.orgId, 'orgId');
+  const runId = assertId(input.runId, 'runId');
+  const { cache, leases, operations } = schemaTables(options);
+  // This is deliberately a soft admission check: concurrent workers may both
+  // observe space and insert. Avoid locks and long transactions; the provider
+  // has its own global safety cap and the 50-minute lease bounds crashes.
+  const observed = await client.query<{ limit: number; active_leases: number }>(
+    `
+      SELECT
+        coalesce(
+          (
+            SELECT CASE
+              WHEN expires_at IS NOT NULL AND expires_at <= clock_timestamp()
+                THEN $2::integer
+              ELSE "limit"
+            END
+            FROM ${cache}
+            WHERE org_id = $1
+          ),
+          $2::integer
+        ) AS "limit",
+        (
+          SELECT count(*)::integer
+          FROM ${leases}
+          WHERE org_id = $1
+            AND released_at IS NULL
+            -- A bound lease represents a provider resource that must remain
+            -- counted until provider-confirmed deletion. The 50-minute TTL
+            -- is only the crash backstop for a create that never became
+            -- durable enough to bind to a sandbox id.
+            AND (sandbox_id IS NOT NULL OR expires_at > clock_timestamp())
+        ) AS active_leases
+    `,
+    [orgId, DEFAULT_PLAY_SANDBOX_CONCURRENCY],
+  );
+  const limit = Number(
+    observed.rows[0]?.limit ?? DEFAULT_PLAY_SANDBOX_CONCURRENCY,
+  );
+  const activeLeases = Number(observed.rows[0]?.active_leases ?? 0);
+  const wouldDefer = activeLeases >= limit;
+  const leaseId = input.leaseId?.trim() || randomUUID();
+  const providerOperationToken =
+    input.attempt === undefined
+      ? null
+      : runtimeSandboxProviderOperationToken(leaseId);
+  if (
+    input.resumePendingDaytonaOperation === true &&
+    input.provider === RuntimeSandboxProvider.Daytona &&
+    input.attempt !== undefined
+  ) {
+    const pending = await client.query<{
+      lease_id: string;
+      provider_operation_token: string | null;
+      attempt: number;
+      environment: 'preview' | 'production';
+    }>(
+      `SELECT lease.lease_id, operation.provider_operation_token,
+              operation.attempt, operation.environment
+         FROM ${operations} AS operation
+         JOIN ${leases} AS lease ON lease.lease_id = operation.operation_key
+        WHERE operation.org_id = $1
+          AND operation.run_id = $2
+          AND operation.provider = $3
+          AND operation.state = $4
+          AND lease.released_at IS NULL
+          AND lease.sandbox_id IS NULL
+          AND lease.expires_at > clock_timestamp()
+        ORDER BY operation.created_at ASC
+        LIMIT 2`,
+      [orgId, runId, input.provider, RuntimeSandboxOperationState.Reserved],
+    );
+    if (pending.rows.length > 1) {
+      throw new Error(
+        'More than one Daytona sandbox operation is pending for this run.',
+      );
+    }
+    const pendingOperation = pending.rows[0];
+    if (pendingOperation) {
+      const pendingToken =
+        pendingOperation.provider_operation_token ??
+        (await backfillReservedProviderOperationToken(
+          client,
+          {
+            leaseId: pendingOperation.lease_id,
+            orgId,
+            runId,
+            provider: input.provider,
+            attempt: pendingOperation.attempt,
+            environment: pendingOperation.environment,
+            providerOperationToken: runtimeSandboxProviderOperationToken(
+              pendingOperation.lease_id,
+            ),
+          },
+          options,
+        ));
+      return {
+        granted: true,
+        leaseId: pendingOperation.lease_id,
+        providerOperationToken: pendingToken,
+        limit,
+        activeLeases,
+        wouldDefer,
+        resumeExistingProviderOperation: true,
+      };
+    }
+  }
+  const existing = await client.query<{
+    org_id: string;
+    run_id: string;
+    provider: RuntimeSandboxProviderId;
+    operation_attempt: number | null;
+    operation_environment: 'preview' | 'production' | null;
+    operation_provider: RuntimeSandboxProviderId | null;
+    provider_operation_token: string | null;
+    operation_state: RuntimeSandboxOperationState | null;
+    released_at: Date | null;
+    sandbox_id: string | null;
+  }>(
+    `SELECT lease.org_id, lease.run_id, lease.provider,
+            operation.attempt AS operation_attempt,
+            operation.environment AS operation_environment,
+            operation.provider AS operation_provider,
+            operation.provider_operation_token,
+            operation.state AS operation_state,
+            lease.released_at, lease.sandbox_id
+       FROM ${leases} AS lease
+       LEFT JOIN ${operations} AS operation
+         ON operation.operation_key = lease.lease_id
+      WHERE lease.lease_id = $1`,
+    [leaseId],
+  );
+  if (existing.rows[0]) {
+    const row = existing.rows[0];
+    if (
+      row.org_id !== orgId ||
+      row.run_id !== runId ||
+      row.provider !== input.provider ||
+      (input.attempt !== undefined &&
+        (row.operation_attempt !== input.attempt ||
+          row.operation_environment !== (input.environment ?? 'production') ||
+          row.operation_provider !== input.provider))
+    ) {
+      throw new Error(
+        'operationKey is already owned by another sandbox operation.',
+      );
+    }
+    if (
+      input.attempt !== undefined &&
+      (row.operation_state !== RuntimeSandboxOperationState.Reserved ||
+        row.released_at !== null ||
+        row.sandbox_id !== null)
+    ) {
+      throw new RuntimeSandboxOperationNotRetryableError(
+        row.operation_state ?? RuntimeSandboxOperationState.Abandoned,
+      );
+    }
+    if (
+      input.attempt !== undefined &&
+      row.provider_operation_token !== null &&
+      row.provider_operation_token !== providerOperationToken
+    ) {
+      throw new Error(
+        'operationKey reservation has an incompatible provider operation token.',
+      );
+    }
+    let resolvedProviderOperationToken = row.provider_operation_token;
+    if (
+      input.attempt !== undefined &&
+      resolvedProviderOperationToken === null
+    ) {
+      if (providerOperationToken === null) {
+        throw new Error(
+          'Current sandbox operation reservation is missing its provider operation token.',
+        );
+      }
+      resolvedProviderOperationToken =
+        await backfillReservedProviderOperationToken(
+          client,
+          {
+            leaseId,
+            orgId,
+            runId,
+            provider: input.provider,
+            attempt: input.attempt,
+            environment: input.environment ?? 'production',
+            providerOperationToken,
+          },
+          options,
+        );
+    }
+    return {
+      granted: true,
+      leaseId,
+      providerOperationToken: resolvedProviderOperationToken,
+      limit,
+      activeLeases,
+      wouldDefer,
+      resumeExistingProviderOperation: false,
+    };
+  }
+  if (wouldDefer && (input.mode ?? 'enforce') === 'enforce') {
+    return { granted: false, limit, activeLeases };
+  }
+  await client.query(
+    `
+      WITH reserved_lease AS (
+        INSERT INTO ${leases} (
+          lease_id, org_id, run_id, provider, expires_at
+        ) VALUES (
+          $1, $2, $3, $4,
+          clock_timestamp() + ($5::integer * interval '1 millisecond')
+        )
+        ON CONFLICT (lease_id) DO NOTHING
+        RETURNING lease_id
+      )
+      INSERT INTO ${operations} (
+        operation_key, provider_operation_token, provider, run_id, attempt, org_id, environment, state
+      )
+      SELECT
+        reserved_lease.lease_id, $9, $4, $3, $6, $2, $7, $8
+      FROM reserved_lease
+      WHERE $6::integer IS NOT NULL
+      ON CONFLICT (operation_key) DO NOTHING
+    `,
+    [
+      leaseId,
+      orgId,
+      runId,
+      input.provider,
+      RUNTIME_SANDBOX_CAPACITY_LEASE_MS,
+      input.attempt ?? null,
+      input.environment ?? 'production',
+      RuntimeSandboxOperationState.Reserved,
+      providerOperationToken,
+    ],
+  );
+  // The caller may have lost the INSERT response after PostgreSQL committed,
+  // or raced another identical retry.  A second read is mandatory: conflict
+  // swallowing alone would manufacture a lease without proving whose intent
+  // it resumes.
+  const persisted = await client.query<{
+    org_id: string;
+    run_id: string;
+    provider: RuntimeSandboxProviderId;
+    operation_attempt: number | null;
+    operation_environment: 'preview' | 'production' | null;
+    provider_operation_token: string | null;
+  }>(
+    `SELECT lease.org_id, lease.run_id, lease.provider,
+            operation.attempt AS operation_attempt,
+            operation.environment AS operation_environment,
+            operation.provider_operation_token
+       FROM ${leases} AS lease
+       LEFT JOIN ${operations} AS operation
+         ON operation.operation_key = lease.lease_id
+      WHERE lease.lease_id = $1`,
+    [leaseId],
+  );
+  const row = persisted.rows[0];
+  if (
+    !row ||
+    row.org_id !== orgId ||
+    row.run_id !== runId ||
+    row.provider !== input.provider ||
+    (input.attempt !== undefined &&
+      (row.operation_attempt !== input.attempt ||
+        row.operation_environment !== (input.environment ?? 'production')))
+  ) {
+    throw new Error(
+      'operationKey reservation did not persist the requested sandbox operation.',
+    );
+  }
+  return {
+    granted: true,
+    leaseId,
+    providerOperationToken: row.provider_operation_token,
+    limit,
+    activeLeases,
+    wouldDefer,
+    resumeExistingProviderOperation: false,
+  };
+}
+
+async function backfillReservedProviderOperationToken(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    leaseId: string;
+    orgId: string;
+    runId: string;
+    provider: RuntimeSandboxProviderId;
+    attempt: number;
+    environment: 'preview' | 'production';
+    providerOperationToken: string;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<string> {
+  const { leases, operations } = schemaTables(options);
+  const restored = await client.query<{ provider_operation_token: string }>(
+    `
+      UPDATE ${operations} AS operation
+      SET provider_operation_token = $2,
+          updated_at = clock_timestamp()
+      FROM ${leases} AS lease
+      WHERE operation.operation_key = $1
+        AND operation.provider = $3
+        AND operation.run_id = $4
+        AND operation.attempt = $5
+        AND operation.org_id = $6
+        AND operation.environment = $7
+        AND operation.state = $8
+        AND operation.provider_operation_token IS NULL
+        AND lease.lease_id = operation.operation_key
+        AND lease.released_at IS NULL
+        AND lease.sandbox_id IS NULL
+        AND lease.expires_at > clock_timestamp()
+      RETURNING operation.provider_operation_token
+    `,
+    [
+      input.leaseId,
+      input.providerOperationToken,
+      input.provider,
+      input.runId,
+      input.attempt,
+      input.orgId,
+      input.environment,
+      RuntimeSandboxOperationState.Reserved,
+    ],
+  );
+  const token = restored.rows[0]?.provider_operation_token;
+  if (token !== input.providerOperationToken) {
+    throw new Error(
+      'Reserved sandbox operation could not be atomically backfilled with its provider operation token.',
+    );
+  }
+  return token;
+}
+
+/**
+ * Strict operation-first admission API for current managed sandbox workers.
+ * The caller owns the deterministic key (run + attempt + provider create
+ * attempt), so retrying after an ambiguous process death resumes one durable
+ * operation instead of manufacturing another reservation.
+ */
+export async function reserveRuntimeSandboxOperation(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    operationKey: string;
+    orgId: string;
+    runId: string;
+    attempt: number;
+    provider: RuntimeSandboxProviderId;
+    environment: 'preview' | 'production';
+    mode?: RuntimeSandboxCapacityAdmissionMode;
+    resumePendingDaytonaOperation?: boolean;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): ReturnType<typeof reserveRuntimeSandboxCapacityLease> {
+  if (!Number.isSafeInteger(input.attempt) || input.attempt < 1) {
+    throw new Error('attempt must be a positive safe integer.');
+  }
+  return reserveRuntimeSandboxCapacityLease(
+    client,
+    {
+      orgId: input.orgId,
+      runId: input.runId,
+      provider: input.provider,
+      leaseId: assertId(input.operationKey, 'operationKey'),
+      attempt: input.attempt,
+      environment: input.environment,
+      mode: input.mode,
+      resumePendingDaytonaOperation: input.resumePendingDaytonaOperation,
+    },
+    options,
+  );
+}
+
+export async function bindRuntimeSandboxCapacityLease(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    leaseId: string;
+    provider: RuntimeSandboxProviderId;
+    sandboxId: string;
+    /** Provider routing identity is part of a physical sandbox identity. */
+    routingDomain: string | null;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const { leases } = schemaTables(options);
+  const result = await client.query<{ lease_id: string }>(
+    `
+      UPDATE ${leases}
+      SET provider = $2, sandbox_id = $3, routing_domain = $4
+      WHERE lease_id = $1 AND released_at IS NULL AND expires_at > clock_timestamp()
+      RETURNING lease_id
+    `,
+    [
+      assertId(input.leaseId, 'leaseId'),
+      input.provider,
+      assertId(input.sandboxId, 'sandboxId'),
+      input.routingDomain?.trim() || null,
+    ],
+  );
+  return result.rows.length === 1;
+}
+
+export async function releaseRuntimeSandboxCapacityLease(
+  client: PostgresSchedulerQueryClient,
+  input: { leaseId: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const { leases } = schemaTables(options);
+  const result = await client.query<{ lease_id: string }>(
+    `
+      UPDATE ${leases}
+      SET released_at = clock_timestamp()
+      WHERE lease_id = $1 AND released_at IS NULL
+      RETURNING lease_id
+    `,
+    [assertId(input.leaseId, 'leaseId')],
+  );
+  return result.rows.length === 1;
+}
+
+/**
+ * Confirmed provider rejection/no-resource outcome.  This is deliberately not
+ * the generic release helper: a reserved operation must become terminal so a
+ * late callback cannot accidentally promote a known rejected create.
+ */
+export async function abandonRuntimeSandboxOperationReservation(
+  client: PostgresSchedulerQueryClient,
+  input: { operationKey: string },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const { leases, operations } = schemaTables(options);
+  const result = await client.query<{ operation_key: string }>(
+    `
+      WITH abandoned AS (
+        UPDATE ${operations} AS operation
+        SET state = $2,
+            updated_at = clock_timestamp()
+        WHERE operation.operation_key = $1
+          AND operation.state = $3
+        RETURNING operation.operation_key
+      ), released AS (
+        UPDATE ${leases} AS lease
+        SET released_at = clock_timestamp()
+        FROM abandoned
+        WHERE lease.lease_id = abandoned.operation_key
+          AND lease.released_at IS NULL
+        RETURNING lease.lease_id
+      )
+      SELECT operation_key FROM abandoned
+    `,
+    [
+      assertId(input.operationKey, 'operationKey'),
+      RuntimeSandboxOperationState.Abandoned,
+      RuntimeSandboxOperationState.Reserved,
+    ],
+  );
+  return result.rows.length === 1;
+}
+
+/** Release only after the provider confirms deletion (or already absent). */
+export async function releaseRuntimeSandboxCapacityLeaseForSandbox(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    runId: string;
+    provider: RuntimeSandboxProviderId;
+    sandboxId: string;
+    routingDomain: string | null;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<number> {
+  const { leases } = schemaTables(options);
+  const result = await client.query<{ lease_id: string }>(
+    `
+      UPDATE ${leases}
+      SET released_at = clock_timestamp()
+      WHERE run_id = $1 AND provider = $2 AND sandbox_id = $3
+        AND routing_domain IS NOT DISTINCT FROM $4
+        AND released_at IS NULL
+      RETURNING lease_id
+    `,
+    [
+      assertId(input.runId, 'runId'),
+      input.provider,
+      assertId(input.sandboxId, 'sandboxId'),
+      input.routingDomain?.trim() || null,
+    ],
+  );
+  return result.rows.length;
+}
+
+export type RuntimeSandboxCapacitySummary = {
+  provider: RuntimeSandboxProviderId;
+  activeSandboxes: number;
+  pendingCleanup: number;
+  deletingCleanup: number;
+  oldestActiveAgeMs: number | null;
+};
+
+/**
+ * Read-only soft-lease occupancy diagnostic. This is intentionally not the
+ * durable live-sandbox operator view: an expired crash-fallback lease does
+ * not prove that the provider resource was deleted.
+ */
+export async function readRuntimeSandboxCapacitySummary(
+  client: PostgresSchedulerQueryClient,
+  input: { provider: RuntimeSandboxProviderId },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxCapacitySummary> {
+  const { leases, cleanup } = schemaTables(options);
+  const result = await client.query<{
+    active_sandboxes: number;
+    pending_cleanup: number;
+    deleting_cleanup: number;
+    oldest_active_age_ms: number | null;
+  }>(
+    `
+      SELECT
+        (
+          SELECT count(*)::integer FROM ${leases}
+          WHERE provider = $1 AND sandbox_id IS NOT NULL
+            AND released_at IS NULL
+        ) AS active_sandboxes,
+        (
+          SELECT count(*)::integer FROM ${cleanup}
+          WHERE provider = $1 AND status = 'pending'
+        ) AS pending_cleanup,
+        (
+          SELECT count(*)::integer FROM ${cleanup}
+          WHERE provider = $1 AND status = 'deleting'
+        ) AS deleting_cleanup,
+        (
+          SELECT floor(extract(epoch FROM (clock_timestamp() - min(created_at))) * 1000)::bigint
+          FROM ${leases}
+          WHERE provider = $1 AND sandbox_id IS NOT NULL
+            AND released_at IS NULL
+        ) AS oldest_active_age_ms
+    `,
+    [input.provider],
+  );
+  const row = result.rows[0];
+  return {
+    provider: input.provider,
+    activeSandboxes: Number(row?.active_sandboxes ?? 0),
+    pendingCleanup: Number(row?.pending_cleanup ?? 0),
+    deletingCleanup: Number(row?.deleting_cleanup ?? 0),
+    oldestActiveAgeMs:
+      row?.oldest_active_age_ms === null ||
+      row?.oldest_active_age_ms === undefined
+        ? null
+        : Number(row.oldest_active_age_ms),
+  };
+}
+
+export async function readRuntimeSandboxCapacitySummaryViaRuntimeScheduler(input: {
+  poolOptions: RuntimeSchedulerPoolOptions;
+  provider: RuntimeSandboxProviderId;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+}): Promise<RuntimeSandboxCapacitySummary> {
+  return await withRuntimeSchedulerClient(input.poolOptions, (client) =>
+    readRuntimeSandboxCapacitySummary(
+      client,
+      { provider: input.provider },
+      input.schedulerOptions,
+    ),
+  );
+}