deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-ledger-reconciliation.ts

@@ -0,0 +1,1218 @@
+import type {
+  PostgresSchedulerOptions,
+  PostgresSchedulerQueryClient,
+} from './postgres';
+import {
+  RuntimeSandboxOperationState,
+  RuntimeSandboxProvider,
+} from './runtime-sandbox-operation';
+
+const DAYTONA_PROVIDER = RuntimeSandboxProvider.Daytona;
+const DEFAULT_BATCH_SIZE = 500;
+const MAX_BATCH_SIZE = 1_000;
+/** Never classify a very recent resource from a provider snapshot as absent. */
+export const RUNTIME_SANDBOX_LEDGER_RECONCILIATION_MINIMUM_AGE_MS =
+  10 * 60 * 1_000;
+
+function quoteIdent(value: string): string {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) {
+    throw new Error(`Invalid runtime scheduler schema: ${value}`);
+  }
+  return `"${value}"`;
+}
+
+function tables(options?: Pick<PostgresSchedulerOptions, 'schema'>) {
+  const schema = quoteIdent(options?.schema ?? 'runtime_scheduler');
+  return {
+    attempts: `${schema}."attempts"`,
+    runs: `${schema}."runs"`,
+    operations: `${schema}."runtime_sandbox_operations"`,
+    cleanup: `${schema}."sandbox_cleanup_jobs_v2"`,
+    cleanupEvents: `${schema}."sandbox_cleanup_events_v2"`,
+    leases: `${schema}."runtime_sandbox_capacity_leases"`,
+    control: `${schema}."control"`,
+  };
+}
+
+/** A provider inventory item is a physical-sandbox identity, never an id-only
+ * guess.  Daytona's operation label lets recovery fence a create whose HTTP
+ * response was lost after Daytona had already accepted it. */
+export type DaytonaSandboxInventoryEntry = {
+  sandboxId: string;
+  operationKey: string | null;
+  routingDomain: string | null;
+};
+
+function normalizeProviderInventory(
+  values: readonly DaytonaSandboxInventoryEntry[],
+): DaytonaSandboxInventoryEntry[] {
+  const byIdentity = new Map<string, DaytonaSandboxInventoryEntry>();
+  for (const value of values) {
+    const sandboxId = value.sandboxId.trim();
+    if (!sandboxId)
+      throw new Error('Provider inventory sandboxId is required.');
+    const operationKey = value.operationKey?.trim() || null;
+    const routingDomain = value.routingDomain?.trim() || null;
+    byIdentity.set(
+      `${sandboxId}\u0000${routingDomain ?? ''}\u0000${operationKey ?? ''}`,
+      { sandboxId, operationKey, routingDomain },
+    );
+  }
+  return [...byIdentity.values()];
+}
+
+function batchSize(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_BATCH_SIZE;
+  if (!Number.isSafeInteger(value) || value < 1 || value > MAX_BATCH_SIZE) {
+    throw new Error(
+      `batchSize must be an integer from 1 through ${MAX_BATCH_SIZE}.`,
+    );
+  }
+  return value;
+}
+
+export type RuntimeSandboxLedgerReconciliationSummary = {
+  provider: RuntimeSandboxProvider.Daytona;
+  providerSandboxCount: number;
+  matchedLiveSandboxes: number;
+  unmatchedProviderSandboxIds: string[];
+  liveLeasesToSeed: number;
+  absentLedgerRows: number;
+  seededLeases: number;
+  /** Historical or raced leases released because deletion was already durable. */
+  releasedTerminalLeases: number;
+  markedDeleted: number;
+  /** Reserved creates recovered from Daytona's exact operation-key label. */
+  recoveredReservedOperations: number;
+  hasMoreAbsentLedgerRows: boolean;
+  dryRun: boolean;
+  diagnostics: readonly RuntimeSandboxLedgerReconciliationDiagnostic[];
+};
+
+export enum RuntimeSandboxLedgerReconciliationPhase {
+  ReleaseTerminalCapacity = 'release_terminal_capacity',
+  MatchProviderInventory = 'match_provider_inventory',
+  RecoverReservedOperations = 'recover_reserved_operations',
+  FindAbsentOperations = 'find_absent_operations',
+  InspectCapacityBindings = 'inspect_capacity_bindings',
+  SeedCapacityBindings = 'seed_capacity_bindings',
+  MarkProviderConfirmedAbsent = 'mark_provider_confirmed_absent',
+}
+
+export type RuntimeSandboxLedgerReconciliationDiagnostic = {
+  phase: RuntimeSandboxLedgerReconciliationPhase;
+  durationMs: number;
+  rowCount: number;
+};
+
+export type RuntimeSandboxOperationBackfillCursor = {
+  /** Exact PostgreSQL timestamptz text; milliseconds are insufficient. */
+  updatedAt: string;
+  runId: string;
+  attempt: number;
+  resourceOrdinal: number;
+};
+
+export type RuntimeSandboxOperationBackfillSummary = {
+  scannedAttempts: number;
+  insertedOperations: number;
+  hasMore: boolean;
+  nextCursor: RuntimeSandboxOperationBackfillCursor | null;
+  /** Redacted, approval-safe resource targets in this bounded page. */
+  targets: readonly RuntimeSandboxOperationBackfillTarget[];
+};
+
+export type RuntimeSandboxOperationBackfillTarget = {
+  runId: string;
+  attempt: number;
+  resourceOrdinal: number;
+  provider: RuntimeSandboxProvider;
+  sandboxId: string;
+  wouldInsert: boolean;
+};
+
+export enum RuntimeSandboxOperationBackfillPhase {
+  Pending = 'pending',
+  Complete = 'complete',
+}
+
+const RUNTIME_SANDBOX_OPERATION_BACKFILL_CONTROL_KEY =
+  'runtime_sandbox_operation_backfill_v1';
+
+type RuntimeSandboxOperationBackfillCheckpoint = {
+  phase: RuntimeSandboxOperationBackfillPhase;
+  cursor: RuntimeSandboxOperationBackfillCursor | null;
+};
+
+async function readRuntimeSandboxOperationBackfillCheckpoint(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxOperationBackfillCheckpoint> {
+  const { control } = tables(options);
+  const result = await client.query<{ value: unknown }>(
+    `SELECT value FROM ${control} WHERE key = $1`,
+    [RUNTIME_SANDBOX_OPERATION_BACKFILL_CONTROL_KEY],
+  );
+  const value = result.rows[0]?.value;
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return {
+      phase: RuntimeSandboxOperationBackfillPhase.Pending,
+      cursor: null,
+    };
+  }
+  const checkpoint = value as { phase?: unknown; cursor?: unknown };
+  if (checkpoint.phase === RuntimeSandboxOperationBackfillPhase.Complete) {
+    return {
+      phase: RuntimeSandboxOperationBackfillPhase.Complete,
+      cursor: null,
+    };
+  }
+  const cursor =
+    checkpoint.cursor as RuntimeSandboxOperationBackfillCursor | null;
+  return {
+    phase: RuntimeSandboxOperationBackfillPhase.Pending,
+    cursor:
+      cursor &&
+      typeof cursor.updatedAt === 'string' &&
+      typeof cursor.runId === 'string' &&
+      typeof cursor.attempt === 'number' &&
+      typeof cursor.resourceOrdinal === 'number'
+        ? cursor
+        : null,
+  };
+}
+
+export type RuntimeSandboxOperationBackfillProgress = {
+  phase: RuntimeSandboxOperationBackfillPhase;
+  dryRun: boolean;
+  batches: number;
+  scannedAttempts: number;
+  insertedOperations: number;
+  projectedOperations: number;
+  targets: readonly RuntimeSandboxOperationBackfillTarget[];
+  nextCursor: RuntimeSandboxOperationBackfillCursor | null;
+};
+
+/** Read one bounded resource-element page without writes. */
+async function previewRuntimeSandboxOperationBackfillPage(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    batchSize?: number;
+    cursor: RuntimeSandboxOperationBackfillCursor | null;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxOperationBackfillSummary> {
+  const limit = batchSize(input.batchSize);
+  const { attempts, operations } = tables(options);
+  const page = await client.query<{
+    run_id: string;
+    attempt: number;
+    updated_at: string;
+    resource_ordinal: number;
+    provider: RuntimeSandboxProvider;
+    sandbox_id: string;
+    would_insert: boolean;
+  }>(
+    `
+      WITH candidates AS (
+        SELECT attempt.run_id, attempt.attempt, attempt.updated_at,
+               resource.ordinal::integer AS resource_ordinal,
+               CASE resource.value->>'kind' WHEN 'modal_sandbox' THEN $6 ELSE $7 END AS provider,
+               resource.value->>'sandboxId' AS sandbox_id,
+               coalesce(CASE resource.value->>'kind'
+                 WHEN 'modal_sandbox' THEN NULLIF(resource.value->>'modalAppId', '')
+                 ELSE NULLIF(resource.value->>'daytonaOrganizationId', '') END, '')
+                 AS routing_domain_key
+        FROM ${attempts} AS attempt
+        CROSS JOIN LATERAL jsonb_array_elements(
+          CASE WHEN jsonb_typeof(attempt.resource_json) = 'array'
+            THEN attempt.resource_json ELSE jsonb_build_array(attempt.resource_json) END
+        ) WITH ORDINALITY AS resource(value, ordinal)
+        WHERE attempt.resource_json IS NOT NULL
+          AND resource.value->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
+          AND NULLIF(resource.value->>'sandboxId', '') IS NOT NULL
+          AND ($1::text IS NULL OR
+            (attempt.updated_at, attempt.run_id, attempt.attempt, resource.ordinal) >
+              ($1::timestamptz, $2::text, $3::integer, $4::bigint))
+      ), page AS (
+        SELECT *, row_number() OVER (
+          PARTITION BY provider, sandbox_id, routing_domain_key
+          ORDER BY updated_at, run_id, attempt, resource_ordinal
+        ) AS conflict_rank
+        FROM candidates
+        ORDER BY updated_at, run_id, attempt, resource_ordinal
+        LIMIT $5
+      )
+      SELECT page.run_id, page.attempt, page.updated_at::text AS updated_at,
+             page.resource_ordinal, page.provider, page.sandbox_id,
+             page.conflict_rank = 1 AND
+             NOT EXISTS (
+               SELECT 1 FROM ${operations} AS operation
+               WHERE operation.provider = page.provider
+                 AND operation.sandbox_id = page.sandbox_id
+                 AND operation.routing_domain_key = page.routing_domain_key
+             ) AS would_insert
+      FROM page
+      ORDER BY page.updated_at, page.run_id, page.attempt, page.resource_ordinal
+    `,
+    [
+      input.cursor?.updatedAt ?? null,
+      input.cursor?.runId ?? '',
+      input.cursor?.attempt ?? 0,
+      input.cursor?.resourceOrdinal ?? 0,
+      limit + 1,
+      RuntimeSandboxProvider.Modal,
+      RuntimeSandboxProvider.Daytona,
+    ],
+  );
+  const hasMore = page.rows.length > limit;
+  const items = page.rows.slice(0, limit);
+  const last = items.at(-1) ?? null;
+  return {
+    scannedAttempts: items.length,
+    insertedOperations: 0,
+    hasMore,
+    nextCursor: last
+      ? {
+          updatedAt: last.updated_at,
+          runId: last.run_id,
+          attempt: Number(last.attempt),
+          resourceOrdinal: Number(last.resource_ordinal),
+        }
+      : null,
+    targets: items.map((item) => ({
+      runId: item.run_id,
+      attempt: Number(item.attempt),
+      resourceOrdinal: Number(item.resource_ordinal),
+      provider: item.provider,
+      sandboxId: item.sandbox_id,
+      wouldInsert: item.would_insert,
+    })),
+  };
+}
+
+/** Persisted, resumable orchestrator. Dry-run performs no JSON expansion or writes. */
+export async function advanceRuntimeSandboxOperationBackfill(
+  client: PostgresSchedulerQueryClient,
+  input: { dryRun: boolean; batchSize?: number; maxBatches?: number },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxOperationBackfillProgress> {
+  const checkpoint = await readRuntimeSandboxOperationBackfillCheckpoint(
+    client,
+    options,
+  );
+  if (input.dryRun) {
+    if (checkpoint.phase === RuntimeSandboxOperationBackfillPhase.Complete) {
+      return {
+        phase: checkpoint.phase,
+        dryRun: true,
+        batches: 0,
+        scannedAttempts: 0,
+        insertedOperations: 0,
+        projectedOperations: 0,
+        targets: [],
+        nextCursor: null,
+      };
+    }
+    const preview = await previewRuntimeSandboxOperationBackfillPage(
+      client,
+      { batchSize: input.batchSize, cursor: checkpoint.cursor },
+      options,
+    );
+    return {
+      phase: RuntimeSandboxOperationBackfillPhase.Pending,
+      dryRun: true,
+      batches: preview.scannedAttempts > 0 ? 1 : 0,
+      scannedAttempts: preview.scannedAttempts,
+      insertedOperations: 0,
+      projectedOperations: preview.targets.filter(
+        (target) => target.wouldInsert,
+      ).length,
+      targets: preview.targets,
+      nextCursor: preview.nextCursor,
+    };
+  }
+  if (checkpoint.phase === RuntimeSandboxOperationBackfillPhase.Complete) {
+    return {
+      phase: checkpoint.phase,
+      dryRun: false,
+      batches: 0,
+      scannedAttempts: 0,
+      insertedOperations: 0,
+      projectedOperations: 0,
+      targets: [],
+      nextCursor: null,
+    };
+  }
+  const maxBatches = Math.max(
+    1,
+    Math.min(200, Math.floor(input.maxBatches ?? 1)),
+  );
+  let cursor = checkpoint.cursor;
+  let scannedAttempts = 0;
+  let insertedOperations = 0;
+  for (let batch = 0; batch < maxBatches; batch += 1) {
+    const page = await backfillRuntimeSandboxOperationsFromAttemptResources(
+      client,
+      { batchSize: input.batchSize, cursor: cursor ?? undefined },
+      options,
+    );
+    scannedAttempts += page.scannedAttempts;
+    insertedOperations += page.insertedOperations;
+    cursor = page.nextCursor;
+    const phase = page.hasMore
+      ? RuntimeSandboxOperationBackfillPhase.Pending
+      : RuntimeSandboxOperationBackfillPhase.Complete;
+    await client.query(
+      `INSERT INTO ${tables(options).control} (key, value)
+       VALUES ($1, $2::jsonb)
+       ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`,
+      [
+        RUNTIME_SANDBOX_OPERATION_BACKFILL_CONTROL_KEY,
+        JSON.stringify({
+          phase,
+          cursor:
+            phase === RuntimeSandboxOperationBackfillPhase.Complete
+              ? null
+              : cursor,
+        }),
+      ],
+    );
+    if (phase === RuntimeSandboxOperationBackfillPhase.Complete) {
+      return {
+        phase,
+        dryRun: false,
+        batches: batch + 1,
+        scannedAttempts,
+        insertedOperations,
+        projectedOperations: scannedAttempts,
+        targets: [],
+        nextCursor: null,
+      };
+    }
+  }
+  return {
+    phase: RuntimeSandboxOperationBackfillPhase.Pending,
+    dryRun: false,
+    batches: maxBatches,
+    scannedAttempts,
+    insertedOperations,
+    projectedOperations: scannedAttempts,
+    targets: [],
+    nextCursor: cursor,
+  };
+}
+
+/**
+ * One bounded compatibility migration from the old audit projection.
+ *
+ * This is intentionally separate from reconciliation: it keyset-scans one
+ * indexed attempt page, expands JSON only inside that page, and writes the
+ * normalized operation table.  Reconciliation itself therefore never starts
+ * a full-history JSON expansion.  Callers must persist `nextCursor` between
+ * batches; a fresh cursor intentionally restarts at the oldest page.
+ */
+export async function backfillRuntimeSandboxOperationsFromAttemptResources(
+  client: PostgresSchedulerQueryClient,
+  input: { batchSize?: number; cursor?: RuntimeSandboxOperationBackfillCursor },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxOperationBackfillSummary> {
+  const limit = batchSize(input.batchSize);
+  const cursor = input.cursor;
+  if (
+    cursor &&
+    (!cursor.updatedAt.trim() ||
+      Number.isNaN(Date.parse(cursor.updatedAt)) ||
+      !cursor.runId.trim() ||
+      !Number.isSafeInteger(cursor.attempt) ||
+      cursor.attempt < 1 ||
+      !Number.isSafeInteger(cursor.resourceOrdinal) ||
+      cursor.resourceOrdinal < 1)
+  ) {
+    throw new Error('Invalid runtime sandbox operation backfill cursor.');
+  }
+  const { attempts, runs, operations } = tables(options);
+  const result = await client.query<{
+    run_id: string;
+    attempt: number;
+    updated_at: string;
+    resource_ordinal: number;
+    provider: RuntimeSandboxProvider;
+    sandbox_id: string;
+    would_insert: boolean;
+    inserted_operations: number;
+  }>(
+    `
+      WITH resources AS MATERIALIZED (
+        SELECT attempt.run_id, attempt.attempt, attempt.updated_at, run.org_id,
+          resource.value AS resource, resource.ordinal::integer AS resource_ordinal,
+          CASE resource.value->>'kind'
+            WHEN 'modal_sandbox' THEN 'modal' ELSE 'daytona' END AS provider,
+          COALESCE(NULLIF(resource.value->>'runtimeEnvironment', ''),
+            NULLIF(resource.value->>'daytonaEnvironment', ''), 'production') AS environment,
+          CASE resource.value->>'kind'
+            WHEN 'modal_sandbox' THEN NULLIF(resource.value->>'modalAppId', '')
+            ELSE NULLIF(resource.value->>'daytonaOrganizationId', '') END AS routing_domain
+        FROM ${attempts} AS attempt
+        JOIN ${runs} AS run ON run.run_id = attempt.run_id
+        CROSS JOIN LATERAL jsonb_array_elements(
+          CASE WHEN jsonb_typeof(attempt.resource_json) = 'array'
+            THEN attempt.resource_json ELSE jsonb_build_array(attempt.resource_json) END
+        ) WITH ORDINALITY AS resource(value, ordinal)
+        WHERE attempt.resource_json IS NOT NULL
+          AND resource.value->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
+          AND NULLIF(resource.value->>'sandboxId', '') IS NOT NULL
+          AND (
+            $1::text IS NULL
+            OR (attempt.updated_at, attempt.run_id, attempt.attempt, resource.ordinal) >
+              ($1::timestamptz, $2::text, $3::integer, $4::bigint)
+          )
+        ORDER BY attempt.updated_at, attempt.run_id, attempt.attempt, resource.ordinal
+        LIMIT $5
+      ), ranked_resources AS (
+        SELECT resources.*,
+               coalesce(routing_domain, '') AS routing_domain_key,
+               row_number() OVER (
+                 PARTITION BY provider, (resource::jsonb)->>'sandboxId', coalesce(routing_domain, '')
+                 ORDER BY updated_at, run_id, attempt, resource_ordinal
+               ) AS conflict_rank
+        FROM resources
+      ), inserted AS (
+        INSERT INTO ${operations} (
+          operation_key, provider, run_id, attempt, org_id, environment,
+          sandbox_id, routing_domain, state, provider_created_at
+        )
+        SELECT
+          'backfill:' || provider || ':' || ((resource::jsonb)->>'sandboxId') || ':' || coalesce(routing_domain, ''),
+          provider,
+          run_id,
+          attempt,
+          org_id,
+          environment,
+          (resource::jsonb)->>'sandboxId',
+          routing_domain,
+          $6,
+          updated_at
+        FROM ranked_resources
+        WHERE conflict_rank = 1
+        ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
+        RETURNING provider, sandbox_id, routing_domain_key
+      )
+      SELECT
+        resource.run_id,
+        resource.attempt,
+        resource.updated_at::text AS updated_at,
+        resource.resource_ordinal,
+        resource.provider,
+        (resource.resource::jsonb)->>'sandboxId' AS sandbox_id,
+        resource.conflict_rank = 1 AND EXISTS (
+          SELECT 1
+          FROM inserted
+          WHERE inserted.provider = resource.provider
+            AND inserted.sandbox_id = (resource.resource::jsonb)->>'sandboxId'
+            AND inserted.routing_domain_key = resource.routing_domain_key
+        ) AS would_insert,
+        (SELECT count(*)::integer FROM inserted) AS inserted_operations
+      FROM ranked_resources AS resource
+      ORDER BY resource.updated_at, resource.run_id, resource.attempt, resource.resource_ordinal
+    `,
+    [
+      cursor?.updatedAt ?? null,
+      cursor?.runId ?? '',
+      cursor?.attempt ?? 0,
+      cursor?.resourceOrdinal ?? 0,
+      limit,
+      RuntimeSandboxOperationState.Live,
+    ],
+  );
+  const last = result.rows[0] ?? null;
+  const scannedAttempts = result.rows.length;
+  // A full page asks callers to fetch once more. This is conservative at an
+  // exact page boundary, but never skips a row by mutating an over-fetch.
+  const hasMore = scannedAttempts === limit;
+  const page = result.rows;
+  const next = page.at(-1) ?? null;
+  return {
+    scannedAttempts: page.length,
+    insertedOperations: Number(last?.inserted_operations ?? 0),
+    hasMore,
+    nextCursor: next
+      ? {
+          updatedAt: next.updated_at,
+          runId: next.run_id,
+          attempt: Number(next.attempt),
+          resourceOrdinal: Number(next.resource_ordinal),
+        }
+      : null,
+    targets: page.map((item) => ({
+      runId: item.run_id,
+      attempt: Number(item.attempt),
+      resourceOrdinal: Number(item.resource_ordinal),
+      provider: item.provider,
+      sandboxId: item.sandbox_id,
+      wouldInsert: item.would_insert,
+    })),
+  };
+}
+
+async function releaseTerminalSandboxCapacityLeases(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<number> {
+  const { cleanup, leases } = tables(options);
+  const released = await client.query<{ lease_id: string }>(
+    `
+      UPDATE ${leases} AS lease
+      SET released_at = clock_timestamp()
+      FROM ${cleanup} AS cleanup
+      WHERE lease.provider = '${DAYTONA_PROVIDER}'
+        AND cleanup.provider = lease.provider
+        AND cleanup.sandbox_id = lease.sandbox_id
+        AND cleanup.routing_domain IS NOT DISTINCT FROM lease.routing_domain
+        AND cleanup.run_id = lease.run_id
+        AND cleanup.status = 'deleted'
+        AND cleanup.deleted_at IS NOT NULL
+        AND lease.released_at IS NULL
+      RETURNING lease.lease_id
+    `,
+  );
+  return released.rows.length;
+}
+
+/**
+ * Reconcile durable sandbox records against one complete Daytona inventory.
+ *
+ * This is intentionally a rollout/migration operation, not a hot-path read:
+ * Daytona's label-scoped list is the authority for provider presence. A live
+ * provider resource is seeded into the soft-capacity lease table. An older
+ * durable resource absent from that inventory is completed in the cleanup
+ * ledger with a provider-confirmed-absent audit event; the resource history is
+ * never deleted or rewritten.
+ */
+export async function reconcileRuntimeDaytonaSandboxLedger(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    /** Complete source-labelled Daytona inventory, retaining ownership labels. */
+    providerSandboxes: readonly DaytonaSandboxInventoryEntry[];
+    /**
+     * Epoch time immediately after the complete provider inventory finished.
+     * Resources recorded later than this snapshot are never classified absent.
+     */
+    inventoryCapturedAtMs: number;
+    dryRun: boolean;
+    batchSize?: number;
+    minimumAgeMs?: number;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<RuntimeSandboxLedgerReconciliationSummary> {
+  const providerSandboxes = normalizeProviderInventory(input.providerSandboxes);
+  const limit = batchSize(input.batchSize);
+  if (
+    !Number.isSafeInteger(input.inventoryCapturedAtMs) ||
+    input.inventoryCapturedAtMs <= 0
+  ) {
+    throw new Error(
+      'inventoryCapturedAtMs must be a positive epoch milliseconds value.',
+    );
+  }
+  const minimumAgeMs =
+    input.minimumAgeMs ?? RUNTIME_SANDBOX_LEDGER_RECONCILIATION_MINIMUM_AGE_MS;
+  if (!Number.isSafeInteger(minimumAgeMs) || minimumAgeMs < 0) {
+    throw new Error('minimumAgeMs must be a non-negative safe integer.');
+  }
+  const { operations, cleanup, cleanupEvents, leases } = tables(options);
+  const operationStateType = `${quoteIdent(
+    options?.schema ?? 'runtime_scheduler',
+  )}.${quoteIdent('runtime_sandbox_operation_state')}`;
+  const diagnostics: RuntimeSandboxLedgerReconciliationDiagnostic[] = [];
+  async function measuredQuery<Row extends Record<string, unknown>>(
+    phase: RuntimeSandboxLedgerReconciliationPhase,
+    sql: string,
+    parameters?: unknown[],
+  ): Promise<{ rows: Row[] }> {
+    const startedAt = performance.now();
+    try {
+      const result = await client.query<Row>(sql, parameters);
+      diagnostics.push({
+        phase,
+        durationMs: performance.now() - startedAt,
+        rowCount: result.rows.length,
+      });
+      return result;
+    } catch (error) {
+      diagnostics.push({
+        phase,
+        durationMs: performance.now() - startedAt,
+        rowCount: 0,
+      });
+      throw error;
+    }
+  }
+  // A previous reconciliation may have observed a sandbox just before its
+  // normal cleanup completed. Repair those terminal rows before calculating
+  // the next snapshot's capacity state. This is idempotent and is deliberately
+  // a migration-only convergence step, never a worker hot-path lock.
+  let releasedTerminalLeases = 0;
+  if (!input.dryRun) {
+    const startedAt = performance.now();
+    releasedTerminalLeases = await releaseTerminalSandboxCapacityLeases(
+      client,
+      options,
+    );
+    diagnostics.push({
+      phase: RuntimeSandboxLedgerReconciliationPhase.ReleaseTerminalCapacity,
+      durationMs: performance.now() - startedAt,
+      rowCount: releasedTerminalLeases,
+    });
+  }
+
+  const inventoryJson = JSON.stringify(
+    providerSandboxes.map((sandbox) => ({
+      sandbox_id: sandbox.sandboxId,
+      operation_key: sandbox.operationKey,
+      routing_domain: sandbox.routingDomain,
+    })),
+  );
+  const duplicateOperationTokens = new Set<string>();
+  const operationTokenCounts = new Map<string, number>();
+  for (const sandbox of providerSandboxes) {
+    if (sandbox.operationKey) {
+      operationTokenCounts.set(
+        sandbox.operationKey,
+        (operationTokenCounts.get(sandbox.operationKey) ?? 0) + 1,
+      );
+    }
+  }
+  for (const [token, count] of operationTokenCounts) {
+    if (count > 1) duplicateOperationTokens.add(token);
+  }
+  // A Daytona create can commit at the provider and lose its response before
+  // the worker records it. The exact operation label is the only selector
+  // allowed to recover that state; an id-only inventory item remains unowned
+  // and fails closed below. Provider acceptance never proves runner liveness.
+  const reservedOwned = providerSandboxes.length
+    ? await measuredQuery<{
+        sandbox_id: string;
+        provider_operation_token: string;
+        routing_domain: string | null;
+      }>(
+        RuntimeSandboxLedgerReconciliationPhase.RecoverReservedOperations,
+        `
+          SELECT item.sandbox_id, operation.provider_operation_token, item.routing_domain
+          FROM jsonb_to_recordset($1::jsonb) AS item(
+            sandbox_id text, operation_key text, routing_domain text
+          )
+          JOIN ${operations} AS operation
+            ON operation.provider_operation_token = item.operation_key
+           AND operation.provider = $2
+           AND operation.environment = 'production'
+           AND operation.state = $3
+          JOIN ${leases} AS lease
+            ON lease.lease_id = operation.operation_key
+           AND lease.released_at IS NULL
+        `,
+        [
+          inventoryJson,
+          DAYTONA_PROVIDER,
+          RuntimeSandboxOperationState.Reserved,
+        ],
+      )
+    : { rows: [] };
+  let recoveredReservedOperations = 0;
+  if (!input.dryRun && reservedOwned.rows.length > 0) {
+    const recovered = await measuredQuery<{ operation_key: string }>(
+      RuntimeSandboxLedgerReconciliationPhase.RecoverReservedOperations,
+      `
+        WITH inventory AS (
+          SELECT item.*
+          FROM jsonb_to_recordset($1::jsonb) AS item(
+            sandbox_id text, operation_key text, routing_domain text
+          )
+          WHERE operation_key IS NOT NULL
+            AND 1 = (
+              SELECT count(*)
+              FROM jsonb_to_recordset($1::jsonb) AS sibling(
+                sandbox_id text, operation_key text, routing_domain text
+              )
+              WHERE sibling.operation_key = item.operation_key
+            )
+        ), promoted AS (
+          UPDATE ${operations} AS operation
+          SET sandbox_id = inventory.sandbox_id,
+              routing_domain = inventory.routing_domain,
+              state = $3::${operationStateType},
+              provider_created_at = clock_timestamp(),
+              updated_at = clock_timestamp()
+          FROM inventory
+          JOIN ${leases} AS lease
+            ON TRUE
+          WHERE operation.provider_operation_token = inventory.operation_key
+            AND operation.provider = $2
+            AND operation.environment = 'production'
+            AND operation.state = $4
+            AND lease.lease_id = operation.operation_key
+            AND lease.released_at IS NULL
+          RETURNING operation.operation_key, operation.run_id, operation.attempt,
+                    operation.environment, operation.sandbox_id,
+                    operation.routing_domain, operation.state
+        ), resource_fact AS (
+          SELECT promoted.*,
+                 jsonb_build_object(
+                   'kind', 'daytona_sandbox',
+                   'sandboxId', promoted.sandbox_id,
+                   'runtimeEnvironment', promoted.environment,
+                   'daytonaOrganizationId', promoted.routing_domain,
+                   'billingStartedAt', extract(epoch FROM clock_timestamp()) * 1000,
+                   'sandboxCapacityLeaseId', promoted.operation_key,
+                   'lateAcquired', true
+                 ) AS resource
+          FROM promoted
+        ), recorded_attempt AS (
+          -- Preserve the provider fact as a compatibility/audit projection in
+          -- the same atomic recovery transition. It is explicitly late: this
+          -- path never re-opens execution.
+          UPDATE ${tables(options).attempts} AS attempt
+          SET resource_json = CASE
+                WHEN attempt.resource_json IS NULL THEN jsonb_build_array(resource_fact.resource)
+                WHEN jsonb_typeof(attempt.resource_json) = 'array'
+                  THEN attempt.resource_json || jsonb_build_array(resource_fact.resource)
+                ELSE jsonb_build_array(attempt.resource_json, resource_fact.resource)
+              END,
+              updated_at = clock_timestamp()
+          FROM resource_fact
+          WHERE attempt.run_id = resource_fact.run_id
+            AND attempt.attempt = resource_fact.attempt
+          RETURNING attempt.run_id, attempt.attempt
+        ), bound AS (
+          UPDATE ${leases} AS lease
+          SET provider = $2,
+              sandbox_id = promoted.sandbox_id,
+              routing_domain = promoted.routing_domain
+          FROM promoted
+          JOIN recorded_attempt
+            ON recorded_attempt.run_id = promoted.run_id
+           AND recorded_attempt.attempt = promoted.attempt
+          WHERE lease.lease_id = promoted.operation_key
+            AND lease.released_at IS NULL
+          RETURNING lease.lease_id
+        ), cleanup_inserted AS (
+          INSERT INTO ${cleanup} (
+            provider, sandbox_id, run_id, attempt, environment, routing_domain
+          )
+          SELECT $2, sandbox_id, run_id, attempt, environment, routing_domain
+          FROM promoted
+          JOIN bound ON bound.lease_id = promoted.operation_key
+          ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
+        )
+        SELECT promoted.operation_key
+        FROM promoted
+        JOIN bound ON bound.lease_id = promoted.operation_key
+      `,
+      [
+        inventoryJson,
+        DAYTONA_PROVIDER,
+        RuntimeSandboxOperationState.Deleting,
+        RuntimeSandboxOperationState.Reserved,
+      ],
+    );
+    recoveredReservedOperations = recovered.rows.length;
+  }
+
+  const liveRows = providerSandboxes.length
+    ? await measuredQuery<{
+        sandbox_id: string;
+        run_id: string;
+        attempt: number;
+        org_id: string;
+        environment: string;
+        routing_domain: string | null;
+        operation_key: string;
+      }>(
+        RuntimeSandboxLedgerReconciliationPhase.MatchProviderInventory,
+        `
+          SELECT operation.sandbox_id, operation.run_id, operation.attempt,
+                 operation.org_id, operation.environment, operation.routing_domain,
+                 operation.operation_key
+          FROM jsonb_to_recordset($4::jsonb) AS item(
+            sandbox_id text, operation_key text, routing_domain text
+          )
+          JOIN ${operations} AS operation
+            ON operation.sandbox_id = item.sandbox_id
+           AND operation.routing_domain IS NOT DISTINCT FROM item.routing_domain
+          WHERE operation.provider = $1
+            AND operation.environment = 'production'
+            AND operation.state IN ($2, $3)
+        `,
+        [
+          DAYTONA_PROVIDER,
+          RuntimeSandboxOperationState.Live,
+          RuntimeSandboxOperationState.Deleting,
+          inventoryJson,
+        ],
+      )
+    : { rows: [] };
+  const live = liveRows.rows.map((row) => ({
+    sandboxId: row.sandbox_id,
+    runId: row.run_id,
+    attempt: Number(row.attempt),
+    orgId: row.org_id,
+    environment: row.environment,
+    routingDomain: row.routing_domain,
+    operationKey: row.operation_key,
+  }));
+  const matchedIdentities = new Set(
+    live.map((row) => `${row.sandboxId}\u0000${row.routingDomain ?? ''}`),
+  );
+  const unmatchedProviderSandboxIds = providerSandboxes
+    .filter(
+      (sandbox) =>
+        !matchedIdentities.has(
+          `${sandbox.sandboxId}\u0000${sandbox.routingDomain ?? ''}`,
+        ) &&
+        !reservedOwned.rows.some(
+          (owned) =>
+            owned.sandbox_id === sandbox.sandboxId &&
+            owned.routing_domain === sandbox.routingDomain &&
+            owned.provider_operation_token === sandbox.operationKey &&
+            !duplicateOperationTokens.has(sandbox.operationKey ?? ''),
+        ),
+    )
+    .map((sandbox) => sandbox.sandboxId);
+
+  // Do not write an incomplete migration: a listed sandbox lacking durable
+  // ownership cannot safely be assigned to an organization for enforcement.
+  if (unmatchedProviderSandboxIds.length > 0) {
+    return {
+      provider: DAYTONA_PROVIDER,
+      providerSandboxCount: providerSandboxes.length,
+      matchedLiveSandboxes: live.length,
+      unmatchedProviderSandboxIds,
+      liveLeasesToSeed: 0,
+      absentLedgerRows: 0,
+      seededLeases: 0,
+      releasedTerminalLeases,
+      markedDeleted: 0,
+      recoveredReservedOperations,
+      hasMoreAbsentLedgerRows: false,
+      dryRun: input.dryRun,
+      diagnostics,
+    };
+  }
+
+  const candidates = await measuredQuery<{
+    sandbox_id: string;
+    run_id: string;
+    attempt: number;
+    org_id: string;
+    environment: string;
+    routing_domain: string | null;
+  }>(
+    RuntimeSandboxLedgerReconciliationPhase.FindAbsentOperations,
+    `
+      SELECT sandbox_id, run_id, attempt, org_id, environment, routing_domain
+      FROM ${operations}
+      WHERE provider = $1
+        AND environment = 'production'
+        AND state = $2
+        AND NOT EXISTS (
+          SELECT 1
+          FROM jsonb_to_recordset($3::jsonb) AS inventory(
+            sandbox_id text, operation_key text, routing_domain text
+          )
+          WHERE inventory.sandbox_id = ${operations}.sandbox_id
+            AND inventory.routing_domain IS NOT DISTINCT FROM ${operations}.routing_domain
+        )
+        AND NOT EXISTS (
+          SELECT 1
+          FROM ${cleanup} AS existing
+          WHERE existing.provider = '${DAYTONA_PROVIDER}'
+            AND existing.sandbox_id = ${operations}.sandbox_id
+            AND existing.routing_domain IS NOT DISTINCT FROM ${operations}.routing_domain
+            AND existing.status = 'deleted'
+        )
+        -- Provider inventory is a point-in-time observation. Never classify
+        -- an operation recorded after it as absent.
+        AND provider_created_at <= to_timestamp($4::double precision / 1000.0)
+        AND provider_created_at <= clock_timestamp() - ($5::integer * interval '1 millisecond')
+      ORDER BY provider_created_at, operation_key
+      LIMIT $6
+    `,
+    [
+      DAYTONA_PROVIDER,
+      RuntimeSandboxOperationState.Live,
+      inventoryJson,
+      input.inventoryCapturedAtMs,
+      minimumAgeMs,
+      limit + 1,
+    ],
+  );
+  const hasMoreAbsentLedgerRows = candidates.rows.length > limit;
+  const absent = candidates.rows.slice(0, limit).map((row) => ({
+    sandboxId: row.sandbox_id,
+    runId: row.run_id,
+    attempt: Number(row.attempt),
+    orgId: row.org_id,
+    environment: row.environment,
+    routingDomain: row.routing_domain,
+  }));
+
+  const leaseRows = live.length
+    ? await measuredQuery<{
+        sandbox_id: string;
+        routing_domain: string | null;
+      }>(
+        RuntimeSandboxLedgerReconciliationPhase.InspectCapacityBindings,
+        `
+          SELECT DISTINCT sandbox_id, routing_domain
+          FROM ${leases}
+          WHERE provider = '${DAYTONA_PROVIDER}'
+            AND sandbox_id = ANY($1::text[])
+        `,
+        [live.map((row) => row.sandboxId)],
+      )
+    : { rows: [] };
+  const existingLeaseIdentities = new Set(
+    leaseRows.rows.map(
+      (row) => `${row.sandbox_id}\u0000${row.routing_domain ?? ''}`,
+    ),
+  );
+  const terminalLiveRows = live.length
+    ? await measuredQuery<{
+        sandbox_id: string;
+        routing_domain: string | null;
+      }>(
+        RuntimeSandboxLedgerReconciliationPhase.InspectCapacityBindings,
+        `
+          SELECT DISTINCT cleanup.sandbox_id, cleanup.routing_domain
+          FROM ${cleanup} AS cleanup
+          WHERE cleanup.provider = '${DAYTONA_PROVIDER}'
+            AND cleanup.status = 'deleted'
+            AND cleanup.deleted_at IS NOT NULL
+            AND cleanup.sandbox_id = ANY($1::text[])
+        `,
+        [live.map((row) => row.sandboxId)],
+      )
+    : { rows: [] };
+  const terminalLiveSandboxIdentities = new Set(
+    terminalLiveRows.rows.map(
+      (row) => `${row.sandbox_id}\u0000${row.routing_domain ?? ''}`,
+    ),
+  );
+  const liveLeasesToSeed = live.filter(
+    (row) =>
+      !existingLeaseIdentities.has(
+        `${row.sandboxId}\u0000${row.routingDomain ?? ''}`,
+      ) &&
+      !terminalLiveSandboxIdentities.has(
+        `${row.sandboxId}\u0000${row.routingDomain ?? ''}`,
+      ),
+  );
+
+  if (input.dryRun) {
+    return {
+      provider: DAYTONA_PROVIDER,
+      providerSandboxCount: providerSandboxes.length,
+      matchedLiveSandboxes: live.length,
+      unmatchedProviderSandboxIds,
+      liveLeasesToSeed: liveLeasesToSeed.length,
+      absentLedgerRows: absent.length,
+      seededLeases: 0,
+      releasedTerminalLeases,
+      markedDeleted: 0,
+      recoveredReservedOperations,
+      hasMoreAbsentLedgerRows,
+      dryRun: true,
+      diagnostics,
+    };
+  }
+
+  let seededLeases = 0;
+  if (liveLeasesToSeed.length > 0) {
+    const seeded = await measuredQuery<{ lease_id: string }>(
+      RuntimeSandboxLedgerReconciliationPhase.SeedCapacityBindings,
+      `
+        INSERT INTO ${leases} (
+          lease_id, org_id, run_id, provider, sandbox_id, routing_domain, expires_at, released_at
+        )
+        SELECT
+          item.operation_key,
+          item.org_id,
+          item.run_id,
+          '${DAYTONA_PROVIDER}',
+          item.sandbox_id,
+          item.routing_domain,
+          clock_timestamp(),
+          NULL
+        FROM jsonb_to_recordset($1::jsonb) AS item(
+          operation_key text, sandbox_id text, org_id text, run_id text,
+          routing_domain text
+        )
+        WHERE NOT EXISTS (
+          SELECT 1
+          FROM ${cleanup} AS terminal_cleanup
+          WHERE terminal_cleanup.provider = '${DAYTONA_PROVIDER}'
+            AND terminal_cleanup.sandbox_id = item.sandbox_id
+            AND terminal_cleanup.run_id = item.run_id
+            AND terminal_cleanup.routing_domain IS NOT DISTINCT FROM item.routing_domain
+            AND terminal_cleanup.status = 'deleted'
+            AND terminal_cleanup.deleted_at IS NOT NULL
+        )
+        ON CONFLICT (lease_id) DO UPDATE
+        SET org_id = EXCLUDED.org_id,
+            run_id = EXCLUDED.run_id,
+            provider = EXCLUDED.provider,
+            sandbox_id = EXCLUDED.sandbox_id,
+            routing_domain = EXCLUDED.routing_domain,
+            expires_at = EXCLUDED.expires_at
+        WHERE ${leases}.released_at IS NULL
+          AND NOT EXISTS (
+            SELECT 1
+            FROM ${cleanup} AS terminal_cleanup
+            WHERE terminal_cleanup.provider = '${DAYTONA_PROVIDER}'
+              AND terminal_cleanup.sandbox_id = EXCLUDED.sandbox_id
+              AND terminal_cleanup.run_id = EXCLUDED.run_id
+              AND terminal_cleanup.routing_domain IS NOT DISTINCT FROM EXCLUDED.routing_domain
+              AND terminal_cleanup.status = 'deleted'
+              AND terminal_cleanup.deleted_at IS NOT NULL
+          )
+        RETURNING lease_id
+      `,
+      [
+        JSON.stringify(
+          liveLeasesToSeed.map((row) => ({
+            operation_key: row.operationKey,
+            sandbox_id: row.sandboxId,
+            org_id: row.orgId,
+            run_id: row.runId,
+            routing_domain: row.routingDomain,
+          })),
+        ),
+      ],
+    );
+    seededLeases = seeded.rows.length;
+  }
+
+  const marked = absent.length
+    ? await measuredQuery<{ sandbox_id: string }>(
+        RuntimeSandboxLedgerReconciliationPhase.MarkProviderConfirmedAbsent,
+        `
+          WITH candidates AS (
+            SELECT *
+            FROM jsonb_to_recordset($1::jsonb) AS item(
+              sandbox_id text,
+              run_id text,
+              attempt integer,
+              environment text,
+              routing_domain text
+            )
+          ),
+          changed AS (
+            INSERT INTO ${cleanup} (
+              provider, sandbox_id, run_id, attempt, environment, routing_domain,
+              status, deleted_at, available_at, updated_at, last_error
+            )
+            SELECT
+              '${DAYTONA_PROVIDER}', sandbox_id, run_id, attempt, environment, routing_domain,
+              'deleted', clock_timestamp(), clock_timestamp(), clock_timestamp(),
+              'reconciled_provider_confirmed_absent'
+            FROM candidates
+            ON CONFLICT (provider, sandbox_id, routing_domain_key) DO UPDATE
+            SET status = 'deleted',
+                deleted_at = clock_timestamp(),
+                lease_owner = NULL,
+                lease_expires_at = NULL,
+                updated_at = clock_timestamp(),
+                last_error = 'reconciled_provider_confirmed_absent'
+            -- A real cleanup worker owns deleting rows. Do not erase its
+            -- lease/fencing state; its normal completion records the outcome.
+            WHERE ${cleanup}.status = 'pending'
+              AND ${cleanup}.lease_owner IS NULL
+            RETURNING sandbox_id, routing_domain, run_id, attempt
+          ),
+          events AS (
+            INSERT INTO ${cleanupEvents} (
+              provider, sandbox_id, routing_domain, run_id, attempt,
+              cleanup_attempt, lease_owner, event_type, details_json
+            )
+            SELECT
+              '${DAYTONA_PROVIDER}', sandbox_id, routing_domain, run_id, attempt,
+              0, NULL, 'reconciled_provider_confirmed_absent',
+              jsonb_build_object('source', 'runtime-sandbox-ledger-reconciliation')
+            FROM changed
+          ),
+          operation_deleted AS (
+            UPDATE ${operations} AS operation
+            SET state = $2,
+                delete_confirmed_at = COALESCE(
+                  operation.delete_confirmed_at,
+                  clock_timestamp()
+                ),
+                updated_at = clock_timestamp()
+            FROM changed
+            WHERE operation.provider = '${DAYTONA_PROVIDER}'
+              AND operation.sandbox_id = changed.sandbox_id
+              AND operation.routing_domain IS NOT DISTINCT FROM changed.routing_domain
+              AND operation.state = $3
+            RETURNING operation.operation_key
+          ),
+          released_leases AS (
+            UPDATE ${leases} AS lease
+            SET released_at = clock_timestamp()
+            FROM changed
+            WHERE lease.provider = '${DAYTONA_PROVIDER}'
+              AND lease.run_id = changed.run_id
+              AND lease.sandbox_id = changed.sandbox_id
+              AND lease.routing_domain IS NOT DISTINCT FROM changed.routing_domain
+              AND lease.released_at IS NULL
+          )
+          SELECT sandbox_id FROM changed
+        `,
+        [
+          JSON.stringify(
+            absent.map((row) => ({
+              sandbox_id: row.sandboxId,
+              run_id: row.runId,
+              attempt: row.attempt,
+              environment: row.environment,
+              routing_domain: row.routingDomain,
+            })),
+          ),
+          RuntimeSandboxOperationState.Deleted,
+          RuntimeSandboxOperationState.Live,
+        ],
+      )
+    : { rows: [] };
+
+  // Covers the narrow race where a normal cleanup reaches provider-confirmed
+  // deletion after the pre-seed repair but before this reconciliation returns.
+  // The worker path also releases its own leases; this makes the migration
+  // converge even when an older worker completed cleanup first.
+  if (!input.dryRun) {
+    const startedAt = performance.now();
+    releasedTerminalLeases += await releaseTerminalSandboxCapacityLeases(
+      client,
+      options,
+    );
+    diagnostics.push({
+      phase: RuntimeSandboxLedgerReconciliationPhase.ReleaseTerminalCapacity,
+      durationMs: performance.now() - startedAt,
+      rowCount: releasedTerminalLeases,
+    });
+  }
+
+  return {
+    provider: DAYTONA_PROVIDER,
+    providerSandboxCount: providerSandboxes.length,
+    matchedLiveSandboxes: live.length,
+    unmatchedProviderSandboxIds,
+    liveLeasesToSeed: liveLeasesToSeed.length,
+    absentLedgerRows: absent.length,
+    seededLeases,
+    releasedTerminalLeases,
+    markedDeleted: marked.rows.length,
+    recoveredReservedOperations,
+    hasMoreAbsentLedgerRows,
+    dryRun: false,
+    diagnostics,
+  };
+}