deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts

@@ -0,0 +1,725 @@
+#!/usr/bin/env bun
+/**
+ * Fleet-wide Daytona sandbox janitor for the absurd play runtime.
+ *
+ * Why this exists
+ * ---------------
+ * One-shot play sandboxes are created with the label `source:
+ * 'deepline-play-runner'` plus `orgId/workflowId/playId/runId`, `ephemeral:
+ * true`, and `autoStopInterval: 5` (see
+ * `packages/play-runtime/runner-backends/backends/daytona-lifecycle.ts`).
+ * Per-run deletion happens through the durable Sandbox Cleanup state machine,
+ * but a worker crash can lose the cleanup enqueue after the physical resource
+ * was recorded. This janitor is the inventory backstop: Daytona labels discover
+ * candidates, while the exact Runtime Sandbox Operation ledger alone decides
+ * ownership and deletion eligibility. The janitor restores cleanup obligations;
+ * it never performs an unledgered provider deletion.
+ *
+ * A candidate is eligible only when the durable ledger has one exact physical
+ * owner at `(daytona, sandboxId, routingDomain)` and either the owning Run is
+ * terminal, deletion was already authorized, or the creation-time configured
+ * execution ceiling plus terminal/grace windows has elapsed. Daytona's runId
+ * label is intentionally excluded from authority: Daytona truncates label
+ * values to 63 characters, so it is only a correlation hint.
+ *
+ * Fleet scoping (IMPORTANT): the sandbox label scope is `source`-only and the
+ * Daytona account is SHARED across every lane (prod, every preview PR fleet,
+ * worktree dev — all use the same DAYTONA_API_KEY and stamp the identical
+ * `source` label; nothing on the sandbox names its lane/schema). A `list` by
+ * `source` therefore returns EVERY lane's sandboxes. The janitor may run only
+ * against the prod-default schema, and even there it may authorize deletion
+ * only for a resource exactly bound in that schema's operation ledger. A
+ * sandbox owned by another lane is therefore unresolved and spared. The worker
+ * wiring and CLI both refuse to sweep on an isolated schema (see
+ * `isIsolatedRuntimeSchedulerSchema`). This module's core assumes that gate has
+ * already been applied by its caller.
+ *
+ * Everything here is best-effort by design. A single ownership read or cleanup
+ * authorization failure is logged and the sweep continues — one wedged sandbox
+ * must never abort the fleet-wide pass. The only hard failure is an inability
+ * to enumerate at all.
+ */
+import { PLAY_RUNNER_TERMINAL_GRACE_SECONDS } from './/runtime-constants';
+import { DAYTONA_PLAY_RUNNER_LABEL_SOURCE } from './/runner-backends/backends/daytona-labels';
+
+export { DAYTONA_PLAY_RUNNER_LABEL_SOURCE };
+
+/**
+ * Slack added on top of the play runtime ceiling before a sandbox is eligible
+ * for reaping. The ceiling is the max user-code runtime; the grace covers
+ * setup, cleanup, and clock skew so we never race a sandbox that is legitimately
+ * still finishing. Constant, not env — this is a tuning number, not a secret.
+ */
+export const REAP_AGE_GRACE_MS = 5 * 60 * 1000;
+
+/**
+ * A single sandbox as seen by the janitor. Structurally a subset of the Daytona
+ * SDK `Sandbox`, so a real `Sandbox` satisfies it directly.
+ */
+export type ReapableSandbox = {
+  id: string;
+  labels: Record<string, string>;
+  /** Physical Daytona organization returned by the provider, not a label. */
+  organizationId?: string;
+  /** ISO-8601 creation timestamp, per the Daytona SDK `Sandbox.createdAt`. */
+  createdAt?: string;
+};
+
+/**
+ * Injected Daytona port. `list` yields our label-scoped sandboxes (the SDK
+ * `list` returns an async iterator that auto-pages, so we consume it lazily).
+ * Inventory is the only Daytona surface the janitor touches.
+ */
+export type DaytonaJanitorClientPort = {
+  list(query: {
+    labels: Record<string, string>;
+  }): AsyncIterable<ReapableSandbox>;
+};
+
+export type DaytonaJanitorOwnershipAlertCode =
+  | 'missing_routing_domain'
+  | 'ownership_missing'
+  | 'routing_domain_conflict'
+  | 'legacy_unbound_ownership'
+  | 'ownership_ambiguous'
+  | 'invalid_execution_ceiling'
+  | 'provider_inventory_conflicts_deleted_ledger'
+  | 'cleanup_ownership_conflict';
+
+export type DaytonaJanitorOwnership =
+  | {
+      kind: 'owned';
+      operationKey: string;
+      runId: string;
+      attempt: number;
+      orgId: string;
+      operationState: 'live' | 'deleting' | 'deleted';
+      runStatus:
+        | 'queued'
+        | 'running'
+        | 'waiting'
+        | 'completed'
+        | 'failed'
+        | 'cancelled';
+      providerCreatedAtMs: number;
+      waitDeadlineMs: number | null;
+      runtimeLimitSeconds: number | null;
+      cleanupStatus: 'pending' | 'deleting' | 'deleted' | null;
+    }
+  | {
+      kind: 'unresolved';
+      code: DaytonaJanitorOwnershipAlertCode;
+      candidateRunIds: string[];
+      candidateRoutingDomains: Array<string | null>;
+    };
+
+export type DaytonaJanitorCleanupAuthorization =
+  | 'scheduled'
+  | 'already_authorized'
+  | 'already_deleted'
+  | 'ownership_conflict'
+  | 'ownership_missing';
+
+/** Durable exact-ownership authority. Provider labels are never inputs. */
+export type DaytonaJanitorOwnershipPort = {
+  inspect(input: {
+    sandboxId: string;
+    routingDomain: string | null;
+  }): Promise<DaytonaJanitorOwnership>;
+  authorizeCleanup(input: {
+    operationKey: string;
+    sandboxId: string;
+    routingDomain: string;
+    runId: string;
+    attempt: number;
+  }): Promise<DaytonaJanitorCleanupAuthorization>;
+};
+
+export type ReapOptions = {
+  /** Injected clock for testability; defaults to Date.now. */
+  now?: () => number;
+  /** When true, decide without restoring a durable cleanup obligation. */
+  dryRun?: boolean;
+  /** Structured logger; defaults to console. */
+  logger?: Pick<Console, 'info' | 'warn'>;
+};
+
+export type ReapSummary = {
+  scanned: number;
+  cleanupEligible: number;
+  cleanupScheduled: number;
+  alreadyAuthorized: number;
+  spared: number;
+  alerts: number;
+  errors: number;
+};
+
+type ReapDecision =
+  | { authorizeCleanup: false; reason: string }
+  | { authorizeCleanup: true; reason: string };
+
+/**
+ * Pure per-sandbox decision. Returns whether to authorize durable cleanup and a
+ * short reason for the structured log line.
+ */
+async function decideSandbox(input: {
+  ownership: Extract<DaytonaJanitorOwnership, { kind: 'owned' }>;
+  nowMs: number;
+}): Promise<ReapDecision> {
+  const { ownership, nowMs } = input;
+  if (ownership.operationState === 'deleting') {
+    return { authorizeCleanup: true, reason: 'deletion_already_authorized' };
+  }
+  if (
+    ownership.runStatus === 'completed' ||
+    ownership.runStatus === 'failed' ||
+    ownership.runStatus === 'cancelled'
+  ) {
+    return { authorizeCleanup: true, reason: 'run_terminal' };
+  }
+  if (
+    ownership.runtimeLimitSeconds === null ||
+    !Number.isSafeInteger(ownership.runtimeLimitSeconds) ||
+    ownership.runtimeLimitSeconds <= 0 ||
+    !Number.isFinite(ownership.providerCreatedAtMs)
+  ) {
+    return { authorizeCleanup: false, reason: 'invalid_execution_ceiling' };
+  }
+  const configuredFloorMs =
+    ownership.providerCreatedAtMs +
+    (ownership.runtimeLimitSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+      1_000 +
+    REAP_AGE_GRACE_MS;
+  const deletionNotBeforeMs = Math.max(
+    configuredFloorMs,
+    ownership.waitDeadlineMs === null
+      ? Number.NEGATIVE_INFINITY
+      : ownership.waitDeadlineMs + REAP_AGE_GRACE_MS,
+  );
+  return nowMs < deletionNotBeforeMs
+    ? { authorizeCleanup: false, reason: 'before_deletion_not_before' }
+    : { authorizeCleanup: true, reason: 'execution_ceiling_elapsed' };
+}
+
+function parseCreatedAtMs(createdAt: string | undefined): number | null {
+  if (!createdAt) return null;
+  const parsed = Date.parse(createdAt);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Pure, testable core. Enumerate our label-scoped sandboxes, decide each
+ * against the scheduler store, and best-effort restore cleanup obligations for
+ * resources that should be dead. Never throws past a single sandbox failure:
+ * an authorization or ownership-read error is logged and the sweep continues.
+ * Enumeration failure is the one hard error — with nothing to iterate there is
+ * no sweep to salvage.
+ */
+export async function reapOrphanDaytonaSandboxes(input: {
+  daytona: DaytonaJanitorClientPort;
+  ownership: DaytonaJanitorOwnershipPort;
+  options?: ReapOptions;
+}): Promise<ReapSummary> {
+  const now = input.options?.now ?? Date.now;
+  const dryRun = input.options?.dryRun ?? false;
+  const logger = input.options?.logger ?? console;
+
+  const summary: ReapSummary = {
+    scanned: 0,
+    cleanupEligible: 0,
+    cleanupScheduled: 0,
+    alreadyAuthorized: 0,
+    spared: 0,
+    alerts: 0,
+    errors: 0,
+  };
+
+  const iterator = input.daytona.list({
+    labels: { source: DAYTONA_PLAY_RUNNER_LABEL_SOURCE },
+  });
+
+  for await (const sandbox of iterator) {
+    summary.scanned += 1;
+    const createdAtMs = parseCreatedAtMs(sandbox.createdAt);
+    const ageMs =
+      createdAtMs === null ? null : Math.max(0, now() - createdAtMs);
+    const labelRunId = sandbox.labels.runId?.trim() || null;
+    const routingDomain = sandbox.organizationId?.trim() || null;
+
+    let ownership: DaytonaJanitorOwnership;
+    try {
+      ownership = await input.ownership.inspect({
+        sandboxId: sandbox.id,
+        routingDomain,
+      });
+    } catch (error) {
+      summary.errors += 1;
+      summary.spared += 1;
+      logger.warn('[daytona-janitor.ownership_read_failed]', {
+        sandboxId: sandbox.id,
+        routingDomain,
+        labelRunId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      continue;
+    }
+
+    if (ownership.kind === 'unresolved') {
+      summary.alerts += 1;
+      summary.spared += 1;
+      logger.warn('[daytona-janitor.ownership_alert]', {
+        provider: 'daytona',
+        code: ownership.code,
+        sandboxId: sandbox.id,
+        routingDomain,
+        labelRunId,
+        candidateRunIds: ownership.candidateRunIds,
+        candidateRoutingDomains: ownership.candidateRoutingDomains,
+      });
+      continue;
+    }
+
+    if (labelRunId && labelRunId !== ownership.runId) {
+      logger.info('[daytona-janitor.label_correlation_mismatch]', {
+        sandboxId: sandbox.id,
+        routingDomain,
+        labelRunId,
+        ownerRunId: ownership.runId,
+      });
+    }
+
+    if (ownership.operationState === 'deleted') {
+      summary.alerts += 1;
+      summary.spared += 1;
+      logger.warn('[daytona-janitor.ownership_alert]', {
+        provider: 'daytona',
+        code: 'provider_inventory_conflicts_deleted_ledger',
+        sandboxId: sandbox.id,
+        routingDomain,
+        labelRunId,
+        ownerRunId: ownership.runId,
+        operationKey: ownership.operationKey,
+      });
+      continue;
+    }
+
+    const decision = await decideSandbox({ ownership, nowMs: now() });
+    if (!decision.authorizeCleanup) {
+      summary.spared += 1;
+      if (decision.reason === 'invalid_execution_ceiling') {
+        summary.alerts += 1;
+        logger.warn('[daytona-janitor.ownership_alert]', {
+          provider: 'daytona',
+          code: 'invalid_execution_ceiling',
+          sandboxId: sandbox.id,
+          routingDomain,
+          labelRunId,
+          ownerRunId: ownership.runId,
+          operationKey: ownership.operationKey,
+        });
+      }
+      continue;
+    }
+
+    summary.cleanupEligible += 1;
+    if (dryRun) {
+      logger.info('[daytona-janitor.would_schedule_cleanup]', {
+        sandboxId: sandbox.id,
+        routingDomain,
+        runId: ownership.runId,
+        ageMs,
+        reason: decision.reason,
+        dryRun: true,
+      });
+      continue;
+    }
+
+    try {
+      const authorization = await input.ownership.authorizeCleanup({
+        operationKey: ownership.operationKey,
+        sandboxId: sandbox.id,
+        routingDomain: routingDomain!,
+        runId: ownership.runId,
+        attempt: ownership.attempt,
+      });
+      if (authorization === 'scheduled') {
+        summary.cleanupScheduled += 1;
+      } else if (
+        authorization === 'already_authorized' ||
+        authorization === 'already_deleted'
+      ) {
+        summary.alreadyAuthorized += 1;
+      } else {
+        summary.alerts += 1;
+        summary.spared += 1;
+        logger.warn('[daytona-janitor.ownership_alert]', {
+          provider: 'daytona',
+          code:
+            authorization === 'ownership_conflict'
+              ? 'cleanup_ownership_conflict'
+              : 'ownership_missing',
+          sandboxId: sandbox.id,
+          routingDomain,
+          labelRunId,
+          ownerRunId: ownership.runId,
+          operationKey: ownership.operationKey,
+        });
+        continue;
+      }
+      logger.info('[daytona-janitor.cleanup_authorized]', {
+        sandboxId: sandbox.id,
+        routingDomain,
+        runId: ownership.runId,
+        ageMs,
+        reason: decision.reason,
+        authorization,
+      });
+    } catch (error) {
+      summary.errors += 1;
+      summary.spared += 1;
+      logger.warn('[daytona-janitor.cleanup_authorization_failed]', {
+        sandboxId: sandbox.id,
+        routingDomain,
+        runId: ownership.runId,
+        ageMs,
+        reason: decision.reason,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  logger.info('[daytona-janitor.summary]', { ...summary, dryRun });
+  return summary;
+}
+
+// ---------------------------------------------------------------------------
+// Production adapters (thin, imported by the CLI entry and the worker wiring).
+// ---------------------------------------------------------------------------
+
+type SchedulerQueryClient = {
+  query<Row extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: unknown[],
+  ): Promise<{ rows: Row[] }>;
+};
+
+function quoteIdent(value: string): string {
+  return `"${value.replace(/"/g, '""')}"`;
+}
+
+type JanitorOwnershipRow = {
+  operation_key: string;
+  run_id: string;
+  attempt: number;
+  org_id: string;
+  routing_domain: string | null;
+  state: 'reserved' | 'live' | 'deleting' | 'deleted' | 'abandoned';
+  provider_created_at_ms: string | null;
+  run_status: Extract<DaytonaJanitorOwnership, { kind: 'owned' }>['runStatus'];
+  wait_deadline_ms: string | null;
+  launch_json: unknown;
+  cleanup_status: 'pending' | 'deleting' | 'deleted' | null;
+  cleanup_run_id: string | null;
+  cleanup_attempt: number | null;
+};
+
+function runtimeLimitSecondsFromLaunch(launch: unknown): number | null {
+  if (!launch || typeof launch !== 'object' || Array.isArray(launch)) {
+    return null;
+  }
+  const contract = (launch as Record<string, unknown>).contractSnapshot;
+  if (!contract || typeof contract !== 'object' || Array.isArray(contract)) {
+    return null;
+  }
+  const runtimeLimit = (contract as Record<string, unknown>).runtimeLimit;
+  if (
+    !runtimeLimit ||
+    typeof runtimeLimit !== 'object' ||
+    Array.isArray(runtimeLimit)
+  ) {
+    return null;
+  }
+  const seconds = (runtimeLimit as Record<string, unknown>).timeoutSeconds;
+  return typeof seconds === 'number' &&
+    Number.isSafeInteger(seconds) &&
+    seconds > 0
+    ? seconds
+    : null;
+}
+
+/** Runtime Postgres Adapter for the janitor's exact ownership authority. */
+export function createPostgresDaytonaJanitorOwnershipPort(input: {
+  client: SchedulerQueryClient;
+  schema: string;
+}): DaytonaJanitorOwnershipPort {
+  const runs = `${quoteIdent(input.schema)}.${quoteIdent('runs')}`;
+  const operations = `${quoteIdent(input.schema)}.${quoteIdent(
+    'runtime_sandbox_operations',
+  )}`;
+  const cleanup = `${quoteIdent(input.schema)}.${quoteIdent(
+    'sandbox_cleanup_jobs_v2',
+  )}`;
+  return {
+    async inspect({ sandboxId, routingDomain }) {
+      if (!routingDomain) {
+        return {
+          kind: 'unresolved',
+          code: 'missing_routing_domain',
+          candidateRunIds: [],
+          candidateRoutingDomains: [],
+        };
+      }
+      const result = await input.client.query<JanitorOwnershipRow>(
+        `
+          SELECT
+            operation.operation_key,
+            operation.run_id,
+            operation.attempt,
+            operation.org_id,
+            operation.routing_domain,
+            operation.state,
+            (extract(epoch FROM operation.provider_created_at) * 1000)::bigint::text
+              AS provider_created_at_ms,
+            run.status AS run_status,
+            (extract(epoch FROM run.wait_deadline) * 1000)::bigint::text
+              AS wait_deadline_ms,
+            run.launch_json,
+            cleanup.status AS cleanup_status,
+            cleanup.run_id AS cleanup_run_id,
+            cleanup.attempt AS cleanup_attempt
+          FROM ${operations} AS operation
+          JOIN ${runs} AS run ON run.run_id = operation.run_id
+          LEFT JOIN ${cleanup} AS cleanup
+            ON cleanup.provider = operation.provider
+           AND cleanup.sandbox_id = operation.sandbox_id
+           AND cleanup.routing_domain IS NOT DISTINCT FROM operation.routing_domain
+          WHERE operation.provider = 'daytona'
+            AND operation.sandbox_id = $1
+          ORDER BY operation.routing_domain_key, operation.operation_key
+        `,
+        [sandboxId],
+      );
+      const exact = result.rows.filter(
+        (row) => row.routing_domain === routingDomain,
+      );
+      if (exact.length !== 1) {
+        const code: DaytonaJanitorOwnershipAlertCode =
+          exact.length > 1
+            ? 'ownership_ambiguous'
+            : result.rows.length === 0
+              ? 'ownership_missing'
+              : result.rows.some((row) => row.routing_domain === null)
+                ? 'legacy_unbound_ownership'
+                : 'routing_domain_conflict';
+        return {
+          kind: 'unresolved',
+          code,
+          candidateRunIds: result.rows.map((row) => row.run_id),
+          candidateRoutingDomains: result.rows.map((row) => row.routing_domain),
+        };
+      }
+      const row = exact[0]!;
+      if (
+        row.state !== 'live' &&
+        row.state !== 'deleting' &&
+        row.state !== 'deleted'
+      ) {
+        return {
+          kind: 'unresolved',
+          code: 'legacy_unbound_ownership',
+          candidateRunIds: [row.run_id],
+          candidateRoutingDomains: [row.routing_domain],
+        };
+      }
+      if (
+        row.cleanup_status !== null &&
+        (row.cleanup_run_id !== row.run_id ||
+          Number(row.cleanup_attempt) !== Number(row.attempt))
+      ) {
+        return {
+          kind: 'unresolved',
+          code: 'cleanup_ownership_conflict',
+          candidateRunIds: [row.run_id, row.cleanup_run_id ?? ''].filter(
+            Boolean,
+          ),
+          candidateRoutingDomains: [row.routing_domain],
+        };
+      }
+      return {
+        kind: 'owned',
+        operationKey: row.operation_key,
+        runId: row.run_id,
+        attempt: Number(row.attempt),
+        orgId: row.org_id,
+        operationState: row.state,
+        runStatus: row.run_status,
+        providerCreatedAtMs:
+          row.provider_created_at_ms === null
+            ? Number.NaN
+            : Number(row.provider_created_at_ms),
+        waitDeadlineMs:
+          row.wait_deadline_ms === null ? null : Number(row.wait_deadline_ms),
+        runtimeLimitSeconds: runtimeLimitSecondsFromLaunch(row.launch_json),
+        cleanupStatus: row.cleanup_status,
+      };
+    },
+    async authorizeCleanup({
+      operationKey,
+      sandboxId,
+      routingDomain,
+      runId,
+      attempt,
+    }) {
+      const result = await input.client.query<{
+        operation_state: string;
+        cleanup_status: string | null;
+        cleanup_run_id: string | null;
+        cleanup_attempt: number | null;
+        inserted: boolean;
+      }>(
+        `
+          WITH owned AS MATERIALIZED (
+            SELECT operation_key, provider, sandbox_id, routing_domain,
+                   run_id, attempt, environment, state
+            FROM ${operations}
+            WHERE operation_key = $1
+              AND provider = 'daytona'
+              AND sandbox_id = $2
+              AND routing_domain = $3
+              AND run_id = $4
+              AND attempt = $5
+            FOR UPDATE
+          ), inserted AS (
+            INSERT INTO ${cleanup} (
+              provider, sandbox_id, run_id, attempt, environment, routing_domain
+            )
+            SELECT provider, sandbox_id, run_id, attempt, environment, routing_domain
+            FROM owned
+            WHERE state IN ('live', 'deleting')
+            ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
+            RETURNING provider, sandbox_id, routing_domain,
+                      status, run_id, attempt
+          ), current_cleanup AS MATERIALIZED (
+            SELECT inserted.status, inserted.run_id, inserted.attempt
+            FROM inserted
+            UNION ALL
+            SELECT cleanup.status, cleanup.run_id, cleanup.attempt
+            FROM ${cleanup} AS cleanup
+            JOIN owned
+              ON cleanup.provider = owned.provider
+             AND cleanup.sandbox_id = owned.sandbox_id
+             AND cleanup.routing_domain IS NOT DISTINCT FROM owned.routing_domain
+            WHERE NOT EXISTS (SELECT 1 FROM inserted)
+          ), transitioned AS (
+            UPDATE ${operations} AS operation
+            SET state = 'deleting', updated_at = clock_timestamp()
+            FROM owned
+            WHERE operation.operation_key = owned.operation_key
+              AND operation.state = 'live'
+              AND EXISTS (
+                SELECT 1 FROM current_cleanup
+                WHERE current_cleanup.run_id = owned.run_id
+                  AND current_cleanup.attempt = owned.attempt
+                  AND current_cleanup.status <> 'deleted'
+              )
+            RETURNING operation.operation_key
+          )
+          SELECT
+            owned.state AS operation_state,
+            current_cleanup.status AS cleanup_status,
+            current_cleanup.run_id AS cleanup_run_id,
+            current_cleanup.attempt AS cleanup_attempt,
+            EXISTS (SELECT 1 FROM inserted) AS inserted
+          FROM owned
+          LEFT JOIN current_cleanup ON TRUE
+        `,
+        [operationKey, sandboxId, routingDomain, runId, attempt],
+      );
+      const row = result.rows[0];
+      if (!row) return 'ownership_missing';
+      if (
+        row.cleanup_status !== null &&
+        (row.cleanup_run_id !== runId ||
+          Number(row.cleanup_attempt) !== attempt)
+      ) {
+        return 'ownership_conflict';
+      }
+      if (
+        row.operation_state === 'deleted' ||
+        row.cleanup_status === 'deleted'
+      ) {
+        return 'already_deleted';
+      }
+      return row.inserted ? 'scheduled' : 'already_authorized';
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry.
+// ---------------------------------------------------------------------------
+
+export async function runDaytonaSandboxJanitorCli(argv: string[]): Promise<void> {
+  const dryRun = argv.includes('--dry-run');
+  // Escape hatch for deliberate debugging only: reaping against an isolated
+  // schema is unsafe because the `source`-scoped fleet is shared across lanes
+  // (see startDaytonaSandboxJanitor). Combine with --dry-run to inspect.
+  const allowIsolated = argv.includes('--allow-isolated-schema');
+
+  // Lazy imports so the pure core above stays free of heavy runtime deps and
+  // remains trivially unit-testable with fake ports.
+  const [
+    { Daytona },
+    { loadDaytonaRequiredConfig },
+    poolModule,
+    schemaModule,
+    { isIsolatedRuntimeSchedulerSchema },
+  ] = await Promise.all([
+    import('@daytonaio/sdk'),
+    import('.//daytona-runtime-config'),
+    import('.//scheduler-backends/postgres-pool'),
+    import('.//scheduler-backends/runtime-scheduler-schema'),
+    import('.//runtime-scheduler-topology'),
+  ]);
+
+  const url = poolModule.resolveWorkflowRuntimePostgresUrl();
+  if (!url) {
+    throw new Error(
+      'WORKFLOW_RUNTIME_POSTGRES is required for the Daytona sandbox janitor.',
+    );
+  }
+  const schema =
+    schemaModule.resolveRuntimeSchedulerSchemaEnv() || 'runtime_scheduler';
+
+  if (isIsolatedRuntimeSchedulerSchema(schema) && !allowIsolated) {
+    // The label scope is shared across every lane on this Daytona account, so a
+    // non-prod schema cannot authoritatively resolve the fleet. Refuse loudly
+    // rather than reap sandboxes owned by prod or another PR.
+    throw new Error(
+      `Refusing to reap on isolated schema "${schema}": the Daytona ` +
+        `source-label scope is shared across lanes, so runs absent from this ` +
+        `store may belong to prod or another PR. Pass --allow-isolated-schema ` +
+        `--dry-run only for deliberate debugging.`,
+    );
+  }
+
+  const { clientOptions } = loadDaytonaRequiredConfig();
+  const daytona = new Daytona(clientOptions);
+
+  const summary = await poolModule.withRuntimeSchedulerClient(
+    { url, applicationName: 'deepline-daytona-janitor' },
+    (client) =>
+      reapOrphanDaytonaSandboxes({
+        daytona: {
+          list: (query) => daytona.list(query),
+        },
+        ownership: createPostgresDaytonaJanitorOwnershipPort({
+          client,
+          schema,
+        }),
+        options: { dryRun },
+      }),
+  );
+
+  console.info('[daytona-janitor.cli_done]', { ...summary, schema, dryRun });
+}