deepline 0.2.0 → 0.2.2

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -0,0 +1,380 @@
+import type { ModalClient } from 'modal';
+import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
+import { RuntimeResourceFenceLostError } from '../types';
+import { buildPlayRunnerBundle } from '../bundle';
+import type {
+  PlayRunnerExecutionConfig,
+  PlayRunnerResult,
+  PlayRunnerRuntimeTiming,
+} from '@shared_libs/play-runtime/protocol';
+import {
+  PLAY_RUNNER_STARTUP_GRACE_SECONDS,
+  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+import {
+  MODAL_SANDBOX_CPU_CORES,
+  MODAL_SANDBOX_MEMORY_MIB,
+  loadModalClientConfig,
+  loadModalRequiredConfig,
+} from '@shared_libs/play-runtime/modal-runtime-config';
+import { validateDaytonaExecutionContext } from './daytona-lifecycle';
+import { stageRunnerPayload } from './daytona-payload-transport';
+import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
+
+const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
+type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
+
+function failed(
+  config: PlayRunnerExecutionConfig,
+  error: unknown,
+): PlayRunnerResult {
+  return {
+    status: 'failed',
+    error: error instanceof Error ? error.message : String(error),
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: config.checkpoint ?? null,
+    tableNamespace: null,
+  };
+}
+
+function emitModalStage(
+  context: PlayRunnerExecutionConfig['context'],
+  stage: string,
+  extra: Record<string, unknown> = {},
+): void {
+  console.info(
+    '[play-runner.modal.stage]',
+    JSON.stringify({
+      workflowId: context.workflowId ?? null,
+      runId: context.runId ?? null,
+      playName: context.playName ?? null,
+      stage,
+      ...extra,
+    }),
+  );
+}
+
+async function confirmDetachedModalRunnerReady(input: {
+  readiness: NonNullable<PlayRunnerCallbacks['readDetachedRunnerReadiness']>;
+  baselineHeartbeatAt: string | null;
+  cancellation?: Promise<never>;
+}): Promise<void> {
+  const cancellation = input.cancellation ?? new Promise<never>(() => {});
+  const deadline = Date.now() + MODAL_RUNNER_READY_TIMEOUT_MS;
+  let delayMs = 50;
+  while (Date.now() < deadline) {
+    const response = await Promise.race([
+      input.readiness(input.baselineHeartbeatAt).catch(() => null),
+      cancellation,
+    ]);
+    if (response?.ready) return;
+    await Promise.race([
+      new Promise<void>((resolve) => setTimeout(resolve, delayMs)),
+      cancellation,
+    ]);
+    delayMs = Math.min(1_000, delayMs * 2);
+  }
+  throw new Error(
+    `RUNTIME_SANDBOX_START_FAILED: Modal accepted the runner command, but the scheduler did not observe play-runner liveness within ${MODAL_RUNNER_READY_TIMEOUT_MS}ms. Customer play execution was not parked.`,
+  );
+}
+
+async function terminateModalSandbox(sandbox: ModalSandbox): Promise<void> {
+  await sandbox.terminate().catch((error) => {
+    console.warn('[play-runner.modal.terminate_failed]', {
+      sandboxId: sandbox.sandboxId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  });
+}
+
+export const modalPlayRunnerBackend: PlayRunnerBackend = {
+  async execute(config, callbacks) {
+    let runtimeResourceRegistrationError: unknown;
+    try {
+      validateDaytonaExecutionContext(config.context);
+      const push = config.context.runnerPushExecution;
+      if (!push) {
+        return failed(
+          config,
+          'Modal runner backend requires push-execution config (context.runnerPushExecution).',
+        );
+      }
+      const readRunnerReadiness = callbacks?.readDetachedRunnerReadiness;
+      if (!readRunnerReadiness) {
+        return failed(
+          config,
+          'Modal runner backend requires the scheduler-owned detached-runner readiness port.',
+        );
+      }
+
+      const startedAt = Date.now();
+      const runtimeTiming: PlayRunnerRuntimeTiming = { backend: 'modal' };
+      emitModalStage(config.context, 'create:start');
+      const modalConfig = await loadModalRequiredConfig({
+        runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
+        limits: config.context.sandboxRuntimeLimits,
+      });
+      const app = await modalConfig.client.apps.fromName(modalConfig.appName, {
+        createIfMissing: true,
+      });
+      const image = modalConfig.client.images.fromRegistry(modalConfig.image);
+      const sandbox = await modalConfig.client.sandboxes.create(app, image, {
+        cpu: MODAL_SANDBOX_CPU_CORES,
+        cpuLimit: MODAL_SANDBOX_CPU_CORES,
+        memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
+        memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
+        timeoutMs:
+          (modalConfig.limits.timeoutSeconds +
+            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
+            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+          1_000,
+        idleTimeoutMs:
+          (modalConfig.limits.timeoutSeconds +
+            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
+            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+          1_000,
+        workdir: modalConfig.workdir,
+        tags: {
+          source: 'deepline-play-runner',
+          orgId: config.context.orgId ?? 'unknown',
+          workflowId: config.context.workflowId ?? 'unknown',
+          runId: config.context.runId ?? 'unknown',
+        },
+        ...(modalConfig.outboundCidrAllowlist
+          ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
+          : {}),
+      });
+      const billingStartedAt = Date.now();
+      runtimeTiming.modalCreateMs = billingStartedAt - startedAt;
+      emitModalStage(config.context, 'create:done', {
+        sandboxId: sandbox.sandboxId,
+        elapsedMs: runtimeTiming.modalCreateMs,
+      });
+      let detached = false;
+      let cancelExecution!: (error: Error) => void;
+      const cancellationPromise = new Promise<never>((_resolve, reject) => {
+        cancelExecution = reject;
+      });
+      const onCancel = () =>
+        cancelExecution(new Error('Modal play runner cancelled'));
+      callbacks?.cancellationSignal?.addEventListener('abort', onCancel, {
+        once: true,
+      });
+
+      try {
+        if (callbacks?.cancellationSignal?.aborted) onCancel();
+        try {
+          await callbacks?.onRuntimeResourceAcquired?.({
+            kind: 'modal_sandbox',
+            sandboxId: sandbox.sandboxId,
+            runtimeEnvironment:
+              process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+                ? 'preview'
+                : 'production',
+            modalAppId: app.appId,
+            billingStartedAt,
+            maxBillingDurationSeconds:
+              modalConfig.limits.timeoutSeconds +
+              PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+            cpu: MODAL_SANDBOX_CPU_CORES,
+            memoryGiB: MODAL_SANDBOX_MEMORY_MIB / 1024,
+            diskGiB: 0,
+          });
+        } catch (error) {
+          runtimeResourceRegistrationError = error;
+          throw error;
+        }
+
+        const uploadStartedAt = Date.now();
+        const payload = await Promise.race([
+          stageRunnerPayload({
+            sandbox: {
+              id: sandbox.sandboxId,
+              uploadFile: (content, path) =>
+                sandbox.filesystem.writeBytes(content, path),
+            },
+            bundlePromise: buildPlayRunnerBundle(),
+            config,
+            workDir: modalConfig.workdir,
+            startedAt,
+            emitStage: (stage, extra) =>
+              emitModalStage(config.context, stage, extra),
+          }),
+          cancellationPromise,
+        ]);
+        runtimeTiming.modalUploadMs = Date.now() - uploadStartedAt;
+        const baselineHeartbeatAt = await Promise.race([
+          captureDetachedDaytonaRunnerReadinessBaseline({
+            readiness: readRunnerReadiness,
+          }),
+          cancellationPromise,
+        ]);
+
+        emitModalStage(config.context, 'execute:start', {
+          sandboxId: sandbox.sandboxId,
+          timeoutSeconds: modalConfig.limits.timeoutSeconds,
+          mode: 'detached',
+        });
+        const executeStartedAt = Date.now();
+        await Promise.race([
+          sandbox.exec(['bash', '-lc', payload.command]),
+          cancellationPromise,
+        ]);
+        runtimeTiming.modalExecuteMs = Date.now() - executeStartedAt;
+        await confirmDetachedModalRunnerReady({
+          readiness: readRunnerReadiness,
+          baselineHeartbeatAt,
+          cancellation: cancellationPromise,
+        });
+        if (callbacks?.cancellationSignal?.aborted) {
+          throw new Error('Modal play runner cancelled');
+        }
+
+        detached = true;
+        sandbox.detach();
+        const runnerAttempt = Math.max(
+          0,
+          Math.floor(config.context.runAttempt ?? 0),
+        );
+        emitModalStage(config.context, 'execute:detached', {
+          sandboxId: sandbox.sandboxId,
+          runnerAttempt,
+          elapsedMs: Date.now() - startedAt,
+        });
+        return {
+          status: 'suspended',
+          suspension: {
+            kind: 'detached_runner',
+            boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
+            runnerAttempt,
+            sandboxProvider: 'modal',
+            runtimeSandboxRef: {
+              schemaVersion: 1,
+              provider: 'modal',
+              resourceId: sandbox.sandboxId,
+              routingDomain: app.appId,
+            },
+            sandboxId: sandbox.sandboxId,
+            sessionId: 'modal',
+            cmdId: sandbox.sandboxId,
+            outputPath: payload.outputPath,
+            exitCodePath: payload.exitCodePath,
+            runtimeCompletedPath: payload.runtimeCompletedPath,
+            startedAtMs: Date.now(),
+            heartbeatTimeoutMs: push.leaseSeconds * 1_000,
+            ceilingMs:
+              (modalConfig.limits.timeoutSeconds +
+                PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+              1_000,
+          },
+          logs: [],
+          stats: {},
+          steps: [],
+          checkpoint: config.checkpoint ?? {
+            completedBatches: {},
+            completedToolBatches: {},
+            resolvedWaterfalls: {},
+            resolvedBoundaries: {},
+          },
+          tableNamespace: null,
+          runtimeTiming,
+        };
+      } finally {
+        callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
+        if (!detached) await terminateModalSandbox(sandbox);
+      }
+    } catch (error) {
+      if (
+        error === runtimeResourceRegistrationError ||
+        error instanceof RuntimeResourceFenceLostError
+      ) {
+        throw error;
+      }
+      emitModalStage(config.context, 'execute:error', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return failed(config, error);
+    }
+  },
+};
+
+export async function deleteModalSandboxById(input: {
+  sandboxId: string;
+  expectedAppId?: string | null;
+}): Promise<
+  | { kind: 'deleted' | 'already_absent'; appId: string }
+  | { kind: 'failed'; appId: string | null; code: string; detail: string }
+> {
+  const expectedAppId = input.expectedAppId?.trim() || null;
+  if (!expectedAppId) {
+    console.warn('[play-runner.modal.reclaim_sandbox_delete_blocked]', {
+      sandboxId: input.sandboxId,
+      reason: 'missing_routing_domain',
+    });
+    return {
+      kind: 'failed',
+      appId: null,
+      code: 'missing_routing_domain',
+      detail: 'Modal sandbox cleanup requires its persisted app id.',
+    };
+  }
+  try {
+    const { client } = await loadModalClientConfig();
+    for await (const sandbox of client.sandboxes.list({
+      appId: expectedAppId,
+    })) {
+      if (sandbox.sandboxId !== input.sandboxId) continue;
+      await sandbox.terminate();
+      return { kind: 'deleted', appId: expectedAppId };
+    }
+    // Cleanup means ensure absent. An exact app-scoped inventory proving the
+    // id is absent satisfies the obligation without an unsafe cross-app get.
+    return { kind: 'already_absent', appId: expectedAppId };
+  } catch (error) {
+    console.warn('[play-runner.modal.reclaim_sandbox_delete_failed]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return {
+      kind: 'failed',
+      appId: expectedAppId,
+      code: 'provider_delete_failed',
+      detail: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export async function readDetachedModalRuntimeCompletion(input: {
+  sandboxId: string;
+  runtimeCompletedPath: string;
+  expectedAppId?: string | null;
+}): Promise<number | null> {
+  const expectedAppId = input.expectedAppId?.trim() || null;
+  if (!expectedAppId) return null;
+  try {
+    const { client } = await loadModalClientConfig();
+    for await (const sandbox of client.sandboxes.list({
+      appId: expectedAppId,
+    })) {
+      if (sandbox.sandboxId !== input.sandboxId) continue;
+      const marker = JSON.parse(
+        await sandbox.filesystem.readText(input.runtimeCompletedPath),
+      ) as { at?: unknown };
+      return typeof marker.at === 'number' && Number.isFinite(marker.at)
+        ? marker.at
+        : null;
+    }
+    return null;
+  } catch (error) {
+    console.warn('[play-runner.modal.runtime_completion_marker_unavailable]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return null;
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts

@@ -1,11 +1,28 @@
 import type { PlayRunnerBackend } from './types';
 import { daytonaPlayRunnerBackend } from './backends/daytona';
+import {
+  daytonaModalFallbackPlayRunnerBackend,
+  daytonaOnlyPlayRunnerBackend,
+  modalOnlyPlayRunnerBackend,
+  resolveDefaultRuntimeSandboxPlacementBackend,
+} from './backends/daytona-modal-fallback';
 import { localProcessPlayRunnerBackend } from './backends/local-process';
+import { modalPlayRunnerBackend } from './backends/modal';
 import {
   PLAY_RUNTIME_BACKENDS,
   normalizePlayRuntimeBackend,
   type PlayRuntimeBackendId,
 } from '@shared_libs/play-runtime/backend';
+import {
+  runtimeSandboxPlacementPolicyForBackend,
+  type RuntimeSandboxPlacementPolicyId,
+} from '@shared_libs/play-runtime/runtime-sandbox-placement-policy';
+
+export function resolveRuntimeSandboxPlacementBackend(
+  policyId: RuntimeSandboxPlacementPolicyId | string,
+): PlayRunnerBackend {
+  return resolveDefaultRuntimeSandboxPlacementBackend(policyId);
+}
 
 export function resolvePlayRunnerBackend(
   backendId?: PlayRuntimeBackendId | string | null,
@@ -13,8 +30,9 @@ export function resolvePlayRunnerBackend(
   const backend = normalizePlayRuntimeBackend(
     backendId ?? process.env.DEEPLINE_PLAY_RUNNER_BACKEND,
   );
-  if (backend === PLAY_RUNTIME_BACKENDS.daytona) {
-    return daytonaPlayRunnerBackend;
+  const placementPolicy = runtimeSandboxPlacementPolicyForBackend(backend);
+  if (placementPolicy) {
+    return resolveRuntimeSandboxPlacementBackend(placementPolicy.id);
   }
   if (backend === PLAY_RUNTIME_BACKENDS.localProcess) {
     return localProcessPlayRunnerBackend;
@@ -22,5 +40,12 @@ export function resolvePlayRunnerBackend(
   throw new Error(`Unsupported play runner backend: ${backend}`);
 }
 
-export { daytonaPlayRunnerBackend, localProcessPlayRunnerBackend };
+export {
+  daytonaPlayRunnerBackend,
+  daytonaModalFallbackPlayRunnerBackend,
+  daytonaOnlyPlayRunnerBackend,
+  localProcessPlayRunnerBackend,
+  modalOnlyPlayRunnerBackend,
+  modalPlayRunnerBackend,
+};
 export type { PlayRunnerBackend };

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts

@@ -0,0 +1,240 @@
+import {
+  deleteDaytonaSandboxByIdWithOutcome,
+  readDetachedDaytonaRuntimeCompletion,
+} from './backends/daytona';
+import {
+  deleteModalSandboxById,
+  readDetachedModalRuntimeCompletion,
+} from './backends/modal';
+import type { DetachedRunnerSuspension } from '@shared_libs/play-runtime/suspension';
+
+export type RuntimeSandboxProviderId = 'daytona' | 'modal';
+
+export type RuntimeSandboxRef = {
+  schemaVersion: 1;
+  provider: RuntimeSandboxProviderId;
+  resourceId: string;
+  routingDomain: string | null;
+};
+
+export type RuntimeSandboxReconciliationAdapter = {
+  provider: RuntimeSandboxProviderId;
+  delete(ref: RuntimeSandboxRef): Promise<RuntimeSandboxDeleteOutcome>;
+  readWorkCompletedAt(
+    ref: RuntimeSandboxRef,
+    path: string,
+  ): Promise<number | null>;
+};
+
+export type RuntimeSandboxReconciliation = {
+  delete(ref: RuntimeSandboxRef): Promise<RuntimeSandboxDeleteOutcome>;
+  readWorkCompletedAt(
+    ref: RuntimeSandboxRef,
+    path: string,
+  ): Promise<number | null>;
+};
+
+export type RuntimeSandboxDeleteOutcome =
+  | {
+      kind: 'deleted' | 'already_absent';
+      routingDomain: string | null;
+    }
+  | {
+      kind: 'timed_out' | 'rate_limited' | 'failed';
+      routingDomain: string | null;
+      code: string;
+      detail: string;
+    };
+
+export function runtimeSandboxDeleteSucceeded(
+  outcome: RuntimeSandboxDeleteOutcome,
+): outcome is Extract<
+  RuntimeSandboxDeleteOutcome,
+  { kind: 'deleted' | 'already_absent' }
+> {
+  return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
+}
+
+/** Stable identity for deduplication and task idempotency at provider seams. */
+export function runtimeSandboxRefIdentityKey(ref: RuntimeSandboxRef): string {
+  return JSON.stringify([ref.provider, ref.resourceId, ref.routingDomain]);
+}
+
+function registry(
+  adapters: readonly RuntimeSandboxReconciliationAdapter[],
+): Map<RuntimeSandboxProviderId, RuntimeSandboxReconciliationAdapter> {
+  const result = new Map<
+    RuntimeSandboxProviderId,
+    RuntimeSandboxReconciliationAdapter
+  >();
+  for (const adapter of adapters) {
+    if (result.has(adapter.provider)) {
+      throw new Error(
+        `Duplicate Runtime Sandbox Adapter: ${adapter.provider}.`,
+      );
+    }
+    result.set(adapter.provider, adapter);
+  }
+  return result;
+}
+
+export function createRuntimeSandboxReconciliation(input: {
+  adapters: readonly RuntimeSandboxReconciliationAdapter[];
+}): RuntimeSandboxReconciliation {
+  const adapters = registry(input.adapters);
+  const resolve = (ref: RuntimeSandboxRef) => {
+    const adapter = adapters.get(ref.provider);
+    if (!adapter) {
+      throw new Error(
+        `No Runtime Sandbox Adapter is registered for ${ref.provider}.`,
+      );
+    }
+    return adapter;
+  };
+  return {
+    delete: (ref) => resolve(ref).delete(ref),
+    readWorkCompletedAt: (ref, path) =>
+      resolve(ref).readWorkCompletedAt(ref, path),
+  };
+}
+
+export function runtimeSandboxRefFromResource(
+  value: unknown,
+): RuntimeSandboxRef | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const resource = value as Record<string, unknown>;
+  const resourceId =
+    typeof resource.sandboxId === 'string' ? resource.sandboxId.trim() : '';
+  if (!resourceId) return null;
+  if (resource.kind === 'daytona_sandbox') {
+    return {
+      schemaVersion: 1,
+      provider: 'daytona',
+      resourceId,
+      routingDomain:
+        typeof resource.daytonaOrganizationId === 'string' &&
+        resource.daytonaOrganizationId.trim()
+          ? resource.daytonaOrganizationId.trim()
+          : null,
+    };
+  }
+  if (resource.kind === 'modal_sandbox') {
+    return {
+      schemaVersion: 1,
+      provider: 'modal',
+      resourceId,
+      routingDomain:
+        typeof resource.modalAppId === 'string' && resource.modalAppId.trim()
+          ? resource.modalAppId.trim()
+          : null,
+    };
+  }
+  return null;
+}
+
+export function runtimeSandboxRefFromSuspension(
+  suspension: DetachedRunnerSuspension,
+): RuntimeSandboxRef {
+  const storedRef = (suspension as { runtimeSandboxRef?: unknown })
+    .runtimeSandboxRef;
+  if (storedRef !== undefined) {
+    if (
+      !storedRef ||
+      typeof storedRef !== 'object' ||
+      Array.isArray(storedRef)
+    ) {
+      throw new Error('Persisted Runtime Sandbox Ref is not an object.');
+    }
+    const candidate = storedRef as Record<string, unknown>;
+    const resourceId =
+      typeof candidate.resourceId === 'string'
+        ? candidate.resourceId.trim()
+        : '';
+    const provider = candidate.provider;
+    const routingDomain = candidate.routingDomain;
+    if (
+      candidate.schemaVersion !== 1 ||
+      (provider !== 'daytona' && provider !== 'modal') ||
+      !resourceId ||
+      resourceId !== suspension.sandboxId.trim() ||
+      (routingDomain !== null && typeof routingDomain !== 'string') ||
+      (typeof routingDomain === 'string' && !routingDomain.trim()) ||
+      (suspension.sandboxProvider !== undefined &&
+        suspension.sandboxProvider !== provider)
+    ) {
+      throw new Error(
+        `Persisted Runtime Sandbox Ref is invalid or inconsistent for sandbox ${suspension.sandboxId}.`,
+      );
+    }
+    return {
+      schemaVersion: 1,
+      provider,
+      resourceId,
+      routingDomain:
+        typeof routingDomain === 'string' ? routingDomain.trim() : null,
+    };
+  }
+  if (
+    suspension.sandboxProvider !== undefined &&
+    suspension.sandboxProvider !== 'daytona' &&
+    suspension.sandboxProvider !== 'modal'
+  ) {
+    throw new Error(
+      `Legacy detached runner has unsupported sandbox provider ${String(suspension.sandboxProvider)}.`,
+    );
+  }
+  return {
+    schemaVersion: 1,
+    provider: suspension.sandboxProvider === 'modal' ? 'modal' : 'daytona',
+    resourceId: suspension.sandboxId,
+    // Legacy suspensions never persisted ownership. Cleanup still uses the
+    // provider-scoped credential; resource-backed cleanup carries the domain.
+    routingDomain: null,
+  };
+}
+
+export const defaultRuntimeSandboxReconciliation =
+  createRuntimeSandboxReconciliation({
+    adapters: [
+      {
+        provider: 'daytona',
+        delete: async (ref) => {
+          const outcome = await deleteDaytonaSandboxByIdWithOutcome({
+            sandboxId: ref.resourceId,
+            timeoutSeconds: 30,
+            expectedOrganizationId: ref.routingDomain,
+            allowUnscopedAlreadyAbsent: ref.routingDomain === null,
+          });
+          return {
+            ...outcome,
+            routingDomain: outcome.organizationId,
+          };
+        },
+        readWorkCompletedAt: (ref, path) =>
+          readDetachedDaytonaRuntimeCompletion({
+            sandboxId: ref.resourceId,
+            runtimeCompletedPath: path,
+            expectedOrganizationId: ref.routingDomain,
+          }),
+      },
+      {
+        provider: 'modal',
+        delete: async (ref) => {
+          const outcome = await deleteModalSandboxById({
+            sandboxId: ref.resourceId,
+            expectedAppId: ref.routingDomain,
+          });
+          return {
+            ...outcome,
+            routingDomain: outcome.appId,
+          };
+        },
+        readWorkCompletedAt: (ref, path) =>
+          readDetachedModalRuntimeCompletion({
+            sandboxId: ref.resourceId,
+            runtimeCompletedPath: path,
+            expectedAppId: ref.routingDomain,
+          }),
+      },
+    ],
+  });