deepline 0.2.0 → 0.2.1

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -24,6 +24,7 @@ import {
 } from '@shared_libs/play-runtime/daytona-runtime-config';
 import {
   DAYTONA_CANCELLED_ERROR,
+  DaytonaSandboxAcquisitionUnavailableError,
   createDaytonaSandboxCleanupManager,
   createOneShotDaytonaSandboxLifecycle,
   type DaytonaClient,
@@ -290,12 +291,29 @@ export async function inspectDetachedDaytonaRunner(input: {
 export async function readDetachedDaytonaRuntimeCompletion(input: {
   sandboxId: string;
   runtimeCompletedPath: string;
+  expectedOrganizationId?: string | null;
 }): Promise<number | null> {
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
     const sandbox = (await daytonaSdkClientFactory
       .createFull(clientOptions)
       .get(input.sandboxId)) as DaytonaSandbox;
+    const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
+    const observedOrganizationId = sandbox.organizationId?.trim() || null;
+    if (
+      expectedOrganizationId &&
+      observedOrganizationId !== expectedOrganizationId
+    ) {
+      console.warn(
+        '[play-runner.daytona.runtime_completion_wrong_routing_domain]',
+        {
+          sandboxId: input.sandboxId,
+          expectedOrganizationId,
+          observedOrganizationId,
+        },
+      );
+      return null;
+    }
     const marker = JSON.parse(
       (await sandbox.fs.downloadFile(input.runtimeCompletedPath, 5)).toString(
         'utf-8',
@@ -790,6 +808,10 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
         await callbacks?.onRuntimeResourceAcquired?.({
           kind: 'daytona_sandbox',
           sandboxId: sandbox.id,
+          runtimeEnvironment:
+            process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+              ? 'preview'
+              : 'production',
           daytonaEnvironment:
             process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
               ? 'preview'
@@ -1031,6 +1053,14 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               kind: 'detached_runner',
               boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
               runnerAttempt,
+              sandboxProvider: 'daytona',
+              runtimeSandboxRef: {
+                schemaVersion: 1,
+                provider: 'daytona',
+                resourceId: sandbox.id,
+                routingDomain:
+                  activeAcquiredResource?.daytonaOrganizationId ?? null,
+              },
               sandboxId: sandbox.id,
               sessionId: start.sessionId,
               cmdId: start.cmdId,
@@ -1121,6 +1151,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
         }
         throw error;
       }
+      if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
+        throw error;
+      }
       emitDaytonaStage(callbacks, config.context, 'execute:error', {
         sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
         error: formatDaytonaError(error),

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -0,0 +1,380 @@
+import type { ModalClient } from 'modal';
+import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
+import { RuntimeResourceFenceLostError } from '../types';
+import { buildPlayRunnerBundle } from '../bundle';
+import type {
+  PlayRunnerExecutionConfig,
+  PlayRunnerResult,
+  PlayRunnerRuntimeTiming,
+} from '@shared_libs/play-runtime/protocol';
+import {
+  PLAY_RUNNER_STARTUP_GRACE_SECONDS,
+  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+import {
+  MODAL_SANDBOX_CPU_CORES,
+  MODAL_SANDBOX_MEMORY_MIB,
+  loadModalClientConfig,
+  loadModalRequiredConfig,
+} from '@shared_libs/play-runtime/modal-runtime-config';
+import { validateDaytonaExecutionContext } from './daytona-lifecycle';
+import { stageRunnerPayload } from './daytona-payload-transport';
+import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
+
+const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
+type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
+
+function failed(
+  config: PlayRunnerExecutionConfig,
+  error: unknown,
+): PlayRunnerResult {
+  return {
+    status: 'failed',
+    error: error instanceof Error ? error.message : String(error),
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: config.checkpoint ?? null,
+    tableNamespace: null,
+  };
+}
+
+function emitModalStage(
+  context: PlayRunnerExecutionConfig['context'],
+  stage: string,
+  extra: Record<string, unknown> = {},
+): void {
+  console.info(
+    '[play-runner.modal.stage]',
+    JSON.stringify({
+      workflowId: context.workflowId ?? null,
+      runId: context.runId ?? null,
+      playName: context.playName ?? null,
+      stage,
+      ...extra,
+    }),
+  );
+}
+
+async function confirmDetachedModalRunnerReady(input: {
+  readiness: NonNullable<PlayRunnerCallbacks['readDetachedRunnerReadiness']>;
+  baselineHeartbeatAt: string | null;
+  cancellation?: Promise<never>;
+}): Promise<void> {
+  const cancellation = input.cancellation ?? new Promise<never>(() => {});
+  const deadline = Date.now() + MODAL_RUNNER_READY_TIMEOUT_MS;
+  let delayMs = 50;
+  while (Date.now() < deadline) {
+    const response = await Promise.race([
+      input.readiness(input.baselineHeartbeatAt).catch(() => null),
+      cancellation,
+    ]);
+    if (response?.ready) return;
+    await Promise.race([
+      new Promise<void>((resolve) => setTimeout(resolve, delayMs)),
+      cancellation,
+    ]);
+    delayMs = Math.min(1_000, delayMs * 2);
+  }
+  throw new Error(
+    `RUNTIME_SANDBOX_START_FAILED: Modal accepted the runner command, but the scheduler did not observe play-runner liveness within ${MODAL_RUNNER_READY_TIMEOUT_MS}ms. Customer play execution was not parked.`,
+  );
+}
+
+async function terminateModalSandbox(sandbox: ModalSandbox): Promise<void> {
+  await sandbox.terminate().catch((error) => {
+    console.warn('[play-runner.modal.terminate_failed]', {
+      sandboxId: sandbox.sandboxId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  });
+}
+
+export const modalPlayRunnerBackend: PlayRunnerBackend = {
+  async execute(config, callbacks) {
+    let runtimeResourceRegistrationError: unknown;
+    try {
+      validateDaytonaExecutionContext(config.context);
+      const push = config.context.runnerPushExecution;
+      if (!push) {
+        return failed(
+          config,
+          'Modal runner backend requires push-execution config (context.runnerPushExecution).',
+        );
+      }
+      const readRunnerReadiness = callbacks?.readDetachedRunnerReadiness;
+      if (!readRunnerReadiness) {
+        return failed(
+          config,
+          'Modal runner backend requires the scheduler-owned detached-runner readiness port.',
+        );
+      }
+
+      const startedAt = Date.now();
+      const runtimeTiming: PlayRunnerRuntimeTiming = { backend: 'modal' };
+      emitModalStage(config.context, 'create:start');
+      const modalConfig = await loadModalRequiredConfig({
+        runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
+        limits: config.context.sandboxRuntimeLimits,
+      });
+      const app = await modalConfig.client.apps.fromName(modalConfig.appName, {
+        createIfMissing: true,
+      });
+      const image = modalConfig.client.images.fromRegistry(modalConfig.image);
+      const sandbox = await modalConfig.client.sandboxes.create(app, image, {
+        cpu: MODAL_SANDBOX_CPU_CORES,
+        cpuLimit: MODAL_SANDBOX_CPU_CORES,
+        memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
+        memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
+        timeoutMs:
+          (modalConfig.limits.timeoutSeconds +
+            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
+            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+          1_000,
+        idleTimeoutMs:
+          (modalConfig.limits.timeoutSeconds +
+            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
+            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+          1_000,
+        workdir: modalConfig.workdir,
+        tags: {
+          source: 'deepline-play-runner',
+          orgId: config.context.orgId ?? 'unknown',
+          workflowId: config.context.workflowId ?? 'unknown',
+          runId: config.context.runId ?? 'unknown',
+        },
+        ...(modalConfig.outboundCidrAllowlist
+          ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
+          : {}),
+      });
+      const billingStartedAt = Date.now();
+      runtimeTiming.modalCreateMs = billingStartedAt - startedAt;
+      emitModalStage(config.context, 'create:done', {
+        sandboxId: sandbox.sandboxId,
+        elapsedMs: runtimeTiming.modalCreateMs,
+      });
+      let detached = false;
+      let cancelExecution!: (error: Error) => void;
+      const cancellationPromise = new Promise<never>((_resolve, reject) => {
+        cancelExecution = reject;
+      });
+      const onCancel = () =>
+        cancelExecution(new Error('Modal play runner cancelled'));
+      callbacks?.cancellationSignal?.addEventListener('abort', onCancel, {
+        once: true,
+      });
+
+      try {
+        if (callbacks?.cancellationSignal?.aborted) onCancel();
+        try {
+          await callbacks?.onRuntimeResourceAcquired?.({
+            kind: 'modal_sandbox',
+            sandboxId: sandbox.sandboxId,
+            runtimeEnvironment:
+              process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+                ? 'preview'
+                : 'production',
+            modalAppId: app.appId,
+            billingStartedAt,
+            maxBillingDurationSeconds:
+              modalConfig.limits.timeoutSeconds +
+              PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+            cpu: MODAL_SANDBOX_CPU_CORES,
+            memoryGiB: MODAL_SANDBOX_MEMORY_MIB / 1024,
+            diskGiB: 0,
+          });
+        } catch (error) {
+          runtimeResourceRegistrationError = error;
+          throw error;
+        }
+
+        const uploadStartedAt = Date.now();
+        const payload = await Promise.race([
+          stageRunnerPayload({
+            sandbox: {
+              id: sandbox.sandboxId,
+              uploadFile: (content, path) =>
+                sandbox.filesystem.writeBytes(content, path),
+            },
+            bundlePromise: buildPlayRunnerBundle(),
+            config,
+            workDir: modalConfig.workdir,
+            startedAt,
+            emitStage: (stage, extra) =>
+              emitModalStage(config.context, stage, extra),
+          }),
+          cancellationPromise,
+        ]);
+        runtimeTiming.modalUploadMs = Date.now() - uploadStartedAt;
+        const baselineHeartbeatAt = await Promise.race([
+          captureDetachedDaytonaRunnerReadinessBaseline({
+            readiness: readRunnerReadiness,
+          }),
+          cancellationPromise,
+        ]);
+
+        emitModalStage(config.context, 'execute:start', {
+          sandboxId: sandbox.sandboxId,
+          timeoutSeconds: modalConfig.limits.timeoutSeconds,
+          mode: 'detached',
+        });
+        const executeStartedAt = Date.now();
+        await Promise.race([
+          sandbox.exec(['bash', '-lc', payload.command]),
+          cancellationPromise,
+        ]);
+        runtimeTiming.modalExecuteMs = Date.now() - executeStartedAt;
+        await confirmDetachedModalRunnerReady({
+          readiness: readRunnerReadiness,
+          baselineHeartbeatAt,
+          cancellation: cancellationPromise,
+        });
+        if (callbacks?.cancellationSignal?.aborted) {
+          throw new Error('Modal play runner cancelled');
+        }
+
+        detached = true;
+        sandbox.detach();
+        const runnerAttempt = Math.max(
+          0,
+          Math.floor(config.context.runAttempt ?? 0),
+        );
+        emitModalStage(config.context, 'execute:detached', {
+          sandboxId: sandbox.sandboxId,
+          runnerAttempt,
+          elapsedMs: Date.now() - startedAt,
+        });
+        return {
+          status: 'suspended',
+          suspension: {
+            kind: 'detached_runner',
+            boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
+            runnerAttempt,
+            sandboxProvider: 'modal',
+            runtimeSandboxRef: {
+              schemaVersion: 1,
+              provider: 'modal',
+              resourceId: sandbox.sandboxId,
+              routingDomain: app.appId,
+            },
+            sandboxId: sandbox.sandboxId,
+            sessionId: 'modal',
+            cmdId: sandbox.sandboxId,
+            outputPath: payload.outputPath,
+            exitCodePath: payload.exitCodePath,
+            runtimeCompletedPath: payload.runtimeCompletedPath,
+            startedAtMs: Date.now(),
+            heartbeatTimeoutMs: push.leaseSeconds * 1_000,
+            ceilingMs:
+              (modalConfig.limits.timeoutSeconds +
+                PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+              1_000,
+          },
+          logs: [],
+          stats: {},
+          steps: [],
+          checkpoint: config.checkpoint ?? {
+            completedBatches: {},
+            completedToolBatches: {},
+            resolvedWaterfalls: {},
+            resolvedBoundaries: {},
+          },
+          tableNamespace: null,
+          runtimeTiming,
+        };
+      } finally {
+        callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
+        if (!detached) await terminateModalSandbox(sandbox);
+      }
+    } catch (error) {
+      if (
+        error === runtimeResourceRegistrationError ||
+        error instanceof RuntimeResourceFenceLostError
+      ) {
+        throw error;
+      }
+      emitModalStage(config.context, 'execute:error', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return failed(config, error);
+    }
+  },
+};
+
+export async function deleteModalSandboxById(input: {
+  sandboxId: string;
+  expectedAppId?: string | null;
+}): Promise<
+  | { kind: 'deleted' | 'already_absent'; appId: string }
+  | { kind: 'failed'; appId: string | null; code: string; detail: string }
+> {
+  const expectedAppId = input.expectedAppId?.trim() || null;
+  if (!expectedAppId) {
+    console.warn('[play-runner.modal.reclaim_sandbox_delete_blocked]', {
+      sandboxId: input.sandboxId,
+      reason: 'missing_routing_domain',
+    });
+    return {
+      kind: 'failed',
+      appId: null,
+      code: 'missing_routing_domain',
+      detail: 'Modal sandbox cleanup requires its persisted app id.',
+    };
+  }
+  try {
+    const { client } = await loadModalClientConfig();
+    for await (const sandbox of client.sandboxes.list({
+      appId: expectedAppId,
+    })) {
+      if (sandbox.sandboxId !== input.sandboxId) continue;
+      await sandbox.terminate();
+      return { kind: 'deleted', appId: expectedAppId };
+    }
+    // Cleanup means ensure absent. An exact app-scoped inventory proving the
+    // id is absent satisfies the obligation without an unsafe cross-app get.
+    return { kind: 'already_absent', appId: expectedAppId };
+  } catch (error) {
+    console.warn('[play-runner.modal.reclaim_sandbox_delete_failed]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return {
+      kind: 'failed',
+      appId: expectedAppId,
+      code: 'provider_delete_failed',
+      detail: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export async function readDetachedModalRuntimeCompletion(input: {
+  sandboxId: string;
+  runtimeCompletedPath: string;
+  expectedAppId?: string | null;
+}): Promise<number | null> {
+  const expectedAppId = input.expectedAppId?.trim() || null;
+  if (!expectedAppId) return null;
+  try {
+    const { client } = await loadModalClientConfig();
+    for await (const sandbox of client.sandboxes.list({
+      appId: expectedAppId,
+    })) {
+      if (sandbox.sandboxId !== input.sandboxId) continue;
+      const marker = JSON.parse(
+        await sandbox.filesystem.readText(input.runtimeCompletedPath),
+      ) as { at?: unknown };
+      return typeof marker.at === 'number' && Number.isFinite(marker.at)
+        ? marker.at
+        : null;
+    }
+    return null;
+  } catch (error) {
+    console.warn('[play-runner.modal.runtime_completion_marker_unavailable]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return null;
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts

@@ -1,11 +1,28 @@
 import type { PlayRunnerBackend } from './types';
 import { daytonaPlayRunnerBackend } from './backends/daytona';
+import {
+  daytonaModalFallbackPlayRunnerBackend,
+  daytonaOnlyPlayRunnerBackend,
+  modalOnlyPlayRunnerBackend,
+  resolveDefaultRuntimeSandboxPlacementBackend,
+} from './backends/daytona-modal-fallback';
 import { localProcessPlayRunnerBackend } from './backends/local-process';
+import { modalPlayRunnerBackend } from './backends/modal';
 import {
   PLAY_RUNTIME_BACKENDS,
   normalizePlayRuntimeBackend,
   type PlayRuntimeBackendId,
 } from '@shared_libs/play-runtime/backend';
+import {
+  runtimeSandboxPlacementPolicyForBackend,
+  type RuntimeSandboxPlacementPolicyId,
+} from '@shared_libs/play-runtime/runtime-sandbox-placement-policy';
+
+export function resolveRuntimeSandboxPlacementBackend(
+  policyId: RuntimeSandboxPlacementPolicyId | string,
+): PlayRunnerBackend {
+  return resolveDefaultRuntimeSandboxPlacementBackend(policyId);
+}
 
 export function resolvePlayRunnerBackend(
   backendId?: PlayRuntimeBackendId | string | null,
@@ -13,8 +30,9 @@ export function resolvePlayRunnerBackend(
   const backend = normalizePlayRuntimeBackend(
     backendId ?? process.env.DEEPLINE_PLAY_RUNNER_BACKEND,
   );
-  if (backend === PLAY_RUNTIME_BACKENDS.daytona) {
-    return daytonaPlayRunnerBackend;
+  const placementPolicy = runtimeSandboxPlacementPolicyForBackend(backend);
+  if (placementPolicy) {
+    return resolveRuntimeSandboxPlacementBackend(placementPolicy.id);
   }
   if (backend === PLAY_RUNTIME_BACKENDS.localProcess) {
     return localProcessPlayRunnerBackend;
@@ -22,5 +40,12 @@ export function resolvePlayRunnerBackend(
   throw new Error(`Unsupported play runner backend: ${backend}`);
 }
 
-export { daytonaPlayRunnerBackend, localProcessPlayRunnerBackend };
+export {
+  daytonaPlayRunnerBackend,
+  daytonaModalFallbackPlayRunnerBackend,
+  daytonaOnlyPlayRunnerBackend,
+  localProcessPlayRunnerBackend,
+  modalOnlyPlayRunnerBackend,
+  modalPlayRunnerBackend,
+};
 export type { PlayRunnerBackend };