deepline 0.1.199 → 0.1.200

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -0,0 +1,433 @@
+import type { Daytona } from '@daytonaio/sdk';
+import type {
+  PlayRunnerDeferredRuntimeTask,
+  PlayRunnerExecutionConfig,
+  PlayRunnerResult,
+} from '@shared_libs/play-runtime/protocol';
+import { shouldCleanupSandboxInBackground } from '@shared_libs/play-runtime/daytona-runtime-config';
+
+const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
+const DAYTONA_CREATE_HEDGE_DELAYS_MS = [0, 200, 400, 700] as const;
+const DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS = 3_500;
+const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = 5;
+const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
+const DAYTONA_SANDBOX_LABEL_SOURCE = 'deepline-play-runner';
+
+export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
+
+export type DaytonaClient = Pick<Daytona, 'create'>;
+export type DaytonaSandbox = Awaited<ReturnType<DaytonaClient['create']>>;
+export type DaytonaExecutionContext = PlayRunnerExecutionConfig['context'];
+export type DaytonaStageEmitter = (
+  stage: string,
+  extra?: Record<string, unknown>,
+) => void;
+
+export type AcquiredDaytonaSandbox = {
+  sandbox: DaytonaSandbox;
+  billingStartedAt: number;
+};
+
+export type OneShotDaytonaSandboxLifecycle = {
+  startedAt: number;
+  acquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
+  createFreshSandbox: () => Promise<AcquiredDaytonaSandbox>;
+  dispose: () => Promise<void>;
+};
+
+type HedgedDaytonaCreateResult = {
+  sandbox: DaytonaSandbox;
+  attempt: number;
+  attemptElapsedMs: number;
+};
+
+function normalizeLabelValue(value: string | null | undefined): string | null {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed.slice(0, 63) : null;
+}
+
+function daytonaOrgIdFromContext(
+  context: DaytonaExecutionContext,
+): string | null {
+  return context.orgId?.trim() || null;
+}
+
+export function validateDaytonaExecutionContext(
+  context: DaytonaExecutionContext,
+): string {
+  const orgId = daytonaOrgIdFromContext(context);
+  if (!orgId) {
+    throw new Error(
+      'Missing required org context for Daytona execution. Refusing to start a shared sandbox without orgId.',
+    );
+  }
+  return orgId;
+}
+
+async function createOneShotDaytonaSandbox(input: {
+  daytona: DaytonaClient;
+  orgId: string;
+  context: DaytonaExecutionContext;
+}): Promise<DaytonaSandbox> {
+  const orgId = normalizeLabelValue(input.orgId);
+  const workflowId = normalizeLabelValue(input.context.workflowId);
+  const playId = normalizeLabelValue(input.context.playId);
+  const runId = normalizeLabelValue(input.context.runId);
+  const labels: {
+    source: string;
+    orgId?: string;
+    workflowId?: string;
+    playId?: string;
+    runId?: string;
+  } = {
+    source: DAYTONA_SANDBOX_LABEL_SOURCE,
+  };
+  if (orgId) labels.orgId = orgId;
+  if (workflowId) labels.workflowId = workflowId;
+  if (playId) labels.playId = playId;
+  if (runId) labels.runId = runId;
+
+  // Intentionally omit image/snapshot so Daytona uses its default fast sandbox
+  // image; custom images would add build/pull tax to one-shot cold starts.
+  return input.daytona.create(
+    {
+      labels,
+      ephemeral: true,
+      autoStopInterval: DAYTONA_AUTO_STOP_INTERVAL_MINUTES,
+      autoArchiveInterval: DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES,
+    },
+    { timeout: DAYTONA_CREATE_TIMEOUT_SECONDS },
+  );
+}
+
+function cleanupLateDaytonaCreate(input: {
+  sandbox: DaytonaSandbox;
+  attempt: number;
+  emitStage: DaytonaStageEmitter;
+}): void {
+  void input.sandbox.delete(30).then(
+    () =>
+      input.emitStage('create:late_cleanup_done', {
+        sandboxId: input.sandbox.id,
+        attempt: input.attempt,
+      }),
+    (error: unknown) => {
+      console.warn('[play-runner.daytona.create_late_cleanup_failed]', {
+        sandboxId: input.sandbox.id,
+        attempt: input.attempt,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    },
+  );
+}
+
+async function createHedgedOneShotDaytonaSandbox(input: {
+  daytona: DaytonaClient;
+  orgId: string;
+  context: DaytonaExecutionContext;
+  emitStage: DaytonaStageEmitter;
+  startedAt: number;
+}): Promise<HedgedDaytonaCreateResult> {
+  return await new Promise<HedgedDaytonaCreateResult>((resolve, reject) => {
+    let settled = false;
+    let startedAttempts = 0;
+    let rejectedAttempts = 0;
+    const errors: string[] = [];
+    const timers: ReturnType<typeof setTimeout>[] = [];
+
+    const clearTimers = () => {
+      while (timers.length > 0) {
+        const timer = timers.pop();
+        if (timer) clearTimeout(timer);
+      }
+    };
+
+    const rejectIfExhausted = () => {
+      if (
+        settled ||
+        rejectedAttempts !== DAYTONA_CREATE_HEDGE_DELAYS_MS.length
+      ) {
+        return;
+      }
+      settled = true;
+      clearTimers();
+      reject(
+        new Error(
+          `Daytona sandbox create failed across ${rejectedAttempts} attempts: ${errors.join('; ')}`,
+        ),
+      );
+    };
+
+    const startAttempt = (attempt: number) => {
+      if (settled) return;
+      startedAttempts = Math.max(startedAttempts, attempt);
+      if (attempt > 1) {
+        input.emitStage('create:retry', {
+          attempt: attempt - 1,
+          elapsedMs: Date.now() - input.startedAt,
+          reason: 'hedge_delay',
+        });
+      }
+      const attemptStartedAt = Date.now();
+      void createOneShotDaytonaSandbox({
+        daytona: input.daytona,
+        orgId: input.orgId,
+        context: input.context,
+      }).then(
+        (sandbox) => {
+          if (settled) {
+            cleanupLateDaytonaCreate({
+              sandbox,
+              attempt,
+              emitStage: input.emitStage,
+            });
+            return;
+          }
+          settled = true;
+          clearTimers();
+          resolve({
+            sandbox,
+            attempt,
+            attemptElapsedMs: Date.now() - attemptStartedAt,
+          });
+        },
+        (error: unknown) => {
+          rejectedAttempts += 1;
+          errors.push(error instanceof Error ? error.message : String(error));
+          console.warn('[play-runner.daytona.create_attempt_failed]', {
+            attempt,
+            error: error instanceof Error ? error.message : String(error),
+          });
+          rejectIfExhausted();
+        },
+      );
+    };
+
+    for (const [index, delayMs] of DAYTONA_CREATE_HEDGE_DELAYS_MS.entries()) {
+      const attempt = index + 1;
+      if (delayMs === 0) {
+        startAttempt(attempt);
+      } else {
+        timers.push(setTimeout(() => startAttempt(attempt), delayMs));
+      }
+    }
+
+    timers.push(
+      setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        clearTimers();
+        reject(
+          new Error(
+            `Daytona sandbox create did not complete within ${DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS}ms after starting ${startedAttempts} attempts.`,
+          ),
+        );
+      }, DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS),
+    );
+  });
+}
+
+async function acquireOneShotDaytonaSandbox(input: {
+  daytona: DaytonaClient;
+  orgId: string;
+  context: DaytonaExecutionContext;
+  emitStage: DaytonaStageEmitter;
+  startedAt: number;
+}): Promise<AcquiredDaytonaSandbox> {
+  input.emitStage('create:start');
+  const result = await createHedgedOneShotDaytonaSandbox(input);
+  const billingStartedAt = Date.now();
+  input.emitStage('create:done', {
+    sandboxId: result.sandbox.id,
+    attempt: result.attempt,
+    elapsedMs: billingStartedAt - input.startedAt,
+    attemptElapsedMs: result.attemptElapsedMs,
+  });
+  return {
+    sandbox: result.sandbox,
+    billingStartedAt,
+  };
+}
+
+export function createOneShotDaytonaSandboxLifecycle(input: {
+  daytona: DaytonaClient;
+  context: DaytonaExecutionContext;
+  emitStage: DaytonaStageEmitter;
+  startedAt?: number;
+}): OneShotDaytonaSandboxLifecycle {
+  const orgId = validateDaytonaExecutionContext(input.context);
+  const startedAt = input.startedAt ?? Date.now();
+  let disposed = false;
+  let latestAcquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
+  const createFreshSandbox = () => {
+    latestAcquiredSandboxPromise = acquireOneShotDaytonaSandbox({
+      daytona: input.daytona,
+      orgId,
+      context: input.context,
+      emitStage: input.emitStage,
+      startedAt,
+    });
+    return latestAcquiredSandboxPromise;
+  };
+  const acquiredSandboxPromise = createFreshSandbox();
+
+  return {
+    startedAt,
+    acquiredSandboxPromise,
+    createFreshSandbox,
+    dispose: async () => {
+      if (disposed) return;
+      disposed = true;
+      try {
+        const acquired = await latestAcquiredSandboxPromise;
+        await acquired.sandbox.delete(30);
+      } catch (error) {
+        console.warn('[play-runner.daytona.dispose_failed_before_acquire]', {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    },
+  };
+}
+
+function attachDeferredDaytonaCleanup(
+  result: PlayRunnerResult,
+  task: PlayRunnerDeferredRuntimeTask | null,
+): PlayRunnerResult {
+  if (!task) {
+    return result;
+  }
+  return {
+    ...result,
+    deferredRuntimeTasks: [...(result.deferredRuntimeTasks ?? []), task],
+  };
+}
+
+export function createDaytonaSandboxCleanupManager(): {
+  activate(acquired: AcquiredDaytonaSandbox): void;
+  currentSandbox(): DaytonaSandbox | null;
+  cleanupActiveSandboxForCancellation(): boolean;
+  stashActiveSandboxForRetry(): void;
+  withCleanup(
+    result: PlayRunnerResult,
+    input: { cancellationCleanupStarted: boolean },
+  ): PlayRunnerResult;
+} {
+  let sandbox: DaytonaSandbox | null = null;
+  let sandboxBillingStartedAt: number | null = null;
+  const deferredCleanupTasksFromPreviousAttempts: PlayRunnerDeferredRuntimeTask[] =
+    [];
+  const backgroundCleanupSandboxIds = new Set<string>();
+
+  const deferredCleanupTask = (): PlayRunnerDeferredRuntimeTask | null => {
+    if (!sandbox || sandboxBillingStartedAt === null) {
+      return null;
+    }
+    return {
+      kind: 'daytona_sandbox_cleanup',
+      sandboxId: sandbox.id,
+      billingStartedAt: sandboxBillingStartedAt,
+      cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : 1,
+      memoryGiB: typeof sandbox.memory === 'number' ? sandbox.memory : 1,
+      diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : 0,
+    };
+  };
+
+  const deleteSandboxOnce = (input: {
+    reason: string;
+    task: PlayRunnerDeferredRuntimeTask | null;
+    sandboxToDelete: DaytonaSandbox | null;
+  }) => {
+    if (
+      !input.task ||
+      !input.sandboxToDelete ||
+      backgroundCleanupSandboxIds.has(input.task.sandboxId)
+    ) {
+      return false;
+    }
+    backgroundCleanupSandboxIds.add(input.task.sandboxId);
+    void input.sandboxToDelete.delete(30).then(
+      () => {
+        console.info('[play-runner.daytona.cleanup_done]', {
+          reason: input.reason,
+          sandboxId: input.task!.sandboxId,
+          elapsedMs: Date.now() - input.task!.billingStartedAt,
+        });
+      },
+      (error: unknown) => {
+        console.warn('[play-runner.daytona.cleanup_failed]', {
+          reason: input.reason,
+          sandboxId: input.task!.sandboxId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      },
+    );
+    return true;
+  };
+
+  const scheduleBackgroundCleanup = (
+    task: PlayRunnerDeferredRuntimeTask | null,
+    sandboxToDelete: DaytonaSandbox | null,
+  ) => {
+    if (
+      !shouldCleanupSandboxInBackground() ||
+      !task ||
+      !sandboxToDelete ||
+      backgroundCleanupSandboxIds.has(task.sandboxId)
+    ) {
+      return false;
+    }
+    return deleteSandboxOnce({
+      reason: 'background',
+      task,
+      sandboxToDelete,
+    });
+  };
+
+  return {
+    activate(acquired) {
+      sandbox = acquired.sandbox;
+      sandboxBillingStartedAt = acquired.billingStartedAt;
+    },
+    currentSandbox() {
+      return sandbox;
+    },
+    cleanupActiveSandboxForCancellation() {
+      return deleteSandboxOnce({
+        reason: 'cancelled',
+        task: deferredCleanupTask(),
+        sandboxToDelete: sandbox,
+      });
+    },
+    stashActiveSandboxForRetry() {
+      const task = deferredCleanupTask();
+      const sandboxToDelete = sandbox;
+      if (!scheduleBackgroundCleanup(task, sandboxToDelete) && task) {
+        deferredCleanupTasksFromPreviousAttempts.push(task);
+      }
+      sandbox = null;
+      sandboxBillingStartedAt = null;
+    },
+    withCleanup(result, input) {
+      const resultWithPreviousCleanup =
+        deferredCleanupTasksFromPreviousAttempts.length === 0
+          ? result
+          : {
+              ...result,
+              deferredRuntimeTasks: [
+                ...deferredCleanupTasksFromPreviousAttempts,
+                ...(result.deferredRuntimeTasks ?? []),
+              ],
+            };
+      const task = deferredCleanupTask();
+      const sandboxToDelete = sandbox;
+      if (input.cancellationCleanupStarted) {
+        return resultWithPreviousCleanup;
+      }
+      if (scheduleBackgroundCleanup(task, sandboxToDelete)) {
+        return resultWithPreviousCleanup;
+      }
+      return attachDeferredDaytonaCleanup(resultWithPreviousCleanup, task);
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -0,0 +1,227 @@
+import { randomUUID } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { gzipSync } from 'node:zlib';
+import type { PlayRunnerExecutionConfig } from '@shared_libs/play-runtime/protocol';
+import { compactPlayArtifactForRuntimeTransport } from '@shared_libs/plays/artifact-transport';
+import type {
+  DaytonaExecutionContext,
+  DaytonaSandbox,
+  DaytonaStageEmitter,
+} from './daytona-lifecycle';
+
+type DaytonaRemoteRuntimeContext = PlayRunnerExecutionConfig['context'];
+
+type DaytonaPayloadEnvelope = {
+  schemaVersion: 1;
+  runnerCode: string;
+  artifactBundledCode: string;
+  config: PlayRunnerExecutionConfig;
+};
+
+export type StagedDaytonaPayload = {
+  workDir: string;
+  command: string;
+  outputPath: string;
+  exitCodePath: string;
+  progressEventPath: string;
+};
+
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+function gzipUtf8(value: string): Buffer {
+  return gzipSync(Buffer.from(value, 'utf-8'));
+}
+
+function gzipJson(value: DaytonaPayloadEnvelope): Buffer {
+  return gzipUtf8(JSON.stringify(value));
+}
+
+function nodeMaterializePayloadCommand(input: {
+  envelopePath: string;
+  runnerPath: string;
+  configPath: string;
+  artifactCodePath: string;
+}): string {
+  const script =
+    "const fs=require('node:fs');const zlib=require('node:zlib');const p=JSON.parse(zlib.gunzipSync(fs.readFileSync(process.argv[1])));if(p.schemaVersion!==1)throw new Error('Unsupported Daytona payload envelope');fs.writeFileSync(process.argv[2],p.runnerCode);fs.writeFileSync(process.argv[3],JSON.stringify(p.config));fs.writeFileSync(process.argv[4],p.artifactBundledCode);";
+  return [
+    'node',
+    '-e',
+    shellQuote(script),
+    shellQuote(input.envelopePath),
+    shellQuote(input.runnerPath),
+    shellQuote(input.configPath),
+    shellQuote(input.artifactCodePath),
+  ].join(' ');
+}
+
+function remoteRuntimeContextForDaytona(
+  context: DaytonaExecutionContext,
+): DaytonaRemoteRuntimeContext {
+  if ((context.preloadedDbSessions?.length ?? 0) > 0) {
+    return {
+      ...context,
+      dbSessionStrategy: 'preloaded',
+      preloadedDbSessions: context.preloadedDbSessions,
+    };
+  }
+  return {
+    ...context,
+    dbSessionStrategy: 'sandbox_public_key',
+    preloadedDbSessions: undefined,
+  };
+}
+
+async function timedDaytonaUpload(input: {
+  sandbox: DaytonaSandbox;
+  emitStage: DaytonaStageEmitter;
+  label: string;
+  path: string;
+  content: Buffer;
+}) {
+  const startedAt = Date.now();
+  await input.sandbox.fs.uploadFile(input.content, input.path);
+  input.emitStage(`upload:${input.label}:done`, {
+    bytes: input.content.byteLength,
+    elapsedMs: Date.now() - startedAt,
+  });
+}
+
+export async function stageDaytonaRunnerPayload(input: {
+  sandbox: DaytonaSandbox;
+  bundlePromise: Promise<string>;
+  config: PlayRunnerExecutionConfig;
+  workDir: string;
+  startedAt: number;
+  emitStage: DaytonaStageEmitter;
+}): Promise<StagedDaytonaPayload> {
+  const workDir = input.workDir.replace(/\/$/, '');
+  const runnerPath = `${workDir}/deepline-play-runner-${randomUUID()}.cjs`;
+  const envelopePath = `${workDir}/deepline-play-payload-${randomUUID()}.json.gz`;
+  const configPath = `${workDir}/deepline-play-config-${randomUUID()}.json`;
+  const artifactCodePath = `${workDir}/deepline-play-artifact-${randomUUID()}.cjs`;
+
+  const remoteMaterializedFiles: Record<string, string> = {};
+  const hasInlineCsv = Boolean(input.config.csvSourceContentBase64);
+  input.emitStage('upload:start', {
+    sandboxId: input.sandbox.id,
+    materializedFileCount: Object.keys(input.config.materializedFiles).length,
+    hasCsv: Boolean(
+      input.config.csvSourceContentBase64 ||
+      input.config.csvSourcePath ||
+      input.config.csvSourceUrl,
+    ),
+  });
+
+  const compactedArtifact = compactPlayArtifactForRuntimeTransport(
+    input.config.artifact,
+  );
+  const uniqueLocalPaths = new Map<string, string>();
+  for (const localPath of Object.values(input.config.materializedFiles)) {
+    if (hasInlineCsv && localPath === input.config.csvSourcePath) {
+      continue;
+    }
+    if (!uniqueLocalPaths.has(localPath)) {
+      uniqueLocalPaths.set(localPath, `${workDir}/files/${randomUUID()}`);
+    }
+  }
+  if (
+    input.config.csvSourcePath &&
+    !hasInlineCsv &&
+    !uniqueLocalPaths.has(input.config.csvSourcePath)
+  ) {
+    uniqueLocalPaths.set(
+      input.config.csvSourcePath,
+      `${workDir}/files/${randomUUID()}`,
+    );
+  }
+
+  await Promise.all(
+    [...uniqueLocalPaths.entries()].map(async ([localPath, remotePath]) => {
+      await timedDaytonaUpload({
+        sandbox: input.sandbox,
+        emitStage: input.emitStage,
+        label:
+          localPath === input.config.csvSourcePath
+            ? 'csv'
+            : 'materialized_file',
+        path: remotePath,
+        content: await readFile(localPath),
+      });
+    }),
+  );
+
+  for (const [logicalPath, localPath] of Object.entries(
+    input.config.materializedFiles,
+  )) {
+    const remotePath = uniqueLocalPaths.get(localPath);
+    if (remotePath) {
+      remoteMaterializedFiles[logicalPath] = remotePath;
+    }
+  }
+
+  const remoteConfig: PlayRunnerExecutionConfig = {
+    ...input.config,
+    artifact: {
+      ...compactedArtifact,
+      bundledCode: '',
+    },
+    artifactTransport: {
+      bundledCodePath: artifactCodePath,
+      bundledCodeEncoding: 'utf8',
+    },
+    workspaceRoot: input.workDir,
+    csvSourcePath: hasInlineCsv
+      ? null
+      : input.config.csvSourcePath
+        ? (uniqueLocalPaths.get(input.config.csvSourcePath) ??
+          input.config.csvSourcePath)
+        : null,
+    csvSourceUrl: input.config.csvSourceUrl ?? null,
+    csvSourceContentBase64: input.config.csvSourceContentBase64 ?? null,
+    materializedFiles: remoteMaterializedFiles,
+    context: remoteRuntimeContextForDaytona(input.config.context),
+  };
+
+  const envelopeUpload = input.bundlePromise.then((bundle) =>
+    timedDaytonaUpload({
+      sandbox: input.sandbox,
+      emitStage: input.emitStage,
+      label: 'payload',
+      path: envelopePath,
+      content: gzipJson({
+        schemaVersion: 1,
+        runnerCode: bundle,
+        artifactBundledCode: compactedArtifact.bundledCode,
+        config: remoteConfig,
+      }),
+    }),
+  );
+  await envelopeUpload;
+  input.emitStage('upload:done', {
+    sandboxId: input.sandbox.id,
+    uploadedFileCount: uniqueLocalPaths.size + 1,
+    elapsedMs: Date.now() - input.startedAt,
+  });
+
+  const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
+  const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
+  const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
+  const runnerCommand = `${nodeMaterializePayloadCommand({
+    envelopePath,
+    runnerPath,
+    configPath,
+    artifactCodePath,
+  })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+
+  return {
+    workDir: input.workDir,
+    command,
+    outputPath,
+    exitCodePath,
+    progressEventPath,
+  };
+}